diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 96dbd9e1d..f2847c405 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -18,6 +18,7 @@ permissions: { } jobs: crate-build: + if: ${{ needs.generate-matrix.outputs.any_builds == 'true' }} needs: - generate-matrix runs-on: ${{ matrix.runner }} @@ -114,6 +115,7 @@ jobs: STEPS_GET_LABELS_OUTPUTS_LABELS: ${{ steps.get-labels.outputs.labels }} build: + if: ${{ needs.generate-matrix.outputs.any_builds == 'true' }} timeout-minutes: 90 needs: - generate-matrix diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index 8dd0240b8..35009efa7 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -82,6 +82,12 @@ rm Lib/ensurepip/_bundled/pip-*.whl cp "${PIP_WHEEL}" Lib/ensurepip/_bundled/ sed "${sed_args[@]}" "s/^_PIP_VERSION = .*/_PIP_VERSION = \"${PIP_VERSION}\"/" Lib/ensurepip/__init__.py +# Build relative runtime library paths into CPython 3.15+. +if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_15}" ]; then + patch -p1 -i "${ROOT}/patch-relocatable-flag.patch" + patch -p1 -i "${ROOT}/patch-relocatable-link.patch" +fi + # configure doesn't support cross-compiling on Apple. Teach it. if [[ "${PYBUILD_PLATFORM}" = macos* && -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_13}" ]]; then if [ "${PYTHON_MAJMIN_VERSION}" = "3.12" ]; then @@ -152,7 +158,9 @@ fi # The default build rule for the macOS dylib doesn't pick up libraries # from modules / makesetup. So patch it accordingly. -if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" ]; then +if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_15}" ]; then + patch -p1 -i "${ROOT}/patch-macos-link-extension-modules-15.patch" +elif [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" ]; then patch -p1 -i "${ROOT}/patch-macos-link-extension-modules-13.patch" else patch -p1 -i "${ROOT}/patch-macos-link-extension-modules.patch" @@ -333,6 +341,10 @@ fi EXTRA_CONFIGURE_FLAGS= +if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_15}" ]; then + EXTRA_CONFIGURE_FLAGS="${EXTRA_CONFIGURE_FLAGS} --enable-relocatable" +fi + if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then CFLAGS="${CFLAGS} -I${TOOLS_PATH}/deps/include/uuid" @@ -818,18 +830,22 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then # Fix the Python binary to reference libpython via @rpath and add # an rpath entry so it can find the library. - install_name_tool \ - -change "/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" "@rpath/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \ - -add_rpath @executable_path/../lib \ - "${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}" + if [ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_14}" ]; then + install_name_tool \ + -change "/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" "@rpath/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \ + -add_rpath @executable_path/../lib \ + "${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}" + fi # Python's build system doesn't make this file writable. chmod 755 "${ROOT}/out/python/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" # Set libpython's install name to @rpath so binaries linking against it # can locate it via their own rpath entries. - install_name_tool \ - -id "@rpath/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \ - "${ROOT}/out/python/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" + if [ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_14}" ]; then + install_name_tool \ + -id "@rpath/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \ + "${ROOT}/out/python/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" + fi # We also normalize /tools/deps/lib/libz.1.dylib to the system location. install_name_tool \ @@ -839,19 +855,19 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then -change /tools/deps/lib/libz.1.dylib /usr/lib/libz.1.dylib \ "${ROOT}/out/python/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" - if [ -n "${PYTHON_BINARY_SUFFIX}" ]; then + if [[ -n "${PYTHON_BINARY_SUFFIX}" && -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_14}" ]]; then install_name_tool \ -change "/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" "@rpath/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \ -add_rpath @executable_path/../lib \ "${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}" fi - # At the moment, python3 and libpython don't have shared-library - # dependencies, but at some point we will want to run this for - # them too. - for module in ${ROOT}/out/python/install/lib/python*/lib-dynload/*.so; do - install_name_tool -add_rpath @loader_path/../.. "$module" - done + # The relocatable backport supplies extension search paths for CPython 3.15+. + if [ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_14}" ]; then + for module in ${ROOT}/out/python/install/lib/python*/lib-dynload/*.so; do + install_name_tool -add_rpath @loader_path/../.. "$module" + done + fi else # (not macos) LIBPYTHON_SHARED_LIBRARY_BASENAME=libpython${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}.so.1.0 LIBPYTHON_SHARED_LIBRARY=${ROOT}/out/python/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME} @@ -917,12 +933,14 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then # empty dummy libpython.so, which allows the link to succeed but # ensures they do not use any unwanted symbols. That might be # worth doing at some point.) - patchelf --force-rpath --set-rpath "\$ORIGIN/../lib" \ - "${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}" - - if [ -n "${PYTHON_BINARY_SUFFIX}" ]; then + if [ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_14}" ]; then patchelf --force-rpath --set-rpath "\$ORIGIN/../lib" \ - "${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}" + "${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}" + + if [ -n "${PYTHON_BINARY_SUFFIX}" ]; then + patchelf --force-rpath --set-rpath "\$ORIGIN/../lib" \ + "${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}" + fi fi # For libpython3.so (the ABI3 library for embedders), we do @@ -941,8 +959,9 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then # libpython3.x.so.1.0 that they want (and it's fine to use # DT_RUNPATH instead of DT_RPATH). if [ "${CC}" == "musl-clang" ]; then - # libpython3.so isn't present in debug builds. - if [ -z "${CPYTHON_DEBUG}" ]; then + # The relocatable backport supplies this path for CPython 3.15+. + # libpython3.so is absent from debug builds. + if [[ -z "${CPYTHON_DEBUG}" && -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_14}" ]]; then patchelf --set-rpath "\$ORIGIN/../lib" \ "${ROOT}/out/python/install/lib/libpython3.so" fi @@ -956,9 +975,11 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then # Let _tkinter locate bundled Tcl/Tk libraries independently of the # interpreter. - for module in "${ROOT}"/out/python/install/lib/python*/lib-dynload/_tkinter*.so; do - patchelf --force-rpath --set-rpath "\$ORIGIN/../.." "$module" - done + if [ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_14}" ]; then + for module in "${ROOT}"/out/python/install/lib/python*/lib-dynload/_tkinter*.so; do + patchelf --force-rpath --set-rpath "\$ORIGIN/../.." "$module" + done + fi fi fi diff --git a/cpython-unix/patch-macos-link-extension-modules-15.patch b/cpython-unix/patch-macos-link-extension-modules-15.patch new file mode 100644 index 000000000..70993e9a6 --- /dev/null +++ b/cpython-unix/patch-macos-link-extension-modules-15.patch @@ -0,0 +1,20 @@ +--- a/Makefile.pre.in ++++ b/Makefile.pre.in +@@ -1053,7 +1053,7 @@ + $(BLDSHARED) $(NO_AS_NEEDED) -o $@ -Wl,-h$@ $^ + + libpython$(LDVERSION).dylib: $(LIBRARY_OBJS) +- $(CC) -dynamiclib $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(SHLIBS) $(LIBC) $(LIBM); \ ++ $(CC) -dynamiclib $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ + + + $(RELOCATABLE_DIR)/libpython$(LDVERSION).so: $(LIBRARY_OBJS) $(DTRACE_OBJS) +@@ -1066,7 +1066,7 @@ + + $(RELOCATABLE_DIR)/libpython$(LDVERSION).dylib: $(LIBRARY_OBJS) + @mkdir -p $(@D) +- $(CC) -dynamiclib $(PY_CORE_LDFLAGS) $(RELOCATABLE_LIBPYTHON_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(RELOCATABLE_LIBPYTHON_INSTALL_NAME) -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(SHLIBS) $(LIBC) $(LIBM) ++ $(CC) -dynamiclib $(PY_CORE_LDFLAGS) $(RELOCATABLE_LIBPYTHON_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(RELOCATABLE_LIBPYTHON_INSTALL_NAME) -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM) + + libpython$(VERSION).sl: $(LIBRARY_OBJS) + $(LDSHARED) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM) diff --git a/cpython-unix/patch-relocatable-flag.patch b/cpython-unix/patch-relocatable-flag.patch new file mode 100644 index 000000000..4e8b54f8e --- /dev/null +++ b/cpython-unix/patch-relocatable-flag.patch @@ -0,0 +1,74 @@ +diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst +index 0d420d8..2cc8e94 100644 +--- a/Doc/using/configure.rst ++++ b/Doc/using/configure.rst +@@ -1073,6 +1073,17 @@ Linker options + + Enable building a shared Python library: ``libpython`` (default is no). + ++.. option:: --enable-relocatable ++ ++ Enable experimental support for relocatable builds (default is no). ++ This option currently has no effect on the build. ++ ++ .. warning:: ++ ++ This feature is under active development. Do not use it in production. ++ ++ .. versionadded:: 3.16 ++ + .. option:: --without-static-libpython + + Do not build ``libpythonMAJOR.MINOR.a`` and do not install ``python.o`` +diff --git a/Makefile.pre.in b/Makefile.pre.in +index 36515db..7dcfc45 100644 +--- a/Makefile.pre.in ++++ b/Makefile.pre.in +@@ -60,6 +60,7 @@ DSYMUTIL= @DSYMUTIL@ + DSYMUTIL_PATH= @DSYMUTIL_PATH@ + + GNULD= @GNULD@ ++RELOCATABLE= @RELOCATABLE@ + + # Shell used by make (some versions default to the login shell, which is bad) + SHELL= /bin/sh -e +diff --git a/configure.ac b/configure.ac +index a8760c9..1afbeba 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -1594,6 +1594,21 @@ if test "x$enable_profiling" = xyes; then + LDFLAGS="-pg $LDFLAGS" + fi + ++AC_MSG_CHECKING([for --enable-relocatable]) ++AC_ARG_ENABLE([relocatable], ++ AS_HELP_STRING([--enable-relocatable], ++ [enable experimental relocatable build support (default is no)]), ++ [], [enable_relocatable=no]) ++AS_CASE([$enable_relocatable], ++ [yes|no], [], ++ [AC_MSG_ERROR([--enable-relocatable must be yes or no])]) ++AC_MSG_RESULT([$enable_relocatable]) ++AS_VAR_IF([enable_relocatable], [yes], [ ++ AC_DEFINE([Py_ENABLE_RELOCATABLE], [1], ++ [Define if --enable-relocatable is used.]) ++]) ++AC_SUBST([RELOCATABLE], [$enable_relocatable]) ++ + AC_MSG_CHECKING([LDLIBRARY]) + + # Apple framework builds need more magic. LDLIBRARY is the dynamic +diff --git a/pyconfig.h.in b/pyconfig.h.in +index 08306a1..4fb029a 100644 +--- a/pyconfig.h.in ++++ b/pyconfig.h.in +@@ -1880,6 +1880,9 @@ + /* Define if you want to build an interpreter with many run-time checks. */ + #undef Py_DEBUG + ++/* Define if --enable-relocatable is used. */ ++#undef Py_ENABLE_RELOCATABLE ++ + /* Defined if Python is built as a shared library. */ + #undef Py_ENABLE_SHARED + diff --git a/cpython-unix/patch-relocatable-link.patch b/cpython-unix/patch-relocatable-link.patch new file mode 100644 index 000000000..adfc13504 --- /dev/null +++ b/cpython-unix/patch-relocatable-link.patch @@ -0,0 +1,337 @@ +diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst +index 2cc8e94..203b04e 100644 +--- a/Doc/using/configure.rst ++++ b/Doc/using/configure.rst +@@ -1076,12 +1076,31 @@ Linker options + .. option:: --enable-relocatable + + Enable experimental support for relocatable builds (default is no). +- This option currently has no effect on the build. + + .. warning:: + + This feature is under active development. Do not use it in production. + ++ Currently, this sets relative runtime library paths on Linux and for ++ non-framework macOS builds. Framework relocation is not implemented yet. ++ Combining this option with :option:`--enable-framework` is an error. ++ ++ Move the installation as a unit. It must use the default ``bin`` and ``lib`` ++ directories under a common prefix. :option:`--with-platlibdir` may select ++ a different directory name for the standard library, but not a nested path. ++ On Linux, these paths use ``DT_RPATH``. ++ ++ Copied virtual environments from non-framework shared builds require ++ directory symlinks to the base installation's libraries. Virtual environments ++ depend on their base installation and must be recreated if it moves. ++ ++ Dependencies stored in other directories, such as ``lib/tcltk``, may ++ need additional loader search paths. ++ ++ This option does not bundle dependencies, adjust data-file lookup for ++ libraries such as Tcl/Tk, or rewrite script shebangs and build-time ++ paths in :mod:`sysconfig`. ++ + .. versionadded:: 3.16 + + .. option:: --without-static-libpython +diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py +index c74ca05..791dc53 100644 +--- a/Lib/test/test_sys.py ++++ b/Lib/test/test_sys.py +@@ -2372,9 +2372,11 @@ def test_remote_exec_deleted_libpython(self): + os.symlink(instsoname, os.path.join(lib_dir, ldlibrary)) + + env = os.environ.copy() +- ld_library_path = env.get('LD_LIBRARY_PATH') +- env['LD_LIBRARY_PATH'] = lib_dir if not ld_library_path else ( +- lib_dir + os.pathsep + ld_library_path) ++ # Preload the copy even when the interpreter's RPATH takes ++ # precedence over LD_LIBRARY_PATH. ++ ld_preload = env.get('LD_PRELOAD') ++ env['LD_PRELOAD'] = copied_libpython if not ld_preload else ( ++ copied_libpython + os.pathsep + ld_preload) + + self._run_remote_exec_with_deleted_mapping(copied_libpython, + env=env) +diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py +index 38e1bfe..b4f1cbd 100644 +--- a/Lib/venv/__init__.py ++++ b/Lib/venv/__init__.py +@@ -310,6 +310,20 @@ def setup_python(self, context): + copier(context.executable, path) + if not os.path.islink(path): + os.chmod(path, 0o755) ++ if (sysconfig.get_config_var('RELOCATABLE') == 'yes' ++ and sysconfig.get_config_var('Py_ENABLE_SHARED') ++ and not sysconfig.get_config_var('PYTHONFRAMEWORK')): ++ # The copied executable searches relative to this environment. ++ # Link the base library directory so it can find libpython. ++ if sysconfig.is_python_build(): ++ library_dir = context.python_dir ++ else: ++ library_dir = os.path.join(sys.base_exec_prefix, 'lib') ++ library_link = os.path.join( ++ context.env_dir, 'lib', 'python-base') ++ if os.path.islink(library_link): ++ os.unlink(library_link) ++ os.symlink(library_dir, library_link, target_is_directory=True) + for suffix in ('python', 'python3', + f'python3.{sys.version_info[1]}'): + path = os.path.join(binpath, suffix) +diff --git a/Makefile.pre.in b/Makefile.pre.in +index 7dcfc45..a090b9c 100644 +--- a/Makefile.pre.in ++++ b/Makefile.pre.in +@@ -61,6 +61,10 @@ DSYMUTIL_PATH= @DSYMUTIL_PATH@ + + GNULD= @GNULD@ + RELOCATABLE= @RELOCATABLE@ ++RELOCATABLE_PYTHON_LDFLAGS=@RELOCATABLE_PYTHON_LDFLAGS@ ++RELOCATABLE_LIBPYTHON_LDFLAGS=@RELOCATABLE_LIBPYTHON_LDFLAGS@ ++RELOCATABLE_EXTENSION_LDFLAGS=@RELOCATABLE_EXTENSION_LDFLAGS@ ++RELOCATABLE_LIBPYTHON_INSTALL_NAME=@RELOCATABLE_LIBPYTHON_INSTALL_NAME@ + + # Shell used by make (some versions default to the login shell, which is bad) + SHELL= /bin/sh -e +@@ -313,6 +317,22 @@ LIBOBJS= @LIBOBJS@ + PYTHON= python$(EXE) + BUILDPYTHON= python$(BUILDEXE) + ++# Build-tree programs run with RUNSHARED where needed. Install variants need ++# runtime paths relative to their installed locations. ++RELOCATABLE_DIR=build/relocatable ++INSTALL_PYTHON=$(BUILDPYTHON) ++INSTALL_LDLIBRARY=$(LDLIBRARY) ++INSTALL_PY3LIBRARY=$(PY3LIBRARY) ++INSTALL_SHAREDMODS=$(SHAREDMODS) ++ifeq ($(RELOCATABLE),yes) ++INSTALL_PYTHON=$(RELOCATABLE_DIR)/$(BUILDPYTHON) ++INSTALL_SHAREDMODS=$(addprefix $(RELOCATABLE_DIR)/,$(SHAREDMODS)) ++ifeq ($(PY_ENABLE_SHARED),1) ++INSTALL_LDLIBRARY=$(RELOCATABLE_DIR)/$(LDLIBRARY) ++INSTALL_PY3LIBRARY=$(addprefix $(RELOCATABLE_DIR)/,$(PY3LIBRARY)) ++endif ++endif ++ + HOSTRUNNER= @HOSTRUNNER@ + + PYTHON_FOR_REGEN?=@PYTHON_FOR_REGEN@ +@@ -809,2 +829,3 @@ list-targets: + .PHONY: build_all ++build_all: $(INSTALL_PYTHON) $(INSTALL_LDLIBRARY) $(INSTALL_PY3LIBRARY) $(INSTALL_SHAREDMODS) + build_all: check-clean-src check-app-store-compliance $(BUILDPYTHON) platform sharedmods \ +@@ -866,6 +887,10 @@ profile-opt: profile-run-stamp + + # List of binaries that BOLT runs on. + BOLT_BINARIES := @BOLT_BINARIES@ ++ifeq ($(RELOCATABLE),yes) ++# Optimize libraries before the interpreter, which depends on their timestamps. ++RELOCATABLE_BOLT_BINARIES = $(addprefix $(RELOCATABLE_DIR)/,$(filter-out $(BUILDPYTHON),$(BOLT_BINARIES)) $(BUILDPYTHON)) ++endif + + BOLT_INSTRUMENT_FLAGS := @BOLT_INSTRUMENT_FLAGS@ + BOLT_APPLY_FLAGS := @BOLT_APPLY_FLAGS@ +@@ -875,13 +900,13 @@ clean-bolt: + # Profile data. + rm -f *.fdata + # Pristine binaries before BOLT optimization. +- rm -f *.prebolt ++ rm -f *.prebolt $(RELOCATABLE_DIR)/*.prebolt + # BOLT instrumented binaries. + rm -f *.bolt_inst + + profile-bolt-stamp: $(BUILDPYTHON) + # Ensure a pristine, pre-BOLT copy of the binary and no profile data from last run. +- for bin in $(BOLT_BINARIES); do \ ++ for bin in $(BOLT_BINARIES) $(RELOCATABLE_BOLT_BINARIES); do \ + prebolt="$${bin}.prebolt"; \ + if [ -e "$${prebolt}" ]; then \ + echo "Restoring pre-BOLT binary $${prebolt}"; \ +@@ -907,6 +932,14 @@ profile-bolt-stamp: $(BUILDPYTHON) + @LLVM_BOLT@ "$${bin}.prebolt" -o "$${bin}.bolt" -data="$${bin}.fdata" $(BOLT_APPLY_FLAGS); \ + mv "$${bin}.bolt" "$${bin}"; \ + done ++ # Install variants are linked from the same objects, so reuse the function profiles. ++ for bin in $(RELOCATABLE_BOLT_BINARIES); do \ ++ @LLVM_BOLT@ "$${bin}.prebolt" -o "$${bin}.bolt" -data="$${bin##*/}.fdata" $(BOLT_APPLY_FLAGS); \ ++ mv "$${bin}.bolt" "$${bin}"; \ ++ done ++ if test "$(RELOCATABLE)" = yes && test -n "$(filter $(INSTSONAME),$(BOLT_BINARIES))"; then \ ++ $(LN) -f $(RELOCATABLE_DIR)/$(INSTSONAME) $(INSTALL_LDLIBRARY); \ ++ fi + touch $@ + + .PHONY: bolt-opt +@@ -974,6 +1007,10 @@ clinic-tests: check-clean-src $(srcdir)/Lib/test/clinic.test.c + $(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) + $(LINKCC) $(PY_CORE_EXE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) + ++$(RELOCATABLE_DIR)/$(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) $(INSTALL_LDLIBRARY) ++ @mkdir -p $(@D) ++ $(LINKCC) $(PY_CORE_EXE_LDFLAGS) $(LINKFORSHARED) $(RELOCATABLE_PYTHON_LDFLAGS) -L$(RELOCATABLE_DIR) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) ++ + platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt + $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform + +@@ -1019,6 +1056,18 @@ libpython$(LDVERSION).dylib: $(LIBRARY_OBJS) + $(CC) -dynamiclib $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(SHLIBS) $(LIBC) $(LIBM); \ + + ++$(RELOCATABLE_DIR)/libpython$(LDVERSION).so: $(LIBRARY_OBJS) $(DTRACE_OBJS) ++ @mkdir -p $(@D) ++ $(BLDSHARED) $(RELOCATABLE_LIBPYTHON_LDFLAGS) -Wl,-h$(INSTSONAME) -o $(RELOCATABLE_DIR)/$(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM) ++ $(LN) -f $(RELOCATABLE_DIR)/$(INSTSONAME) $@ ++ ++$(RELOCATABLE_DIR)/libpython3.so: $(RELOCATABLE_DIR)/libpython$(LDVERSION).so ++ $(BLDSHARED) $(RELOCATABLE_LIBPYTHON_LDFLAGS) $(NO_AS_NEEDED) -o $@ -Wl,-hlibpython3.so $^ ++ ++$(RELOCATABLE_DIR)/libpython$(LDVERSION).dylib: $(LIBRARY_OBJS) ++ @mkdir -p $(@D) ++ $(CC) -dynamiclib $(PY_CORE_LDFLAGS) $(RELOCATABLE_LIBPYTHON_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(RELOCATABLE_LIBPYTHON_INSTALL_NAME) -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(SHLIBS) $(LIBC) $(LIBM) ++ + libpython$(VERSION).sl: $(LIBRARY_OBJS) + $(LDSHARED) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM) + +@@ -2394,7 +2443,7 @@ sharedinstall: all + else true; \ + fi; \ + done +- @for i in X $(SHAREDMODS); do \ ++ @for i in X $(INSTALL_SHAREDMODS); do \ + if test $$i != X; then \ + echo $(INSTALL_SHARED) $$i $(DESTSHARED)/`basename $$i`; \ + $(INSTALL_SHARED) $$i $(DESTDIR)$(DESTSHARED)/`basename $$i`; \ +@@ -2408,7 +2457,7 @@ sharedinstall: all + # Install the interpreter with $(VERSION) affixed + # This goes into $(exec_prefix) + .PHONY: altbininstall +-altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ ++altbininstall: all @FRAMEWORKPYTHONW@ + @for i in $(BINDIR) $(LIBDIR); \ + do \ + if test ! -d $(DESTDIR)$$i; then \ +@@ -2418,7 +2467,7 @@ altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ + fi; \ + done + if test "$(PYTHONFRAMEWORKDIR)" = "no-framework" ; then \ +- $(INSTALL_PROGRAM) $(BUILDPYTHON) $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE); \ ++ $(INSTALL_PROGRAM) $(INSTALL_PYTHON) $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE); \ + else \ + $(INSTALL_PROGRAM) $(STRIPFLAG) Mac/pythonw $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE); \ + fi +@@ -2433,13 +2482,13 @@ altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ + if test -n "$(DLLLIBRARY)" ; then \ + $(INSTALL_SHARED) $(DLLLIBRARY) $(DESTDIR)$(BINDIR); \ + else \ +- $(INSTALL_SHARED) $(LDLIBRARY) $(DESTDIR)$(LIBDIR)/$(INSTSONAME); \ ++ $(INSTALL_SHARED) $(INSTALL_LDLIBRARY) $(DESTDIR)$(LIBDIR)/$(INSTSONAME); \ + if test $(LDLIBRARY) != $(INSTSONAME); then \ + (cd $(DESTDIR)$(LIBDIR); $(LN) -sf $(INSTSONAME) $(LDLIBRARY)) \ + fi \ + fi; \ + if test -n "$(PY3LIBRARY)"; then \ +- $(INSTALL_SHARED) $(PY3LIBRARY) $(DESTDIR)$(LIBDIR)/$(PY3LIBRARY); \ ++ $(INSTALL_SHARED) $(INSTALL_PY3LIBRARY) $(DESTDIR)$(LIBDIR)/$(PY3LIBRARY); \ + fi; \ + else true; \ + fi; \ +diff --git a/Modules/makesetup b/Modules/makesetup +index 8b28946..88c01c6 100755 +--- a/Modules/makesetup ++++ b/Modules/makesetup +@@ -279,6 +279,9 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | + rule="$file: $objs \$(MODULE_${mods_upper}_LDEPS)" + rule="$rule; \$(BLDSHARED) $objs $libs \$(MODULE_LDFLAGS_SHARED) -o $file" + echo "$rule" >>$rulesf ++ rule="\$(RELOCATABLE_DIR)/$file: $objs \$(MODULE_${mods_upper}_LDEPS)" ++ rule="$rule; @mkdir -p \$(@D); \$(BLDSHARED) $objs $libs \$(MODULE_LDFLAGS_SHARED) \$(RELOCATABLE_EXTENSION_LDFLAGS) -o \$@" ++ echo "$rule" >>$rulesf + done + done + +diff --git a/configure.ac b/configure.ac +index 1afbeba..47741e6 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -6874,6 +6874,54 @@ AC_SUBST([BINLIBDEST]) + LIBDEST='${prefix}/${PLATLIBDIR}/python$(VERSION)$(ABI_THREAD)' + BINLIBDEST='${exec_prefix}/${PLATLIBDIR}/python$(VERSION)$(ABI_THREAD)' + ++# Runtime paths are specific to the installed location of each kind of binary. ++# Do not add them to LDSHARED: third-party extensions may be installed elsewhere. ++RELOCATABLE_PYTHON_LDFLAGS= ++RELOCATABLE_LIBPYTHON_LDFLAGS= ++RELOCATABLE_EXTENSION_LDFLAGS= ++RELOCATABLE_LIBPYTHON_INSTALL_NAME= ++AS_IF([test "$enable_relocatable" = yes && test -n "$enable_framework"], [ ++ AC_MSG_ERROR([--enable-relocatable does not support framework builds yet]) ++]) ++AS_IF([test "$enable_relocatable" = yes && test -z "$enable_framework"], [ ++ AS_CASE([$ac_sys_system], ++ [Linux|Darwin], [], ++ [AC_MSG_ERROR([relocatable builds are not implemented on $ac_sys_system yet])]) ++ AS_VAR_IF([prefix], [NONE], [relocatable_prefix=$ac_default_prefix], ++ [relocatable_prefix=$prefix]) ++ AS_IF([test "$exec_prefix" != NONE && test "$exec_prefix" != '${prefix}' && test "$exec_prefix" != "$relocatable_prefix"], ++ [AC_MSG_ERROR([--enable-relocatable requires a common prefix and exec-prefix])]) ++ AS_IF([test "$bindir" != '${exec_prefix}/bin' && test "$bindir" != '${prefix}/bin' && test "$bindir" != "$relocatable_prefix/bin"], ++ [AC_MSG_ERROR([--enable-relocatable requires the default bin directory])]) ++ AS_IF([test "$libdir" != '${exec_prefix}/lib' && test "$libdir" != '${prefix}/lib' && test "$libdir" != "$relocatable_prefix/lib"], ++ [AC_MSG_ERROR([--enable-relocatable requires the default lib directory])]) ++ AS_CASE([$PLATLIBDIR], [*/*|.|..], ++ [AC_MSG_ERROR([--enable-relocatable requires a single directory name for --with-platlibdir])]) ++ AS_CASE([$ac_sys_system], ++ [Linux], [ ++ # Keep the dollar sign through configure, make, and the shell. ++ # DT_RPATH entries can also resolve indirect dependencies. ++ RELOCATABLE_PYTHON_LDFLAGS="-Wl,--disable-new-dtags,-rpath,'\$\$ORIGIN/../lib:\$\$ORIGIN/../lib/python-base'" ++ RELOCATABLE_LIBPYTHON_LDFLAGS="-Wl,--disable-new-dtags,-rpath,'\$\$ORIGIN'" ++ RELOCATABLE_EXTENSION_LDFLAGS="-Wl,--disable-new-dtags,-rpath,'\$\$ORIGIN/../../../lib'" ++ save_LDFLAGS=$LDFLAGS ++ LDFLAGS="$LDFLAGS -Wl,--disable-new-dtags" ++ AC_LINK_IFELSE([AC_LANG_PROGRAM([], [])], [], ++ [AC_MSG_ERROR([--enable-relocatable requires linker support for --disable-new-dtags])]) ++ LDFLAGS=$save_LDFLAGS ++ ], ++ [Darwin], [ ++ RELOCATABLE_LIBPYTHON_INSTALL_NAME='@rpath/libpython$(LDVERSION).dylib' ++ RELOCATABLE_PYTHON_LDFLAGS='-Wl,-rpath,@executable_path/../lib -Wl,-rpath,@executable_path/../lib/python-base' ++ RELOCATABLE_LIBPYTHON_LDFLAGS='-Wl,-rpath,@loader_path' ++ RELOCATABLE_EXTENSION_LDFLAGS='-Wl,-rpath,@loader_path/../../../lib' ++ ]) ++]) ++AC_SUBST([RELOCATABLE_PYTHON_LDFLAGS]) ++AC_SUBST([RELOCATABLE_LIBPYTHON_LDFLAGS]) ++AC_SUBST([RELOCATABLE_EXTENSION_LDFLAGS]) ++AC_SUBST([RELOCATABLE_LIBPYTHON_INSTALL_NAME]) ++ + dnl define LIBPL after ABIFLAGS and LDVERSION is defined. + AC_SUBST([PY_ENABLE_SHARED]) + if test x$PLATFORM_TRIPLET = x; then +diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py +index 2f30d31..5af4856 100644 +--- a/Lib/test/test_venv.py ++++ b/Lib/test/test_venv.py +@@ -809,6 +809,8 @@ def test_zippath_from_non_installed_posix(self): + # LD_LIBRARY_PATH so the non-installed python can find and link + # libpython.so + ld_library_path = sysconfig.get_config_var("LIBDIR") ++ if sysconfig.get_config_var("RELOCATABLE") == "yes": ++ ld_library_path = os.path.join(sys.base_exec_prefix, "lib") + if not ld_library_path or sysconfig.is_python_build(): + ld_library_path = os.path.abspath(os.path.dirname(sys.executable)) + if sys.platform == 'darwin': +@@ -827,7 +829,11 @@ def test_zippath_from_non_installed_posix(self): + # correct zip path in pythonpath. + target_python = os.path.join(self.env_dir, self.bindir, python_exe) + cmd = [target_python, '-S', '-c', 'import sys; print(sys.path)'] +- out, err = check_output(cmd) ++ # This interpreter also depends on libraries outside the incomplete ++ # installation used by this test. ++ with EnvironmentVarGuard() as env: ++ env[ld_library_path_env] = ld_library_path ++ out, err = check_output(cmd) + self.assertTrue(zip_landmark.encode() in out) + + @requireVenvCreate diff --git a/pythonbuild/disttests/__init__.py b/pythonbuild/disttests/__init__.py index d6ad302b8..b076efaac 100644 --- a/pythonbuild/disttests/__init__.py +++ b/pythonbuild/disttests/__init__.py @@ -45,6 +45,69 @@ class TestPythonInterpreter(unittest.TestCase): + @unittest.skipUnless( + sys.platform in {"linux", "darwin"} and sys.version_info >= (3, 15), + "Relocatable build mode is enabled on Unix in Python 3.15+", + ) + def test_relocatable_build_mode(self): + import sysconfig + + self.assertEqual(sysconfig.get_config_var("RELOCATABLE"), "yes") + self.assertEqual(sysconfig.get_config_var("Py_ENABLE_RELOCATABLE"), 1) + + @unittest.skipUnless( + sys.platform in {"linux", "darwin"}, "Unix shared-library search paths" + ) + @unittest.skipIf( + "static" in os.environ["BUILD_OPTIONS"], + "Static builds have no shared libraries", + ) + def test_extension_libraries_without_executable_rpath(self): + # PBS links libpython statically into the interpreter. A copy outside the + # installation cannot supply the extension's library path through its rpath. + with tempfile.TemporaryDirectory(prefix="disttests-relocatable-") as temp_dir: + root = Path(temp_dir) + executable = root / "bin" / Path(sys.executable).name + executable.parent.mkdir() + shutil.copy2(sys.executable, executable) + env = { + key: value + for key, value in os.environ.items() + if not key.startswith(("PYTHON", "LD_", "DYLD_")) + } + env["PYTHONHOME"] = sys.base_prefix + + if "-linux-gnu" in os.environ["TARGET_TRIPLE"]: + # An incompatible ambient library must not override a bundled one. + poison = root / "poison" + poison.mkdir() + for pattern in ("libpython*.so*", "libtcl*.so*", "libtk*.so*"): + for library in (Path(sys.base_prefix) / "lib").glob(pattern): + (poison / library.name).write_bytes(b"not an ELF library") + env["LD_LIBRARY_PATH"] = str(poison) + + probe = """ +import ctypes +import pathlib +import sys +import _tkinter + +assert _tkinter.__file__ +abi3 = pathlib.Path(sys.base_prefix) / "lib" / "libpython3.so" +if abi3.exists(): + ctypes.CDLL(str(abi3)) +""" + result = subprocess.run( + [str(executable), "-S", "-c", probe], + cwd=root, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=30, + ) + self.assertEqual(result.returncode, 0, result.stdout) + @unittest.skipUnless(os.name == "nt", "Windows executable names") def test_windows_executables(self): build_options = os.environ["BUILD_OPTIONS"].split("+")