POC: Add a relocatable CPython build mode

.github/workflows/windows.yml

@@ -18,6 +18,7 @@ permissions: { }
 
 jobs:
   crate-build:
+    if: ${{ needs.generate-matrix.outputs.any_builds == 'true' }}
     needs:
       - generate-matrix
     runs-on: ${{ matrix.runner }}
@@ -114,6 +115,7 @@ jobs:
           STEPS_GET_LABELS_OUTPUTS_LABELS: ${{ steps.get-labels.outputs.labels }}
 
   build:
+    if: ${{ needs.generate-matrix.outputs.any_builds == 'true' }}
     timeout-minutes: 90
     needs:
       - generate-matrix

cpython-unix/build-cpython.sh

@@ -82,6 +82,12 @@ rm Lib/ensurepip/_bundled/pip-*.whl
 cp "${PIP_WHEEL}" Lib/ensurepip/_bundled/
 sed "${sed_args[@]}" "s/^_PIP_VERSION = .*/_PIP_VERSION = \"${PIP_VERSION}\"/" Lib/ensurepip/__init__.py
 
+# Build relative runtime library paths into CPython 3.15+.
+if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_15}" ]; then
+    patch -p1 -i "${ROOT}/patch-relocatable-flag.patch"
+    patch -p1 -i "${ROOT}/patch-relocatable-link.patch"
+fi
+
 # configure doesn't support cross-compiling on Apple. Teach it.
 if [[ "${PYBUILD_PLATFORM}" = macos* && -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_13}" ]]; then
     if [ "${PYTHON_MAJMIN_VERSION}" = "3.12" ]; then
@@ -152,7 +158,9 @@ fi
 
 # The default build rule for the macOS dylib doesn't pick up libraries
 # from modules / makesetup. So patch it accordingly.
-if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" ]; then
+if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_15}" ]; then
+    patch -p1 -i "${ROOT}/patch-macos-link-extension-modules-15.patch"
+elif [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" ]; then
     patch -p1 -i "${ROOT}/patch-macos-link-extension-modules-13.patch"
 else
     patch -p1 -i "${ROOT}/patch-macos-link-extension-modules.patch"
@@ -333,6 +341,10 @@ fi
 
 EXTRA_CONFIGURE_FLAGS=
 
+if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_15}" ]; then
+    EXTRA_CONFIGURE_FLAGS="${EXTRA_CONFIGURE_FLAGS} --enable-relocatable"
+fi
+
 if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then
     CFLAGS="${CFLAGS} -I${TOOLS_PATH}/deps/include/uuid"
 
@@ -818,18 +830,22 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then
 
         # Fix the Python binary to reference libpython via @rpath and add
         # an rpath entry so it can find the library.
-        install_name_tool \
-            -change "/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" "@rpath/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \
-            -add_rpath @executable_path/../lib \
-            "${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}"
+        if [ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_14}" ]; then
+            install_name_tool \
+                -change "/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" "@rpath/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \
+                -add_rpath @executable_path/../lib \
+                "${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}"
+        fi
 
         # Python's build system doesn't make this file writable.
         chmod 755 "${ROOT}/out/python/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}"
         # Set libpython's install name to @rpath so binaries linking against it
         # can locate it via their own rpath entries.
-        install_name_tool \
-            -id "@rpath/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \
-            "${ROOT}/out/python/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}"
+        if [ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_14}" ]; then
+            install_name_tool \
+                -id "@rpath/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \
+                "${ROOT}/out/python/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}"
+        fi
 
         # We also normalize /tools/deps/lib/libz.1.dylib to the system location.
         install_name_tool \
@@ -839,19 +855,19 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then
             -change /tools/deps/lib/libz.1.dylib /usr/lib/libz.1.dylib \
             "${ROOT}/out/python/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}"
 
-        if [ -n "${PYTHON_BINARY_SUFFIX}" ]; then
+        if [[ -n "${PYTHON_BINARY_SUFFIX}" && -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_14}" ]]; then
             install_name_tool \
                 -change "/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" "@rpath/${LIBPYTHON_SHARED_LIBRARY_BASENAME}" \
                 -add_rpath @executable_path/../lib \
                 "${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}"
         fi
 
-        # At the moment, python3 and libpython don't have shared-library
-        # dependencies, but at some point we will want to run this for
-        # them too.
-        for module in ${ROOT}/out/python/install/lib/python*/lib-dynload/*.so; do
-            install_name_tool -add_rpath @loader_path/../.. "$module"
-        done
+        # The relocatable backport supplies extension search paths for CPython 3.15+.
+        if [ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_14}" ]; then
+            for module in ${ROOT}/out/python/install/lib/python*/lib-dynload/*.so; do
+                install_name_tool -add_rpath @loader_path/../.. "$module"
+            done
+        fi
     else # (not macos)
         LIBPYTHON_SHARED_LIBRARY_BASENAME=libpython${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}.so.1.0
         LIBPYTHON_SHARED_LIBRARY=${ROOT}/out/python/install/lib/${LIBPYTHON_SHARED_LIBRARY_BASENAME}
@@ -917,12 +933,14 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then
         # empty dummy libpython.so, which allows the link to succeed but
         # ensures they do not use any unwanted symbols. That might be
         # worth doing at some point.)
-        patchelf --force-rpath --set-rpath "\$ORIGIN/../lib" \
-            "${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}"
-
-        if [ -n "${PYTHON_BINARY_SUFFIX}" ]; then
+        if [ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_14}" ]; then
             patchelf --force-rpath --set-rpath "\$ORIGIN/../lib" \
-                "${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}"
+                "${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}"
+
+            if [ -n "${PYTHON_BINARY_SUFFIX}" ]; then
+                patchelf --force-rpath --set-rpath "\$ORIGIN/../lib" \
+                    "${ROOT}/out/python/install/bin/python${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}"
+            fi
         fi
 
         # For libpython3.so (the ABI3 library for embedders), we do
@@ -941,8 +959,9 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then
         # libpython3.x.so.1.0 that they want (and it's fine to use
         # DT_RUNPATH instead of DT_RPATH).
         if [ "${CC}" == "musl-clang" ]; then
-            # libpython3.so isn't present in debug builds.
-            if [ -z "${CPYTHON_DEBUG}" ]; then
+            # The relocatable backport supplies this path for CPython 3.15+.
+            # libpython3.so is absent from debug builds.
+            if [[ -z "${CPYTHON_DEBUG}" && -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_14}" ]]; then
                 patchelf --set-rpath "\$ORIGIN/../lib" \
                     "${ROOT}/out/python/install/lib/libpython3.so"
             fi
@@ -956,9 +975,11 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then
 
         # Let _tkinter locate bundled Tcl/Tk libraries independently of the
         # interpreter.
-        for module in "${ROOT}"/out/python/install/lib/python*/lib-dynload/_tkinter*.so; do
-            patchelf --force-rpath --set-rpath "\$ORIGIN/../.." "$module"
-        done
+        if [ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_14}" ]; then
+            for module in "${ROOT}"/out/python/install/lib/python*/lib-dynload/_tkinter*.so; do
+                patchelf --force-rpath --set-rpath "\$ORIGIN/../.." "$module"
+            done
+        fi
     fi
 fi
 

cpython-unix/patch-macos-link-extension-modules-15.patch

@@ -0,0 +1,20 @@
+--- a/Makefile.pre.in
++++ b/Makefile.pre.in
+@@ -1053,7 +1053,7 @@
+ 	$(BLDSHARED) $(NO_AS_NEEDED) -o $@ -Wl,-h$@ $^
+
+ libpython$(LDVERSION).dylib: $(LIBRARY_OBJS)
+-	 $(CC) -dynamiclib $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(SHLIBS) $(LIBC) $(LIBM); \
++	 $(CC) -dynamiclib $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \
+
+
+ $(RELOCATABLE_DIR)/libpython$(LDVERSION).so: $(LIBRARY_OBJS) $(DTRACE_OBJS)
+@@ -1066,7 +1066,7 @@
+
+ $(RELOCATABLE_DIR)/libpython$(LDVERSION).dylib: $(LIBRARY_OBJS)
+ 	@mkdir -p $(@D)
+-	$(CC) -dynamiclib $(PY_CORE_LDFLAGS) $(RELOCATABLE_LIBPYTHON_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(RELOCATABLE_LIBPYTHON_INSTALL_NAME) -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(SHLIBS) $(LIBC) $(LIBM)
++	$(CC) -dynamiclib $(PY_CORE_LDFLAGS) $(RELOCATABLE_LIBPYTHON_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(RELOCATABLE_LIBPYTHON_INSTALL_NAME) -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM)
+
+ libpython$(VERSION).sl: $(LIBRARY_OBJS)
+ 	$(LDSHARED) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM)

cpython-unix/patch-relocatable-flag.patch

@@ -0,0 +1,74 @@
+diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst
+index 0d420d8..2cc8e94 100644
+--- a/Doc/using/configure.rst
++++ b/Doc/using/configure.rst
+@@ -1073,6 +1073,17 @@ Linker options
+
+    Enable building a shared Python library: ``libpython`` (default is no).
+
++.. option:: --enable-relocatable
++
++   Enable experimental support for relocatable builds (default is no).
++   This option currently has no effect on the build.
++
++   .. warning::
++
++      This feature is under active development. Do not use it in production.
++
++   .. versionadded:: 3.16
++
+ .. option:: --without-static-libpython
+
+    Do not build ``libpythonMAJOR.MINOR.a`` and do not install ``python.o``
+diff --git a/Makefile.pre.in b/Makefile.pre.in
+index 36515db..7dcfc45 100644
+--- a/Makefile.pre.in
++++ b/Makefile.pre.in
+@@ -60,6 +60,7 @@ DSYMUTIL=       @DSYMUTIL@
+ DSYMUTIL_PATH=  @DSYMUTIL_PATH@
+
+ GNULD=		@GNULD@
++RELOCATABLE=	@RELOCATABLE@
+
+ # Shell used by make (some versions default to the login shell, which is bad)
+ SHELL=		/bin/sh -e
+diff --git a/configure.ac b/configure.ac
+index a8760c9..1afbeba 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -1594,6 +1594,21 @@ if test "x$enable_profiling" = xyes; then
+   LDFLAGS="-pg $LDFLAGS"
+ fi
+
++AC_MSG_CHECKING([for --enable-relocatable])
++AC_ARG_ENABLE([relocatable],
++  AS_HELP_STRING([--enable-relocatable],
++                 [enable experimental relocatable build support (default is no)]),
++  [], [enable_relocatable=no])
++AS_CASE([$enable_relocatable],
++  [yes|no], [],
++  [AC_MSG_ERROR([--enable-relocatable must be yes or no])])
++AC_MSG_RESULT([$enable_relocatable])
++AS_VAR_IF([enable_relocatable], [yes], [
++  AC_DEFINE([Py_ENABLE_RELOCATABLE], [1],
++            [Define if --enable-relocatable is used.])
++])
++AC_SUBST([RELOCATABLE], [$enable_relocatable])
++
+ AC_MSG_CHECKING([LDLIBRARY])
+
+ # Apple framework builds need more magic. LDLIBRARY is the dynamic
+diff --git a/pyconfig.h.in b/pyconfig.h.in
+index 08306a1..4fb029a 100644
+--- a/pyconfig.h.in
++++ b/pyconfig.h.in
+@@ -1880,6 +1880,9 @@
+ /* Define if you want to build an interpreter with many run-time checks. */
+ #undef Py_DEBUG
+
++/* Define if --enable-relocatable is used. */
++#undef Py_ENABLE_RELOCATABLE
++
+ /* Defined if Python is built as a shared library. */
+ #undef Py_ENABLE_SHARED
+