fix(toolchain): exclude libpython from runtime in recent releases

news/4091.fixed.md

@@ -0,0 +1,5 @@
+(toolchain) users can now request libpython to be not implemented via
+{target}`//python/config_settings:py_runtime_include_libpython=no`.
+The default is to include it, but it may change in the future. Ensure
+that you include `libpython` via the `<toolchain_repo>//:libpython` if
+it is actually needed at runtime.

news/fixed.4091.md

@@ -0,0 +1,4 @@
+(toolchain) {obj}`python_repository` now attempts to auto-detect the version
+for the hermetic toolchain and exclude the `libpython` from the runtime saving
+a little bit of MBs from the sandbox. Addresses
+([#3534](https://github.com/bazel-contrib/rules_python/issues/3534))

python/config_settings/BUILD.bazel

@@ -10,6 +10,7 @@ load(
     "LibcFlag",
     "PrecompileFlag",
     "PrecompileSourceRetentionFlag",
+    "PyRuntimeIncludeLibPython",
     "ValidateTestMainFlag",
     "VenvsSitePackages",
     "VenvsUseDeclareSymlinkFlag",
@@ -161,6 +162,40 @@ string_flag(
     visibility = ["//visibility:public"],
 )
 
+string_flag(
+    name = "py_runtime_include_libpython",
+    build_setting_default = PyRuntimeIncludeLibPython.YES,
+    values = PyRuntimeIncludeLibPython.flag_values(),
+    visibility = ["//visibility:public"],
+)
+
+config_setting(
+    name = "_is_py_runtime_include_libpython_auto",
+    flag_values = {
+        ":py_runtime_include_libpython": PyRuntimeIncludeLibPython.AUTO,
+    },
+    # NOTE: Only public because it is used in python_repository repos.
+    visibility = NOT_ACTUALLY_PUBLIC,
+)
+
+config_setting(
+    name = "_is_py_runtime_include_libpython_yes",
+    flag_values = {
+        ":py_runtime_include_libpython": PyRuntimeIncludeLibPython.YES,
+    },
+    # NOTE: Only public because it is used in python_repository repos.
+    visibility = NOT_ACTUALLY_PUBLIC,
+)
+
+config_setting(
+    name = "_is_py_runtime_include_libpython_no",
+    flag_values = {
+        ":py_runtime_include_libpython": PyRuntimeIncludeLibPython.NO,
+    },
+    # NOTE: Only public because it is used in python_repository repos.
+    visibility = NOT_ACTUALLY_PUBLIC,
+)
+
 # pip.parse related flags
 
 string_flag(

python/private/common_labels.bzl

@@ -29,6 +29,7 @@ labels = struct(
     PYTHON_VERSION_MAJOR_MINOR = str(Label("//python/config_settings:python_version_major_minor")),
     PY_FREETHREADED = str(Label("//python/config_settings:py_freethreaded")),
     PY_LINUX_LIBC = str(Label("//python/config_settings:py_linux_libc")),
+    PY_RUNTIME_INCLUDE_LIBPYTHON = str(Label("//python/config_settings:py_runtime_include_libpython")),
     REPL_DEP = str(Label("//python/bin:repl_dep")),
     VALIDATE_TEST_MAIN = str(Label("//python/config_settings:validate_test_main")),
     VENV = str(Label("//python/config_settings:venv")),

python/private/flags.bzl

@@ -261,3 +261,18 @@ LibcFlag = FlagEnum(
     MUSL = "musl",
     get_value = _libc_flag_get_value,
 )
+
+# Used for selectively including the libpython into the targets. By default
+# the hermetic Python toolchain statically links python3 binary, so that it
+# may be optional to include the shared library unless there are extensions
+# dynamically linking, which requires this at runtime.
+#
+# buildifier: disable=name-conventions
+PyRuntimeIncludeLibPython = FlagEnum(
+    # Automatically do the right thing - currently the same as yes.
+    AUTO = "auto",
+    # Include libpython
+    YES = "yes",
+    # Do not include libpython
+    NO = "no",
+)

python/private/hermetic_runtime_repo_setup.bzl

@@ -23,6 +23,9 @@ load(":version.bzl", "version")
 
 _IS_FREETHREADED_YES = Label("//python/config_settings:_is_py_freethreaded_yes")
 _IS_FREETHREADED_NO = Label("//python/config_settings:_is_py_freethreaded_no")
+_IS_PY_RUNTIME_INCLUDE_LIBPYTHON_AUTO = Label("//python/config_settings:_is_py_runtime_include_libpython_auto")
+_IS_PY_RUNTIME_INCLUDE_LIBPYTHON_YES = Label("//python/config_settings:_is_py_runtime_include_libpython_yes")
+_IS_PY_RUNTIME_INCLUDE_LIBPYTHON_NO = Label("//python/config_settings:_is_py_runtime_include_libpython_no")
 
 def define_hermetic_runtime_toolchain_impl(
         *,
@@ -31,7 +34,8 @@ def define_hermetic_runtime_toolchain_impl(
         extra_files_glob_exclude,
         python_version,
         python_bin,
-        coverage_tool):
+        coverage_tool,
+        python3_statically_links_libpython = True):
     """Define a toolchain implementation for a python-build-standalone repo.
 
     It expected this macro is called in the top-level package of an extracted
@@ -51,6 +55,13 @@ def define_hermetic_runtime_toolchain_impl(
             repository.
         coverage_tool: {type}`str` optional target to the coverage tool to
             use.
+        python3_statically_links_libpython: {type}`bool` a flag to enable omitting libpython
+            from the py_runtime registration because it is statically linked into `python3`.
+            This is switched via config flag
+            {target}`//python/config_settings/py_runtime_include_libpython`. Do this per-target
+            via transitions or globally.
+            :::{versionadded} VERSION_NEXT_FEATURE
+            :::
     """
     _ = name  # @unused
     version_info = version.parse(python_version)
@@ -67,9 +78,11 @@ def define_hermetic_runtime_toolchain_impl(
     ]
     files_include += extra_files_glob_include
     files_exclude = [
-        # Unused shared libraries. `python` executable and the `:libpython` target
-        # depend on `libpython{python_version}.so.1.0`.
-        "lib/libpython{major}.{minor}*.so".format(**version_dict),
+        # Unused shared libraries.
+        # `python` executable and the `:libpython` target depend on
+        # `libpython{python_version}.so.1.0`.
+        # we include
+        "lib/libpython*",
         # static libraries
         "lib/**/*.a",
         # tests for the standard libraries.
@@ -80,15 +93,28 @@ def define_hermetic_runtime_toolchain_impl(
     ]
     files_exclude += extra_files_glob_exclude
 
-    native.filegroup(
+    native.alias(
         name = "files",
+        actual = "files_all",
+    )
+    native.filegroup(
+        name = "files_all",
         srcs = native.glob(
             include = files_include,
             # Platform-agnostic filegroup can't match on all patterns.
             allow_empty = True,
             exclude = files_exclude,
         ),
     )
+    native.filegroup(
+        name = "files_no_libpython",
+        srcs = native.glob(
+            include = files_include,
+            # Platform-agnostic filegroup can't match on all patterns.
+            allow_empty = True,
+            exclude = files_exclude + ["lib/libpython*"],
+        ),
+    )
     cc_import(
         name = "interface",
         interface_library = select({
@@ -217,9 +243,22 @@ def define_hermetic_runtime_toolchain_impl(
             "rc": "candidate",
         }.get(version_info.pre[0])
 
+    if python3_statically_links_libpython:
+        no_libpython_requested = _IS_PY_RUNTIME_INCLUDE_LIBPYTHON_NO
+    else:
+        # We cannot omit it libpython even if the user requests it
+        no_libpython_requested = "@platforms//:incompatible"
+
     py_runtime(
         name = "py3_runtime",
-        files = [":files"],
+        files = select(
+            {
+                _IS_PY_RUNTIME_INCLUDE_LIBPYTHON_YES: [":files_all"],
+                _IS_PY_RUNTIME_INCLUDE_LIBPYTHON_AUTO: [":files_all"],
+                no_libpython_requested: [":files_no_libpython"],
+            },
+            no_match_error = "the archive does not support not including libpython",
+        ),
         interpreter = python_bin,
         interpreter_version_info = {
             "major": str(version_info.release[0]),
@@ -243,9 +282,7 @@ def define_hermetic_runtime_toolchain_impl(
         # On Windows, a symlink-style venv requires supporting .dll files.
         venv_bin_files = select({
             "@platforms//os:windows": native.glob(
-                include = [
-                    "*.dll",
-                ],
+                include = ["*.dll"],
                 # This must be true because glob empty-ness is checked
                 # during loading phase, before select() filters it out.
                 allow_empty = True,

python/private/python_repository.bzl

@@ -200,6 +200,17 @@ def _python_repository_impl(rctx):
     elif rctx.attr.distutils_content:
         rctx.file(distutils_path, rctx.attr.distutils_content)
 
+    # Support not including libraries into runtime
+    release = None
+    for url in urls:
+        head_and_release, _, _ = url.rpartition("/")
+        _, _, maybe_release = head_and_release.rpartition("/")
+        if not maybe_release.isdigit():
+            # Maybe this is some custom toolchain, so skip this
+            break
+
+        release = int(maybe_release)
+
     if "darwin" in platform and "osx" == repo_utils.get_platforms_os_name(rctx):
         # Fix up the Python distribution's LC_ID_DYLIB field.
         # It points to a build directory local to the GitHub Actions
@@ -218,26 +229,25 @@ def _python_repository_impl(rctx):
     _create_pycache_symlinks(rctx, logger)
     python_bin = "python.exe" if ("windows" in platform) else "bin/python3"
 
-    if "linux" in platform:
+    if "linux" in platform and release and release >= 20240224:
         # Workaround around https://github.com/astral-sh/python-build-standalone/issues/231
-        for url in urls:
-            head_and_release, _, _ = url.rpartition("/")
-            _, _, release = head_and_release.rpartition("/")
-            if not release.isdigit():
-                # Maybe this is some custom toolchain, so skip this
-                break
-
-            if int(release) >= 20240224:
-                # Starting with this release the Linux toolchains have infinite symlink loop
-                # on host platforms that are not Linux. Delete the files no
-                # matter the host platform so that the cross-built artifacts
-                # are the same irrespective of the host platform we are
-                # building on.
-                #
-                # Link to the first affected release:
-                # https://github.com/astral-sh/python-build-standalone/releases/tag/20240224
-                rctx.delete("share/terminfo")
-                break
+
+        # Starting with this release the Linux toolchains have infinite symlink loop
+        # on host platforms that are not Linux. Delete the files no
+        # matter the host platform so that the cross-built artifacts
+        # are the same irrespective of the host platform we are
+        # building on.
+        #
+        # Link to the first affected release:
+        # https://github.com/astral-sh/python-build-standalone/releases/tag/20240224
+        rctx.delete("share/terminfo")
+
+    if release and release >= 20250517:
+        # Starting with 20250517 we have python3 linked statically
+        # https://github.com/astral-sh/python-build-standalone/issues/941
+        python3_statically_links_libpython = False
+    else:
+        python3_statically_links_libpython = True
 
     glob_include = []
     glob_exclude = [
@@ -283,13 +293,15 @@ define_hermetic_runtime_toolchain_impl(
   python_version = {python_version},
   python_bin = {python_bin},
   coverage_tool = {coverage_tool},
+  python3_statically_links_libpython = {python3_statically_links_libpython}
 )
 """.format(
         extra_files_glob_exclude = render.list(glob_exclude),
         extra_files_glob_include = render.list(glob_include),
         python_bin = render.str(python_bin),
         python_version = render.str(rctx.attr.python_version),
         coverage_tool = render.str(coverage_tool),
+        python3_statically_links_libpython = python3_statically_links_libpython,
     )
     rctx.delete("python")
     rctx.symlink(python_bin, "python")