From e27b109c35f9c985396a173cfd7d0af65e26b069 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 22 Aug 2026 23:33:48 +0900 Subject: [PATCH 1/5] fix(toolchain): exclude libpython from runtime in recent releases Use the already existing auto-detection and exclude libpython and friends as suggested in the upstream ticket. Fixes #3534 --- news/fixed.4091.md | 4 ++ .../private/hermetic_runtime_repo_setup.bzl | 9 +++- python/private/python_repository.bzl | 50 ++++++++++++------- 3 files changed, 43 insertions(+), 20 deletions(-) create mode 100644 news/fixed.4091.md diff --git a/news/fixed.4091.md b/news/fixed.4091.md new file mode 100644 index 0000000000..5cac638a11 --- /dev/null +++ b/news/fixed.4091.md @@ -0,0 +1,4 @@ +(toolchain) {obj}`python_repository` now attempts to auto-detect the version +for the hermetic toolchain and exclude the `libpython` from the runtime saving +a little bit of MBs from the sandbox. Addresses +([#3534](https://github.com/bazel-contrib/rules_python/issues/3534)) diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl index 20b0324894..ff8d626898 100644 --- a/python/private/hermetic_runtime_repo_setup.bzl +++ b/python/private/hermetic_runtime_repo_setup.bzl @@ -31,7 +31,8 @@ def define_hermetic_runtime_toolchain_impl( extra_files_glob_exclude, python_version, python_bin, - coverage_tool): + coverage_tool, + runtime_include_libpython = True): """Define a toolchain implementation for a python-build-standalone repo. It expected this macro is called in the top-level package of an extracted @@ -51,6 +52,10 @@ def define_hermetic_runtime_toolchain_impl( repository. coverage_tool: {type}`str` optional target to the coverage tool to use. + runtime_include_libpython: {type}`bool` a flag to consider python3 to + either include it (legacy behaviour) or not include it into the + runtime because it is statically linked. If needed, include it via + "libpython" target separately. """ _ = name # @unused version_info = version.parse(python_version) @@ -79,6 +84,8 @@ def define_hermetic_runtime_toolchain_impl( "**/__pycache__/*.pyc.*", ] files_exclude += extra_files_glob_exclude + if runtime_include_libpython: + files_exclude.append("lib/libpython*") native.filegroup( name = "files", diff --git a/python/private/python_repository.bzl b/python/private/python_repository.bzl index 9c44971117..3c0e7ae6c6 100644 --- a/python/private/python_repository.bzl +++ b/python/private/python_repository.bzl @@ -200,6 +200,17 @@ def _python_repository_impl(rctx): elif rctx.attr.distutils_content: rctx.file(distutils_path, rctx.attr.distutils_content) + # Support not including libraries into runtime + release = None + for url in urls: + head_and_release, _, _ = url.rpartition("/") + _, _, maybe_release = head_and_release.rpartition("/") + if not maybe_release.isdigit(): + # Maybe this is some custom toolchain, so skip this + break + + release = int(maybe_release) + if "darwin" in platform and "osx" == repo_utils.get_platforms_os_name(rctx): # Fix up the Python distribution's LC_ID_DYLIB field. # It points to a build directory local to the GitHub Actions @@ -218,26 +229,25 @@ def _python_repository_impl(rctx): _create_pycache_symlinks(rctx, logger) python_bin = "python.exe" if ("windows" in platform) else "bin/python3" - if "linux" in platform: + if "linux" in platform and release and release >= 20240224: # Workaround around https://github.com/astral-sh/python-build-standalone/issues/231 - for url in urls: - head_and_release, _, _ = url.rpartition("/") - _, _, release = head_and_release.rpartition("/") - if not release.isdigit(): - # Maybe this is some custom toolchain, so skip this - break - - if int(release) >= 20240224: - # Starting with this release the Linux toolchains have infinite symlink loop - # on host platforms that are not Linux. Delete the files no - # matter the host platform so that the cross-built artifacts - # are the same irrespective of the host platform we are - # building on. - # - # Link to the first affected release: - # https://github.com/astral-sh/python-build-standalone/releases/tag/20240224 - rctx.delete("share/terminfo") - break + + # Starting with this release the Linux toolchains have infinite symlink loop + # on host platforms that are not Linux. Delete the files no + # matter the host platform so that the cross-built artifacts + # are the same irrespective of the host platform we are + # building on. + # + # Link to the first affected release: + # https://github.com/astral-sh/python-build-standalone/releases/tag/20240224 + rctx.delete("share/terminfo") + + if release and release >= 20250517: + # Starting with 20250517 we have python3 linked statically + # https://github.com/astral-sh/python-build-standalone/issues/941 + runtime_include_libpython = False + else: + runtime_include_libpython = True glob_include = [] glob_exclude = [ @@ -283,6 +293,7 @@ define_hermetic_runtime_toolchain_impl( python_version = {python_version}, python_bin = {python_bin}, coverage_tool = {coverage_tool}, + runtime_include_libpython = {runtime_include_libpython} ) """.format( extra_files_glob_exclude = render.list(glob_exclude), @@ -290,6 +301,7 @@ define_hermetic_runtime_toolchain_impl( python_bin = render.str(python_bin), python_version = render.str(rctx.attr.python_version), coverage_tool = render.str(coverage_tool), + runtime_include_libpython = runtime_include_libpython, ) rctx.delete("python") rctx.symlink(python_bin, "python") From 34eba94a9eaf7f962128c4d3684ae62505d1e97d Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 23 Aug 2026 01:04:33 +0900 Subject: [PATCH 2/5] add news --- news/4091.fixed.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 news/4091.fixed.md diff --git a/news/4091.fixed.md b/news/4091.fixed.md new file mode 100644 index 0000000000..ec0662e277 --- /dev/null +++ b/news/4091.fixed.md @@ -0,0 +1,5 @@ +(toolchain) users can now request libpython to be not implemented via +{target}`//python/config_settings:py_runtime_include_libpython=no`. +The default is to include it, but it may change in the future. Ensure +that you include `libpython` via the `//:libpython` if +it is actually needed at runtime. From 64b99725fbf78069132072d239f545d7d21ac836 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 23 Aug 2026 01:05:42 +0900 Subject: [PATCH 3/5] sketch the interface --- python/config_settings/BUILD.bazel | 26 +++++++++++ python/private/flags.bzl | 15 +++++++ .../private/hermetic_runtime_repo_setup.bzl | 44 ++++++++++++------- 3 files changed, 69 insertions(+), 16 deletions(-) diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel index d92aa4261c..54c61ae8c9 100644 --- a/python/config_settings/BUILD.bazel +++ b/python/config_settings/BUILD.bazel @@ -10,6 +10,7 @@ load( "LibcFlag", "PrecompileFlag", "PrecompileSourceRetentionFlag", + "PyRuntimeIncludeLibPython", "ValidateTestMainFlag", "VenvsSitePackages", "VenvsUseDeclareSymlinkFlag", @@ -161,6 +162,31 @@ string_flag( visibility = ["//visibility:public"], ) +string_flag( + name = "py_runtime_include_libpython", + build_setting_default = PyRuntimeIncludeLibPython.YES, + values = PyRuntimeIncludeLibPython.flag_values(), + visibility = ["//visibility:public"], +) + +config_setting( + name = "_is_py_runtime_include_libpython_yes", + flag_values = { + ":py_runtime_include_libpython": PyRuntimeIncludeLibPython.YES, + }, + # NOTE: Only public because it is used in python_repository repos. + visibility = NOT_ACTUALLY_PUBLIC, +) + +config_setting( + name = "_is_py_runtime_include_libpython_auto", + flag_values = { + ":py_runtime_include_libpython": PyRuntimeIncludeLibPython.AUTO, + }, + # NOTE: Only public because it is used in python_repository repos. + visibility = NOT_ACTUALLY_PUBLIC, +) + # pip.parse related flags string_flag( diff --git a/python/private/flags.bzl b/python/private/flags.bzl index 042e4e9838..0a708c7bd0 100644 --- a/python/private/flags.bzl +++ b/python/private/flags.bzl @@ -261,3 +261,18 @@ LibcFlag = FlagEnum( MUSL = "musl", get_value = _libc_flag_get_value, ) + +# Used for selectively including the libpython into the targets. By default +# the hermetic Python toolchain statically links python3 binary, so that it +# may be optional to include the shared library unless there are extensions +# dynamically linking, which requires this at runtime. +# +# buildifier: disable=name-conventions +PyRuntimeIncludeLibPython = FlagEnum( + # Automatically do the right thing - currently the same as yes. + AUTO = "auto", + # Include libpython + YES = "yes", + # Do not include libpython + NO = "no", +) diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl index ff8d626898..1605d324c3 100644 --- a/python/private/hermetic_runtime_repo_setup.bzl +++ b/python/private/hermetic_runtime_repo_setup.bzl @@ -23,6 +23,8 @@ load(":version.bzl", "version") _IS_FREETHREADED_YES = Label("//python/config_settings:_is_py_freethreaded_yes") _IS_FREETHREADED_NO = Label("//python/config_settings:_is_py_freethreaded_no") +_IS_PY_RUNTIME_INCLUDE_LIBPYTHON_YES = Label("//python/config_settings:_is_py_runtime_include_libpython_yes") +_IS_PY_RUNTIME_INCLUDE_LIBPYTHON_AUTO = Label("//python/config_settings:_is_py_runtime_include_libpython_auto") def define_hermetic_runtime_toolchain_impl( *, @@ -31,8 +33,7 @@ def define_hermetic_runtime_toolchain_impl( extra_files_glob_exclude, python_version, python_bin, - coverage_tool, - runtime_include_libpython = True): + coverage_tool): """Define a toolchain implementation for a python-build-standalone repo. It expected this macro is called in the top-level package of an extracted @@ -52,10 +53,6 @@ def define_hermetic_runtime_toolchain_impl( repository. coverage_tool: {type}`str` optional target to the coverage tool to use. - runtime_include_libpython: {type}`bool` a flag to consider python3 to - either include it (legacy behaviour) or not include it into the - runtime because it is statically linked. If needed, include it via - "libpython" target separately. """ _ = name # @unused version_info = version.parse(python_version) @@ -72,9 +69,11 @@ def define_hermetic_runtime_toolchain_impl( ] files_include += extra_files_glob_include files_exclude = [ - # Unused shared libraries. `python` executable and the `:libpython` target - # depend on `libpython{python_version}.so.1.0`. - "lib/libpython{major}.{minor}*.so".format(**version_dict), + # Unused shared libraries. + # `python` executable and the `:libpython` target depend on + # `libpython{python_version}.so.1.0`. + # we include + "lib/libpython*", # static libraries "lib/**/*.a", # tests for the standard libraries. @@ -84,11 +83,13 @@ def define_hermetic_runtime_toolchain_impl( "**/__pycache__/*.pyc.*", ] files_exclude += extra_files_glob_exclude - if runtime_include_libpython: - files_exclude.append("lib/libpython*") - native.filegroup( + native.alias( name = "files", + actual = "files_all", + ) + native.filegroup( + name = "files_all", srcs = native.glob( include = files_include, # Platform-agnostic filegroup can't match on all patterns. @@ -96,6 +97,15 @@ def define_hermetic_runtime_toolchain_impl( exclude = files_exclude, ), ) + native.filegroup( + name = "files_no_libpython", + srcs = native.glob( + include = files_include, + # Platform-agnostic filegroup can't match on all patterns. + allow_empty = True, + exclude = files_exclude + ["lib/libpython*"], + ), + ) cc_import( name = "interface", interface_library = select({ @@ -226,7 +236,11 @@ def define_hermetic_runtime_toolchain_impl( py_runtime( name = "py3_runtime", - files = [":files"], + files = select({ + _IS_PY_RUNTIME_INCLUDE_LIBPYTHON_YES: [":files_all"], + _IS_PY_RUNTIME_INCLUDE_LIBPYTHON_AUTO: [":files_all"], + "//conditions:default": [":files_no_libpython"], + }), interpreter = python_bin, interpreter_version_info = { "major": str(version_info.release[0]), @@ -250,9 +264,7 @@ def define_hermetic_runtime_toolchain_impl( # On Windows, a symlink-style venv requires supporting .dll files. venv_bin_files = select({ "@platforms//os:windows": native.glob( - include = [ - "*.dll", - ], + include = ["*.dll"], # This must be true because glob empty-ness is checked # during loading phase, before select() filters it out. allow_empty = True, From 2e7c65e383122dc429fcdc254b4e2e333c611aa8 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 23 Aug 2026 01:17:37 +0900 Subject: [PATCH 4/5] sketch the interface --- python/config_settings/BUILD.bazel | 13 +++++-- .../private/hermetic_runtime_repo_setup.bzl | 34 ++++++++++++++----- python/private/python_repository.bzl | 8 ++--- 3 files changed, 41 insertions(+), 14 deletions(-) diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel index 54c61ae8c9..1939e37f03 100644 --- a/python/config_settings/BUILD.bazel +++ b/python/config_settings/BUILD.bazel @@ -169,6 +169,15 @@ string_flag( visibility = ["//visibility:public"], ) +config_setting( + name = "_is_py_runtime_include_libpython_auto", + flag_values = { + ":py_runtime_include_libpython": PyRuntimeIncludeLibPython.AUTO, + }, + # NOTE: Only public because it is used in python_repository repos. + visibility = NOT_ACTUALLY_PUBLIC, +) + config_setting( name = "_is_py_runtime_include_libpython_yes", flag_values = { @@ -179,9 +188,9 @@ config_setting( ) config_setting( - name = "_is_py_runtime_include_libpython_auto", + name = "_is_py_runtime_include_libpython_no", flag_values = { - ":py_runtime_include_libpython": PyRuntimeIncludeLibPython.AUTO, + ":py_runtime_include_libpython": PyRuntimeIncludeLibPython.NO, }, # NOTE: Only public because it is used in python_repository repos. visibility = NOT_ACTUALLY_PUBLIC, diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl index 1605d324c3..c847f0527c 100644 --- a/python/private/hermetic_runtime_repo_setup.bzl +++ b/python/private/hermetic_runtime_repo_setup.bzl @@ -23,8 +23,9 @@ load(":version.bzl", "version") _IS_FREETHREADED_YES = Label("//python/config_settings:_is_py_freethreaded_yes") _IS_FREETHREADED_NO = Label("//python/config_settings:_is_py_freethreaded_no") -_IS_PY_RUNTIME_INCLUDE_LIBPYTHON_YES = Label("//python/config_settings:_is_py_runtime_include_libpython_yes") -_IS_PY_RUNTIME_INCLUDE_LIBPYTHON_AUTO = Label("//python/config_settings:_is_py_runtime_include_libpython_auto") +_IS_PY_RUNTIME_INCLUDE_LIBPYTHON_AUTO = Label("//python/config_settings:_is_py_runtime_include_lib_python_auto") +_IS_PY_RUNTIME_INCLUDE_LIBPYTHON_YES = Label("//python/config_settings:_is_py_runtime_include_lib_python_yes") +_IS_PY_RUNTIME_INCLUDE_LIBPYTHON_NO = Label("//python/config_settings:_is_py_runtime_include_lib_python_no") def define_hermetic_runtime_toolchain_impl( *, @@ -33,7 +34,8 @@ def define_hermetic_runtime_toolchain_impl( extra_files_glob_exclude, python_version, python_bin, - coverage_tool): + coverage_tool, + python3_statically_links_libpython = True): """Define a toolchain implementation for a python-build-standalone repo. It expected this macro is called in the top-level package of an extracted @@ -53,6 +55,13 @@ def define_hermetic_runtime_toolchain_impl( repository. coverage_tool: {type}`str` optional target to the coverage tool to use. + python3_statically_links_libpython: {type}`bool` a flag to enable omitting libpython + from the py_runtime registration because it is statically linked into `python3`. + This is switched via config flag + {target}`//python/config_settings/py_runtime_include_libpython`. Do this per-target + via transitions or globally. + :::{versionadded} VERSION_NEXT_FEATURE + ::: """ _ = name # @unused version_info = version.parse(python_version) @@ -234,13 +243,22 @@ def define_hermetic_runtime_toolchain_impl( "rc": "candidate", }.get(version_info.pre[0]) + if python3_statically_links_libpython: + no_libpython_requested = _IS_PY_RUNTIME_INCLUDE_LIBPYTHON_NO + else: + # We cannot omit it libpython even if the user requests it + no_libpython_requested = "@platforms//:incompatible" + py_runtime( name = "py3_runtime", - files = select({ - _IS_PY_RUNTIME_INCLUDE_LIBPYTHON_YES: [":files_all"], - _IS_PY_RUNTIME_INCLUDE_LIBPYTHON_AUTO: [":files_all"], - "//conditions:default": [":files_no_libpython"], - }), + files = select( + { + _IS_PY_RUNTIME_INCLUDE_LIBPYTHON_YES: [":files_all"], + _IS_PY_RUNTIME_INCLUDE_LIBPYTHON_AUTO: [":files_all"], + no_libpython_requested: [":files_no_libpython"], + }, + error = "BUG", + ), interpreter = python_bin, interpreter_version_info = { "major": str(version_info.release[0]), diff --git a/python/private/python_repository.bzl b/python/private/python_repository.bzl index 3c0e7ae6c6..48badce77b 100644 --- a/python/private/python_repository.bzl +++ b/python/private/python_repository.bzl @@ -245,9 +245,9 @@ def _python_repository_impl(rctx): if release and release >= 20250517: # Starting with 20250517 we have python3 linked statically # https://github.com/astral-sh/python-build-standalone/issues/941 - runtime_include_libpython = False + python3_statically_links_libpython = False else: - runtime_include_libpython = True + python3_statically_links_libpython = True glob_include = [] glob_exclude = [ @@ -293,7 +293,7 @@ define_hermetic_runtime_toolchain_impl( python_version = {python_version}, python_bin = {python_bin}, coverage_tool = {coverage_tool}, - runtime_include_libpython = {runtime_include_libpython} + python3_statically_links_libpython = {python3_statically_links_libpython} ) """.format( extra_files_glob_exclude = render.list(glob_exclude), @@ -301,7 +301,7 @@ define_hermetic_runtime_toolchain_impl( python_bin = render.str(python_bin), python_version = render.str(rctx.attr.python_version), coverage_tool = render.str(coverage_tool), - runtime_include_libpython = runtime_include_libpython, + python3_statically_links_libpython = python3_statically_links_libpython, ) rctx.delete("python") rctx.symlink(python_bin, "python") From 77d36a6581649193768b15b81a05286279b3b43e Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 23 Aug 2026 01:44:06 +0900 Subject: [PATCH 5/5] fixup --- python/private/common_labels.bzl | 1 + python/private/hermetic_runtime_repo_setup.bzl | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/python/private/common_labels.bzl b/python/private/common_labels.bzl index db4a00ba0a..4bd53fe689 100644 --- a/python/private/common_labels.bzl +++ b/python/private/common_labels.bzl @@ -29,6 +29,7 @@ labels = struct( PYTHON_VERSION_MAJOR_MINOR = str(Label("//python/config_settings:python_version_major_minor")), PY_FREETHREADED = str(Label("//python/config_settings:py_freethreaded")), PY_LINUX_LIBC = str(Label("//python/config_settings:py_linux_libc")), + PY_RUNTIME_INCLUDE_LIBPYTHON = str(Label("//python/config_settings:py_runtime_include_libpython")), REPL_DEP = str(Label("//python/bin:repl_dep")), VALIDATE_TEST_MAIN = str(Label("//python/config_settings:validate_test_main")), VENV = str(Label("//python/config_settings:venv")), diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl index c847f0527c..cdd9ec5170 100644 --- a/python/private/hermetic_runtime_repo_setup.bzl +++ b/python/private/hermetic_runtime_repo_setup.bzl @@ -23,9 +23,9 @@ load(":version.bzl", "version") _IS_FREETHREADED_YES = Label("//python/config_settings:_is_py_freethreaded_yes") _IS_FREETHREADED_NO = Label("//python/config_settings:_is_py_freethreaded_no") -_IS_PY_RUNTIME_INCLUDE_LIBPYTHON_AUTO = Label("//python/config_settings:_is_py_runtime_include_lib_python_auto") -_IS_PY_RUNTIME_INCLUDE_LIBPYTHON_YES = Label("//python/config_settings:_is_py_runtime_include_lib_python_yes") -_IS_PY_RUNTIME_INCLUDE_LIBPYTHON_NO = Label("//python/config_settings:_is_py_runtime_include_lib_python_no") +_IS_PY_RUNTIME_INCLUDE_LIBPYTHON_AUTO = Label("//python/config_settings:_is_py_runtime_include_libpython_auto") +_IS_PY_RUNTIME_INCLUDE_LIBPYTHON_YES = Label("//python/config_settings:_is_py_runtime_include_libpython_yes") +_IS_PY_RUNTIME_INCLUDE_LIBPYTHON_NO = Label("//python/config_settings:_is_py_runtime_include_libpython_no") def define_hermetic_runtime_toolchain_impl( *, @@ -257,7 +257,7 @@ def define_hermetic_runtime_toolchain_impl( _IS_PY_RUNTIME_INCLUDE_LIBPYTHON_AUTO: [":files_all"], no_libpython_requested: [":files_no_libpython"], }, - error = "BUG", + no_match_error = "the archive does not support not including libpython", ), interpreter = python_bin, interpreter_version_info = {