diff --git a/news/4091.fixed.md b/news/4091.fixed.md new file mode 100644 index 0000000000..ec0662e277 --- /dev/null +++ b/news/4091.fixed.md @@ -0,0 +1,5 @@ +(toolchain) users can now request libpython to be not implemented via +{target}`//python/config_settings:py_runtime_include_libpython=no`. +The default is to include it, but it may change in the future. Ensure +that you include `libpython` via the `//:libpython` if +it is actually needed at runtime. diff --git a/news/fixed.4091.md b/news/fixed.4091.md new file mode 100644 index 0000000000..5cac638a11 --- /dev/null +++ b/news/fixed.4091.md @@ -0,0 +1,4 @@ +(toolchain) {obj}`python_repository` now attempts to auto-detect the version +for the hermetic toolchain and exclude the `libpython` from the runtime saving +a little bit of MBs from the sandbox. Addresses +([#3534](https://github.com/bazel-contrib/rules_python/issues/3534)) diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel index d92aa4261c..1939e37f03 100644 --- a/python/config_settings/BUILD.bazel +++ b/python/config_settings/BUILD.bazel @@ -10,6 +10,7 @@ load( "LibcFlag", "PrecompileFlag", "PrecompileSourceRetentionFlag", + "PyRuntimeIncludeLibPython", "ValidateTestMainFlag", "VenvsSitePackages", "VenvsUseDeclareSymlinkFlag", @@ -161,6 +162,40 @@ string_flag( visibility = ["//visibility:public"], ) +string_flag( + name = "py_runtime_include_libpython", + build_setting_default = PyRuntimeIncludeLibPython.YES, + values = PyRuntimeIncludeLibPython.flag_values(), + visibility = ["//visibility:public"], +) + +config_setting( + name = "_is_py_runtime_include_libpython_auto", + flag_values = { + ":py_runtime_include_libpython": PyRuntimeIncludeLibPython.AUTO, + }, + # NOTE: Only public because it is used in python_repository repos. + visibility = NOT_ACTUALLY_PUBLIC, +) + +config_setting( + name = "_is_py_runtime_include_libpython_yes", + flag_values = { + ":py_runtime_include_libpython": PyRuntimeIncludeLibPython.YES, + }, + # NOTE: Only public because it is used in python_repository repos. + visibility = NOT_ACTUALLY_PUBLIC, +) + +config_setting( + name = "_is_py_runtime_include_libpython_no", + flag_values = { + ":py_runtime_include_libpython": PyRuntimeIncludeLibPython.NO, + }, + # NOTE: Only public because it is used in python_repository repos. + visibility = NOT_ACTUALLY_PUBLIC, +) + # pip.parse related flags string_flag( diff --git a/python/private/common_labels.bzl b/python/private/common_labels.bzl index db4a00ba0a..4bd53fe689 100644 --- a/python/private/common_labels.bzl +++ b/python/private/common_labels.bzl @@ -29,6 +29,7 @@ labels = struct( PYTHON_VERSION_MAJOR_MINOR = str(Label("//python/config_settings:python_version_major_minor")), PY_FREETHREADED = str(Label("//python/config_settings:py_freethreaded")), PY_LINUX_LIBC = str(Label("//python/config_settings:py_linux_libc")), + PY_RUNTIME_INCLUDE_LIBPYTHON = str(Label("//python/config_settings:py_runtime_include_libpython")), REPL_DEP = str(Label("//python/bin:repl_dep")), VALIDATE_TEST_MAIN = str(Label("//python/config_settings:validate_test_main")), VENV = str(Label("//python/config_settings:venv")), diff --git a/python/private/flags.bzl b/python/private/flags.bzl index 042e4e9838..0a708c7bd0 100644 --- a/python/private/flags.bzl +++ b/python/private/flags.bzl @@ -261,3 +261,18 @@ LibcFlag = FlagEnum( MUSL = "musl", get_value = _libc_flag_get_value, ) + +# Used for selectively including the libpython into the targets. By default +# the hermetic Python toolchain statically links python3 binary, so that it +# may be optional to include the shared library unless there are extensions +# dynamically linking, which requires this at runtime. +# +# buildifier: disable=name-conventions +PyRuntimeIncludeLibPython = FlagEnum( + # Automatically do the right thing - currently the same as yes. + AUTO = "auto", + # Include libpython + YES = "yes", + # Do not include libpython + NO = "no", +) diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl index 20b0324894..cdd9ec5170 100644 --- a/python/private/hermetic_runtime_repo_setup.bzl +++ b/python/private/hermetic_runtime_repo_setup.bzl @@ -23,6 +23,9 @@ load(":version.bzl", "version") _IS_FREETHREADED_YES = Label("//python/config_settings:_is_py_freethreaded_yes") _IS_FREETHREADED_NO = Label("//python/config_settings:_is_py_freethreaded_no") +_IS_PY_RUNTIME_INCLUDE_LIBPYTHON_AUTO = Label("//python/config_settings:_is_py_runtime_include_libpython_auto") +_IS_PY_RUNTIME_INCLUDE_LIBPYTHON_YES = Label("//python/config_settings:_is_py_runtime_include_libpython_yes") +_IS_PY_RUNTIME_INCLUDE_LIBPYTHON_NO = Label("//python/config_settings:_is_py_runtime_include_libpython_no") def define_hermetic_runtime_toolchain_impl( *, @@ -31,7 +34,8 @@ def define_hermetic_runtime_toolchain_impl( extra_files_glob_exclude, python_version, python_bin, - coverage_tool): + coverage_tool, + python3_statically_links_libpython = True): """Define a toolchain implementation for a python-build-standalone repo. It expected this macro is called in the top-level package of an extracted @@ -51,6 +55,13 @@ def define_hermetic_runtime_toolchain_impl( repository. coverage_tool: {type}`str` optional target to the coverage tool to use. + python3_statically_links_libpython: {type}`bool` a flag to enable omitting libpython + from the py_runtime registration because it is statically linked into `python3`. + This is switched via config flag + {target}`//python/config_settings/py_runtime_include_libpython`. Do this per-target + via transitions or globally. + :::{versionadded} VERSION_NEXT_FEATURE + ::: """ _ = name # @unused version_info = version.parse(python_version) @@ -67,9 +78,11 @@ def define_hermetic_runtime_toolchain_impl( ] files_include += extra_files_glob_include files_exclude = [ - # Unused shared libraries. `python` executable and the `:libpython` target - # depend on `libpython{python_version}.so.1.0`. - "lib/libpython{major}.{minor}*.so".format(**version_dict), + # Unused shared libraries. + # `python` executable and the `:libpython` target depend on + # `libpython{python_version}.so.1.0`. + # we include + "lib/libpython*", # static libraries "lib/**/*.a", # tests for the standard libraries. @@ -80,8 +93,12 @@ def define_hermetic_runtime_toolchain_impl( ] files_exclude += extra_files_glob_exclude - native.filegroup( + native.alias( name = "files", + actual = "files_all", + ) + native.filegroup( + name = "files_all", srcs = native.glob( include = files_include, # Platform-agnostic filegroup can't match on all patterns. @@ -89,6 +106,15 @@ def define_hermetic_runtime_toolchain_impl( exclude = files_exclude, ), ) + native.filegroup( + name = "files_no_libpython", + srcs = native.glob( + include = files_include, + # Platform-agnostic filegroup can't match on all patterns. + allow_empty = True, + exclude = files_exclude + ["lib/libpython*"], + ), + ) cc_import( name = "interface", interface_library = select({ @@ -217,9 +243,22 @@ def define_hermetic_runtime_toolchain_impl( "rc": "candidate", }.get(version_info.pre[0]) + if python3_statically_links_libpython: + no_libpython_requested = _IS_PY_RUNTIME_INCLUDE_LIBPYTHON_NO + else: + # We cannot omit it libpython even if the user requests it + no_libpython_requested = "@platforms//:incompatible" + py_runtime( name = "py3_runtime", - files = [":files"], + files = select( + { + _IS_PY_RUNTIME_INCLUDE_LIBPYTHON_YES: [":files_all"], + _IS_PY_RUNTIME_INCLUDE_LIBPYTHON_AUTO: [":files_all"], + no_libpython_requested: [":files_no_libpython"], + }, + no_match_error = "the archive does not support not including libpython", + ), interpreter = python_bin, interpreter_version_info = { "major": str(version_info.release[0]), @@ -243,9 +282,7 @@ def define_hermetic_runtime_toolchain_impl( # On Windows, a symlink-style venv requires supporting .dll files. venv_bin_files = select({ "@platforms//os:windows": native.glob( - include = [ - "*.dll", - ], + include = ["*.dll"], # This must be true because glob empty-ness is checked # during loading phase, before select() filters it out. allow_empty = True, diff --git a/python/private/python_repository.bzl b/python/private/python_repository.bzl index 9c44971117..48badce77b 100644 --- a/python/private/python_repository.bzl +++ b/python/private/python_repository.bzl @@ -200,6 +200,17 @@ def _python_repository_impl(rctx): elif rctx.attr.distutils_content: rctx.file(distutils_path, rctx.attr.distutils_content) + # Support not including libraries into runtime + release = None + for url in urls: + head_and_release, _, _ = url.rpartition("/") + _, _, maybe_release = head_and_release.rpartition("/") + if not maybe_release.isdigit(): + # Maybe this is some custom toolchain, so skip this + break + + release = int(maybe_release) + if "darwin" in platform and "osx" == repo_utils.get_platforms_os_name(rctx): # Fix up the Python distribution's LC_ID_DYLIB field. # It points to a build directory local to the GitHub Actions @@ -218,26 +229,25 @@ def _python_repository_impl(rctx): _create_pycache_symlinks(rctx, logger) python_bin = "python.exe" if ("windows" in platform) else "bin/python3" - if "linux" in platform: + if "linux" in platform and release and release >= 20240224: # Workaround around https://github.com/astral-sh/python-build-standalone/issues/231 - for url in urls: - head_and_release, _, _ = url.rpartition("/") - _, _, release = head_and_release.rpartition("/") - if not release.isdigit(): - # Maybe this is some custom toolchain, so skip this - break - - if int(release) >= 20240224: - # Starting with this release the Linux toolchains have infinite symlink loop - # on host platforms that are not Linux. Delete the files no - # matter the host platform so that the cross-built artifacts - # are the same irrespective of the host platform we are - # building on. - # - # Link to the first affected release: - # https://github.com/astral-sh/python-build-standalone/releases/tag/20240224 - rctx.delete("share/terminfo") - break + + # Starting with this release the Linux toolchains have infinite symlink loop + # on host platforms that are not Linux. Delete the files no + # matter the host platform so that the cross-built artifacts + # are the same irrespective of the host platform we are + # building on. + # + # Link to the first affected release: + # https://github.com/astral-sh/python-build-standalone/releases/tag/20240224 + rctx.delete("share/terminfo") + + if release and release >= 20250517: + # Starting with 20250517 we have python3 linked statically + # https://github.com/astral-sh/python-build-standalone/issues/941 + python3_statically_links_libpython = False + else: + python3_statically_links_libpython = True glob_include = [] glob_exclude = [ @@ -283,6 +293,7 @@ define_hermetic_runtime_toolchain_impl( python_version = {python_version}, python_bin = {python_bin}, coverage_tool = {coverage_tool}, + python3_statically_links_libpython = {python3_statically_links_libpython} ) """.format( extra_files_glob_exclude = render.list(glob_exclude), @@ -290,6 +301,7 @@ define_hermetic_runtime_toolchain_impl( python_bin = render.str(python_bin), python_version = render.str(rctx.attr.python_version), coverage_tool = render.str(coverage_tool), + python3_statically_links_libpython = python3_statically_links_libpython, ) rctx.delete("python") rctx.symlink(python_bin, "python")