fix(zipapp): reduce bundled interpreter size (#4165) Bazel reports source symlinks as regular files, causing Python executable aliases to be copied into zipapps as full binaries. Hermetic runtimes also include shared libpython artifacts even when the interpreter links Python statically. Preserve source symlinks through sandbox indirection and exclude libpython shared objects from runtime files while retaining them for explicit native dependencies. Add regression coverage for archive structure and native extension loading. Fixes #3534 Fixes #4163 --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: Richard Levasseur <richardlev@gmail.com> (cherry picked from commit fd22cfff38697b2c10b3feff88ea55d6d57bed7f) Work towards #4175
diff --git a/CHANGELOG.md b/CHANGELOG.md index 013d33c..fbef417 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md
@@ -47,6 +47,15 @@ in {obj}`python.defaults`, {obj}`python_register_toolchains`, and {obj}`python_register_multi_toolchains` ([#4023](https://github.com/bazel-contrib/rules_python/pull/4023)). +* (toolchains) Shared `libpython*.so*` files are now excluded by default from + hermetic Python runtime files for recognized Astral `python-build-standalone` + builds (`20250517` and newer), which statically link `libpython` into the + interpreter. Older, custom, or unrecognized runtimes retain them. Set + `libpython = "include"` or `"exclude"` on {obj}`python.override`, + {obj}`python.single_version_override`, + {obj}`python.single_version_platform_override`, or + {obj}`python_register_toolchains` to override this behavior + ([#4165](https://github.com/bazel-contrib/rules_python/pull/4165)). {#v2-4-0-fixed} ### Fixed @@ -81,6 +90,10 @@ * (toolchain) Fixed a crash on Windows when precompiling is enabled: the precompiler's interpreter couldn't find its DLLs once relocated ([#4082](https://github.com/bazel-contrib/rules_python/issues/4082)). +* (zipapp) Reduced self-contained archive sizes by preserving Python executable + symlinks instead of storing each alias as another copy of the interpreter, and + by omitting shared `libpython` files from recognized statically linked Astral + runtime builds. {#v2-4-0-added} ### Added @@ -92,6 +105,7 @@ ([#4029](https://github.com/bazel-contrib/rules_python/issues/4029)). + {#v2-3-3} ## [2.3.3] - 2026-09-04
diff --git a/docs/toolchains.md b/docs/toolchains.md index de495d5..3ddce7b 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md
@@ -240,12 +240,22 @@ via {attr}`python.override.minor_mapping`. * Per-version control of the coverage tool used using {attr}`python.single_version_platform_override.coverage_tool`. +* Control of shared `libpython` files using the `libpython` attribute on + {bzl:obj}`python.override`, {bzl:obj}`python.single_version_override`, or + {bzl:obj}`python.single_version_platform_override`. * Adding additional Python versions via {bzl:obj}`python.single_version_override` or {bzl:obj}`python.single_version_platform_override`. * Adding additional Python versions dynamically from a manifest file or URL via {attr}`python.override.add_runtime_manifest_files` or {attr}`python.override.add_runtime_manifest_urls`. +The `libpython` attribute accepts `auto`, `include`, or `exclude`, and defaults +to `auto`. In automatic mode, shared `libpython` files are excluded only for +recognized Astral Python Standalone builds from `20250517` onward, which are +known to statically link `libpython` into the interpreter. Unknown, custom, and +older runtimes retain the shared libraries. Use `include` or `exclude` to +override this behavior when mirroring or customizing a runtime. + ### Registering custom runtimes Because the python-build-standalone project has _thousands_ of prebuilt runtimes
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index b4ff84f..b88fb72 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel
@@ -780,6 +780,7 @@ srcs = ["python_repository.bzl"], deps = [ ":auth", + ":pbs_manifest", ":repo_utils", ":text_util", "//python:versions",
diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl index 20b0324..d61a040 100644 --- a/python/private/hermetic_runtime_repo_setup.bzl +++ b/python/private/hermetic_runtime_repo_setup.bzl
@@ -29,6 +29,7 @@ name, extra_files_glob_include, extra_files_glob_exclude, + interpreter_has_static_libpython = False, python_version, python_bin, coverage_tool): @@ -45,6 +46,8 @@ binaries). extra_files_glob_exclude: {type}`list[str]` additional glob exclude patterns for the target runtime files. + interpreter_has_static_libpython: {type}`bool` whether the interpreter + includes libpython statically. python_version: {type}`str` The Python version, in `major.minor.micro` format. python_bin: {type}`str` The path to the Python binary within the @@ -78,6 +81,8 @@ # During pyc creation, temp files named *.pyc.NNN are created "**/__pycache__/*.pyc.*", ] + if interpreter_has_static_libpython: + files_exclude.append("lib/libpython*.so*") files_exclude += extra_files_glob_exclude native.filegroup(
diff --git a/python/private/pbs_manifest.bzl b/python/private/pbs_manifest.bzl index 86434dc..ad019df 100644 --- a/python/private/pbs_manifest.bzl +++ b/python/private/pbs_manifest.bzl
@@ -1,5 +1,11 @@ """Helper functions to parse python-build-standalone manifests.""" +_ASTRAL_STATIC_LIBPYTHON_RELEASE = 20250517 +_ASTRAL_RELEASE_URL_PREFIXES = [ + "https://github.com/astral-sh/python-build-standalone/releases/download/", + "https://releases.astral.sh/github/python-build-standalone/releases/download/", +] + def parse_filename(filename): """Parses a python-build-standalone filename (or URL) into its components. @@ -112,6 +118,28 @@ "vendor": vendor, } +# buildifier: disable=function-docstring-args +# buildifier: disable=function-docstring-return +# urls: list[str], release_filename: str -> bool +def is_astral_static_libpython_build(urls, release_filename): + """Whether an Astral build includes libpython statically in its interpreter.""" + parsed = parse_filename(release_filename) + if not parsed: + return False + + build_version = parsed["build_version"] + if ( + not build_version.isdigit() or + int(build_version) < _ASTRAL_STATIC_LIBPYTHON_RELEASE + ): + return False + + return any([ + url.startswith(prefix) + for url in urls + for prefix in _ASTRAL_RELEASE_URL_PREFIXES + ]) + def parse_runtime_manifest(content): """Parses the SHA256SUMS file content into a list of structs.
diff --git a/python/private/python.bzl b/python/private/python.bzl index 9fed939..8256e1b 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl
@@ -612,6 +612,8 @@ kwargs.setdefault(tag.python_version, {})["distutils_content"] = tag.distutils_content if tag.distutils: kwargs.setdefault(tag.python_version, {})["distutils"] = tag.distutils + if tag.libpython and tag.libpython != "auto": + kwargs.setdefault(tag.python_version, {})["libpython"] = tag.libpython def _process_single_version_platform_overrides(*, tag, _fail = fail, default): if not _validate_version(tag.python_version, _fail = _fail): @@ -641,6 +643,8 @@ if tag.urls: available_versions[tag.python_version].setdefault("url", {})[tag.platform] = tag.urls + if tag.libpython and tag.libpython != "auto": + available_versions[tag.python_version].setdefault("libpython", {})[tag.platform] = tag.libpython # If platform is customized, or doesn't exist, (re)define one. if ((tag.target_compatible_with or tag.target_settings or tag.os_name or tag.arch) or @@ -719,6 +723,7 @@ forwarded_attrs = sorted(AUTH_ATTRS) + [ "base_urls", + "libpython", "register_all_versions", ] for key in forwarded_attrs: @@ -1348,6 +1353,19 @@ doc = """Deprecated; do not use. This attribute has no effect.""", mandatory = False, ), + "libpython": attr.string( + default = "auto", + doc = """Whether to include shared libpython files. + +Valid values are `auto`, `include`, and `exclude`. With `auto`, recognized +Astral Python Standalone builds from 20250517 onward exclude the shared +libraries; other runtimes retain them. + +:::{versionadded} 2.4.0 +::: +""", + values = ["auto", "include", "exclude"], + ), "minor_mapping": attr.string_dict( mandatory = False, doc = """\ @@ -1424,6 +1442,19 @@ "Either {attr}`distutils` or {attr}`distutils_content` can be specified, but not both.", mandatory = False, ), + "libpython": attr.string( + default = "auto", + doc = """Whether to include shared libpython files. + +Valid values are `auto`, `include`, and `exclude`. With `auto`, recognized +Astral Python Standalone builds from 20250517 onward exclude the shared +libraries; other runtimes retain them. + +:::{versionadded} 2.4.0 +::: +""", + values = ["auto", "include", "exclude"], + ), "patch_strip": attr.int( mandatory = False, doc = "Same as the --strip argument of Unix patch.", @@ -1493,6 +1524,19 @@ `rules_python` defaults. """, ), + "libpython": attr.string( + default = "auto", + doc = """Whether to include shared libpython files. + +Valid values are `auto`, `include`, and `exclude`. With `auto`, recognized +Astral Python Standalone builds from 20250517 onward exclude the shared +libraries; other runtimes retain them. + +:::{versionadded} 2.4.0 +::: +""", + values = ["auto", "include", "exclude"], + ), "os_name": attr.string( doc = """ The host OS the runtime is compatible with.
diff --git a/python/private/python_register_toolchains.bzl b/python/private/python_register_toolchains.bzl index c6f827b..770c6a6 100644 --- a/python/private/python_register_toolchains.bzl +++ b/python/private/python_register_toolchains.bzl
@@ -40,6 +40,7 @@ register_toolchains = True, register_coverage_tool = False, set_python_version_constraint = False, + libpython = None, tool_versions = None, platforms = PLATFORMS, minor_mapping = None, @@ -68,6 +69,8 @@ set_python_version_constraint: {type}`bool` When set to `True`, `target_compatible_with` for the toolchains will include a version constraint. + libpython: {type}`str` controls shared libpython files: `auto`, + `include`, or `exclude`. tool_versions: {type}`dict` contains a mapping of version with SHASUM and platform info. If not supplied, the defaults in python/versions.bzl will be used. @@ -145,6 +148,7 @@ urls = urls, strip_prefix = strip_prefix, coverage_tool = coverage_tool, + libpython = libpython or tool_versions[python_version].get("libpython", {}).get(platform, "auto"), **kwargs ) if register_toolchains:
diff --git a/python/private/python_repository.bzl b/python/private/python_repository.bzl index 9c44971..3fd0928 100644 --- a/python/private/python_repository.bzl +++ b/python/private/python_repository.bzl
@@ -17,6 +17,7 @@ load("//python:versions.bzl", "FREETHREADED", "INSTALL_ONLY") load(":auth.bzl", "get_auth") +load(":pbs_manifest.bzl", "is_astral_static_libpython_build") load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") load(":text_util.bzl", "render") @@ -162,6 +163,12 @@ *python_version_info ) urls = rctx.attr.urls or [rctx.attr.url] + if rctx.attr.libpython == "include": + interpreter_has_static_libpython = False + elif rctx.attr.libpython == "exclude": + interpreter_has_static_libpython = True + else: + interpreter_has_static_libpython = is_astral_static_libpython_build(urls, release_filename) auth = get_auth(rctx, urls) if INSTALL_ONLY in release_filename: @@ -277,16 +284,18 @@ package(default_visibility = ["//visibility:public"]) define_hermetic_runtime_toolchain_impl( - name = "define_runtime", - extra_files_glob_include = {extra_files_glob_include}, - extra_files_glob_exclude = {extra_files_glob_exclude}, - python_version = {python_version}, - python_bin = {python_bin}, - coverage_tool = {coverage_tool}, + name = "define_runtime", + extra_files_glob_include = {extra_files_glob_include}, + extra_files_glob_exclude = {extra_files_glob_exclude}, + interpreter_has_static_libpython = {interpreter_has_static_libpython}, + python_version = {python_version}, + python_bin = {python_bin}, + coverage_tool = {coverage_tool}, ) """.format( extra_files_glob_exclude = render.list(glob_exclude), extra_files_glob_include = render.list(glob_include), + interpreter_has_static_libpython = str(interpreter_has_static_libpython), python_bin = render.str(python_bin), python_version = render.str(rctx.attr.python_version), coverage_tool = render.str(coverage_tool), @@ -361,6 +370,11 @@ doc = "Noop, will be removed in the next major release", mandatory = False, ), + "libpython": attr.string( + default = "auto", + doc = "Whether to include shared libpython files: auto, include, or exclude.", + values = ["auto", "include", "exclude"], + ), "netrc": attr.string( doc = ".netrc file to use for authentication; mirrors the eponymous attribute from http_archive", ),
diff --git a/tests/py_zipapp/venv_zipapp_test.py b/tests/py_zipapp/venv_zipapp_test.py index bd26d53..b906073 100644 --- a/tests/py_zipapp/venv_zipapp_test.py +++ b/tests/py_zipapp/venv_zipapp_test.py
@@ -84,6 +84,12 @@ # On Windows, pyvenv.cfg and bin/python3 are generated at runtime. if os.name != "nt": + self.assertFalse( + any( + "/lib/libpython" in name and ".so" in name for name in namelist + ), + "Statically linked Python should not bundle libpython", + ) self.assertHasPathMatchingSuffix(namelist, "/pyvenv.cfg") # The venv directory name depends on the target name, so find it
diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index ef7ccf0..22cd5c2 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl
@@ -419,6 +419,28 @@ _tests.append(_test_auth_overrides) +def _test_libpython_override(env): + py = parse_modules( + module_ctx = python_ext.mctx( + python_ext.module( + name = "my_module", + is_root = True, + override = [ + python_ext.override(libpython = "exclude"), + ], + toolchain = [python_ext.toolchain(python_version = "3.12")], + ), + _rules_python_module(), + ), + logger = repo_utils.logger(verbosity_level = 0, name = "python"), + ) + + env.expect.that_dict(py.config.default).contains_at_least({ + "libpython": "exclude", + }) + +_tests.append(_test_libpython_override) + def _test_add_target_settings(env): py = parse_modules( module_ctx = python_ext.mctx(
diff --git a/tests/python_bzlmod_ext/parse_runtime_manifest_tests.bzl b/tests/python_bzlmod_ext/parse_runtime_manifest_tests.bzl index 4493576..8c9176a 100644 --- a/tests/python_bzlmod_ext/parse_runtime_manifest_tests.bzl +++ b/tests/python_bzlmod_ext/parse_runtime_manifest_tests.bzl
@@ -4,7 +4,7 @@ load("@rules_testing//lib:analysis_test.bzl", "analysis_test") load("@rules_testing//lib:test_suite.bzl", "test_suite") load("@rules_testing//lib:util.bzl", rt_util = "util") -load("//python/private:pbs_manifest.bzl", "parse_filename", "parse_runtime_manifest") # buildifier: disable=bzl-visibility +load("//python/private:pbs_manifest.bzl", "is_astral_static_libpython_build", "parse_filename", "parse_runtime_manifest") # buildifier: disable=bzl-visibility _tests = [] @@ -97,6 +97,47 @@ _tests.append(_test_parse_filename_baseline) +def _test_is_astral_static_libpython_build(name): + rt_util.helper_target( + native.filegroup, + name = name + "_subject", + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_is_astral_static_libpython_build_impl, + ) + +def _test_is_astral_static_libpython_build_impl(env, target): + _ = target # @unused + + github_url = "https://github.com/astral-sh/python-build-standalone/releases/download/20250517/archive.tar.gz" + mirror_url = "https://releases.astral.sh/github/python-build-standalone/releases/download/20250517/archive.tar.gz" + cutoff_filename = "cpython-3.13.4+20250517-x86_64-unknown-linux-gnu-install_only.tar.gz" + + env.expect.that_bool(is_astral_static_libpython_build( + [github_url], + cutoff_filename, + )).equals(True) + env.expect.that_bool(is_astral_static_libpython_build( + [mirror_url], + cutoff_filename, + )).equals(True) + env.expect.that_bool(is_astral_static_libpython_build( + [github_url], + "cpython-3.13.3+20250516-x86_64-unknown-linux-gnu-install_only.tar.gz", + )).equals(False) + env.expect.that_bool(is_astral_static_libpython_build( + ["https://example.com/20250604/archive.tar.gz"], + cutoff_filename, + )).equals(False) + env.expect.that_bool(is_astral_static_libpython_build( + [github_url], + "custom-python.tar.gz", + )).equals(False) + +_tests.append(_test_is_astral_static_libpython_build) + def _test_parse_runtime_manifest(name): """Sets up the manifest file parsing test.
diff --git a/tests/support/mocks/python_ext.bzl b/tests/support/mocks/python_ext.bzl index 4c68e87..419afe8 100644 --- a/tests/support/mocks/python_ext.bzl +++ b/tests/support/mocks/python_ext.bzl
@@ -32,6 +32,7 @@ "available_python_versions": [], "base_urls": ["https://github.com/astral-sh/python-build-standalone/releases/download"], "ignore_root_user_error": True, + "libpython": "auto", "minor_mapping": {}, "register_all_versions": False, "runtime_manifest_sha": "", @@ -53,6 +54,7 @@ attrs = { "distutils": None, "distutils_content": "", + "libpython": "auto", "patch_strip": 0, "patches": [], "python_version": "", @@ -68,6 +70,7 @@ attrs = { "arch": "", "coverage_tool": None, + "libpython": "auto", "os_name": "", "patch_strip": 0, "patches": [],
diff --git a/tests/tools/zipapp/zipper_test.py b/tests/tools/zipapp/zipper_test.py index e4f25c5..e0ff155 100644 --- a/tests/tools/zipapp/zipper_test.py +++ b/tests/tools/zipapp/zipper_test.py
@@ -115,6 +115,65 @@ ) +def test_create_zip_with_source_symlink_marked_as_regular(tmp_path): + manifest_path = tmp_path / "manifest.txt" + output_zip = tmp_path / "output.zip" + + target_path = tmp_path / "python3.14" + target_path.write_text("python") + symlink_path = tmp_path / "python" + symlink_path.symlink_to(target_path.name) + manifest_path.write_text(f"rf-file|0|bin/python|{symlink_path}") + + create_zip(manifest_path, output_zip) + + with zipfile.ZipFile(output_zip, "r") as zf: + assert_zip_file_content( + zf, + "runfiles/my_ws/bin/python", + is_symlink_file=True, + target=target_path.name, + ) + + +def test_create_zip_with_sandboxed_source_symlink(tmp_path): + manifest_path = tmp_path / "manifest.txt" + output_zip = tmp_path / "output.zip" + + source_dir = tmp_path / "source" + source_dir.mkdir() + target_path = source_dir / "python3.14" + target_path.write_text("python") + source_symlink = source_dir / "python" + source_symlink.symlink_to(target_path.name) + + sandbox_dir = tmp_path / "sandbox" + sandbox_dir.mkdir() + sandbox_symlink = sandbox_dir / "python" + sandbox_symlink.symlink_to(source_symlink) + sandbox_regular = sandbox_dir / "python3.14" + sandbox_regular.symlink_to(target_path) + manifest_path.write_text( + "\n".join( + [ + f"rf-file|0|bin/python|{sandbox_symlink}", + f"rf-file|0|bin/python3.14|{sandbox_regular}", + ] + ) + ) + + create_zip(manifest_path, output_zip) + + with zipfile.ZipFile(output_zip, "r") as zf: + assert_zip_file_content( + zf, + "runfiles/my_ws/bin/python", + is_symlink_file=True, + target=target_path.name, + ) + assert_zip_file_content(zf, "runfiles/my_ws/bin/python3.14", content="python") + + def test_pathsep_normalization(tmp_path): manifest_path = tmp_path / "manifest.txt" output_zip = tmp_path / "output.zip"
diff --git a/tools/zipapp/zipper.py b/tools/zipapp/zipper.py index 5a8eb8c..c0e5472 100644 --- a/tools/zipapp/zipper.py +++ b/tools/zipapp/zipper.py
@@ -118,6 +118,23 @@ return path.replace("\\", "/") +def _source_symlink_target(content_path, is_symlink_str): + if is_symlink_str == "1": + return os.readlink(content_path) + if is_symlink_str != "0" or not os.path.islink(content_path): + return None + + target = os.readlink(content_path) + if not os.path.isabs(target): + return target + + # Bazel sandboxes expose regular inputs as absolute symlinks. Look through + # that indirection to detect whether the original source is also a symlink. + if os.path.islink(target): + return os.readlink(target) + return None + + def _write_entry(zf, entry, compress_type, seen, platform_pathsep): type_, is_symlink_str, zip_path, content_path = entry # Normalize slashes, otherwise the `seen` logic doesn't @@ -155,14 +172,13 @@ else: is_symlink_str = "0" - is_symlink = is_symlink_str == "1" - - if is_symlink: + symlink_target = _source_symlink_target(content_path, is_symlink_str) + if symlink_target is not None: zi = zipfile.ZipInfo(zip_path) zi.date_time = (1980, 1, 1, 0, 0, 0) zi.create_system = 3 # Unix zi.compress_type = compress_type - target = convert_symlink_target(os.readlink(content_path), platform_pathsep) + target = convert_symlink_target(symlink_target, platform_pathsep) # Set permissions to 777 for symlink (standard) zi.external_attr = (S_IFLNK | 0o777) << 16 zf.writestr(zi, target)