fix(pypi): fetch retained wheel archives lazily (#4316) URL-resolved wheels currently retain both the downloaded archive and extracted files even when a build only consumes the Python library. Download the archive temporarily for extraction and remove it afterward, while preserving the existing `whl` and `whl_file` targets through a lazy sibling `<repo>_whl` repository. Ordinary Python builds do not fetch that sibling repository. Apply matching patches in the separate wheel-file repository and extract from its patched archive. This keeps both original and patched archives out of the extracted repository. Preserve caller-owned wheel inputs, using a distinct `_patched_whl` repository for their patched replacements to avoid collisions with existing source repositories. Source-built wheels and direct `whl_archive` URL calls retain their current archive behavior. Both download paths share checksum validation, authentication, and URL/environment substitution. Extend the existing wheel-library integration test with a generated wheel served over HTTP. Separate Bazel invocations check archive cleanup, lazy repository creation, both wheel targets, matching and nonmatching patches, preservation of caller-owned inputs, and the full `pip.parse(uv_lock = ...)` path. Retain the existing pip-backend, source-distribution, direct `whl_archive`, and dependency-wiring assertions in the same integration target. Add analysis coverage for archive ownership and exclusion from runtime files. Gazelle's module mapping still consumes wheel archives and therefore materializes their archive repositories. Changing that consumer to use extracted libraries is separate follow-up work. Validation: - All 31 bzlmod example tests pass, including an assertion that the public wheel archive contains the same patched metadata as the extracted library. Both `bazel vendor //...` and building all targets with the resulting vendor directory pass locally. - All 24 affected Bazel test targets pass with `--config=fast-tests`, including the consolidated `//tests/integration:whl_library_test_bazel_self` target and its five test methods. The nested integration target is run with tag filters overridden and local test concurrency limited to one. - The storage analysis suite passes under bzlmod. Under Bazel 7.4.1 WORKSPACE mode, the bzlmod-specific ownership assertion is skipped and runtime archive exclusion passes. - All applicable pre-commit hooks pass, including buildifier, Ruff, deleted-package generation, and Gazelle. - The NumPy 2.3.3 PoC imports successfully without materializing an archive repository; explicitly building its `whl` target materializes the archive. Its extracted repository uses 57.74 MiB instead of 73.61 MiB allocated on the test filesystem. Shared download caches are outside that measurement. Implements the delete-by-default / optional-archive-access suggestion in https://github.com/bazel-contrib/rules_python/issues/3091#issuecomment-3074481951. Preserves the wheel-consumer functionality introduced in #364 and fixed in #458. Related to #3968 and complements the extraction-reuse work in #3856.
diff --git a/examples/bzlmod/whl_mods/BUILD.bazel b/examples/bzlmod/whl_mods/BUILD.bazel index 7c5ab50..d141561 100644 --- a/examples/bzlmod/whl_mods/BUILD.bazel +++ b/examples/bzlmod/whl_mods/BUILD.bazel
@@ -8,8 +8,10 @@ py_test( name = "pip_whl_mods_test", srcs = ["pip_whl_mods_test.py"], + data = ["@pip//requests:whl"], env = { "REQUESTS_PKG": "$(rlocationpaths @pip//requests:pkg)", + "REQUESTS_WHL": "$(rlocationpath @pip//requests:whl)", "WHEEL_PKG": "$(rlocationpaths @pip//wheel:pkg)", }, main = "pip_whl_mods_test.py",
diff --git a/examples/bzlmod/whl_mods/appended_build_content.BUILD b/examples/bzlmod/whl_mods/appended_build_content.BUILD index 0ca118d..e751dcb 100644 --- a/examples/bzlmod/whl_mods/appended_build_content.BUILD +++ b/examples/bzlmod/whl_mods/appended_build_content.BUILD
@@ -7,10 +7,8 @@ ) filegroup( - name = "whl_orig", - srcs = glob( - ["*.whl"], - allow_empty = False, - exclude = ["*-patched-*.whl"], - ), + name = "whl_archive", + # The archive may live in a separate repository. Use the wheel target + # instead of assuming that the archive is beside the extracted files. + srcs = [":whl_file"], )
diff --git a/examples/bzlmod/whl_mods/pip_whl_mods_test.py b/examples/bzlmod/whl_mods/pip_whl_mods_test.py index 8cd7232..972a7df 100644 --- a/examples/bzlmod/whl_mods/pip_whl_mods_test.py +++ b/examples/bzlmod/whl_mods/pip_whl_mods_test.py
@@ -19,6 +19,7 @@ import subprocess import sys import unittest +import zipfile from pathlib import Path from python.runfiles import runfiles @@ -145,6 +146,14 @@ "Summary: Python HTTP for Humans. Patched.", metadata.read_text().splitlines(), ) + # The public wheel target must expose the same patches as the library, + # even when its archive is stored outside the extracted repository. + archive = Path(r.Rlocation(os.environ["REQUESTS_WHL"])) + with zipfile.ZipFile(archive) as wheel: + self.assertEqual( + wheel.read(f"requests-{current_wheel_version}.dist-info/METADATA"), + metadata.read_bytes(), + ) if __name__ == "__main__":
diff --git a/news/lazy-wheel-archives.changed.md b/news/lazy-wheel-archives.changed.md new file mode 100644 index 0000000..08226ef --- /dev/null +++ b/news/lazy-wheel-archives.changed.md
@@ -0,0 +1,6 @@ +(pypi) Downloaded wheels are removed after extraction when their original +archive can be fetched independently. The existing `whl` and `whl_file` targets +fetch the archive from a separate repository on demand. Matching patches are +applied in the separate wheel-file repository, which also supplies the patched +archive for extraction. Wheels built from source retain their existing archive +behavior.
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 96b68e7..149074d 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel
@@ -479,8 +479,10 @@ name = "whl_library", srcs = ["whl_library.bzl"], deps = [ + ":patch_whl", ":pip_archive", ":whl_archive", + ":whl_file_repo", ], ) @@ -555,8 +557,20 @@ deps = [ ":attrs", ":patch_and_extract_whl", - ":urllib", ":whl_deps_repo", + ":whl_file_repo", + "//python/private:auth", + "//python/private:repo_utils", + ], +) + +bzl_library( + name = "whl_file_repo", + srcs = ["whl_file_repo.bzl"], + deps = [ + ":attrs", + ":patch_whl", + ":urllib", "//python/private:auth", "//python/private:repo_utils", ],
diff --git a/python/private/pypi/patch_and_extract_whl.bzl b/python/private/pypi/patch_and_extract_whl.bzl index b4ecb2f..37c90e3 100644 --- a/python/private/pypi/patch_and_extract_whl.bzl +++ b/python/private/pypi/patch_and_extract_whl.bzl
@@ -3,7 +3,7 @@ load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:repo_utils.bzl", "repo_utils") load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") -load(":patch_whl.bzl", "patch_whl") +load(":patch_whl.bzl", "get_whl_patches", "patch_whl") load(":pep508_requirement.bzl", "requirement") load(":pypi_repo_utils.bzl", "pypi_repo_utils") load(":whl_extract.bzl", "whl_extract") @@ -69,7 +69,7 @@ elif path.is_dir: paths.extend(path.readdir()) -def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None): +def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None, whl_file = None): """Extract the wheel, apply patches and generate BUILD.bazel files. Reused in pip and http wheel download code. @@ -79,27 +79,30 @@ whl_path: the whl path to extract. logger: The logger to use sdist_filename: The filename to ignore in the BUILD.bazel files as sources. + whl_file: The label of an externally stored wheel, if any. Returns: The repository metadata if the extraction is reproducible """ if rctx.attr.whl_patches: - patches = {} - for patch_file, json_args in rctx.attr.whl_patches.items(): - patch_dst = struct(**json.decode(json_args)) - if whl_path.basename in patch_dst.whls: - patches[patch_file] = patch_dst.patch_strip + patches = get_whl_patches(rctx.attr.whl_patches, whl_path.basename) if patches: + # patch_whl deletes its input before repacking. Never delete an + # archive owned by another repository. + if whl_path.dirname != rctx.path("."): + rctx.symlink(whl_path, whl_path.basename) + whl_path = rctx.path(whl_path.basename) whl_path = patch_whl( rctx, whl_path = whl_path, patches = patches, ) + whl_file = None whl_extract(rctx, whl_path = whl_path, logger = logger) - install_dir_path = whl_path.dirname.get_child("site-packages") + install_dir_path = rctx.path("site-packages") metadata = whl_metadata( install_dir = install_dir_path, read_fn = rctx.read, @@ -139,8 +142,14 @@ metadata = metadata, filename = sdist_filename or whl_path.basename, ), + **({"whl_file": str(whl_file)} if whl_file else {}) ) + # The generated wheel target can obtain the original archive independently. + # Do not retain the temporary download or delete a caller-owned wheel. + if whl_file and whl_path.dirname == rctx.path("."): + rctx.delete(whl_path) + # Delete these in case the wheel had them. They generally don't cause # a problem, but let's avoid the chance of that happening. rctx.file("WORKSPACE")
diff --git a/python/private/pypi/patch_whl.bzl b/python/private/pypi/patch_whl.bzl index e8d76ee..b257f2b 100644 --- a/python/private/pypi/patch_whl.bzl +++ b/python/private/pypi/patch_whl.bzl
@@ -58,6 +58,23 @@ platform_tag = parsed_whl.platform_tag, ) +def get_whl_patches(whl_patches, filename): + """Select the patches that apply to a wheel filename. + + Args: + whl_patches: Patch labels mapped to JSON-encoded wheel names and strip levels. + filename: The wheel filename to match. + + Returns: + A dictionary of matching patch labels mapped to strip levels. + """ + patches = {} + for patch_file, json_args in whl_patches.items(): + patch_dst = struct(**json.decode(json_args)) + if filename in patch_dst.whls: + patches[patch_file] = patch_dst.patch_strip + return patches + def patch_whl(rctx, *, whl_path, patches): """Patch a whl file and repack it natively.
diff --git a/python/private/pypi/whl_archive.bzl b/python/private/pypi/whl_archive.bzl index 242104f..3ba32b1 100644 --- a/python/private/pypi/whl_archive.bzl +++ b/python/private/pypi/whl_archive.bzl
@@ -1,11 +1,11 @@ "" -load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") +load("//python/private:auth.bzl", "AUTH_ATTRS") load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") load(":attrs.bzl", "ATTRS") load(":patch_and_extract_whl.bzl", "patch_and_extract_whl") -load(":urllib.bzl", "urllib") load(":whl_deps_repo.bzl", "whl_deps_attrs") +load(":whl_file_repo.bzl", "download_whl") def _whl_archive_impl(rctx): logger = repo_utils.logger(rctx) @@ -14,47 +14,17 @@ if rctx.attr.whl_file: rctx.watch(rctx.attr.whl_file) whl_path = rctx.path(rctx.attr.whl_file) - - # Simulate the behaviour where the whl is present in the current directory. - rctx.symlink(whl_path, whl_path.basename) - whl_path = rctx.path(whl_path.basename) elif rctx.attr.urls and rctx.attr.filename: - filename = rctx.attr.filename - urls = rctx.attr.urls - urls = [ - urllib.absolute_url( - rctx.attr.index_url, - url, - envsubst = rctx.attr.envsubst, - getenv = rctx.getenv, - ) - for url in urls - ] - result = rctx.download( - url = urls, - output = filename, - sha256 = rctx.attr.sha256, - integrity = rctx.attr.integrity if not rctx.attr.sha256 else "", - auth = get_auth(rctx, urls), - ) - if not rctx.attr.sha256 and not rctx.attr.integrity: - # this is only seen when there is a direct URL reference without a hash - logger.warn("Please update the requirement line to include the hash:\n{} \\\n --hash=sha256:{}".format( - rctx.attr.requirement, - result.sha256, - )) - - if not result.success: - fail("could not download the '{}' from {}:\n{}".format(filename, urls, result)) - - if filename.endswith(".whl"): - whl_path = rctx.path(filename) - else: - fail("Only wheels are supported") + whl_path = download_whl(rctx, logger = logger) else: fail("Either 'whl_file' or 'urls' and 'filename' needs to be specified") - return patch_and_extract_whl(rctx, whl_path = whl_path, logger = logger) + return patch_and_extract_whl( + rctx, + whl_path = whl_path, + whl_file = rctx.attr.whl_file or rctx.attr.downloaded_whl, + logger = logger, + ) whl_archive_attrs = whl_deps_attrs | { "annotation": attr.label( @@ -64,6 +34,9 @@ ), allow_files = True, ), + "downloaded_whl": attr.string( + doc = "Label of the lazy archive target, emitted in BUILD.bazel without fetching it. INTERNAL USE ONLY.", + ), "filename": attr.string( doc = "Download the whl file to this filename. Only used when the `urls` is passed. If not specified, will be auto-detected from the `urls`.", ),
diff --git a/python/private/pypi/whl_file_repo.bzl b/python/private/pypi/whl_file_repo.bzl new file mode 100644 index 0000000..40d79ac --- /dev/null +++ b/python/private/pypi/whl_file_repo.bzl
@@ -0,0 +1,93 @@ +"""Store an original or patched wheel separately from its extracted files.""" + +load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") +load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") +load(":attrs.bzl", "ATTRS") +load(":patch_whl.bzl", "patch_whl") +load(":urllib.bzl", "urllib") + +def download_whl(rctx, *, logger): + """Download a wheel and return its path, preserving index authentication. + + Args: + rctx: The repository context. + logger: The repository logger. + + Returns: + The path of the downloaded wheel. + """ + filename = rctx.attr.filename + if not filename.endswith(".whl"): + fail("Only wheels are supported") + urls = [ + urllib.absolute_url( + rctx.attr.index_url, + url, + envsubst = rctx.attr.envsubst, + getenv = rctx.getenv, + ) + for url in rctx.attr.urls + ] + result = rctx.download( + url = urls, + output = filename, + sha256 = rctx.attr.sha256, + integrity = rctx.attr.integrity if not rctx.attr.sha256 else "", + auth = get_auth(rctx, urls), + ) + if not rctx.attr.sha256 and not rctx.attr.integrity: + logger.warn("Please update the requirement line to include the hash:\n{} \\\n --hash=sha256:{}".format( + rctx.attr.requirement, + result.sha256, + )) + if not result.success: + fail("could not download the '{}' from {}:\n{}".format(filename, urls, result)) + return rctx.path(filename) + +whl_file_repo_attrs = { + "envsubst": ATTRS["envsubst"], + "filename": attr.string(doc = "The filename of the downloaded wheel."), + "index_url": attr.string(doc = "The index URL used to resolve relative wheel URLs."), + "integrity": attr.string(doc = "The expected SRI checksum, used when sha256 is unset."), + "patches": attr.label_keyed_string_dict(doc = "Patch labels mapped to their strip levels."), + "requirement": attr.string(doc = "The requirement used in download diagnostics."), + "sha256": attr.string(doc = "The expected SHA256 checksum."), + "urls": attr.string_list(doc = "The wheel download URLs."), + "whl_file": attr.label(doc = "A caller-owned wheel to use instead of downloading."), +} | AUTH_ATTRS + +def _whl_file_repo_impl(rctx): + if rctx.attr.whl_file: + rctx.watch(rctx.attr.whl_file) + whl_path = rctx.path(rctx.attr.whl_file) + + # patch_whl deletes its input before repacking. Keep caller-owned + # wheels intact by giving it a repository-local symlink. + rctx.symlink(whl_path, whl_path.basename) + whl_path = rctx.path(whl_path.basename) + else: + whl_path = download_whl(rctx, logger = repo_utils.logger(rctx)) + if rctx.attr.patches: + whl_path = patch_whl( + rctx, + whl_path = whl_path, + patches = {patch: int(strip) for patch, strip in rctx.attr.patches.items()}, + ) + + # Patching temporarily unpacks the input. Retain only the replacement + # wheel; the installation is extracted in the consuming repository. + for path in rctx.path(".").readdir(): + if path.basename != whl_path.basename: + rctx.delete(path) + rctx.file("BUILD.bazel", "exports_files([{}])\n".format(repr(whl_path.basename))) + if hasattr(rctx, "repo_metadata"): + return rctx.repo_metadata(reproducible = True) + return None + +whl_file_repo = repository_rule( + implementation = _whl_file_repo_impl, + attrs = whl_file_repo_attrs | { + "_rule_name": attr.string(default = "whl_file_repo"), + }, + environ = [REPO_DEBUG_ENV_VAR], +)
diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 3f874b4..3441a2c 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl
@@ -14,8 +14,10 @@ "" +load(":patch_whl.bzl", "get_whl_patches", "patched_whl_name") load(":pip_archive.bzl", "pip_archive", "pip_archive_attrs") load(":whl_archive.bzl", "whl_archive", "whl_archive_attrs") +load(":whl_file_repo.bzl", "whl_file_repo", "whl_file_repo_attrs") def _filter(kwargs, subset, debug = False): dropped = {} @@ -38,6 +40,12 @@ * {obj}`whl_archive` * {obj}`pip_archive` + URL wheels are downloaded temporarily for extraction, then removed. + A sibling `<name>_whl` repository supplies the original archive lazily, + when a wheel target is requested. Extraction does not depend on it. + Matching patches are applied in the wheel-file repository instead; + extraction and wheel targets then consume the same patched archive. + Args: name: {type}`str` The name of the repo. repo: Unused, will be dropped in the next major release. @@ -56,6 +64,22 @@ kwargs.setdefault("dep_template", "@{}{{name}}//:{{target}}".format(kwargs.pop("repo_prefix", ""))) if whl_file or (urls and filename and filename.endswith(".whl")): + wheel_filename = filename or str(whl_file).split(":")[-1].split("/")[-1] + patches = get_whl_patches(kwargs.get("whl_patches", {}), wheel_filename) + if patches: + # Caller-owned wheels are often already supplied by <name>_whl. + # Give their patched output a distinct repository name. + file_repo = name + ("_patched_whl" if whl_file else "_whl") + whl_file_repo( + name = file_repo, + patches = {patch: str(strip) for patch, strip in patches.items()}, + **_filter(kwargs, whl_file_repo_attrs) + ) + kwargs["whl_file"] = "@{}//:{}".format(file_repo, patched_whl_name(wheel_filename)) + kwargs["whl_patches"] = {} + elif not whl_file: + whl_file_repo(name = name + "_whl", **_filter(kwargs, whl_file_repo_attrs)) + kwargs["downloaded_whl"] = "@{}//:{}".format(name + "_whl", filename) whl_archive(name = name, **_filter(kwargs, whl_archive_attrs)) else: pip_archive(name = name, **_filter(kwargs, pip_archive_attrs))
diff --git a/python/private/pypi/whl_library_srcs.bzl b/python/private/pypi/whl_library_srcs.bzl index 0556219..44d47eb 100644 --- a/python/private/pypi/whl_library_srcs.bzl +++ b/python/private/pypi/whl_library_srcs.bzl
@@ -31,6 +31,7 @@ def whl_library_srcs( *, name, + whl_file = None, sdist_filename = None, data_exclude = [], srcs_exclude = [], @@ -57,6 +58,7 @@ Args: name: {type}`str` The file to match for including it into the `whl` filegroup. This may be also parsed to generate extra metadata. + whl_file: {type}`str | None` The label of an externally stored wheel. sdist_filename: {type}`str | None` If the wheel was built from an sdist, the filename of the sdist. visibility: {type}`list[str]` The visibility of the source targets. @@ -178,7 +180,7 @@ if hasattr(native, "filegroup"): native.filegroup( name = WHEEL_FILE, - srcs = [name], + srcs = [whl_file or name], visibility = visibility, )
diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 4f7bf8e..a1cc727 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl
@@ -23,6 +23,7 @@ def whl_library_targets( *, name, + whl_file = None, metadata_name = "", requires_dist = [], extras = [], @@ -46,6 +47,7 @@ Args: name: {type}`str` The wheel filename + whl_file: {type}`str | None` The label of a wheel stored in another repository. metadata_name: {type}`str` The package name as written in wheel `METADATA`. group_deps: {type}`list[str]` names of fellow members of the group (if any). These will be excluded from generated deps lists so as to avoid @@ -72,6 +74,7 @@ """ whl_library_srcs( name = name, + whl_file = whl_file, sdist_filename = sdist_filename, data_exclude = data_exclude, srcs_exclude = srcs_exclude,
diff --git a/tests/integration/BUILD.bazel b/tests/integration/BUILD.bazel index 6c5d248..21ef8fa 100644 --- a/tests/integration/BUILD.bazel +++ b/tests/integration/BUILD.bazel
@@ -149,6 +149,7 @@ rules_python_integration_test( name = "whl_library_test", + py_main = "whl_library_test.py", ) py_library(
diff --git a/tests/integration/whl_library/BUILD.bazel b/tests/integration/whl_library/BUILD.bazel index a53bcc7..0d2085b 100644 --- a/tests/integration/whl_library/BUILD.bazel +++ b/tests/integration/whl_library/BUILD.bazel
@@ -1,5 +1,38 @@ +load("@rules_python//python:py_binary.bzl", "py_binary") load("@rules_python//python:py_test.bzl", "py_test") +exports_files([ + "storage_pkg-1.0-py3-none-any.whl", + "wheel.sha256", + "modify_pkg.patch", + "uv.lock", +]) + +[ + py_binary( + name = name, + srcs = ["main.py"], + legacy_create_init = 0, + main = "main.py", + deps = ["@{}//:pkg".format(name)], + ) + for name in [ + "plain", + "patched", + "nonmatching", + "provided", + "provided_patched", + ] +] + +py_binary( + name = "uv_locked", + srcs = ["main.py"], + legacy_create_init = 0, + main = "main.py", + deps = ["@uv_pip//storage_pkg"], +) + [ alias( name = "{}_pkg".format(pkg),
diff --git a/tests/integration/whl_library/MODULE.bazel b/tests/integration/whl_library/MODULE.bazel index d3886d7..9059879 100644 --- a/tests/integration/whl_library/MODULE.bazel +++ b/tests/integration/whl_library/MODULE.bazel
@@ -59,3 +59,14 @@ metadata_file = "@whl_archive//:metadata.json", requirement = "requests", ) + +wheels = use_extension("//:wheels.bzl", "wheels") +use_repo(wheels, "nonmatching", "patched", "plain", "provided", "provided_patched") + +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") +pip.parse( + hub_name = "uv_pip", + python_version = "3.14", + uv_lock = "//:uv.lock", +) +use_repo(pip, "uv_pip")
diff --git a/tests/integration/whl_library/main.py b/tests/integration/whl_library/main.py new file mode 100644 index 0000000..ba55731 --- /dev/null +++ b/tests/integration/whl_library/main.py
@@ -0,0 +1,3 @@ +import storage_pkg + +print(f"PATCHED={storage_pkg.PATCHED}")
diff --git a/tests/integration/whl_library/modify_pkg.patch b/tests/integration/whl_library/modify_pkg.patch new file mode 100644 index 0000000..61c310b --- /dev/null +++ b/tests/integration/whl_library/modify_pkg.patch
@@ -0,0 +1,5 @@ +--- storage_pkg/__init__.py ++++ storage_pkg/__init__.py +@@ -1 +1 @@ +-PATCHED = False ++PATCHED = True
diff --git a/tests/integration/whl_library/wheels.bzl b/tests/integration/whl_library/wheels.bzl new file mode 100644 index 0000000..b82f0c0 --- /dev/null +++ b/tests/integration/whl_library/wheels.bzl
@@ -0,0 +1,38 @@ +"""Local HTTP and caller-provided wheel fixtures.""" + +load("@rules_python//python/private/pypi:whl_library.bzl", "whl_library") # buildifier: disable=bzl-visibility + +_FILENAME = "storage_pkg-1.0-py3-none-any.whl" + +def _patches(filename): + return { + Label("//:modify_pkg.patch"): json.encode({ + "patch_strip": 0, + "whls": [filename], + }), + } + +def _wheels_impl(mrctx): + common = { + "envsubst": ["WHEEL_TEST_ADDRESS"], + "filename": _FILENAME, + "requirement": "storage-pkg", + "sha256": mrctx.read(Label("//:wheel.sha256")).strip(), + "urls": ["http://${WHEEL_TEST_ADDRESS}/" + _FILENAME], + } + whl_library(name = "plain", **common) + whl_library(name = "patched", whl_patches = _patches(_FILENAME), **common) + whl_library(name = "nonmatching", whl_patches = _patches("other.whl"), **common) + whl_library( + name = "provided", + requirement = "storage-pkg", + whl_file = Label("//:" + _FILENAME), + ) + whl_library( + name = "provided_patched", + requirement = "storage-pkg", + whl_file = Label("//:" + _FILENAME), + whl_patches = _patches(_FILENAME), + ) + +wheels = module_extension(implementation = _wheels_impl)
diff --git a/tests/integration/whl_library_test.py b/tests/integration/whl_library_test.py new file mode 100644 index 0000000..f61ad21 --- /dev/null +++ b/tests/integration/whl_library_test.py
@@ -0,0 +1,154 @@ +"""Exercise lazy wheel retention across separate Bazel invocations.""" + +import functools +import hashlib +import http.server +import threading +import unittest +import zipfile +from pathlib import Path + +from tests.integration import runner + + +class WheelLibraryTest(runner.TestCase): + def setUp(self): + super().setUp() + self.wheel = self.repo_root / "storage_pkg-1.0-py3-none-any.whl" + with zipfile.ZipFile(self.wheel, "w") as wheel: + wheel.writestr("storage_pkg/__init__.py", "PATCHED = False\n") + wheel.writestr( + "storage_pkg-1.0.dist-info/METADATA", + "Metadata-Version: 2.1\nName: storage-pkg\nVersion: 1.0\n", + ) + wheel.writestr( + "storage_pkg-1.0.dist-info/WHEEL", + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + ) + wheel.writestr("storage_pkg-1.0.dist-info/RECORD", "") + self.original_wheel = self.wheel.read_bytes() + (self.repo_root / "wheel.sha256").write_text( + hashlib.sha256(self.original_wheel).hexdigest() + ) + handler = functools.partial( + http.server.SimpleHTTPRequestHandler, directory=str(self.repo_root) + ) + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + self.addCleanup(server.server_close) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join) + self.addCleanup(server.shutdown) + self.bazel_env["WHEEL_TEST_ADDRESS"] = f"127.0.0.1:{server.server_port}" + base_url = f"http://127.0.0.1:{server.server_port}" + digest = hashlib.sha256(self.original_wheel).hexdigest() + (self.repo_root / "uv.lock").write_text( + f"""\ +version = 1 +revision = 3 +requires-python = ">=3.14" + +[[package]] +name = "storage-pkg" +version = "1.0" +source = {{ registry = "{base_url}/simple" }} +wheels = [ + {{ url = "{base_url}/{self.wheel.name}", hash = "sha256:{digest}", size = {len(self.original_wheel)} }}, +] +""" + ) + output_base = self.run_bazel("info", "output_base").stdout.strip() + self.external = Path(output_base) / "external" + + def repositories(self, name): + return [ + path + for path in self.external.iterdir() + if path.is_dir() and path.name.endswith(("+" + name, "~" + name)) + ] + + def repository(self, name): + repositories = self.repositories(name) + self.assertEqual(len(repositories), 1, repositories) + return repositories[0].resolve() + + def assert_runs(self, name, *, patched): + result = self.run_bazel("run", "//:" + name) + self.assert_result_matches(result, f"PATCHED={patched}") + + def test_lazy_archive(self): + self.assert_runs("plain", patched=False) + extracted = self.repository("plain") + self.assertEqual(list(extracted.glob("*.whl")), []) + self.assertEqual(self.repositories("plain_whl"), []) + + self.run_bazel("build", "@plain//:whl", "@plain//:whl_file") + archive = self.repository("plain_whl") / self.wheel.name + self.assertEqual(archive.read_bytes(), self.original_wheel) + self.assertEqual(list(extracted.glob("*.whl")), []) + + def test_matching_and_nonmatching_patches(self): + self.assert_runs("patched", patched=True) + patched = self.repository("patched") + self.assertEqual(list(patched.glob("*.whl")), []) + self.run_bazel("build", "@patched//:whl", "@patched//:whl_file") + with zipfile.ZipFile( + self.repository("patched_whl") / "storage_pkg-1.0+patched-py3-none-any.whl" + ) as wheel: + self.assertEqual(wheel.read("storage_pkg/__init__.py"), b"PATCHED = True\n") + self.assertFalse((self.repository("patched_whl") / self.wheel.name).exists()) + self.assertFalse((self.repository("patched_whl") / "storage_pkg").exists()) + self.assertEqual(list(patched.glob("*.whl")), []) + + self.assert_runs("nonmatching", patched=False) + self.assertEqual(list(self.repository("nonmatching").glob("*.whl")), []) + self.assertEqual(self.repositories("nonmatching_whl"), []) + self.run_bazel("build", "@nonmatching//:whl") + archive = self.repository("nonmatching_whl") / self.wheel.name + self.assertEqual(archive.read_bytes(), self.original_wheel) + + def test_caller_owned_wheel_is_preserved(self): + for name, patched in [("provided", False), ("provided_patched", True)]: + with self.subTest(name=name): + self.assert_runs(name, patched=patched) + self.run_bazel("build", f"@{name}//:whl", f"@{name}//:whl_file") + self.assertEqual(self.wheel.read_bytes(), self.original_wheel) + self.assertEqual(list(self.repository(name).glob("*.whl")), []) + self.assertEqual(self.repositories(name + "_whl"), []) + if patched: + with zipfile.ZipFile( + self.repository(name + "_patched_whl") + / "storage_pkg-1.0+patched-py3-none-any.whl" + ) as wheel: + self.assertEqual( + wheel.read("storage_pkg/__init__.py"), b"PATCHED = True\n" + ) + + def test_repository_contents(self): + # Retain the existing direct whl_archive, pip_archive, sdist, and + # dependency-wiring assertions in the same nested Bazel test. + self.run_bazel("test", "//:test_contents") + + def test_uv_lock_lazy_archive(self): + self.assert_runs("uv_locked", patched=False) + repositories = [ + path + for path in self.external.glob("*uv_pip_*") + if path.is_dir() and not path.name.endswith("_whl") + ] + self.assertEqual(len(repositories), 1, repositories) + extracted = repositories[0].resolve() + archive_repo = self.external / (repositories[0].name + "_whl") + self.assertEqual(list(extracted.glob("*.whl")), []) + self.assertFalse(archive_repo.exists()) + self.assertFalse(archive_repo.is_symlink()) + + self.run_bazel("build", "@uv_pip//storage_pkg:whl") + self.assertEqual( + (archive_repo / self.wheel.name).read_bytes(), self.original_wheel + ) + self.assertEqual(list(extracted.glob("*.whl")), []) + + +if __name__ == "__main__": + unittest.main()
diff --git a/tests/pypi/integration/BUILD.bazel b/tests/pypi/integration/BUILD.bazel index 316abe8..6a060cf 100644 --- a/tests/pypi/integration/BUILD.bazel +++ b/tests/pypi/integration/BUILD.bazel
@@ -1,5 +1,8 @@ load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_python_publish_deps//:requirements.bzl", "all_requirements") +load(":whl_storage_tests.bzl", "whl_storage_test_suite") + +whl_storage_test_suite(name = "whl_storage_tests") build_test( name = "all_requirements_build_test",
diff --git a/tests/pypi/integration/whl_storage_tests.bzl b/tests/pypi/integration/whl_storage_tests.bzl new file mode 100644 index 0000000..de469b1 --- /dev/null +++ b/tests/pypi/integration/whl_storage_tests.bzl
@@ -0,0 +1,48 @@ +"""Check archive storage and runtime dependencies for a downloaded wheel.""" + +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//tests/support:support.bzl", "SUPPORTS_BZLMOD") + +def _test_downloaded_archive(name): + analysis_test( + name = name, + target = "@rules_python_publish_deps//packaging:whl", + impl = _test_downloaded_archive_impl, + # WORKSPACE resolves this fixture through pip_archive, which retains + # its existing archive layout. Only bzlmod supplies a known wheel URL. + attr_values = {"target_compatible_with": SUPPORTS_BZLMOD}, + ) + +def _test_downloaded_archive_impl(env, target): + files = target[DefaultInfo].files.to_list() + env.expect.that_collection(files).has_size(1) + for file in files: + env.expect.that_bool(file.basename.endswith(".whl")).equals(True) + env.expect.that_bool(file.owner.workspace_name.endswith("_whl")).equals(True) + +def _test_runtime_excludes_archive(name): + analysis_test( + name = name, + target = "@rules_python_publish_deps//packaging:pkg", + impl = _test_runtime_excludes_archive_impl, + ) + +def _test_runtime_excludes_archive_impl(env, target): + files = target[DefaultInfo].default_runfiles.files.to_list() + env.expect.that_bool(bool(files)).equals(True) + env.expect.that_collection([ + file.short_path + for file in files + if file.extension == "whl" + ]).contains_exactly([]) + +def whl_storage_test_suite(name): + """Create tests for separate archive storage and wheel target compatibility.""" + test_suite( + name = name, + tests = [ + _test_downloaded_archive, + _test_runtime_excludes_archive, + ], + )
diff --git a/tests/pypi/whl_library_srcs/whl_library_srcs_tests.bzl b/tests/pypi/whl_library_srcs/whl_library_srcs_tests.bzl index fe4342b..740d70e 100644 --- a/tests/pypi/whl_library_srcs/whl_library_srcs_tests.bzl +++ b/tests/pypi/whl_library_srcs/whl_library_srcs_tests.bzl
@@ -67,6 +67,26 @@ _tests.append(_test_filegroups) +def _test_external_whl_file(env): + calls = [] + whl_library_srcs( + name = "example-1.0-py3-none-any.whl", + whl_file = "@example_whl//:example-1.0-py3-none-any.whl", + native = struct( + filegroup = lambda **kwargs: calls.append(kwargs), + glob = lambda *args, **kwargs: [], + ), + rules = struct(), + ) + wheel = [call for call in calls if call["name"] == "whl_file"][0] + env.expect.that_collection(wheel["srcs"]).contains_exactly([ + "@example_whl//:example-1.0-py3-none-any.whl", + ]) + extracted = [call for call in calls if call["name"] == "extracted_whl_files"][0] + env.expect.that_collection(extracted["srcs"]).contains_exactly([]) + +_tests.append(_test_external_whl_file) + def _test_copy(env): calls = []