feat(pypi): generate requirements.bzl in the unified @pypi hub (#4223)
The unified `@pypi` hub (#3837) does not generate a `requirements.bzl`.
That makes renaming a hub away from the now-reserved `pypi` name an
all-at-once change: the moment the concrete hub stops being `@pypi`,
every `load("@pypi//:requirements.bzl", "requirement")` in the repo
fails to load. In our monorepo that is ~940 BUILD files, and any BUILD
file that lands on main mid-migration re-breaks it. The same thing would
happen to every such repo if `RULES_PYTHON_PYPI_HUB_RESERVED` is flipped
on by default, as discussed in #3837.
This adds an opt-in, `pip.default(unified_hub_requirements_bzl = True)`
(root module only, off by default), that generates a `requirements.bzl`
in the unified hub with the per-package macros of a concrete hub's. It's
opt-in because the docs already steer users to `@pypi//<pkg>` labels
over `requirement()`; this is a migration aid for repos whose hub used
to be named `pypi`, not a new default surface. With it on:
* `requirement()`, `whl_requirement()`, `data_requirement()` and
`dist_info_requirement()` return labels in the unified hub
(`@@<unified>//<pkg>:pkg` etc., the same canonical-name form the
concrete hubs use), so they route through
`--@rules_python//python/config_settings:venv` exactly like a plain
`@pypi//<pkg>` label. All of those targets already exist in every
unified package (`_STANDARD_ALIASES`).
* The `all_*` lists (`all_requirements`,
`all_whl_requirements(_by_package)`, `all_data_requirements`) are
deliberately not generated. They are fixed at loading time, before the
venv flag is known, so they could only list one hub's packages and would
silently not follow the flag like the rest of the unified hub. Code that
wants a whole lock loads them from that concrete hub, and a stale
`load("@pypi//:requirements.bzl", "all_requirements")` fails loudly
instead.
Before: after renaming `hub_name = "pypi"` to e.g. `"pypi_main"` with
`pip.default(default_hub = "pypi_main")`,
`load("@pypi//:requirements.bzl", ...)` fails with "no such file".
After, with the opt-in: it keeps working and resolves to the same
wheels, and targets can migrate to `"@pypi//<pkg>"` labels incrementally
(we're moving our own repo to labels and gating new `requirement()`
calls in CI). Without the opt-in nothing changes.
Tests:
* `tests/pypi/extension`: the setting is off by default, on when the
root module sets it, and ignored from a non-root module.
* `tests/integration/unified_pypi` (which now opts in): `requirement()`
with a non-normalized name on the default hub, `requirement()` under a
venv transition, and loading `@pypi//:requirements.bzl` failing once the
opt-in is removed. Passes on `bazel_self` and `bazel_7.7.0`.
Context: we're carrying this as a local patch in a large monorepo, where
renaming our main hub and loading `requirement` from the unified hub
leaves the resolved wheel set unchanged. #4172 also touches
`tests/integration/unified_pypi`; the two should merge independently,
but one of them may need a trivial rebase.
(Done with help from an agent.)
🤖 Generated with [Claude Code](https://claude.com/claude-code)
---------
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
diff --git a/docs/pypi/download.md b/docs/pypi/download.md
index e4ab387..b9904d4 100644
--- a/docs/pypi/download.md
+++ b/docs/pypi/download.md
@@ -121,6 +121,17 @@
`@pypi//numpy`), and the dependency will automatically resolve to the correct
wheel version from the active hub during the build.
+To migrate a hub that used to be named `pypi`, the root module can opt into a
+`requirements.bzl` in the unified hub with
+`pip.default(unified_hub_requirements_bzl = True)`. It has the per-package
+macros of a concrete hub's (`requirement`, `whl_requirement`, `data_requirement`
+and `dist_info_requirement`), so existing
+`load("@pypi//:requirements.bzl", "requirement")` calls keep working after the
+rename, and `requirement("numpy")` routes through the active hub like
+`@pypi//numpy`. It has no `all_requirements` or other `all_*` lists: those are
+fixed at loading time, before the active hub is known, so load them from a
+concrete hub instead. Prefer `@pypi//<pkg>` labels in new code.
+
### Declaring Abstract Dependencies (pip.dep)
:::{versionadded} 2.2.0
diff --git a/news/4223.added.md b/news/4223.added.md
new file mode 100644
index 0000000..de37e0e
--- /dev/null
+++ b/news/4223.added.md
@@ -0,0 +1,9 @@
+(pypi) Added `pip.default(unified_hub_requirements_bzl = True)`, which makes the
+unified `@pypi` hub generate a `requirements.bzl` with the per-package macros of
+a concrete hub's (`requirement`, `whl_requirement`, `data_requirement` and
+`dist_info_requirement`). A repo that renames its hub away from the reserved
+`pypi` name keeps its `load("@pypi//:requirements.bzl", "requirement")` users
+working, and the returned labels route through
+{obj}`--@rules_python//python/config_settings:venv`. The `all_*` lists are not
+provided, since they could only list one hub's packages; load them from a
+concrete hub.
diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl
index cafe245..1b82780 100644
--- a/python/private/pypi/extension.bzl
+++ b/python/private/pypi/extension.bzl
@@ -207,6 +207,7 @@
A struct with the configuration.
"""
default_hub = None
+ unified_hub_requirements_bzl = False
defaults = {
"platforms": default_platforms(),
"python_version": None,
@@ -221,6 +222,8 @@
if default_hub:
fail("Duplicate pip.default tag: only one explicit default PyPI hub is allowed.")
default_hub = tag.default_hub
+ if tag.unified_hub_requirements_bzl and mod.is_root:
+ unified_hub_requirements_bzl = True
pyproject_toml = tag.pyproject_toml
if pyproject_toml:
pyproject = read_pyproject(module_ctx, pyproject_toml)
@@ -270,6 +273,7 @@
},
enable_pipstar_extract = enable_pipstar_extract,
toml_decode = toml.decode,
+ unified_hub_requirements_bzl = unified_hub_requirements_bzl,
)
def parse_modules(
@@ -482,6 +486,7 @@
facts = simpleapi_cache.get_facts(),
hub_group_map = hub_group_map,
hub_whl_map = hub_whl_map,
+ unified_hub_requirements_bzl = config.unified_hub_requirements_bzl,
whl_libraries = whl_libraries,
whl_mods = whl_mods,
platform_config_settings = {
@@ -535,6 +540,7 @@
extra_aliases = extra_aliases,
hubs = hubs,
packages = packages,
+ requirements_bzl = mods.unified_hub_requirements_bzl,
)
def _pip_impl(module_ctx):
@@ -755,6 +761,26 @@
:::
""",
),
+ "unified_hub_requirements_bzl": attr.bool(
+ default = False,
+ doc = """\
+Generate a `requirements.bzl` in the [Unified @pypi Hub](unified-pypi-hub) with
+the per-package macros of a concrete hub's: `requirement`, `whl_requirement`,
+`data_requirement` and `dist_info_requirement`. The labels they return are in
+the unified hub, so they follow {flag}`--venv` like `@pypi//<pkg>`.
+
+This is a migration aid for a repository whose hub used to be named `pypi`:
+after renaming it, its existing `load("@pypi//:requirements.bzl", "requirement")`
+calls keep working and can move to `@pypi//<pkg>` labels one at a time. The
+`all_*` lists are not generated, since they are fixed at loading time and could
+only list one hub's packages; load those from a concrete hub.
+
+Only the root module's value is used.
+
+:::{versionadded} VERSION_NEXT_FEATURE
+:::
+""",
+ ),
"whl_abi_tags": attr.string_list(
doc = """\
A list of ABIs to select wheels for. The values can be either strings or include template
diff --git a/python/private/pypi/unified_hub_repo.bzl b/python/private/pypi/unified_hub_repo.bzl
index f88db75..ddd15c0 100644
--- a/python/private/pypi/unified_hub_repo.bzl
+++ b/python/private/pypi/unified_hub_repo.bzl
@@ -7,7 +7,7 @@
load("@rules_python//python/private/pypi:unified_hub_setup.bzl", "define_venv_flag_config_settings")
package(default_visibility = ["//visibility:public"])
-
+{exports_requirements_bzl}
define_venv_flag_config_settings(
name = "venv_config_settings",
hubs = {hubs},
@@ -28,6 +28,35 @@
)
"""
+_EXPORTS_REQUIREMENTS_BZL = """
+# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it
+exports_files(["requirements.bzl"])
+"""
+
+# The per-package macros of a concrete hub's requirements.bzl. The `all_*`
+# lists are left out on purpose: they are fixed at loading time, before the
+# venv flag is known, so they could only ever list one hub's packages.
+_REQUIREMENTS_BZL_TMPL = '''\
+"""Starlark representation of the unified PyPI hub's packages.
+
+@generated by rules_python pip.parse bzlmod extension.
+"""
+
+load("@rules_python//python:pip.bzl", "pip_utils")
+
+def requirement(name):
+ return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "pkg")
+
+def whl_requirement(name):
+ return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "whl")
+
+def data_requirement(name):
+ return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "data")
+
+def dist_info_requirement(name):
+ return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "dist_info")
+'''
+
def _unified_hub_repo_impl(rctx):
hubs = rctx.attr.hubs
default_hub = rctx.attr.default_hub or None
@@ -35,7 +64,10 @@
# 1. Generate Root BUILD.bazel with shared config settings
rctx.file(
"BUILD.bazel",
- _ROOT_BUILD_TMPL.format(hubs = hubs),
+ _ROOT_BUILD_TMPL.format(
+ exports_requirements_bzl = _EXPORTS_REQUIREMENTS_BZL if rctx.attr.requirements_bzl else "",
+ hubs = hubs,
+ ),
)
# 2. Organize extra aliases by package
@@ -60,6 +92,23 @@
),
)
+ # 4. Optionally generate requirements.bzl with a concrete hub's per-package
+ # macros, so a hub renamed away from the reserved "pypi" name keeps its
+ # `load("@pypi//:requirements.bzl", "requirement")` users working. The
+ # macros return labels in this repo, so they route through the venv flag
+ # like any other `@pypi//<pkg>` label.
+ #
+ # NOTE: we are using the canonical name with the double '@' in order to
+ # always uniquely identify a repository, as the labels are being passed as
+ # a string and the resolution of the label happens at the call-site of the
+ # `requirement`, et al. macros.
+ if rctx.attr.requirements_bzl:
+ macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name)
+ rctx.file(
+ "requirements.bzl",
+ _REQUIREMENTS_BZL_TMPL.replace("%%MACRO_TMPL%%", macro_tmpl),
+ )
+
unified_hub_repo = repository_rule(
implementation = _unified_hub_repo_impl,
attrs = {
@@ -77,11 +126,15 @@
mandatory = True,
doc = "Dictionary mapping package names to a list of hubs that contain them.",
),
+ "requirements_bzl": attr.bool(
+ default = False,
+ doc = "Whether to generate a requirements.bzl with the per-package `requirement` macros.",
+ ),
},
doc = "Private repository rule creating the automatic Unified PyPI Hub.",
)
-def unified_workspace_hub_repo(name, hubs, default_hub = None, extra_aliases = {}):
+def unified_workspace_hub_repo(name, hubs, default_hub = None, extra_aliases = {}, requirements_bzl = False):
"""Creates a Unified PyPI Hub repository for WORKSPACE mode by loading requirements from hubs.
Args:
@@ -90,6 +143,8 @@
e.g. {"dev_pip": dev_pip_requirements, "pypi_alpha": pypi_alpha_requirements}
default_hub: Optional default hub name.
extra_aliases: Dictionary mapping 'package:alias' to a list of hubs that support it.
+ requirements_bzl: Whether to generate a requirements.bzl with the
+ per-package `requirement` macros.
"""
packages = {}
for hub_name, req_map in hubs.items():
@@ -108,4 +163,5 @@
extra_aliases = extra_aliases,
hubs = sorted(hubs.keys()),
packages = packages,
+ requirements_bzl = requirements_bzl,
)
diff --git a/tests/integration/unified_pypi/BUILD.bazel b/tests/integration/unified_pypi/BUILD.bazel
index c0b9905..c0aee2d 100644
--- a/tests/integration/unified_pypi/BUILD.bazel
+++ b/tests/integration/unified_pypi/BUILD.bazel
@@ -1,8 +1,28 @@
+load("@pypi//:requirements.bzl", "requirement")
load("@rules_python//python:py_binary.bzl", "py_binary")
load("@rules_python//python:py_test.bzl", "py_test")
package(default_visibility = ["//visibility:public"])
+# `requirement()` from the unified hub normalizes the name and resolves
+# through the venv flag like a plain `@pypi//<pkg>` label.
+py_test(
+ name = "test_requirement_default",
+ srcs = ["test_default.py"],
+ main = "test_default.py",
+ deps = [requirement("Colorama")],
+)
+
+py_test(
+ name = "test_requirement_a",
+ srcs = ["test_a.py"],
+ config_settings = {
+ "@rules_python//python/config_settings:venv": "pypi_a",
+ },
+ main = "test_a.py",
+ deps = [requirement("colorama")],
+)
+
py_test(
name = "test_default",
srcs = ["test_default.py"],
diff --git a/tests/integration/unified_pypi/MODULE.bazel b/tests/integration/unified_pypi/MODULE.bazel
index 6a4b87e..6dd2e25 100644
--- a/tests/integration/unified_pypi/MODULE.bazel
+++ b/tests/integration/unified_pypi/MODULE.bazel
@@ -45,6 +45,7 @@
use_repo(pip, "pypi_b")
pip.default(default_hub = "pypi_b")
+pip.default(unified_hub_requirements_bzl = True)
pip.dep(
name = "declared-only-pkg",
extra_targets = ["declared-only-alias"],
diff --git a/tests/integration/unified_pypi_test.py b/tests/integration/unified_pypi_test.py
index a19a9fc..08f7b02 100644
--- a/tests/integration/unified_pypi_test.py
+++ b/tests/integration/unified_pypi_test.py
@@ -13,6 +13,31 @@
def test_transitioned_hub(self):
self.run_bazel("test", "//:test_a")
+ def test_requirement_macro_default_hub(self):
+ self.run_bazel("test", "//:test_requirement_default")
+
+ def test_requirement_macro_transitioned_hub(self):
+ self.run_bazel("test", "//:test_requirement_a")
+
+ def test_requirements_bzl_requires_opt_in(self):
+ module_bazel = self.repo_root / "MODULE.bazel"
+ without_opt_in = module_bazel.read_text().replace(
+ "pip.default(unified_hub_requirements_bzl = True)\n",
+ "",
+ )
+ with self._temp_modify_file(module_bazel, without_opt_in):
+ result = self.run_bazel(
+ "cquery", "//:test_requirement_default", check=False
+ )
+ self.assertNotEqual(
+ result.exit_code,
+ 0,
+ "Expected loading @pypi//:requirements.bzl to fail without the opt-in",
+ )
+ self.assert_result_matches(
+ result, r"cannot load '[^']*//:requirements\.bzl': no such file"
+ )
+
def test_cli_override(self):
self.run_bazel(
"run",
diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl
index 97849e8..6d7e971 100644
--- a/tests/pypi/extension/extension_tests.bzl
+++ b/tests/pypi/extension/extension_tests.bzl
@@ -52,6 +52,7 @@
os_name = None,
platform = None,
pyproject_toml = None,
+ unified_hub_requirements_bzl = False,
whl_platform_tags = None,
whl_abi_tags = None):
return struct(
@@ -66,6 +67,7 @@
os_name = os_name,
platform = platform,
pyproject_toml = pyproject_toml,
+ unified_hub_requirements_bzl = unified_hub_requirements_bzl,
whl_abi_tags = whl_abi_tags or [],
whl_platform_tags = whl_platform_tags or [],
)
@@ -120,6 +122,7 @@
exposed_packages = subjects.dict,
hub_group_map = subjects.dict,
hub_whl_map = subjects.dict,
+ unified_hub_requirements_bzl = subjects.bool,
whl_libraries = subjects.dict,
whl_mods = subjects.dict,
),
@@ -497,6 +500,66 @@
_tests.append(_test_default_hub_precedence)
+def _parse_unified_hub_requirements_bzl(env, *modules):
+ return _parse_modules(
+ env,
+ module_ctx = _pypi_mock_mctx(
+ os_name = "linux",
+ arch_name = "x86_64",
+ *modules
+ ),
+ available_interpreters = {
+ "python_3_15_host": "unit_test_interpreter_target",
+ },
+ minor_mapping = {"3.15": "3.15.19"},
+ )
+
+def _unified_hub_requirements_bzl_mod(name, is_root, unified_hub_requirements_bzl):
+ return _mod(
+ name = name,
+ is_root = is_root,
+ default = _default_tags_default + [
+ _default(unified_hub_requirements_bzl = unified_hub_requirements_bzl),
+ ],
+ parse = [
+ _parse(
+ hub_name = name + "_pypi",
+ python_version = "3.15",
+ simpleapi_skip = ["simple"],
+ requirements_lock = "requirements.txt",
+ ),
+ ],
+ )
+
+def _test_unified_hub_requirements_bzl_off_by_default(env):
+ pypi = _parse_unified_hub_requirements_bzl(
+ env,
+ _unified_hub_requirements_bzl_mod("root", is_root = True, unified_hub_requirements_bzl = False),
+ )
+ pypi.unified_hub_requirements_bzl().equals(False)
+
+_tests.append(_test_unified_hub_requirements_bzl_off_by_default)
+
+def _test_unified_hub_requirements_bzl_root_opt_in(env):
+ pypi = _parse_unified_hub_requirements_bzl(
+ env,
+ _unified_hub_requirements_bzl_mod("root", is_root = True, unified_hub_requirements_bzl = True),
+ )
+ pypi.unified_hub_requirements_bzl().equals(True)
+
+_tests.append(_test_unified_hub_requirements_bzl_root_opt_in)
+
+def _test_unified_hub_requirements_bzl_ignored_outside_root(env):
+ """Only the root module decides whether the unified hub gets a requirements.bzl."""
+ pypi = _parse_unified_hub_requirements_bzl(
+ env,
+ _unified_hub_requirements_bzl_mod("root", is_root = True, unified_hub_requirements_bzl = False),
+ _unified_hub_requirements_bzl_mod("rules_python", is_root = False, unified_hub_requirements_bzl = True),
+ )
+ pypi.unified_hub_requirements_bzl().equals(False)
+
+_tests.append(_test_unified_hub_requirements_bzl_ignored_outside_root)
+
def _test_extension_dep(env):
pypi = _parse_modules(
env,
