fix(pypi): enable pip.default.envsubst for pip.default.index_url (#4438) It turns out that pip.parse.envsubst could not reasonably be used together with pip.default.index_url, as there could be multiple pip.parse tags with varying contents, so pip.default.index_url would not always be processed in the same way. To cope with this, we here introduce pip.default.envsubst, which can effectively be used together with pip.default.index_url. This also fixes a testing bug where envsubst was not a list. Closes #4123 --------- Co-authored-by: Ignas Anikevicius <240938+aignas@users.noreply.github.com> (cherry picked from commit 5f64d496c1da4a610c9feb8a0d19869b701d2b04) Work towards #4388
diff --git a/CHANGELOG.md b/CHANGELOG.md index 1534137..c21f14e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md
@@ -147,6 +147,9 @@ {obj}`--@rules_python//python/config_settings:venv`. The `all_*` lists are not provided, since they could only list one hub's packages; load them from a concrete hub. +* (pypi) Added {attr}`pip.default.envsubst` to expand environment variables in + {attr}`pip.default.index_url` + ([#4123](https://github.com/bazel-contrib/rules_python/issues/4123)). * (rules) Added {obj}`PyInfo.direct_implicit_pyc_files` and {obj}`PyExecutableInfo.venv_runfiles`. * (rules) Added {obj}`pytest_test`, which facilitates running of pytest tests. @@ -172,6 +175,7 @@ + {#v2-4-6} ## [2.4.6] - 2026-10-29
diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 5a57114..bcfe44c 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl
@@ -252,6 +252,7 @@ _configure( defaults, override = mod.is_root, + envsubst = tag.envsubst, index_url = tag.index_url, # extra values that we just add auth_patterns = tag.auth_patterns, @@ -264,6 +265,7 @@ return struct( auth_patterns = defaults.get("auth_patterns", {}), default_hub = default_hub, + envsubst = defaults.get("envsubst", []), index_url = defaults.get("index_url", "https://pypi.org/simple").rstrip("/"), netrc = defaults.get("netrc", None), python_version = defaults.get("python_version", None), @@ -443,6 +445,7 @@ builder.pip_parse( module_ctx, pip_attr = pip_attr, + default_envsubst = config.envsubst, python_version = python_version, ) @@ -686,14 +689,29 @@ * `sys_platform`, defaults to a value inferred from the {attr}`os_name`. """, ), + "envsubst": attr.string_list( + mandatory = False, + doc = """\ +A list of environment variables to substitute (e.g. `["PIP_INDEX_URL", +"PIP_RETRIES"]`). The corresponding variables are expanded in `index_url` +using the syntax `$VARNAME` or `${VARNAME}` (expanding to empty string if unset) +or `${VARNAME:-default}` (expanding to default if the variable is unset or empty +in the environment). Note: On Bazel 7 changes to the variables named here do +not cause packages to be re-fetched. Don't fetch different things based on the +value of these variables. + +:::{versionadded} 2.5.0 +::: +""", + ), "index_url": attr.string( doc = """\ The index URL to use as a default when downloading packages from PyPI. This is used if nothing is specified via `--index-url` or `--extra-index-url` parameters in the `requirements.txt` file or via the {attr}`pip.parse.extra_pip_args`. -This value is going to be subject to `envsubst` substitutions if necessary, look at the -{attr}`pip.parse.envsubst` documentation for more information.. +This value is subject to `envsubst` substitutions if necessary, look at the +{attr}`pip.default.envsubst` documentation for more information. The indexes must support Simple API as described here: <https://packaging.python.org/en/latest/specifications/simple-repository-api/>
diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 2500a34..ef6a724 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl
@@ -145,7 +145,7 @@ whl_libraries = self._whl_libraries, ) -def _pip_parse(self, module_ctx, pip_attr, python_version = None): +def _pip_parse(self, module_ctx, pip_attr, default_envsubst = [], python_version = None): python_version = python_version or pip_attr.python_version if python_version in self._platforms: fail(( @@ -202,7 +202,7 @@ )) return - _set_get_index_urls(self, module_ctx, pip_attr, python_version) + _set_get_index_urls(self, module_ctx, pip_attr, default_envsubst, python_version) self._platforms[python_version] = _platforms( module_ctx, python_version = full_python_version, @@ -375,7 +375,10 @@ ### end of setters, below we have various functions to implement the public methods -def _set_get_index_urls(self, mctx, pip_attr, python_version): +def _set_get_index_urls(self, mctx, pip_attr, default_envsubst, python_version): + # Combine default_envsubst and pip_attr.envsubst without duplications. + combined_envsubst = {k: True for k in default_envsubst + pip_attr.envsubst}.keys() + # Resolve the index URL through envsubst so the ``$VAR`` / ``${VAR:-default}`` # form is honored when deciding whether the experimental index-url mode is # active. Without this, an unsubstituted template like ``$RULES_PYTHON_PIP_INDEX_URL`` @@ -383,7 +386,7 @@ # would expand to the empty string. default_index_url = envsubst( pip_attr.experimental_index_url, - pip_attr.envsubst, + combined_envsubst, mctx.getenv, ) or self._config.index_url default_extra_index_urls = pip_attr.experimental_extra_index_urls or [] @@ -400,7 +403,11 @@ self._get_index_urls[python_version] = lambda ctx, distributions, *, index_url = None, extra_index_urls = None: self._simpleapi_download_fn( ctx, attr = struct( - index_url = _select_default_index(index_url, default_index_url).rstrip("/"), + index_url = envsubst( + _select_default_index(index_url, default_index_url).rstrip("/"), + combined_envsubst, + mctx.getenv, + ), extra_index_urls = [ x.rstrip("/") for x in (extra_index_urls or default_extra_index_urls) @@ -411,7 +418,7 @@ for d, versions in distributions.items() if _use_downloader(self, python_version, d) }, - envsubst = pip_attr.envsubst, + envsubst = combined_envsubst, # Auth related info netrc = self._config.netrc or pip_attr.netrc, auth_patterns = self._config.auth_patterns or pip_attr.auth_patterns,
diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 11d4c31..7c8f968 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl
@@ -46,6 +46,7 @@ config_settings = None, default_hub = "", env = None, + envsubst = None, index_url = None, marker = None, netrc = None, @@ -61,6 +62,7 @@ config_settings = config_settings, default_hub = default_hub, env = env or {}, + envsubst = envsubst or [], index_url = index_url or "", marker = marker or "", netrc = netrc, @@ -724,10 +726,57 @@ # `my_module` is setting the default. want_index_url = "https://pypi.internal.org/simple", ), + struct( + modules = [ + _mod( + name = "my_module", + default = [ + _default( + index_url = "${RULES_PYTHON_PIP_INDEX_URL}", + envsubst = ["RULES_PYTHON_PIP_INDEX_URL"], + ), + ], + parse = [ + _parse( + hub_name = "pypi_a", + python_version = "3.15", + requirements_lock = "requirements.txt", + ), + ], + ), + ], + environ = {"RULES_PYTHON_PIP_INDEX_URL": "https://pypi.internal.org/simple"}, + want_index_url = "https://pypi.internal.org/simple", + ), + struct( + modules = [ + _mod( + name = "my_module", + default = [ + _default( + index_url = "${RULES_PYTHON_PIP_INDEX_URL}", + envsubst = ["RULES_PYTHON_PIP_INDEX_URL"], + ), + ], + parse = [ + _parse( + hub_name = "pypi_a", + python_version = "3.15", + requirements_lock = "requirements_with_index_url.txt", + ), + ], + ), + ], + environ = {"RULES_PYTHON_PIP_INDEX_URL": "https://pypi.internal.org/simple"}, + # We want the file to have lower precedence than our default if the + # `my_module` is setting the default. + want_index_url = "https://pypi.internal.org/simple", + ), ]: pypi = _parse_modules( env, module_ctx = _pypi_mock_mctx( + environ = getattr(test, "environ", {}), os_name = "linux", arch_name = "x86_64", mock_files = {
diff --git a/tests/pypi/extension/pip_parse.bzl b/tests/pypi/extension/pip_parse.bzl index 7b5bdfd..638836e 100644 --- a/tests/pypi/extension/pip_parse.bzl +++ b/tests/pypi/extension/pip_parse.bzl
@@ -9,7 +9,7 @@ download_only = False, enable_implicit_namespace_pkgs = False, environment = {}, - envsubst = {}, + envsubst = [], experimental_extra_index_urls = [], experimental_index_url = "", experimental_requirement_cycles = {},
diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 48f6c44..073ccc0 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl
@@ -1395,7 +1395,7 @@ { "attr": struct( auth_patterns = {}, - envsubst = {}, + envsubst = [], extra_index_urls = [], index_url = "https://pypi.org/simple", index_url_overrides = {},