feat: Add support for envsubst in extra_pip_args (#1673)
Since `MODULE.bazel` files don't support `load()`, it is more difficult
to grab the value of an environment variable such as `PIP_INDEX_URL` or
`PIP_RETRIES` and feed it to `pip.parse()`. Some of these `PIP_*`
variables are useful in `extra_pip_args`. To enable this use case, also
for the bzlmod scenario, add the `envsubst` attribute, which is a list
of environment variables to substitute in `extra_pip_args`. If
`$VARNAME`, `${VARNAME}` or `${VARNAME:-default}` is found in
`extra_pip_args`, substitute the value of the variable.diff --git a/CHANGELOG.md b/CHANGELOG.md
index dfcdf36..01e7c07 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -27,6 +27,12 @@
This fixes issues due to pyc files being created at runtime and affecting the
definition of what files were considered part of the runtime.
+* (pip_parse) Added the `envsubst` parameter, which enables environment variable
+ substitutions in the `extra_pip_args` attribute.
+
+* (pip_repository) Added the `envsubst` parameter, which enables environment
+ variable substitutions in the `extra_pip_args` attribute.
+
### Fixed
* (bzlmod) pip.parse now does not fail with an empty `requirements.txt`.
diff --git a/examples/pip_parse_vendored/WORKSPACE b/examples/pip_parse_vendored/WORKSPACE
index e0b7c86..7904724 100644
--- a/examples/pip_parse_vendored/WORKSPACE
+++ b/examples/pip_parse_vendored/WORKSPACE
@@ -18,8 +18,14 @@
# This repository isn't referenced, except by our test that asserts the requirements.bzl is updated.
# It also wouldn't be needed by users of this ruleset.
+# If you're using envsubst with extra_pip_args, as we do below, the value of the environment
+# variables at the time we generate requirements.bzl don't make it into the file, as you may
+# verify by inspection; the environment variables at a later time, when we download the
+# packages, will be the ones that take effect.
pip_parse(
name = "pip",
+ envsubst = ["PIP_RETRIES"],
+ extra_pip_args = ["--retries=${PIP_RETRIES:-5}"],
python_interpreter_target = "@python39_host//:python",
requirements_lock = "//:requirements.txt",
)
diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl
index 206f8bb..de5d187 100644
--- a/examples/pip_parse_vendored/requirements.bzl
+++ b/examples/pip_parse_vendored/requirements.bzl
@@ -16,7 +16,7 @@
all_data_requirements = ["@pip//certifi:data", "@pip//charset_normalizer:data", "@pip//idna:data", "@pip//requests:data", "@pip//urllib3:data"]
_packages = [("pip_certifi", "certifi==2023.7.22 --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"), ("pip_charset_normalizer", "charset-normalizer==2.1.1 --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"), ("pip_idna", "idna==3.4 --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"), ("pip_requests", "requests==2.28.1 --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"), ("pip_urllib3", "urllib3==1.26.13 --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8")]
-_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "extra_pip_args": [], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": "@python39_host//:python", "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600}
+_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "envsubst": ["PIP_RETRIES"], "extra_pip_args": ["--retries=${PIP_RETRIES:-5}"], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": "@python39_host//:python", "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600}
_annotations = {}
def requirement(name):
diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel
index 4bcd5b8..d90fb2c 100644
--- a/python/pip_install/BUILD.bazel
+++ b/python/pip_install/BUILD.bazel
@@ -30,6 +30,7 @@
"//python/pip_install/private:generate_whl_library_build_bazel_bzl",
"//python/pip_install/private:srcs_bzl",
"//python/private:bzlmod_enabled_bzl",
+ "//python/private:envsubst_bzl",
"//python/private:normalize_name_bzl",
"//python/private:parse_whl_name_bzl",
"//python/private:patch_whl_bzl",
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl
index 8f65d13..382855c 100644
--- a/python/pip_install/pip_repository.bzl
+++ b/python/pip_install/pip_repository.bzl
@@ -22,6 +22,7 @@
load("//python/pip_install/private:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel")
load("//python/pip_install/private:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel")
load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS")
+load("//python/private:envsubst.bzl", "envsubst")
load("//python/private:normalize_name.bzl", "normalize_name")
load("//python/private:parse_whl_name.bzl", "parse_whl_name")
load("//python/private:patch_whl.bzl", "patch_whl")
@@ -195,12 +196,24 @@
if use_isolated(rctx, rctx.attr):
args.append("--isolated")
+ # At the time of writing, the very latest Bazel, as in `USE_BAZEL_VERSION=last_green bazelisk`
+ # supports rctx.getenv(name, default): When building incrementally, any change to the value of
+ # the variable named by name will cause this repository to be re-fetched. That hasn't yet made
+ # its way into the official releases, though.
+ if "getenv" in dir(rctx):
+ getenv = rctx.getenv
+ else:
+ getenv = rctx.os.environ.get
+
# Check for None so we use empty default types from our attrs.
# Some args want to be list, and some want to be dict.
if rctx.attr.extra_pip_args != None:
args += [
"--extra_pip_args",
- json.encode(struct(arg = rctx.attr.extra_pip_args)),
+ json.encode(struct(arg = [
+ envsubst(pip_arg, rctx.attr.envsubst, getenv)
+ for pip_arg in rctx.attr.extra_pip_args
+ ])),
]
if rctx.attr.download_only:
@@ -338,6 +351,7 @@
"download_only": rctx.attr.download_only,
"enable_implicit_namespace_pkgs": rctx.attr.enable_implicit_namespace_pkgs,
"environment": rctx.attr.environment,
+ "envsubst": rctx.attr.envsubst,
"extra_pip_args": options,
"isolated": use_isolated(rctx, rctx.attr),
"pip_data_exclude": rctx.attr.pip_data_exclude,
@@ -418,10 +432,23 @@
Can be used to set common variables such as `http_proxy`, `https_proxy` and `no_proxy`
Note that pip is run with "--isolated" on the CLI so `PIP_<VAR>_<NAME>`
style env vars are ignored, but env vars that control requests and urllib3
-can be passed.
+can be passed. If you need `PIP_<VAR>_<NAME>`, take a look at `extra_pip_args`
+and `envsubst`.
""",
default = {},
),
+ "envsubst": attr.string_list(
+ mandatory = False,
+ doc = """\
+A list of environment variables to substitute (e.g. `["PIP_INDEX_URL",
+"PIP_RETRIES"]`). The corresponding variables are expanded in `extra_pip_args`
+using the syntax `$VARNAME` or `${VARNAME}` (expanding to empty string if unset)
+or `${VARNAME:-default}` (expanding to default if the variable is unset or empty
+in the environment). Note: On Bazel 6 and Bazel 7 changes to the variables named
+here do not cause packages to be re-fetched. Don't fetch different things based
+on the value of these variables.
+""",
+ ),
"experimental_requirement_cycles": attr.string_list_dict(
default = {},
doc = """\
@@ -509,7 +536,14 @@
""",
),
"extra_pip_args": attr.string_list(
- doc = "Extra arguments to pass on to pip. Must not contain spaces.",
+ doc = """Extra arguments to pass on to pip. Must not contain spaces.
+
+Supports environment variables using the syntax `$VARNAME` or
+`${VARNAME}` (expanding to empty string if unset) or
+`${VARNAME:-default}` (expanding to default if the variable is unset
+or empty in the environment), if `"VARNAME"` is listed in the
+`envsubst` attribute. See also `envsubst`.
+""",
),
"isolated": attr.bool(
doc = """\
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
index 1017b09..20e260d 100644
--- a/python/private/BUILD.bazel
+++ b/python/private/BUILD.bazel
@@ -78,6 +78,11 @@
)
bzl_library(
+ name = "envsubst_bzl",
+ srcs = ["envsubst.bzl"],
+)
+
+bzl_library(
name = "full_version_bzl",
srcs = ["full_version.bzl"],
deps = ["//python:versions_bzl"],
diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl
index ce3ddde..b4dbf2f 100644
--- a/python/private/bzlmod/pip.bzl
+++ b/python/private/bzlmod/pip.bzl
@@ -200,6 +200,7 @@
pip_data_exclude = pip_attr.pip_data_exclude,
enable_implicit_namespace_pkgs = pip_attr.enable_implicit_namespace_pkgs,
environment = pip_attr.environment,
+ envsubst = pip_attr.envsubst,
group_name = group_name,
group_deps = group_deps,
)
diff --git a/python/private/envsubst.bzl b/python/private/envsubst.bzl
new file mode 100644
index 0000000..b2fdb99
--- /dev/null
+++ b/python/private/envsubst.bzl
@@ -0,0 +1,65 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Substitute environment variables in shell format strings."""
+
+def envsubst(template_string, varnames, getenv):
+ """Helper function to substitute environment variables.
+
+ Supports `$VARNAME`, `${VARNAME}` and `${VARNAME:-default}`
+ syntaxes in the `template_string`, looking up each `VARNAME`
+ listed in the `varnames` list in the environment defined by the
+ `getenv` function. Typically called with `getenv = rctx.getenv`
+ (if it is available) or `getenv = rctx.os.environ.get` (on e.g.
+ Bazel 6 or Bazel 7, which don't have `rctx.getenv` yet).
+
+ Limitations: Unlike the shell, we don't support `${VARNAME}` and
+ `${VARNAME:-default}` in the default expression for a different
+ environment variable expansion. We do support the braceless syntax
+ in the default, so an expression such as `${HOME:-/home/$USER}` is
+ valid.
+
+ Args:
+ template_string: String that may contain variables to be expanded.
+ varnames: List of variable names of variables to expand in
+ `template_string`.
+ getenv: Callable mapping variable names (in the first argument)
+ to their values, or returns the default (provided in the
+ second argument to `getenv`) if a value wasn't found.
+
+ Returns:
+ `template_string` with environment variables expanded according
+ to their values as determined by `getenv`.
+ """
+
+ if not varnames:
+ return template_string
+
+ for varname in varnames:
+ value = getenv(varname, "")
+ template_string = template_string.replace("$%s" % varname, value)
+ template_string = template_string.replace("${%s}" % varname, value)
+ segments = template_string.split("${%s:-" % varname)
+ template_string = segments.pop(0)
+ for segment in segments:
+ default_value, separator, rest = segment.partition("}")
+ if "{" in default_value:
+ fail("Environment substitution expression " +
+ "\"${%s:-\" has an opening \"{\" " % varname +
+ "in default value \"%s\"." % default_value)
+ if not separator:
+ fail("Environment substitution expression " +
+ "\"${%s:-\" is missing the final \"}\"" % varname)
+ template_string += (value if value else default_value) + rest
+ return template_string
diff --git a/tests/private/envsubst/BUILD.bazel b/tests/private/envsubst/BUILD.bazel
new file mode 100644
index 0000000..ec99705
--- /dev/null
+++ b/tests/private/envsubst/BUILD.bazel
@@ -0,0 +1,19 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for envsubsts."""
+
+load(":envsubst_tests.bzl", "envsubst_test_suite")
+
+envsubst_test_suite(name = "envsubst_tests")
diff --git a/tests/private/envsubst/envsubst_tests.bzl b/tests/private/envsubst/envsubst_tests.bzl
new file mode 100644
index 0000000..dd5e706
--- /dev/null
+++ b/tests/private/envsubst/envsubst_tests.bzl
@@ -0,0 +1,126 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Test for py_wheel."""
+
+load("@rules_testing//lib:analysis_test.bzl", "test_suite")
+load("//python/private:envsubst.bzl", "envsubst") # buildifier: disable=bzl-visibility
+
+_basic_tests = []
+
+def _test_envsubst_braceless(env):
+ env.expect.that_str(
+ envsubst("--retries=$PIP_RETRIES", ["PIP_RETRIES"], {"PIP_RETRIES": "5"}.get),
+ ).equals("--retries=5")
+
+ env.expect.that_str(
+ envsubst("--retries=$PIP_RETRIES", [], {"PIP_RETRIES": "5"}.get),
+ ).equals("--retries=$PIP_RETRIES")
+
+ env.expect.that_str(
+ envsubst("--retries=$PIP_RETRIES", ["PIP_RETRIES"], {}.get),
+ ).equals("--retries=")
+
+_basic_tests.append(_test_envsubst_braceless)
+
+def _test_envsubst_braces_without_default(env):
+ env.expect.that_str(
+ envsubst("--retries=${PIP_RETRIES}", ["PIP_RETRIES"], {"PIP_RETRIES": "5"}.get),
+ ).equals("--retries=5")
+
+ env.expect.that_str(
+ envsubst("--retries=${PIP_RETRIES}", [], {"PIP_RETRIES": "5"}.get),
+ ).equals("--retries=${PIP_RETRIES}")
+
+ env.expect.that_str(
+ envsubst("--retries=${PIP_RETRIES}", ["PIP_RETRIES"], {}.get),
+ ).equals("--retries=")
+
+_basic_tests.append(_test_envsubst_braces_without_default)
+
+def _test_envsubst_braces_with_default(env):
+ env.expect.that_str(
+ envsubst("--retries=${PIP_RETRIES:-6}", ["PIP_RETRIES"], {"PIP_RETRIES": "5"}.get),
+ ).equals("--retries=5")
+
+ env.expect.that_str(
+ envsubst("--retries=${PIP_RETRIES:-6}", [], {"PIP_RETRIES": "5"}.get),
+ ).equals("--retries=${PIP_RETRIES:-6}")
+
+ env.expect.that_str(
+ envsubst("--retries=${PIP_RETRIES:-6}", ["PIP_RETRIES"], {}.get),
+ ).equals("--retries=6")
+
+_basic_tests.append(_test_envsubst_braces_with_default)
+
+def _test_envsubst_nested_both_vars(env):
+ env.expect.that_str(
+ envsubst(
+ "${HOME:-/home/$USER}",
+ ["HOME", "USER"],
+ {"HOME": "/home/testuser", "USER": "mockuser"}.get,
+ ),
+ ).equals("/home/testuser")
+
+_basic_tests.append(_test_envsubst_nested_both_vars)
+
+def _test_envsubst_nested_outer_var(env):
+ env.expect.that_str(
+ envsubst(
+ "${HOME:-/home/$USER}",
+ ["HOME"],
+ {"HOME": "/home/testuser", "USER": "mockuser"}.get,
+ ),
+ ).equals("/home/testuser")
+
+_basic_tests.append(_test_envsubst_nested_outer_var)
+
+def _test_envsubst_nested_no_vars(env):
+ env.expect.that_str(
+ envsubst(
+ "${HOME:-/home/$USER}",
+ [],
+ {"HOME": "/home/testuser", "USER": "mockuser"}.get,
+ ),
+ ).equals("${HOME:-/home/$USER}")
+
+ env.expect.that_str(
+ envsubst("${HOME:-/home/$USER}", ["HOME", "USER"], {}.get),
+ ).equals("/home/")
+
+_basic_tests.append(_test_envsubst_nested_no_vars)
+
+def _test_envsubst_nested_braces_inner_var(env):
+ env.expect.that_str(
+ envsubst(
+ "Home directory is ${HOME:-/home/$USER}.",
+ ["HOME", "USER"],
+ {"USER": "mockuser"}.get,
+ ),
+ ).equals("Home directory is /home/mockuser.")
+
+ env.expect.that_str(
+ envsubst(
+ "Home directory is ${HOME:-/home/$USER}.",
+ ["USER"],
+ {"USER": "mockuser"}.get,
+ ),
+ ).equals("Home directory is ${HOME:-/home/mockuser}.")
+
+_basic_tests.append(_test_envsubst_nested_braces_inner_var)
+
+def envsubst_test_suite(name):
+ test_suite(
+ name = name,
+ basic_tests = _basic_tests,
+ )
