fix(pypi): use python -B for repo-phase invocations (#2641)
Before this change we would just invoke the Python interpreter. This
means that in the `rules_python` directory there would be `__pycache__`
folders created in the source tree and the same `__pycache__` folders
would be created in the python interpreter repository rules if the
directories were writable.
This change ensures that we are executing `python` with `-B` in those
contexts and reduces any likelihood of us doing the wrong thing.
Work towards #1169.
---------
Co-authored-by: Richard Levasseur <richardlev@gmail.com>
diff --git a/CHANGELOG.md b/CHANGELOG.md
index da77574..8f97eef 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -67,6 +67,9 @@
([#1169](https://github.com/bazelbuild/rules_python/issues/1169)).
* (gazelle) Don't collapse depsets to a list or into args when generating the modules mapping file.
Support spilling modules mapping args into a params file.
+* (pypi) From now on `python` invocations in repository and module extension
+ evaluation contexts will invoke Python interpreter with `-B` to avoid
+ creating `.pyc` files.
{#v0-0-0-added}
### Added
diff --git a/python/private/pypi/evaluate_markers.bzl b/python/private/pypi/evaluate_markers.bzl
index ec5f576..028657f 100644
--- a/python/private/pypi/evaluate_markers.bzl
+++ b/python/private/pypi/evaluate_markers.bzl
@@ -55,12 +55,12 @@
pypi_repo_utils.execute_checked(
mrctx,
op = "ResolveRequirementEnvMarkers({})".format(in_file),
+ python = pypi_repo_utils.resolve_python_interpreter(
+ mrctx,
+ python_interpreter = python_interpreter,
+ python_interpreter_target = python_interpreter_target,
+ ),
arguments = [
- pypi_repo_utils.resolve_python_interpreter(
- mrctx,
- python_interpreter = python_interpreter,
- python_interpreter_target = python_interpreter_target,
- ),
"-m",
"python.private.pypi.requirements_parser.resolve_target_platforms",
in_file,
diff --git a/python/private/pypi/patch_whl.bzl b/python/private/pypi/patch_whl.bzl
index a7da224..c839f2e 100644
--- a/python/private/pypi/patch_whl.bzl
+++ b/python/private/pypi/patch_whl.bzl
@@ -27,8 +27,8 @@
within the wheel.
"""
-load("//python/private:repo_utils.bzl", "repo_utils")
load(":parse_whl_name.bzl", "parse_whl_name")
+load(":pypi_repo_utils.bzl", "pypi_repo_utils")
_rules_python_root = Label("//:BUILD.bazel")
@@ -102,10 +102,14 @@
record_patch = rctx.path("RECORD.patch")
whl_patched = patched_whl_name(whl_input.basename)
- repo_utils.execute_checked(
+ pypi_repo_utils.execute_checked(
rctx,
+ python = python_interpreter,
+ srcs = [
+ Label("//python/private/pypi:repack_whl.py"),
+ Label("//tools:wheelmaker.py"),
+ ],
arguments = [
- python_interpreter,
"-m",
"python.private.pypi.repack_whl",
"--record-patch",
diff --git a/python/private/pypi/pypi_repo_utils.bzl b/python/private/pypi/pypi_repo_utils.bzl
index 1964316..bb2acc8 100644
--- a/python/private/pypi/pypi_repo_utils.bzl
+++ b/python/private/pypi/pypi_repo_utils.bzl
@@ -104,11 +104,30 @@
])
return pypath
-def _execute_checked(mrctx, *, srcs, **kwargs):
+def _execute_prep(mrctx, *, python, srcs, **kwargs):
+ for src in srcs:
+ # This will ensure that we will re-evaluate the bzlmod extension or
+ # refetch the repository_rule when the srcs change. This should work on
+ # Bazel versions without `mrctx.watch` as well.
+ repo_utils.watch(mrctx, mrctx.path(src))
+
+ environment = kwargs.pop("environment", {})
+ pythonpath = environment.get("PYTHONPATH", "")
+ if pythonpath and not types.is_string(pythonpath):
+ environment["PYTHONPATH"] = _construct_pypath(mrctx, entries = pythonpath)
+ kwargs["environment"] = environment
+
+ # -B is added to prevent the repo-phase invocation from creating timestamp
+ # based pyc files, which contributes to race conditions and non-determinism
+ kwargs["arguments"] = [python, "-B"] + kwargs.get("arguments", [])
+ return kwargs
+
+def _execute_checked(mrctx, *, python, srcs, **kwargs):
"""Helper function to run a python script and modify the PYTHONPATH to include external deps.
Args:
mrctx: Handle to the module_ctx or repository_ctx.
+ python: The python interpreter to use.
srcs: The src files that the script depends on. This is important to
ensure that the Bazel repository cache or the bzlmod lock file gets
invalidated when any one file changes. It is advisable to use
@@ -118,26 +137,34 @@
the `environment` has a value `PYTHONPATH` and it is a list, then
it will be passed to `construct_pythonpath` function.
"""
-
- for src in srcs:
- # This will ensure that we will re-evaluate the bzlmod extension or
- # refetch the repository_rule when the srcs change. This should work on
- # Bazel versions without `mrctx.watch` as well.
- repo_utils.watch(mrctx, mrctx.path(src))
-
- env = kwargs.pop("environment", {})
- pythonpath = env.get("PYTHONPATH", "")
- if pythonpath and not types.is_string(pythonpath):
- env["PYTHONPATH"] = _construct_pypath(mrctx, entries = pythonpath)
-
return repo_utils.execute_checked(
mrctx,
- environment = env,
- **kwargs
+ **_execute_prep(mrctx, python = python, srcs = srcs, **kwargs)
+ )
+
+def _execute_checked_stdout(mrctx, *, python, srcs, **kwargs):
+ """Helper function to run a python script and modify the PYTHONPATH to include external deps.
+
+ Args:
+ mrctx: Handle to the module_ctx or repository_ctx.
+ python: The python interpreter to use.
+ srcs: The src files that the script depends on. This is important to
+ ensure that the Bazel repository cache or the bzlmod lock file gets
+ invalidated when any one file changes. It is advisable to use
+ `RECORD` files for external deps and the list of srcs from the
+ rules_python repo for any scripts.
+ **kwargs: Arguments forwarded to `repo_utils.execute_checked`. If
+ the `environment` has a value `PYTHONPATH` and it is a list, then
+ it will be passed to `construct_pythonpath` function.
+ """
+ return repo_utils.execute_checked_stdout(
+ mrctx,
+ **_execute_prep(mrctx, python = python, srcs = srcs, **kwargs)
)
pypi_repo_utils = struct(
construct_pythonpath = _construct_pypath,
execute_checked = _execute_checked,
+ execute_checked_stdout = _execute_checked_stdout,
resolve_python_interpreter = _resolve_python_interpreter,
)
diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl
index ef4077f..bdcf784 100644
--- a/python/private/pypi/whl_library.bzl
+++ b/python/private/pypi/whl_library.bzl
@@ -75,14 +75,15 @@
if not is_standalone_interpreter(rctx, python_interpreter, logger = logger):
return []
- stdout = repo_utils.execute_checked_stdout(
+ stdout = pypi_repo_utils.execute_checked_stdout(
rctx,
op = "GetPythonVersionForUnixCflags",
+ python = python_interpreter,
arguments = [
- python_interpreter,
"-c",
"import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}', end='')",
],
+ srcs = [],
)
_python_version = stdout
include_path = "{}/include/python{}".format(
@@ -181,7 +182,6 @@
python_interpreter_target = rctx.attr.python_interpreter_target,
)
args = [
- python_interpreter,
"-m",
"python.private.pypi.whl_installer.wheel_installer",
"--requirement",
@@ -247,6 +247,7 @@
# truncate the requirement value when logging it / reporting
# progress since it may contain several ' --hash=sha256:...
# --hash=sha256:...' substrings that fill up the console
+ python = python_interpreter,
op = op_tmpl.format(name = rctx.attr.name, requirement = rctx.attr.requirement.split(" ", 1)[0]),
arguments = args,
environment = environment,
@@ -295,6 +296,7 @@
pypi_repo_utils.execute_checked(
rctx,
op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path),
+ python = python_interpreter,
arguments = args + [
"--whl-file",
whl_path,
