fix(venv_shebang_rewriter): avoid depending on host coreutils (#4125)
Two scripts used as build actions in the pip integration ran directly
(`ctx.actions.run`, no shell) with no declared `PATH`. On sandboxed
environments without an FHS-style `/bin:/usr/bin` (e.g. NixOS), these
fail with `<tool>: command not found`, even though the action's own
executable resolves fine.
- `venv_shebang_rewriter.sh` resolved `head`/`tail`/`chmod` from `PATH`.
- `wheel_record_rewriter.sh` resolved `awk` from `PATH`.
## Fix
Both are ported to plain Python scripts exposed as `py_binary` targets,
per this project's own documented guidance (`PyExecToolsInfo`'s
`exec_interpreter` docs recommend a `py_binary` + `cfg=exec` over manual
interpreter wiring). This sidesteps host `PATH` concerns entirely.
---------
Co-authored-by: Richard Levasseur <richardlev@gmail.com>
diff --git a/news/4125.fixed.md b/news/4125.fixed.md
new file mode 100644
index 0000000..54c2096
--- /dev/null
+++ b/news/4125.fixed.md
@@ -0,0 +1,3 @@
+(pypi) Venv mode now works on NixOS and no longer requires coreutils to process
+wheel scripts and RECORD files
+([#4125](https://github.com/bazel-contrib/rules_python/pull/4125)).
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
index 38ecabc..35286c5 100644
--- a/python/private/pypi/BUILD.bazel
+++ b/python/private/pypi/BUILD.bazel
@@ -13,6 +13,7 @@
# limitations under the License.
load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+load("//python:py_binary.bzl", "py_binary")
package(default_visibility = ["//:__subpackages__"])
@@ -32,20 +33,34 @@
visibility = ["//visibility:public"],
)
+py_binary(
+ name = "venv_shebang_rewriter_py",
+ srcs = ["venv_shebang_rewriter.py"],
+ legacy_create_init = False,
+ main = "venv_shebang_rewriter.py",
+)
+
alias(
name = "venv_shebang_rewriter",
actual = select({
"@platforms//os:windows": "venv_shebang_rewriter.ps1",
- "//conditions:default": "venv_shebang_rewriter.sh",
+ "//conditions:default": "venv_shebang_rewriter_py",
}),
visibility = ["//visibility:public"],
)
+py_binary(
+ name = "wheel_record_rewriter_py",
+ srcs = ["wheel_record_rewriter.py"],
+ legacy_create_init = False,
+ main = "wheel_record_rewriter.py",
+)
+
alias(
name = "wheel_record_rewriter",
actual = select({
"@platforms//os:windows": "wheel_record_rewriter.ps1",
- "//conditions:default": "wheel_record_rewriter.sh",
+ "//conditions:default": "wheel_record_rewriter_py",
}),
visibility = ["//visibility:public"],
)
diff --git a/python/private/pypi/venv_shebang_rewriter.py b/python/private/pypi/venv_shebang_rewriter.py
new file mode 100644
index 0000000..48e615b
--- /dev/null
+++ b/python/private/pypi/venv_shebang_rewriter.py
@@ -0,0 +1,42 @@
+"""Rewrites a console_script wrapper's shebang into a batch/shell-Python polyglot."""
+
+import os
+import sys
+
+
+def main(argv):
+ in_path, out_path, target_os = argv[1:4]
+
+ with open(in_path, "rb") as in_file:
+ first_line = in_file.readline()
+ rest = in_file.read()
+
+ with open(out_path, "wb") as out_file:
+ if target_os == "windows":
+ python_exe = (
+ b"pythonw.exe" if first_line.startswith(b"#!pythonw") else b"python.exe"
+ )
+ # A Batch-Python polyglot. Batch executes the first line and exits,
+ # while Python (via -x) ignores the first line and executes the rest.
+ out_file.write(
+ b'@setlocal enabledelayedexpansion & "%~dp0'
+ + python_exe
+ + b'" -x "%~f0" %* & exit /b !ERRORLEVEL!\r\n',
+ )
+ else:
+ out_file.write(b"#!/bin/sh\n")
+ # A Shell-Python polyglot. The shell executes the triple-quoted 'exec'
+ # command, re-running the script with python3 from the scripts directory.
+ # Python ignores the triple-quoted string and continues.
+ out_file.write(
+ b"'''exec' \"$(dirname \"$0\")/python3\" \"$0\" \"$@\"\n' '''\n"
+ )
+
+ out_file.write(rest)
+
+ mode = os.stat(out_path).st_mode
+ os.chmod(out_path, mode | 0o111)
+
+
+if __name__ == "__main__":
+ main(sys.argv)
diff --git a/python/private/pypi/venv_shebang_rewriter.sh b/python/private/pypi/venv_shebang_rewriter.sh
deleted file mode 100755
index d4391d3..0000000
--- a/python/private/pypi/venv_shebang_rewriter.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/bin/sh
-set -eu
-
-IN="$1"
-OUT="$2"
-TARGET_OS="$3"
-
-FIRST_LINE=$(head -n 1 "$IN")
-
-if [ "$TARGET_OS" = "windows" ]; then
- case "$FIRST_LINE" in
- "#!pythonw"*) PYTHON_EXE="pythonw.exe" ;;
- *) PYTHON_EXE="python.exe" ;;
- esac
- # A Batch-Python polyglot. Batch executes the first line and exits,
- # while Python (via -x) ignores the first line and executes the rest.
- printf "@setlocal enabledelayedexpansion & \"%%~dp0$PYTHON_EXE\" -x \"%%~f0\" %%* & exit /b !ERRORLEVEL!\r\n" > "$OUT"
-else
- printf "#!/bin/sh\n" > "$OUT"
- # A Shell-Python polyglot. The shell executes the triple-quoted 'exec'
- # command, re-running the script with python3 from the scripts directory.
- # Python ignores the triple-quoted string and continues.
- printf "'''exec' \"\$(dirname \"\$0\")/python3\" \"\$0\" \"\$@\"\n' '''\n" >> "$OUT"
-fi
-
-tail -n +2 "$IN" >> "$OUT"
-chmod +x "$OUT"
diff --git a/python/private/pypi/wheel_record_rewriter.py b/python/private/pypi/wheel_record_rewriter.py
new file mode 100644
index 0000000..8931b30
--- /dev/null
+++ b/python/private/pypi/wheel_record_rewriter.py
@@ -0,0 +1,75 @@
+"""Rewrites a wheel's RECORD file to reflect its final installed layout."""
+
+import sys
+
+
+def _rewrite(in_path, out_path, target_os, data_dir_basename, rewritten_scripts):
+ data_prefix = data_dir_basename + "/"
+ quoted_data_prefix = '"' + data_prefix
+
+ if target_os == "windows":
+ data_repl = "../../"
+ headers_repl = "../../Include/"
+ platlib_repl = ""
+ purelib_repl = ""
+ scripts_repl = "../../Scripts/"
+ else:
+ data_repl = "../../../"
+ headers_repl = "../../../include/"
+ platlib_repl = ""
+ purelib_repl = ""
+ scripts_repl = "../../../bin/"
+
+ # PEP 427 specifies archive filenames are UTF-8, and while the encoding of
+ # dist-info files isn't formally standardized in PEP 376, packaging tools
+ # like pip and importlib.metadata treat them as UTF-8 in practice.
+ # See: https://discuss.python.org/t/encoding-of-files-in-the-dist-info-directory/6734/4
+ with open(in_path, encoding="utf-8") as in_file, open(
+ out_path, "w", encoding="utf-8", newline="\n"
+ ) as out_file:
+ for raw_line in in_file:
+ line = raw_line.rstrip("\r\n")
+
+ if line.startswith(quoted_data_prefix):
+ quote = '"'
+ rest = line[len(quoted_data_prefix) :]
+ elif line.startswith(data_prefix):
+ quote = ""
+ rest = line[len(data_prefix) :]
+ else:
+ out_file.write(line + "\n")
+ continue
+
+ if rest.startswith("purelib/"):
+ out_file.write(quote + purelib_repl + rest[len("purelib/") :] + "\n")
+ elif rest.startswith("platlib/"):
+ out_file.write(quote + platlib_repl + rest[len("platlib/") :] + "\n")
+ elif rest.startswith("scripts/"):
+ entry = rest[len("scripts/") :]
+ if target_os == "windows":
+ if quote == '"':
+ idx = entry.index('"')
+ else:
+ idx = entry.index(",")
+ spath, suffix = entry[:idx], entry[idx:]
+ if spath in rewritten_scripts:
+ spath += ".bat"
+ out_file.write(quote + scripts_repl + spath + suffix + "\n")
+ else:
+ out_file.write(quote + scripts_repl + entry + "\n")
+ elif rest.startswith("headers/"):
+ out_file.write(quote + headers_repl + rest[len("headers/") :] + "\n")
+ elif rest.startswith("data/"):
+ out_file.write(quote + data_repl + rest[len("data/") :] + "\n")
+ else:
+ out_file.write(line + "\n")
+
+
+def main(argv):
+ in_path, out_path, target_os, data_dir_basename = argv[1:5]
+ rewritten_scripts = set(argv[5:])
+ _rewrite(in_path, out_path, target_os, data_dir_basename, rewritten_scripts)
+
+
+if __name__ == "__main__":
+ main(sys.argv)
diff --git a/python/private/pypi/wheel_record_rewriter.sh b/python/private/pypi/wheel_record_rewriter.sh
deleted file mode 100755
index 4d93f2a..0000000
--- a/python/private/pypi/wheel_record_rewriter.sh
+++ /dev/null
@@ -1,85 +0,0 @@
-#!/bin/sh
-set -eu
-
-IN="$1"
-OUT="$2"
-TARGET_OS="$3"
-DATA_DIR_BASENAME="$4"
-shift 4
-
-DATA_PREFIX="${DATA_DIR_BASENAME}/"
-QUOTED_DATA_PREFIX="\"${DATA_DIR_BASENAME}/"
-
-if [ "$TARGET_OS" = "windows" ]; then
- DATA_REPL="../../"
- HEADERS_REPL="../../Include/"
- PLATLIB_REPL=""
- PURELIB_REPL=""
- SCRIPTS_REPL="../../Scripts/"
-else
- DATA_REPL="../../../"
- HEADERS_REPL="../../../include/"
- PLATLIB_REPL=""
- PURELIB_REPL=""
- SCRIPTS_REPL="../../../bin/"
-fi
-
-awk -v data_prefix="$DATA_PREFIX" \
- -v quoted_data_prefix="$QUOTED_DATA_PREFIX" \
- -v data_repl="$DATA_REPL" \
- -v headers_repl="$HEADERS_REPL" \
- -v platlib_repl="$PLATLIB_REPL" \
- -v purelib_repl="$PURELIB_REPL" \
- -v scripts_repl="$SCRIPTS_REPL" \
- -v target_os="$TARGET_OS" '
-BEGIN {
- for (i = 2; i < ARGC; i++) {
- rewritten[ARGV[i]] = 1
- }
- ARGC = 2
-}
-{
- line = $0
- quote = ""
- if (substr(line, 1, length(quoted_data_prefix)) == quoted_data_prefix) {
- quote = "\""
- rest = substr(line, length(quoted_data_prefix) + 1)
- } else if (substr(line, 1, length(data_prefix)) == data_prefix) {
- rest = substr(line, length(data_prefix) + 1)
- } else {
- print line
- next
- }
-
- if (substr(rest, 1, 8) == "purelib/") {
- print quote purelib_repl substr(rest, 9)
- } else if (substr(rest, 1, 8) == "platlib/") {
- print quote platlib_repl substr(rest, 9)
- } else if (substr(rest, 1, 8) == "scripts/") {
- entry = substr(rest, 9)
- if (target_os == "windows") {
- if (quote == "\"") {
- idx = index(entry, "\"")
- spath = substr(entry, 1, idx - 1)
- suffix = substr(entry, idx)
- } else {
- idx = index(entry, ",")
- spath = substr(entry, 1, idx - 1)
- suffix = substr(entry, idx)
- }
- if (spath in rewritten) {
- spath = spath ".bat"
- }
- print quote scripts_repl spath suffix
- } else {
- print quote scripts_repl entry
- }
- } else if (substr(rest, 1, 8) == "headers/") {
- print quote headers_repl substr(rest, 9)
- } else if (substr(rest, 1, 5) == "data/") {
- print quote data_repl substr(rest, 6)
- } else {
- print line
- }
-}
-' "$IN" "$@" > "$OUT"
