Parse requirements files for global pip flags. (#456)
Parse requirements files for global pip flags, then add them to the extra_pip_args array.
Pass a copy of each cleaned up line (comments and line-breaks removed) to
each whl_library repo verbatim. Then link based requirements or requirement
lines with requirement specific flags work.
Due to the pip restriction on using requirement specific flags in
requirements.txt files only, write each requirement line to a temp file
before invoking pip in each whl_library repo.
Fixes #438 #447
diff --git a/examples/pip_parse/WORKSPACE b/examples/pip_parse/WORKSPACE
index fc9256b..0a48ebd 100644
--- a/examples/pip_parse/WORKSPACE
+++ b/examples/pip_parse/WORKSPACE
@@ -29,7 +29,7 @@
# (Optional) You can set quiet to False if you want to see pip output.
#quiet = False,
- # Uses the default repository name "pip_incremental"
+ # Uses the default repository name "pip_parsed_deps"
requirements_lock = "//:requirements_lock.txt",
)
diff --git a/examples/pip_parse/requirements_lock.txt b/examples/pip_parse/requirements_lock.txt
index b0d5b9e..b4dff9f 100644
--- a/examples/pip_parse/requirements_lock.txt
+++ b/examples/pip_parse/requirements_lock.txt
@@ -2,15 +2,27 @@
# This file is autogenerated by pip-compile
# To update, run:
#
-# pip-compile --output-file=requirements_lock.txt requirements.txt
+# pip-compile --generate-hashes --output-file=requirements_lock.txt requirements.txt
#
-certifi==2020.12.5
+--extra-index-url https://pypi.org/simple
+
+certifi==2020.12.5 \
+ --hash=sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c \
+ --hash=sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830 \
# via requests
-chardet==3.0.4
+chardet==3.0.4 \
+ --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
+ --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
# via requests
-idna==2.10
+idna==2.10 \
+ --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \
+ --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 \
# via requests
-requests==2.24.0
+requests==2.24.0 \
+ --hash=sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b \
+ --hash=sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898 \
# via -r requirements.txt
-urllib3==1.25.11
+urllib3==1.25.11 \
+ --hash=sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2 \
+ --hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e \
# via requests
diff --git a/python/pip_install/extract_wheels/lib/arguments_test.py b/python/pip_install/extract_wheels/lib/arguments_test.py
index 0d6a6af..c0338bd 100644
--- a/python/pip_install/extract_wheels/lib/arguments_test.py
+++ b/python/pip_install/extract_wheels/lib/arguments_test.py
@@ -17,7 +17,7 @@
args_dict = deserialize_structured_args(args_dict)
self.assertIn("repo", args_dict)
self.assertIn("extra_pip_args", args_dict)
- self.assertEqual(args_dict["pip_data_exclude"], None)
+ self.assertEqual(args_dict["pip_data_exclude"], [])
self.assertEqual(args_dict["enable_implicit_namespace_pkgs"], False)
self.assertEqual(args_dict["repo"], repo_name)
self.assertEqual(args_dict["extra_pip_args"], index_url)
diff --git a/python/pip_install/parse_requirements_to_bzl/__init__.py b/python/pip_install/parse_requirements_to_bzl/__init__.py
index e38f9b0..66e6f5e 100644
--- a/python/pip_install/parse_requirements_to_bzl/__init__.py
+++ b/python/pip_install/parse_requirements_to_bzl/__init__.py
@@ -2,28 +2,47 @@
import json
import textwrap
import sys
+import shlex
from typing import List, Tuple
from python.pip_install.extract_wheels.lib import bazel, arguments
from pip._internal.req import parse_requirements, constructors
from pip._internal.req.req_install import InstallRequirement
+from pip._internal.req.req_file import get_file_content, preprocess, handle_line, get_line_parser, RequirementsFileParser
from pip._internal.network.session import PipSession
-def parse_install_requirements(requirements_lock: str) -> List[InstallRequirement]:
- return [
- constructors.install_req_from_parsed_requirement(pr)
- for pr in parse_requirements(requirements_lock, session=PipSession())
- ]
+def parse_install_requirements(requirements_lock: str, extra_pip_args: List[str]) -> List[Tuple[InstallRequirement, str]]:
+ ps = PipSession()
+ # This is roughly taken from pip._internal.req.req_file.parse_requirements
+ # (https://github.com/pypa/pip/blob/21.0.1/src/pip/_internal/req/req_file.py#L127) in order to keep
+ # the original line (sort-of, its preprocessed) from the requirements_lock file around, to pass to sub repos
+ # as the requirement.
+ line_parser = get_line_parser(finder=None)
+ parser = RequirementsFileParser(ps, line_parser)
+ install_req_and_lines: List[Tuple[InstallRequirement, str]] = []
+ _, content = get_file_content(requirements_lock, ps)
+ for parsed_line, (_, line) in zip(parser.parse(requirements_lock, constraint=False), preprocess(content)):
+ if parsed_line.is_requirement:
+ install_req_and_lines.append(
+ (
+ constructors.install_req_from_line(parsed_line.requirement),
+ line
+ )
+ )
+
+ else:
+ extra_pip_args.extend(shlex.split(line))
+ return install_req_and_lines
-def repo_names_and_requirements(install_reqs: List[InstallRequirement], repo_prefix: str) -> List[Tuple[str, str]]:
+def repo_names_and_requirements(install_reqs: List[Tuple[InstallRequirement, str]], repo_prefix: str) -> List[Tuple[str, str]]:
return [
(
bazel.sanitise_name(ir.name, prefix=repo_prefix),
- str(ir.req)
+ line,
)
- for ir in install_reqs
+ for ir, line in install_reqs
]
def deserialize_structured_args(args):
@@ -35,6 +54,8 @@
for arg_name in structured_args:
if args.get(arg_name) is not None:
args[arg_name] = json.loads(args[arg_name])["args"]
+ else:
+ args[arg_name] = []
return args
@@ -54,13 +75,13 @@
requirements_lock = args.pop("requirements_lock")
repo_prefix = bazel.whl_library_repo_prefix(args["repo"])
- install_reqs = parse_install_requirements(requirements_lock)
- repo_names_and_reqs = repo_names_and_requirements(install_reqs, repo_prefix)
+ install_req_and_lines = parse_install_requirements(requirements_lock, args["extra_pip_args"])
+ repo_names_and_reqs = repo_names_and_requirements(install_req_and_lines, repo_prefix)
all_requirements = ", ".join(
- [bazel.sanitised_repo_library_label(ir.name, repo_prefix=repo_prefix) for ir in install_reqs]
+ [bazel.sanitised_repo_library_label(ir.name, repo_prefix=repo_prefix) for ir, _ in install_req_and_lines]
)
all_whl_requirements = ", ".join(
- [bazel.sanitised_repo_file_label(ir.name, repo_prefix=repo_prefix) for ir in install_reqs]
+ [bazel.sanitised_repo_file_label(ir.name, repo_prefix=repo_prefix) for ir, _ in install_req_and_lines]
)
return textwrap.dedent("""\
load("@rules_python//python/pip_install:pip_repository.bzl", "whl_library")
diff --git a/python/pip_install/parse_requirements_to_bzl/extract_single_wheel/__init__.py b/python/pip_install/parse_requirements_to_bzl/extract_single_wheel/__init__.py
index d2b9413..884b8ad 100644
--- a/python/pip_install/parse_requirements_to_bzl/extract_single_wheel/__init__.py
+++ b/python/pip_install/parse_requirements_to_bzl/extract_single_wheel/__init__.py
@@ -4,6 +4,8 @@
import subprocess
import json
+from tempfile import NamedTemporaryFile
+
from python.pip_install.extract_wheels.lib import bazel, requirements, arguments
from python.pip_install.extract_wheels import configure_reproducible_wheels
@@ -27,10 +29,15 @@
if args.extra_pip_args:
pip_args += json.loads(args.extra_pip_args)["args"]
- pip_args.append(args.requirement)
+ with NamedTemporaryFile(mode='wb') as requirement_file:
+ requirement_file.write(args.requirement.encode("utf-8"))
+ requirement_file.flush()
+ # Requirement specific args like --hash can only be passed in a requirements file,
+ # so write our single requirement into a temp file in case it has any of those flags.
+ pip_args.extend(["-r", requirement_file.name])
- # Assumes any errors are logged by pip so do nothing. This command will fail if pip fails
- subprocess.run(pip_args, check=True)
+ # Assumes any errors are logged by pip so do nothing. This command will fail if pip fails
+ subprocess.run(pip_args, check=True)
name, extras_for_pkg = requirements._parse_requirement_for_extra(args.requirement)
extras = {name: extras_for_pkg} if extras_for_pkg and name else dict()
diff --git a/python/pip_install/parse_requirements_to_bzl/parse_requirements_to_bzl_test.py b/python/pip_install/parse_requirements_to_bzl/parse_requirements_to_bzl_test.py
index 4b474d4..7199cea 100644
--- a/python/pip_install/parse_requirements_to_bzl/parse_requirements_to_bzl_test.py
+++ b/python/pip_install/parse_requirements_to_bzl/parse_requirements_to_bzl_test.py
@@ -15,8 +15,9 @@
def test_generated_requirements_bzl(self) -> None:
with NamedTemporaryFile() as requirements_lock:
- requirement_string = "foo==0.0.0"
- requirements_lock.write(bytes(requirement_string, encoding="utf-8"))
+ comments_and_flags = "#comment\n--require-hashes True\n"
+ requirement_string = "foo==0.0.0 --hash=sha256:hashofFoowhl"
+ requirements_lock.write(bytes(comments_and_flags + requirement_string, encoding="utf-8"))
requirements_lock.flush()
args = argparse.Namespace()
args.requirements_lock = requirements_lock.name
@@ -32,7 +33,8 @@
self.assertIn(all_whl_requirements, contents, contents)
self.assertIn(requirement_string, contents, contents)
self.assertIn(requirement_string, contents, contents)
- self.assertIn("'extra_pip_args': {}".format(repr(extra_pip_args)), contents, contents)
+ all_flags = extra_pip_args + ["--require-hashes", "True"]
+ self.assertIn("'extra_pip_args': {}".format(repr(all_flags)), contents, contents)
if __name__ == "__main__":
