refactor/docs: improve compile_pip_requirements error message and docs (#2792)
Resolution failure is the most common error from pip-compile, so we
should make sure the error message is as clean as it can be. Previously,
the output was cluttered with the exception traceback, which makes the
actual error hard to see (several nested traceback).
The new output shortens it with a nicer message:
```
Checking _main/requirements_lock.txt
ERROR: Cannot install requests<2.24 and requests~=2.25.1 because these package versions have conflicting dependencies.
ResolutionImpossible: for help visit https://pip.pypa.io/en/latest/topics/dependency-resolution/#dealing-with-dependency-conflicts
```
Fixes #2763
---------
Co-authored-by: Richard Levasseur <rlevasseur@google.com>
diff --git a/docs/pypi-dependencies.md b/docs/pypi-dependencies.md
index 6cc0da6..4ec40bc 100644
--- a/docs/pypi-dependencies.md
+++ b/docs/pypi-dependencies.md
@@ -5,8 +5,40 @@
Using PyPI packages (aka "pip install") involves two main steps.
-1. [Installing third party packages](#installing-third-party-packages)
-2. [Using third party packages as dependencies](#using-third-party-packages)
+1. [Generating requirements file](#generating-requirements-file)
+2. [Installing third party packages](#installing-third-party-packages)
+3. [Using third party packages as dependencies](#using-third-party-packages)
+
+{#generating-requirements-file}
+## Generating requirements file
+
+Generally, when working on a Python project, you'll have some dependencies that themselves have other dependencies. You might also specify dependency bounds instead of specific versions. So you'll need to generate a full list of all transitive dependencies and pinned versions for every dependency.
+
+Typically, you'd have your dependencies specified in `pyproject.toml` or `requirements.in` and generate the full pinned list of dependencies in `requirements_lock.txt`, which you can manage with the `compile_pip_requirements` Bazel rule:
+
+```starlark
+load("@rules_python//python:pip.bzl", "compile_pip_requirements")
+
+compile_pip_requirements(
+ name = "requirements",
+ src = "requirements.in",
+ requirements_txt = "requirements_lock.txt",
+)
+```
+
+This rule generates two targets:
+- `bazel run [name].update` will regenerate the `requirements_txt` file
+- `bazel test [name]_test` will test that the `requirements_txt` file is up to date
+
+For more documentation, see the API docs under {obj}`@rules_python//python:pip.bzl`.
+
+Once you generate this fully specified list of requirements, you can install the requirements with the instructions in [Installing third party packages](#installing-third-party-packages).
+
+:::{warning}
+If you're specifying dependencies in `pyproject.toml`, make sure to include the `[build-system]` configuration, with pinned dependencies. `compile_pip_requirements` will use the build system specified to read your project's metadata, and you might see non-hermetic behavior if you don't pin the build system.
+
+Not specifying `[build-system]` at all will result in using a default `[build-system]` configuration, which uses unpinned versions ([ref](https://peps.python.org/pep-0518/#build-system-table)).
+:::
{#installing-third-party-packages}
## Installing third party packages
@@ -27,8 +59,7 @@
)
use_repo(pip, "my_deps")
```
-For more documentation, including how the rules can update/create a requirements
-file, see the bzlmod examples under the {gh-path}`examples` folder or the documentation
+For more documentation, see the bzlmod examples under the {gh-path}`examples` folder or the documentation
for the {obj}`@rules_python//python/extensions:pip.bzl` extension.
```{note}
diff --git a/python/private/pypi/dependency_resolver/dependency_resolver.py b/python/private/pypi/dependency_resolver/dependency_resolver.py
index 89c9123..ada0763 100644
--- a/python/private/pypi/dependency_resolver/dependency_resolver.py
+++ b/python/private/pypi/dependency_resolver/dependency_resolver.py
@@ -15,14 +15,17 @@
"Set defaults for the pip-compile command to run it under Bazel"
import atexit
+import functools
import os
import shutil
import sys
from pathlib import Path
-from typing import Optional, Tuple
+from typing import List, Optional, Tuple
import click
import piptools.writer as piptools_writer
+from pip._internal.exceptions import DistributionNotFound
+from pip._vendor.resolvelib.resolvers import ResolutionImpossible
from piptools.scripts.compile import cli
from python.runfiles import runfiles
@@ -82,7 +85,7 @@
@click.command(context_settings={"ignore_unknown_options": True})
@click.option("--src", "srcs", multiple=True, required=True)
@click.argument("requirements_txt")
-@click.argument("update_target_label")
+@click.argument("target_label_prefix")
@click.option("--requirements-linux")
@click.option("--requirements-darwin")
@click.option("--requirements-windows")
@@ -90,7 +93,7 @@
def main(
srcs: Tuple[str, ...],
requirements_txt: str,
- update_target_label: str,
+ target_label_prefix: str,
requirements_linux: Optional[str],
requirements_darwin: Optional[str],
requirements_windows: Optional[str],
@@ -152,9 +155,10 @@
# or shutil.copyfile, as they will fail with OSError: [Errno 18] Invalid cross-device link.
shutil.copy(resolved_requirements_file, requirements_out)
- update_command = os.getenv("CUSTOM_COMPILE_COMMAND") or "bazel run %s" % (
- update_target_label,
+ update_command = (
+ os.getenv("CUSTOM_COMPILE_COMMAND") or f"bazel run {target_label_prefix}.update"
)
+ test_command = f"bazel test {target_label_prefix}_test"
os.environ["CUSTOM_COMPILE_COMMAND"] = update_command
os.environ["PIP_CONFIG_FILE"] = os.getenv("PIP_CONFIG_FILE") or os.devnull
@@ -168,6 +172,12 @@
)
argv.extend(extra_args)
+ _run_pip_compile = functools.partial(
+ run_pip_compile,
+ argv,
+ srcs_relative=srcs_relative,
+ )
+
if UPDATE:
print("Updating " + requirements_file_relative)
@@ -187,49 +197,66 @@
atexit.register(
lambda: shutil.copy(absolute_output_file, requirements_file_tree)
)
- cli(argv, standalone_mode=False)
+ _run_pip_compile(verbose_command=f"{update_command} -- --verbose")
requirements_file_relative_path = Path(requirements_file_relative)
content = requirements_file_relative_path.read_text()
content = content.replace(absolute_path_prefix, "")
requirements_file_relative_path.write_text(content)
else:
- # cli will exit(0) on success
- try:
- print("Checking " + requirements_file)
- cli(argv)
- print("cli() should exit", file=sys.stderr)
- sys.exit(1)
- except SystemExit as e:
- if e.code == 2:
- print(
- "pip-compile exited with code 2. This means that pip-compile found "
- "incompatible requirements or could not find a version that matches "
- f"the install requirement in one of {srcs_relative}.",
- file=sys.stderr,
- )
- sys.exit(1)
- elif e.code == 0:
- golden = open(_locate(bazel_runfiles, requirements_file)).readlines()
- out = open(requirements_out).readlines()
- out = [line.replace(absolute_path_prefix, "") for line in out]
- if golden != out:
- import difflib
+ print("Checking " + requirements_file)
+ sys.stdout.flush()
+ _run_pip_compile(verbose_command=f"{test_command} --test_arg=--verbose")
+ golden = open(_locate(bazel_runfiles, requirements_file)).readlines()
+ out = open(requirements_out).readlines()
+ out = [line.replace(absolute_path_prefix, "") for line in out]
+ if golden != out:
+ import difflib
- print("".join(difflib.unified_diff(golden, out)), file=sys.stderr)
- print(
- "Lock file out of date. Run '"
- + update_command
- + "' to update.",
- file=sys.stderr,
- )
- sys.exit(1)
- sys.exit(0)
- else:
- print(
- f"pip-compile unexpectedly exited with code {e.code}.",
- file=sys.stderr,
- )
- sys.exit(1)
+ print("".join(difflib.unified_diff(golden, out)), file=sys.stderr)
+ print(
+ f"Lock file out of date. Run '{update_command}' to update.",
+ file=sys.stderr,
+ )
+ sys.exit(1)
+
+
+def run_pip_compile(
+ args: List[str],
+ *,
+ srcs_relative: List[str],
+ verbose_command: str,
+) -> None:
+ try:
+ cli(args, standalone_mode=False)
+ except DistributionNotFound as e:
+ if isinstance(e.__cause__, ResolutionImpossible):
+ # pip logs an informative error to stderr already
+ # just render the error and exit
+ print(e)
+ sys.exit(1)
+ else:
+ raise
+ except SystemExit as e:
+ if e.code == 0:
+ return # shouldn't happen, but just in case
+ elif e.code == 2:
+ print(
+ "pip-compile exited with code 2. This means that pip-compile found "
+ "incompatible requirements or could not find a version that matches "
+ f"the install requirement in one of {srcs_relative}.\n"
+ "Try re-running with verbose:\n"
+ f" {verbose_command}",
+ file=sys.stderr,
+ )
+ sys.exit(1)
+ else:
+ print(
+ f"pip-compile unexpectedly exited with code {e.code}.\n"
+ "Try re-running with verbose:\n"
+ f" {verbose_command}",
+ file=sys.stderr,
+ )
+ sys.exit(1)
if __name__ == "__main__":
diff --git a/python/private/pypi/pip_compile.bzl b/python/private/pypi/pip_compile.bzl
index 8e46947..7edbf7d 100644
--- a/python/private/pypi/pip_compile.bzl
+++ b/python/private/pypi/pip_compile.bzl
@@ -110,7 +110,7 @@
args = ["--src=%s" % loc.format(src) for src in srcs] + [
loc.format(requirements_txt),
- "//%s:%s.update" % (native.package_name(), name),
+ "//%s:%s" % (native.package_name(), name),
"--resolver=backtracking",
"--allow-unsafe",
]
