feat: support pyproject.toml in compile_pip_requirements (#1519)
With this PR we can also use `pyproject.toml` in addition
to `requirements.in` which helps in making the requirements
in a more structured form. For example, we could parse the
toml itself and create aliases in the hub repos only for the
packages outlined in the `pyproject.toml` file. The same for
`gazelle`, we could restrict `gazelle_python.yaml` contents
to only the dependencies listed in `pyproject.toml`.
Examples can be migrated once we agree on the interface.
Summary:
- feat: support pyproject.toml in compile_pip_requirements
- chore: use pyproject.toml for sphinx doc requirements
---------
Co-authored-by: Richard Levasseur <rlevasseur@google.com>
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 185ac37..ebbde9a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -48,6 +48,10 @@
default, which will cause `gazelle` to change third-party dependency labels
from `@pip_foo//:pkg` to `@pip//foo` by default.
+* The `compile_pip_requirements` now defaults to `pyproject.toml` if the `src`
+ or `requirements_in` attributes are unspecified, matching the upstream
+ `pip-compile` behaviour more closely.
+
Breaking changes:
* (pip) `pip_install` repository rule in this release has been disabled and
@@ -76,6 +80,9 @@
* (bzlmod) Added `.whl` patching support via `patches` and `patch_strip`
arguments to the new `pip.override` tag class.
+* (pip) Support for using [PEP621](https://peps.python.org/pep-0621/) compliant
+ `pyproject.toml` for creating a resolved `requirements.txt` file.
+
## [0.26.0] - 2023-10-06
### Changed
diff --git a/docs/sphinx/BUILD.bazel b/docs/sphinx/BUILD.bazel
index 1990269..7c99f77 100644
--- a/docs/sphinx/BUILD.bazel
+++ b/docs/sphinx/BUILD.bazel
@@ -102,8 +102,8 @@
# Run bazel run //docs/sphinx:requirements.update
compile_pip_requirements(
name = "requirements",
+ src = "pyproject.toml",
requirements_darwin = "requirements_darwin.txt",
- requirements_in = "requirements.in",
requirements_txt = "requirements_linux.txt",
target_compatible_with = _TARGET_COMPATIBLE_WITH,
)
diff --git a/docs/sphinx/pyproject.toml b/docs/sphinx/pyproject.toml
new file mode 100644
index 0000000..02e0f36
--- /dev/null
+++ b/docs/sphinx/pyproject.toml
@@ -0,0 +1,12 @@
+[project]
+name = "rules_python_docs"
+version = "0.0.0"
+
+dependencies = [
+ # NOTE: This is only used as input to create the resolved requirements.txt
+ # file, which is what builds, both Bazel and Readthedocs, both use.
+ "sphinx",
+ "myst-parser",
+ "sphinx_rtd_theme",
+ "readthedocs-sphinx-ext",
+]
diff --git a/docs/sphinx/requirements.in b/docs/sphinx/requirements.in
deleted file mode 100644
index c403778..0000000
--- a/docs/sphinx/requirements.in
+++ /dev/null
@@ -1,6 +0,0 @@
-# NOTE: This is only used as input to create the resolved requirements.txt file,
-# which is what builds, both Bazel and Readthedocs, both use.
-sphinx
-myst-parser
-sphinx_rtd_theme
-readthedocs-sphinx-ext
diff --git a/docs/sphinx/requirements_darwin.txt b/docs/sphinx/requirements_darwin.txt
index 1f47b83..5e3fd19 100644
--- a/docs/sphinx/requirements_darwin.txt
+++ b/docs/sphinx/requirements_darwin.txt
@@ -184,7 +184,7 @@
myst-parser==1.0.0 \
--hash=sha256:502845659313099542bd38a2ae62f01360e7dd4b1310f025dd014dfc0439cdae \
--hash=sha256:69fb40a586c6fa68995e6521ac0a525793935db7e724ca9bac1d33be51be9a4c
- # via -r docs/sphinx/requirements.in
+ # via rules-python-docs (docs/sphinx/pyproject.toml)
packaging==23.0 \
--hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \
--hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97
@@ -240,7 +240,7 @@
readthedocs-sphinx-ext==2.2.3 \
--hash=sha256:6583c26791a5853ee9e57ce9db864e2fb06808ba470f805d74d53fc50811e012 \
--hash=sha256:e9d911792789b88ae12e2be94d88c619f89a4fa1fe9e42c1505c9930a07163d8
- # via -r docs/sphinx/requirements.in
+ # via rules-python-docs (docs/sphinx/pyproject.toml)
requests==2.31.0 \
--hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \
--hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1
@@ -255,14 +255,14 @@
--hash=sha256:0dac3b698538ffef41716cf97ba26c1c7788dba73ce6f150c1ff5b4720786dd2 \
--hash=sha256:807d1cb3d6be87eb78a381c3e70ebd8d346b9a25f3753e9947e866b2786865fc
# via
- # -r docs/sphinx/requirements.in
# myst-parser
+ # rules-python-docs (docs/sphinx/pyproject.toml)
# sphinx-rtd-theme
# sphinxcontrib-jquery
sphinx-rtd-theme==1.2.0 \
--hash=sha256:a0d8bd1a2ed52e0b338cbe19c4b2eef3c5e7a048769753dac6a9f059c7b641b8 \
--hash=sha256:f823f7e71890abe0ac6aaa6013361ea2696fc8d3e1fa798f463e82bdb77eeff2
- # via -r docs/sphinx/requirements.in
+ # via rules-python-docs (docs/sphinx/pyproject.toml)
sphinxcontrib-applehelp==1.0.4 \
--hash=sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228 \
--hash=sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e
diff --git a/docs/sphinx/requirements_linux.txt b/docs/sphinx/requirements_linux.txt
index 1f47b83..5e3fd19 100644
--- a/docs/sphinx/requirements_linux.txt
+++ b/docs/sphinx/requirements_linux.txt
@@ -184,7 +184,7 @@
myst-parser==1.0.0 \
--hash=sha256:502845659313099542bd38a2ae62f01360e7dd4b1310f025dd014dfc0439cdae \
--hash=sha256:69fb40a586c6fa68995e6521ac0a525793935db7e724ca9bac1d33be51be9a4c
- # via -r docs/sphinx/requirements.in
+ # via rules-python-docs (docs/sphinx/pyproject.toml)
packaging==23.0 \
--hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \
--hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97
@@ -240,7 +240,7 @@
readthedocs-sphinx-ext==2.2.3 \
--hash=sha256:6583c26791a5853ee9e57ce9db864e2fb06808ba470f805d74d53fc50811e012 \
--hash=sha256:e9d911792789b88ae12e2be94d88c619f89a4fa1fe9e42c1505c9930a07163d8
- # via -r docs/sphinx/requirements.in
+ # via rules-python-docs (docs/sphinx/pyproject.toml)
requests==2.31.0 \
--hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \
--hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1
@@ -255,14 +255,14 @@
--hash=sha256:0dac3b698538ffef41716cf97ba26c1c7788dba73ce6f150c1ff5b4720786dd2 \
--hash=sha256:807d1cb3d6be87eb78a381c3e70ebd8d346b9a25f3753e9947e866b2786865fc
# via
- # -r docs/sphinx/requirements.in
# myst-parser
+ # rules-python-docs (docs/sphinx/pyproject.toml)
# sphinx-rtd-theme
# sphinxcontrib-jquery
sphinx-rtd-theme==1.2.0 \
--hash=sha256:a0d8bd1a2ed52e0b338cbe19c4b2eef3c5e7a048769753dac6a9f059c7b641b8 \
--hash=sha256:f823f7e71890abe0ac6aaa6013361ea2696fc8d3e1fa798f463e82bdb77eeff2
- # via -r docs/sphinx/requirements.in
+ # via rules-python-docs (docs/sphinx/pyproject.toml)
sphinxcontrib-applehelp==1.0.4 \
--hash=sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228 \
--hash=sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e
diff --git a/examples/build_file_generation/BUILD.bazel b/examples/build_file_generation/BUILD.bazel
index 5b01215..7b9766e 100644
--- a/examples/build_file_generation/BUILD.bazel
+++ b/examples/build_file_generation/BUILD.bazel
@@ -11,7 +11,7 @@
compile_pip_requirements(
name = "requirements",
- requirements_in = "requirements.in",
+ src = "requirements.in",
requirements_txt = "requirements_lock.txt",
requirements_windows = "requirements_windows.txt",
)
diff --git a/examples/bzlmod/BUILD.bazel b/examples/bzlmod/BUILD.bazel
index ff14016..5e2509a 100644
--- a/examples/bzlmod/BUILD.bazel
+++ b/examples/bzlmod/BUILD.bazel
@@ -16,7 +16,7 @@
# with pip-compile.
compile_pip_requirements_3_9(
name = "requirements_3_9",
- requirements_in = "requirements.in",
+ src = "requirements.in",
requirements_txt = "requirements_lock_3_9.txt",
requirements_windows = "requirements_windows_3_9.txt",
)
@@ -25,7 +25,7 @@
# with pip-compile.
compile_pip_requirements_3_10(
name = "requirements_3_10",
- requirements_in = "requirements.in",
+ src = "requirements.in",
requirements_txt = "requirements_lock_3_10.txt",
requirements_windows = "requirements_windows_3_10.txt",
)
diff --git a/examples/bzlmod/other_module/BUILD.bazel b/examples/bzlmod/other_module/BUILD.bazel
index d50a3a0..a93b92a 100644
--- a/examples/bzlmod/other_module/BUILD.bazel
+++ b/examples/bzlmod/other_module/BUILD.bazel
@@ -4,6 +4,6 @@
# override in the MODULE.bazel.
compile_pip_requirements_311(
name = "requirements",
- requirements_in = "requirements.in",
+ src = "requirements.in",
requirements_txt = "requirements_lock_3_11.txt",
)
diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel
index 1205817..bca3b36 100644
--- a/examples/bzlmod_build_file_generation/BUILD.bazel
+++ b/examples/bzlmod_build_file_generation/BUILD.bazel
@@ -16,7 +16,7 @@
# with pip-compile.
compile_pip_requirements(
name = "requirements",
- requirements_in = "requirements.in",
+ src = "requirements.in",
requirements_txt = "requirements_lock.txt",
requirements_windows = "requirements_windows.txt",
)
diff --git a/examples/multi_python_versions/requirements/BUILD.bazel b/examples/multi_python_versions/requirements/BUILD.bazel
index e3e821a..f67333a 100644
--- a/examples/multi_python_versions/requirements/BUILD.bazel
+++ b/examples/multi_python_versions/requirements/BUILD.bazel
@@ -5,24 +5,24 @@
compile_pip_requirements_3_8(
name = "requirements_3_8",
- requirements_in = "requirements.in",
+ src = "requirements.in",
requirements_txt = "requirements_lock_3_8.txt",
)
compile_pip_requirements_3_9(
name = "requirements_3_9",
- requirements_in = "requirements.in",
+ src = "requirements.in",
requirements_txt = "requirements_lock_3_9.txt",
)
compile_pip_requirements_3_10(
name = "requirements_3_10",
- requirements_in = "requirements.in",
+ src = "requirements.in",
requirements_txt = "requirements_lock_3_10.txt",
)
compile_pip_requirements_3_11(
name = "requirements_3_11",
- requirements_in = "requirements.in",
+ src = "requirements.in",
requirements_txt = "requirements_lock_3_11.txt",
)
diff --git a/examples/pip_parse/BUILD.bazel b/examples/pip_parse/BUILD.bazel
index cf5d0f6..c2cc9a3 100644
--- a/examples/pip_parse/BUILD.bazel
+++ b/examples/pip_parse/BUILD.bazel
@@ -53,7 +53,7 @@
# This rule adds a convenient way to update the requirements file.
compile_pip_requirements(
name = "requirements",
- requirements_in = "requirements.in",
+ src = "requirements.in",
requirements_txt = "requirements_lock.txt",
)
diff --git a/examples/pip_parse_vendored/BUILD.bazel b/examples/pip_parse_vendored/BUILD.bazel
index b87b2aa..8741c5a 100644
--- a/examples/pip_parse_vendored/BUILD.bazel
+++ b/examples/pip_parse_vendored/BUILD.bazel
@@ -4,7 +4,10 @@
# This rule adds a convenient way to update the requirements.txt
# lockfile based on the requirements.in.
-compile_pip_requirements(name = "requirements")
+compile_pip_requirements(
+ name = "requirements",
+ src = "requirements.in",
+)
# The requirements.bzl file is generated with a reference to the interpreter for the host platform.
# In order to check in a platform-agnostic file, we have to replace that reference with the symbol
diff --git a/examples/pip_repository_annotations/BUILD.bazel b/examples/pip_repository_annotations/BUILD.bazel
index 5b924e1..bdf9df1 100644
--- a/examples/pip_repository_annotations/BUILD.bazel
+++ b/examples/pip_repository_annotations/BUILD.bazel
@@ -9,6 +9,7 @@
# This rule adds a convenient way to update the requirements file.
compile_pip_requirements(
name = "requirements",
+ src = "requirements.in",
)
py_test(
diff --git a/python/pip_install/requirements.bzl b/python/pip_install/requirements.bzl
index 3935add..5caf762 100644
--- a/python/pip_install/requirements.bzl
+++ b/python/pip_install/requirements.bzl
@@ -19,6 +19,7 @@
def compile_pip_requirements(
name,
+ src = None,
extra_args = [],
extra_deps = [],
generate_hashes = True,
@@ -48,12 +49,17 @@
Args:
name: base name for generated targets, typically "requirements".
+ src: file containing inputs to dependency resolution. If not specified,
+ defaults to `pyproject.toml`. Supported formats are:
+ * a requirements text file, usually named `requirements.in`
+ * A `.toml` file, where the `project.dependencies` list is used as per
+ [PEP621](https://peps.python.org/pep-0621/).
extra_args: passed to pip-compile.
extra_deps: extra dependencies passed to pip-compile.
generate_hashes: whether to put hashes in the requirements_txt file.
py_binary: the py_binary rule to be used.
py_test: the py_test rule to be used.
- requirements_in: file expressing desired dependencies.
+ requirements_in: file expressing desired dependencies. Deprecated, use src instead.
requirements_txt: result of "compiling" the requirements.in file.
requirements_linux: File of linux specific resolve output to check validate if requirement.in has changes.
requirements_darwin: File of darwin specific resolve output to check validate if requirement.in has changes.
@@ -62,7 +68,11 @@
visibility: passed to both the _test and .update rules.
**kwargs: other bazel attributes passed to the "_test" rule.
"""
- requirements_in = name + ".in" if requirements_in == None else requirements_in
+ if requirements_in and src:
+ fail("Only one of 'src' and 'requirements_in' attributes can be used")
+ else:
+ src = requirements_in or src or "pyproject.toml"
+
requirements_txt = name + ".txt" if requirements_txt == None else requirements_txt
# "Default" target produced by this macro
@@ -74,7 +84,7 @@
visibility = visibility,
)
- data = [name, requirements_in, requirements_txt] + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None]
+ data = [name, requirements_txt, src] + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None]
# Use the Label constructor so this is expanded in the context of the file
# where it appears, which is to say, in @rules_python
@@ -83,7 +93,7 @@
loc = "$(rlocationpath {})"
args = [
- loc.format(requirements_in),
+ loc.format(src),
loc.format(requirements_txt),
"//%s:%s.update" % (native.package_name(), name),
"--resolver=backtracking",
@@ -105,6 +115,7 @@
requirement("colorama"),
requirement("importlib_metadata"),
requirement("more_itertools"),
+ requirement("packaging"),
requirement("pep517"),
requirement("pip"),
requirement("pip_tools"),
diff --git a/tests/compile_pip_requirements/BUILD.bazel b/tests/compile_pip_requirements/BUILD.bazel
index cadb59a..6df46b8 100644
--- a/tests/compile_pip_requirements/BUILD.bazel
+++ b/tests/compile_pip_requirements/BUILD.bazel
@@ -21,22 +21,22 @@
compile_pip_requirements(
name = "requirements",
+ src = "requirements.txt",
data = [
"requirements.in",
"requirements_extra.in",
],
- requirements_in = "requirements.txt",
requirements_txt = "requirements_lock.txt",
)
compile_pip_requirements(
name = "requirements_nohashes",
+ src = "requirements.txt",
data = [
"requirements.in",
"requirements_extra.in",
],
generate_hashes = False,
- requirements_in = "requirements.txt",
requirements_txt = "requirements_nohashes_lock.txt",
)
@@ -55,12 +55,12 @@
compile_pip_requirements(
name = "os_specific_requirements",
+ src = "requirements_os_specific.in",
data = [
"requirements_extra.in",
"requirements_os_specific.in",
],
requirements_darwin = "requirements_lock_darwin.txt",
- requirements_in = "requirements_os_specific.in",
requirements_linux = "requirements_lock_linux.txt",
requirements_txt = "requirements_lock.txt",
requirements_windows = "requirements_lock_windows.txt",
diff --git a/tests/pip_repository_entry_points/BUILD.bazel b/tests/pip_repository_entry_points/BUILD.bazel
index f0204ca..c39b1f0 100644
--- a/tests/pip_repository_entry_points/BUILD.bazel
+++ b/tests/pip_repository_entry_points/BUILD.bazel
@@ -5,6 +5,7 @@
# This rule adds a convenient way to update the requirements file.
compile_pip_requirements(
name = "requirements",
+ src = "requirements.in",
requirements_windows = ":requirements_windows.txt",
)
diff --git a/tools/publish/BUILD.bazel b/tools/publish/BUILD.bazel
index 065e56b..4759a31 100644
--- a/tools/publish/BUILD.bazel
+++ b/tools/publish/BUILD.bazel
@@ -2,6 +2,7 @@
compile_pip_requirements(
name = "requirements",
+ src = "requirements.in",
requirements_darwin = "requirements_darwin.txt",
requirements_windows = "requirements_windows.txt",
)
