feat: add a tool to update internal dependencies (#1321)
Before this change the updates to the dependencies would happen very
seldomly, with this script, I propose we do it before each minor version
release. Adding a shell script and adding a reminder to the release
process may help with that.
diff --git a/DEVELOPING.md b/DEVELOPING.md
index 2972d96..3c9e89d 100644
--- a/DEVELOPING.md
+++ b/DEVELOPING.md
@@ -1,5 +1,16 @@
# For Developers
+## Updating internal dependencies
+
+1. Modify the `./python/pip_install/tools/requirements.txt` file and run:
+ ```
+ bazel run //tools/private/update_deps:update_pip_deps
+ ```
+1. Bump the coverage dependencies using the script using:
+ ```
+ bazel run //tools/private/update_deps:update_coverage_deps <VERSION>
+ ```
+
## Releasing
Start from a clean checkout at `main`.
diff --git a/MODULE.bazel b/MODULE.bazel
index b7a0411..aaa5c86 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -15,6 +15,7 @@
internal_deps.install()
use_repo(
internal_deps,
+ # START: maintained by 'bazel run //tools/private:update_pip_deps'
"pypi__build",
"pypi__click",
"pypi__colorama",
@@ -29,6 +30,7 @@
"pypi__tomli",
"pypi__wheel",
"pypi__zipp",
+ # END: maintained by 'bazel run //tools/private:update_pip_deps'
)
# We need to do another use_extension call to expose the "pythons_hub"
diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel
index 179fd62..4e4fbb4 100644
--- a/python/pip_install/BUILD.bazel
+++ b/python/pip_install/BUILD.bazel
@@ -10,6 +10,18 @@
)
filegroup(
+ name = "repositories",
+ srcs = ["repositories.bzl"],
+ visibility = ["//tools/private/update_deps:__pkg__"],
+)
+
+filegroup(
+ name = "requirements_txt",
+ srcs = ["tools/requirements.txt"],
+ visibility = ["//tools/private/update_deps:__pkg__"],
+)
+
+filegroup(
name = "bzl",
srcs = glob(["*.bzl"]) + [
"//python/pip_install/private:bzl_srcs",
diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl
index efe3bc7..4b209b3 100644
--- a/python/pip_install/repositories.bzl
+++ b/python/pip_install/repositories.bzl
@@ -20,6 +20,7 @@
load("//:version.bzl", "MINIMUM_BAZEL_VERSION")
_RULE_DEPS = [
+ # START: maintained by 'bazel run //tools/private:update_pip_deps'
(
"pypi__build",
"https://files.pythonhosted.org/packages/03/97/f58c723ff036a8d8b4d3115377c0a37ed05c1f68dd9a0d66dab5e82c5c1c/build-0.9.0-py3-none-any.whl",
@@ -36,11 +37,21 @@
"4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6",
),
(
+ "pypi__importlib_metadata",
+ "https://files.pythonhosted.org/packages/d7/31/74dcb59a601b95fce3b0334e8fc9db758f78e43075f22aeb3677dfb19f4c/importlib_metadata-1.4.0-py2.py3-none-any.whl",
+ "bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359",
+ ),
+ (
"pypi__installer",
"https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl",
"05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53",
),
(
+ "pypi__more_itertools",
+ "https://files.pythonhosted.org/packages/bd/3f/c4b3dbd315e248f84c388bd4a72b131a29f123ecacc37ffb2b3834546e42/more_itertools-8.13.0-py3-none-any.whl",
+ "c5122bffc5f104d37c1626b8615b511f3427aa5389b94d61e5ef8236bfbc3ddb",
+ ),
+ (
"pypi__packaging",
"https://files.pythonhosted.org/packages/8f/7b/42582927d281d7cb035609cd3a543ffac89b74f3f4ee8e1c50914bcb57eb/packaging-22.0-py3-none-any.whl",
"957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3",
@@ -76,20 +87,11 @@
"b60533f3f5d530e971d6737ca6d58681ee434818fab630c83a734bb10c083ce8",
),
(
- "pypi__importlib_metadata",
- "https://files.pythonhosted.org/packages/d7/31/74dcb59a601b95fce3b0334e8fc9db758f78e43075f22aeb3677dfb19f4c/importlib_metadata-1.4.0-py2.py3-none-any.whl",
- "bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359",
- ),
- (
"pypi__zipp",
"https://files.pythonhosted.org/packages/f4/50/cc72c5bcd48f6e98219fc4a88a5227e9e28b81637a99c49feba1d51f4d50/zipp-1.0.0-py2.py3-none-any.whl",
"8dda78f06bd1674bd8720df8a50bb47b6e1233c503a4eed8e7810686bde37656",
),
- (
- "pypi__more_itertools",
- "https://files.pythonhosted.org/packages/bd/3f/c4b3dbd315e248f84c388bd4a72b131a29f123ecacc37ffb2b3834546e42/more_itertools-8.13.0-py3-none-any.whl",
- "c5122bffc5f104d37c1626b8615b511f3427aa5389b94d61e5ef8236bfbc3ddb",
- ),
+ # END: maintained by 'bazel run //tools/private:update_pip_deps'
]
_GENERIC_WHEEL = """\
diff --git a/python/pip_install/tools/requirements.txt b/python/pip_install/tools/requirements.txt
new file mode 100755
index 0000000..e8de112
--- /dev/null
+++ b/python/pip_install/tools/requirements.txt
@@ -0,0 +1,14 @@
+build==0.9
+click==8.0.1
+colorama
+importlib_metadata==1.4.0
+installer
+more_itertools==8.13.0
+packaging==22.0
+pep517
+pip==22.3.1
+pip_tools==6.12.1
+setuptools==60.10
+tomli
+wheel==0.38.4
+zipp==1.0.0
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
index 10af17e..7220ccf 100644
--- a/python/private/BUILD.bazel
+++ b/python/private/BUILD.bazel
@@ -24,6 +24,12 @@
visibility = ["//python:__pkg__"],
)
+filegroup(
+ name = "coverage_deps",
+ srcs = ["coverage_deps.bzl"],
+ visibility = ["//tools/private/update_deps:__pkg__"],
+)
+
# Filegroup of bzl files that can be used by downstream rules for documentation generation
filegroup(
name = "bzl",
diff --git a/python/private/coverage_deps.bzl b/python/private/coverage_deps.bzl
index 93938e9..863d496 100644
--- a/python/private/coverage_deps.bzl
+++ b/python/private/coverage_deps.bzl
@@ -19,8 +19,7 @@
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
load("//python/private:version_label.bzl", "version_label")
-# Update with './tools/update_coverage_deps.py <version>'
-#START: managed by update_coverage_deps.py script
+# START: maintained by 'bazel run //tools/private:update_coverage_deps'
_coverage_deps = {
"cp310": {
"aarch64-apple-darwin": (
@@ -95,7 +94,7 @@
),
},
}
-#END: managed by update_coverage_deps.py script
+# END: maintained by 'bazel run //tools/private:update_coverage_deps'
_coverage_patch = Label("//python/private:coverage.patch")
diff --git a/tools/private/update_deps/BUILD.bazel b/tools/private/update_deps/BUILD.bazel
new file mode 100644
index 0000000..2ab7cc7
--- /dev/null
+++ b/tools/private/update_deps/BUILD.bazel
@@ -0,0 +1,76 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+load("//python:py_binary.bzl", "py_binary")
+load("//python:py_library.bzl", "py_library")
+load("//python:py_test.bzl", "py_test")
+
+licenses(["notice"])
+
+py_library(
+ name = "args",
+ srcs = ["args.py"],
+ imports = ["../../.."],
+ deps = ["//python/runfiles"],
+)
+
+py_library(
+ name = "update_file",
+ srcs = ["update_file.py"],
+ imports = ["../../.."],
+)
+
+py_binary(
+ name = "update_coverage_deps",
+ srcs = ["update_coverage_deps.py"],
+ data = [
+ "//python/private:coverage_deps",
+ ],
+ env = {
+ "UPDATE_FILE": "$(rlocationpath //python/private:coverage_deps)",
+ },
+ imports = ["../../.."],
+ deps = [
+ ":args",
+ ":update_file",
+ ],
+)
+
+py_binary(
+ name = "update_pip_deps",
+ srcs = ["update_pip_deps.py"],
+ data = [
+ "//:MODULE.bazel",
+ "//python/pip_install:repositories",
+ "//python/pip_install:requirements_txt",
+ ],
+ env = {
+ "MODULE_BAZEL": "$(rlocationpath //:MODULE.bazel)",
+ "REPOSITORIES_BZL": "$(rlocationpath //python/pip_install:repositories)",
+ "REQUIREMENTS_TXT": "$(rlocationpath //python/pip_install:requirements_txt)",
+ },
+ imports = ["../../.."],
+ deps = [
+ ":args",
+ ":update_file",
+ ],
+)
+
+py_test(
+ name = "update_file_test",
+ srcs = ["update_file_test.py"],
+ imports = ["../../.."],
+ deps = [
+ ":update_file",
+ ],
+)
diff --git a/tools/private/update_deps/args.py b/tools/private/update_deps/args.py
new file mode 100644
index 0000000..293294c
--- /dev/null
+++ b/tools/private/update_deps/args.py
@@ -0,0 +1,35 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A small library for common arguments when updating files."""
+
+import pathlib
+
+from python.runfiles import runfiles
+
+
+def path_from_runfiles(input: str) -> pathlib.Path:
+ """A helper to create a path from runfiles.
+
+ Args:
+ input: the string input to construct a path.
+
+ Returns:
+ the pathlib.Path path to a file which is verified to exist.
+ """
+ path = pathlib.Path(runfiles.Create().Rlocation(input))
+ if not path.exists():
+ raise ValueError(f"Path '{path}' does not exist")
+
+ return path
diff --git a/tools/update_coverage_deps.py b/tools/private/update_deps/update_coverage_deps.py
similarity index 75%
rename from tools/update_coverage_deps.py
rename to tools/private/update_deps/update_coverage_deps.py
index 57b7850..72baa44 100755
--- a/tools/update_coverage_deps.py
+++ b/tools/private/update_deps/update_coverage_deps.py
@@ -22,6 +22,7 @@
import argparse
import difflib
import json
+import os
import pathlib
import sys
import textwrap
@@ -30,6 +31,9 @@
from typing import Any
from urllib import request
+from tools.private.update_deps.args import path_from_runfiles
+from tools.private.update_deps.update_file import update_file
+
# This should be kept in sync with //python:versions.bzl
_supported_platforms = {
# Windows is unsupported right now
@@ -110,64 +114,6 @@
)
-def _writelines(path: pathlib.Path, lines: list[str]):
- with open(path, "w") as f:
- f.writelines(lines)
-
-
-def _difflines(path: pathlib.Path, lines: list[str]):
- with open(path) as f:
- input = f.readlines()
-
- rules_python = pathlib.Path(__file__).parent.parent
- p = path.relative_to(rules_python)
-
- print(f"Diff of the changes that would be made to '{p}':")
- for line in difflib.unified_diff(
- input,
- lines,
- fromfile=f"a/{p}",
- tofile=f"b/{p}",
- ):
- print(line, end="")
-
- # Add an empty line at the end of the diff
- print()
-
-
-def _update_file(
- path: pathlib.Path,
- snippet: str,
- start_marker: str,
- end_marker: str,
- dry_run: bool = True,
-):
- with open(path) as f:
- input = f.readlines()
-
- out = []
- skip = False
- for line in input:
- if skip:
- if not line.startswith(end_marker):
- continue
-
- skip = False
-
- out.append(line)
-
- if not line.startswith(start_marker):
- continue
-
- skip = True
- out.extend([f"{line}\n" for line in snippet.splitlines()])
-
- if dry_run:
- _difflines(path, out)
- else:
- _writelines(path, out)
-
-
def _parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(__doc__)
parser.add_argument(
@@ -193,6 +139,12 @@
action="store_true",
help="Wether to write to files",
)
+ parser.add_argument(
+ "--update-file",
+ type=path_from_runfiles,
+ default=os.environ.get("UPDATE_FILE"),
+ help="The path for the file to be updated, defaults to the value taken from UPDATE_FILE",
+ )
return parser.parse_args()
@@ -230,14 +182,12 @@
urls.sort(key=lambda x: f"{x.python}_{x.platform}")
- rules_python = pathlib.Path(__file__).parent.parent
-
# Update the coverage_deps, which are used to register deps
- _update_file(
- path=rules_python / "python" / "private" / "coverage_deps.bzl",
+ update_file(
+ path=args.update_file,
snippet=f"_coverage_deps = {repr(Deps(urls))}\n",
- start_marker="#START: managed by update_coverage_deps.py script",
- end_marker="#END: managed by update_coverage_deps.py script",
+ start_marker="# START: maintained by 'bazel run //tools/private:update_coverage_deps'",
+ end_marker="# END: maintained by 'bazel run //tools/private:update_coverage_deps'",
dry_run=args.dry_run,
)
diff --git a/tools/private/update_deps/update_file.py b/tools/private/update_deps/update_file.py
new file mode 100644
index 0000000..ab3e8a8
--- /dev/null
+++ b/tools/private/update_deps/update_file.py
@@ -0,0 +1,114 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A small library to update bazel files within the repo.
+
+This is reused in other files updating coverage deps and pip deps.
+"""
+
+import argparse
+import difflib
+import pathlib
+import sys
+
+
+def _writelines(path: pathlib.Path, out: str):
+ with open(path, "w") as f:
+ f.write(out)
+
+
+def unified_diff(name: str, a: str, b: str) -> str:
+ return "".join(
+ difflib.unified_diff(
+ a.splitlines(keepends=True),
+ b.splitlines(keepends=True),
+ fromfile=f"a/{name}",
+ tofile=f"b/{name}",
+ )
+ ).strip()
+
+
+def replace_snippet(
+ current: str,
+ snippet: str,
+ start_marker: str,
+ end_marker: str,
+) -> str:
+ """Update a file on disk to replace text in a file between two markers.
+
+ Args:
+ path: pathlib.Path, the path to the file to be modified.
+ snippet: str, the snippet of code to insert between the markers.
+ start_marker: str, the text that marks the start of the region to be replaced.
+ end_markr: str, the text that marks the end of the region to be replaced.
+ dry_run: bool, if set to True, then the file will not be written and instead we are going to print a diff to
+ stdout.
+ """
+ lines = []
+ skip = False
+ found_match = False
+ for line in current.splitlines(keepends=True):
+ if line.lstrip().startswith(start_marker.lstrip()):
+ found_match = True
+ lines.append(line)
+ lines.append(snippet.rstrip() + "\n")
+ skip = True
+ elif skip and line.lstrip().startswith(end_marker):
+ skip = False
+ lines.append(line)
+ continue
+ elif not skip:
+ lines.append(line)
+
+ if not found_match:
+ raise RuntimeError(f"Start marker '{start_marker}' was not found")
+ if skip:
+ raise RuntimeError(f"End marker '{end_marker}' was not found")
+
+ return "".join(lines)
+
+
+def update_file(
+ path: pathlib.Path,
+ snippet: str,
+ start_marker: str,
+ end_marker: str,
+ dry_run: bool = True,
+):
+ """update a file on disk to replace text in a file between two markers.
+
+ Args:
+ path: pathlib.Path, the path to the file to be modified.
+ snippet: str, the snippet of code to insert between the markers.
+ start_marker: str, the text that marks the start of the region to be replaced.
+ end_markr: str, the text that marks the end of the region to be replaced.
+ dry_run: bool, if set to True, then the file will not be written and instead we are going to print a diff to
+ stdout.
+ """
+ current = path.read_text()
+ out = replace_snippet(current, snippet, start_marker, end_marker)
+
+ if not dry_run:
+ _writelines(path, out)
+ return
+
+ relative = path.relative_to(
+ pathlib.Path(__file__).resolve().parent.parent.parent.parent
+ )
+ name = f"{relative}"
+ diff = unified_diff(name, current, out)
+ if diff:
+ print(f"Diff of the changes that would be made to '{name}':\n{diff}")
+ else:
+ print(f"'{name}' is up to date")
diff --git a/tools/private/update_deps/update_file_test.py b/tools/private/update_deps/update_file_test.py
new file mode 100644
index 0000000..01c6ec7
--- /dev/null
+++ b/tools/private/update_deps/update_file_test.py
@@ -0,0 +1,128 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from tools.private.update_deps.update_file import replace_snippet, unified_diff
+
+
+class TestReplaceSnippet(unittest.TestCase):
+ def test_replace_simple(self):
+ current = """\
+Before the snippet
+
+# Start marker
+To be replaced
+It may have the '# Start marker' or '# End marker' in the middle,
+But it has to be in the beginning of the line to mark the end of a region.
+# End marker
+
+After the snippet
+"""
+ snippet = "Replaced"
+ got = replace_snippet(
+ current=current,
+ snippet="Replaced",
+ start_marker="# Start marker",
+ end_marker="# End marker",
+ )
+
+ want = """\
+Before the snippet
+
+# Start marker
+Replaced
+# End marker
+
+After the snippet
+"""
+ self.assertEqual(want, got)
+
+ def test_replace_indented(self):
+ current = """\
+Before the snippet
+
+ # Start marker
+ To be replaced
+ # End marker
+
+After the snippet
+"""
+ got = replace_snippet(
+ current=current,
+ snippet=" Replaced",
+ start_marker="# Start marker",
+ end_marker="# End marker",
+ )
+
+ want = """\
+Before the snippet
+
+ # Start marker
+ Replaced
+ # End marker
+
+After the snippet
+"""
+ self.assertEqual(want, got)
+
+ def test_raises_if_start_is_not_found(self):
+ with self.assertRaises(RuntimeError) as exc:
+ replace_snippet(
+ current="foo",
+ snippet="",
+ start_marker="start",
+ end_marker="end",
+ )
+
+ self.assertEqual(exc.exception.args[0], "Start marker 'start' was not found")
+
+ def test_raises_if_end_is_not_found(self):
+ with self.assertRaises(RuntimeError) as exc:
+ replace_snippet(
+ current="start",
+ snippet="",
+ start_marker="start",
+ end_marker="end",
+ )
+
+ self.assertEqual(exc.exception.args[0], "End marker 'end' was not found")
+
+
+class TestUnifiedDiff(unittest.TestCase):
+ def test_diff(self):
+ give_a = """\
+First line
+second line
+Third line
+"""
+ give_b = """\
+First line
+Second line
+Third line
+"""
+ got = unified_diff("filename", give_a, give_b)
+ want = """\
+--- a/filename
++++ b/filename
+@@ -1,3 +1,3 @@
+ First line
+-second line
++Second line
+ Third line"""
+ self.assertEqual(want, got)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tools/private/update_deps/update_pip_deps.py b/tools/private/update_deps/update_pip_deps.py
new file mode 100755
index 0000000..8a2dd5f
--- /dev/null
+++ b/tools/private/update_deps/update_pip_deps.py
@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A script to manage internal pip dependencies."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import pathlib
+import re
+import sys
+import tempfile
+import textwrap
+from dataclasses import dataclass
+
+from pip._internal.cli.main import main as pip_main
+
+from tools.private.update_deps.args import path_from_runfiles
+from tools.private.update_deps.update_file import update_file
+
+
+@dataclass
+class Dep:
+ name: str
+ url: str
+ sha256: str
+
+
+def _dep_snippet(deps: list[Dep]) -> str:
+ lines = []
+ for dep in deps:
+ lines.extend(
+ [
+ "(\n",
+ f' "{dep.name}",\n',
+ f' "{dep.url}",\n',
+ f' "{dep.sha256}",\n',
+ "),\n",
+ ]
+ )
+
+ return textwrap.indent("".join(lines), " " * 4)
+
+
+def _module_snippet(deps: list[Dep]) -> str:
+ lines = []
+ for dep in deps:
+ lines.append(f'"{dep.name}",\n')
+
+ return textwrap.indent("".join(lines), " " * 4)
+
+
+def _generate_report(requirements_txt: pathlib.Path) -> dict:
+ with tempfile.NamedTemporaryFile() as tmp:
+ tmp_path = pathlib.Path(tmp.name)
+ sys.argv = [
+ "pip",
+ "install",
+ "--dry-run",
+ "--ignore-installed",
+ "--report",
+ f"{tmp_path}",
+ "-r",
+ f"{requirements_txt}",
+ ]
+ pip_main()
+ with open(tmp_path) as f:
+ return json.load(f)
+
+
+def _get_deps(report: dict) -> list[Dep]:
+ deps = []
+ for dep in report["install"]:
+ try:
+ dep = Dep(
+ name="pypi__"
+ + re.sub(
+ "[._-]+",
+ "_",
+ dep["metadata"]["name"],
+ ),
+ url=dep["download_info"]["url"],
+ sha256=dep["download_info"]["archive_info"]["hash"][len("sha256=") :],
+ )
+ except:
+ debug_dep = textwrap.indent(json.dumps(dep, indent=4), " " * 4)
+ print(f"Could not parse the response from 'pip':\n{debug_dep}")
+ raise
+
+ deps.append(dep)
+
+ return sorted(deps, key=lambda dep: dep.name)
+
+
+def main():
+ parser = argparse.ArgumentParser(__doc__)
+ parser.add_argument(
+ "--start",
+ type=str,
+ default="# START: maintained by 'bazel run //tools/private:update_pip_deps'",
+ help="The text to match in a file when updating them.",
+ )
+ parser.add_argument(
+ "--end",
+ type=str,
+ default="# END: maintained by 'bazel run //tools/private:update_pip_deps'",
+ help="The text to match in a file when updating them.",
+ )
+ parser.add_argument(
+ "--dry-run",
+ action="store_true",
+ help="Wether to write to files",
+ )
+ parser.add_argument(
+ "--requirements-txt",
+ type=path_from_runfiles,
+ default=os.environ.get("REQUIREMENTS_TXT"),
+ help="The requirements.txt path for the pip_install tools, defaults to the value taken from REQUIREMENTS_TXT",
+ )
+ parser.add_argument(
+ "--module-bazel",
+ type=path_from_runfiles,
+ default=os.environ.get("MODULE_BAZEL"),
+ help="The path for the file to be updated, defaults to the value taken from MODULE_BAZEL",
+ )
+ parser.add_argument(
+ "--repositories-bzl",
+ type=path_from_runfiles,
+ default=os.environ.get("REPOSITORIES_BZL"),
+ help="The path for the file to be updated, defaults to the value taken from REPOSITORIES_BZL",
+ )
+ args = parser.parse_args()
+
+ report = _generate_report(args.requirements_txt)
+ deps = _get_deps(report)
+
+ update_file(
+ path=args.repositories_bzl,
+ snippet=_dep_snippet(deps),
+ start_marker=args.start,
+ end_marker=args.end,
+ dry_run=args.dry_run,
+ )
+
+ update_file(
+ path=args.module_bazel,
+ snippet=_module_snippet(deps),
+ start_marker=args.start,
+ end_marker=args.end,
+ dry_run=args.dry_run,
+ )
+
+
+if __name__ == "__main__":
+ main()
