feat(coverage): Register coverage.py to hermetic toolchains (#977)
This allows including the coverage package as part of the toolchain dependencies, which is mixed into a test's dependencies when `bazel coverage` is run (if coverage is not enabled, no extra dependency is added)
For now, it's disabled by default because enabling it poses the risk of having two versions of coverage installed (one from the toolchain, one from the user's dependencies).
The user can turn the coverage_tool setting by passing
`register_coverage_tool=(True|False)` to `python_register_toolchains` or
`python_register_multi_toolchains` call or specifying the
`coverage_tool` label as described in the `versions.bzl` file.
Use coverage.py v6.5.0 because the latest has `types.py` in the package
directory, which imports from Python's stdlib `types` [1]. Somehow the
Python interpreter is thinking that the `from types import FrameType` is
referring to the currently interpreted file and everything breaks. I
would have expected the package to use absolute imports and only attempt
to import from `coverage.types` if we use `coverage.types` and not just
a plain `types` import.
NOTE: Coverage is only for non-windows platforms.
Update tests to:
- ensure that we can still use the toolchain as previously.
- ensure that we are not downloading extra deps if they are not needed.
* Also changes the projects bazelrc to use a remotejdk, which makes it easier for contributors because they don't have to locally install a jdk to get going.
[1]: https://github.com/nedbat/coveragepy/blob/master/coverage/types.py
[3]: https://github.com/bazelbuild/bazel/issues/15835
diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml
index 95fdea6..a400360 100644
--- a/.bazelci/presubmit.yml
+++ b/.bazelci/presubmit.yml
@@ -34,6 +34,20 @@
.reusable_build_test_all: &reusable_build_test_all
build_targets: ["..."]
test_targets: ["..."]
+.coverage_targets_example_bzlmod: &coverage_targets_example_bzlmod
+ coverage_targets: ["//:test"]
+.coverage_targets_example_multi_python: &coverage_targets_example_multi_python
+ coverage_targets:
+ - //tests:my_lib_3_10_test
+ - //tests:my_lib_3_11_test
+ - //tests:my_lib_3_8_test
+ - //tests:my_lib_3_9_test
+ - //tests:my_lib_default_test
+ - //tests:version_3_10_test
+ - //tests:version_3_11_test
+ - //tests:version_3_8_test
+ - //tests:version_3_9_test
+ - //tests:version_default_test
tasks:
gazelle_extension:
name: Test the Gazelle extension
@@ -89,42 +103,50 @@
integration_test_bzlmod_ubuntu:
<<: *reusable_build_test_all
+ <<: *coverage_targets_example_bzlmod
name: bzlmod integration tests on Ubuntu
working_directory: examples/bzlmod
platform: ubuntu2004
integration_test_bzlmod_debian:
<<: *reusable_build_test_all
+ <<: *coverage_targets_example_bzlmod
name: bzlmod integration tests on Debian
working_directory: examples/bzlmod
platform: debian11
integration_test_bzlmod_macos:
<<: *reusable_build_test_all
+ <<: *coverage_targets_example_bzlmod
name: bzlmod integration tests on macOS
working_directory: examples/bzlmod
platform: macos
integration_test_bzlmod_windows:
<<: *reusable_build_test_all
+ # coverage is not supported on Windows
name: bzlmod integration tests on Windows
working_directory: examples/bzlmod
platform: windows
integration_test_multi_python_versions_ubuntu:
<<: *reusable_build_test_all
+ <<: *coverage_targets_example_multi_python
name: multi_python_versions integration tests on Ubuntu
working_directory: examples/multi_python_versions
platform: ubuntu2004
integration_test_multi_python_versions_debian:
<<: *reusable_build_test_all
+ <<: *coverage_targets_example_multi_python
name: multi_python_versions integration tests on Debian
working_directory: examples/multi_python_versions
platform: debian11
integration_test_multi_python_versions_macos:
<<: *reusable_build_test_all
+ <<: *coverage_targets_example_multi_python
name: multi_python_versions integration tests on macOS
working_directory: examples/multi_python_versions
platform: macos
integration_test_multi_python_versions_windows:
<<: *reusable_build_test_all
+ # coverage is not supported on Windows
name: multi_python_versions integration tests on Windows
working_directory: examples/multi_python_versions
platform: windows
diff --git a/MODULE.bazel b/MODULE.bazel
index af5d2e0..92da402 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -29,4 +29,20 @@
"pypi__tomli",
"pypi__wheel",
"pypi__zipp",
+ # coverage_deps managed by running ./tools/update_coverage_deps.py <version>
+ "pypi__coverage_cp310_aarch64-apple-darwin",
+ "pypi__coverage_cp310_aarch64-unknown-linux-gnu",
+ "pypi__coverage_cp310_x86_64-apple-darwin",
+ "pypi__coverage_cp310_x86_64-unknown-linux-gnu",
+ "pypi__coverage_cp311_aarch64-unknown-linux-gnu",
+ "pypi__coverage_cp311_x86_64-apple-darwin",
+ "pypi__coverage_cp311_x86_64-unknown-linux-gnu",
+ "pypi__coverage_cp38_aarch64-apple-darwin",
+ "pypi__coverage_cp38_aarch64-unknown-linux-gnu",
+ "pypi__coverage_cp38_x86_64-apple-darwin",
+ "pypi__coverage_cp38_x86_64-unknown-linux-gnu",
+ "pypi__coverage_cp39_aarch64-apple-darwin",
+ "pypi__coverage_cp39_aarch64-unknown-linux-gnu",
+ "pypi__coverage_cp39_x86_64-apple-darwin",
+ "pypi__coverage_cp39_x86_64-unknown-linux-gnu",
)
diff --git a/examples/bzlmod/.bazelrc b/examples/bzlmod/.bazelrc
index b3a24e8..b8c233f 100644
--- a/examples/bzlmod/.bazelrc
+++ b/examples/bzlmod/.bazelrc
@@ -1 +1,3 @@
common --experimental_enable_bzlmod
+
+coverage --java_runtime_version=remotejdk_11
diff --git a/examples/bzlmod/BUILD.bazel b/examples/bzlmod/BUILD.bazel
index 2094567..7b7566b 100644
--- a/examples/bzlmod/BUILD.bazel
+++ b/examples/bzlmod/BUILD.bazel
@@ -12,7 +12,7 @@
py_library(
name = "lib",
- srcs = ["__init__.py"],
+ srcs = ["lib.py"],
deps = [
requirement("pylint"),
requirement("tabulate"),
diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel
index 48fb4cb..5f984c3 100644
--- a/examples/bzlmod/MODULE.bazel
+++ b/examples/bzlmod/MODULE.bazel
@@ -13,6 +13,7 @@
python = use_extension("@rules_python//python:extensions.bzl", "python")
python.toolchain(
name = "python3_9",
+ configure_coverage_tool = True,
python_version = "3.9",
)
use_repo(python, "python3_9_toolchains")
diff --git a/examples/bzlmod/__main__.py b/examples/bzlmod/__main__.py
index b173bd6..099493b 100644
--- a/examples/bzlmod/__main__.py
+++ b/examples/bzlmod/__main__.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __init__ import main
+from lib import main
if __name__ == "__main__":
print(main([["A", 1], ["B", 2]]))
diff --git a/examples/bzlmod/__init__.py b/examples/bzlmod/lib.py
similarity index 100%
rename from examples/bzlmod/__init__.py
rename to examples/bzlmod/lib.py
diff --git a/examples/bzlmod/test.py b/examples/bzlmod/test.py
index b6038a7..cdc1c89 100644
--- a/examples/bzlmod/test.py
+++ b/examples/bzlmod/test.py
@@ -14,7 +14,7 @@
import unittest
-from __init__ import main
+from lib import main
class ExampleTest(unittest.TestCase):
diff --git a/examples/multi_python_versions/.bazelrc b/examples/multi_python_versions/.bazelrc
index f23315a..3fd6365 100644
--- a/examples/multi_python_versions/.bazelrc
+++ b/examples/multi_python_versions/.bazelrc
@@ -3,3 +3,5 @@
# Windows requires these for multi-python support:
build --enable_runfiles
startup --windows_enable_symlinks
+
+coverage --java_runtime_version=remotejdk_11
diff --git a/examples/multi_python_versions/WORKSPACE b/examples/multi_python_versions/WORKSPACE
index 41c8880..35855ca 100644
--- a/examples/multi_python_versions/WORKSPACE
+++ b/examples/multi_python_versions/WORKSPACE
@@ -22,11 +22,14 @@
"3.8",
"3.9",
"3.10",
+ "3.11",
],
+ register_coverage_tool = True,
)
load("@python//:pip.bzl", "multi_pip_parse")
load("@python//3.10:defs.bzl", interpreter_3_10 = "interpreter")
+load("@python//3.11:defs.bzl", interpreter_3_11 = "interpreter")
load("@python//3.8:defs.bzl", interpreter_3_8 = "interpreter")
load("@python//3.9:defs.bzl", interpreter_3_9 = "interpreter")
@@ -35,11 +38,13 @@
default_version = default_python_version,
python_interpreter_target = {
"3.10": interpreter_3_10,
+ "3.11": interpreter_3_11,
"3.8": interpreter_3_8,
"3.9": interpreter_3_9,
},
requirements_lock = {
"3.10": "//requirements:requirements_lock_3_10.txt",
+ "3.11": "//requirements:requirements_lock_3_11.txt",
"3.8": "//requirements:requirements_lock_3_8.txt",
"3.9": "//requirements:requirements_lock_3_9.txt",
},
diff --git a/examples/multi_python_versions/requirements/BUILD.bazel b/examples/multi_python_versions/requirements/BUILD.bazel
index 4848fab..e3184c8 100644
--- a/examples/multi_python_versions/requirements/BUILD.bazel
+++ b/examples/multi_python_versions/requirements/BUILD.bazel
@@ -1,4 +1,5 @@
load("@python//3.10:defs.bzl", compile_pip_requirements_3_10 = "compile_pip_requirements")
+load("@python//3.11:defs.bzl", compile_pip_requirements_3_11 = "compile_pip_requirements")
load("@python//3.8:defs.bzl", compile_pip_requirements_3_8 = "compile_pip_requirements")
load("@python//3.9:defs.bzl", compile_pip_requirements_3_9 = "compile_pip_requirements")
@@ -22,3 +23,10 @@
requirements_in = "requirements.in",
requirements_txt = "requirements_lock_3_10.txt",
)
+
+compile_pip_requirements_3_11(
+ name = "requirements_3_11",
+ extra_args = ["--allow-unsafe"],
+ requirements_in = "requirements.in",
+ requirements_txt = "requirements_lock_3_11.txt",
+)
diff --git a/examples/multi_python_versions/requirements/requirements_lock_3_11.txt b/examples/multi_python_versions/requirements/requirements_lock_3_11.txt
new file mode 100644
index 0000000..a437a39
--- /dev/null
+++ b/examples/multi_python_versions/requirements/requirements_lock_3_11.txt
@@ -0,0 +1,56 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+# bazel run //requirements:requirements_3_11.update
+#
+websockets==10.3 \
+ --hash=sha256:07cdc0a5b2549bcfbadb585ad8471ebdc7bdf91e32e34ae3889001c1c106a6af \
+ --hash=sha256:210aad7fdd381c52e58777560860c7e6110b6174488ef1d4b681c08b68bf7f8c \
+ --hash=sha256:28dd20b938a57c3124028680dc1600c197294da5db4292c76a0b48efb3ed7f76 \
+ --hash=sha256:2f94fa3ae454a63ea3a19f73b95deeebc9f02ba2d5617ca16f0bbdae375cda47 \
+ --hash=sha256:31564a67c3e4005f27815634343df688b25705cccb22bc1db621c781ddc64c69 \
+ --hash=sha256:347974105bbd4ea068106ec65e8e8ebd86f28c19e529d115d89bd8cc5cda3079 \
+ --hash=sha256:379e03422178436af4f3abe0aa8f401aa77ae2487843738542a75faf44a31f0c \
+ --hash=sha256:3eda1cb7e9da1b22588cefff09f0951771d6ee9fa8dbe66f5ae04cc5f26b2b55 \
+ --hash=sha256:51695d3b199cd03098ae5b42833006a0f43dc5418d3102972addc593a783bc02 \
+ --hash=sha256:54c000abeaff6d8771a4e2cef40900919908ea7b6b6a30eae72752607c6db559 \
+ --hash=sha256:5b936bf552e4f6357f5727579072ff1e1324717902127ffe60c92d29b67b7be3 \
+ --hash=sha256:6075fd24df23133c1b078e08a9b04a3bc40b31a8def4ee0b9f2c8865acce913e \
+ --hash=sha256:661f641b44ed315556a2fa630239adfd77bd1b11cb0b9d96ed8ad90b0b1e4978 \
+ --hash=sha256:6ea6b300a6bdd782e49922d690e11c3669828fe36fc2471408c58b93b5535a98 \
+ --hash=sha256:6ed1d6f791eabfd9808afea1e068f5e59418e55721db8b7f3bfc39dc831c42ae \
+ --hash=sha256:7934e055fd5cd9dee60f11d16c8d79c4567315824bacb1246d0208a47eca9755 \
+ --hash=sha256:7ab36e17af592eec5747c68ef2722a74c1a4a70f3772bc661079baf4ae30e40d \
+ --hash=sha256:7f6d96fdb0975044fdd7953b35d003b03f9e2bcf85f2d2cf86285ece53e9f991 \
+ --hash=sha256:83e5ca0d5b743cde3d29fda74ccab37bdd0911f25bd4cdf09ff8b51b7b4f2fa1 \
+ --hash=sha256:85506b3328a9e083cc0a0fb3ba27e33c8db78341b3eb12eb72e8afd166c36680 \
+ --hash=sha256:8af75085b4bc0b5c40c4a3c0e113fa95e84c60f4ed6786cbb675aeb1ee128247 \
+ --hash=sha256:8b1359aba0ff810d5830d5ab8e2c4a02bebf98a60aa0124fb29aa78cfdb8031f \
+ --hash=sha256:8fbd7d77f8aba46d43245e86dd91a8970eac4fb74c473f8e30e9c07581f852b2 \
+ --hash=sha256:907e8247480f287aa9bbc9391bd6de23c906d48af54c8c421df84655eef66af7 \
+ --hash=sha256:93d5ea0b5da8d66d868b32c614d2b52d14304444e39e13a59566d4acb8d6e2e4 \
+ --hash=sha256:97bc9d41e69a7521a358f9b8e44871f6cdeb42af31815c17aed36372d4eec667 \
+ --hash=sha256:994cdb1942a7a4c2e10098d9162948c9e7b235df755de91ca33f6e0481366fdb \
+ --hash=sha256:a141de3d5a92188234afa61653ed0bbd2dde46ad47b15c3042ffb89548e77094 \
+ --hash=sha256:a1e15b230c3613e8ea82c9fc6941b2093e8eb939dd794c02754d33980ba81e36 \
+ --hash=sha256:aad5e300ab32036eb3fdc350ad30877210e2f51bceaca83fb7fef4d2b6c72b79 \
+ --hash=sha256:b529fdfa881b69fe563dbd98acce84f3e5a67df13de415e143ef053ff006d500 \
+ --hash=sha256:b9c77f0d1436ea4b4dc089ed8335fa141e6a251a92f75f675056dac4ab47a71e \
+ --hash=sha256:bb621ec2dbbbe8df78a27dbd9dd7919f9b7d32a73fafcb4d9252fc4637343582 \
+ --hash=sha256:c7250848ce69559756ad0086a37b82c986cd33c2d344ab87fea596c5ac6d9442 \
+ --hash=sha256:c8d1d14aa0f600b5be363077b621b1b4d1eb3fbf90af83f9281cda668e6ff7fd \
+ --hash=sha256:d1655a6fc7aecd333b079d00fb3c8132d18988e47f19740c69303bf02e9883c6 \
+ --hash=sha256:d6353ba89cfc657a3f5beabb3b69be226adbb5c6c7a66398e17809b0ce3c4731 \
+ --hash=sha256:da4377904a3379f0c1b75a965fff23b28315bcd516d27f99a803720dfebd94d4 \
+ --hash=sha256:e49ea4c1a9543d2bd8a747ff24411509c29e4bdcde05b5b0895e2120cb1a761d \
+ --hash=sha256:e4e08305bfd76ba8edab08dcc6496f40674f44eb9d5e23153efa0a35750337e8 \
+ --hash=sha256:e6fa05a680e35d0fcc1470cb070b10e6fe247af54768f488ed93542e71339d6f \
+ --hash=sha256:e7e6f2d6fd48422071cc8a6f8542016f350b79cc782752de531577d35e9bd677 \
+ --hash=sha256:e904c0381c014b914136c492c8fa711ca4cced4e9b3d110e5e7d436d0fc289e8 \
+ --hash=sha256:ec2b0ab7edc8cd4b0eb428b38ed89079bdc20c6bdb5f889d353011038caac2f9 \
+ --hash=sha256:ef5ce841e102278c1c2e98f043db99d6755b1c58bde475516aef3a008ed7f28e \
+ --hash=sha256:f351c7d7d92f67c0609329ab2735eee0426a03022771b00102816a72715bb00b \
+ --hash=sha256:fab7c640815812ed5f10fbee7abbf58788d602046b7bb3af9b1ac753a6d5e916 \
+ --hash=sha256:fc06cc8073c8e87072138ba1e431300e2d408f054b27047d047b549455066ff4
+ # via -r requirements/requirements.in
diff --git a/examples/multi_python_versions/tests/BUILD.bazel b/examples/multi_python_versions/tests/BUILD.bazel
index 7219ca5..2292d53 100644
--- a/examples/multi_python_versions/tests/BUILD.bazel
+++ b/examples/multi_python_versions/tests/BUILD.bazel
@@ -1,4 +1,5 @@
load("@python//3.10:defs.bzl", py_binary_3_10 = "py_binary", py_test_3_10 = "py_test")
+load("@python//3.11:defs.bzl", py_binary_3_11 = "py_binary", py_test_3_11 = "py_test")
load("@python//3.8:defs.bzl", py_binary_3_8 = "py_binary", py_test_3_8 = "py_test")
load("@python//3.9:defs.bzl", py_binary_3_9 = "py_binary", py_test_3_9 = "py_test")
load("@rules_python//python:defs.bzl", "py_binary", "py_test")
@@ -27,6 +28,12 @@
main = "version.py",
)
+py_binary_3_11(
+ name = "version_3_11",
+ srcs = ["version.py"],
+ main = "version.py",
+)
+
py_test(
name = "my_lib_default_test",
srcs = ["my_lib_test.py"],
@@ -55,6 +62,13 @@
deps = ["//libs/my_lib"],
)
+py_test_3_11(
+ name = "my_lib_3_11_test",
+ srcs = ["my_lib_test.py"],
+ main = "my_lib_test.py",
+ deps = ["//libs/my_lib"],
+)
+
py_test(
name = "version_default_test",
srcs = ["version_test.py"],
@@ -83,6 +97,13 @@
main = "version_test.py",
)
+py_test_3_11(
+ name = "version_3_11_test",
+ srcs = ["version_test.py"],
+ env = {"VERSION_CHECK": "3.11"},
+ main = "version_test.py",
+)
+
py_test(
name = "version_default_takes_3_10_subprocess_test",
srcs = ["cross_version_test.py"],
diff --git a/python/extensions.bzl b/python/extensions.bzl
index 0c9ad07..bc0d570 100644
--- a/python/extensions.bzl
+++ b/python/extensions.bzl
@@ -19,6 +19,7 @@
load("@rules_python//python/pip_install:pip_repository.bzl", "locked_requirements_label", "pip_repository_attrs", "use_isolated", "whl_library")
load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies")
load("@rules_python//python/pip_install:requirements_parser.bzl", parse_requirements = "parse")
+load("@rules_python//python/private:coverage_deps.bzl", "install_coverage_deps")
def _python_impl(module_ctx):
for mod in module_ctx.modules:
@@ -26,20 +27,32 @@
python_register_toolchains(
name = attr.name,
python_version = attr.python_version,
+ bzlmod = True,
# Toolchain registration in bzlmod is done in MODULE file
register_toolchains = False,
+ register_coverage_tool = attr.configure_coverage_tool,
)
python = module_extension(
implementation = _python_impl,
tag_classes = {
- "toolchain": tag_class(attrs = dict({"name": attr.string(mandatory = True), "python_version": attr.string(mandatory = True)})),
+ "toolchain": tag_class(
+ attrs = {
+ "configure_coverage_tool": attr.bool(
+ mandatory = False,
+ doc = "Whether or not to configure the default coverage tool for the toolchains.",
+ ),
+ "name": attr.string(mandatory = True),
+ "python_version": attr.string(mandatory = True),
+ },
+ ),
},
)
# buildifier: disable=unused-variable
def _internal_deps_impl(module_ctx):
pip_install_dependencies()
+ install_coverage_deps()
internal_deps = module_extension(
implementation = _internal_deps_impl,
diff --git a/python/private/coverage_deps.bzl b/python/private/coverage_deps.bzl
new file mode 100644
index 0000000..d6092e6
--- /dev/null
+++ b/python/private/coverage_deps.bzl
@@ -0,0 +1,179 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Dependencies for coverage.py used by the hermetic toolchain.
+"""
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
+load(
+ "//python:versions.bzl",
+ "MINOR_MAPPING",
+ "PLATFORMS",
+)
+
+# Update with './tools/update_coverage_deps.py <version>'
+#START: managed by update_coverage_deps.py script
+_coverage_deps = {
+ "cp310": {
+ "aarch64-apple-darwin": (
+ "https://files.pythonhosted.org/packages/89/a2/cbf599e50bb4be416e0408c4cf523c354c51d7da39935461a9687e039481/coverage-6.5.0-cp310-cp310-macosx_11_0_arm64.whl",
+ "784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660",
+ ),
+ "aarch64-unknown-linux-gnu": (
+ "https://files.pythonhosted.org/packages/15/b0/3639d84ee8a900da0cf6450ab46e22517e4688b6cec0ba8ab6f8166103a2/coverage-6.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
+ "b4a5be1748d538a710f87542f22c2cad22f80545a847ad91ce45e77417293eb4",
+ ),
+ "x86_64-apple-darwin": (
+ "https://files.pythonhosted.org/packages/c4/8d/5ec7d08f4601d2d792563fe31db5e9322c306848fec1e65ec8885927f739/coverage-6.5.0-cp310-cp310-macosx_10_9_x86_64.whl",
+ "ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53",
+ ),
+ "x86_64-unknown-linux-gnu": (
+ "https://files.pythonhosted.org/packages/3c/7d/d5211ea782b193ab8064b06dc0cc042cf1a4ca9c93a530071459172c550f/coverage-6.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+ "af4fffaffc4067232253715065e30c5a7ec6faac36f8fc8d6f64263b15f74db0",
+ ),
+ },
+ "cp311": {
+ "aarch64-unknown-linux-gnu": (
+ "https://files.pythonhosted.org/packages/36/f3/5cbd79cf4cd059c80b59104aca33b8d05af4ad5bf5b1547645ecee716378/coverage-6.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
+ "c4ed2820d919351f4167e52425e096af41bfabacb1857186c1ea32ff9983ed75",
+ ),
+ "x86_64-apple-darwin": (
+ "https://files.pythonhosted.org/packages/50/cf/455930004231fa87efe8be06d13512f34e070ddfee8b8bf5a050cdc47ab3/coverage-6.5.0-cp311-cp311-macosx_10_9_x86_64.whl",
+ "4a5375e28c5191ac38cca59b38edd33ef4cc914732c916f2929029b4bfb50795",
+ ),
+ "x86_64-unknown-linux-gnu": (
+ "https://files.pythonhosted.org/packages/6a/63/8e82513b7e4a1b8d887b4e85c1c2b6c9b754a581b187c0b084f3330ac479/coverage-6.5.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+ "a8fb6cf131ac4070c9c5a3e21de0f7dc5a0fbe8bc77c9456ced896c12fcdad91",
+ ),
+ },
+ "cp38": {
+ "aarch64-apple-darwin": (
+ "https://files.pythonhosted.org/packages/07/82/79fa21ceca9a9b091eb3c67e27eb648dade27b2c9e1eb23af47232a2a365/coverage-6.5.0-cp38-cp38-macosx_11_0_arm64.whl",
+ "2198ea6fc548de52adc826f62cb18554caedfb1d26548c1b7c88d8f7faa8f6ba",
+ ),
+ "aarch64-unknown-linux-gnu": (
+ "https://files.pythonhosted.org/packages/40/3b/cd68cb278c4966df00158811ec1e357b9a7d132790c240fc65da57e10013/coverage-6.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
+ "6c4459b3de97b75e3bd6b7d4b7f0db13f17f504f3d13e2a7c623786289dd670e",
+ ),
+ "x86_64-apple-darwin": (
+ "https://files.pythonhosted.org/packages/05/63/a789b462075395d34f8152229dccf92b25ca73eac05b3f6cd75fa5017095/coverage-6.5.0-cp38-cp38-macosx_10_9_x86_64.whl",
+ "d900bb429fdfd7f511f868cedd03a6bbb142f3f9118c09b99ef8dc9bf9643c3c",
+ ),
+ "x86_64-unknown-linux-gnu": (
+ "https://files.pythonhosted.org/packages/bd/a0/e263b115808226fdb2658f1887808c06ac3f1b579ef5dda02309e0d54459/coverage-6.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+ "6b07130585d54fe8dff3d97b93b0e20290de974dc8177c320aeaf23459219c0b",
+ ),
+ },
+ "cp39": {
+ "aarch64-apple-darwin": (
+ "https://files.pythonhosted.org/packages/63/e9/f23e8664ec4032d7802a1cf920853196bcbdce7b56408e3efe1b2da08f3c/coverage-6.5.0-cp39-cp39-macosx_11_0_arm64.whl",
+ "95203854f974e07af96358c0b261f1048d8e1083f2de9b1c565e1be4a3a48cfc",
+ ),
+ "aarch64-unknown-linux-gnu": (
+ "https://files.pythonhosted.org/packages/18/95/27f80dcd8273171b781a19d109aeaed7f13d78ef6d1e2f7134a5826fd1b4/coverage-6.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
+ "b9023e237f4c02ff739581ef35969c3739445fb059b060ca51771e69101efffe",
+ ),
+ "x86_64-apple-darwin": (
+ "https://files.pythonhosted.org/packages/ea/52/c08080405329326a7ff16c0dfdb4feefaa8edd7446413df67386fe1bbfe0/coverage-6.5.0-cp39-cp39-macosx_10_9_x86_64.whl",
+ "633713d70ad6bfc49b34ead4060531658dc6dfc9b3eb7d8a716d5873377ab745",
+ ),
+ "x86_64-unknown-linux-gnu": (
+ "https://files.pythonhosted.org/packages/6b/f2/919f0fdc93d3991ca074894402074d847be8ac1e1d78e7e9e1c371b69a6f/coverage-6.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+ "8f830ed581b45b82451a40faabb89c84e1a998124ee4212d440e9c6cf70083e5",
+ ),
+ },
+}
+#END: managed by update_coverage_deps.py script
+
+def coverage_dep(name, python_version, platform, visibility, install = True):
+ """Register a singe coverage dependency based on the python version and platform.
+
+ Args:
+ name: The name of the registered repository.
+ python_version: The full python version.
+ platform: The platform, which can be found in //python:versions.bzl PLATFORMS dict.
+ visibility: The visibility of the coverage tool.
+ install: should we install the dependency with a given name or generate the label
+ of the bzlmod dependency fallback, which is hard-coded in MODULE.bazel?
+
+ Returns:
+ The label of the coverage tool if the platform is supported, otherwise - None.
+ """
+ if "windows" in platform:
+ # NOTE @aignas 2023-01-19: currently we do not support windows as the
+ # upstream coverage wrapper is written in shell. Do not log any warning
+ # for now as it is not actionable.
+ return None
+
+ python_short_version = python_version.rpartition(".")[0]
+ abi = python_short_version.replace("3.", "cp3")
+ url, sha256 = _coverage_deps.get(abi, {}).get(platform, (None, ""))
+
+ if url == None:
+ # Some wheels are not present for some builds, so let's silently ignore those.
+ return None
+
+ if not install:
+ # FIXME @aignas 2023-01-19: right now we use globally installed coverage
+ # which has visibility set to public, but is hidden due to repo remapping.
+ #
+ # The name of the toolchain is not known when registering the coverage tooling,
+ # so we use this as a workaround for now.
+ return Label("@pypi__coverage_{abi}_{platform}//:coverage".format(
+ abi = abi,
+ platform = platform,
+ ))
+
+ maybe(
+ http_archive,
+ name = name,
+ build_file_content = """
+filegroup(
+ name = "coverage",
+ srcs = ["coverage/__main__.py"],
+ data = glob(["coverage/*.py", "coverage/**/*.py", "coverage/*.so"]),
+ visibility = {visibility},
+)
+ """.format(
+ visibility = visibility,
+ ),
+ sha256 = sha256,
+ type = "zip",
+ urls = [url],
+ )
+
+ return Label("@@{name}//:coverage".format(name = name))
+
+def install_coverage_deps():
+ """Register the dependency for the coverage dep.
+
+ This is only used under bzlmod.
+ """
+
+ for python_version in MINOR_MAPPING.values():
+ for platform in PLATFORMS.keys():
+ if "windows" in platform:
+ continue
+
+ coverage_dep(
+ name = "pypi__coverage_cp{version_no_dot}_{platform}".format(
+ version_no_dot = python_version.rpartition(".")[0].replace(".", ""),
+ platform = platform,
+ ),
+ python_version = python_version,
+ platform = platform,
+ visibility = ["//visibility:public"],
+ install = True,
+ )
diff --git a/python/repositories.bzl b/python/repositories.bzl
index 7589640..faaec64 100644
--- a/python/repositories.bzl
+++ b/python/repositories.bzl
@@ -19,6 +19,7 @@
load("@bazel_tools//tools/build_defs/repo:http.bzl", _http_archive = "http_archive")
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
+load("//python/private:coverage_deps.bzl", "coverage_dep")
load(
"//python/private:toolchains_repo.bzl",
"multi_toolchain_aliases",
@@ -295,9 +296,21 @@
exports_files(["python", "{python_path}"])
+# Used to only download coverage toolchain when the coverage is collected by
+# bazel.
+config_setting(
+ name = "coverage_enabled",
+ values = {{"collect_code_coverage": "true"}},
+ visibility = ["//visibility:private"],
+)
+
py_runtime(
name = "py3_runtime",
files = [":files"],
+ coverage_tool = select({{
+ ":coverage_enabled": {coverage_tool},
+ "//conditions:default": None,
+ }}),
interpreter = "{python_path}",
python_version = "PY3",
)
@@ -312,6 +325,7 @@
python_path = python_bin,
python_version = python_short_version,
python_version_nodot = python_short_version.replace(".", ""),
+ coverage_tool = rctx.attr.coverage_tool if rctx.attr.coverage_tool == None or "windows" in rctx.os.name else "\"{}\"".format(rctx.attr.coverage_tool),
)
rctx.delete("python")
rctx.symlink(python_bin, "python")
@@ -319,6 +333,7 @@
rctx.file("BUILD.bazel", build_content)
return {
+ "coverage_tool": rctx.attr.coverage_tool,
"distutils": rctx.attr.distutils,
"distutils_content": rctx.attr.distutils_content,
"ignore_root_user_error": rctx.attr.ignore_root_user_error,
@@ -336,6 +351,28 @@
_python_repository_impl,
doc = "Fetches the external tools needed for the Python toolchain.",
attrs = {
+ "coverage_tool": attr.label(
+ # Mirrors the definition at
+ # https://github.com/bazelbuild/bazel/blob/master/src/main/starlark/builtins_bzl/common/python/py_runtime_rule.bzl
+ allow_files = False,
+ doc = """
+This is a target to use for collecting code coverage information from `py_binary`
+and `py_test` targets.
+
+If set, the target must either produce a single file or be an executable target.
+The path to the single file, or the executable if the target is executable,
+determines the entry point for the python coverage tool. The target and its
+runfiles will be added to the runfiles when coverage is enabled.
+
+The entry point for the tool must be loadable by a Python interpreter (e.g. a
+`.py` or `.pyc` file). It must accept the command line arguments
+of coverage.py (https://coverage.readthedocs.io), at least including
+the `run` and `lcov` subcommands.
+
+For more information see the official bazel docs
+(https://bazel.build/reference/be/python#py_runtime.coverage_tool).
+""",
+ ),
"distutils": attr.label(
allow_single_file = True,
doc = "A distutils.cfg file to be included in the Python installation. " +
@@ -399,8 +436,10 @@
distutils = None,
distutils_content = None,
register_toolchains = True,
+ register_coverage_tool = False,
set_python_version_constraint = False,
tool_versions = TOOL_VERSIONS,
+ bzlmod = False,
**kwargs):
"""Convenience macro for users which does typical setup.
@@ -417,9 +456,11 @@
distutils: see the distutils attribute in the python_repository repository rule.
distutils_content: see the distutils_content attribute in the python_repository repository rule.
register_toolchains: Whether or not to register the downloaded toolchains.
+ register_coverage_tool: Whether or not to register the downloaded coverage tool to the toolchains.
set_python_version_constraint: When set to true, target_compatible_with for the toolchains will include a version constraint.
tool_versions: a dict containing a mapping of version with SHASUM and platform info. If not supplied, the defaults
- in python/versions.bzl will be used
+ in python/versions.bzl will be used.
+ bzlmod: Whether this rule is being run under a bzlmod module extension.
**kwargs: passed to each python_repositories call.
"""
base_url = kwargs.pop("base_url", DEFAULT_RELEASE_BASE_URL)
@@ -436,6 +477,24 @@
(release_filename, url, strip_prefix, patches) = get_release_info(platform, python_version, base_url, tool_versions)
+ # allow passing in a tool version
+ coverage_tool = None
+ coverage_tool = tool_versions[python_version].get("coverage_tool", {}).get(platform, None)
+ if register_coverage_tool and coverage_tool == None:
+ coverage_tool = coverage_dep(
+ name = "{name}_{platform}_coverage".format(
+ name = name,
+ platform = platform,
+ ),
+ python_version = python_version,
+ platform = platform,
+ visibility = ["@@{name}_{platform}//:__subpackages__".format(
+ name = name,
+ platform = platform,
+ )],
+ install = not bzlmod,
+ )
+
python_repository(
name = "{name}_{platform}".format(
name = name,
@@ -450,6 +509,7 @@
distutils = distutils,
distutils_content = distutils_content,
strip_prefix = strip_prefix,
+ coverage_tool = coverage_tool,
**kwargs
)
if register_toolchains:
diff --git a/python/versions.bzl b/python/versions.bzl
index f843e47..4feeeae 100644
--- a/python/versions.bzl
+++ b/python/versions.bzl
@@ -26,6 +26,21 @@
# the hashes:
# bazel run //python/private:print_toolchains_checksums
#
+# Note, to users looking at how to specify their tool versions, coverage_tool version for each
+# interpreter can be specified by:
+# "3.8.10": {
+# "url": "20210506/cpython-{python_version}-{platform}-pgo+lto-20210506T0943.tar.zst",
+# "sha256": {
+# "x86_64-apple-darwin": "8d06bec08db8cdd0f64f4f05ee892cf2fcbc58cfb1dd69da2caab78fac420238",
+# "x86_64-unknown-linux-gnu": "aec8c4c53373b90be7e2131093caa26063be6d9d826f599c935c0e1042af3355",
+# },
+# "coverage_tool": {
+# "x86_64-apple-darwin": "<label_for_darwin>"",
+# "x86_64-unknown-linux-gnu": "<label_for_linux>"",
+# },
+# "strip_prefix": "python",
+# },
+#
# buildifier: disable=unsorted-dict-items
TOOL_VERSIONS = {
"3.8.10": {
diff --git a/tools/update_coverage_deps.py b/tools/update_coverage_deps.py
new file mode 100755
index 0000000..4cf1e94
--- /dev/null
+++ b/tools/update_coverage_deps.py
@@ -0,0 +1,258 @@
+#!/usr/bin/python3 -B
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A small script to update bazel files within the repo.
+
+We are not running this with 'bazel run' to keep the dependencies minimal
+"""
+
+# NOTE @aignas 2023-01-09: We should only depend on core Python 3 packages.
+import argparse
+import difflib
+import json
+import pathlib
+import sys
+import textwrap
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Any
+from urllib import request
+
+# This should be kept in sync with //python:versions.bzl
+_supported_platforms = {
+ # Windows is unsupported right now
+ # "win_amd64": "x86_64-pc-windows-msvc",
+ "manylinux2014_x86_64": "x86_64-unknown-linux-gnu",
+ "manylinux2014_aarch64": "aarch64-unknown-linux-gnu",
+ "macosx_11_0_arm64": "aarch64-apple-darwin",
+ "macosx_10_9_x86_64": "x86_64-apple-darwin",
+}
+
+
+@dataclass
+class Dep:
+ name: str
+ platform: str
+ python: str
+ url: str
+ sha256: str
+
+ @property
+ def repo_name(self):
+ return f"pypi__{self.name}_{self.python}_{self.platform}"
+
+ def __repr__(self):
+ return "\n".join(
+ [
+ "(",
+ f' "{self.url}",',
+ f' "{self.sha256}",',
+ ")",
+ ]
+ )
+
+
+@dataclass
+class Deps:
+ deps: list[Dep]
+
+ def __repr__(self):
+ deps = defaultdict(dict)
+ for d in self.deps:
+ deps[d.python][d.platform] = d
+
+ parts = []
+ for python, contents in deps.items():
+ inner = textwrap.indent(
+ "\n".join([f'"{platform}": {d},' for platform, d in contents.items()]),
+ prefix=" ",
+ )
+ parts.append('"{}": {{\n{}\n}},'.format(python, inner))
+ return "{{\n{}\n}}".format(textwrap.indent("\n".join(parts), prefix=" "))
+
+
+def _get_platforms(filename: str, name: str, version: str, python_version: str):
+ return filename[
+ len(f"{name}-{version}-{python_version}-{python_version}-") : -len(".whl")
+ ].split(".")
+
+
+def _map(
+ name: str,
+ filename: str,
+ python_version: str,
+ url: str,
+ digests: list,
+ platform: str,
+ **kwargs: Any,
+):
+ if platform not in _supported_platforms:
+ return None
+
+ return Dep(
+ name=name,
+ platform=_supported_platforms[platform],
+ python=python_version,
+ url=url,
+ sha256=digests["sha256"],
+ )
+
+
+def _writelines(path: pathlib.Path, lines: list[str]):
+ with open(path, "w") as f:
+ f.writelines(lines)
+
+
+def _difflines(path: pathlib.Path, lines: list[str]):
+ with open(path) as f:
+ input = f.readlines()
+
+ rules_python = pathlib.Path(__file__).parent.parent
+ p = path.relative_to(rules_python)
+
+ print(f"Diff of the changes that would be made to '{p}':")
+ for line in difflib.unified_diff(
+ input,
+ lines,
+ fromfile=f"a/{p}",
+ tofile=f"b/{p}",
+ ):
+ print(line, end="")
+
+ # Add an empty line at the end of the diff
+ print()
+
+
+def _update_file(
+ path: pathlib.Path,
+ snippet: str,
+ start_marker: str,
+ end_marker: str,
+ dry_run: bool = True,
+):
+ with open(path) as f:
+ input = f.readlines()
+
+ out = []
+ skip = False
+ for line in input:
+ if skip:
+ if not line.startswith(end_marker):
+ continue
+
+ skip = False
+
+ out.append(line)
+
+ if not line.startswith(start_marker):
+ continue
+
+ skip = True
+ out.extend([f"{line}\n" for line in snippet.splitlines()])
+
+ if dry_run:
+ _difflines(path, out)
+ else:
+ _writelines(path, out)
+
+
+def _parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(__doc__)
+ parser.add_argument(
+ "--name",
+ default="coverage",
+ type=str,
+ help="The name of the package",
+ )
+ parser.add_argument(
+ "version",
+ type=str,
+ help="The version of the package to download",
+ )
+ parser.add_argument(
+ "--py",
+ nargs="+",
+ type=str,
+ default=["cp38", "cp39", "cp310", "cp311"],
+ help="Supported python versions",
+ )
+ parser.add_argument(
+ "--dry-run",
+ action="store_true",
+ help="Wether to write to files",
+ )
+ return parser.parse_args()
+
+
+def main():
+ args = _parse_args()
+
+ api_url = f"https://pypi.python.org/pypi/{args.name}/{args.version}/json"
+ req = request.Request(api_url)
+ with request.urlopen(req) as response:
+ data = json.loads(response.read().decode("utf-8"))
+
+ urls = []
+ for u in data["urls"]:
+ if u["yanked"]:
+ continue
+
+ if not u["filename"].endswith(".whl"):
+ continue
+
+ if u["python_version"] not in args.py:
+ continue
+
+ if f'_{u["python_version"]}m_' in u["filename"]:
+ continue
+
+ platforms = _get_platforms(
+ u["filename"],
+ args.name,
+ args.version,
+ u["python_version"],
+ )
+
+ result = [_map(name=args.name, platform=p, **u) for p in platforms]
+ urls.extend(filter(None, result))
+
+ urls.sort(key=lambda x: f"{x.python}_{x.platform}")
+
+ rules_python = pathlib.Path(__file__).parent.parent
+
+ # Update the coverage_deps, which are used to register deps
+ _update_file(
+ path=rules_python / "python" / "private" / "coverage_deps.bzl",
+ snippet=f"_coverage_deps = {repr(Deps(urls))}\n",
+ start_marker="#START: managed by update_coverage_deps.py script",
+ end_marker="#END: managed by update_coverage_deps.py script",
+ dry_run=args.dry_run,
+ )
+
+ # Update the MODULE.bazel, which needs to expose the dependencies to the toolchain
+ # repositories
+ _update_file(
+ path=rules_python / "MODULE.bazel",
+ snippet="".join(sorted([f' "{u.repo_name}",\n' for u in urls])),
+ start_marker=" # coverage_deps managed by running",
+ end_marker=")",
+ dry_run=args.dry_run,
+ )
+
+ return
+
+
+if __name__ == "__main__":
+ main()
