Added support for annotating rendered pip dependencies (#589)
diff --git a/.bazelrc b/.bazelrc index e668a68..b900d7d 100644 --- a/.bazelrc +++ b/.bazelrc
@@ -3,8 +3,8 @@ # This lets us glob() up all the files inside the examples to make them inputs to tests # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh -build --deleted_packages=examples/build_file_generation,examples/pip_install,examples/pip_parse,examples/py_import,examples/relative_requirements -query --deleted_packages=examples/build_file_generation,examples/pip_install,examples/pip_parse,examples/py_import,examples/relative_requirements +build --deleted_packages=examples/build_file_generation,examples/pip_install,examples/pip_parse,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements +query --deleted_packages=examples/build_file_generation,examples/pip_install,examples/pip_parse,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements test --test_output=errors
diff --git a/BUILD b/BUILD index 6ba643c..cd19d36 100644 --- a/BUILD +++ b/BUILD
@@ -32,6 +32,8 @@ "//python:distribution", "//python/pip_install:distribution", "//third_party/github.com/bazelbuild/bazel-skylib/lib:distribution", + "//third_party/github.com/bazelbuild/bazel-skylib/rules:distribution", + "//third_party/github.com/bazelbuild/bazel-skylib/rules/private:distribution", "//tools:distribution", ], visibility = ["//examples:__pkg__"],
diff --git a/docs/pip.md b/docs/pip.md index ce865a6..0d657b4 100644 --- a/docs/pip.md +++ b/docs/pip.md
@@ -35,6 +35,33 @@ | kwargs | other bazel attributes passed to the "_test" rule | none | +<a name="#package_annotation"></a> + +## package_annotation + +<pre> +package_annotation(<a href="#package_annotation-additive_build_content">additive_build_content</a>, <a href="#package_annotation-copy_files">copy_files</a>, <a href="#package_annotation-copy_executables">copy_executables</a>, <a href="#package_annotation-data">data</a>, <a href="#package_annotation-data_exclude_glob">data_exclude_glob</a>, + <a href="#package_annotation-srcs_exclude_glob">srcs_exclude_glob</a>) +</pre> + +Annotations to apply to the BUILD file content from package generated from a `pip_repository` rule. + +[cf]: https://github.com/bazelbuild/bazel-skylib/blob/main/docs/copy_file_doc.md + + +**PARAMETERS** + + +| Name | Description | Default Value | +| :-------------: | :-------------: | :-------------: | +| additive_build_content | Raw text to add to the generated <code>BUILD</code> file of a package. | <code>None</code> | +| copy_files | A mapping of <code>src</code> and <code>out</code> files for [@bazel_skylib//rules:copy_file.bzl][cf] | <code>{}</code> | +| copy_executables | A mapping of <code>src</code> and <code>out</code> files for [@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as executable. | <code>{}</code> | +| data | A list of labels to add as <code>data</code> dependencies to the generated <code>py_library</code> target. | <code>[]</code> | +| data_exclude_glob | A list of exclude glob patterns to add as <code>data</code> to the generated <code>py_library</code> target. | <code>[]</code> | +| srcs_exclude_glob | A list of labels to add as <code>srcs</code> to the generated <code>py_library</code> target. | <code>[]</code> | + + <a name="#pip_install"></a> ## pip_install
diff --git a/examples/BUILD b/examples/BUILD index 8188ca7..44147e5 100644 --- a/examples/BUILD +++ b/examples/BUILD
@@ -28,6 +28,11 @@ ) bazel_integration_test( + name = "pip_repository_annotations_example", + timeout = "long", +) + +bazel_integration_test( name = "py_import_example", timeout = "long", )
diff --git a/examples/pip_repository_annotations/.bazelrc b/examples/pip_repository_annotations/.bazelrc new file mode 100644 index 0000000..9e7ef37 --- /dev/null +++ b/examples/pip_repository_annotations/.bazelrc
@@ -0,0 +1,2 @@ +# https://docs.bazel.build/versions/main/best-practices.html#using-the-bazelrc-file +try-import %workspace%/user.bazelrc
diff --git a/examples/pip_repository_annotations/BUILD b/examples/pip_repository_annotations/BUILD new file mode 100644 index 0000000..a5a0561 --- /dev/null +++ b/examples/pip_repository_annotations/BUILD
@@ -0,0 +1,30 @@ +load("@pip_installed//:requirements.bzl", "requirement") +load("@rules_python//python:defs.bzl", "py_test") +load("@rules_python//python:pip.bzl", "compile_pip_requirements") + +exports_files( + glob(["data/**"]), + visibility = ["//visibility:public"], +) + +# This rule adds a convenient way to update the requirements file. +compile_pip_requirements( + name = "requirements", + extra_args = ["--allow-unsafe"], +) + +py_test( + name = "pip_parse_annotations_test", + srcs = ["pip_repository_annotations_test.py"], + env = {"WHEEL_PKG_DIR": "pip_parsed_wheel"}, + main = "pip_repository_annotations_test.py", + deps = ["@pip_parsed_wheel//:pkg"], +) + +py_test( + name = "pip_install_annotations_test", + srcs = ["pip_repository_annotations_test.py"], + env = {"WHEEL_PKG_DIR": "pip_installed/pypi__wheel"}, + main = "pip_repository_annotations_test.py", + deps = [requirement("wheel")], +)
diff --git a/examples/pip_repository_annotations/WORKSPACE b/examples/pip_repository_annotations/WORKSPACE new file mode 100644 index 0000000..d09ed69 --- /dev/null +++ b/examples/pip_repository_annotations/WORKSPACE
@@ -0,0 +1,58 @@ +workspace(name = "pip_repository_annotations_example") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "rules_python", + sha256 = "cd6730ed53a002c56ce4e2f396ba3b3be262fd7cb68339f0377a45e8227fe332", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.5.0/rules_python-0.5.0.tar.gz", +) + +http_archive( + name = "bazel_skylib", + sha256 = "c6966ec828da198c5d9adbaa94c05e3a1c7f21bd012a0b29ba8ddbccb2c93b0d", + urls = [ + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.1.1/bazel-skylib-1.1.1.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.1.1/bazel-skylib-1.1.1.tar.gz", + ], +) + +load("@rules_python//python:pip.bzl", "package_annotation", "pip_install", "pip_parse") + +# Here we can see an example of annotations being applied to an arbitrary +# package. For details on `package_annotation` and it's uses, see the +# docs at @rules_python//docs:pip.md`. +ANNOTATIONS = { + "wheel": package_annotation( + additive_build_content = """\ +load("@bazel_skylib//rules:write_file.bzl", "write_file") +write_file( + name = "generated_file", + out = "generated_file.txt", + content = ["Hello world from build content file"], +) +""", + copy_executables = {"@pip_repository_annotations_example//:data/copy_executable.py": "copied_content/executable.py"}, + copy_files = {"@pip_repository_annotations_example//:data/copy_file.txt": "copied_content/file.txt"}, + data = [":generated_file"], + data_exclude_glob = ["*.dist-info/RECORD"], + ), +} + +# For a more thorough example of `pip_parse`. See `@rules_python//examples/pip_parse` +pip_parse( + name = "pip_parsed", + annotations = ANNOTATIONS, + requirements_lock = "//:requirements.txt", +) + +load("@pip_parsed//:requirements.bzl", "install_deps") + +install_deps() + +# For a more thorough example of `pip_install`. See `@rules_python//examples/pip_install` +pip_install( + name = "pip_installed", + annotations = ANNOTATIONS, + requirements = "//:requirements.txt", +)
diff --git a/examples/pip_repository_annotations/data/copy_executable.py b/examples/pip_repository_annotations/data/copy_executable.py new file mode 100755 index 0000000..20c6651 --- /dev/null +++ b/examples/pip_repository_annotations/data/copy_executable.py
@@ -0,0 +1,4 @@ +#!/usr/bin/env python + +if __name__ == "__main__": + print("Hello world from copied executable")
diff --git a/examples/pip_repository_annotations/data/copy_file.txt b/examples/pip_repository_annotations/data/copy_file.txt new file mode 100644 index 0000000..b1020f7 --- /dev/null +++ b/examples/pip_repository_annotations/data/copy_file.txt
@@ -0,0 +1 @@ +Hello world from copied file
diff --git a/examples/pip_repository_annotations/pip_repository_annotations_test.py b/examples/pip_repository_annotations/pip_repository_annotations_test.py new file mode 100644 index 0000000..a8f0863 --- /dev/null +++ b/examples/pip_repository_annotations/pip_repository_annotations_test.py
@@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +import os +import subprocess +import unittest +from glob import glob +from pathlib import Path + + +class PipRepositoryAnnotationsTest(unittest.TestCase): + maxDiff = None + + def wheel_pkg_dir(self) -> str: + env = os.environ.get("WHEEL_PKG_DIR") + self.assertIsNotNone(env) + return env + + def test_build_content_and_data(self): + generated_file = ( + Path.cwd() / "external" / self.wheel_pkg_dir() / "generated_file.txt" + ) + self.assertTrue(generated_file.exists()) + + content = generated_file.read_text().rstrip() + self.assertEqual(content, "Hello world from build content file") + + def test_copy_files(self): + copied_file = ( + Path.cwd() / "external" / self.wheel_pkg_dir() / "copied_content/file.txt" + ) + self.assertTrue(copied_file.exists()) + + content = copied_file.read_text().rstrip() + self.assertEqual(content, "Hello world from copied file") + + def test_copy_executables(self): + executable = ( + Path.cwd() + / "external" + / self.wheel_pkg_dir() + / "copied_content/executable.py" + ) + self.assertTrue(executable.exists()) + + proc = subprocess.run( + [executable], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + stdout = proc.stdout.decode("utf-8").strip() + self.assertEqual(stdout, "Hello world from copied executable") + + def test_data_exclude_glob(self): + files = glob("external/" + self.wheel_pkg_dir() + "/wheel-*.dist-info/*") + basenames = [Path(path).name for path in files] + self.assertIn("WHEEL", basenames) + self.assertNotIn("RECORD", basenames) + + +if __name__ == "__main__": + unittest.main()
diff --git a/examples/pip_repository_annotations/requirements.in b/examples/pip_repository_annotations/requirements.in new file mode 100644 index 0000000..2309722 --- /dev/null +++ b/examples/pip_repository_annotations/requirements.in
@@ -0,0 +1 @@ +wheel
diff --git a/examples/pip_repository_annotations/requirements.txt b/examples/pip_repository_annotations/requirements.txt new file mode 100644 index 0000000..51d1dfc --- /dev/null +++ b/examples/pip_repository_annotations/requirements.txt
@@ -0,0 +1,10 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# bazel run //:requirements.update +# +wheel==0.37.1 \ + --hash=sha256:4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a \ + --hash=sha256:e9a504e793efbca1b8e0e9cb979a249cf4a0a7b5b8c9e8b65a5e39d49529c1c4 + # via -r requirements.in
diff --git a/python/pip.bzl b/python/pip.bzl index 7776efe..0530a2c 100644 --- a/python/pip.bzl +++ b/python/pip.bzl
@@ -13,11 +13,12 @@ # limitations under the License. """Import pip requirements into Bazel.""" -load("//python/pip_install:pip_repository.bzl", "pip_repository") +load("//python/pip_install:pip_repository.bzl", "pip_repository", _package_annotation = "package_annotation") load("//python/pip_install:repositories.bzl", "pip_install_dependencies") load("//python/pip_install:requirements.bzl", _compile_pip_requirements = "compile_pip_requirements") compile_pip_requirements = _compile_pip_requirements +package_annotation = _package_annotation def pip_install(requirements, name = "pip", **kwargs): """Accepts a `requirements.txt` file and installs the dependencies listed within.
diff --git a/python/pip_install/extract_wheels/__init__.py b/python/pip_install/extract_wheels/__init__.py index f1b7254..cda6d54 100644 --- a/python/pip_install/extract_wheels/__init__.py +++ b/python/pip_install/extract_wheels/__init__.py
@@ -12,7 +12,13 @@ import subprocess import sys -from python.pip_install.extract_wheels.lib import arguments, bazel, requirements +from python.pip_install.extract_wheels.lib import ( + annotation, + arguments, + bazel, + requirements, + wheel, +) def configure_reproducible_wheels() -> None: @@ -58,6 +64,11 @@ required=True, help="Path to requirements.txt from where to install dependencies", ) + parser.add_argument( + "--annotations", + type=annotation.annotations_map_from_str_path, + help="A json encoded file containing annotations for rendered packages.", + ) arguments.parse_common_args(parser) args = parser.parse_args() deserialized_args = dict(vars(args)) @@ -89,18 +100,26 @@ repo_label = "@%s" % args.repo + # Locate all wheels + wheels = [whl for whl in glob.glob("*.whl")] + + # Collect all annotations + reqs = {whl: wheel.Wheel(whl).name for whl in wheels} + annotations = args.annotations.collect(reqs.values()) + targets = [ '"{}{}"'.format( repo_label, bazel.extract_wheel( - whl, - extras, - deserialized_args["pip_data_exclude"], - args.enable_implicit_namespace_pkgs, - args.repo_prefix, + wheel_file=whl, + extras=extras, + pip_data_exclude=deserialized_args["pip_data_exclude"], + enable_implicit_namespace_pkgs=args.enable_implicit_namespace_pkgs, + repo_prefix=args.repo_prefix, + annotation=annotations.get(name), ), ) - for whl in glob.glob("*.whl") + for whl, name in reqs.items() ] with open("requirements.bzl", "w") as requirement_file:
diff --git a/python/pip_install/extract_wheels/lib/BUILD b/python/pip_install/extract_wheels/lib/BUILD index 4758f15..49ed483 100644 --- a/python/pip_install/extract_wheels/lib/BUILD +++ b/python/pip_install/extract_wheels/lib/BUILD
@@ -1,9 +1,11 @@ load("@rules_python//python:defs.bzl", "py_library", "py_test") load("//python/pip_install:repositories.bzl", "requirement") +load(":annotations_test_helpers.bzl", "package_annotation", "package_annotations_file") py_library( name = "lib", srcs = [ + "annotation.py", "arguments.py", "bazel.py", "namespace_pkgs.py", @@ -21,6 +23,43 @@ ], ) +package_annotations_file( + name = "mock_annotations", + annotations = { + "pkg_a": package_annotation(), + "pkg_b": package_annotation( + data_exclude_glob = [ + "*.foo", + "*.bar", + ], + ), + "pkg_c": package_annotation( + additive_build_content = """\ +cc_library( + name = "my_target", + hdrs = glob(["**/*.h"]), + srcs = glob(["**/*.cc"]), +) +""", + data = [":my_target"], + ), + "pkg_d": package_annotation( + srcs_exclude_glob = ["pkg_d/tests/**"], + ), + }, + tags = ["manual"], +) + +py_test( + name = "annotations_test", + size = "small", + srcs = ["annotations_test.py"], + data = [":mock_annotations"], + env = {"MOCK_ANNOTATIONS": "$(rootpath :mock_annotations)"}, + tags = ["unit"], + deps = [":lib"], +) + py_test( name = "bazel_test", size = "small",
diff --git a/python/pip_install/extract_wheels/lib/annotation.py b/python/pip_install/extract_wheels/lib/annotation.py new file mode 100644 index 0000000..977668e --- /dev/null +++ b/python/pip_install/extract_wheels/lib/annotation.py
@@ -0,0 +1,112 @@ +import json +import logging +from collections import OrderedDict +from pathlib import Path +from typing import Any, Dict, List + + +class Annotation(OrderedDict): + """A python representation of `@rules_python//python:pip.bzl%package_annotation`""" + + def __init__(self, content: Dict[str, Any]) -> None: + + missing = [] + ordered_content = OrderedDict() + for field in ( + "additive_build_content", + "copy_executables", + "copy_files", + "data", + "data_exclude_glob", + "srcs_exclude_glob", + ): + if field not in content: + missing.append(field) + continue + ordered_content.update({field: content.pop(field)}) + + if missing: + raise ValueError("Data missing from initial annotation: {}".format(missing)) + + if content: + raise ValueError( + "Unexpected data passed to annotations: {}".format( + sorted(list(content.keys())) + ) + ) + + return OrderedDict.__init__(self, ordered_content) + + @property + def additive_build_content(self) -> str: + return self["additive_build_content"] + + @property + def copy_executables(self) -> Dict[str, str]: + return self["copy_executables"] + + @property + def copy_files(self) -> Dict[str, str]: + return self["copy_files"] + + @property + def data(self) -> List[str]: + return self["data"] + + @property + def data_exclude_glob(self) -> List[str]: + return self["data_exclude_glob"] + + @property + def srcs_exclude_glob(self) -> List[str]: + return self["srcs_exclude_glob"] + + +class AnnotationsMap: + """A mapping of python package names to [Annotation]""" + + def __init__(self, json_file: Path): + content = json.loads(json_file.read_text()) + + self._annotations = {pkg: Annotation(data) for (pkg, data) in content.items()} + + @property + def annotations(self) -> Dict[str, Annotation]: + return self._annotations + + def collect(self, requirements: List[str]) -> Dict[str, Annotation]: + unused = self.annotations + collection = {} + for pkg in requirements: + if pkg in unused: + collection.update({pkg: unused.pop(pkg)}) + + logging.warning("Unused annotations: {}".format(sorted(list(unused.keys())))) + + return collection + + +def annotation_from_str_path(path: str) -> Annotation: + """Load an annotation from a json encoded file + + Args: + path (str): The path to a json encoded file + + Returns: + Annotation: The deserialized annotations + """ + json_file = Path(path) + content = json.loads(json_file.read_text()) + return Annotation(content) + + +def annotations_map_from_str_path(path: str) -> AnnotationsMap: + """Load an annotations map from a json encoded file + + Args: + path (str): The path to a json encoded file + + Returns: + AnnotationsMap: The deserialized annotations map + """ + return AnnotationsMap(Path(path))
diff --git a/python/pip_install/extract_wheels/lib/annotations_test.py b/python/pip_install/extract_wheels/lib/annotations_test.py new file mode 100644 index 0000000..96573e1 --- /dev/null +++ b/python/pip_install/extract_wheels/lib/annotations_test.py
@@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +import os +import textwrap +import unittest +from pathlib import Path + +from python.pip_install.extract_wheels.lib.annotation import Annotation, AnnotationsMap + + +class AnnotationsTestCase(unittest.TestCase): + + maxDiff = None + + def test_annotations_constructor(self) -> None: + annotations_env = os.environ.get("MOCK_ANNOTATIONS") + self.assertIsNotNone(annotations_env) + + annotations_path = Path.cwd() / annotations_env + self.assertTrue(annotations_path.exists()) + + annotations_map = AnnotationsMap(annotations_path) + self.assertListEqual( + list(annotations_map.annotations.keys()), + ["pkg_a", "pkg_b", "pkg_c", "pkg_d"], + ) + + collection = annotations_map.collect(["pkg_a", "pkg_b", "pkg_c", "pkg_d"]) + + self.assertEqual( + collection["pkg_a"], + Annotation( + { + "additive_build_content": None, + "copy_executables": {}, + "copy_files": {}, + "data": [], + "data_exclude_glob": [], + "srcs_exclude_glob": [], + } + ), + ) + + self.assertEqual( + collection["pkg_b"], + Annotation( + { + "additive_build_content": None, + "copy_executables": {}, + "copy_files": {}, + "data": [], + "data_exclude_glob": ["*.foo", "*.bar"], + "srcs_exclude_glob": [], + } + ), + ) + + self.assertEqual( + collection["pkg_c"], + Annotation( + { + "additive_build_content": textwrap.dedent( + """\ + cc_library( + name = "my_target", + hdrs = glob(["**/*.h"]), + srcs = glob(["**/*.cc"]), + ) + """ + ), + "copy_executables": {}, + "copy_files": {}, + "data": [":my_target"], + "data_exclude_glob": [], + "srcs_exclude_glob": [], + } + ), + ) + + self.assertEqual( + collection["pkg_d"], + Annotation( + { + "additive_build_content": None, + "copy_executables": {}, + "copy_files": {}, + "data": [], + "data_exclude_glob": [], + "srcs_exclude_glob": ["pkg_d/tests/**"], + } + ), + ) + + +if __name__ == "__main__": + unittest.main()
diff --git a/python/pip_install/extract_wheels/lib/annotations_test_helpers.bzl b/python/pip_install/extract_wheels/lib/annotations_test_helpers.bzl new file mode 100644 index 0000000..dbd1124 --- /dev/null +++ b/python/pip_install/extract_wheels/lib/annotations_test_helpers.bzl
@@ -0,0 +1,33 @@ +"""Helper macros and rules for testing the `annotations` module of `extract_wheels`""" + +load("//python:pip.bzl", _package_annotation = "package_annotation") + +package_annotation = _package_annotation + +def _package_annotations_file_impl(ctx): + output = ctx.actions.declare_file(ctx.label.name + ".annotations.json") + + annotations = {package: json.decode(data) for (package, data) in ctx.attr.annotations.items()} + ctx.actions.write( + output = output, + content = json.encode_indent(annotations, indent = " " * 4), + ) + + return DefaultInfo( + files = depset([output]), + runfiles = ctx.runfiles(files = [output]), + ) + +package_annotations_file = rule( + implementation = _package_annotations_file_impl, + doc = ( + "Consumes `package_annotation` definitions in the same way " + + "`pip_repository` rules do to produce an annotations file." + ), + attrs = { + "annotations": attr.string_dict( + doc = "See `@rules_python//python:pip.bzl%package_annotation", + mandatory = True, + ), + }, +)
diff --git a/python/pip_install/extract_wheels/lib/bazel.py b/python/pip_install/extract_wheels/lib/bazel.py index e880c20..e967b39 100644 --- a/python/pip_install/extract_wheels/lib/bazel.py +++ b/python/pip_install/extract_wheels/lib/bazel.py
@@ -6,7 +6,12 @@ from pathlib import Path from typing import Dict, Iterable, List, Optional, Set -from python.pip_install.extract_wheels.lib import namespace_pkgs, purelib, wheel +from python.pip_install.extract_wheels.lib import ( + annotation, + namespace_pkgs, + purelib, + wheel, +) WHEEL_FILE_LABEL = "whl" PY_LIBRARY_LABEL = "pkg" @@ -77,13 +82,45 @@ ) +def generate_copy_commands(src, dest, is_executable=False) -> str: + """Generate a [@bazel_skylib//rules:copy_file.bzl%copy_file][cf] target + + [cf]: https://github.com/bazelbuild/bazel-skylib/blob/1.1.1/docs/copy_file_doc.md + + Args: + src (str): The label for the `src` attribute of [copy_file][cf] + dest (str): The label for the `out` attribute of [copy_file][cf] + is_executable (bool, optional): Whether or not the file being copied is executable. + sets `is_executable` for [copy_file][cf] + + Returns: + str: A `copy_file` instantiation. + """ + return textwrap.dedent( + """\ + copy_file( + name = "{dest}.copy", + src = "{src}", + out = "{dest}", + is_executable = {is_executable}, + ) + """.format( + src=src, + dest=dest, + is_executable=is_executable, + ) + ) + + def generate_build_file_contents( name: str, dependencies: List[str], whl_file_deps: List[str], - pip_data_exclude: List[str], + data_exclude: List[str], tags: List[str], - additional_targets: List[str] = [], + srcs_exclude: List[str] = [], + data: List[str] = [], + additional_content: List[str] = [], ) -> str: """Generate a BUILD file for an unzipped Wheel @@ -91,9 +128,9 @@ name: the target name of the py_library dependencies: a list of Bazel labels pointing to dependencies of the library whl_file_deps: a list of Bazel labels pointing to wheel file dependencies of this wheel. - pip_data_exclude: more patterns to exclude from the data attribute of generated py_library rules. + data_exclude: more patterns to exclude from the data attribute of generated py_library rules. tags: list of tags to apply to generated py_library rules. - additional_targets: A list of additional targets to append to the BUILD file contents. + additional_content: A list of additional content to append to the BUILD file. Returns: A complete BUILD file as a string @@ -102,22 +139,28 @@ there may be no Python sources whatsoever (e.g. packages written in Cython: like `pymssql`). """ - data_exclude = [ - "*.whl", - "**/__pycache__/**", - "**/*.py", - "**/*.pyc", - f"{WHEEL_ENTRY_POINT_PREFIX}*.py", - "**/* *", - "BUILD.bazel", - "WORKSPACE", - ] + pip_data_exclude + data_exclude = list( + set( + [ + "*.whl", + "**/__pycache__/**", + "**/* *", + "**/*.py", + "**/*.pyc", + "BUILD.bazel", + "WORKSPACE", + f"{WHEEL_ENTRY_POINT_PREFIX}*.py", + ] + + data_exclude + ) + ) return "\n".join( [ textwrap.dedent( """\ load("@rules_python//python:defs.bzl", "py_library", "py_binary") + load("@rules_python//third_party/github.com/bazelbuild/bazel-skylib/rules:copy_file.bzl", "copy_file") package(default_visibility = ["//visibility:public"]) @@ -139,8 +182,8 @@ py_library( name = "{name}", - srcs = glob(["**/*.py"], exclude=["{entry_point_prefix}*.py", "**/__pycache__/**"], allow_empty = True), - data = glob(["**/*"], exclude={data_exclude}), + srcs = glob(["**/*.py"], exclude={srcs_exclude}, allow_empty = True), + data = {data} + glob(["**/*"], exclude={data_exclude}), # This makes this directory a top-level in the python import # search path for anything that depends on this. imports = ["."], @@ -149,18 +192,20 @@ ) """.format( name=name, - dependencies=",".join(dependencies), - data_exclude=json.dumps(data_exclude), + dependencies=",".join(sorted(dependencies)), + data_exclude=json.dumps(sorted(data_exclude)), whl_file_label=WHEEL_FILE_LABEL, - whl_file_deps=",".join(whl_file_deps), - tags=",".join(['"%s"' % t for t in tags]), + whl_file_deps=",".join(sorted(whl_file_deps)), + tags=",".join(sorted(['"%s"' % t for t in tags])), data_label=DATA_LABEL, dist_info_label=DIST_INFO_LABEL, entry_point_prefix=WHEEL_ENTRY_POINT_PREFIX, + srcs_exclude=json.dumps(sorted(srcs_exclude)), + data=json.dumps(sorted(data)), ) ) ] - + additional_targets + + additional_content ) @@ -297,6 +342,7 @@ repo_prefix: str, incremental: bool = False, incremental_dir: Path = Path("."), + annotation: Optional[annotation.Annotation] = None, ) -> Optional[str]: """Extracts wheel into given directory and creates py_library and filegroup targets. @@ -308,6 +354,7 @@ incremental: If true the extract the wheel in a format suitable for an external repository. This effects the names of libraries and their dependencies, which point to other external repositories. incremental_dir: An optional override for the working directory of incremental builds. + annotation: An optional set of annotations to apply to the BUILD contents of the wheel. Returns: The Bazel label for the extracted wheel, in the form '//path/to/wheel'. @@ -367,13 +414,36 @@ ) with open(os.path.join(directory, "BUILD.bazel"), "w") as build_file: + additional_content = entry_points + data = [] + data_exclude = pip_data_exclude + srcs_exclude = [] + if annotation: + for src, dest in annotation.copy_files.items(): + data.append(dest) + additional_content.append(generate_copy_commands(src, dest)) + for src, dest in annotation.copy_executables.items(): + data.append(dest) + additional_content.append( + generate_copy_commands(src, dest, is_executable=True) + ) + data.extend(annotation.data) + data_exclude.extend(annotation.data_exclude_glob) + srcs_exclude.extend(annotation.srcs_exclude_glob) + if annotation.additive_build_content: + additional_content.append(annotation.additive_build_content) + contents = generate_build_file_contents( - PY_LIBRARY_LABEL if incremental else sanitise_name(whl.name, repo_prefix), - sanitised_dependencies, - sanitised_wheel_file_dependencies, - pip_data_exclude, - ["pypi_name=" + whl.name, "pypi_version=" + whl.metadata.version], - entry_points, + name=PY_LIBRARY_LABEL + if incremental + else sanitise_name(whl.name, repo_prefix), + dependencies=sanitised_dependencies, + whl_file_deps=sanitised_wheel_file_dependencies, + data_exclude=data_exclude, + data=data, + srcs_exclude=srcs_exclude, + tags=["pypi_name=" + whl.name, "pypi_version=" + whl.metadata.version], + additional_content=additional_content, ) build_file.write(contents)
diff --git a/python/pip_install/parse_requirements_to_bzl/__init__.py b/python/pip_install/parse_requirements_to_bzl/__init__.py index 1e1261d..7a23384 100644 --- a/python/pip_install/parse_requirements_to_bzl/__init__.py +++ b/python/pip_install/parse_requirements_to_bzl/__init__.py
@@ -3,20 +3,20 @@ import shlex import sys import textwrap -from typing import List, Tuple +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple from pip._internal.network.session import PipSession -from pip._internal.req import constructors, parse_requirements +from pip._internal.req import constructors from pip._internal.req.req_file import ( RequirementsFileParser, get_file_content, get_line_parser, - handle_line, preprocess, ) from pip._internal.req.req_install import InstallRequirement -from python.pip_install.extract_wheels.lib import arguments, bazel +from python.pip_install.extract_wheels.lib import annotation, arguments, bazel def parse_install_requirements( @@ -56,7 +56,25 @@ ] -def generate_parsed_requirements_contents(all_args: argparse.Namespace) -> str: +def parse_whl_library_args(args: argparse.Namespace) -> Dict[str, Any]: + whl_library_args = dict(vars(args)) + whl_library_args = arguments.deserialize_structured_args(whl_library_args) + whl_library_args.setdefault("python_interpreter", sys.executable) + + # These arguments are not used by `whl_library` + for arg in ("requirements_lock", "annotations"): + if arg in whl_library_args: + whl_library_args.pop(arg) + + return whl_library_args + + +def generate_parsed_requirements_contents( + requirements_lock: Path, + repo_prefix: str, + whl_library_args: Dict[str, Any], + annotations: Dict[str, str] = dict(), +) -> str: """ Parse each requirement from the requirements_lock file, and prepare arguments for each repository rule, which will represent the individual requirements. @@ -64,28 +82,21 @@ Generates a requirements.bzl file containing a macro (install_deps()) which instantiates a repository rule for each requirment in the lock file. """ - - args = dict(vars(all_args)) - args = arguments.deserialize_structured_args(args) - args.setdefault("python_interpreter", sys.executable) - # Pop this off because it wont be used as a config argument to the whl_library rule. - requirements_lock = args.pop("requirements_lock") - install_req_and_lines = parse_install_requirements( - requirements_lock, args["extra_pip_args"] + requirements_lock, whl_library_args["extra_pip_args"] ) repo_names_and_reqs = repo_names_and_requirements( - install_req_and_lines, args["repo_prefix"] + install_req_and_lines, repo_prefix ) all_requirements = ", ".join( [ - bazel.sanitised_repo_library_label(ir.name, repo_prefix=args["repo_prefix"]) + bazel.sanitised_repo_library_label(ir.name, repo_prefix=repo_prefix) for ir, _ in install_req_and_lines ] ) all_whl_requirements = ", ".join( [ - bazel.sanitised_repo_file_label(ir.name, repo_prefix=args["repo_prefix"]) + bazel.sanitised_repo_file_label(ir.name, repo_prefix=repo_prefix) for ir, _ in install_req_and_lines ] ) @@ -99,6 +110,7 @@ _packages = {repo_names_and_reqs} _config = {args} + _annotations = {annotations} def _clean_name(name): return name.replace("-", "_").replace(".", "_").lower() @@ -120,24 +132,32 @@ script = pkg return "@{repo_prefix}" + _clean_name(pkg) + "//:{entry_point_prefix}_" + script + def _get_annotation(requirement): + # This expects to parse `setuptools==58.2.0 --hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11` + # down wo `setuptools`. + name = requirement.split(" ")[0].split("=")[0] + return _annotations.get(name) + def install_deps(): for name, requirement in _packages: whl_library( name = name, requirement = requirement, + annotation = _get_annotation(requirement), **_config, ) """.format( all_requirements=all_requirements, all_whl_requirements=all_whl_requirements, - repo_names_and_reqs=repo_names_and_reqs, - args=args, - repo_prefix=args["repo_prefix"], - py_library_label=bazel.PY_LIBRARY_LABEL, - wheel_file_label=bazel.WHEEL_FILE_LABEL, + annotations=json.dumps(annotations), + args=whl_library_args, data_label=bazel.DATA_LABEL, dist_info_label=bazel.DIST_INFO_LABEL, entry_point_prefix=bazel.WHEEL_ENTRY_POINT_PREFIX, + py_library_label=bazel.PY_LIBRARY_LABEL, + repo_names_and_reqs=repo_names_and_reqs, + repo_prefix=repo_prefix, + wheel_file_label=bazel.WHEEL_FILE_LABEL, ) ) @@ -181,8 +201,42 @@ required=True, help="timeout to use for pip operation.", ) + parser.add_argument( + "--annotations", + type=annotation.annotations_map_from_str_path, + help="A json encoded file containing annotations for rendered packages.", + ) arguments.parse_common_args(parser) args = parser.parse_args() + # Check for any annotations which match packages in the locked requirements file + install_requirements = parse_install_requirements( + args.requirements_lock, args.extra_pip_args + ) + req_names = sorted([req.name for req, _ in install_requirements]) + annotations = args.annotations.collect(req_names) + + # Write all rendered annotation files and generate a list of the labels to write to the requirements file + annotated_requirements = dict() + for name, content in annotations.items(): + annotation_path = Path(name + ".annotation.json") + annotation_path.write_text(json.dumps(content, indent=4)) + annotated_requirements.update( + { + name: "@{}//:{}.annotation.json".format( + args.repo_prefix.rstrip("_"), name + ) + } + ) + with open("requirements.bzl", "w") as requirement_file: - requirement_file.write(generate_parsed_requirements_contents(args)) + whl_library_args = parse_whl_library_args(args) + + requirement_file.write( + generate_parsed_requirements_contents( + requirements_lock=args.requirements_lock, + repo_prefix=args.repo_prefix, + whl_library_args=whl_library_args, + annotations=annotated_requirements, + ) + )
diff --git a/python/pip_install/parse_requirements_to_bzl/extract_single_wheel/__init__.py b/python/pip_install/parse_requirements_to_bzl/extract_single_wheel/__init__.py index 2c03ff3..198aefa 100644 --- a/python/pip_install/parse_requirements_to_bzl/extract_single_wheel/__init__.py +++ b/python/pip_install/parse_requirements_to_bzl/extract_single_wheel/__init__.py
@@ -8,6 +8,7 @@ from python.pip_install.extract_wheels import configure_reproducible_wheels from python.pip_install.extract_wheels.lib import arguments, bazel, requirements +from python.pip_install.extract_wheels.lib.annotation import annotation_from_str_path def main() -> None: @@ -20,6 +21,11 @@ required=True, help="A single PEP508 requirement specifier string.", ) + parser.add_argument( + "--annotation", + type=annotation_from_str_path, + help="A json encoded file containing annotations for rendered packages.", + ) arguments.parse_common_args(parser) args = parser.parse_args() deserialized_args = dict(vars(args)) @@ -61,10 +67,11 @@ whl = next(iter(glob.glob("*.whl"))) bazel.extract_wheel( - whl, - extras, - deserialized_args["pip_data_exclude"], - args.enable_implicit_namespace_pkgs, + wheel_file=whl, + extras=extras, + pip_data_exclude=deserialized_args["pip_data_exclude"], + enable_implicit_namespace_pkgs=args.enable_implicit_namespace_pkgs, incremental=True, repo_prefix=args.repo_prefix, + annotation=args.annotation, )
diff --git a/python/pip_install/parse_requirements_to_bzl/parse_requirements_to_bzl_test.py b/python/pip_install/parse_requirements_to_bzl/parse_requirements_to_bzl_test.py index 9619af5..c0608bf 100644 --- a/python/pip_install/parse_requirements_to_bzl/parse_requirements_to_bzl_test.py +++ b/python/pip_install/parse_requirements_to_bzl/parse_requirements_to_bzl_test.py
@@ -6,6 +6,7 @@ from python.pip_install.parse_requirements_to_bzl import ( generate_parsed_requirements_contents, + parse_whl_library_args, ) @@ -28,7 +29,12 @@ args.python_interpreter = "/custom/python3" args.python_interpreter_target = "@custom_python//:exec" args.environment = json.dumps({"arg": {}}) - contents = generate_parsed_requirements_contents(args) + whl_library_args = parse_whl_library_args(args) + contents = generate_parsed_requirements_contents( + requirements_lock=args.requirements_lock, + repo_prefix=args.repo_prefix, + whl_library_args=whl_library_args, + ) library_target = "@pip_parsed_deps_pypi__foo//:pkg" whl_target = "@pip_parsed_deps_pypi__foo//:whl" all_requirements = 'all_requirements = ["{library_target}"]'.format(
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 3d0710a..7166d25 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl
@@ -128,6 +128,11 @@ # We need a BUILD file to load the generated requirements.bzl rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) + # Write the annotations file to pass to the wheel maker + annotations = {package: json.decode(data) for (package, data) in rctx.attr.annotations.items()} + annotations_file = rctx.path("annotations.json") + rctx.file(annotations_file, json.encode_indent(annotations, indent = " " * 4)) + pypath = _construct_pypath(rctx) if rctx.attr.incremental: @@ -142,6 +147,8 @@ str(rctx.attr.quiet), "--timeout", str(rctx.attr.timeout), + "--annotations", + annotations_file, ] args += ["--python_interpreter", _get_python_interpreter_attr(rctx)] @@ -155,6 +162,8 @@ "python.pip_install.extract_wheels", "--requirements", rctx.path(rctx.attr.requirements), + "--annotations", + annotations_file, ] args += ["--repo", rctx.attr.name, "--repo-prefix", rctx.attr.repo_prefix] @@ -259,6 +268,9 @@ } pip_repository_attrs = { + "annotations": attr.string_dict( + doc = "Optional annotations to apply to packages", + ), "incremental": attr.bool( default = False, doc = "Create the repository in incremental mode.", @@ -324,7 +336,7 @@ environ = common_env, ) -def _impl_whl_library(rctx): +def _whl_library_impl(rctx): python_interpreter = _resolve_python_interpreter(rctx) # pointer to parent repo so these rules rerun if the definitions in requirements.bzl change. @@ -340,6 +352,12 @@ "--repo-prefix", rctx.attr.repo_prefix, ] + if rctx.attr.annotation: + args.extend([ + "--annotation", + rctx.path(rctx.attr.annotation), + ]) + args = _parse_optional_attrs(rctx, args) result = rctx.execute( @@ -356,6 +374,13 @@ return whl_library_attrs = { + "annotation": attr.label( + doc = ( + "Optional json encoded file containing annotation to apply to the extracted wheel. " + + "See `package_annotation`" + ), + allow_files = True, + ), "repo": attr.string( mandatory = True, doc = "Pointer to parent repo name. Used to make these rules rerun if the parent repo changes.", @@ -373,6 +398,40 @@ doc = """ Download and extracts a single wheel based into a bazel repo based on the requirement string passed in. Instantiated from pip_repository and inherits config options from there.""", - implementation = _impl_whl_library, + implementation = _whl_library_impl, environ = common_env, ) + +def package_annotation( + additive_build_content = None, + copy_files = {}, + copy_executables = {}, + data = [], + data_exclude_glob = [], + srcs_exclude_glob = []): + """Annotations to apply to the BUILD file content from package generated from a `pip_repository` rule. + + [cf]: https://github.com/bazelbuild/bazel-skylib/blob/main/docs/copy_file_doc.md + + Args: + additive_build_content (str, optional): Raw text to add to the generated `BUILD` file of a package. + copy_files (dict, optional): A mapping of `src` and `out` files for [@bazel_skylib//rules:copy_file.bzl][cf] + copy_executables (dict, optional): A mapping of `src` and `out` files for + [@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as + executable. + data (list, optional): A list of labels to add as `data` dependencies to the generated `py_library` target. + data_exclude_glob (list, optional): A list of exclude glob patterns to add as `data` to the generated + `py_library` target. + srcs_exclude_glob (list, optional): A list of labels to add as `srcs` to the generated `py_library` target. + + Returns: + str: A json encoded string of the provided content. + """ + return json.encode(struct( + additive_build_content = additive_build_content, + copy_files = copy_files, + copy_executables = copy_executables, + data = data, + data_exclude_glob = data_exclude_glob, + srcs_exclude_glob = srcs_exclude_glob, + ))
diff --git a/python/pip_install/private/srcs.bzl b/python/pip_install/private/srcs.bzl index 3784fa8..3f20c45 100644 --- a/python/pip_install/private/srcs.bzl +++ b/python/pip_install/private/srcs.bzl
@@ -10,6 +10,7 @@ "@rules_python//python/pip_install/extract_wheels:__init__.py", "@rules_python//python/pip_install/extract_wheels:__main__.py", "@rules_python//python/pip_install/extract_wheels/lib:__init__.py", + "@rules_python//python/pip_install/extract_wheels/lib:annotation.py", "@rules_python//python/pip_install/extract_wheels/lib:arguments.py", "@rules_python//python/pip_install/extract_wheels/lib:bazel.py", "@rules_python//python/pip_install/extract_wheels/lib:namespace_pkgs.py",
diff --git a/python/pip_install/private/test/BUILD b/python/pip_install/private/test/BUILD index 90d1846..60d25de 100644 --- a/python/pip_install/private/test/BUILD +++ b/python/pip_install/private/test/BUILD
@@ -3,8 +3,8 @@ diff_test( name = "srcs_diff_test", failure_message = ( - "Please run `bazel run //python/pip_install/private:srcs_module.update` " + - "to update the `srcs.bzl` module found in the same package." + "Please run 'bazel run //python/pip_install/private:srcs_module.update' " + + "to update the 'srcs.bzl' module found in the same package." ), file1 = "//python/pip_install/private:srcs_module", file2 = "//python/pip_install/private:srcs.bzl",
diff --git a/third_party/github.com/bazelbuild/bazel-skylib/rules/BUILD b/third_party/github.com/bazelbuild/bazel-skylib/rules/BUILD new file mode 100644 index 0000000..6857449 --- /dev/null +++ b/third_party/github.com/bazelbuild/bazel-skylib/rules/BUILD
@@ -0,0 +1,36 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + +licenses(["notice"]) + +package(default_visibility = ["//visibility:public"]) + +bzl_library( + name = "copy_file", + srcs = ["copy_file.bzl"], + deps = ["//third_party/github.com/bazelbuild/bazel-skylib/rules/private:copy_file_private"], +) + +filegroup( + name = "test_deps", + testonly = True, + srcs = [ + "BUILD", + ] + glob(["*.bzl"]), +) + +# The files needed for distribution +filegroup( + name = "distribution", + srcs = [ + "BUILD", + ] + glob(["*.bzl"]), + visibility = [ + "//:__pkg__", + ], +) + +# export bzl files for the documentation +exports_files( + glob(["*.bzl"]), + visibility = ["//:__subpackages__"], +)
diff --git a/third_party/github.com/bazelbuild/bazel-skylib/rules/copy_file.bzl b/third_party/github.com/bazelbuild/bazel-skylib/rules/copy_file.bzl new file mode 100644 index 0000000..2908fa6 --- /dev/null +++ b/third_party/github.com/bazelbuild/bazel-skylib/rules/copy_file.bzl
@@ -0,0 +1,29 @@ +# Copyright 2019 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A rule that copies a file to another place. + +native.genrule() is sometimes used to copy files (often wishing to rename them). +The 'copy_file' rule does this with a simpler interface than genrule. + +The rule uses a Bash command on Linux/macOS/non-Windows, and a cmd.exe command +on Windows (no Bash is required). +""" + +load( + "@rules_python//third_party/github.com/bazelbuild/bazel-skylib/rules/private:copy_file_private.bzl", + _copy_file = "copy_file", +) + +copy_file = _copy_file
diff --git a/third_party/github.com/bazelbuild/bazel-skylib/rules/private/BUILD b/third_party/github.com/bazelbuild/bazel-skylib/rules/private/BUILD new file mode 100644 index 0000000..a1aeb39 --- /dev/null +++ b/third_party/github.com/bazelbuild/bazel-skylib/rules/private/BUILD
@@ -0,0 +1,18 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + +licenses(["notice"]) + +bzl_library( + name = "copy_file_private", + srcs = ["copy_file_private.bzl"], + visibility = ["//third_party/github.com/bazelbuild/bazel-skylib/rules:__pkg__"], +) + +# The files needed for distribution +filegroup( + name = "distribution", + srcs = glob(["*"]), + visibility = [ + "//:__subpackages__", + ], +)
diff --git a/third_party/github.com/bazelbuild/bazel-skylib/rules/private/copy_file_private.bzl b/third_party/github.com/bazelbuild/bazel-skylib/rules/private/copy_file_private.bzl new file mode 100644 index 0000000..d044c97 --- /dev/null +++ b/third_party/github.com/bazelbuild/bazel-skylib/rules/private/copy_file_private.bzl
@@ -0,0 +1,141 @@ +# Copyright 2019 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Implementation of copy_file macro and underlying rules. + +These rules copy a file to another location using Bash (on Linux/macOS) or +cmd.exe (on Windows). '_copy_xfile' marks the resulting file executable, +'_copy_file' does not. +""" + +def copy_cmd(ctx, src, dst): + # Most Windows binaries built with MSVC use a certain argument quoting + # scheme. Bazel uses that scheme too to quote arguments. However, + # cmd.exe uses different semantics, so Bazel's quoting is wrong here. + # To fix that we write the command to a .bat file so no command line + # quoting or escaping is required. + bat = ctx.actions.declare_file(ctx.label.name + "-cmd.bat") + ctx.actions.write( + output = bat, + # Do not use lib/shell.bzl's shell.quote() method, because that uses + # Bash quoting syntax, which is different from cmd.exe's syntax. + content = "@copy /Y \"%s\" \"%s\" >NUL" % ( + src.path.replace("/", "\\"), + dst.path.replace("/", "\\"), + ), + is_executable = True, + ) + ctx.actions.run( + inputs = [src], + tools = [bat], + outputs = [dst], + executable = "cmd.exe", + arguments = ["/C", bat.path.replace("/", "\\")], + mnemonic = "CopyFile", + progress_message = "Copying files", + use_default_shell_env = True, + ) + +def copy_bash(ctx, src, dst): + ctx.actions.run_shell( + tools = [src], + outputs = [dst], + command = "cp -f \"$1\" \"$2\"", + arguments = [src.path, dst.path], + mnemonic = "CopyFile", + progress_message = "Copying files", + use_default_shell_env = True, + ) + +def _copy_file_impl(ctx): + if ctx.attr.allow_symlink: + ctx.actions.symlink( + output = ctx.outputs.out, + target_file = ctx.file.src, + is_executable = ctx.attr.is_executable, + ) + elif ctx.attr.is_windows: + copy_cmd(ctx, ctx.file.src, ctx.outputs.out) + else: + copy_bash(ctx, ctx.file.src, ctx.outputs.out) + + files = depset(direct = [ctx.outputs.out]) + runfiles = ctx.runfiles(files = [ctx.outputs.out]) + if ctx.attr.is_executable: + return [DefaultInfo(files = files, runfiles = runfiles, executable = ctx.outputs.out)] + else: + return [DefaultInfo(files = files, runfiles = runfiles)] + +_ATTRS = { + "allow_symlink": attr.bool(mandatory = True), + "is_executable": attr.bool(mandatory = True), + "is_windows": attr.bool(mandatory = True), + "out": attr.output(mandatory = True), + "src": attr.label(mandatory = True, allow_single_file = True), +} + +_copy_file = rule( + implementation = _copy_file_impl, + provides = [DefaultInfo], + attrs = _ATTRS, +) + +_copy_xfile = rule( + implementation = _copy_file_impl, + executable = True, + provides = [DefaultInfo], + attrs = _ATTRS, +) + +def copy_file(name, src, out, is_executable = False, allow_symlink = False, **kwargs): + """Copies a file to another location. + + `native.genrule()` is sometimes used to copy files (often wishing to rename them). The 'copy_file' rule does this with a simpler interface than genrule. + + This rule uses a Bash command on Linux/macOS/non-Windows, and a cmd.exe command on Windows (no Bash is required). + + Args: + name: Name of the rule. + src: A Label. The file to make a copy of. (Can also be the label of a rule + that generates a file.) + out: Path of the output file, relative to this package. + is_executable: A boolean. Whether to make the output file executable. When + True, the rule's output can be executed using `bazel run` and can be + in the srcs of binary and test rules that require executable sources. + WARNING: If `allow_symlink` is True, `src` must also be executable. + allow_symlink: A boolean. Whether to allow symlinking instead of copying. + When False, the output is always a hard copy. When True, the output + *can* be a symlink, but there is no guarantee that a symlink is + created (i.e., at the time of writing, we don't create symlinks on + Windows). Set this to True if you need fast copying and your tools can + handle symlinks (which most UNIX tools can). + **kwargs: further keyword arguments, e.g. `visibility` + """ + + copy_file_impl = _copy_file + if is_executable: + copy_file_impl = _copy_xfile + + copy_file_impl( + name = name, + src = src, + out = out, + is_windows = select({ + "@bazel_tools//src/conditions:host_windows": True, + "//conditions:default": False, + }), + is_executable = is_executable, + allow_symlink = allow_symlink, + **kwargs + )