Implement py_proto_library (#832)
* Add py_proto_library
* Bump versions of rules_proto and protobuf
* Update documentation
* Bump rules_pkg version
diff --git a/MODULE.bazel b/MODULE.bazel
index 6afe365..f337db3 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -6,6 +6,10 @@
bazel_dep(name = "platforms", version = "0.0.4")
+# Those are loaded only when using py_proto_library
+bazel_dep(name = "rules_proto", version = "5.3.0-21.7")
+bazel_dep(name = "protobuf", repo_name = "com_google_protobuf", version = "21.7")
+
internal_deps = use_extension("@rules_python//python:extensions.bzl", "internal_deps")
internal_deps.install()
use_repo(
diff --git a/README.md b/README.md
index 53f1956..a509e28 100644
--- a/README.md
+++ b/README.md
@@ -6,7 +6,7 @@
## Overview
This repository is the home of the core Python rules -- `py_library`,
-`py_binary`, `py_test`, and related symbols that provide the basis for Python
+`py_binary`, `py_test`, `py_proto_library`, and related symbols that provide the basis for Python
support in Bazel. It also contains package installation rules for integrating with PyPI and other package indices. Documentation lives in the
[`docs/`](https://github.com/bazelbuild/rules_python/tree/main/docs)
directory and in the
diff --git a/internal_deps.bzl b/internal_deps.bzl
index a41d5fb..942a872 100644
--- a/internal_deps.bzl
+++ b/internal_deps.bzl
@@ -20,10 +20,10 @@
http_archive,
name = "rules_pkg",
urls = [
- "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.2.4/rules_pkg-0.2.4.tar.gz",
- "https://github.com/bazelbuild/rules_pkg/releases/download/0.2.4/rules_pkg-0.2.4.tar.gz",
+ "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz",
+ "https://github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz",
],
- sha256 = "4ba8f4ab0ff85f2484287ab06c0d871dcb31cc54d439457d28fd4ae14b18450a",
+ sha256 = "8a298e832762eda1830597d64fe7db58178aa84cd5926d76d5b744d6558941c2",
)
maybe(
@@ -124,3 +124,24 @@
strip_prefix = "bazel-integration-testing-165440b2dbda885f8d1ccb8d0f417e6cf8c54f17",
sha256 = "2401b1369ef44cc42f91dc94443ef491208dbd06da1e1e10b702d8c189f098e3",
)
+
+ maybe(
+ http_archive,
+ name = "rules_proto",
+ sha256 = "dc3fb206a2cb3441b485eb1e423165b231235a1ea9b031b4433cf7bc1fa460dd",
+ strip_prefix = "rules_proto-5.3.0-21.7",
+ urls = [
+ "https://github.com/bazelbuild/rules_proto/archive/refs/tags/5.3.0-21.7.tar.gz",
+ ],
+ )
+
+ maybe(
+ http_archive,
+ name = "com_google_protobuf",
+ sha256 = "75be42bd736f4df6d702a0e4e4d30de9ee40eac024c4b845d17ae4cc831fe4ae",
+ strip_prefix = "protobuf-21.7",
+ urls = [
+ "https://mirror.bazel.build/github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz",
+ "https://github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz",
+ ],
+ )
diff --git a/internal_setup.bzl b/internal_setup.bzl
index f4d3a1a..57eecc2 100644
--- a/internal_setup.bzl
+++ b/internal_setup.bzl
@@ -17,7 +17,9 @@
load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
load("@build_bazel_integration_testing//tools:repositories.bzl", "bazel_binaries")
+load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
+load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
load("//:version.bzl", "SUPPORTED_BAZEL_VERSIONS")
load("//gazelle:deps.bzl", _go_repositories = "gazelle_deps")
load("//python/pip_install:repositories.bzl", "pip_install_dependencies")
@@ -41,3 +43,8 @@
go_register_toolchains(version = "1.19.2")
gazelle_dependencies()
+
+ rules_proto_dependencies()
+ rules_proto_toolchains()
+
+ protobuf_deps()
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
index c99b040..75e48bc 100644
--- a/python/private/BUILD.bazel
+++ b/python/private/BUILD.bazel
@@ -19,7 +19,7 @@
filegroup(
name = "distribution",
- srcs = glob(["**"]),
+ srcs = glob(["**"]) + ["//python/private/proto:distribution"],
visibility = ["//python:__pkg__"],
)
diff --git a/python/private/proto/BUILD b/python/private/proto/BUILD
new file mode 100644
index 0000000..8483d19
--- /dev/null
+++ b/python/private/proto/BUILD
@@ -0,0 +1,32 @@
+# Copyright 2022 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain")
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"]) # Apache 2.0
+
+filegroup(
+ name = "distribution",
+ srcs = glob(["**"]),
+ visibility = ["//python/private:__pkg__"],
+)
+
+proto_lang_toolchain(
+ name = "python_toolchain",
+ command_line = "--python_out=%s",
+ progress_message = "Generating Python proto_library %{label}",
+ runtime = "@com_google_protobuf//:protobuf_python",
+)
diff --git a/python/private/proto/py_proto_library.bzl b/python/private/proto/py_proto_library.bzl
new file mode 100644
index 0000000..ef5f2ca
--- /dev/null
+++ b/python/private/proto/py_proto_library.bzl
@@ -0,0 +1,191 @@
+# Copyright 2022 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""The implementation of the `py_proto_library` rule and its aspect."""
+
+load("@rules_proto//proto:defs.bzl", "ProtoInfo", "proto_common")
+load("//python:defs.bzl", "PyInfo")
+
+ProtoLangToolchainInfo = proto_common.ProtoLangToolchainInfo
+
+_PyProtoInfo = provider(
+ doc = "Encapsulates information needed by the Python proto rules.",
+ fields = {
+ "runfiles_from_proto_deps": """
+ (depset[File]) Files from the transitive closure implicit proto
+ dependencies""",
+ "transitive_sources": """(depset[File]) The Python sources.""",
+ },
+)
+
+def _filter_provider(provider, *attrs):
+ return [dep[provider] for attr in attrs for dep in attr if provider in dep]
+
+def _py_proto_aspect_impl(target, ctx):
+ """Generates and compiles Python code for a proto_library.
+
+ The function runs protobuf compiler on the `proto_library` target generating
+ a .py file for each .proto file.
+
+ Args:
+ target: (Target) A target providing `ProtoInfo`. Usually this means a
+ `proto_library` target, but not always; you must expect to visit
+ non-`proto_library` targets, too.
+ ctx: (RuleContext) The rule context.
+
+ Returns:
+ ([_PyProtoInfo]) Providers collecting transitive information about
+ generated files.
+ """
+
+ _proto_library = ctx.rule.attr
+
+ # Check Proto file names
+ for proto in target[ProtoInfo].direct_sources:
+ if proto.is_source and "-" in proto.dirname:
+ fail("Cannot generate Python code for a .proto whose path contains '-' ({}).".format(
+ proto.path,
+ ))
+
+ proto_lang_toolchain_info = ctx.attr._aspect_proto_toolchain[ProtoLangToolchainInfo]
+ api_deps = [proto_lang_toolchain_info.runtime]
+
+ generated_sources = []
+ proto_info = target[ProtoInfo]
+ if proto_info.direct_sources:
+ # Generate py files
+ generated_sources = proto_common.declare_generated_files(
+ actions = ctx.actions,
+ proto_info = proto_info,
+ extension = "_pb2.py",
+ name_mapper = lambda name: name.replace("-", "_").replace(".", "/"),
+ )
+
+ proto_common.compile(
+ actions = ctx.actions,
+ proto_info = proto_info,
+ proto_lang_toolchain_info = proto_lang_toolchain_info,
+ generated_files = generated_sources,
+ plugin_output = ctx.bin_dir.path,
+ )
+
+ # Generated sources == Python sources
+ python_sources = generated_sources
+
+ deps = _filter_provider(_PyProtoInfo, getattr(_proto_library, "deps", []))
+ runfiles_from_proto_deps = depset(
+ transitive = [dep[DefaultInfo].default_runfiles.files for dep in api_deps] +
+ [dep.runfiles_from_proto_deps for dep in deps],
+ )
+ transitive_sources = depset(
+ direct = python_sources,
+ transitive = [dep.transitive_sources for dep in deps],
+ )
+
+ return [
+ _PyProtoInfo(
+ runfiles_from_proto_deps = runfiles_from_proto_deps,
+ transitive_sources = transitive_sources,
+ ),
+ ]
+
+_py_proto_aspect = aspect(
+ implementation = _py_proto_aspect_impl,
+ attrs = {
+ "_aspect_proto_toolchain": attr.label(
+ default = ":python_toolchain",
+ ),
+ },
+ attr_aspects = ["deps"],
+ required_providers = [ProtoInfo],
+ provides = [_PyProtoInfo],
+)
+
+def _py_proto_library_rule(ctx):
+ """Merges results of `py_proto_aspect` in `deps`.
+
+ Args:
+ ctx: (RuleContext) The rule context.
+ Returns:
+ ([PyInfo, DefaultInfo, OutputGroupInfo])
+ """
+ if not ctx.attr.deps:
+ fail("'deps' attribute mustn't be empty.")
+
+ pyproto_infos = _filter_provider(_PyProtoInfo, ctx.attr.deps)
+ default_outputs = depset(
+ transitive = [info.transitive_sources for info in pyproto_infos],
+ )
+
+ return [
+ DefaultInfo(
+ files = default_outputs,
+ default_runfiles = ctx.runfiles(transitive_files = depset(
+ transitive =
+ [default_outputs] +
+ [info.runfiles_from_proto_deps for info in pyproto_infos],
+ )),
+ ),
+ OutputGroupInfo(
+ default = depset(),
+ ),
+ PyInfo(
+ transitive_sources = default_outputs,
+ # Proto always produces 2- and 3- compatible source files
+ has_py2_only_sources = False,
+ has_py3_only_sources = False,
+ ),
+ ]
+
+py_proto_library = rule(
+ implementation = _py_proto_library_rule,
+ doc = """
+ Use `py_proto_library` to generate Python libraries from `.proto` files.
+
+ The convention is to name the `py_proto_library` rule `foo_py_pb2`,
+ when it is wrapping `proto_library` rule `foo_proto`.
+
+ `deps` must point to a `proto_library` rule.
+
+ Example:
+
+```starlark
+py_library(
+ name = "lib",
+ deps = [":foo_py_pb2"],
+)
+
+py_proto_library(
+ name = "foo_py_pb2",
+ deps = [":foo_proto"],
+)
+
+proto_library(
+ name = "foo_proto",
+ srcs = ["foo.proto"],
+)
+```""",
+ attrs = {
+ "deps": attr.label_list(
+ doc = """
+ The list of `proto_library` rules to generate Python libraries for.
+
+ Usually this is just the one target: the proto library of interest.
+ It can be any target providing `ProtoInfo`.""",
+ providers = [ProtoInfo],
+ aspects = [_py_proto_aspect],
+ ),
+ },
+ provides = [PyInfo],
+)
diff --git a/python/proto.bzl b/python/proto.bzl
new file mode 100644
index 0000000..3f455ae
--- /dev/null
+++ b/python/proto.bzl
@@ -0,0 +1,21 @@
+# Copyright 2022 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Python proto library.
+"""
+
+load("//python/private/proto:py_proto_library.bzl", _py_proto_library = "py_proto_library")
+
+py_proto_library = _py_proto_library
