feat: Expose Python C headers through the toolchain. (#1287)
This allows getting a build's `cc_library` of Python headers through
toolchain resolution instead of having to use the underlying toolchain's
repository `:python_headers` target directly.
Without this feature, it's not possible to reliably and correctly get
the C information about the runtime a build is going to use. Existing
solutions require carefully setting up repo names, external state,
and/or using specific build rules. In comparison, with this feature,
consumers are able to simply ask for the current headers via a helper
target or manually lookup the toolchain and pull the relevant
information; toolchain resolution handles finding the correct headers.
The basic way this works is by registering a second toolchain to carry
C/C++ related information; as such, it is named `py_cc_toolchain`. The
py cc toolchain has the same constraint settings as the regular py
toolchain; an expected invariant is that there is a 1:1 correspondence
between the two. This base functionality allows a consuming rule
implementation to use toolchain resolution to find the Python C
toolchain information.
Usually what downstream consumers need are the headers to feed into
another `cc_library` (or equivalent) target, so, rather than have every
project re-implement the same "lookup and forward cc_library info"
logic,
this is provided by the `//python/cc:current_py_cc_headers` target.
Targets that need the headers can then depend on that target as if it
was a `cc_library` target.
Work towards https://github.com/bazelbuild/rules_python/issues/824
diff --git a/python/BUILD.bazel b/python/BUILD.bazel
index d75889d..c5f2580 100644
--- a/python/BUILD.bazel
+++ b/python/BUILD.bazel
@@ -33,6 +33,7 @@
filegroup(
name = "distribution",
srcs = glob(["**"]) + [
+ "//python/cc:distribution",
"//python/config_settings:distribution",
"//python/constraints:distribution",
"//python/private:distribution",
diff --git a/python/cc/BUILD.bazel b/python/cc/BUILD.bazel
new file mode 100644
index 0000000..d4a6bb8
--- /dev/null
+++ b/python/cc/BUILD.bazel
@@ -0,0 +1,44 @@
+# Package for C/C++ specific functionality of the Python rules.
+
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+load("//python/private:current_py_cc_headers.bzl", "current_py_cc_headers")
+load("//python/private:util.bzl", "BZLMOD_ENABLED")
+
+package(
+ default_visibility = ["//:__subpackages__"],
+)
+
+# This target provides the C headers for whatever the current toolchain is
+# for the consuming rule. It basically acts like a cc_library by forwarding
+# on the providers for the underlying cc_library that the toolchain is using.
+current_py_cc_headers(
+ name = "current_py_cc_headers",
+ # Building this directly will fail unless a py cc toolchain is registered,
+ # and it's only under bzlmod that one is registered by default.
+ tags = [] if BZLMOD_ENABLED else ["manual"],
+ visibility = ["//visibility:public"],
+)
+
+toolchain_type(
+ name = "toolchain_type",
+ visibility = ["//visibility:public"],
+)
+
+bzl_library(
+ name = "py_cc_toolchain_bzl",
+ srcs = ["py_cc_toolchain.bzl"],
+ visibility = ["//visibility:public"],
+ deps = ["//python/private:py_cc_toolchain_bzl"],
+)
+
+bzl_library(
+ name = "py_cc_toolchain_info_bzl",
+ srcs = ["py_cc_toolchain_info.bzl"],
+ visibility = ["//visibility:public"],
+ deps = ["//python/private:py_cc_toolchain_info_bzl"],
+)
+
+filegroup(
+ name = "distribution",
+ srcs = glob(["**"]),
+)
diff --git a/python/cc/py_cc_toolchain.bzl b/python/cc/py_cc_toolchain.bzl
new file mode 100644
index 0000000..2e782ef
--- /dev/null
+++ b/python/cc/py_cc_toolchain.bzl
@@ -0,0 +1,19 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Public entry point for py_cc_toolchain rule."""
+
+load("//python/private:py_cc_toolchain_macro.bzl", _py_cc_toolchain = "py_cc_toolchain")
+
+py_cc_toolchain = _py_cc_toolchain
diff --git a/python/cc/py_cc_toolchain_info.bzl b/python/cc/py_cc_toolchain_info.bzl
new file mode 100644
index 0000000..9ea394a
--- /dev/null
+++ b/python/cc/py_cc_toolchain_info.bzl
@@ -0,0 +1,23 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Provider for C/C++ information about the Python runtime.
+
+NOTE: This is a beta-quality feature. APIs subject to change until
+https://github.com/bazelbuild/rules_python/issues/824 is considered done.
+"""
+
+load("//python/private:py_cc_toolchain_info.bzl", _PyCcToolchainInfo = "PyCcToolchainInfo")
+
+PyCcToolchainInfo = _PyCcToolchainInfo
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
index f454f42..10af17e 100644
--- a/python/private/BUILD.bazel
+++ b/python/private/BUILD.bazel
@@ -51,6 +51,28 @@
deps = ["@bazel_skylib//lib:types"],
)
+bzl_library(
+ name = "py_cc_toolchain_bzl",
+ srcs = [
+ "py_cc_toolchain_macro.bzl",
+ "py_cc_toolchain_rule.bzl",
+ ],
+ visibility = [
+ "//docs:__subpackages__",
+ "//python/cc:__pkg__",
+ ],
+ deps = [
+ ":py_cc_toolchain_info_bzl",
+ ":util_bzl",
+ ],
+)
+
+bzl_library(
+ name = "py_cc_toolchain_info_bzl",
+ srcs = ["py_cc_toolchain_info.bzl"],
+ visibility = ["//python/cc:__pkg__"],
+)
+
# @bazel_tools can't define bzl_library itself, so we just put a wrapper around it.
bzl_library(
name = "bazel_tools_bzl",
@@ -73,6 +95,7 @@
"reexports.bzl",
"stamp.bzl",
"util.bzl",
+ "py_cc_toolchain_rule.bzl",
],
visibility = ["//docs:__pkg__"],
)
diff --git a/python/private/current_py_cc_headers.bzl b/python/private/current_py_cc_headers.bzl
new file mode 100644
index 0000000..be7f8f8
--- /dev/null
+++ b/python/private/current_py_cc_headers.bzl
@@ -0,0 +1,41 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Implementation of current_py_cc_headers rule."""
+
+def _current_py_cc_headers_impl(ctx):
+ py_cc_toolchain = ctx.toolchains["//python/cc:toolchain_type"].py_cc_toolchain
+ return py_cc_toolchain.headers.providers_map.values()
+
+current_py_cc_headers = rule(
+ implementation = _current_py_cc_headers_impl,
+ toolchains = ["//python/cc:toolchain_type"],
+ provides = [CcInfo],
+ doc = """\
+Provides the currently active Python toolchain's C headers.
+
+This is a wrapper around the underlying `cc_library()` for the
+C headers for the consuming target's currently active Python toolchain.
+
+To use, simply depend on this target where you would have wanted the
+toolchain's underlying `:python_headers` target:
+
+```starlark
+cc_library(
+ name = "foo",
+ deps = ["@rules_python//python/cc:current_py_cc_headers"]
+)
+```
+""",
+)
diff --git a/python/private/py_cc_toolchain_info.bzl b/python/private/py_cc_toolchain_info.bzl
new file mode 100644
index 0000000..e7afc10
--- /dev/null
+++ b/python/private/py_cc_toolchain_info.bzl
@@ -0,0 +1,43 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Implementation of PyCcToolchainInfo."""
+
+PyCcToolchainInfo = provider(
+ doc = "C/C++ information about the Python runtime.",
+ fields = {
+ "headers": """\
+(struct) Information about the header files, with fields:
+ * providers_map: a dict of string to provider instances. The key should be
+ a fully qualified name (e.g. `@rules_foo//bar:baz.bzl#MyInfo`) of the
+ provider to uniquely identify its type.
+
+ The following keys are always present:
+ * CcInfo: the CcInfo provider instance for the headers.
+ * DefaultInfo: the DefaultInfo provider instance for the headers.
+
+ A map is used to allow additional providers from the originating headers
+ target (typically a `cc_library`) to be propagated to consumers (directly
+ exposing a Target object can cause memory issues and is an anti-pattern).
+
+ When consuming this map, it's suggested to use `providers_map.values()` to
+ return all providers; or copy the map and filter out or replace keys as
+ appropriate. Note that any keys begining with `_` (underscore) are
+ considered private and should be forward along as-is (this better allows
+ e.g. `:current_py_cc_headers` to act as the underlying headers target it
+ represents).
+""",
+ "python_version": "(str) The Python Major.Minor version.",
+ },
+)
diff --git a/python/private/py_cc_toolchain_macro.bzl b/python/private/py_cc_toolchain_macro.bzl
new file mode 100644
index 0000000..35276f7
--- /dev/null
+++ b/python/private/py_cc_toolchain_macro.bzl
@@ -0,0 +1,31 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Fronting macro for the py_cc_toolchain rule."""
+
+load(":py_cc_toolchain_rule.bzl", _py_cc_toolchain = "py_cc_toolchain")
+load(":util.bzl", "add_tag")
+
+# A fronting macro is used because macros have user-observable behavior;
+# using one from the onset avoids introducing those changes in the future.
+def py_cc_toolchain(**kwargs):
+ """Creates a py_cc_toolchain target.
+
+ Args:
+ **kwargs: Keyword args to pass onto underlying rule.
+ """
+
+ # This tag is added to easily identify usages through other macros.
+ add_tag(kwargs, "@rules_python//python:py_cc_toolchain")
+ _py_cc_toolchain(**kwargs)
diff --git a/python/private/py_cc_toolchain_rule.bzl b/python/private/py_cc_toolchain_rule.bzl
new file mode 100644
index 0000000..c80f845
--- /dev/null
+++ b/python/private/py_cc_toolchain_rule.bzl
@@ -0,0 +1,57 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Implementation of py_cc_toolchain rule.
+
+NOTE: This is a beta-quality feature. APIs subject to change until
+https://github.com/bazelbuild/rules_python/issues/824 is considered done.
+"""
+
+load(":py_cc_toolchain_info.bzl", "PyCcToolchainInfo")
+
+def _py_cc_toolchain_impl(ctx):
+ py_cc_toolchain = PyCcToolchainInfo(
+ headers = struct(
+ providers_map = {
+ "CcInfo": ctx.attr.headers[CcInfo],
+ "DefaultInfo": ctx.attr.headers[DefaultInfo],
+ },
+ ),
+ python_version = ctx.attr.python_version,
+ )
+ return [platform_common.ToolchainInfo(
+ py_cc_toolchain = py_cc_toolchain,
+ )]
+
+py_cc_toolchain = rule(
+ implementation = _py_cc_toolchain_impl,
+ attrs = {
+ "headers": attr.label(
+ doc = ("Target that provides the Python headers. Typically this " +
+ "is a cc_library target."),
+ providers = [CcInfo],
+ mandatory = True,
+ ),
+ "python_version": attr.string(
+ doc = "The Major.minor Python version, e.g. 3.11",
+ mandatory = True,
+ ),
+ },
+ doc = """\
+A toolchain for a Python runtime's C/C++ information (e.g. headers)
+
+This rule carries information about the C/C++ side of a Python runtime, e.g.
+headers, shared libraries, etc.
+""",
+)
diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl
index f47ea8f..5923787 100644
--- a/python/private/toolchains_repo.bzl
+++ b/python/private/toolchains_repo.bzl
@@ -83,6 +83,15 @@
toolchain = "@{user_repository_name}_{platform}//:python_runtimes",
toolchain_type = "@bazel_tools//tools/python:toolchain_type",
)
+
+toolchain(
+ name = "{prefix}{platform}_py_cc_toolchain",
+ target_compatible_with = {compatible_with},
+ target_settings = {target_settings},
+ toolchain = "@{user_repository_name}_{platform}//:py_cc_toolchain",
+ toolchain_type = "@rules_python//python/cc:toolchain_type",
+
+)
""".format(
compatible_with = meta.compatible_with,
platform = platform,
diff --git a/python/private/util.bzl b/python/private/util.bzl
index f0d4373..4c4b8fc 100644
--- a/python/private/util.bzl
+++ b/python/private/util.bzl
@@ -1,7 +1,25 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
"""Functionality shared by multiple pieces of code."""
load("@bazel_skylib//lib:types.bzl", "types")
+# When bzlmod is enabled, canonical repos names have @@ in them, while under
+# workspace builds, there is never a @@ in labels.
+BZLMOD_ENABLED = "@@" in str(Label("//:unused"))
+
def copy_propagating_kwargs(from_kwargs, into_kwargs = None):
"""Copies args that must be compatible between two targets with a dependency relationship.
@@ -46,15 +64,26 @@
Returns:
The same `attrs` object, but modified.
"""
+ add_tag(attrs, _MIGRATION_TAG)
+ return attrs
+
+def add_tag(attrs, tag):
+ """Adds `tag` to `attrs["tags"]`.
+
+ Args:
+ attrs: dict of keyword args. It is modified in place.
+ tag: str, the tag to add.
+ """
if "tags" in attrs and attrs["tags"] != None:
tags = attrs["tags"]
# Preserve the input type: this allows a test verifying the underlying
# rule can accept the tuple for the tags argument.
if types.is_tuple(tags):
- attrs["tags"] = tags + (_MIGRATION_TAG,)
+ attrs["tags"] = tags + (tag,)
else:
- attrs["tags"] = tags + [_MIGRATION_TAG]
+ # List concatenation is necessary because the original value
+ # may be a frozen list.
+ attrs["tags"] = tags + [tag]
else:
- attrs["tags"] = [_MIGRATION_TAG]
- return attrs
+ attrs["tags"] = [tag]
diff --git a/python/repositories.bzl b/python/repositories.bzl
index 04de657..38a580e 100644
--- a/python/repositories.bzl
+++ b/python/repositories.bzl
@@ -265,6 +265,7 @@
# Generated by python/repositories.bzl
load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair")
+load("@rules_python//python/cc:py_cc_toolchain.bzl", "py_cc_toolchain")
package(default_visibility = ["//visibility:public"])
@@ -336,6 +337,12 @@
py2_runtime = None,
py3_runtime = ":py3_runtime",
)
+
+py_cc_toolchain(
+ name = "py_cc_toolchain",
+ headers = ":python_headers",
+ python_version = "{python_version}",
+)
""".format(
glob_exclude = repr(glob_exclude),
glob_include = repr(glob_include),
