refactor: make python extension generate platform toolchains (#2875)
This makes the python bzlmod extension handle generating the
platform-specific toolchain
entries ("python_3_10_{platform}"). This is to eventually allow adding
additional
toolchains that aren't part of the PLATFORMS mapping in versions.bzl and
have
their own custom constraints.
The main things this refactor does are:
1. The bzlmod phase passes the full list of implementation toolchains
to create (previously, it relied on `hub_repo` to generate the
implementation
names).
2. The name of a toolchain (the toolchain.name arg) and the repo that
implements
it (the py_toolchain_suite.user_repository_repo arg) are separate. This
allows
future work to mixin toolchains that point to arbitrary repos.
3. The platform meta data uses a list of target settings instead of dict
of
flag values. This allows more arbitrary target settings. For now, flag
values
on the platform metadata is still looked for because it's known that
users
patch python/versions.bzl.
Along the way:
* Factor out a platform_info helper in versions.bzl
* Factor out a NOT_ACTUALLY_PUBLIC constants to better denote things
that
are public visibility, but actually internal.
* Add some docs to some internals so we don't have to chase down their
definitions.
Work towards https://github.com/bazel-contrib/rules_python/issues/2081diff --git a/python/private/config_settings.bzl b/python/private/config_settings.bzl
index 1685195..5eb858e 100644
--- a/python/private/config_settings.bzl
+++ b/python/private/config_settings.bzl
@@ -31,6 +31,10 @@
{docs_url}/python/config_settings
"""
+# Indicates something needs public visibility so that other generated code can
+# access it, but it's not intended for general public usage.
+_NOT_ACTUALLY_PUBLIC = ["//visibility:public"]
+
def construct_config_settings(*, name, default_version, versions, minor_mapping, documented_flags): # buildifier: disable=function-docstring
"""Create a 'python_version' config flag and construct all config settings used in rules_python.
@@ -128,7 +132,30 @@
# `whl_library` in the hub repo created by `pip.parse`.
flag_values = {"current_config": "will-never-match"},
# Only public so that PyPI hub repo can access it
- visibility = ["//visibility:public"],
+ visibility = _NOT_ACTUALLY_PUBLIC,
+ )
+
+ libc = Label("//python/config_settings:py_linux_libc")
+ native.config_setting(
+ name = "_is_py_linux_libc_glibc",
+ flag_values = {libc: "glibc"},
+ visibility = _NOT_ACTUALLY_PUBLIC,
+ )
+ native.config_setting(
+ name = "_is_py_linux_libc_musl",
+ flag_values = {libc: "glibc"},
+ visibility = _NOT_ACTUALLY_PUBLIC,
+ )
+ freethreaded = Label("//python/config_settings:py_freethreaded")
+ native.config_setting(
+ name = "_is_py_freethreaded_yes",
+ flag_values = {freethreaded: "yes"},
+ visibility = _NOT_ACTUALLY_PUBLIC,
+ )
+ native.config_setting(
+ name = "_is_py_freethreaded_no",
+ flag_values = {freethreaded: "no"},
+ visibility = _NOT_ACTUALLY_PUBLIC,
)
def _python_version_flag_impl(ctx):
diff --git a/python/private/py_repositories.bzl b/python/private/py_repositories.bzl
index 46ca903..b5bd93b 100644
--- a/python/private/py_repositories.bzl
+++ b/python/private/py_repositories.bzl
@@ -39,11 +39,15 @@
name = "pythons_hub",
minor_mapping = MINOR_MAPPING,
default_python_version = "",
- toolchain_prefixes = [],
- toolchain_python_versions = [],
- toolchain_set_python_version_constraints = [],
- toolchain_user_repository_names = [],
python_versions = sorted(TOOL_VERSIONS.keys()),
+ toolchain_names = [],
+ toolchain_repo_names = {},
+ toolchain_target_compatible_with_map = {},
+ toolchain_target_settings_map = {},
+ toolchain_platform_keys = {},
+ toolchain_python_versions = {},
+ toolchain_set_python_version_constraints = {},
+ base_toolchain_repo_names = [],
)
http_archive(
name = "bazel_skylib",
diff --git a/python/private/py_toolchain_suite.bzl b/python/private/py_toolchain_suite.bzl
index e71882d..fa73d5d 100644
--- a/python/private/py_toolchain_suite.bzl
+++ b/python/private/py_toolchain_suite.bzl
@@ -34,15 +34,20 @@
python_version,
set_python_version_constraint,
flag_values,
+ target_settings = [],
target_compatible_with = []):
"""For internal use only.
Args:
prefix: Prefix for toolchain target names.
- user_repository_name: The name of the user repository.
+ user_repository_name: The name of the repository with the toolchain
+ implementation (it's assumed to have particular target names within
+ it). Does not include the leading "@".
python_version: The full (X.Y.Z) version of the interpreter.
set_python_version_constraint: True or False as a string.
- flag_values: Extra flag values to match for this toolchain.
+ flag_values: Extra flag values to match for this toolchain. These
+ are prepended to target_settings.
+ target_settings: Extra target_settings to match for this toolchain.
target_compatible_with: list constraints the toolchains are compatible with.
"""
@@ -82,7 +87,7 @@
match_any = match_any,
visibility = ["//visibility:private"],
)
- target_settings = [name]
+ target_settings = [name] + target_settings
else:
fail(("Invalid set_python_version_constraint value: got {} {}, wanted " +
"either the string 'True' or the string 'False'; " +
diff --git a/python/private/python.bzl b/python/private/python.bzl
index f49fb26..53cd5e9 100644
--- a/python/private/python.bzl
+++ b/python/private/python.bzl
@@ -22,15 +22,9 @@
load(":pythons_hub.bzl", "hub_repo")
load(":repo_utils.bzl", "repo_utils")
load(":semver.bzl", "semver")
-load(":text_util.bzl", "render")
load(":toolchains_repo.bzl", "multi_toolchain_aliases")
load(":util.bzl", "IS_BAZEL_6_4_OR_HIGHER")
-# This limit can be increased essentially arbitrarily, but doing so will cause a rebuild of all
-# targets using any of these toolchains due to the changed repository name.
-_MAX_NUM_TOOLCHAINS = 9999
-_TOOLCHAIN_INDEX_PAD_LENGTH = len(str(_MAX_NUM_TOOLCHAINS))
-
def parse_modules(*, module_ctx, _fail = fail):
"""Parse the modules and return a struct for registrations.
@@ -240,9 +234,6 @@
# toolchain. We need the default last.
toolchains.append(default_toolchain)
- if len(toolchains) > _MAX_NUM_TOOLCHAINS:
- fail("more than {} python versions are not supported".format(_MAX_NUM_TOOLCHAINS))
-
# sort the toolchains so that the toolchain versions that are in the
# `minor_mapping` are coming first. This ensures that `python_version =
# "3.X"` transitions work as expected.
@@ -275,6 +266,9 @@
def _python_impl(module_ctx):
py = parse_modules(module_ctx = module_ctx)
+ # dict[str version, list[str] platforms]; where version is full
+ # python version string ("3.4.5"), and platforms are keys from
+ # the PLATFORMS global.
loaded_platforms = {}
for toolchain_info in py.toolchains:
# Ensure that we pass the full version here.
@@ -297,30 +291,82 @@
**kwargs
)
- # Create the pythons_hub repo for the interpreter meta data and the
- # the various toolchains.
+ # List of the base names ("python_3_10") for the toolchain repos
+ base_toolchain_repo_names = []
+
+ # list[str] The infix to use for the resulting toolchain() `name` arg.
+ toolchain_names = []
+
+ # dict[str i, str repo]; where repo is the full repo name
+ # ("python_3_10_unknown-linux-x86_64") for the toolchain
+ # i corresponds to index `i` in toolchain_names
+ toolchain_repo_names = {}
+
+ # dict[str i, list[str] constraints]; where constraints is a list
+ # of labels for target_compatible_with
+ # i corresponds to index `i` in toolchain_names
+ toolchain_tcw_map = {}
+
+ # dict[str i, list[str] settings]; where settings is a list
+ # of labels for target_settings
+ # i corresponds to index `i` in toolchain_names
+ toolchain_ts_map = {}
+
+ # dict[str i, str set_constraint]; where set_constraint is the string
+ # "True" or "False".
+ # i corresponds to index `i` in toolchain_names
+ toolchain_set_python_version_constraints = {}
+
+ # dict[str i, str python_version]; where python_version is the full
+ # python version ("3.4.5").
+ toolchain_python_versions = {}
+
+ # dict[str i, str platform_key]; where platform_key is the key within
+ # the PLATFORMS global for this toolchain
+ toolchain_platform_keys = {}
+
+ # Split the toolchain info into separate objects so they can be passed onto
+ # the repository rule.
+ for i, t in enumerate(py.toolchains):
+ is_last = (i + 1) == len(py.toolchains)
+ base_name = t.name
+ base_toolchain_repo_names.append(base_name)
+ fv = full_version(version = t.python_version, minor_mapping = py.config.minor_mapping)
+ for platform in loaded_platforms[fv]:
+ if platform not in PLATFORMS:
+ continue
+ key = str(len(toolchain_names))
+
+ full_name = "{}_{}".format(base_name, platform)
+ toolchain_names.append(full_name)
+ toolchain_repo_names[key] = full_name
+ toolchain_tcw_map[key] = PLATFORMS[platform].compatible_with
+
+ # The target_settings attribute may not be present for users
+ # patching python/versions.bzl.
+ toolchain_ts_map[key] = getattr(PLATFORMS[platform], "target_settings", [])
+ toolchain_platform_keys[key] = platform
+ toolchain_python_versions[key] = fv
+
+ # The last toolchain is the default; it can't have version constraints
+ # Despite the implication of the arg name, the values are strs, not bools
+ toolchain_set_python_version_constraints[key] = (
+ "True" if not is_last else "False"
+ )
+
hub_repo(
name = "pythons_hub",
- # Last toolchain is default
+ toolchain_names = toolchain_names,
+ toolchain_repo_names = toolchain_repo_names,
+ toolchain_target_compatible_with_map = toolchain_tcw_map,
+ toolchain_target_settings_map = toolchain_ts_map,
+ toolchain_platform_keys = toolchain_platform_keys,
+ toolchain_python_versions = toolchain_python_versions,
+ toolchain_set_python_version_constraints = toolchain_set_python_version_constraints,
+ base_toolchain_repo_names = [t.name for t in py.toolchains],
default_python_version = py.default_python_version,
minor_mapping = py.config.minor_mapping,
python_versions = list(py.config.default["tool_versions"].keys()),
- toolchain_prefixes = [
- render.toolchain_prefix(index, toolchain.name, _TOOLCHAIN_INDEX_PAD_LENGTH)
- for index, toolchain in enumerate(py.toolchains)
- ],
- toolchain_python_versions = [
- full_version(version = t.python_version, minor_mapping = py.config.minor_mapping)
- for t in py.toolchains
- ],
- # The last toolchain is the default; it can't have version constraints
- # Despite the implication of the arg name, the values are strs, not bools
- toolchain_set_python_version_constraints = [
- "True" if i != len(py.toolchains) - 1 else "False"
- for i in range(len(py.toolchains))
- ],
- toolchain_user_repository_names = [t.name for t in py.toolchains],
- loaded_platforms = loaded_platforms,
)
# This is require in order to support multiple version py_test
diff --git a/python/private/pythons_hub.bzl b/python/private/pythons_hub.bzl
index b448d53..53351ca 100644
--- a/python/private/pythons_hub.bzl
+++ b/python/private/pythons_hub.bzl
@@ -16,7 +16,7 @@
load("//python:versions.bzl", "PLATFORMS")
load(":text_util.bzl", "render")
-load(":toolchains_repo.bzl", "python_toolchain_build_file_content")
+load(":toolchains_repo.bzl", "toolchain_suite_content")
def _have_same_length(*lists):
if not lists:
@@ -24,8 +24,10 @@
return len({len(length): None for length in lists}) == 1
_HUB_BUILD_FILE_TEMPLATE = """\
-load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+# Generated by @rules_python//python/private:pythons_hub.bzl
+
load("@@{rules_python}//python/private:py_toolchain_suite.bzl", "py_toolchain_suite")
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
bzl_library(
name = "interpreters_bzl",
@@ -42,44 +44,43 @@
{toolchains}
"""
-def _hub_build_file_content(
- prefixes,
- python_versions,
- set_python_version_constraints,
- user_repository_names,
- workspace_location,
- loaded_platforms):
- """This macro iterates over each of the lists and returns the toolchain content.
-
- python_toolchain_build_file_content is called to generate each of the toolchain
- definitions.
- """
-
- if not _have_same_length(python_versions, set_python_version_constraints, user_repository_names):
+def _hub_build_file_content(rctx):
+ # Verify a precondition. If these don't match, then something went wrong.
+ if not _have_same_length(
+ rctx.attr.toolchain_names,
+ rctx.attr.toolchain_platform_keys,
+ rctx.attr.toolchain_repo_names,
+ rctx.attr.toolchain_target_compatible_with_map,
+ rctx.attr.toolchain_target_settings_map,
+ rctx.attr.toolchain_set_python_version_constraints,
+ rctx.attr.toolchain_python_versions,
+ ):
fail("all lists must have the same length")
- # Iterate over the length of python_versions and call
- # build the toolchain content by calling python_toolchain_build_file_content
- toolchains = "\n".join(
- [
- python_toolchain_build_file_content(
- prefix = prefixes[i],
- python_version = python_versions[i],
- set_python_version_constraint = set_python_version_constraints[i],
- user_repository_name = user_repository_names[i],
- loaded_platforms = {
- k: v
- for k, v in PLATFORMS.items()
- if k in loaded_platforms[python_versions[i]]
- },
- )
- for i in range(len(python_versions))
- ],
- )
+ #pad_length = len(str(len(rctx.attr.toolchain_names))) + 1
+ pad_length = 4
+ toolchains = []
+ for i, base_name in enumerate(rctx.attr.toolchain_names):
+ key = str(i)
+ platform = rctx.attr.toolchain_platform_keys[key]
+ if platform in PLATFORMS:
+ flag_values = PLATFORMS[platform].flag_values
+ else:
+ flag_values = {}
+
+ toolchains.append(toolchain_suite_content(
+ prefix = "_{}_{}".format(render.left_pad_zero(i, pad_length), base_name),
+ user_repository_name = rctx.attr.toolchain_repo_names[key],
+ target_compatible_with = rctx.attr.toolchain_target_compatible_with_map[key],
+ flag_values = flag_values,
+ target_settings = rctx.attr.toolchain_target_settings_map[key],
+ set_python_version_constraint = rctx.attr.toolchain_set_python_version_constraints[key],
+ python_version = rctx.attr.toolchain_python_versions[key],
+ ))
return _HUB_BUILD_FILE_TEMPLATE.format(
- toolchains = toolchains,
- rules_python = workspace_location.repo_name,
+ toolchains = "\n".join(toolchains),
+ rules_python = rctx.attr._rules_python_workspace.repo_name,
)
_interpreters_bzl_template = """
@@ -103,14 +104,7 @@
# write them to the BUILD file.
rctx.file(
"BUILD.bazel",
- _hub_build_file_content(
- rctx.attr.toolchain_prefixes,
- rctx.attr.toolchain_python_versions,
- rctx.attr.toolchain_set_python_version_constraints,
- rctx.attr.toolchain_user_repository_names,
- rctx.attr._rules_python_workspace,
- rctx.attr.loaded_platforms,
- ),
+ _hub_build_file_content(rctx),
executable = False,
)
@@ -118,7 +112,7 @@
# a symlink to a interpreter.
interpreter_labels = "".join([
_line_for_hub_template.format(name = name)
- for name in rctx.attr.toolchain_user_repository_names
+ for name in rctx.attr.base_toolchain_repo_names
])
rctx.file(
@@ -150,13 +144,15 @@
""",
implementation = _hub_repo_impl,
attrs = {
+ "base_toolchain_repo_names": attr.string_list(
+ doc = "The base repo name for toolchains ('python_3_10', no " +
+ "platform suffix)",
+ mandatory = True,
+ ),
"default_python_version": attr.string(
doc = "Default Python version for the build in `X.Y` or `X.Y.Z` format.",
mandatory = True,
),
- "loaded_platforms": attr.string_list_dict(
- doc = "The list of loaded platforms keyed by the toolchain full python version",
- ),
"minor_mapping": attr.string_dict(
doc = "The minor mapping of the `X.Y` to `X.Y.Z` format that is used in config settings.",
mandatory = True,
@@ -165,20 +161,32 @@
doc = "The list of python versions to include in the `interpreters.bzl` if the toolchains are not specified. Used in `WORKSPACE` builds.",
mandatory = False,
),
- "toolchain_prefixes": attr.string_list(
- doc = "List prefixed for the toolchains",
+ "toolchain_names": attr.string_list(
+ doc = "Names of toolchains",
mandatory = True,
),
- "toolchain_python_versions": attr.string_list(
+ "toolchain_platform_keys": attr.string_dict(
+ doc = "The platform key in PLATFORMS for toolchains.",
+ mandatory = True,
+ ),
+ "toolchain_python_versions": attr.string_dict(
doc = "List of Python versions for the toolchains. In `X.Y.Z` format.",
mandatory = True,
),
- "toolchain_set_python_version_constraints": attr.string_list(
+ "toolchain_repo_names": attr.string_dict(
+ doc = "The repo names containing toolchain implementations.",
+ mandatory = True,
+ ),
+ "toolchain_set_python_version_constraints": attr.string_dict(
doc = "List of version contraints for the toolchains",
mandatory = True,
),
- "toolchain_user_repository_names": attr.string_list(
- doc = "List of the user repo names for the toolchains",
+ "toolchain_target_compatible_with_map": attr.string_list_dict(
+ doc = "The target_compatible_with settings for toolchains.",
+ mandatory = True,
+ ),
+ "toolchain_target_settings_map": attr.string_list_dict(
+ doc = "The target_settings for toolchains",
mandatory = True,
),
"_rules_python_workspace": attr.label(default = Label("//:does_not_matter_what_this_name_is")),
diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl
index 23c4643..d0814b6 100644
--- a/python/private/toolchains_repo.bzl
+++ b/python/private/toolchains_repo.bzl
@@ -31,6 +31,18 @@
load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
load(":text_util.bzl", "render")
+_SUITE_TEMPLATE = """
+py_toolchain_suite(
+ flag_values = {flag_values},
+ target_settings = {target_settings},
+ prefix = {prefix},
+ python_version = {python_version},
+ set_python_version_constraint = {set_python_version_constraint},
+ target_compatible_with = {target_compatible_with},
+ user_repository_name = {user_repository_name},
+)
+""".lstrip()
+
def python_toolchain_build_file_content(
prefix,
python_version,
@@ -53,29 +65,40 @@
build_content: Text containing toolchain definitions
"""
- return "\n\n".join([
- """\
-py_toolchain_suite(
- user_repository_name = "{user_repository_name}_{platform}",
- prefix = "{prefix}{platform}",
- target_compatible_with = {compatible_with},
- flag_values = {flag_values},
- python_version = "{python_version}",
- set_python_version_constraint = "{set_python_version_constraint}",
-)""".format(
- compatible_with = render.indent(render.list(meta.compatible_with)).lstrip(),
- flag_values = render.indent(render.dict(
- meta.flag_values,
- key_repr = lambda x: repr(str(x)), # this is to correctly display labels
- )).lstrip(),
- platform = platform,
- set_python_version_constraint = set_python_version_constraint,
- user_repository_name = user_repository_name,
- prefix = prefix,
+ entries = []
+ for platform, meta in loaded_platforms.items():
+ entries.append(toolchain_suite_content(
+ target_compatible_with = meta.compatible_with,
+ flag_values = meta.flag_values,
+ prefix = "{}{}".format(prefix, platform),
+ user_repository_name = "{}_{}".format(user_repository_name, platform),
python_version = python_version,
- )
- for platform, meta in loaded_platforms.items()
- ])
+ set_python_version_constraint = set_python_version_constraint,
+ target_settings = [],
+ ))
+ return "\n\n".join(entries)
+
+def toolchain_suite_content(
+ *,
+ flag_values,
+ prefix,
+ python_version,
+ set_python_version_constraint,
+ target_compatible_with,
+ target_settings,
+ user_repository_name):
+ return _SUITE_TEMPLATE.format(
+ prefix = render.str(prefix),
+ user_repository_name = render.str(user_repository_name),
+ target_compatible_with = render.indent(render.list(target_compatible_with)).lstrip(),
+ flag_values = render.indent(render.dict(
+ flag_values,
+ key_repr = lambda x: repr(str(x)), # this is to correctly display labels
+ )).lstrip(),
+ target_settings = render.list(target_settings, hanging_indent = " "),
+ set_python_version_constraint = render.str(set_python_version_constraint),
+ python_version = render.str(python_version),
+ )
def _toolchains_repo_impl(rctx):
build_content = """\
