internal(config_settings): make config_setting creation reusable (#1750)
The PR #1743 explored the idea of creating extra config settings for
each target platform that our toolchain is targetting, however that has
a drawback of not being usable in `bzlmod` if someone built Python for
a platform that we don't provide a toolchain for and tried to use the
`pip.parse` machinery with that by providing the
`python_interpreter_target`. That is a niche usecase, but `rules_python`
is a core ruleset that should only provide abstractions/helpers that
work in all cases or make it possible to extend things.
This explores a way to decouple the definition of the available
`config_settings` values and how they are constructed by adding an extra
`is_python_config_setting` macro, that could be used to declare the
config settings from within the `pip.parse` hub repo. This makes the
work in #1744 to support whl-only hub repos more self-contained.
Supersedes #1743.
---------
Co-authored-by: Richard Levasseur <rlevasseur@google.com>
diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel
index 4f12ef4..a017f97 100644
--- a/python/config_settings/BUILD.bazel
+++ b/python/config_settings/BUILD.bazel
@@ -1,4 +1,3 @@
-load("//python:versions.bzl", "TOOL_VERSIONS")
load(":config_settings.bzl", "construct_config_settings")
filegroup(
@@ -12,5 +11,4 @@
construct_config_settings(
name = "construct_config_settings",
- python_versions = TOOL_VERSIONS.keys(),
)
diff --git a/python/config_settings/config_settings.bzl b/python/config_settings/config_settings.bzl
index 9e6bbd6..f1d2ff0 100644
--- a/python/config_settings/config_settings.bzl
+++ b/python/config_settings/config_settings.bzl
@@ -15,105 +15,16 @@
"""This module is used to construct the config settings in the BUILD file in this same package.
"""
-load("@bazel_skylib//lib:selects.bzl", "selects")
-load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
-load("//python:versions.bzl", "MINOR_MAPPING")
+load(
+ "//python/private:config_settings.bzl",
+ _construct_config_settings = "construct_config_settings",
+ _is_python_config_setting = "is_python_config_setting",
+)
-def construct_config_settings(name, python_versions):
- """Constructs a set of configs for all Python versions.
+# This is exposed only for cases where the pip hub repo needs to use this rule
+# to define hub-repo scoped config_settings for platform specific wheel
+# support.
+is_python_config_setting = _is_python_config_setting
- Args:
- name: str, unused; only specified to satisfy buildifier lint checks
- and allow programatic modification of the target.
- python_versions: list of all (x.y.z) Python versions supported by rules_python.
- """
-
- # Maps e.g. "3.8" -> ["3.8.1", "3.8.2", etc]
- minor_to_micro_versions = {}
-
- allowed_flag_values = []
- for micro_version in python_versions:
- minor, _, _ = micro_version.rpartition(".")
- minor_to_micro_versions.setdefault(minor, []).append(micro_version)
- allowed_flag_values.append(micro_version)
-
- allowed_flag_values.extend(list(minor_to_micro_versions))
-
- string_flag(
- name = "python_version",
- # TODO: The default here should somehow match the MODULE config. Until
- # then, use the empty string to indicate an unknown version. This
- # also prevents version-unaware targets from inadvertently matching
- # a select condition when they shouldn't.
- build_setting_default = "",
- values = [""] + sorted(allowed_flag_values),
- visibility = ["//visibility:public"],
- )
-
- for minor_version, micro_versions in minor_to_micro_versions.items():
- # This matches the raw flag value, e.g. --//python/config_settings:python_version=3.8
- # It's private because matching the concept of e.g. "3.8" value is done
- # using the `is_python_X.Y` config setting group, which is aware of the
- # minor versions that could match instead.
- equals_minor_version_name = "_python_version_flag_equals_" + minor_version
- native.config_setting(
- name = equals_minor_version_name,
- flag_values = {":python_version": minor_version},
- )
- matches_minor_version_names = [equals_minor_version_name]
-
- default_micro_version = MINOR_MAPPING[minor_version]
-
- for micro_version in micro_versions:
- is_micro_version_name = "is_python_" + micro_version
- if default_micro_version != micro_version:
- native.config_setting(
- name = is_micro_version_name,
- flag_values = {":python_version": micro_version},
- visibility = ["//visibility:public"],
- )
- matches_minor_version_names.append(is_micro_version_name)
- continue
-
- # Ensure that is_python_3.9.8 is matched if python_version is set
- # to 3.9 if MINOR_MAPPING points to 3.9.8
- equals_micro_name = "_python_version_flag_equals_" + micro_version
- native.config_setting(
- name = equals_micro_name,
- flag_values = {":python_version": micro_version},
- )
-
- # An alias pointing to an underscore-prefixed config_setting_group
- # is used because config_setting_group creates
- # `is_{minor}_N` targets, which are easily confused with the
- # `is_{minor}.{micro}` (dot) targets.
- selects.config_setting_group(
- name = "_" + is_micro_version_name,
- match_any = [
- equals_micro_name,
- equals_minor_version_name,
- ],
- )
- native.alias(
- name = is_micro_version_name,
- actual = "_" + is_micro_version_name,
- visibility = ["//visibility:public"],
- )
- matches_minor_version_names.append(equals_micro_name)
-
- # This is prefixed with an underscore to prevent confusion due to how
- # config_setting_group is implemented and how our micro-version targets
- # are named. config_setting_group will generate targets like
- # "is_python_3.10_1" (where the `_N` suffix is len(match_any).
- # Meanwhile, the micro-version tarets are named "is_python_3.10.1" --
- # just a single dot vs underscore character difference.
- selects.config_setting_group(
- name = "_is_python_" + minor_version,
- match_any = matches_minor_version_names,
- )
-
- native.alias(
- name = "is_python_" + minor_version,
- actual = "_is_python_" + minor_version,
- visibility = ["//visibility:public"],
- )
+# This is exposed for usage in rules_python only.
+construct_config_settings = _construct_config_settings
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
index 221c3b7..d3d6e76 100644
--- a/python/private/BUILD.bazel
+++ b/python/private/BUILD.bazel
@@ -69,6 +69,15 @@
)
bzl_library(
+ name = "config_settings_bzl",
+ srcs = ["config_settings.bzl"],
+ deps = [
+ "//python:versions_bzl",
+ "@bazel_skylib//lib:selects",
+ ],
+)
+
+bzl_library(
name = "coverage_deps_bzl",
srcs = ["coverage_deps.bzl"],
deps = [
diff --git a/python/private/config_settings.bzl b/python/private/config_settings.bzl
new file mode 100644
index 0000000..75f88de
--- /dev/null
+++ b/python/private/config_settings.bzl
@@ -0,0 +1,190 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This module is used to construct the config settings in the BUILD file in this same package.
+"""
+
+load("@bazel_skylib//lib:selects.bzl", "selects")
+load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
+load("//python:versions.bzl", "MINOR_MAPPING", "TOOL_VERSIONS")
+
+_PYTHON_VERSION_FLAG = str(Label("//python/config_settings:python_version"))
+
+def _ver_key(s):
+ major, _, s = s.partition(".")
+ minor, _, s = s.partition(".")
+ micro, _, s = s.partition(".")
+ return (int(major), int(minor), int(micro))
+
+def _flag_values(python_versions):
+ """Construct a map of python_version to a list of toolchain values.
+
+ This mapping maps the concept of a config setting to a list of compatible toolchain versions.
+ For using this in the code, the VERSION_FLAG_VALUES should be used instead.
+
+ Args:
+ python_versions: list of strings; all X.Y.Z python versions
+
+ Returns:
+ A `map[str, list[str]]`. Each key is a python_version flag value. Each value
+ is a list of the python_version flag values that should match when for the
+ `key`. For example:
+ ```
+ "3.8" -> ["3.8", "3.8.1", "3.8.2", ..., "3.8.19"] # All 3.8 versions
+ "3.8.2" -> ["3.8.2"] # Only 3.8.2
+ "3.8.19" -> ["3.8.19", "3.8"] # The latest version should also match 3.8 so
+ as when the `3.8` toolchain is used we just use the latest `3.8` toolchain.
+ this makes the `select("is_python_3.8.19")` work no matter how the user
+ specifies the latest python version to use.
+ ```
+ """
+ ret = {}
+
+ for micro_version in sorted(python_versions, key = _ver_key):
+ minor_version, _, _ = micro_version.rpartition(".")
+
+ # This matches the raw flag value, e.g. --//python/config_settings:python_version=3.8
+ # It's private because matching the concept of e.g. "3.8" value is done
+ # using the `is_python_X.Y` config setting group, which is aware of the
+ # minor versions that could match instead.
+ ret.setdefault(minor_version, [minor_version]).append(micro_version)
+
+ # Ensure that is_python_3.9.8 is matched if python_version is set
+ # to 3.9 if MINOR_MAPPING points to 3.9.8
+ default_micro_version = MINOR_MAPPING[minor_version]
+ ret[micro_version] = [micro_version, minor_version] if default_micro_version == micro_version else [micro_version]
+
+ return ret
+
+VERSION_FLAG_VALUES = _flag_values(TOOL_VERSIONS.keys())
+
+def is_python_config_setting(name, *, python_version, reuse_conditions = None, **kwargs):
+ """Create a config setting for matching 'python_version' configuration flag.
+
+ This function is mainly intended for internal use within the `whl_library` and `pip_parse`
+ machinery.
+
+ The matching of the 'python_version' flag depends on the value passed in
+ `python_version` and here is the example for `3.8` (but the same applies
+ to other python versions present in @//python:versions.bzl#TOOL_VERSIONS):
+ * "3.8" -> ["3.8", "3.8.1", "3.8.2", ..., "3.8.19"] # All 3.8 versions
+ * "3.8.2" -> ["3.8.2"] # Only 3.8.2
+ * "3.8.19" -> ["3.8.19", "3.8"] # The latest version should also match 3.8 so
+ as when the `3.8` toolchain is used we just use the latest `3.8` toolchain.
+ this makes the `select("is_python_3.8.19")` work no matter how the user
+ specifies the latest python version to use.
+
+ Args:
+ name: name for the target that will be created to be used in select statements.
+ python_version: The python_version to be passed in the `flag_values` in the
+ `config_setting`. Depending on the version, the matching python version list
+ can be as described above.
+ reuse_conditions: A dict of version to version label for which we should
+ reuse config_setting targets instead of creating them from scratch. This
+ is useful when using is_python_config_setting multiple times in the
+ same package with the same `major.minor` python versions.
+ **kwargs: extra kwargs passed to the `config_setting`.
+ """
+ if python_version not in name:
+ fail("The name '{}' must have the python version '{}' in it".format(name, python_version))
+
+ if python_version not in VERSION_FLAG_VALUES:
+ fail("The 'python_version' must be known to 'rules_python', choose from the values: {}".format(VERSION_FLAG_VALUES.keys()))
+
+ python_versions = VERSION_FLAG_VALUES[python_version]
+ if len(python_versions) == 1:
+ native.config_setting(
+ name = name,
+ flag_values = {
+ _PYTHON_VERSION_FLAG: python_version,
+ },
+ **kwargs
+ )
+ return
+
+ reuse_conditions = reuse_conditions or {}
+ create_config_settings = {
+ "_{}".format(name).replace(python_version, version): {_PYTHON_VERSION_FLAG: version}
+ for version in python_versions
+ if not reuse_conditions or version not in reuse_conditions
+ }
+ match_any = list(create_config_settings.keys())
+ for version, condition in reuse_conditions.items():
+ if len(VERSION_FLAG_VALUES[version]) == 1:
+ match_any.append(condition)
+ continue
+
+ # Convert the name to an internal label that this function would create,
+ # so that we are hitting the config_setting and not the config_setting_group.
+ condition = Label(condition)
+ if hasattr(condition, "same_package_label"):
+ condition = condition.same_package_label("_" + condition.name)
+ else:
+ condition = condition.relative("_" + condition.name)
+
+ match_any.append(condition)
+
+ for name_, flag_values_ in create_config_settings.items():
+ native.config_setting(
+ name = name_,
+ flag_values = flag_values_,
+ **kwargs
+ )
+
+ # An alias pointing to an underscore-prefixed config_setting_group
+ # is used because config_setting_group creates
+ # `is_{version}_N` targets, which are easily confused with the
+ # `is_{minor}.{micro}` (dot) targets.
+ selects.config_setting_group(
+ name = "_{}_group".format(name),
+ match_any = match_any,
+ visibility = ["//visibility:private"],
+ )
+ native.alias(
+ name = name,
+ actual = "_{}_group".format(name),
+ visibility = kwargs.get("visibility", []),
+ )
+
+def construct_config_settings(name = None): # buildifier: disable=function-docstring
+ """Create a 'python_version' config flag and construct all config settings used in rules_python.
+
+ This mainly includes the targets that are used in the toolchain and pip hub
+ repositories that only match on the 'python_version' flag values.
+
+ Args:
+ name(str): A dummy name value that is no-op for now.
+ """
+ string_flag(
+ name = "python_version",
+ # TODO: The default here should somehow match the MODULE config. Until
+ # then, use the empty string to indicate an unknown version. This
+ # also prevents version-unaware targets from inadvertently matching
+ # a select condition when they shouldn't.
+ build_setting_default = "",
+ values = [""] + VERSION_FLAG_VALUES.keys(),
+ visibility = ["//visibility:public"],
+ )
+
+ for version, matching_versions in VERSION_FLAG_VALUES.items():
+ is_python_config_setting(
+ name = "is_python_{}".format(version),
+ python_version = version,
+ reuse_conditions = {
+ v: native.package_relative_label("is_python_{}".format(v))
+ for v in matching_versions
+ if v != version
+ },
+ visibility = ["//visibility:public"],
+ )
diff --git a/tests/config_settings/construct_config_settings_tests.bzl b/tests/config_settings/construct_config_settings_tests.bzl
index 61beb9c..b1b2e06 100644
--- a/tests/config_settings/construct_config_settings_tests.bzl
+++ b/tests/config_settings/construct_config_settings_tests.bzl
@@ -18,6 +18,7 @@
load("@rules_testing//lib:test_suite.bzl", "test_suite")
load("@rules_testing//lib:truth.bzl", "subjects")
load("@rules_testing//lib:util.bzl", rt_util = "util")
+load("//python/config_settings:config_settings.bzl", "is_python_config_setting")
_tests = []
@@ -28,25 +29,51 @@
_subject = rule(
implementation = _subject_impl,
attrs = {
+ "match_cpu": attr.string(),
"match_micro": attr.string(),
"match_minor": attr.string(),
+ "match_os": attr.string(),
+ "match_os_cpu": attr.string(),
"no_match": attr.string(),
"no_match_micro": attr.string(),
},
)
def _test_minor_version_matching(name):
+ minor_matches = {
+ # Having it here ensures that we can mix and match config settings defined in
+ # the repo and elsewhere
+ str(Label("//python/config_settings:is_python_3.11")): "matched-3.11",
+ "//conditions:default": "matched-default",
+ }
+ minor_cpu_matches = {
+ str(Label(":is_python_3.11_aarch64")): "matched-3.11-aarch64",
+ str(Label(":is_python_3.11_ppc")): "matched-3.11-ppc",
+ str(Label(":is_python_3.11_s390x")): "matched-3.11-s390x",
+ str(Label(":is_python_3.11_x86_64")): "matched-3.11-x86_64",
+ }
+ minor_os_matches = {
+ str(Label(":is_python_3.11_linux")): "matched-3.11-linux",
+ str(Label(":is_python_3.11_osx")): "matched-3.11-osx",
+ str(Label(":is_python_3.11_windows")): "matched-3.11-windows",
+ }
+ minor_os_cpu_matches = {
+ str(Label(":is_python_3.11_linux_aarch64")): "matched-3.11-linux-aarch64",
+ str(Label(":is_python_3.11_linux_ppc")): "matched-3.11-linux-ppc",
+ str(Label(":is_python_3.11_linux_s390x")): "matched-3.11-linux-s390x",
+ str(Label(":is_python_3.11_linux_x86_64")): "matched-3.11-linux-x86_64",
+ str(Label(":is_python_3.11_osx_aarch64")): "matched-3.11-osx-aarch64",
+ str(Label(":is_python_3.11_osx_x86_64")): "matched-3.11-osx-x86_64",
+ str(Label(":is_python_3.11_windows_x86_64")): "matched-3.11-windows-x86_64",
+ }
+
rt_util.helper_target(
_subject,
name = name + "_subject",
- match_minor = select({
- "//python/config_settings:is_python_3.11": "matched-3.11",
- "//conditions:default": "matched-default",
- }),
- match_micro = select({
- "//python/config_settings:is_python_3.11": "matched-3.11",
- "//conditions:default": "matched-default",
- }),
+ match_minor = select(minor_matches),
+ match_cpu = select(minor_matches | minor_cpu_matches),
+ match_os = select(minor_matches | minor_os_matches),
+ match_os_cpu = select(minor_matches | minor_cpu_matches | minor_os_matches | minor_os_cpu_matches),
no_match = select({
"//python/config_settings:is_python_3.12": "matched-3.12",
"//conditions:default": "matched-default",
@@ -59,16 +86,23 @@
impl = _test_minor_version_matching_impl,
config_settings = {
str(Label("//python/config_settings:python_version")): "3.11.1",
+ "//command_line_option:platforms": str(Label("//tests/config_settings:linux_aarch64")),
},
)
def _test_minor_version_matching_impl(env, target):
target = env.expect.that_target(target)
+ target.attr("match_cpu", factory = subjects.str).equals(
+ "matched-3.11-aarch64",
+ )
target.attr("match_minor", factory = subjects.str).equals(
"matched-3.11",
)
- target.attr("match_micro", factory = subjects.str).equals(
- "matched-3.11",
+ target.attr("match_os", factory = subjects.str).equals(
+ "matched-3.11-linux",
+ )
+ target.attr("match_os_cpu", factory = subjects.str).equals(
+ "matched-3.11-linux-aarch64",
)
target.attr("no_match", factory = subjects.str).equals(
"matched-default",
@@ -124,8 +158,54 @@
_tests.append(_test_latest_micro_version_matching)
-def construct_config_settings_test_suite(name):
+def construct_config_settings_test_suite(name): # buildifier: disable=function-docstring
+ # We have CI runners running on a great deal of the platforms from the list below,
+ # hence use all of them within tests.
+ for os in ["linux", "osx", "windows"]:
+ is_python_config_setting(
+ name = "is_python_3.11_" + os,
+ constraint_values = [
+ "@platforms//os:" + os,
+ ],
+ python_version = "3.11",
+ )
+
+ for cpu in ["s390x", "ppc", "x86_64", "aarch64"]:
+ is_python_config_setting(
+ name = "is_python_3.11_" + cpu,
+ constraint_values = [
+ "@platforms//cpu:" + cpu,
+ ],
+ python_version = "3.11",
+ )
+
+ for (os, cpu) in [
+ ("linux", "aarch64"),
+ ("linux", "ppc"),
+ ("linux", "s390x"),
+ ("linux", "x86_64"),
+ ("osx", "aarch64"),
+ ("osx", "x86_64"),
+ ("windows", "x86_64"),
+ ]:
+ is_python_config_setting(
+ name = "is_python_3.11_{}_{}".format(os, cpu),
+ constraint_values = [
+ "@platforms//cpu:" + cpu,
+ "@platforms//os:" + os,
+ ],
+ python_version = "3.11",
+ )
+
test_suite(
name = name,
tests = _tests,
)
+
+ native.platform(
+ name = "linux_aarch64",
+ constraint_values = [
+ "@platforms//os:linux",
+ "@platforms//cpu:aarch64",
+ ],
+ )
