refactor: lookup exec interpreter using toolchain resolution (#1997)
This changes the exec tools toolchain to use toolchain resolution to
find the interpreter.
The main benefit of this is it avoids having to duplicate specifying
where the interpreter
is. Instead, it is automatically found by toolchain resolution.
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 42c5d76..af6e527 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -28,6 +28,9 @@
* `protobuf`/`com_google_protobuf` dependency bumped to `v24.4`
* (bzlmod): optimize the creation of config settings used in pip to
reduce the total number of targets in the hub repo.
+* (toolchains) The exec tools toolchain now finds its interpreter by reusing
+ the regular interpreter toolchain. This avoids having to duplicate specifying
+ where the runtime for the exec tools toolchain is.
### Fixed
* (bzlmod): Targets in `all_requirements` now use the same form as targets returned by the `requirement` macro.
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
index e2a2bc0..cd385e3 100644
--- a/python/private/BUILD.bazel
+++ b/python/private/BUILD.bazel
@@ -16,6 +16,7 @@
load("//python:py_binary.bzl", "py_binary")
load("//python:py_library.bzl", "py_library")
load("//python:versions.bzl", "print_toolchains_checksums")
+load(":py_exec_tools_toolchain.bzl", "current_interpreter_executable")
load(":stamp.bzl", "stamp_build_setting")
package(
@@ -464,3 +465,12 @@
"//tests/entry_points:__pkg__",
],
)
+
+# The current toolchain's interpreter as an excutable, usable with
+# executable=True attributes.
+current_interpreter_executable(
+ name = "current_interpreter_executable",
+ # Not actually public. Only public because it's an implicit dependency of
+ # py_exec_tools_toolchain.
+ visibility = ["//visibility:public"],
+)
diff --git a/python/private/common/py_library.bzl b/python/private/common/py_library.bzl
index 977bdb3..673beed 100644
--- a/python/private/common/py_library.bzl
+++ b/python/private/common/py_library.bzl
@@ -64,6 +64,7 @@
"""
check_native_allowed(ctx)
direct_sources = filter_to_py_srcs(ctx.files.srcs)
+
precompile_result = semantics.maybe_precompile(ctx, direct_sources)
direct_pyc_files = depset(precompile_result.pyc_files)
default_outputs = depset(precompile_result.keep_srcs, transitive = [direct_pyc_files])
diff --git a/python/private/py_exec_tools_info.bzl b/python/private/py_exec_tools_info.bzl
index 3011f53..2998543 100644
--- a/python/private/py_exec_tools_info.bzl
+++ b/python/private/py_exec_tools_info.bzl
@@ -32,12 +32,6 @@
the proper target constraints are being applied when obtaining this from
the toolchain.
""",
- "exec_interpreter_version_info": """
-struct of interpreter version info for `exec_interpreter`. Note this
-is for the exec interpreter, not the target interpreter. For version information
-about the target Python runtime, use the `//python:toolchain_type` toolchain
-information.
-""",
"precompiler": """
Optional Target. The tool to use for generating pyc files. If not available,
precompiling will not be available.
diff --git a/python/private/py_exec_tools_toolchain.bzl b/python/private/py_exec_tools_toolchain.bzl
index 6036db4..5c17b89 100644
--- a/python/private/py_exec_tools_toolchain.bzl
+++ b/python/private/py_exec_tools_toolchain.bzl
@@ -14,15 +14,12 @@
"""Rule that defines a toolchain for build tools."""
-load("//python/private/common:providers.bzl", "interpreter_version_info_struct_from_dict")
+load("//python/private:toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE")
load(":py_exec_tools_info.bzl", "PyExecToolsInfo")
def _py_exec_tools_toolchain_impl(ctx):
return [platform_common.ToolchainInfo(exec_tools = PyExecToolsInfo(
exec_interpreter = ctx.attr.exec_interpreter,
- exec_interpreter_version_info = interpreter_version_info_struct_from_dict(
- ctx.attr.exec_interpreter_version_info,
- ),
precompiler = ctx.attr.precompiler,
))]
@@ -30,13 +27,9 @@
implementation = _py_exec_tools_toolchain_impl,
attrs = {
"exec_interpreter": attr.label(
+ default = "//python/private:current_interpreter_executable",
cfg = "exec",
- allow_files = True,
- doc = "See PyExecToolsInfo.exec_interpreter",
- executable = True,
- ),
- "exec_interpreter_version_info": attr.string_dict(
- doc = "See PyExecToolsInfo.exec_interpreter_version_info",
+ doc = "See PyexecToolsInfo.exec_interpreter.",
),
"precompiler": attr.label(
allow_files = True,
@@ -45,3 +38,26 @@
),
},
)
+
+def _current_interpreter_executable_impl(ctx):
+ toolchain = ctx.toolchains[TARGET_TOOLCHAIN_TYPE]
+ runtime = toolchain.py3_runtime
+ if runtime.interpreter:
+ executable = ctx.actions.declare_file(ctx.label.name)
+ ctx.actions.symlink(output = executable, target_file = runtime.interpreter, is_executable = True)
+ else:
+ executable = ctx.actions.declare_symlink(ctx.label.name)
+ ctx.actions.symlink(output = executable, target_path = runtime.interpreter_path)
+ return [
+ toolchain,
+ DefaultInfo(
+ executable = executable,
+ runfiles = ctx.runfiles([executable], transitive_files = runtime.files),
+ ),
+ ]
+
+current_interpreter_executable = rule(
+ implementation = _current_interpreter_executable_impl,
+ toolchains = [TARGET_TOOLCHAIN_TYPE],
+ executable = True,
+)
diff --git a/python/repositories.bzl b/python/repositories.bzl
index 4ffadd0..245aae2 100644
--- a/python/repositories.bzl
+++ b/python/repositories.bzl
@@ -399,12 +399,6 @@
py_exec_tools_toolchain(
name = "py_exec_tools_toolchain",
- exec_interpreter = "{python_path}",
- exec_interpreter_version_info = {{
- "major": "{interpreter_version_info_major}",
- "minor": "{interpreter_version_info_minor}",
- "micro": "{interpreter_version_info_micro}",
- }},
precompiler = "@rules_python//tools/precompiler:precompiler",
)
""".format(
diff --git a/tests/exec_toolchain_matching/BUILD.bazel b/tests/exec_toolchain_matching/BUILD.bazel
new file mode 100644
index 0000000..ce04bf7
--- /dev/null
+++ b/tests/exec_toolchain_matching/BUILD.bazel
@@ -0,0 +1,76 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("//python/private:py_exec_tools_toolchain.bzl", "py_exec_tools_toolchain") # buildifier: disable=bzl-visibility
+load(
+ ":exec_toolchain_matching_tests.bzl",
+ "define_py_runtime",
+ "exec_toolchain_matching_test_suite",
+)
+
+exec_toolchain_matching_test_suite(
+ name = "exec_toolchain_matching_tests",
+)
+
+define_py_runtime(
+ name = "target_3.12_linux",
+ interpreter_path = "/linux/python3.12",
+ interpreter_version_info = {
+ "major": "3",
+ "minor": "12",
+ },
+)
+
+define_py_runtime(
+ name = "target_3.12_mac",
+ interpreter_path = "/mac/python3.12",
+ interpreter_version_info = {
+ "major": "3",
+ "minor": "12",
+ },
+)
+
+define_py_runtime(
+ name = "target_3.12_any",
+ interpreter_path = "/any/python3.11",
+ interpreter_version_info = {
+ "major": "3",
+ "minor": "11",
+ },
+)
+
+define_py_runtime(
+ name = "target_default",
+ interpreter_path = "/should_not_match_anything",
+ interpreter_version_info = {
+ "major": "-1",
+ "minor": "-1",
+ },
+)
+
+# While these have the same definition, we register duplicates with different
+# names because it makes understanding toolchain resolution easier. Toolchain
+# resolution debug output shows the implementation name, not the toolchain()
+# call that was being evaluated.
+py_exec_tools_toolchain(
+ name = "exec_3.12",
+)
+
+py_exec_tools_toolchain(
+ name = "exec_3.11_any",
+)
+
+py_exec_tools_toolchain(
+ name = "exec_default",
+)
diff --git a/tests/exec_toolchain_matching/exec_toolchain_matching_tests.bzl b/tests/exec_toolchain_matching/exec_toolchain_matching_tests.bzl
new file mode 100644
index 0000000..f6eae5a
--- /dev/null
+++ b/tests/exec_toolchain_matching/exec_toolchain_matching_tests.bzl
@@ -0,0 +1,152 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Starlark tests for PyRuntimeInfo provider."""
+
+load("@rules_testing//lib:analysis_test.bzl", "analysis_test")
+load("@rules_testing//lib:test_suite.bzl", "test_suite")
+load("@rules_testing//lib:util.bzl", rt_util = "util")
+load("//python:py_runtime.bzl", "py_runtime")
+load("//python:py_runtime_pair.bzl", "py_runtime_pair")
+load("//python/private:toolchain_types.bzl", "EXEC_TOOLS_TOOLCHAIN_TYPE", "TARGET_TOOLCHAIN_TYPE") # buildifier: disable=bzl-visibility
+load("//python/private:util.bzl", "IS_BAZEL_7_OR_HIGHER") # buildifier: disable=bzl-visibility
+load("//tests/support:support.bzl", "LINUX", "MAC", "PYTHON_VERSION")
+
+_LookupInfo = provider() # buildifier: disable=provider-params
+
+def _lookup_toolchains_impl(ctx):
+ return [_LookupInfo(
+ target = ctx.toolchains[TARGET_TOOLCHAIN_TYPE],
+ exec = ctx.toolchains[EXEC_TOOLS_TOOLCHAIN_TYPE],
+ )]
+
+_lookup_toolchains = rule(
+ implementation = _lookup_toolchains_impl,
+ toolchains = [TARGET_TOOLCHAIN_TYPE, EXEC_TOOLS_TOOLCHAIN_TYPE],
+ attrs = {"_use_auto_exec_groups": attr.bool(default = True)},
+)
+
+def define_py_runtime(name, **kwargs):
+ py_runtime(
+ name = name + "_runtime",
+ **kwargs
+ )
+ py_runtime_pair(
+ name = name,
+ py3_runtime = name + "_runtime",
+ )
+
+_tests = []
+
+def _test_exec_matches_target_python_version(name):
+ rt_util.helper_target(
+ _lookup_toolchains,
+ name = name + "_subject",
+ )
+
+ # ==== Target toolchains =====
+
+ # This is never matched. It comes first to ensure the python version
+ # constraint is being respected.
+ native.toolchain(
+ name = "00_target_3.11_any",
+ toolchain_type = TARGET_TOOLCHAIN_TYPE,
+ toolchain = ":target_3.12_linux",
+ target_settings = ["//python/config_settings:is_python_3.11"],
+ )
+
+ # This is matched by the top-level target being built in what --platforms
+ # specifies.
+ native.toolchain(
+ name = "10_target_3.12_linux",
+ toolchain_type = TARGET_TOOLCHAIN_TYPE,
+ toolchain = ":target_3.12_linux",
+ target_compatible_with = ["@platforms//os:linux"],
+ target_settings = ["//python/config_settings:is_python_3.12"],
+ )
+
+ # This is matched when the exec config switches to the mac platform and
+ # then looks for a Python runtime for itself.
+ native.toolchain(
+ name = "15_target_3.12_mac",
+ toolchain_type = TARGET_TOOLCHAIN_TYPE,
+ toolchain = ":target_3.12_mac",
+ target_compatible_with = ["@platforms//os:macos"],
+ target_settings = ["//python/config_settings:is_python_3.12"],
+ )
+
+ # This is never matched. It's just here so that toolchains from the
+ # environment don't match.
+ native.toolchain(
+ name = "99_target_default",
+ toolchain_type = TARGET_TOOLCHAIN_TYPE,
+ toolchain = ":target_default",
+ )
+
+ # ==== Exec tools toolchains =====
+
+ # Register a 3.11 before to ensure it the python version is respected
+ native.toolchain(
+ name = "00_exec_3.11_any",
+ toolchain_type = EXEC_TOOLS_TOOLCHAIN_TYPE,
+ toolchain = ":exec_3.11_any",
+ target_settings = ["//python/config_settings:is_python_3.11"],
+ )
+
+ # Note that mac comes first. This is so it matches instead of linux
+ # We only ever look for mac ones, so no need to register others.
+ native.toolchain(
+ name = "10_exec_3.12_mac",
+ toolchain_type = EXEC_TOOLS_TOOLCHAIN_TYPE,
+ toolchain = ":exec_3.12",
+ exec_compatible_with = ["@platforms//os:macos"],
+ target_settings = ["//python/config_settings:is_python_3.12"],
+ )
+
+ # This is never matched. It's just here so that toolchains from the
+ # environment don't match.
+ native.toolchain(
+ name = "99_exec_default",
+ toolchain_type = EXEC_TOOLS_TOOLCHAIN_TYPE,
+ toolchain = ":exec_default",
+ )
+
+ analysis_test(
+ name = name,
+ target = name + "_subject",
+ impl = _test_exec_matches_target_python_version_impl,
+ config_settings = {
+ "//command_line_option:extra_execution_platforms": [str(MAC)],
+ "//command_line_option:extra_toolchains": ["//tests/exec_toolchain_matching:all"],
+ "//command_line_option:platforms": [str(LINUX)],
+ PYTHON_VERSION: "3.12",
+ },
+ )
+
+_tests.append(_test_exec_matches_target_python_version)
+
+def _test_exec_matches_target_python_version_impl(env, target):
+ target_runtime = target[_LookupInfo].target.py3_runtime
+ exec_runtime = target[_LookupInfo].exec.exec_tools.exec_interpreter[platform_common.ToolchainInfo].py3_runtime
+
+ env.expect.that_str(target_runtime.interpreter_path).equals("/linux/python3.12")
+ env.expect.that_str(exec_runtime.interpreter_path).equals("/mac/python3.12")
+
+ if IS_BAZEL_7_OR_HIGHER:
+ target_version = target_runtime.interpreter_version_info
+ exec_version = exec_runtime.interpreter_version_info
+
+ env.expect.that_bool(target_version == exec_version)
+
+def exec_toolchain_matching_test_suite(name):
+ test_suite(name = name, tests = _tests)
diff --git a/tests/support/support.bzl b/tests/support/support.bzl
index efcc43a..2e58203 100644
--- a/tests/support/support.bzl
+++ b/tests/support/support.bzl
@@ -33,6 +33,7 @@
# doesn't accept yet Label objects.
EXEC_TOOLS_TOOLCHAIN = str(Label("//python/config_settings:exec_tools_toolchain"))
PRECOMPILE = str(Label("//python/config_settings:precompile"))
-PYC_COLLECTION = str(Label("//python/config_settings:pyc_collection"))
-PRECOMPILE_SOURCE_RETENTION = str(Label("//python/config_settings:precompile_source_retention"))
PRECOMPILE_ADD_TO_RUNFILES = str(Label("//python/config_settings:precompile_add_to_runfiles"))
+PRECOMPILE_SOURCE_RETENTION = str(Label("//python/config_settings:precompile_source_retention"))
+PYC_COLLECTION = str(Label("//python/config_settings:pyc_collection"))
+PYTHON_VERSION = str(Label("//python/config_settings:python_version"))
