internal: repos to create a toolchain from a locally installed Python (#2000)
This adds the primitives for defining a toolchain based on a locally
installed Python.
Doing this consists of two parts:
* A repo rule to define a Python runtime pointing to a local Python
installation.
* A repo rule to define toolchains for those runtimes.
The runtime repos create platform runtimes, i.e, it sets
py_runtime.interpreter_path.
This means the runtime isn't included in the runfiles.
Note that these repo rules are largely implementation details, and are
definitely not
stable API-wise. Creating public APIs to use them through WORKSPACE or
bzlmod will
be done in a separate change (there's a few design and behavior
questions to discuss).
This is definitely experimental quality. In particular, the code that
tries
to figure out the C headers/libraries is very finicky. I couldn't find
solid docs about
how to do this, and there's a lot of undocumented settings, so what's
there is what
I was able to piece together from my laptop's behavior.
Misc other changes:
* Also fixes a bug if a pyenv-backed interpreter path is used for
precompiling:
pyenv uses `$0` to determine what to re-exec. The
`:current_interpreter_executable`
target used its own name, which pyenv didn't understand.
* The repo logger now also accepts a string. This should help prevent
accidentally
passing a string causing an error. It's also just a bit more convenient
when
doing development.
* Repo loggers will automatically include their rule name and repo name.
This
makes following logging output easier.
* Makes `repo_utils.execute()` report progress.
* Adds `repo_utils.getenv`, `repo_utils.watch`, and
`repo_utils.watch_tree`:
backwards compatibility functions for their `rctx` equivalents.
* Adds `repo_utils.which_unchecked`: calls `which`, but allows for
failure.
* Adds `repo_utils.get_platforms_os_name()`: Returns the name used in
`@platforms` for
the OS reported by `rctx`.
* Makes several repo util functions call `watch()` or `getenv()`, if
available. This
makes repository rules better respect environmental changes.
* Adds more detail to the definition of an in-build vs platform runtime
* Adds a README for the integration tests directory. Setting up and
using one is a bit
more involved than other tests, so some docs help.
* Allows integration tests to specify bazel versions to use.
diff --git a/python/private/full_version.bzl b/python/private/full_version.bzl
index 68c9694..98eeee5 100644
--- a/python/private/full_version.bzl
+++ b/python/private/full_version.bzl
@@ -40,4 +40,4 @@
),
)
else:
- fail("Unknown version format: {}".format(version))
+ fail("Unknown version format: '{}'".format(version))
diff --git a/python/private/get_local_runtime_info.py b/python/private/get_local_runtime_info.py
new file mode 100644
index 0000000..0207f56
--- /dev/null
+++ b/python/private/get_local_runtime_info.py
@@ -0,0 +1,49 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import sys
+import sysconfig
+
+data = {
+ "major": sys.version_info.major,
+ "minor": sys.version_info.minor,
+ "micro": sys.version_info.micro,
+ "include": sysconfig.get_path("include"),
+ "implementation_name": sys.implementation.name,
+}
+
+config_vars = [
+ # The libpythonX.Y.so file. Usually?
+ # It might be a static archive (.a) file instead.
+ "LDLIBRARY",
+ # The directory with library files. Supposedly.
+ # It's not entirely clear how to get the directory with libraries.
+ # There's several types of libraries with different names and a plethora
+ # of settings.
+ # https://stackoverflow.com/questions/47423246/get-pythons-lib-path
+ # For now, it seems LIBDIR has what is needed, so just use that.
+ "LIBDIR",
+ # The versioned libpythonX.Y.so.N file. Usually?
+ # It might be a static archive (.a) file instead.
+ "INSTSONAME",
+ # The libpythonX.so file. Usually?
+ # It might be a static archive (a.) file instead.
+ "PY3LIBRARY",
+ # The platform-specific filename suffix for library files.
+ # Includes the dot, e.g. `.so`
+ "SHLIB_SUFFIX",
+]
+data.update(zip(config_vars, sysconfig.get_config_vars(*config_vars)))
+print(json.dumps(data))
diff --git a/python/private/local_runtime_repo.bzl b/python/private/local_runtime_repo.bzl
new file mode 100644
index 0000000..f6bca6c
--- /dev/null
+++ b/python/private/local_runtime_repo.bzl
@@ -0,0 +1,252 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Create a repository for a locally installed Python runtime."""
+
+load("//python/private:enum.bzl", "enum")
+load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
+
+# buildifier: disable=name-conventions
+_OnFailure = enum(
+ SKIP = "skip",
+ WARN = "warn",
+ FAIL = "fail",
+)
+
+_TOOLCHAIN_IMPL_TEMPLATE = """\
+# Generated by python/private/local_runtime_repo.bzl
+
+load("@rules_python//python/private:local_runtime_repo_setup.bzl", "define_local_runtime_toolchain_impl")
+
+define_local_runtime_toolchain_impl(
+ name = "local_runtime",
+ lib_ext = "{lib_ext}",
+ major = "{major}",
+ minor = "{minor}",
+ micro = "{micro}",
+ interpreter_path = "{interpreter_path}",
+ implementation_name = "{implementation_name}",
+ os = "{os}",
+)
+"""
+
+def _local_runtime_repo_impl(rctx):
+ logger = repo_utils.logger(rctx)
+ on_failure = rctx.attr.on_failure
+
+ platforms_os_name = repo_utils.get_platforms_os_name(rctx)
+ if not platforms_os_name:
+ if on_failure == "fail":
+ fail("Unrecognized host platform '{}': cannot determine OS constraint".format(
+ rctx.os.name,
+ ))
+
+ if on_failure == "warn":
+ logger.warn(lambda: "Unrecognized host platform '{}': cannot determine OS constraint".format(
+ rctx.os.name,
+ ))
+
+ # else, on_failure must be skip
+ rctx.file("BUILD.bazel", _expand_incompatible_template())
+ return
+
+ result = _resolve_interpreter_path(rctx)
+ if not result.resolved_path:
+ if on_failure == "fail":
+ fail("interpreter not found: {}".format(result.describe_failure()))
+
+ if on_failure == "warn":
+ logger.warn(lambda: "interpreter not found: {}".format(result.describe_failure()))
+
+ # else, on_failure must be skip
+ rctx.file("BUILD.bazel", _expand_incompatible_template())
+ return
+ else:
+ interpreter_path = result.resolved_path
+
+ logger.info(lambda: "resolved interpreter {} to {}".format(rctx.attr.interpreter_path, interpreter_path))
+
+ exec_result = repo_utils.execute_unchecked(
+ rctx,
+ op = "local_runtime_repo.GetPythonInfo({})".format(rctx.name),
+ arguments = [
+ interpreter_path,
+ rctx.path(rctx.attr._get_local_runtime_info),
+ ],
+ quiet = True,
+ )
+ if exec_result.return_code != 0:
+ if on_failure == "fail":
+ fail("GetPythonInfo failed: {}".format(exec_result.describe_failure()))
+ if on_failure == "warn":
+ logger.warn(lambda: "GetPythonInfo failed: {}".format(exec_result.describe_failure()))
+
+ # else, on_failure must be skip
+ rctx.file("BUILD.bazel", _expand_incompatible_template())
+ return
+
+ info = json.decode(exec_result.stdout)
+ logger.info(lambda: _format_get_info_result(info))
+
+ # NOTE: Keep in sync with recursive glob in define_local_runtime_toolchain_impl
+ repo_utils.watch_tree(rctx, rctx.path(info["include"]))
+
+ # The cc_library.includes values have to be non-absolute paths, otherwise
+ # the toolchain will give an error. Work around this error by making them
+ # appear as part of this repo.
+ rctx.symlink(info["include"], "include")
+
+ shared_lib_names = [
+ info["PY3LIBRARY"],
+ info["LDLIBRARY"],
+ info["INSTSONAME"],
+ ]
+
+ # In some cases, the value may be empty. Not clear why.
+ shared_lib_names = [v for v in shared_lib_names if v]
+
+ # In some cases, the same value is returned for multiple keys. Not clear why.
+ shared_lib_names = {v: None for v in shared_lib_names}.keys()
+ shared_lib_dir = info["LIBDIR"]
+
+ # The specific files are symlinked instead of the whole directory
+ # because it can point to a directory that has more than just
+ # the Python runtime shared libraries, e.g. /usr/lib, or a Python
+ # specific directory with pip-installed shared libraries.
+ rctx.report_progress("Symlinking external Python shared libraries")
+ for name in shared_lib_names:
+ origin = rctx.path("{}/{}".format(shared_lib_dir, name))
+
+ # The reported names don't always exist; it depends on the particulars
+ # of the runtime installation.
+ if origin.exists:
+ repo_utils.watch(rctx, origin)
+ rctx.symlink(origin, "lib/" + name)
+
+ rctx.file("WORKSPACE", "")
+ rctx.file("MODULE.bazel", "")
+ rctx.file("REPO.bazel", "")
+ rctx.file("BUILD.bazel", _TOOLCHAIN_IMPL_TEMPLATE.format(
+ major = info["major"],
+ minor = info["minor"],
+ micro = info["micro"],
+ interpreter_path = interpreter_path,
+ lib_ext = info["SHLIB_SUFFIX"],
+ implementation_name = info["implementation_name"],
+ os = "@platforms//os:{}".format(repo_utils.get_platforms_os_name(rctx)),
+ ))
+
+local_runtime_repo = repository_rule(
+ implementation = _local_runtime_repo_impl,
+ doc = """
+Use a locally installed Python runtime as a toolchain implementation.
+
+Note this uses the runtime as a *platform runtime*. A platform runtime means
+means targets don't include the runtime itself as part of their runfiles or
+inputs. Instead, users must assure that where the targets run have the runtime
+pre-installed or otherwise available.
+
+This results in lighter weight binaries (in particular, Bazel doesn't have to
+create thousands of files for every `py_test`), at the risk of having to rely on
+a system having the necessary Python installed.
+""",
+ attrs = {
+ "interpreter_path": attr.string(
+ doc = """
+An absolute path or program name on the `PATH` env var.
+
+Values with slashes are assumed to be the path to a program. Otherwise, it is
+treated as something to search for on `PATH`
+
+Note that, when a plain program name is used, the path to the interpreter is
+resolved at repository evalution time, not runtime of any resulting binaries.
+""",
+ default = "python3",
+ ),
+ "on_failure": attr.string(
+ default = _OnFailure.SKIP,
+ values = sorted(_OnFailure.__members__.values()),
+ doc = """
+How to handle errors when trying to automatically determine settings.
+
+* `skip` will silently skip creating a runtime. Instead, a non-functional
+ runtime will be generated and marked as incompatible so it cannot be used.
+ This is best if a local runtime is known not to work or be available
+ in certain cases and that's OK. e.g., one use windows paths when there
+ are people running on linux.
+* `warn` will print a warning message. This is useful when you expect
+ a runtime to be available, but are OK with it missing and falling back
+ to some other runtime.
+* `fail` will result in a failure. This is only recommended if you must
+ ensure the runtime is available.
+""",
+ ),
+ "_get_local_runtime_info": attr.label(
+ allow_single_file = True,
+ default = "//python/private:get_local_runtime_info.py",
+ ),
+ "_rule_name": attr.string(default = "local_runtime_repo"),
+ },
+ environ = ["PATH", REPO_DEBUG_ENV_VAR],
+)
+
+def _expand_incompatible_template():
+ return _TOOLCHAIN_IMPL_TEMPLATE.format(
+ interpreter_path = "/incompatible",
+ implementation_name = "incompatible",
+ lib_ext = "incompatible",
+ major = "0",
+ minor = "0",
+ micro = "0",
+ os = "@platforms//:incompatible",
+ )
+
+def _resolve_interpreter_path(rctx):
+ """Find the absolute path for an interpreter.
+
+ Args:
+ rctx: A repository_ctx object
+
+ Returns:
+ `struct` with the following fields:
+ * `resolved_path`: `path` object of a path that exists
+ * `describe_failure`: `Callable | None`. If a path that doesn't exist,
+ returns a description of why it couldn't be resolved
+ A path object or None. The path may not exist.
+ """
+ if "/" not in rctx.attr.interpreter_path and "\\" not in rctx.attr.interpreter_path:
+ # Provide a bit nicer integration with pyenv: recalculate the runtime if the
+ # user changes the python version using e.g. `pyenv shell`
+ repo_utils.getenv(rctx, "PYENV_VERSION")
+ result = repo_utils.which_unchecked(rctx, rctx.attr.interpreter_path)
+ resolved_path = result.binary
+ describe_failure = result.describe_failure
+ else:
+ repo_utils.watch(rctx, rctx.attr.interpreter_path)
+ resolved_path = rctx.path(rctx.attr.interpreter_path)
+ if not resolved_path.exists:
+ describe_failure = lambda: "Path not found: {}".format(repr(rctx.attr.interpreter_path))
+ else:
+ describe_failure = None
+
+ return struct(
+ resolved_path = resolved_path,
+ describe_failure = describe_failure,
+ )
+
+def _format_get_info_result(info):
+ lines = ["GetPythonInfo result:"]
+ for key, value in sorted(info.items()):
+ lines.append(" {}: {}".format(key, value if value != "" else "<empty string>"))
+ return "\n".join(lines)
diff --git a/python/private/local_runtime_repo_setup.bzl b/python/private/local_runtime_repo_setup.bzl
new file mode 100644
index 0000000..23fa99d
--- /dev/null
+++ b/python/private/local_runtime_repo_setup.bzl
@@ -0,0 +1,141 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Setup code called by the code generated by `local_runtime_repo`."""
+
+load("@bazel_skylib//lib:selects.bzl", "selects")
+load("@rules_cc//cc:defs.bzl", "cc_library")
+load("@rules_python//python:py_runtime.bzl", "py_runtime")
+load("@rules_python//python:py_runtime_pair.bzl", "py_runtime_pair")
+load("@rules_python//python/cc:py_cc_toolchain.bzl", "py_cc_toolchain")
+load("@rules_python//python/private:py_exec_tools_toolchain.bzl", "py_exec_tools_toolchain")
+
+_PYTHON_VERSION_FLAG = Label("@rules_python//python/config_settings:python_version")
+
+def define_local_runtime_toolchain_impl(
+ name,
+ lib_ext,
+ major,
+ minor,
+ micro,
+ interpreter_path,
+ implementation_name,
+ os):
+ """Defines a toolchain implementation for a local Python runtime.
+
+ Generates public targets:
+ * `python_runtimes`: The target toolchain type implementation
+ * `py_exec_tools_toolchain`: The exec tools toolchain type implementation
+ * `py_cc_toolchain`: The py cc toolchain type implementation
+ * `os`: A constraint (or alias to one) for the `target_compatible_with` this
+ toolchain is compatible with.
+ * `is_matching_python_version`: A `config_setting` for `target_settings`
+ this toolchain is compatible with.
+
+ Args:
+ name: `str` Only present to satisfy tooling
+ lib_ext: `str` The file extension for the `libpython` shared libraries
+ major: `str` The major Python version, e.g. `3` of `3.9.1`.
+ minor: `str` The minor Python version, e.g. `9` of `3.9.1`.
+ micro: `str` The micro Python version, e.g. "1" of `3.9.1`.
+ interpreter_path: `str` Absolute path to the interpreter.
+ implementation_name: `str` The implementation name, as returned by
+ `sys.implementation.name`.
+ os: `str` A label to the OS constraint (e.g. `@platforms//os:linux`) for
+ this runtime.
+ """
+ major_minor = "{}.{}".format(major, minor)
+ major_minor_micro = "{}.{}".format(major_minor, micro)
+
+ cc_library(
+ name = "_python_headers",
+ # NOTE: Keep in sync with watch_tree() called in local_runtime_repo
+ srcs = native.glob(["include/**/*.h"]),
+ includes = ["include"],
+ )
+
+ cc_library(
+ name = "_libpython",
+ # Don't use a recursive glob because the lib/ directory usually contains
+ # a subdirectory of the stdlib -- lots of unrelated files
+ srcs = native.glob([
+ "lib/*{}".format(lib_ext), # Match libpython*.so
+ "lib/*{}*".format(lib_ext), # Also match libpython*.so.1.0
+ ]),
+ hdrs = [":_python_headers"],
+ )
+
+ py_runtime(
+ name = "_py3_runtime",
+ interpreter_path = interpreter_path,
+ python_version = "PY3",
+ interpreter_version_info = {
+ "major": major,
+ "micro": micro,
+ "minor": minor,
+ },
+ implementation_name = implementation_name,
+ )
+
+ py_runtime_pair(
+ name = "python_runtimes",
+ py2_runtime = None,
+ py3_runtime = ":_py3_runtime",
+ visibility = ["//visibility:public"],
+ )
+
+ py_exec_tools_toolchain(
+ name = "py_exec_tools_toolchain",
+ visibility = ["//visibility:public"],
+ precompiler = "@rules_python//tools/precompiler:precompiler",
+ )
+
+ py_cc_toolchain(
+ name = "py_cc_toolchain",
+ headers = ":_python_headers",
+ libs = ":_libpython",
+ python_version = major_minor_micro,
+ visibility = ["//visibility:public"],
+ )
+
+ native.alias(
+ name = "os",
+ # Call Label() to force the string to evaluate in the context of
+ # rules_python, not the calling BUILD-file code. This is because
+ # the value is an `@platforms//foo` string, which @rules_python has
+ # visibility to, but the calling repo may not.
+ actual = Label(os),
+ visibility = ["//visibility:public"],
+ )
+
+ native.config_setting(
+ name = "_is_major_minor",
+ flag_values = {
+ _PYTHON_VERSION_FLAG: major_minor,
+ },
+ )
+ native.config_setting(
+ name = "_is_major_minor_micro",
+ flag_values = {
+ _PYTHON_VERSION_FLAG: major_minor_micro,
+ },
+ )
+ selects.config_setting_group(
+ name = "is_matching_python_version",
+ match_any = [
+ ":_is_major_minor",
+ ":_is_major_minor_micro",
+ ],
+ visibility = ["//visibility:public"],
+ )
diff --git a/python/private/local_runtime_toolchains_repo.bzl b/python/private/local_runtime_toolchains_repo.bzl
new file mode 100644
index 0000000..880fbfe
--- /dev/null
+++ b/python/private/local_runtime_toolchains_repo.bzl
@@ -0,0 +1,93 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Create a repository to hold a local Python toolchain definitions."""
+
+load("//python/private:text_util.bzl", "render")
+load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
+
+_TOOLCHAIN_TEMPLATE = """
+# Generated by local_runtime_toolchains_repo.bzl
+
+load("@rules_python//python/private:py_toolchain_suite.bzl", "define_local_toolchain_suites")
+
+define_local_toolchain_suites(
+ name = "toolchains",
+ version_aware_repo_names = {version_aware_names},
+ version_unaware_repo_names = {version_unaware_names},
+)
+"""
+
+def _local_runtime_toolchains_repo(rctx):
+ logger = repo_utils.logger(rctx)
+ rctx.file("WORKSPACE", "")
+ rctx.file("MODULE.bazel", "")
+ rctx.file("REPO.bazel", "")
+
+ logger.info(lambda: _format_toolchains_for_logging(rctx))
+
+ rctx.file("BUILD.bazel", _TOOLCHAIN_TEMPLATE.format(
+ version_aware_names = render.list(rctx.attr.runtimes),
+ version_unaware_names = render.list(rctx.attr.default_runtimes or rctx.attr.runtimes),
+ ))
+
+local_runtime_toolchains_repo = repository_rule(
+ implementation = _local_runtime_toolchains_repo,
+ doc = """
+Create a repo of toolchains definitions for local runtimes.
+
+This is intended to be used on the toolchain implemenations generated by
+`local_runtime_repo`.
+
+NOTE: This does not call `native.register_toolchains` -- the caller is
+responsible for registering the toolchains this defines.
+""",
+ attrs = {
+ "default_runtimes": attr.string_list(
+ doc = """
+The repo names of `local_runtime_repo` repos to define as toolchains.
+
+These will be defined as *version-unaware* toolchains. This means they will
+match any Python version. As such, they are registered after the version-aware
+toolchains defined by the `runtimes` attribute.
+
+Note that order matters: it determines the toolchain priority within the
+package.
+""",
+ ),
+ "runtimes": attr.string_list(
+ doc = """
+The repo names of `local_runtime_repo` repos to define as toolchains.
+
+These will be defined as *version-aware* toolchains. This means they require the
+`--//python/config_settings:python_version` to be set in order to match. These
+are registered before `default_runtimes`.
+
+Note that order matters: it determines the toolchain priority within the
+package.
+""",
+ ),
+ "_rule_name": attr.string(default = "local_toolchains_repo"),
+ },
+ environ = [REPO_DEBUG_ENV_VAR],
+)
+
+def _format_toolchains_for_logging(rctx):
+ lines = ["Local toolchain priority order:"]
+ i = 0
+ for i, name in enumerate(rctx.attr.runtimes, start = i):
+ lines.append(" {}: {} (version aware)".format(i, name))
+ for i, name in enumerate(rctx.attr.default_runtimes, start = i):
+ lines.append(" {}: {} (version unaware)".format(i, name))
+ return "\n".join(lines)
diff --git a/python/private/py_exec_tools_toolchain.bzl b/python/private/py_exec_tools_toolchain.bzl
index b3d0fb2..a4516d8 100644
--- a/python/private/py_exec_tools_toolchain.bzl
+++ b/python/private/py_exec_tools_toolchain.bzl
@@ -14,6 +14,7 @@
"""Rule that defines a toolchain for build tools."""
+load("@bazel_skylib//lib:paths.bzl", "paths")
load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo")
load("//python/private:toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE")
load(":py_exec_tools_info.bzl", "PyExecToolsInfo")
@@ -53,11 +54,15 @@
def _current_interpreter_executable_impl(ctx):
toolchain = ctx.toolchains[TARGET_TOOLCHAIN_TYPE]
runtime = toolchain.py3_runtime
+
+ # NOTE: We name the output filename after the underlying file name
+ # because of things like pyenv: they use $0 to determine what to
+ # re-exec. If it's not a recognized name, then they fail.
if runtime.interpreter:
- executable = ctx.actions.declare_file(ctx.label.name)
+ executable = ctx.actions.declare_file(runtime.interpreter.basename)
ctx.actions.symlink(output = executable, target_file = runtime.interpreter, is_executable = True)
else:
- executable = ctx.actions.declare_symlink(ctx.label.name)
+ executable = ctx.actions.declare_symlink(paths.basename(runtime.interpreter_path))
ctx.actions.symlink(output = executable, target_path = runtime.interpreter_path)
return [
toolchain,
diff --git a/python/private/py_toolchain_suite.bzl b/python/private/py_toolchain_suite.bzl
index 564e0ee..3fead95 100644
--- a/python/private/py_toolchain_suite.bzl
+++ b/python/private/py_toolchain_suite.bzl
@@ -15,6 +15,7 @@
"""Create the toolchain defs in a BUILD.bazel file."""
load("@bazel_skylib//lib:selects.bzl", "selects")
+load("//python/private:text_util.bzl", "render")
load(
":toolchain_types.bzl",
"EXEC_TOOLS_TOOLCHAIN_TYPE",
@@ -24,7 +25,15 @@
_IS_EXEC_TOOLCHAIN_ENABLED = Label("//python/config_settings:is_exec_tools_toolchain_enabled")
-def py_toolchain_suite(*, prefix, user_repository_name, python_version, set_python_version_constraint, flag_values, **kwargs):
+# buildifier: disable=unnamed-macro
+def py_toolchain_suite(
+ *,
+ prefix,
+ user_repository_name,
+ python_version,
+ set_python_version_constraint,
+ flag_values,
+ target_compatible_with = []):
"""For internal use only.
Args:
@@ -33,8 +42,7 @@
python_version: The full (X.Y.Z) version of the interpreter.
set_python_version_constraint: True or False as a string.
flag_values: Extra flag values to match for this toolchain.
- **kwargs: extra args passed to the `toolchain` calls.
-
+ target_compatible_with: list constraints the toolchains are compatible with.
"""
# We have to use a String value here because bzlmod is passing in a
@@ -82,30 +90,38 @@
repr(set_python_version_constraint),
))
+ _internal_toolchain_suite(
+ prefix = prefix,
+ runtime_repo_name = user_repository_name,
+ target_settings = target_settings,
+ target_compatible_with = target_compatible_with,
+ )
+
+def _internal_toolchain_suite(prefix, runtime_repo_name, target_compatible_with, target_settings):
native.toolchain(
name = "{prefix}_toolchain".format(prefix = prefix),
- toolchain = "@{user_repository_name}//:python_runtimes".format(
- user_repository_name = user_repository_name,
+ toolchain = "@{runtime_repo_name}//:python_runtimes".format(
+ runtime_repo_name = runtime_repo_name,
),
toolchain_type = TARGET_TOOLCHAIN_TYPE,
target_settings = target_settings,
- **kwargs
+ target_compatible_with = target_compatible_with,
)
native.toolchain(
name = "{prefix}_py_cc_toolchain".format(prefix = prefix),
- toolchain = "@{user_repository_name}//:py_cc_toolchain".format(
- user_repository_name = user_repository_name,
+ toolchain = "@{runtime_repo_name}//:py_cc_toolchain".format(
+ runtime_repo_name = runtime_repo_name,
),
toolchain_type = PY_CC_TOOLCHAIN_TYPE,
target_settings = target_settings,
- **kwargs
+ target_compatible_with = target_compatible_with,
)
native.toolchain(
name = "{prefix}_py_exec_tools_toolchain".format(prefix = prefix),
- toolchain = "@{user_repository_name}//:py_exec_tools_toolchain".format(
- user_repository_name = user_repository_name,
+ toolchain = "@{runtime_repo_name}//:py_exec_tools_toolchain".format(
+ runtime_repo_name = runtime_repo_name,
),
toolchain_type = EXEC_TOOLS_TOOLCHAIN_TYPE,
target_settings = select({
@@ -118,10 +134,46 @@
# the RHS must be a `config_setting`.
"//conditions:default": [_IS_EXEC_TOOLCHAIN_ENABLED],
}),
- exec_compatible_with = kwargs.get("target_compatible_with"),
+ exec_compatible_with = target_compatible_with,
)
# NOTE: When adding a new toolchain, for WORKSPACE builds to see the
# toolchain, the name must be added to the native.register_toolchains()
# call in python/repositories.bzl. Bzlmod doesn't need anything; it will
# register `:all`.
+
+def define_local_toolchain_suites(name, version_aware_repo_names, version_unaware_repo_names):
+ """Define toolchains for `local_runtime_repo` backed toolchains.
+
+ This generates `toolchain` targets that can be registered using `:all`. The
+ specific names of the toolchain targets are not defined. The priority order
+ of the toolchains is the order that is passed in, with version-aware having
+ higher priority than version-unaware.
+
+ Args:
+ name: `str` Unused; only present to satisfy tooling.
+ version_aware_repo_names: `list[str]` of the repo names that will have
+ version-aware toolchains defined.
+ version_unaware_repo_names: `list[str]` of the repo names that will have
+ version-unaware toolchains defined.
+ """
+ i = 0
+ for i, repo in enumerate(version_aware_repo_names, start = i):
+ prefix = render.left_pad_zero(i, 4)
+ _internal_toolchain_suite(
+ prefix = prefix,
+ runtime_repo_name = repo,
+ target_compatible_with = ["@{}//:os".format(repo)],
+ target_settings = ["@{}//:is_matching_python_version".format(repo)],
+ )
+
+ # The version unaware entries must go last because they will match any Python
+ # version.
+ for i, repo in enumerate(version_unaware_repo_names, start = i + 1):
+ prefix = render.left_pad_zero(i, 4)
+ _internal_toolchain_suite(
+ prefix = prefix,
+ runtime_repo_name = repo,
+ target_settings = [],
+ target_compatible_with = ["@{}//:os".format(repo)],
+ )
diff --git a/python/private/repo_utils.bzl b/python/private/repo_utils.bzl
index 54ad45c..9d76e19 100644
--- a/python/private/repo_utils.bzl
+++ b/python/private/repo_utils.bzl
@@ -29,7 +29,7 @@
Returns:
True if enabled, False if not.
"""
- return rctx.os.environ.get(REPO_DEBUG_ENV_VAR) == "1"
+ return _getenv(rctx, REPO_DEBUG_ENV_VAR) == "1"
def _debug_print(rctx, message_cb):
"""Prints a message if repo debugging is enabled.
@@ -46,7 +46,8 @@
"""Creates a logger instance for printing messages.
Args:
- rctx: repository_ctx object.
+ rctx: repository_ctx object. If the attribute `_rule_name` is
+ present, it will be included in log messages.
Returns:
A struct with attributes logging: trace, debug, info, warn, fail.
@@ -65,11 +66,20 @@
"TRACE": 3,
}.get(verbosity_level, 0)
- def _log(enabled_on_verbosity, level, message_cb):
+ def _log(enabled_on_verbosity, level, message_cb_or_str):
if verbosity < enabled_on_verbosity:
return
+ rule_name = getattr(rctx.attr, "_rule_name", "?")
+ if type(message_cb_or_str) == "string":
+ message = message_cb_or_str
+ else:
+ message = message_cb_or_str()
- print("\nrules_python: {}: ".format(level.upper()), message_cb()) # buildifier: disable=print
+ print("\nrules_python:{}(@@{}) {}:".format(
+ rule_name,
+ rctx.name,
+ level.upper(),
+ ), message) # buildifier: disable=print
return struct(
trace = lambda message_cb: _log(3, "TRACE", message_cb),
@@ -117,6 +127,7 @@
env_str = _env_to_str(environment),
))
+ rctx.report_progress("Running {}".format(op))
result = rctx.execute(arguments, environment = environment, **kwargs)
if fail_on_error and result.return_code != 0:
@@ -150,7 +161,18 @@
output = _outputs_to_str(result),
))
- return result
+ result_kwargs = {k: getattr(result, k) for k in dir(result)}
+ return struct(
+ describe_failure = lambda: _execute_describe_failure(
+ op = op,
+ arguments = arguments,
+ result = result,
+ rctx = rctx,
+ kwargs = kwargs,
+ environment = environment,
+ ),
+ **result_kwargs
+ )
def _execute_unchecked(*args, **kwargs):
"""Execute a subprocess.
@@ -185,6 +207,25 @@
"""Calls execute_checked, but only returns the stdout value."""
return _execute_checked(*args, **kwargs).stdout
+def _execute_describe_failure(*, op, arguments, result, rctx, kwargs, environment):
+ return (
+ "repo.execute: {op}: failure:\n" +
+ " command: {cmd}\n" +
+ " return code: {return_code}\n" +
+ " working dir: {cwd}\n" +
+ " timeout: {timeout}\n" +
+ " environment:{env_str}\n" +
+ "{output}"
+ ).format(
+ op = op,
+ cmd = _args_to_str(arguments),
+ return_code = result.return_code,
+ cwd = _cwd_to_str(rctx, kwargs),
+ timeout = _timeout_to_str(kwargs),
+ env_str = _env_to_str(environment),
+ output = _outputs_to_str(result),
+ )
+
def _which_checked(rctx, binary_name):
"""Tests to see if a binary exists, and otherwise fails with a message.
@@ -195,16 +236,54 @@
Returns:
rctx.Path for the binary.
"""
+ result = _which_unchecked(rctx, binary_name)
+ if result.binary == None:
+ fail(result.describe_failure())
+ return result.binary
+
+def _which_unchecked(rctx, binary_name):
+ """Tests to see if a binary exists.
+
+ This is also watch the `PATH` environment variable.
+
+ Args:
+ binary_name: name of the binary to find.
+ rctx: repository context.
+
+ Returns:
+ `struct` with attributes:
+ * `binary`: `repository_ctx.Path`
+ * `describe_failure`: `Callable | None`; takes no args. If the
+ binary couldn't be found, provides a detailed error description.
+ """
+ path = _getenv(rctx, "PATH", "")
binary = rctx.which(binary_name)
- if binary == None:
- fail((
- "Unable to find the binary '{binary_name}' on PATH.\n" +
- " PATH = {path}"
- ).format(
- binary_name = binary_name,
- path = rctx.os.environ.get("PATH"),
- ))
- return binary
+ if binary:
+ _watch(rctx, binary)
+ describe_failure = None
+ else:
+ describe_failure = lambda: _which_describe_failure(binary_name, path)
+
+ return struct(
+ binary = binary,
+ describe_failure = describe_failure,
+ )
+
+def _which_describe_failure(binary_name, path):
+ return (
+ "Unable to find the binary '{binary_name}' on PATH.\n" +
+ " PATH = {path}"
+ ).format(
+ binary_name = binary_name,
+ path = path,
+ )
+
+def _getenv(rctx, name, default = None):
+ # Bazel 7+ API
+ if hasattr(rctx, "getenv"):
+ return rctx.getenv(name, default)
+ else:
+ return rctx.os.environ.get("PATH", default)
def _args_to_str(arguments):
return " ".join([_arg_repr(a) for a in arguments])
@@ -262,12 +341,50 @@
lines.append("<{} empty>".format(name))
return "\n".join(lines)
+def _get_platforms_os_name(rctx):
+ """Return the name in @platforms//os for the host os.
+
+ Args:
+ rctx: repository_ctx
+
+ Returns:
+ `str | None`. The target name if it maps to known platforms
+ value, otherwise None.
+ """
+ os = rctx.os.name.lower()
+ if "linux" in os:
+ return os
+ if "windows" in os:
+ return "windows"
+ if "mac" in os:
+ return "osx"
+
+ return None
+
+# TODO: Remove after Bazel 6 support dropped
+def _watch(rctx, *args, **kwargs):
+ """Calls rctx.watch, if available."""
+ if hasattr(rctx, "watch"):
+ rctx.watch(*args, **kwargs)
+
+# TODO: Remove after Bazel 6 support dropped
+def _watch_tree(rctx, *args, **kwargs):
+ """Calls rctx.watch_tree, if available."""
+ if hasattr(rctx, "watch_tree"):
+ rctx.watch_tree(*args, **kwargs)
+
repo_utils = struct(
- execute_checked = _execute_checked,
- execute_unchecked = _execute_unchecked,
- execute_checked_stdout = _execute_checked_stdout,
- is_repo_debug_enabled = _is_repo_debug_enabled,
+ # keep sorted
debug_print = _debug_print,
- which_checked = _which_checked,
+ execute_checked = _execute_checked,
+ execute_checked_stdout = _execute_checked_stdout,
+ execute_unchecked = _execute_unchecked,
+ get_platforms_os_name = _get_platforms_os_name,
+ getenv = _getenv,
+ is_repo_debug_enabled = _is_repo_debug_enabled,
logger = _logger,
+ watch = _watch,
+ watch_tree = _watch_tree,
+ which_checked = _which_checked,
+ which_unchecked = _which_unchecked,
)
diff --git a/python/private/text_util.bzl b/python/private/text_util.bzl
index 8a018e7..38f2b0e 100644
--- a/python/private/text_util.bzl
+++ b/python/private/text_util.bzl
@@ -20,6 +20,15 @@
return "\n".join([indent + line for line in text.splitlines()])
+def _hanging_indent(text, indent = " " * 4):
+ if "\n" not in text:
+ return text
+
+ lines = text.splitlines()
+ for i, line in enumerate(lines):
+ lines[i] = (indent if i != 0 else "") + line
+ return "\n".join(lines)
+
def _render_alias(name, actual, *, visibility = None):
args = [
"name = \"{}\",".format(name),
@@ -67,14 +76,24 @@
return "{}({})".format(name, args)
-def _render_list(items):
+def _render_list(items, *, hanging_indent = ""):
+ """Convert a list to formatted text.
+
+ Args:
+ items: list of items.
+ hanging_indent: str, indent to apply to second and following lines of
+ the formatted text.
+
+ Returns:
+ The list pretty formatted as a string.
+ """
if not items:
return "[]"
if len(items) == 1:
return "[{}]".format(repr(items[0]))
- return "\n".join([
+ text = "\n".join([
"[",
_indent("\n".join([
"{},".format(repr(item))
@@ -82,6 +101,12 @@
])),
"]",
])
+ if hanging_indent:
+ text = _hanging_indent(text, hanging_indent)
+ return text
+
+def _render_str(value):
+ return repr(value)
def _render_tuple(items, *, value_repr = repr):
if not items:
@@ -116,10 +141,12 @@
render = struct(
alias = _render_alias,
dict = _render_dict,
+ hanging_indent = _hanging_indent,
indent = _indent,
left_pad_zero = _left_pad_zero,
list = _render_list,
select = _render_select,
- tuple = _render_tuple,
+ str = _render_str,
toolchain_prefix = _toolchain_prefix,
+ tuple = _render_tuple,
)
