python/private/toolchains_repo.bzl

# Copyright 2022 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Create a repository to hold the toolchains.

This follows guidance here:
https://docs.bazel.build/versions/main/skylark/deploying.html#registering-toolchains

The "complex computation" in our case is simply downloading large artifacts.
This guidance tells us how to avoid that: we put the toolchain targets in the
alias repository with only the toolchain attribute pointing into the
platform-specific repositories.
"""

load(
    "//python:versions.bzl",
    "PLATFORMS",
    "WINDOWS_NAME",
)
load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
load(":text_util.bzl", "render")

def get_repository_name(repository_workspace):
    dummy_label = "//:_"
    return str(repository_workspace.relative(dummy_label))[:-len(dummy_label)] or "@"

def python_toolchain_build_file_content(
        prefix,
        python_version,
        set_python_version_constraint,
        user_repository_name,
        loaded_platforms):
    """Creates the content for toolchain definitions for a build file.

    Args:
        prefix: Python toolchain name prefixes
        python_version: Python versions for the toolchains
        set_python_version_constraint: string, "True" if the toolchain should
            have the Python version constraint added as a requirement for
            matching the toolchain, "False" if not.
        user_repository_name: names for the user repos
        loaded_platforms: {type}`struct` the list of platform structs defining the
            loaded platforms. It is as they are defined in `//python:versions.bzl`.

    Returns:
        build_content: Text containing toolchain definitions
    """

    return "\n\n".join([
        """\
py_toolchain_suite(
    user_repository_name = "{user_repository_name}_{platform}",
    prefix = "{prefix}{platform}",
    target_compatible_with = {compatible_with},
    flag_values = {flag_values},
    python_version = "{python_version}",
    set_python_version_constraint = "{set_python_version_constraint}",
)""".format(
            compatible_with = render.indent(render.list(meta.compatible_with)).lstrip(),
            flag_values = render.indent(render.dict(
                meta.flag_values,
                key_repr = lambda x: repr(str(x)),  # this is to correctly display labels
            )).lstrip(),
            platform = platform,
            set_python_version_constraint = set_python_version_constraint,
            user_repository_name = user_repository_name,
            prefix = prefix,
            python_version = python_version,
        )
        for platform, meta in loaded_platforms.items()
    ])

def _toolchains_repo_impl(rctx):
    build_content = """\
# Generated by python/private/toolchains_repo.bzl
#
# These can be registered in the workspace file or passed to --extra_toolchains
# flag. By default all these toolchains are registered by the
# python_register_toolchains macro so you don't normally need to interact with
# these targets.

load("@{rules_python}//python/private:py_toolchain_suite.bzl", "py_toolchain_suite")

""".format(
        rules_python = rctx.attr._rules_python_workspace.workspace_name,
    )

    toolchains = python_toolchain_build_file_content(
        prefix = "",
        python_version = rctx.attr.python_version,
        set_python_version_constraint = str(rctx.attr.set_python_version_constraint),
        user_repository_name = rctx.attr.user_repository_name,
        loaded_platforms = {
            k: v
            for k, v in PLATFORMS.items()
            if k in rctx.attr.platforms
        },
    )

    rctx.file("BUILD.bazel", build_content + toolchains)

toolchains_repo = repository_rule(
    _toolchains_repo_impl,
    doc = "Creates a repository with toolchain definitions for all known platforms " +
          "which can be registered or selected.",
    attrs = {
        "platforms": attr.string_list(doc = "List of platforms for which the toolchain definitions shall be created"),
        "python_version": attr.string(doc = "The Python version."),
        "set_python_version_constraint": attr.bool(doc = "if target_compatible_with for the toolchain should set the version constraint"),
        "user_repository_name": attr.string(doc = "what the user chose for the base name"),
        "_rules_python_workspace": attr.label(default = Label("//:WORKSPACE")),
    },
)

def _toolchain_aliases_impl(rctx):
    # Base BUILD file for this repository.
    build_contents = """\
# Generated by python/private/toolchains_repo.bzl
load("@rules_python//python/private:toolchain_aliases.bzl", "toolchain_aliases")

package(default_visibility = ["//visibility:public"])

exports_files(["defs.bzl"])

PLATFORMS = [
{loaded_platforms}
]
toolchain_aliases(
    name = "{py_repository}",
    platforms = PLATFORMS,
)
""".format(
        py_repository = rctx.attr.user_repository_name,
        loaded_platforms = "\n".join(["    \"{}\",".format(p) for p in rctx.attr.platforms]),
    )
    rctx.file("BUILD.bazel", build_contents)

    # Expose a Starlark file so rules can know what host platform we used and where to find an interpreter
    # when using repository_ctx.path, which doesn't understand aliases.
    rctx.file("defs.bzl", content = """\
# Generated by python/private/toolchains_repo.bzl

load(
    "{rules_python}//python/config_settings:transition.bzl",
    _py_binary = "py_binary",
    _py_test = "py_test",
)
load(
    "{rules_python}//python/entry_points:py_console_script_binary.bzl",
    _py_console_script_binary = "py_console_script_binary",
)
load("{rules_python}//python:pip.bzl", _compile_pip_requirements = "compile_pip_requirements")

def py_binary(name, **kwargs):
    return _py_binary(
        name = name,
        python_version = "{python_version}",
        **kwargs
    )

def py_console_script_binary(name, **kwargs):
    return _py_console_script_binary(
        name = name,
        binary_rule = py_binary,
        **kwargs
    )

def py_test(name, **kwargs):
    return _py_test(
        name = name,
        python_version = "{python_version}",
        **kwargs
    )

def compile_pip_requirements(name, **kwargs):
    return _compile_pip_requirements(
        name = name,
        py_binary = py_binary,
        py_test = py_test,
        **kwargs
    )

""".format(
        python_version = rctx.attr.python_version,
        rules_python = get_repository_name(rctx.attr._rules_python_workspace),
    ))

toolchain_aliases = repository_rule(
    _toolchain_aliases_impl,
    doc = """\
Creates a repository with a shorter name only referencing the python version,
it contains a BUILD.bazel file declaring aliases to the host platform's targets
and is a great fit for any usage related to setting up toolchains for build
actions.""",
    attrs = {
        "platforms": attr.string_list(
            doc = "List of platforms for which aliases shall be created",
        ),
        "python_version": attr.string(doc = "The Python version."),
        "user_repository_name": attr.string(
            mandatory = True,
            doc = "The base name for all created repositories, like 'python38'.",
        ),
        "_rules_python_workspace": attr.label(default = Label("//:WORKSPACE")),
    },
    environ = [REPO_DEBUG_ENV_VAR],
)

def _host_toolchain_impl(rctx):
    rctx.file("BUILD.bazel", """\
# Generated by python/private/toolchains_repo.bzl

exports_files(["python"], visibility = ["//visibility:public"])
""")

    os_name = repo_utils.get_platforms_os_name(rctx)
    host_platform = _get_host_platform(
        rctx = rctx,
        logger = repo_utils.logger(rctx),
        python_version = rctx.attr.python_version,
        os_name = os_name,
        cpu_name = repo_utils.get_platforms_cpu_name(rctx),
        platforms = rctx.attr.platforms,
    )
    repo = "@@{py_repository}_{host_platform}".format(
        py_repository = rctx.attr.name[:-len("_host")],
        host_platform = host_platform,
    )

    rctx.report_progress("Symlinking interpreter files to the target platform")
    host_python_repo = rctx.path(Label("{repo}//:BUILD.bazel".format(repo = repo)))

    # The interpreter might not work on platfroms that don't have symlink support if
    # we just symlink the interpreter itself. rctx.symlink does a copy in such cases
    # so we can just attempt to symlink all of the directories in the host interpreter
    # repo, which should be faster than re-downloading it.
    for p in host_python_repo.dirname.readdir():
        if p.basename in [
            # ignore special files created by the repo rule automatically
            "BUILD.bazel",
            "MODULE.bazel",
            "REPO.bazel",
            "WORKSPACE",
            "WORKSPACE.bazel",
            "WORKSPACE.bzlmod",
        ]:
            continue

        # symlink works on all platforms that bazel supports, so it should work on
        # UNIX and Windows with and without symlink support. For better performance
        # users should enable the symlink startup option, however that requires admin
        # privileges.
        rctx.symlink(p, p.basename)

    is_windows = (os_name == WINDOWS_NAME)
    python_binary = "python.exe" if is_windows else "python"

    # Ensure that we can run the interpreter and check that we are not
    # using the host interpreter.
    python_tester_contents = """\
from pathlib import Path
import sys

python = Path(sys.executable)
want_python = str(Path("{python}").resolve())
got_python = str(Path(sys.executable).resolve())

assert want_python == got_python, \
    "Expected to use a different interpreter:\\nwant: '{{}}'\\n got: '{{}}'".format(
        want_python,
        got_python,
    )
""".format(repo = repo.strip("@"), python = python_binary)
    python_tester = rctx.path("python_tester.py")
    rctx.file(python_tester, python_tester_contents)
    repo_utils.execute_checked(
        rctx,
        op = "CheckHostInterpreter",
        arguments = [rctx.path(python_binary), python_tester],
    )
    if not rctx.delete(python_tester):
        fail("Failed to delete the python tester")

host_toolchain = repository_rule(
    _host_toolchain_impl,
    doc = """\
Creates a repository with a shorter name meant to be used in the repository_ctx,
which needs to have `symlinks` for the interpreter. This is separate from the
toolchain_aliases repo because referencing the `python` interpreter target from
this repo causes an eager fetch of the toolchain for the host platform.
    """,
    attrs = {
        "platforms": attr.string_list(mandatory = True),
        "python_version": attr.string(mandatory = True),
        "_rule_name": attr.string(default = "host_toolchain"),
        "_rules_python_workspace": attr.label(default = Label("//:WORKSPACE")),
    },
)

def _multi_toolchain_aliases_impl(rctx):
    rules_python = rctx.attr._rules_python_workspace.workspace_name

    for python_version, repository_name in rctx.attr.python_versions.items():
        file = "{}/defs.bzl".format(python_version)
        rctx.file(file, content = """\
# Generated by python/private/toolchains_repo.bzl

load(
    "@{repository_name}//:defs.bzl",
    _compile_pip_requirements = "compile_pip_requirements",
    _py_binary = "py_binary",
    _py_console_script_binary = "py_console_script_binary",
    _py_test = "py_test",
)

compile_pip_requirements = _compile_pip_requirements
py_binary = _py_binary
py_console_script_binary = _py_console_script_binary
py_test = _py_test
""".format(
            repository_name = repository_name,
        ))
        rctx.file("{}/BUILD.bazel".format(python_version), "")

    pip_bzl = """\
# Generated by python/private/toolchains_repo.bzl

load("@{rules_python}//python:pip.bzl", "pip_parse", _multi_pip_parse = "multi_pip_parse")

def multi_pip_parse(name, requirements_lock, **kwargs):
    return _multi_pip_parse(
        name = name,
        python_versions = {python_versions},
        requirements_lock = requirements_lock,
        minor_mapping = {minor_mapping},
        **kwargs
    )

""".format(
        python_versions = rctx.attr.python_versions.keys(),
        minor_mapping = render.indent(render.dict(rctx.attr.minor_mapping), indent = " " * 8).lstrip(),
        rules_python = rules_python,
    )
    rctx.file("pip.bzl", content = pip_bzl)
    rctx.file("BUILD.bazel", "")

multi_toolchain_aliases = repository_rule(
    _multi_toolchain_aliases_impl,
    attrs = {
        "minor_mapping": attr.string_dict(doc = "The mapping between `X.Y` and `X.Y.Z` python version values"),
        "python_versions": attr.string_dict(doc = "The Python versions."),
        "_rules_python_workspace": attr.label(default = Label("//:WORKSPACE")),
    },
)

def sanitize_platform_name(platform):
    return platform.replace("-", "_")

def _get_host_platform(*, rctx, logger, python_version, os_name, cpu_name, platforms):
    """Gets the host platform.

    Args:
        rctx: {type}`repository_ctx`.
        logger: {type}`struct`.
        python_version: {type}`string`.
        os_name: {type}`str` the host OS name.
        cpu_name: {type}`str` the host CPU name.
        platforms: {type}`list[str]` the list of loaded platforms.
    Returns:
        The host platform.
    """
    candidates = []
    for platform in platforms:
        meta = PLATFORMS[platform]

        if meta.os_name == os_name and meta.arch == cpu_name:
            candidates.append(platform)

    if len(candidates) == 1:
        return candidates[0]

    if candidates:
        env_var = "RULES_PYTHON_REPO_TOOLCHAIN_{}_{}_{}".format(
            python_version.replace(".", "_"),
            os_name.upper(),
            cpu_name.upper(),
        )
        preference = repo_utils.getenv(rctx, env_var)
        if preference == None:
            logger.info("Consider using '{}' to select from one of the platforms: {}".format(
                env_var,
                candidates,
            ))
        elif preference not in candidates:
            return logger.fail("Please choose a preferred interpreter out of the following platforms: {}".format(candidates))
        else:
            candidates = [preference]

    if candidates:
        return candidates[0]

    return logger.fail("Could not find a compatible 'host' python for '{os_name}', '{cpu_name}' from the loaded platforms: {platforms}".format(
        os_name = os_name,
        cpu_name = cpu_name,
        platforms = platforms,
    ))