refactor(bzlmod): move bzlmod code to private/bzlmod (#1477)
This PR just moves all of the private `bzlmod` code to
`python/private/bzlmod`
and adds minimal `bzl_library` bindings to make the docs the same. Once
#1476
is merged, we can start exposing documentation for `module_extension`.
This includes extras in `pip_install/pip_repository.bzl` just to make it
possible to review and merge #1476 and this in parallel.
diff --git a/MODULE.bazel b/MODULE.bazel
index 5d77839..efff733 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -14,7 +14,7 @@
bazel_dep(name = "stardoc", version = "0.6.2", dev_dependency = True, repo_name = "io_bazel_stardoc")
-internal_deps = use_extension("@rules_python//python/extensions/private:internal_deps.bzl", "internal_deps")
+internal_deps = use_extension("@rules_python//python/private/bzlmod:internal_deps.bzl", "internal_deps")
internal_deps.install()
use_repo(
internal_deps,
diff --git a/python/extensions/pip.bzl b/python/extensions/pip.bzl
index a0559ff..a69ee34 100644
--- a/python/extensions/pip.bzl
+++ b/python/extensions/pip.bzl
@@ -14,443 +14,6 @@
"pip module extension for use with bzlmod"
-load("@bazel_features//:features.bzl", "bazel_features")
-load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS")
-load(
- "//python/pip_install:pip_repository.bzl",
- "locked_requirements_label",
- "pip_hub_repository_bzlmod",
- "pip_repository_attrs",
- "use_isolated",
- "whl_library",
-)
-load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse")
-load("//python/private:full_version.bzl", "full_version")
-load("//python/private:normalize_name.bzl", "normalize_name")
-load("//python/private:version_label.bzl", "version_label")
+load("//python/private/bzlmod:pip.bzl", _pip = "pip")
-def _whl_mods_impl(mctx):
- """Implementation of the pip.whl_mods tag class.
-
- This creates the JSON files used to modify the creation of different wheels.
-"""
- whl_mods_dict = {}
- for mod in mctx.modules:
- for whl_mod_attr in mod.tags.whl_mods:
- if whl_mod_attr.hub_name not in whl_mods_dict.keys():
- whl_mods_dict[whl_mod_attr.hub_name] = {whl_mod_attr.whl_name: whl_mod_attr}
- elif whl_mod_attr.whl_name in whl_mods_dict[whl_mod_attr.hub_name].keys():
- # We cannot have the same wheel name in the same hub, as we
- # will create the same JSON file name.
- fail("""\
-Found same whl_name '{}' in the same hub '{}', please use a different hub_name.""".format(
- whl_mod_attr.whl_name,
- whl_mod_attr.hub_name,
- ))
- else:
- whl_mods_dict[whl_mod_attr.hub_name][whl_mod_attr.whl_name] = whl_mod_attr
-
- for hub_name, whl_maps in whl_mods_dict.items():
- whl_mods = {}
-
- # create a struct that we can pass to the _whl_mods_repo rule
- # to create the different JSON files.
- for whl_name, mods in whl_maps.items():
- build_content = mods.additive_build_content
- if mods.additive_build_content_file != None and mods.additive_build_content != "":
- fail("""\
-You cannot use both the additive_build_content and additive_build_content_file arguments at the same time.
-""")
- elif mods.additive_build_content_file != None:
- build_content = mctx.read(mods.additive_build_content_file)
-
- whl_mods[whl_name] = json.encode(struct(
- additive_build_content = build_content,
- copy_files = mods.copy_files,
- copy_executables = mods.copy_executables,
- data = mods.data,
- data_exclude_glob = mods.data_exclude_glob,
- srcs_exclude_glob = mods.srcs_exclude_glob,
- ))
-
- _whl_mods_repo(
- name = hub_name,
- whl_mods = whl_mods,
- )
-
-def _create_whl_repos(module_ctx, pip_attr, whl_map):
- python_interpreter_target = pip_attr.python_interpreter_target
-
- # if we do not have the python_interpreter set in the attributes
- # we programmatically find it.
- hub_name = pip_attr.hub_name
- if python_interpreter_target == None:
- python_name = "python_" + version_label(pip_attr.python_version, sep = "_")
- if python_name not in INTERPRETER_LABELS.keys():
- fail((
- "Unable to find interpreter for pip hub '{hub_name}' for " +
- "python_version={version}: Make sure a corresponding " +
- '`python.toolchain(python_version="{version}")` call exists'
- ).format(
- hub_name = hub_name,
- version = pip_attr.python_version,
- ))
- python_interpreter_target = INTERPRETER_LABELS[python_name]
-
- pip_name = "{}_{}".format(
- hub_name,
- version_label(pip_attr.python_version),
- )
- requrements_lock = locked_requirements_label(module_ctx, pip_attr)
-
- # Parse the requirements file directly in starlark to get the information
- # needed for the whl_libary declarations below.
- requirements_lock_content = module_ctx.read(requrements_lock)
- parse_result = parse_requirements(requirements_lock_content)
- requirements = parse_result.requirements
- extra_pip_args = pip_attr.extra_pip_args + parse_result.options
-
- if hub_name not in whl_map:
- whl_map[hub_name] = {}
-
- whl_modifications = {}
- if pip_attr.whl_modifications != None:
- for mod, whl_name in pip_attr.whl_modifications.items():
- whl_modifications[whl_name] = mod
-
- # Create a new wheel library for each of the different whls
- for whl_name, requirement_line in requirements:
- # We are not using the "sanitized name" because the user
- # would need to guess what name we modified the whl name
- # to.
- annotation = whl_modifications.get(whl_name)
- whl_name = normalize_name(whl_name)
- whl_library(
- name = "%s_%s" % (pip_name, whl_name),
- requirement = requirement_line,
- repo = pip_name,
- repo_prefix = pip_name + "_",
- annotation = annotation,
- python_interpreter = pip_attr.python_interpreter,
- python_interpreter_target = python_interpreter_target,
- quiet = pip_attr.quiet,
- timeout = pip_attr.timeout,
- isolated = use_isolated(module_ctx, pip_attr),
- extra_pip_args = extra_pip_args,
- download_only = pip_attr.download_only,
- pip_data_exclude = pip_attr.pip_data_exclude,
- enable_implicit_namespace_pkgs = pip_attr.enable_implicit_namespace_pkgs,
- environment = pip_attr.environment,
- )
-
- if whl_name not in whl_map[hub_name]:
- whl_map[hub_name][whl_name] = {}
-
- whl_map[hub_name][whl_name][full_version(pip_attr.python_version)] = pip_name + "_"
-
-def _pip_impl(module_ctx):
- """Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories.
-
- This implementation iterates through all of the `pip.parse` calls and creates
- different pip hub repositories based on the "hub_name". Each of the
- pip calls create spoke repos that uses a specific Python interpreter.
-
- In a MODULES.bazel file we have:
-
- pip.parse(
- hub_name = "pip",
- python_version = 3.9,
- requirements_lock = "//:requirements_lock_3_9.txt",
- requirements_windows = "//:requirements_windows_3_9.txt",
- )
- pip.parse(
- hub_name = "pip",
- python_version = 3.10,
- requirements_lock = "//:requirements_lock_3_10.txt",
- requirements_windows = "//:requirements_windows_3_10.txt",
- )
-
- For instance, we have a hub with the name of "pip".
- A repository named the following is created. It is actually called last when
- all of the pip spokes are collected.
-
- - @@rules_python~override~pip~pip
-
- As shown in the example code above we have the following.
- Two different pip.parse statements exist in MODULE.bazel provide the hub_name "pip".
- These definitions create two different pip spoke repositories that are
- related to the hub "pip".
- One spoke uses Python 3.9 and the other uses Python 3.10. This code automatically
- determines the Python version and the interpreter.
- Both of these pip spokes contain requirements files that includes websocket
- and its dependencies.
-
- We also need repositories for the wheels that the different pip spokes contain.
- For each Python version a different wheel repository is created. In our example
- each pip spoke had a requirements file that contained websockets. We
- then create two different wheel repositories that are named the following.
-
- - @@rules_python~override~pip~pip_39_websockets
- - @@rules_python~override~pip~pip_310_websockets
-
- And if the wheel has any other dependencies subsequent wheels are created in the same fashion.
-
- The hub repository has aliases for `pkg`, `data`, etc, which have a select that resolves to
- a spoke repository depending on the Python version.
-
- Also we may have more than one hub as defined in a MODULES.bazel file. So we could have multiple
- hubs pointing to various different pip spokes.
-
- Some other business rules notes. A hub can only have one spoke per Python version. We cannot
- have a hub named "pip" that has two spokes that use the Python 3.9 interpreter. Second
- we cannot have the same hub name used in sub-modules. The hub name has to be globally
- unique.
-
- This implementation also handles the creation of whl_modification JSON files that are used
- during the creation of wheel libraries. These JSON files used via the annotations argument
- when calling wheel_installer.py.
-
- Args:
- module_ctx: module contents
- """
-
- # Build all of the wheel modifications if the tag class is called.
- _whl_mods_impl(module_ctx)
-
- # Used to track all the different pip hubs and the spoke pip Python
- # versions.
- pip_hub_map = {}
-
- # Keeps track of all the hub's whl repos across the different versions.
- # dict[hub, dict[whl, dict[version, str pip]]]
- # Where hub, whl, and pip are the repo names
- hub_whl_map = {}
-
- for mod in module_ctx.modules:
- for pip_attr in mod.tags.parse:
- hub_name = pip_attr.hub_name
- if hub_name not in pip_hub_map:
- pip_hub_map[pip_attr.hub_name] = struct(
- module_name = mod.name,
- python_versions = [pip_attr.python_version],
- )
- elif pip_hub_map[hub_name].module_name != mod.name:
- # We cannot have two hubs with the same name in different
- # modules.
- fail((
- "Duplicate cross-module pip hub named '{hub}': pip hub " +
- "names must be unique across modules. First defined " +
- "by module '{first_module}', second attempted by " +
- "module '{second_module}'"
- ).format(
- hub = hub_name,
- first_module = pip_hub_map[hub_name].module_name,
- second_module = mod.name,
- ))
-
- elif pip_attr.python_version in pip_hub_map[hub_name].python_versions:
- fail((
- "Duplicate pip python version '{version}' for hub " +
- "'{hub}' in module '{module}': the Python versions " +
- "used for a hub must be unique"
- ).format(
- hub = hub_name,
- module = mod.name,
- version = pip_attr.python_version,
- ))
- else:
- pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version)
-
- _create_whl_repos(module_ctx, pip_attr, hub_whl_map)
-
- for hub_name, whl_map in hub_whl_map.items():
- pip_hub_repository_bzlmod(
- name = hub_name,
- repo_name = hub_name,
- whl_map = whl_map,
- default_version = full_version(DEFAULT_PYTHON_VERSION),
- )
-
-def _pip_parse_ext_attrs():
- attrs = dict({
- "hub_name": attr.string(
- mandatory = True,
- doc = """
-The name of the repo pip dependencies will be accessible from.
-
-This name must be unique between modules; unless your module is guaranteed to
-always be the root module, it's highly recommended to include your module name
-in the hub name. Repo mapping, `use_repo(..., pip="my_modules_pip_deps")`, can
-be used for shorter local names within your module.
-
-Within a module, the same `hub_name` can be specified to group different Python
-versions of pip dependencies under one repository name. This allows using a
-Python version-agnostic name when referring to pip dependencies; the
-correct version will be automatically selected.
-
-Typically, a module will only have a single hub of pip dependencies, but this
-is not required. Each hub is a separate resolution of pip dependencies. This
-means if different programs need different versions of some library, separate
-hubs can be created, and each program can use its respective hub's targets.
-Targets from different hubs should not be used together.
-""",
- ),
- "python_version": attr.string(
- mandatory = True,
- doc = """
-The Python version to use for resolving the pip dependencies, in Major.Minor
-format (e.g. "3.11"). Patch level granularity (e.g. "3.11.1") is not supported.
-If not specified, then the default Python version (as set by the root module or
-rules_python) will be used.
-
-The version specified here must have a corresponding `python.toolchain()`
-configured.
-""",
- ),
- "whl_modifications": attr.label_keyed_string_dict(
- mandatory = False,
- doc = """\
-A dict of labels to wheel names that is typically generated by the whl_modifications.
-The labels are JSON config files describing the modifications.
-""",
- ),
- }, **pip_repository_attrs)
-
- # Like the pip_repository rule, we end up setting this manually so
- # don't allow users to override it.
- attrs.pop("repo_prefix")
-
- # incompatible_generate_aliases is always True in bzlmod
- attrs.pop("incompatible_generate_aliases")
-
- return attrs
-
-def _whl_mod_attrs():
- attrs = {
- "additive_build_content": attr.string(
- doc = "(str, optional): Raw text to add to the generated `BUILD` file of a package.",
- ),
- "additive_build_content_file": attr.label(
- doc = """\
-(label, optional): path to a BUILD file to add to the generated
-`BUILD` file of a package. You cannot use both additive_build_content and additive_build_content_file
-arguments at the same time.""",
- ),
- "copy_executables": attr.string_dict(
- doc = """\
-(dict, optional): A mapping of `src` and `out` files for
-[@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as
-executable.""",
- ),
- "copy_files": attr.string_dict(
- doc = """\
-(dict, optional): A mapping of `src` and `out` files for
-[@bazel_skylib//rules:copy_file.bzl][cf]""",
- ),
- "data": attr.string_list(
- doc = """\
-(list, optional): A list of labels to add as `data` dependencies to
-the generated `py_library` target.""",
- ),
- "data_exclude_glob": attr.string_list(
- doc = """\
-(list, optional): A list of exclude glob patterns to add as `data` to
-the generated `py_library` target.""",
- ),
- "hub_name": attr.string(
- doc = """\
-Name of the whl modification, hub we use this name to set the modifications for
-pip.parse. If you have different pip hubs you can use a different name,
-otherwise it is best practice to just use one.
-
-You cannot have the same `hub_name` in different modules. You can reuse the same
-name in the same module for different wheels that you put in the same hub, but you
-cannot have a child module that uses the same `hub_name`.
-""",
- mandatory = True,
- ),
- "srcs_exclude_glob": attr.string_list(
- doc = """\
-(list, optional): A list of labels to add as `srcs` to the generated
-`py_library` target.""",
- ),
- "whl_name": attr.string(
- doc = "The whl name that the modifications are used for.",
- mandatory = True,
- ),
- }
- return attrs
-
-def _extension_extra_args():
- args = {}
-
- if bazel_features.external_deps.module_extension_has_os_arch_dependent:
- args = args | {
- "arch_dependent": True,
- "os_dependent": True,
- }
-
- return args
-
-pip = module_extension(
- doc = """\
-This extension is used to make dependencies from pip available.
-
-pip.parse:
-To use, call `pip.parse()` and specify `hub_name` and your requirements file.
-Dependencies will be downloaded and made available in a repo named after the
-`hub_name` argument.
-
-Each `pip.parse()` call configures a particular Python version. Multiple calls
-can be made to configure different Python versions, and will be grouped by
-the `hub_name` argument. This allows the same logical name, e.g. `@pip//numpy`
-to automatically resolve to different, Python version-specific, libraries.
-
-pip.whl_mods:
-This tag class is used to help create JSON files to describe modifications to
-the BUILD files for wheels.
-""",
- implementation = _pip_impl,
- tag_classes = {
- "parse": tag_class(
- attrs = _pip_parse_ext_attrs(),
- doc = """\
-This tag class is used to create a pip hub and all of the spokes that are part of that hub.
-This tag class reuses most of the pip attributes that are found in
-@rules_python//python/pip_install:pip_repository.bzl.
-The exceptions are it does not use the args 'repo_prefix',
-and 'incompatible_generate_aliases'. We set the repository prefix
-for the user and the alias arg is always True in bzlmod.
-""",
- ),
- "whl_mods": tag_class(
- attrs = _whl_mod_attrs(),
- doc = """\
-This tag class is used to create JSON file that are used when calling wheel_builder.py. These
-JSON files contain instructions on how to modify a wheel's project. Each of the attributes
-create different modifications based on the type of attribute. Previously to bzlmod these
-JSON files where referred to as annotations, and were renamed to whl_modifications in this
-extension.
-""",
- ),
- },
- **_extension_extra_args()
-)
-
-def _whl_mods_repo_impl(rctx):
- rctx.file("BUILD.bazel", "")
- for whl_name, mods in rctx.attr.whl_mods.items():
- rctx.file("{}.json".format(whl_name), mods)
-
-_whl_mods_repo = repository_rule(
- doc = """\
-This rule creates json files based on the whl_mods attribute.
-""",
- implementation = _whl_mods_repo_impl,
- attrs = {
- "whl_mods": attr.string_dict(
- mandatory = True,
- doc = "JSON endcoded string that is provided to wheel_builder.py",
- ),
- },
-)
+pip = _pip
diff --git a/python/extensions/python.bzl b/python/extensions/python.bzl
index c7c2c82..5428b75 100644
--- a/python/extensions/python.bzl
+++ b/python/extensions/python.bzl
@@ -14,253 +14,6 @@
"Python toolchain module extensions for use with bzlmod"
-load("//python:repositories.bzl", "python_register_toolchains")
-load("//python/extensions/private:pythons_hub.bzl", "hub_repo")
-load("//python/private:toolchains_repo.bzl", "multi_toolchain_aliases")
+load("//python/private/bzlmod:python.bzl", _python = "python")
-# This limit can be increased essentially arbitrarily, but doing so will cause a rebuild of all
-# targets using any of these toolchains due to the changed repository name.
-_MAX_NUM_TOOLCHAINS = 9999
-_TOOLCHAIN_INDEX_PAD_LENGTH = len(str(_MAX_NUM_TOOLCHAINS))
-
-def _toolchain_prefix(index, name):
- """Prefixes the given name with the index, padded with zeros to ensure lexicographic sorting.
-
- Examples:
- _toolchain_prefix( 2, "foo") == "_0002_foo_"
- _toolchain_prefix(2000, "foo") == "_2000_foo_"
- """
- return "_{}_{}_".format(_left_pad_zero(index, _TOOLCHAIN_INDEX_PAD_LENGTH), name)
-
-def _left_pad_zero(index, length):
- if index < 0:
- fail("index must be non-negative")
- return ("0" * length + str(index))[-length:]
-
-# Printing a warning msg not debugging, so we have to disable
-# the buildifier check.
-# buildifier: disable=print
-def _print_warn(msg):
- print("WARNING:", msg)
-
-def _python_register_toolchains(name, toolchain_attr, version_constraint):
- """Calls python_register_toolchains and returns a struct used to collect the toolchains.
- """
- python_register_toolchains(
- name = name,
- python_version = toolchain_attr.python_version,
- register_coverage_tool = toolchain_attr.configure_coverage_tool,
- ignore_root_user_error = toolchain_attr.ignore_root_user_error,
- set_python_version_constraint = version_constraint,
- )
- return struct(
- python_version = toolchain_attr.python_version,
- set_python_version_constraint = str(version_constraint),
- name = name,
- )
-
-def _python_impl(module_ctx):
- # The toolchain info structs to register, in the order to register them in.
- toolchains = []
-
- # We store the default toolchain separately to ensure it is the last
- # toolchain added to toolchains.
- default_toolchain = None
-
- # Map of string Major.Minor to the toolchain name and module name
- global_toolchain_versions = {}
-
- for mod in module_ctx.modules:
- module_toolchain_versions = []
-
- for toolchain_attr in mod.tags.toolchain:
- toolchain_version = toolchain_attr.python_version
- toolchain_name = "python_" + toolchain_version.replace(".", "_")
-
- # Duplicate versions within a module indicate a misconfigured module.
- if toolchain_version in module_toolchain_versions:
- _fail_duplicate_module_toolchain_version(toolchain_version, mod.name)
- module_toolchain_versions.append(toolchain_version)
-
- # Ignore version collisions in the global scope because there isn't
- # much else that can be done. Modules don't know and can't control
- # what other modules do, so the first in the dependency graph wins.
- if toolchain_version in global_toolchain_versions:
- # If the python version is explicitly provided by the root
- # module, they should not be warned for choosing the same
- # version that rules_python provides as default.
- first = global_toolchain_versions[toolchain_version]
- if mod.name != "rules_python" or not first.is_root:
- _warn_duplicate_global_toolchain_version(
- toolchain_version,
- first = first,
- second_toolchain_name = toolchain_name,
- second_module_name = mod.name,
- )
- continue
- global_toolchain_versions[toolchain_version] = struct(
- toolchain_name = toolchain_name,
- module_name = mod.name,
- is_root = mod.is_root,
- )
-
- # Only the root module and rules_python are allowed to specify the default
- # toolchain for a couple reasons:
- # * It prevents submodules from specifying different defaults and only
- # one of them winning.
- # * rules_python needs to set a soft default in case the root module doesn't,
- # e.g. if the root module doesn't use Python itself.
- # * The root module is allowed to override the rules_python default.
- if mod.is_root:
- # A single toolchain is treated as the default because it's unambiguous.
- is_default = toolchain_attr.is_default or len(mod.tags.toolchain) == 1
- elif mod.name == "rules_python" and not default_toolchain:
- # We don't do the len() check because we want the default that rules_python
- # sets to be clearly visible.
- is_default = toolchain_attr.is_default
- else:
- is_default = False
-
- # We have already found one default toolchain, and we can only have
- # one.
- if is_default and default_toolchain != None:
- _fail_multiple_default_toolchains(
- first = default_toolchain.name,
- second = toolchain_name,
- )
-
- toolchain_info = _python_register_toolchains(
- toolchain_name,
- toolchain_attr,
- version_constraint = not is_default,
- )
-
- if is_default:
- default_toolchain = toolchain_info
- else:
- toolchains.append(toolchain_info)
-
- # A default toolchain is required so that the non-version-specific rules
- # are able to match a toolchain.
- if default_toolchain == None:
- fail("No default Python toolchain configured. Is rules_python missing `is_default=True`?")
-
- # The last toolchain in the BUILD file is set as the default
- # toolchain. We need the default last.
- toolchains.append(default_toolchain)
-
- if len(toolchains) > _MAX_NUM_TOOLCHAINS:
- fail("more than {} python versions are not supported".format(_MAX_NUM_TOOLCHAINS))
-
- # Create the pythons_hub repo for the interpreter meta data and the
- # the various toolchains.
- hub_repo(
- name = "pythons_hub",
- default_python_version = default_toolchain.python_version,
- toolchain_prefixes = [
- _toolchain_prefix(index, toolchain.name)
- for index, toolchain in enumerate(toolchains)
- ],
- toolchain_python_versions = [t.python_version for t in toolchains],
- toolchain_set_python_version_constraints = [t.set_python_version_constraint for t in toolchains],
- toolchain_user_repository_names = [t.name for t in toolchains],
- )
-
- # This is require in order to support multiple version py_test
- # and py_binary
- multi_toolchain_aliases(
- name = "python_versions",
- python_versions = {
- version: entry.toolchain_name
- for version, entry in global_toolchain_versions.items()
- },
- )
-
-def _fail_duplicate_module_toolchain_version(version, module):
- fail(("Duplicate module toolchain version: module '{module}' attempted " +
- "to use version '{version}' multiple times in itself").format(
- version = version,
- module = module,
- ))
-
-def _warn_duplicate_global_toolchain_version(version, first, second_toolchain_name, second_module_name):
- _print_warn((
- "Ignoring toolchain '{second_toolchain}' from module '{second_module}': " +
- "Toolchain '{first_toolchain}' from module '{first_module}' " +
- "already registered Python version {version} and has precedence"
- ).format(
- first_toolchain = first.toolchain_name,
- first_module = first.module_name,
- second_module = second_module_name,
- second_toolchain = second_toolchain_name,
- version = version,
- ))
-
-def _fail_multiple_default_toolchains(first, second):
- fail(("Multiple default toolchains: only one toolchain " +
- "can have is_default=True. First default " +
- "was toolchain '{first}'. Second was '{second}'").format(
- first = first,
- second = second,
- ))
-
-python = module_extension(
- doc = """Bzlmod extension that is used to register Python toolchains.
-""",
- implementation = _python_impl,
- tag_classes = {
- "toolchain": tag_class(
- doc = """Tag class used to register Python toolchains.
-Use this tag class to register one or more Python toolchains. This class
-is also potentially called by sub modules. The following covers different
-business rules and use cases.
-
-Toolchains in the Root Module
-
-This class registers all toolchains in the root module.
-
-Toolchains in Sub Modules
-
-It will create a toolchain that is in a sub module, if the toolchain
-of the same name does not exist in the root module. The extension stops name
-clashing between toolchains in the root module and toolchains in sub modules.
-You cannot configure more than one toolchain as the default toolchain.
-
-Toolchain set as the default version
-
-This extension will not create a toolchain that exists in a sub module,
-if the sub module toolchain is marked as the default version. If you have
-more than one toolchain in your root module, you need to set one of the
-toolchains as the default version. If there is only one toolchain it
-is set as the default toolchain.
-
-Toolchain repository name
-
-A toolchain's repository name uses the format `python_{major}_{minor}`, e.g.
-`python_3_10`. The `major` and `minor` components are
-`major` and `minor` are the Python version from the `python_version` attribute.
-""",
- attrs = {
- "configure_coverage_tool": attr.bool(
- mandatory = False,
- doc = "Whether or not to configure the default coverage tool for the toolchains.",
- ),
- "ignore_root_user_error": attr.bool(
- default = False,
- doc = "Whether the check for root should be ignored or not. This causes cache misses with .pyc files.",
- mandatory = False,
- ),
- "is_default": attr.bool(
- mandatory = False,
- doc = "Whether the toolchain is the default version",
- ),
- "python_version": attr.string(
- mandatory = True,
- doc = "The Python version, in `major.minor` format, e.g " +
- "'3.12', to create a toolchain for. Patch level " +
- "granularity (e.g. '3.12.1') is not supported.",
- ),
- },
- ),
- },
-)
+python = _python
diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel
index c071033..271cad5 100644
--- a/python/pip_install/BUILD.bazel
+++ b/python/pip_install/BUILD.bazel
@@ -38,6 +38,7 @@
"//python/private:render_pkg_aliases_bzl",
"//python/private:toolchains_repo_bzl",
"//python/private:which_bzl",
+ "//python/private/bzlmod:pip_repository_bzl",
],
)
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl
index ea8b9eb..5f829a9 100644
--- a/python/pip_install/pip_repository.bzl
+++ b/python/pip_install/pip_repository.bzl
@@ -25,6 +25,7 @@
load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases")
load("//python/private:toolchains_repo.bzl", "get_host_os_arch")
load("//python/private:which.bzl", "which_with_fail")
+load("//python/private/bzlmod:pip_repository.bzl", _pip_hub_repository_bzlmod = "pip_repository")
CPPFLAGS = "CPPFLAGS"
@@ -32,6 +33,9 @@
_WHEEL_ENTRY_POINT_PREFIX = "rules_python_wheel_entry_point"
+# Kept for not creating merge conflicts with PR#1476, can be removed later.
+pip_hub_repository_bzlmod = _pip_hub_repository_bzlmod
+
def _construct_pypath(rctx):
"""Helper function to construct a PYTHONPATH.
@@ -267,68 +271,6 @@
""")
return requirements_txt
-def _pip_hub_repository_bzlmod_impl(rctx):
- bzl_packages = rctx.attr.whl_map.keys()
- aliases = render_pkg_aliases(
- repo_name = rctx.attr.repo_name,
- rules_python = rctx.attr._template.workspace_name,
- default_version = rctx.attr.default_version,
- whl_map = rctx.attr.whl_map,
- )
- for path, contents in aliases.items():
- rctx.file(path, contents)
-
- # NOTE: we are using the canonical name with the double '@' in order to
- # always uniquely identify a repository, as the labels are being passed as
- # a string and the resolution of the label happens at the call-site of the
- # `requirement`, et al. macros.
- macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name)
-
- rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS)
- rctx.template("requirements.bzl", rctx.attr._template, substitutions = {
- "%%ALL_DATA_REQUIREMENTS%%": _format_repr_list([
- macro_tmpl.format(p, "data")
- for p in bzl_packages
- ]),
- "%%ALL_REQUIREMENTS%%": _format_repr_list([
- macro_tmpl.format(p, p)
- for p in bzl_packages
- ]),
- "%%ALL_WHL_REQUIREMENTS%%": _format_repr_list([
- macro_tmpl.format(p, "whl")
- for p in bzl_packages
- ]),
- "%%MACRO_TMPL%%": macro_tmpl,
- "%%NAME%%": rctx.attr.name,
- })
-
-pip_hub_repository_bzlmod_attrs = {
- "default_version": attr.string(
- mandatory = True,
- doc = """\
-This is the default python version in the format of X.Y.Z. This should match
-what is setup by the 'python' extension using the 'is_default = True'
-setting.""",
- ),
- "repo_name": attr.string(
- mandatory = True,
- doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.",
- ),
- "whl_map": attr.string_list_dict(
- mandatory = True,
- doc = "The wheel map where values are python versions",
- ),
- "_template": attr.label(
- default = ":pip_repository_requirements_bzlmod.bzl.tmpl",
- ),
-}
-
-pip_hub_repository_bzlmod = repository_rule(
- attrs = pip_hub_repository_bzlmod_attrs,
- doc = """A rule for bzlmod mulitple pip repository creation. PRIVATE USE ONLY.""",
- implementation = _pip_hub_repository_bzlmod_impl,
-)
-
def _pip_repository_impl(rctx):
requirements_txt = locked_requirements_label(rctx, rctx.attr)
content = rctx.read(requirements_txt)
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
index beda50f..b8b8e51 100644
--- a/python/private/BUILD.bazel
+++ b/python/private/BUILD.bazel
@@ -27,6 +27,7 @@
filegroup(
name = "distribution",
srcs = glob(["**"]) + [
+ "//python/private/bzlmod:distribution",
"//python/private/common:distribution",
"//python/private/proto:distribution",
"//tools/build_defs/python/private:distribution",
diff --git a/python/extensions/private/BUILD.bazel b/python/private/bzlmod/BUILD.bazel
similarity index 68%
rename from python/extensions/private/BUILD.bazel
rename to python/private/bzlmod/BUILD.bazel
index f367b71..fc8449e 100644
--- a/python/extensions/private/BUILD.bazel
+++ b/python/private/bzlmod/BUILD.bazel
@@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+
package(default_visibility = ["//visibility:private"])
licenses(["notice"])
@@ -19,5 +21,15 @@
filegroup(
name = "distribution",
srcs = glob(["**"]),
- visibility = ["//python/extensions/private:__pkg__"],
+ visibility = ["//python/private:__pkg__"],
+)
+
+bzl_library(
+ name = "pip_repository_bzl",
+ srcs = ["pip_repository.bzl"],
+ visibility = ["//:__subpackages__"],
+ deps = [
+ "//python/private:render_pkg_aliases_bzl",
+ "//python/private:text_util_bzl",
+ ],
)
diff --git a/python/extensions/private/internal_deps.bzl b/python/private/bzlmod/internal_deps.bzl
similarity index 100%
rename from python/extensions/private/internal_deps.bzl
rename to python/private/bzlmod/internal_deps.bzl
diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl
new file mode 100644
index 0000000..3630648
--- /dev/null
+++ b/python/private/bzlmod/pip.bzl
@@ -0,0 +1,456 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"pip module extension for use with bzlmod"
+
+load("@bazel_features//:features.bzl", "bazel_features")
+load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS")
+load(
+ "//python/pip_install:pip_repository.bzl",
+ "locked_requirements_label",
+ "pip_repository_attrs",
+ "use_isolated",
+ "whl_library",
+)
+load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse")
+load("//python/private:full_version.bzl", "full_version")
+load("//python/private:normalize_name.bzl", "normalize_name")
+load("//python/private:version_label.bzl", "version_label")
+load(":pip_repository.bzl", "pip_repository")
+
+def _whl_mods_impl(mctx):
+ """Implementation of the pip.whl_mods tag class.
+
+ This creates the JSON files used to modify the creation of different wheels.
+"""
+ whl_mods_dict = {}
+ for mod in mctx.modules:
+ for whl_mod_attr in mod.tags.whl_mods:
+ if whl_mod_attr.hub_name not in whl_mods_dict.keys():
+ whl_mods_dict[whl_mod_attr.hub_name] = {whl_mod_attr.whl_name: whl_mod_attr}
+ elif whl_mod_attr.whl_name in whl_mods_dict[whl_mod_attr.hub_name].keys():
+ # We cannot have the same wheel name in the same hub, as we
+ # will create the same JSON file name.
+ fail("""\
+Found same whl_name '{}' in the same hub '{}', please use a different hub_name.""".format(
+ whl_mod_attr.whl_name,
+ whl_mod_attr.hub_name,
+ ))
+ else:
+ whl_mods_dict[whl_mod_attr.hub_name][whl_mod_attr.whl_name] = whl_mod_attr
+
+ for hub_name, whl_maps in whl_mods_dict.items():
+ whl_mods = {}
+
+ # create a struct that we can pass to the _whl_mods_repo rule
+ # to create the different JSON files.
+ for whl_name, mods in whl_maps.items():
+ build_content = mods.additive_build_content
+ if mods.additive_build_content_file != None and mods.additive_build_content != "":
+ fail("""\
+You cannot use both the additive_build_content and additive_build_content_file arguments at the same time.
+""")
+ elif mods.additive_build_content_file != None:
+ build_content = mctx.read(mods.additive_build_content_file)
+
+ whl_mods[whl_name] = json.encode(struct(
+ additive_build_content = build_content,
+ copy_files = mods.copy_files,
+ copy_executables = mods.copy_executables,
+ data = mods.data,
+ data_exclude_glob = mods.data_exclude_glob,
+ srcs_exclude_glob = mods.srcs_exclude_glob,
+ ))
+
+ _whl_mods_repo(
+ name = hub_name,
+ whl_mods = whl_mods,
+ )
+
+def _create_whl_repos(module_ctx, pip_attr, whl_map):
+ python_interpreter_target = pip_attr.python_interpreter_target
+
+ # if we do not have the python_interpreter set in the attributes
+ # we programmatically find it.
+ hub_name = pip_attr.hub_name
+ if python_interpreter_target == None:
+ python_name = "python_" + version_label(pip_attr.python_version, sep = "_")
+ if python_name not in INTERPRETER_LABELS.keys():
+ fail((
+ "Unable to find interpreter for pip hub '{hub_name}' for " +
+ "python_version={version}: Make sure a corresponding " +
+ '`python.toolchain(python_version="{version}")` call exists'
+ ).format(
+ hub_name = hub_name,
+ version = pip_attr.python_version,
+ ))
+ python_interpreter_target = INTERPRETER_LABELS[python_name]
+
+ pip_name = "{}_{}".format(
+ hub_name,
+ version_label(pip_attr.python_version),
+ )
+ requrements_lock = locked_requirements_label(module_ctx, pip_attr)
+
+ # Parse the requirements file directly in starlark to get the information
+ # needed for the whl_libary declarations below.
+ requirements_lock_content = module_ctx.read(requrements_lock)
+ parse_result = parse_requirements(requirements_lock_content)
+ requirements = parse_result.requirements
+ extra_pip_args = pip_attr.extra_pip_args + parse_result.options
+
+ if hub_name not in whl_map:
+ whl_map[hub_name] = {}
+
+ whl_modifications = {}
+ if pip_attr.whl_modifications != None:
+ for mod, whl_name in pip_attr.whl_modifications.items():
+ whl_modifications[whl_name] = mod
+
+ # Create a new wheel library for each of the different whls
+ for whl_name, requirement_line in requirements:
+ # We are not using the "sanitized name" because the user
+ # would need to guess what name we modified the whl name
+ # to.
+ annotation = whl_modifications.get(whl_name)
+ whl_name = normalize_name(whl_name)
+ whl_library(
+ name = "%s_%s" % (pip_name, whl_name),
+ requirement = requirement_line,
+ repo = pip_name,
+ repo_prefix = pip_name + "_",
+ annotation = annotation,
+ python_interpreter = pip_attr.python_interpreter,
+ python_interpreter_target = python_interpreter_target,
+ quiet = pip_attr.quiet,
+ timeout = pip_attr.timeout,
+ isolated = use_isolated(module_ctx, pip_attr),
+ extra_pip_args = extra_pip_args,
+ download_only = pip_attr.download_only,
+ pip_data_exclude = pip_attr.pip_data_exclude,
+ enable_implicit_namespace_pkgs = pip_attr.enable_implicit_namespace_pkgs,
+ environment = pip_attr.environment,
+ )
+
+ if whl_name not in whl_map[hub_name]:
+ whl_map[hub_name][whl_name] = {}
+
+ whl_map[hub_name][whl_name][full_version(pip_attr.python_version)] = pip_name + "_"
+
+def _pip_impl(module_ctx):
+ """Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories.
+
+ This implementation iterates through all of the `pip.parse` calls and creates
+ different pip hub repositories based on the "hub_name". Each of the
+ pip calls create spoke repos that uses a specific Python interpreter.
+
+ In a MODULES.bazel file we have:
+
+ pip.parse(
+ hub_name = "pip",
+ python_version = 3.9,
+ requirements_lock = "//:requirements_lock_3_9.txt",
+ requirements_windows = "//:requirements_windows_3_9.txt",
+ )
+ pip.parse(
+ hub_name = "pip",
+ python_version = 3.10,
+ requirements_lock = "//:requirements_lock_3_10.txt",
+ requirements_windows = "//:requirements_windows_3_10.txt",
+ )
+
+ For instance, we have a hub with the name of "pip".
+ A repository named the following is created. It is actually called last when
+ all of the pip spokes are collected.
+
+ - @@rules_python~override~pip~pip
+
+ As shown in the example code above we have the following.
+ Two different pip.parse statements exist in MODULE.bazel provide the hub_name "pip".
+ These definitions create two different pip spoke repositories that are
+ related to the hub "pip".
+ One spoke uses Python 3.9 and the other uses Python 3.10. This code automatically
+ determines the Python version and the interpreter.
+ Both of these pip spokes contain requirements files that includes websocket
+ and its dependencies.
+
+ We also need repositories for the wheels that the different pip spokes contain.
+ For each Python version a different wheel repository is created. In our example
+ each pip spoke had a requirements file that contained websockets. We
+ then create two different wheel repositories that are named the following.
+
+ - @@rules_python~override~pip~pip_39_websockets
+ - @@rules_python~override~pip~pip_310_websockets
+
+ And if the wheel has any other dependencies subsequent wheels are created in the same fashion.
+
+ The hub repository has aliases for `pkg`, `data`, etc, which have a select that resolves to
+ a spoke repository depending on the Python version.
+
+ Also we may have more than one hub as defined in a MODULES.bazel file. So we could have multiple
+ hubs pointing to various different pip spokes.
+
+ Some other business rules notes. A hub can only have one spoke per Python version. We cannot
+ have a hub named "pip" that has two spokes that use the Python 3.9 interpreter. Second
+ we cannot have the same hub name used in sub-modules. The hub name has to be globally
+ unique.
+
+ This implementation also handles the creation of whl_modification JSON files that are used
+ during the creation of wheel libraries. These JSON files used via the annotations argument
+ when calling wheel_installer.py.
+
+ Args:
+ module_ctx: module contents
+ """
+
+ # Build all of the wheel modifications if the tag class is called.
+ _whl_mods_impl(module_ctx)
+
+ # Used to track all the different pip hubs and the spoke pip Python
+ # versions.
+ pip_hub_map = {}
+
+ # Keeps track of all the hub's whl repos across the different versions.
+ # dict[hub, dict[whl, dict[version, str pip]]]
+ # Where hub, whl, and pip are the repo names
+ hub_whl_map = {}
+
+ for mod in module_ctx.modules:
+ for pip_attr in mod.tags.parse:
+ hub_name = pip_attr.hub_name
+ if hub_name not in pip_hub_map:
+ pip_hub_map[pip_attr.hub_name] = struct(
+ module_name = mod.name,
+ python_versions = [pip_attr.python_version],
+ )
+ elif pip_hub_map[hub_name].module_name != mod.name:
+ # We cannot have two hubs with the same name in different
+ # modules.
+ fail((
+ "Duplicate cross-module pip hub named '{hub}': pip hub " +
+ "names must be unique across modules. First defined " +
+ "by module '{first_module}', second attempted by " +
+ "module '{second_module}'"
+ ).format(
+ hub = hub_name,
+ first_module = pip_hub_map[hub_name].module_name,
+ second_module = mod.name,
+ ))
+
+ elif pip_attr.python_version in pip_hub_map[hub_name].python_versions:
+ fail((
+ "Duplicate pip python version '{version}' for hub " +
+ "'{hub}' in module '{module}': the Python versions " +
+ "used for a hub must be unique"
+ ).format(
+ hub = hub_name,
+ module = mod.name,
+ version = pip_attr.python_version,
+ ))
+ else:
+ pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version)
+
+ _create_whl_repos(module_ctx, pip_attr, hub_whl_map)
+
+ for hub_name, whl_map in hub_whl_map.items():
+ pip_repository(
+ name = hub_name,
+ repo_name = hub_name,
+ whl_map = whl_map,
+ default_version = full_version(DEFAULT_PYTHON_VERSION),
+ )
+
+def _pip_parse_ext_attrs():
+ attrs = dict({
+ "hub_name": attr.string(
+ mandatory = True,
+ doc = """
+The name of the repo pip dependencies will be accessible from.
+
+This name must be unique between modules; unless your module is guaranteed to
+always be the root module, it's highly recommended to include your module name
+in the hub name. Repo mapping, `use_repo(..., pip="my_modules_pip_deps")`, can
+be used for shorter local names within your module.
+
+Within a module, the same `hub_name` can be specified to group different Python
+versions of pip dependencies under one repository name. This allows using a
+Python version-agnostic name when referring to pip dependencies; the
+correct version will be automatically selected.
+
+Typically, a module will only have a single hub of pip dependencies, but this
+is not required. Each hub is a separate resolution of pip dependencies. This
+means if different programs need different versions of some library, separate
+hubs can be created, and each program can use its respective hub's targets.
+Targets from different hubs should not be used together.
+""",
+ ),
+ "python_version": attr.string(
+ mandatory = True,
+ doc = """
+The Python version to use for resolving the pip dependencies, in Major.Minor
+format (e.g. "3.11"). Patch level granularity (e.g. "3.11.1") is not supported.
+If not specified, then the default Python version (as set by the root module or
+rules_python) will be used.
+
+The version specified here must have a corresponding `python.toolchain()`
+configured.
+""",
+ ),
+ "whl_modifications": attr.label_keyed_string_dict(
+ mandatory = False,
+ doc = """\
+A dict of labels to wheel names that is typically generated by the whl_modifications.
+The labels are JSON config files describing the modifications.
+""",
+ ),
+ }, **pip_repository_attrs)
+
+ # Like the pip_repository rule, we end up setting this manually so
+ # don't allow users to override it.
+ attrs.pop("repo_prefix")
+
+ # incompatible_generate_aliases is always True in bzlmod
+ attrs.pop("incompatible_generate_aliases")
+
+ return attrs
+
+def _whl_mod_attrs():
+ attrs = {
+ "additive_build_content": attr.string(
+ doc = "(str, optional): Raw text to add to the generated `BUILD` file of a package.",
+ ),
+ "additive_build_content_file": attr.label(
+ doc = """\
+(label, optional): path to a BUILD file to add to the generated
+`BUILD` file of a package. You cannot use both additive_build_content and additive_build_content_file
+arguments at the same time.""",
+ ),
+ "copy_executables": attr.string_dict(
+ doc = """\
+(dict, optional): A mapping of `src` and `out` files for
+[@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as
+executable.""",
+ ),
+ "copy_files": attr.string_dict(
+ doc = """\
+(dict, optional): A mapping of `src` and `out` files for
+[@bazel_skylib//rules:copy_file.bzl][cf]""",
+ ),
+ "data": attr.string_list(
+ doc = """\
+(list, optional): A list of labels to add as `data` dependencies to
+the generated `py_library` target.""",
+ ),
+ "data_exclude_glob": attr.string_list(
+ doc = """\
+(list, optional): A list of exclude glob patterns to add as `data` to
+the generated `py_library` target.""",
+ ),
+ "hub_name": attr.string(
+ doc = """\
+Name of the whl modification, hub we use this name to set the modifications for
+pip.parse. If you have different pip hubs you can use a different name,
+otherwise it is best practice to just use one.
+
+You cannot have the same `hub_name` in different modules. You can reuse the same
+name in the same module for different wheels that you put in the same hub, but you
+cannot have a child module that uses the same `hub_name`.
+""",
+ mandatory = True,
+ ),
+ "srcs_exclude_glob": attr.string_list(
+ doc = """\
+(list, optional): A list of labels to add as `srcs` to the generated
+`py_library` target.""",
+ ),
+ "whl_name": attr.string(
+ doc = "The whl name that the modifications are used for.",
+ mandatory = True,
+ ),
+ }
+ return attrs
+
+def _extension_extra_args():
+ args = {}
+
+ if bazel_features.external_deps.module_extension_has_os_arch_dependent:
+ args = args | {
+ "arch_dependent": True,
+ "os_dependent": True,
+ }
+
+ return args
+
+pip = module_extension(
+ doc = """\
+This extension is used to make dependencies from pip available.
+
+pip.parse:
+To use, call `pip.parse()` and specify `hub_name` and your requirements file.
+Dependencies will be downloaded and made available in a repo named after the
+`hub_name` argument.
+
+Each `pip.parse()` call configures a particular Python version. Multiple calls
+can be made to configure different Python versions, and will be grouped by
+the `hub_name` argument. This allows the same logical name, e.g. `@pip//numpy`
+to automatically resolve to different, Python version-specific, libraries.
+
+pip.whl_mods:
+This tag class is used to help create JSON files to describe modifications to
+the BUILD files for wheels.
+""",
+ implementation = _pip_impl,
+ tag_classes = {
+ "parse": tag_class(
+ attrs = _pip_parse_ext_attrs(),
+ doc = """\
+This tag class is used to create a pip hub and all of the spokes that are part of that hub.
+This tag class reuses most of the pip attributes that are found in
+@rules_python//python/pip_install:pip_repository.bzl.
+The exceptions are it does not use the args 'repo_prefix',
+and 'incompatible_generate_aliases'. We set the repository prefix
+for the user and the alias arg is always True in bzlmod.
+""",
+ ),
+ "whl_mods": tag_class(
+ attrs = _whl_mod_attrs(),
+ doc = """\
+This tag class is used to create JSON file that are used when calling wheel_builder.py. These
+JSON files contain instructions on how to modify a wheel's project. Each of the attributes
+create different modifications based on the type of attribute. Previously to bzlmod these
+JSON files where referred to as annotations, and were renamed to whl_modifications in this
+extension.
+""",
+ ),
+ },
+ **_extension_extra_args()
+)
+
+def _whl_mods_repo_impl(rctx):
+ rctx.file("BUILD.bazel", "")
+ for whl_name, mods in rctx.attr.whl_mods.items():
+ rctx.file("{}.json".format(whl_name), mods)
+
+_whl_mods_repo = repository_rule(
+ doc = """\
+This rule creates json files based on the whl_mods attribute.
+""",
+ implementation = _whl_mods_repo_impl,
+ attrs = {
+ "whl_mods": attr.string_dict(
+ mandatory = True,
+ doc = "JSON endcoded string that is provided to wheel_builder.py",
+ ),
+ },
+)
diff --git a/python/private/bzlmod/pip_repository.bzl b/python/private/bzlmod/pip_repository.bzl
new file mode 100644
index 0000000..f5bb46f
--- /dev/null
+++ b/python/private/bzlmod/pip_repository.bzl
@@ -0,0 +1,87 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""
+
+load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases")
+load("//python/private:text_util.bzl", "render")
+
+_BUILD_FILE_CONTENTS = """\
+package(default_visibility = ["//visibility:public"])
+
+# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it
+exports_files(["requirements.bzl"])
+"""
+
+def _pip_repository_impl(rctx):
+ bzl_packages = rctx.attr.whl_map.keys()
+ aliases = render_pkg_aliases(
+ repo_name = rctx.attr.repo_name,
+ rules_python = rctx.attr._template.workspace_name,
+ default_version = rctx.attr.default_version,
+ whl_map = rctx.attr.whl_map,
+ )
+ for path, contents in aliases.items():
+ rctx.file(path, contents)
+
+ # NOTE: we are using the canonical name with the double '@' in order to
+ # always uniquely identify a repository, as the labels are being passed as
+ # a string and the resolution of the label happens at the call-site of the
+ # `requirement`, et al. macros.
+ macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name)
+
+ rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS)
+ rctx.template("requirements.bzl", rctx.attr._template, substitutions = {
+ "%%ALL_DATA_REQUIREMENTS%%": render.list([
+ macro_tmpl.format(p, "data")
+ for p in bzl_packages
+ ]),
+ "%%ALL_REQUIREMENTS%%": render.list([
+ macro_tmpl.format(p, p)
+ for p in bzl_packages
+ ]),
+ "%%ALL_WHL_REQUIREMENTS%%": render.list([
+ macro_tmpl.format(p, "whl")
+ for p in bzl_packages
+ ]),
+ "%%MACRO_TMPL%%": macro_tmpl,
+ "%%NAME%%": rctx.attr.name,
+ })
+
+pip_repository_attrs = {
+ "default_version": attr.string(
+ mandatory = True,
+ doc = """\
+This is the default python version in the format of X.Y.Z. This should match
+what is setup by the 'python' extension using the 'is_default = True'
+setting.""",
+ ),
+ "repo_name": attr.string(
+ mandatory = True,
+ doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.",
+ ),
+ "whl_map": attr.string_list_dict(
+ mandatory = True,
+ doc = "The wheel map where values are python versions",
+ ),
+ "_template": attr.label(
+ default = ":requirements.bzl.tmpl",
+ ),
+}
+
+pip_repository = repository_rule(
+ attrs = pip_repository_attrs,
+ doc = """A rule for bzlmod mulitple pip repository creation. PRIVATE USE ONLY.""",
+ implementation = _pip_repository_impl,
+)
diff --git a/python/private/bzlmod/python.bzl b/python/private/bzlmod/python.bzl
new file mode 100644
index 0000000..be5c083
--- /dev/null
+++ b/python/private/bzlmod/python.bzl
@@ -0,0 +1,266 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"Python toolchain module extensions for use with bzlmod"
+
+load("//python:repositories.bzl", "python_register_toolchains")
+load("//python/private:toolchains_repo.bzl", "multi_toolchain_aliases")
+load(":pythons_hub.bzl", "hub_repo")
+
+# This limit can be increased essentially arbitrarily, but doing so will cause a rebuild of all
+# targets using any of these toolchains due to the changed repository name.
+_MAX_NUM_TOOLCHAINS = 9999
+_TOOLCHAIN_INDEX_PAD_LENGTH = len(str(_MAX_NUM_TOOLCHAINS))
+
+def _toolchain_prefix(index, name):
+ """Prefixes the given name with the index, padded with zeros to ensure lexicographic sorting.
+
+ Examples:
+ _toolchain_prefix( 2, "foo") == "_0002_foo_"
+ _toolchain_prefix(2000, "foo") == "_2000_foo_"
+ """
+ return "_{}_{}_".format(_left_pad_zero(index, _TOOLCHAIN_INDEX_PAD_LENGTH), name)
+
+def _left_pad_zero(index, length):
+ if index < 0:
+ fail("index must be non-negative")
+ return ("0" * length + str(index))[-length:]
+
+# Printing a warning msg not debugging, so we have to disable
+# the buildifier check.
+# buildifier: disable=print
+def _print_warn(msg):
+ print("WARNING:", msg)
+
+def _python_register_toolchains(name, toolchain_attr, version_constraint):
+ """Calls python_register_toolchains and returns a struct used to collect the toolchains.
+ """
+ python_register_toolchains(
+ name = name,
+ python_version = toolchain_attr.python_version,
+ register_coverage_tool = toolchain_attr.configure_coverage_tool,
+ ignore_root_user_error = toolchain_attr.ignore_root_user_error,
+ set_python_version_constraint = version_constraint,
+ )
+ return struct(
+ python_version = toolchain_attr.python_version,
+ set_python_version_constraint = str(version_constraint),
+ name = name,
+ )
+
+def _python_impl(module_ctx):
+ # The toolchain info structs to register, in the order to register them in.
+ toolchains = []
+
+ # We store the default toolchain separately to ensure it is the last
+ # toolchain added to toolchains.
+ default_toolchain = None
+
+ # Map of string Major.Minor to the toolchain name and module name
+ global_toolchain_versions = {}
+
+ for mod in module_ctx.modules:
+ module_toolchain_versions = []
+
+ for toolchain_attr in mod.tags.toolchain:
+ toolchain_version = toolchain_attr.python_version
+ toolchain_name = "python_" + toolchain_version.replace(".", "_")
+
+ # Duplicate versions within a module indicate a misconfigured module.
+ if toolchain_version in module_toolchain_versions:
+ _fail_duplicate_module_toolchain_version(toolchain_version, mod.name)
+ module_toolchain_versions.append(toolchain_version)
+
+ # Ignore version collisions in the global scope because there isn't
+ # much else that can be done. Modules don't know and can't control
+ # what other modules do, so the first in the dependency graph wins.
+ if toolchain_version in global_toolchain_versions:
+ # If the python version is explicitly provided by the root
+ # module, they should not be warned for choosing the same
+ # version that rules_python provides as default.
+ first = global_toolchain_versions[toolchain_version]
+ if mod.name != "rules_python" or not first.is_root:
+ _warn_duplicate_global_toolchain_version(
+ toolchain_version,
+ first = first,
+ second_toolchain_name = toolchain_name,
+ second_module_name = mod.name,
+ )
+ continue
+ global_toolchain_versions[toolchain_version] = struct(
+ toolchain_name = toolchain_name,
+ module_name = mod.name,
+ is_root = mod.is_root,
+ )
+
+ # Only the root module and rules_python are allowed to specify the default
+ # toolchain for a couple reasons:
+ # * It prevents submodules from specifying different defaults and only
+ # one of them winning.
+ # * rules_python needs to set a soft default in case the root module doesn't,
+ # e.g. if the root module doesn't use Python itself.
+ # * The root module is allowed to override the rules_python default.
+ if mod.is_root:
+ # A single toolchain is treated as the default because it's unambiguous.
+ is_default = toolchain_attr.is_default or len(mod.tags.toolchain) == 1
+ elif mod.name == "rules_python" and not default_toolchain:
+ # We don't do the len() check because we want the default that rules_python
+ # sets to be clearly visible.
+ is_default = toolchain_attr.is_default
+ else:
+ is_default = False
+
+ # We have already found one default toolchain, and we can only have
+ # one.
+ if is_default and default_toolchain != None:
+ _fail_multiple_default_toolchains(
+ first = default_toolchain.name,
+ second = toolchain_name,
+ )
+
+ toolchain_info = _python_register_toolchains(
+ toolchain_name,
+ toolchain_attr,
+ version_constraint = not is_default,
+ )
+
+ if is_default:
+ default_toolchain = toolchain_info
+ else:
+ toolchains.append(toolchain_info)
+
+ # A default toolchain is required so that the non-version-specific rules
+ # are able to match a toolchain.
+ if default_toolchain == None:
+ fail("No default Python toolchain configured. Is rules_python missing `is_default=True`?")
+
+ # The last toolchain in the BUILD file is set as the default
+ # toolchain. We need the default last.
+ toolchains.append(default_toolchain)
+
+ if len(toolchains) > _MAX_NUM_TOOLCHAINS:
+ fail("more than {} python versions are not supported".format(_MAX_NUM_TOOLCHAINS))
+
+ # Create the pythons_hub repo for the interpreter meta data and the
+ # the various toolchains.
+ hub_repo(
+ name = "pythons_hub",
+ default_python_version = default_toolchain.python_version,
+ toolchain_prefixes = [
+ _toolchain_prefix(index, toolchain.name)
+ for index, toolchain in enumerate(toolchains)
+ ],
+ toolchain_python_versions = [t.python_version for t in toolchains],
+ toolchain_set_python_version_constraints = [t.set_python_version_constraint for t in toolchains],
+ toolchain_user_repository_names = [t.name for t in toolchains],
+ )
+
+ # This is require in order to support multiple version py_test
+ # and py_binary
+ multi_toolchain_aliases(
+ name = "python_versions",
+ python_versions = {
+ version: entry.toolchain_name
+ for version, entry in global_toolchain_versions.items()
+ },
+ )
+
+def _fail_duplicate_module_toolchain_version(version, module):
+ fail(("Duplicate module toolchain version: module '{module}' attempted " +
+ "to use version '{version}' multiple times in itself").format(
+ version = version,
+ module = module,
+ ))
+
+def _warn_duplicate_global_toolchain_version(version, first, second_toolchain_name, second_module_name):
+ _print_warn((
+ "Ignoring toolchain '{second_toolchain}' from module '{second_module}': " +
+ "Toolchain '{first_toolchain}' from module '{first_module}' " +
+ "already registered Python version {version} and has precedence"
+ ).format(
+ first_toolchain = first.toolchain_name,
+ first_module = first.module_name,
+ second_module = second_module_name,
+ second_toolchain = second_toolchain_name,
+ version = version,
+ ))
+
+def _fail_multiple_default_toolchains(first, second):
+ fail(("Multiple default toolchains: only one toolchain " +
+ "can have is_default=True. First default " +
+ "was toolchain '{first}'. Second was '{second}'").format(
+ first = first,
+ second = second,
+ ))
+
+python = module_extension(
+ doc = """Bzlmod extension that is used to register Python toolchains.
+""",
+ implementation = _python_impl,
+ tag_classes = {
+ "toolchain": tag_class(
+ doc = """Tag class used to register Python toolchains.
+Use this tag class to register one or more Python toolchains. This class
+is also potentially called by sub modules. The following covers different
+business rules and use cases.
+
+Toolchains in the Root Module
+
+This class registers all toolchains in the root module.
+
+Toolchains in Sub Modules
+
+It will create a toolchain that is in a sub module, if the toolchain
+of the same name does not exist in the root module. The extension stops name
+clashing between toolchains in the root module and toolchains in sub modules.
+You cannot configure more than one toolchain as the default toolchain.
+
+Toolchain set as the default version
+
+This extension will not create a toolchain that exists in a sub module,
+if the sub module toolchain is marked as the default version. If you have
+more than one toolchain in your root module, you need to set one of the
+toolchains as the default version. If there is only one toolchain it
+is set as the default toolchain.
+
+Toolchain repository name
+
+A toolchain's repository name uses the format `python_{major}_{minor}`, e.g.
+`python_3_10`. The `major` and `minor` components are
+`major` and `minor` are the Python version from the `python_version` attribute.
+""",
+ attrs = {
+ "configure_coverage_tool": attr.bool(
+ mandatory = False,
+ doc = "Whether or not to configure the default coverage tool for the toolchains.",
+ ),
+ "ignore_root_user_error": attr.bool(
+ default = False,
+ doc = "Whether the check for root should be ignored or not. This causes cache misses with .pyc files.",
+ mandatory = False,
+ ),
+ "is_default": attr.bool(
+ mandatory = False,
+ doc = "Whether the toolchain is the default version",
+ ),
+ "python_version": attr.string(
+ mandatory = True,
+ doc = "The Python version, in `major.minor` format, e.g " +
+ "'3.12', to create a toolchain for. Patch level " +
+ "granularity (e.g. '3.12.1') is not supported.",
+ ),
+ },
+ ),
+ },
+)
diff --git a/python/extensions/private/pythons_hub.bzl b/python/private/bzlmod/pythons_hub.bzl
similarity index 100%
rename from python/extensions/private/pythons_hub.bzl
rename to python/private/bzlmod/pythons_hub.bzl
diff --git a/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl b/python/private/bzlmod/requirements.bzl.tmpl
similarity index 100%
rename from python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl
rename to python/private/bzlmod/requirements.bzl.tmpl
diff --git a/python/private/text_util.bzl b/python/private/text_util.bzl
index 3d72b8d..da67001 100644
--- a/python/private/text_util.bzl
+++ b/python/private/text_util.bzl
@@ -57,9 +57,20 @@
return "select({})".format(args)
+def _render_list(items):
+ return "\n".join([
+ "[",
+ _indent("\n".join([
+ "{},".format(repr(item))
+ for item in items
+ ])),
+ "]",
+ ])
+
render = struct(
- indent = _indent,
alias = _render_alias,
dict = _render_dict,
+ indent = _indent,
+ list = _render_list,
select = _render_select,
)
