refactor: move PyPI related extension and repository_rule setup to its own dir (#2003)
This is to ensure that future work is easier and its clear where to add
tests.
Now all of the unit tests can be run by just `bazel test
//tests/pypi/...`.
Refactor summary:
- chore: add aignas to CODEOWNERS
- chore: add a new directory for storing PyPI related code
- move pypi_index_sources.bzl to private/pypi
- chore: move parse_requirements_txt to private/pypi
- move parse_whl_name to private/pypi
- move whl_target_platforms to private/pypi
- move parse_requirements to private/pypi
- move pip_repo_name to private/pypi
- remove unused file
- move pip_config_settings to private/pypi
- move pypi_index to pypi/private and rename
- move labels.bzl to private/pypi
- move generate_build_bazel to private/pypi
- move render_pkg_aliases.bzl to private/pypi
- move patch_whl.bzl to private/pypi
- Move envsubst and render_tests to top level of tests
- move pip_install_srcs to private/pypi
- split and move pip_install/pip_repository.bzl to private/pypi
- move the bzlmod extension to private/pypi
diff --git a/python/BUILD.bazel b/python/BUILD.bazel
index cbf2996..29b495b 100644
--- a/python/BUILD.bazel
+++ b/python/BUILD.bazel
@@ -101,7 +101,7 @@
"//python/pip_install:requirements_bzl",
"//python/private:bzlmod_enabled_bzl",
"//python/private:full_version_bzl",
- "//python/private:render_pkg_aliases_bzl",
+ "//python/private/pypi:render_pkg_aliases_bzl",
"//python/private/whl_filegroup:whl_filegroup_bzl",
],
)
diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel
index 1003efb..f6f46db 100644
--- a/python/config_settings/BUILD.bazel
+++ b/python/config_settings/BUILD.bazel
@@ -9,7 +9,7 @@
"PycCollectionFlag",
)
load(
- "//python/private:pip_flags.bzl",
+ "//python/private/pypi:flags.bzl",
"INTERNAL_FLAGS",
"UniversalWhlFlag",
"UseWhlFlag",
diff --git a/python/extensions/BUILD.bazel b/python/extensions/BUILD.bazel
index eb095ab..1bc2a71 100644
--- a/python/extensions/BUILD.bazel
+++ b/python/extensions/BUILD.bazel
@@ -36,8 +36,6 @@
srcs = ["python.bzl"],
visibility = ["//:__subpackages__"],
deps = [
- "//python/private:util_bzl",
- "//python/private/bzlmod:bazel_features_bzl",
"//python/private/bzlmod:python_bzl",
],
)
diff --git a/python/pip.bzl b/python/pip.bzl
index 8cc091d..f1c74dd 100644
--- a/python/pip.bzl
+++ b/python/pip.bzl
@@ -19,12 +19,13 @@
for internal use only.
"""
-load("//python/pip_install:pip_repository.bzl", "pip_repository", _package_annotation = "package_annotation")
load("//python/pip_install:requirements.bzl", _compile_pip_requirements = "compile_pip_requirements")
load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")
load("//python/private:full_version.bzl", "full_version")
load("//python/private:normalize_name.bzl", "normalize_name")
-load("//python/private:render_pkg_aliases.bzl", "NO_MATCH_ERROR_MESSAGE_TEMPLATE")
+load("//python/private/pypi:package_annotation.bzl", _package_annotation = "package_annotation")
+load("//python/private/pypi:pip_repository.bzl", "pip_repository")
+load("//python/private/pypi:render_pkg_aliases.bzl", "NO_MATCH_ERROR_MESSAGE_TEMPLATE")
load("//python/private/whl_filegroup:whl_filegroup.bzl", _whl_filegroup = "whl_filegroup")
compile_pip_requirements = _compile_pip_requirements
diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel
index 91f2ec7..1894c4d 100644
--- a/python/pip_install/BUILD.bazel
+++ b/python/pip_install/BUILD.bazel
@@ -22,23 +22,10 @@
name = "pip_repository_bzl",
srcs = ["pip_repository.bzl"],
deps = [
- ":repositories_bzl",
- "//python:repositories_bzl",
- "//python:versions_bzl",
- "//python/pip_install/private:generate_group_library_build_bazel_bzl",
- "//python/pip_install/private:generate_whl_library_build_bazel_bzl",
- "//python/pip_install/private:srcs_bzl",
- "//python/private:bzlmod_enabled_bzl",
- "//python/private:envsubst_bzl",
- "//python/private:normalize_name_bzl",
- "//python/private:parse_requirements_bzl",
- "//python/private:parse_whl_name_bzl",
- "//python/private:patch_whl_bzl",
- "//python/private:render_pkg_aliases_bzl",
- "//python/private:repo_utils_bzl",
- "//python/private:toolchains_repo_bzl",
- "//python/private:whl_target_platforms_bzl",
- "@bazel_skylib//lib:sets",
+ "//python/private/pypi:group_library_bzl",
+ "//python/private/pypi:package_annotation_bzl",
+ "//python/private/pypi:pip_repository_bzl",
+ "//python/private/pypi:whl_library_bzl",
],
)
@@ -52,11 +39,6 @@
)
bzl_library(
- name = "requirements_parser_bzl",
- srcs = ["requirements_parser.bzl"],
-)
-
-bzl_library(
name = "repositories_bzl",
srcs = ["repositories.bzl"],
deps = [
@@ -71,7 +53,6 @@
srcs = glob(["*.bzl"]) + [
"BUILD.bazel",
"pip_repository_requirements.bzl.tmpl",
- "//python/pip_install/private:distribution",
"//python/pip_install/tools/dependency_resolver:distribution",
"//python/pip_install/tools/wheel_installer:distribution",
],
@@ -92,23 +73,10 @@
filegroup(
name = "bzl",
- srcs = glob(["*.bzl"]) + [
- "//python/pip_install/private:bzl_srcs",
- ],
+ srcs = glob(["*.bzl"]),
visibility = ["//:__subpackages__"],
)
-filegroup(
- name = "py_srcs",
- srcs = [
- "//python/pip_install/tools/dependency_resolver:py_srcs",
- "//python/pip_install/tools/wheel_installer:py_srcs",
- "//python/private:repack_whl.py",
- "//tools:wheelmaker.py",
- ],
- visibility = ["//python/pip_install/private:__pkg__"],
-)
-
exports_files(
glob(["*.bzl"]),
visibility = ["//docs:__pkg__"],
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl
index d6c8d91..18deee1 100644
--- a/python/pip_install/pip_repository.bzl
+++ b/python/pip_install/pip_repository.bzl
@@ -14,1099 +14,13 @@
""
-load("@bazel_skylib//lib:sets.bzl", "sets")
-load("//python:repositories.bzl", "is_standalone_interpreter")
-load("//python:versions.bzl", "WINDOWS_NAME")
-load("//python/pip_install:repositories.bzl", "all_requirements")
-load("//python/pip_install/private:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel")
-load("//python/pip_install/private:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel")
-load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS")
-load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth")
-load("//python/private:envsubst.bzl", "envsubst")
-load("//python/private:normalize_name.bzl", "normalize_name")
-load("//python/private:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
-load("//python/private:parse_whl_name.bzl", "parse_whl_name")
-load("//python/private:patch_whl.bzl", "patch_whl")
-load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias")
-load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
-load("//python/private:toolchains_repo.bzl", "get_host_os_arch")
-load("//python/private:whl_target_platforms.bzl", "whl_target_platforms")
+load("//python/private/pypi:group_library.bzl", _group_library = "group_library")
+load("//python/private/pypi:package_annotation.bzl", _package_annotation = "package_annotation")
+load("//python/private/pypi:pip_repository.bzl", _pip_repository = "pip_repository")
+load("//python/private/pypi:whl_library.bzl", _whl_library = "whl_library")
-CPPFLAGS = "CPPFLAGS"
-
-COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools"
-
-_WHEEL_ENTRY_POINT_PREFIX = "rules_python_wheel_entry_point"
-
-def _construct_pypath(rctx):
- """Helper function to construct a PYTHONPATH.
-
- Contains entries for code in this repo as well as packages downloaded from //python/pip_install:repositories.bzl.
- This allows us to run python code inside repository rule implementations.
-
- Args:
- rctx: Handle to the repository_context.
-
- Returns: String of the PYTHONPATH.
- """
-
- separator = ":" if not "windows" in rctx.os.name.lower() else ";"
- pypath = separator.join([
- str(rctx.path(entry).dirname)
- for entry in rctx.attr._python_path_entries
- ])
- return pypath
-
-def _get_python_interpreter_attr(rctx):
- """A helper function for getting the `python_interpreter` attribute or it's default
-
- Args:
- rctx (repository_ctx): Handle to the rule repository context.
-
- Returns:
- str: The attribute value or it's default
- """
- if rctx.attr.python_interpreter:
- return rctx.attr.python_interpreter
-
- if "win" in rctx.os.name:
- return "python.exe"
- else:
- return "python3"
-
-def _resolve_python_interpreter(rctx):
- """Helper function to find the python interpreter from the common attributes
-
- Args:
- rctx: Handle to the rule repository context.
-
- Returns:
- `path` object, for the resolved path to the Python interpreter.
- """
- python_interpreter = _get_python_interpreter_attr(rctx)
-
- if rctx.attr.python_interpreter_target != None:
- python_interpreter = rctx.path(rctx.attr.python_interpreter_target)
-
- (os, _) = get_host_os_arch(rctx)
-
- # On Windows, the symlink doesn't work because Windows attempts to find
- # Python DLLs where the symlink is, not where the symlink points.
- if os == WINDOWS_NAME:
- python_interpreter = python_interpreter.realpath
- elif "/" not in python_interpreter:
- # It's a plain command, e.g. "python3", to look up in the environment.
- found_python_interpreter = rctx.which(python_interpreter)
- if not found_python_interpreter:
- fail("python interpreter `{}` not found in PATH".format(python_interpreter))
- python_interpreter = found_python_interpreter
- else:
- python_interpreter = rctx.path(python_interpreter)
- return python_interpreter
-
-def _get_xcode_location_cflags(rctx):
- """Query the xcode sdk location to update cflags
-
- Figure out if this interpreter target comes from rules_python, and patch the xcode sdk location if so.
- Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg
- otherwise. See https://github.com/indygreg/python-build-standalone/issues/103
- """
-
- # Only run on MacOS hosts
- if not rctx.os.name.lower().startswith("mac os"):
- return []
-
- xcode_sdk_location = repo_utils.execute_unchecked(
- rctx,
- op = "GetXcodeLocation",
- arguments = [repo_utils.which_checked(rctx, "xcode-select"), "--print-path"],
- )
- if xcode_sdk_location.return_code != 0:
- return []
-
- xcode_root = xcode_sdk_location.stdout.strip()
- if COMMAND_LINE_TOOLS_PATH_SLUG not in xcode_root.lower():
- # This is a full xcode installation somewhere like /Applications/Xcode13.0.app/Contents/Developer
- # so we need to change the path to to the macos specific tools which are in a different relative
- # path than xcode installed command line tools.
- xcode_root = "{}/Platforms/MacOSX.platform/Developer".format(xcode_root)
- return [
- "-isysroot {}/SDKs/MacOSX.sdk".format(xcode_root),
- ]
-
-def _get_toolchain_unix_cflags(rctx, python_interpreter):
- """Gather cflags from a standalone toolchain for unix systems.
-
- Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg
- otherwise. See https://github.com/indygreg/python-build-standalone/issues/103
- """
-
- # Only run on Unix systems
- if not rctx.os.name.lower().startswith(("mac os", "linux")):
- return []
-
- # Only update the location when using a standalone toolchain.
- if not is_standalone_interpreter(rctx, python_interpreter):
- return []
-
- stdout = repo_utils.execute_checked_stdout(
- rctx,
- op = "GetPythonVersionForUnixCflags",
- arguments = [
- python_interpreter,
- "-c",
- "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}', end='')",
- ],
- )
- _python_version = stdout
- include_path = "{}/include/python{}".format(
- python_interpreter.dirname,
- _python_version,
- )
-
- return ["-isystem {}".format(include_path)]
-
-def use_isolated(ctx, attr):
- """Determine whether or not to pass the pip `--isolated` flag to the pip invocation.
-
- Args:
- ctx: repository or module context
- attr: attributes for the repo rule or tag extension
-
- Returns:
- True if --isolated should be passed
- """
- use_isolated = attr.isolated
-
- # The environment variable will take precedence over the attribute
- isolated_env = ctx.os.environ.get("RULES_PYTHON_PIP_ISOLATED", None)
- if isolated_env != None:
- if isolated_env.lower() in ("0", "false"):
- use_isolated = False
- else:
- use_isolated = True
-
- return use_isolated
-
-def _parse_optional_attrs(rctx, args, extra_pip_args = None):
- """Helper function to parse common attributes of pip_repository and whl_library repository rules.
-
- This function also serializes the structured arguments as JSON
- so they can be passed on the command line to subprocesses.
-
- Args:
- rctx: Handle to the rule repository context.
- args: A list of parsed args for the rule.
- extra_pip_args: The pip args to pass.
- Returns: Augmented args list.
- """
-
- if use_isolated(rctx, rctx.attr):
- args.append("--isolated")
-
- # Bazel version 7.1.0 and later (and rolling releases from version 8.0.0-pre.20240128.3)
- # support rctx.getenv(name, default): When building incrementally, any change to the value of
- # the variable named by name will cause this repository to be re-fetched.
- if "getenv" in dir(rctx):
- getenv = rctx.getenv
- else:
- getenv = rctx.os.environ.get
-
- # Check for None so we use empty default types from our attrs.
- # Some args want to be list, and some want to be dict.
- if extra_pip_args != None:
- args += [
- "--extra_pip_args",
- json.encode(struct(arg = [
- envsubst(pip_arg, rctx.attr.envsubst, getenv)
- for pip_arg in rctx.attr.extra_pip_args
- ])),
- ]
-
- if rctx.attr.download_only:
- args.append("--download_only")
-
- if rctx.attr.pip_data_exclude != None:
- args += [
- "--pip_data_exclude",
- json.encode(struct(arg = rctx.attr.pip_data_exclude)),
- ]
-
- if rctx.attr.enable_implicit_namespace_pkgs:
- args.append("--enable_implicit_namespace_pkgs")
-
- if rctx.attr.environment != None:
- args += [
- "--environment",
- json.encode(struct(arg = rctx.attr.environment)),
- ]
-
- return args
-
-def _create_repository_execution_environment(rctx, python_interpreter):
- """Create a environment dictionary for processes we spawn with rctx.execute.
-
- Args:
- rctx (repository_ctx): The repository context.
- python_interpreter (path): The resolved python interpreter.
- Returns:
- Dictionary of environment variable suitable to pass to rctx.execute.
- """
-
- # Gather any available CPPFLAGS values
- cppflags = []
- cppflags.extend(_get_xcode_location_cflags(rctx))
- cppflags.extend(_get_toolchain_unix_cflags(rctx, python_interpreter))
-
- env = {
- "PYTHONPATH": _construct_pypath(rctx),
- CPPFLAGS: " ".join(cppflags),
- }
-
- return env
-
-_BUILD_FILE_CONTENTS = """\
-package(default_visibility = ["//visibility:public"])
-
-# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it
-exports_files(["requirements.bzl"])
-"""
-
-def _pip_repository_impl(rctx):
- requirements_by_platform = parse_requirements(
- rctx,
- requirements_by_platform = rctx.attr.requirements_by_platform,
- requirements_linux = rctx.attr.requirements_linux,
- requirements_lock = rctx.attr.requirements_lock,
- requirements_osx = rctx.attr.requirements_darwin,
- requirements_windows = rctx.attr.requirements_windows,
- extra_pip_args = rctx.attr.extra_pip_args,
- )
- selected_requirements = {}
- options = None
- repository_platform = host_platform(rctx.os)
- for name, requirements in requirements_by_platform.items():
- r = select_requirement(
- requirements,
- platform = repository_platform,
- )
- if not r:
- continue
- options = options or r.extra_pip_args
- selected_requirements[name] = r.requirement_line
-
- bzl_packages = sorted(selected_requirements.keys())
-
- # Normalize cycles first
- requirement_cycles = {
- name: sorted(sets.to_list(sets.make(deps)))
- for name, deps in rctx.attr.experimental_requirement_cycles.items()
- }
-
- # Check for conflicts between cycles _before_ we normalize package names so
- # that reported errors use the names the user specified
- for i in range(len(requirement_cycles)):
- left_group = requirement_cycles.keys()[i]
- left_deps = requirement_cycles.values()[i]
- for j in range(len(requirement_cycles) - (i + 1)):
- right_deps = requirement_cycles.values()[1 + i + j]
- right_group = requirement_cycles.keys()[1 + i + j]
- for d in left_deps:
- if d in right_deps:
- fail("Error: Requirement %s cannot be repeated between cycles %s and %s; please merge the cycles." % (d, left_group, right_group))
-
- # And normalize the names as used in the cycle specs
- #
- # NOTE: We must check that a listed dependency is actually in the actual
- # requirements set for the current platform so that we can support cycles in
- # platform-conditional requirements. Otherwise we'll blindly generate a
- # label referencing a package which may not be installed on the current
- # platform.
- requirement_cycles = {
- normalize_name(name): sorted([normalize_name(d) for d in group if normalize_name(d) in bzl_packages])
- for name, group in requirement_cycles.items()
- }
-
- imports = [
- # NOTE: Maintain the order consistent with `buildifier`
- 'load("@rules_python//python:pip.bzl", "pip_utils")',
- 'load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_library")',
- ]
-
- annotations = {}
- for pkg, annotation in rctx.attr.annotations.items():
- filename = "{}.annotation.json".format(normalize_name(pkg))
- rctx.file(filename, json.encode_indent(json.decode(annotation)))
- annotations[pkg] = "@{name}//:{filename}".format(name = rctx.attr.name, filename = filename)
-
- config = {
- "download_only": rctx.attr.download_only,
- "enable_implicit_namespace_pkgs": rctx.attr.enable_implicit_namespace_pkgs,
- "environment": rctx.attr.environment,
- "envsubst": rctx.attr.envsubst,
- "extra_pip_args": options,
- "isolated": use_isolated(rctx, rctx.attr),
- "pip_data_exclude": rctx.attr.pip_data_exclude,
- "python_interpreter": _get_python_interpreter_attr(rctx),
- "quiet": rctx.attr.quiet,
- "repo": rctx.attr.name,
- "timeout": rctx.attr.timeout,
- }
- if rctx.attr.use_hub_alias_dependencies:
- config["dep_template"] = "@{}//{{name}}:{{target}}".format(rctx.attr.name)
- else:
- config["repo_prefix"] = "{}_".format(rctx.attr.name)
-
- if rctx.attr.python_interpreter_target:
- config["python_interpreter_target"] = str(rctx.attr.python_interpreter_target)
- if rctx.attr.experimental_target_platforms:
- config["experimental_target_platforms"] = rctx.attr.experimental_target_platforms
-
- macro_tmpl = "@%s//{}:{}" % rctx.attr.name
-
- aliases = render_pkg_aliases(
- aliases = {
- pkg: [whl_alias(repo = rctx.attr.name + "_" + pkg)]
- for pkg in bzl_packages or []
- },
- )
- for path, contents in aliases.items():
- rctx.file(path, contents)
-
- rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS)
- rctx.template("requirements.bzl", rctx.attr._template, substitutions = {
- " # %%GROUP_LIBRARY%%": """\
- group_repo = "{name}__groups"
- group_library(
- name = group_repo,
- repo_prefix = "{name}_",
- groups = all_requirement_groups,
- )""".format(name = rctx.attr.name) if not rctx.attr.use_hub_alias_dependencies else "",
- "%%ALL_DATA_REQUIREMENTS%%": _format_repr_list([
- macro_tmpl.format(p, "data")
- for p in bzl_packages
- ]),
- "%%ALL_REQUIREMENTS%%": _format_repr_list([
- macro_tmpl.format(p, "pkg")
- for p in bzl_packages
- ]),
- "%%ALL_REQUIREMENT_GROUPS%%": _format_dict(_repr_dict(requirement_cycles)),
- "%%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%": _format_dict(_repr_dict({
- p: macro_tmpl.format(p, "whl")
- for p in bzl_packages
- })),
- "%%ANNOTATIONS%%": _format_dict(_repr_dict(annotations)),
- "%%CONFIG%%": _format_dict(_repr_dict(config)),
- "%%EXTRA_PIP_ARGS%%": json.encode(options),
- "%%IMPORTS%%": "\n".join(imports),
- "%%MACRO_TMPL%%": macro_tmpl,
- "%%NAME%%": rctx.attr.name,
- "%%PACKAGES%%": _format_repr_list(
- [
- ("{}_{}".format(rctx.attr.name, p), r)
- for p, r in sorted(selected_requirements.items())
- ],
- ),
- })
-
- return
-
-common_env = [
- "RULES_PYTHON_PIP_ISOLATED",
- REPO_DEBUG_ENV_VAR,
-]
-
-common_attrs = {
- "download_only": attr.bool(
- doc = """
-Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of
---platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different
-platform from the host platform.
- """,
- ),
- "enable_implicit_namespace_pkgs": attr.bool(
- default = False,
- doc = """
-If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary
-and py_test targets must specify either `legacy_create_init=False` or the global Bazel option
-`--incompatible_default_to_explicit_init_py` to prevent `__init__.py` being automatically generated in every directory.
-
-This option is required to support some packages which cannot handle the conversion to pkg-util style.
- """,
- ),
- "environment": attr.string_dict(
- doc = """
-Environment variables to set in the pip subprocess.
-Can be used to set common variables such as `http_proxy`, `https_proxy` and `no_proxy`
-Note that pip is run with "--isolated" on the CLI so `PIP_<VAR>_<NAME>`
-style env vars are ignored, but env vars that control requests and urllib3
-can be passed. If you need `PIP_<VAR>_<NAME>`, take a look at `extra_pip_args`
-and `envsubst`.
- """,
- default = {},
- ),
- "envsubst": attr.string_list(
- mandatory = False,
- doc = """\
-A list of environment variables to substitute (e.g. `["PIP_INDEX_URL",
-"PIP_RETRIES"]`). The corresponding variables are expanded in `extra_pip_args`
-using the syntax `$VARNAME` or `${VARNAME}` (expanding to empty string if unset)
-or `${VARNAME:-default}` (expanding to default if the variable is unset or empty
-in the environment). Note: On Bazel 6 and Bazel 7.0 changes to the variables named
-here do not cause packages to be re-fetched. Don't fetch different things based
-on the value of these variables.
-""",
- ),
- "experimental_requirement_cycles": attr.string_list_dict(
- default = {},
- doc = """\
-A mapping of dependency cycle names to a list of requirements which form that cycle.
-
-Requirements which form cycles will be installed together and taken as
-dependencies together in order to ensure that the cycle is always satisified.
-
-Example:
- `sphinx` depends on `sphinxcontrib-serializinghtml`
- When listing both as requirements, ala
-
- ```
- py_binary(
- name = "doctool",
- ...
- deps = [
- "@pypi//sphinx:pkg",
- "@pypi//sphinxcontrib_serializinghtml",
- ]
- )
- ```
-
- Will produce a Bazel error such as
-
- ```
- ERROR: .../external/pypi_sphinxcontrib_serializinghtml/BUILD.bazel:44:6: in alias rule @pypi_sphinxcontrib_serializinghtml//:pkg: cycle in dependency graph:
- //:doctool (...)
- @pypi//sphinxcontrib_serializinghtml:pkg (...)
- .-> @pypi_sphinxcontrib_serializinghtml//:pkg (...)
- | @pypi_sphinxcontrib_serializinghtml//:_pkg (...)
- | @pypi_sphinx//:pkg (...)
- | @pypi_sphinx//:_pkg (...)
- `-- @pypi_sphinxcontrib_serializinghtml//:pkg (...)
- ```
-
- Which we can resolve by configuring these two requirements to be installed together as a cycle
-
- ```
- pip_parse(
- ...
- experimental_requirement_cycles = {
- "sphinx": [
- "sphinx",
- "sphinxcontrib-serializinghtml",
- ]
- },
- )
- ```
-
-Warning:
- If a dependency participates in multiple cycles, all of those cycles must be
- collapsed down to one. For instance `a <-> b` and `a <-> c` cannot be listed
- as two separate cycles.
-""",
- ),
- "experimental_target_platforms": attr.string_list(
- default = [],
- doc = """\
-A list of platforms that we will generate the conditional dependency graph for
-cross platform wheels by parsing the wheel metadata. This will generate the
-correct dependencies for packages like `sphinx` or `pylint`, which include
-`colorama` when installed and used on Windows platforms.
-
-An empty list means falling back to the legacy behaviour where the host
-platform is the target platform.
-
-WARNING: It may not work as expected in cases where the python interpreter
-implementation that is being used at runtime is different between different platforms.
-This has been tested for CPython only.
-
-For specific target platforms use values of the form `<os>_<arch>` where `<os>`
-is one of `linux`, `osx`, `windows` and arch is one of `x86_64`, `x86_32`,
-`aarch64`, `s390x` and `ppc64le`.
-
-You can also target a specific Python version by using `cp3<minor_version>_<os>_<arch>`.
-If multiple python versions are specified as target platforms, then select statements
-of the `lib` and `whl` targets will include usage of version aware toolchain config
-settings like `@rules_python//python/config_settings:is_python_3.y`.
-
-Special values: `host` (for generating deps for the host platform only) and
-`<prefix>_*` values. For example, `cp39_*`, `linux_*`, `cp39_linux_*`.
-
-NOTE: this is not for cross-compiling Python wheels but rather for parsing the `whl` METADATA correctly.
-""",
- ),
- "extra_pip_args": attr.string_list(
- doc = """Extra arguments to pass on to pip. Must not contain spaces.
-
-Supports environment variables using the syntax `$VARNAME` or
-`${VARNAME}` (expanding to empty string if unset) or
-`${VARNAME:-default}` (expanding to default if the variable is unset
-or empty in the environment), if `"VARNAME"` is listed in the
-`envsubst` attribute. See also `envsubst`.
-""",
- ),
- "isolated": attr.bool(
- doc = """\
-Whether or not to pass the [--isolated](https://pip.pypa.io/en/stable/cli/pip/#cmdoption-isolated) flag to
-the underlying pip command. Alternatively, the `RULES_PYTHON_PIP_ISOLATED` environment variable can be used
-to control this flag.
-""",
- default = True,
- ),
- "pip_data_exclude": attr.string_list(
- doc = "Additional data exclusion parameters to add to the pip packages BUILD file.",
- ),
- "python_interpreter": attr.string(
- doc = """\
-The python interpreter to use. This can either be an absolute path or the name
-of a binary found on the host's `PATH` environment variable. If no value is set
-`python3` is defaulted for Unix systems and `python.exe` for Windows.
-""",
- # NOTE: This attribute should not have a default. See `_get_python_interpreter_attr`
- # default = "python3"
- ),
- "python_interpreter_target": attr.label(
- allow_single_file = True,
- doc = """
-If you are using a custom python interpreter built by another repository rule,
-use this attribute to specify its BUILD target. This allows pip_repository to invoke
-pip using the same interpreter as your toolchain. If set, takes precedence over
-python_interpreter. An example value: "@python3_x86_64-unknown-linux-gnu//:python".
-""",
- ),
- "quiet": attr.bool(
- default = True,
- doc = """\
-If True, suppress printing stdout and stderr output to the terminal.
-
-If you would like to get more diagnostic output, please use:
-
- RULES_PYTHON_REPO_DEBUG=1
-
-or
-
- RULES_PYTHON_REPO_DEBUG_VERBOSITY=<INFO|DEBUG|TRACE>
-""",
- ),
- "repo_prefix": attr.string(
- doc = """
-Prefix for the generated packages will be of the form `@<prefix><sanitized-package-name>//...`
-
-DEPRECATED. Only left for people who vendor requirements.bzl.
-""",
- ),
- # 600 is documented as default here: https://docs.bazel.build/versions/master/skylark/lib/repository_ctx.html#execute
- "timeout": attr.int(
- default = 600,
- doc = "Timeout (in seconds) on the rule's execution duration.",
- ),
- "_py_srcs": attr.label_list(
- doc = "Python sources used in the repository rule",
- allow_files = True,
- default = PIP_INSTALL_PY_SRCS,
- ),
-}
-
-pip_repository_attrs = {
- "annotations": attr.string_dict(
- doc = "Optional annotations to apply to packages",
- ),
- "requirements_by_platform": attr.label_keyed_string_dict(
- doc = """\
-The requirements files and the comma delimited list of target platforms as values.
-
-The keys are the requirement files and the values are comma-separated platform
-identifiers. For now we only support `<os>_<cpu>` values that are present in
-`@platforms//os` and `@platforms//cpu` packages respectively.
-""",
- ),
- "requirements_darwin": attr.label(
- allow_single_file = True,
- doc = "Override the requirements_lock attribute when the host platform is Mac OS",
- ),
- "requirements_linux": attr.label(
- allow_single_file = True,
- doc = "Override the requirements_lock attribute when the host platform is Linux",
- ),
- "requirements_lock": attr.label(
- allow_single_file = True,
- doc = """\
-A fully resolved 'requirements.txt' pip requirement file containing the
-transitive set of your dependencies. If this file is passed instead of
-'requirements' no resolve will take place and pip_repository will create
-individual repositories for each of your dependencies so that wheels are
-fetched/built only for the targets specified by 'build/run/test'. Note that if
-your lockfile is platform-dependent, you can use the `requirements_[platform]`
-attributes.
-
-Note, that in general requirements files are compiled for a specific platform,
-but sometimes they can work for multiple platforms. `rules_python` right now
-supports requirements files that are created for a particular platform without
-platform markers.
-""",
- ),
- "requirements_windows": attr.label(
- allow_single_file = True,
- doc = "Override the requirements_lock attribute when the host platform is Windows",
- ),
- "use_hub_alias_dependencies": attr.bool(
- default = False,
- doc = """\
-Controls if the hub alias dependencies are used. If set to true, then the
-group_library will be included in the hub repo.
-
-True will become default in a subsequent release.
-""",
- ),
- "_template": attr.label(
- default = ":pip_repository_requirements.bzl.tmpl",
- ),
-}
-
-pip_repository_attrs.update(**common_attrs)
-
-pip_repository = repository_rule(
- attrs = pip_repository_attrs,
- doc = """Accepts a locked/compiled requirements file and installs the dependencies listed within.
-
-Those dependencies become available in a generated `requirements.bzl` file.
-You can instead check this `requirements.bzl` file into your repo, see the "vendoring" section below.
-
-In your WORKSPACE file:
-
-```starlark
-load("@rules_python//python:pip.bzl", "pip_parse")
-
-pip_parse(
- name = "pypi",
- requirements_lock = ":requirements.txt",
-)
-
-load("@pypi//:requirements.bzl", "install_deps")
-
-install_deps()
-```
-
-You can then reference installed dependencies from a `BUILD` file with the alias targets generated in the same repo, for example, for `PyYAML` we would have the following:
-- `@pypi//pyyaml` and `@pypi//pyyaml:pkg` both point to the `py_library`
- created after extracting the `PyYAML` package.
-- `@pypi//pyyaml:data` points to the extra data included in the package.
-- `@pypi//pyyaml:dist_info` points to the `dist-info` files in the package.
-- `@pypi//pyyaml:whl` points to the wheel file that was extracted.
-
-```starlark
-py_library(
- name = "bar",
- ...
- deps = [
- "//my/other:dep",
- "@pypi//numpy",
- "@pypi//requests",
- ],
-)
-```
-
-or
-
-```starlark
-load("@pypi//:requirements.bzl", "requirement")
-
-py_library(
- name = "bar",
- ...
- deps = [
- "//my/other:dep",
- requirement("numpy"),
- requirement("requests"),
- ],
-)
-```
-
-In addition to the `requirement` macro, which is used to access the generated `py_library`
-target generated from a package's wheel, The generated `requirements.bzl` file contains
-functionality for exposing [entry points][whl_ep] as `py_binary` targets as well.
-
-[whl_ep]: https://packaging.python.org/specifications/entry-points/
-
-```starlark
-load("@pypi//:requirements.bzl", "entry_point")
-
-alias(
- name = "pip-compile",
- actual = entry_point(
- pkg = "pip-tools",
- script = "pip-compile",
- ),
-)
-```
-
-Note that for packages whose name and script are the same, only the name of the package
-is needed when calling the `entry_point` macro.
-
-```starlark
-load("@pip//:requirements.bzl", "entry_point")
-
-alias(
- name = "flake8",
- actual = entry_point("flake8"),
-)
-```
-
-### Vendoring the requirements.bzl file
-
-In some cases you may not want to generate the requirements.bzl file as a repository rule
-while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module
-such as a ruleset, you may want to include the requirements.bzl file rather than make your users
-install the WORKSPACE setup to generate it.
-See https://github.com/bazelbuild/rules_python/issues/608
-
-This is the same workflow as Gazelle, which creates `go_repository` rules with
-[`update-repos`](https://github.com/bazelbuild/bazel-gazelle#update-repos)
-
-To do this, use the "write to source file" pattern documented in
-https://blog.aspect.dev/bazel-can-write-to-the-source-folder
-to put a copy of the generated requirements.bzl into your project.
-Then load the requirements.bzl file directly rather than from the generated repository.
-See the example in rules_python/examples/pip_parse_vendored.
-""",
- implementation = _pip_repository_impl,
- environ = common_env,
-)
-
-def _whl_library_impl(rctx):
- python_interpreter = _resolve_python_interpreter(rctx)
- args = [
- python_interpreter,
- "-m",
- "python.pip_install.tools.wheel_installer.wheel_installer",
- "--requirement",
- rctx.attr.requirement,
- ]
- extra_pip_args = []
- extra_pip_args.extend(rctx.attr.extra_pip_args)
-
- # Manually construct the PYTHONPATH since we cannot use the toolchain here
- environment = _create_repository_execution_environment(rctx, python_interpreter)
-
- whl_path = None
- if rctx.attr.whl_file:
- whl_path = rctx.path(rctx.attr.whl_file)
-
- # Simulate the behaviour where the whl is present in the current directory.
- rctx.symlink(whl_path, whl_path.basename)
- whl_path = rctx.path(whl_path.basename)
- elif rctx.attr.urls:
- filename = rctx.attr.filename
- urls = rctx.attr.urls
- if not filename:
- _, _, filename = urls[0].rpartition("/")
-
- if not (filename.endswith(".whl") or filename.endswith("tar.gz") or filename.endswith(".zip")):
- if rctx.attr.filename:
- msg = "got '{}'".format(filename)
- else:
- msg = "detected '{}' from url:\n{}".format(filename, urls[0])
- fail("Only '.whl', '.tar.gz' or '.zip' files are supported, {}".format(msg))
-
- result = rctx.download(
- url = urls,
- output = filename,
- sha256 = rctx.attr.sha256,
- auth = get_auth(rctx, urls),
- )
-
- if not result.success:
- fail("could not download the '{}' from {}:\n{}".format(filename, urls, result))
-
- if filename.endswith(".whl"):
- whl_path = rctx.path(rctx.attr.filename)
- else:
- # It is an sdist and we need to tell PyPI to use a file in this directory
- # and not use any indexes.
- extra_pip_args.extend(["--no-index", "--find-links", "."])
-
- args = _parse_optional_attrs(rctx, args, extra_pip_args)
-
- if not whl_path:
- repo_utils.execute_checked(
- rctx,
- op = "whl_library.ResolveRequirement({}, {})".format(rctx.attr.name, rctx.attr.requirement),
- arguments = args,
- environment = environment,
- quiet = rctx.attr.quiet,
- timeout = rctx.attr.timeout,
- )
-
- whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"])
- if not rctx.delete("whl_file.json"):
- fail("failed to delete the whl_file.json file")
-
- if rctx.attr.whl_patches:
- patches = {}
- for patch_file, json_args in rctx.attr.whl_patches.items():
- patch_dst = struct(**json.decode(json_args))
- if whl_path.basename in patch_dst.whls:
- patches[patch_file] = patch_dst.patch_strip
-
- whl_path = patch_whl(
- rctx,
- python_interpreter = python_interpreter,
- whl_path = whl_path,
- patches = patches,
- quiet = rctx.attr.quiet,
- timeout = rctx.attr.timeout,
- )
-
- target_platforms = rctx.attr.experimental_target_platforms
- if target_platforms:
- parsed_whl = parse_whl_name(whl_path.basename)
- if parsed_whl.platform_tag != "any":
- # NOTE @aignas 2023-12-04: if the wheel is a platform specific
- # wheel, we only include deps for that target platform
- target_platforms = [
- p.target_platform
- for p in whl_target_platforms(
- platform_tag = parsed_whl.platform_tag,
- abi_tag = parsed_whl.abi_tag,
- )
- ]
-
- repo_utils.execute_checked(
- rctx,
- op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path),
- arguments = args + [
- "--whl-file",
- whl_path,
- ] + ["--platform={}".format(p) for p in target_platforms],
- environment = environment,
- quiet = rctx.attr.quiet,
- timeout = rctx.attr.timeout,
- )
-
- metadata = json.decode(rctx.read("metadata.json"))
- rctx.delete("metadata.json")
-
- entry_points = {}
- for item in metadata["entry_points"]:
- name = item["name"]
- module = item["module"]
- attribute = item["attribute"]
-
- # There is an extreme edge-case with entry_points that end with `.py`
- # See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174
- entry_point_without_py = name[:-3] + "_py" if name.endswith(".py") else name
- entry_point_target_name = (
- _WHEEL_ENTRY_POINT_PREFIX + "_" + entry_point_without_py
- )
- entry_point_script_name = entry_point_target_name + ".py"
-
- rctx.file(
- entry_point_script_name,
- _generate_entry_point_contents(module, attribute),
- )
- entry_points[entry_point_without_py] = entry_point_script_name
-
- build_file_contents = generate_whl_library_build_bazel(
- dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix),
- whl_name = whl_path.basename,
- dependencies = metadata["deps"],
- dependencies_by_platform = metadata["deps_by_platform"],
- group_name = rctx.attr.group_name,
- group_deps = rctx.attr.group_deps,
- data_exclude = rctx.attr.pip_data_exclude,
- tags = [
- "pypi_name=" + metadata["name"],
- "pypi_version=" + metadata["version"],
- ],
- entry_points = entry_points,
- annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))),
- )
- rctx.file("BUILD.bazel", build_file_contents)
-
- return
-
-def _generate_entry_point_contents(
- module,
- attribute,
- shebang = "#!/usr/bin/env python3"):
- """Generate the contents of an entry point script.
-
- Args:
- module (str): The name of the module to use.
- attribute (str): The name of the attribute to call.
- shebang (str, optional): The shebang to use for the entry point python
- file.
-
- Returns:
- str: A string of python code.
- """
- contents = """\
-{shebang}
-import sys
-from {module} import {attribute}
-if __name__ == "__main__":
- sys.exit({attribute}())
-""".format(
- shebang = shebang,
- module = module,
- attribute = attribute,
- )
- return contents
-
-# NOTE @aignas 2024-03-21: The usage of dict({}, **common) ensures that all args to `dict` are unique
-whl_library_attrs = dict({
- "annotation": attr.label(
- doc = (
- "Optional json encoded file containing annotation to apply to the extracted wheel. " +
- "See `package_annotation`"
- ),
- allow_files = True,
- ),
- "dep_template": attr.string(
- doc = """
-The dep template to use for referencing the dependencies. It should have `{name}`
-and `{target}` tokens that will be replaced with the normalized distribution name
-and the target that we need respectively.
-""",
- ),
- "filename": attr.string(
- doc = "Download the whl file to this filename. Only used when the `urls` is passed. If not specified, will be auto-detected from the `urls`.",
- ),
- "group_deps": attr.string_list(
- doc = "List of dependencies to skip in order to break the cycles within a dependency group.",
- default = [],
- ),
- "group_name": attr.string(
- doc = "Name of the group, if any.",
- ),
- "repo": attr.string(
- mandatory = True,
- doc = "Pointer to parent repo name. Used to make these rules rerun if the parent repo changes.",
- ),
- "requirement": attr.string(
- mandatory = True,
- doc = "Python requirement string describing the package to make available, if 'urls' or 'whl_file' is given, then this only needs to include foo[any_extras] as a bare minimum.",
- ),
- "sha256": attr.string(
- doc = "The sha256 of the downloaded whl. Only used when the `urls` is passed.",
- ),
- "urls": attr.string_list(
- doc = """\
-The list of urls of the whl to be downloaded using bazel downloader. Using this
-attr makes `extra_pip_args` and `download_only` ignored.""",
- ),
- "whl_file": attr.label(
- doc = "The whl file that should be used instead of downloading or building the whl.",
- ),
- "whl_patches": attr.label_keyed_string_dict(
- doc = """a label-keyed-string dict that has
- json.encode(struct([whl_file], patch_strip]) as values. This
- is to maintain flexibility and correct bzlmod extension interface
- until we have a better way to define whl_library and move whl
- patching to a separate place. INTERNAL USE ONLY.""",
- ),
- "_python_path_entries": attr.label_list(
- # Get the root directory of these rules and keep them as a default attribute
- # in order to avoid unnecessary repository fetching restarts.
- #
- # This is very similar to what was done in https://github.com/bazelbuild/rules_go/pull/3478
- default = [
- Label("//:BUILD.bazel"),
- ] + [
- # Includes all the external dependencies from repositories.bzl
- Label("@" + repo + "//:BUILD.bazel")
- for repo in all_requirements
- ],
- ),
-}, **common_attrs)
-whl_library_attrs.update(AUTH_ATTRS)
-
-whl_library = repository_rule(
- attrs = whl_library_attrs,
- doc = """
-Download and extracts a single wheel based into a bazel repo based on the requirement string passed in.
-Instantiated from pip_repository and inherits config options from there.""",
- implementation = _whl_library_impl,
- environ = common_env,
-)
-
-def package_annotation(
- additive_build_content = None,
- copy_files = {},
- copy_executables = {},
- data = [],
- data_exclude_glob = [],
- srcs_exclude_glob = []):
- """Annotations to apply to the BUILD file content from package generated from a `pip_repository` rule.
-
- [cf]: https://github.com/bazelbuild/bazel-skylib/blob/main/docs/copy_file_doc.md
-
- Args:
- additive_build_content (str, optional): Raw text to add to the generated `BUILD` file of a package.
- copy_files (dict, optional): A mapping of `src` and `out` files for [@bazel_skylib//rules:copy_file.bzl][cf]
- copy_executables (dict, optional): A mapping of `src` and `out` files for
- [@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as
- executable.
- data (list, optional): A list of labels to add as `data` dependencies to the generated `py_library` target.
- data_exclude_glob (list, optional): A list of exclude glob patterns to add as `data` to the generated
- `py_library` target.
- srcs_exclude_glob (list, optional): A list of labels to add as `srcs` to the generated `py_library` target.
-
- Returns:
- str: A json encoded string of the provided content.
- """
- return json.encode(struct(
- additive_build_content = additive_build_content,
- copy_files = copy_files,
- copy_executables = copy_executables,
- data = data,
- data_exclude_glob = data_exclude_glob,
- srcs_exclude_glob = srcs_exclude_glob,
- ))
-
-def _group_library_impl(rctx):
- build_file_contents = generate_group_library_build_bazel(
- repo_prefix = rctx.attr.repo_prefix,
- groups = rctx.attr.groups,
- )
- rctx.file("BUILD.bazel", build_file_contents)
-
-group_library = repository_rule(
- attrs = {
- "groups": attr.string_list_dict(
- doc = "A mapping of group names to requirements within that group.",
- ),
- "repo_prefix": attr.string(
- doc = "Prefix used for the whl_library created components of each group",
- ),
- },
- implementation = _group_library_impl,
- doc = """
-Create a package containing only wrapper py_library and whl_library rules for implementing dependency groups.
-This is an implementation detail of dependency groups and should not be used alone.
- """,
-)
-
-# pip_repository implementation
-
-def _format_list(items):
- return "[{}]".format(", ".join(items))
-
-def _format_repr_list(strings):
- return _format_list(
- [repr(s) for s in strings],
- )
-
-def _repr_dict(items):
- return {k: repr(v) for k, v in items.items()}
-
-def _format_dict(items):
- return "{{{}}}".format(", ".join(sorted(['"{}": {}'.format(k, v) for k, v in items.items()])))
+# Re-exports for backwards compatibility
+group_library = _group_library
+pip_repository = _pip_repository
+whl_library = _whl_library
+package_annotation = _package_annotation
diff --git a/python/pip_install/private/BUILD.bazel b/python/pip_install/private/BUILD.bazel
deleted file mode 100644
index 887d2d3..0000000
--- a/python/pip_install/private/BUILD.bazel
+++ /dev/null
@@ -1,48 +0,0 @@
-load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
-load(":pip_install_utils.bzl", "srcs_module")
-
-package(default_visibility = ["//:__subpackages__"])
-
-exports_files([
- "srcs.bzl",
-])
-
-filegroup(
- name = "distribution",
- srcs = glob(["*"]),
- visibility = ["//python/pip_install:__subpackages__"],
-)
-
-filegroup(
- name = "bzl_srcs",
- srcs = glob(["*.bzl"]),
-)
-
-srcs_module(
- name = "srcs_module",
- srcs = "//python/pip_install:py_srcs",
- dest = ":srcs.bzl",
-)
-
-bzl_library(
- name = "generate_whl_library_build_bazel_bzl",
- srcs = ["generate_whl_library_build_bazel.bzl"],
- deps = [
- "//python/private:labels_bzl",
- "//python/private:normalize_name_bzl",
- ],
-)
-
-bzl_library(
- name = "generate_group_library_build_bazel_bzl",
- srcs = ["generate_group_library_build_bazel.bzl"],
- deps = [
- "//python/private:labels_bzl",
- "//python/private:normalize_name_bzl",
- ],
-)
-
-bzl_library(
- name = "srcs_bzl",
- srcs = ["srcs.bzl"],
-)
diff --git a/python/pip_install/private/pip_install_utils.bzl b/python/pip_install/private/pip_install_utils.bzl
deleted file mode 100644
index 488583d..0000000
--- a/python/pip_install/private/pip_install_utils.bzl
+++ /dev/null
@@ -1,132 +0,0 @@
-# Copyright 2023 The Bazel Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Utilities for `rules_python` pip rules"""
-
-_SRCS_TEMPLATE = """\
-\"\"\"A generated file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules
-
-This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.update` target. Please
-`bazel run` this target to apply any updates. Note that doing so will discard any local modifications.
-"\"\"
-
-# Each source file is tracked as a target so `pip_repository` rules will know to automatically rebuild if any of the
-# sources changed.
-PIP_INSTALL_PY_SRCS = [
- {srcs}
-]
-"""
-
-def _src_label(file):
- dir_path, file_name = file.short_path.rsplit("/", 1)
-
- return "@rules_python//{}:{}".format(
- dir_path,
- file_name,
- )
-
-def _srcs_module_impl(ctx):
- srcs = [_src_label(src) for src in ctx.files.srcs]
- if not srcs:
- fail("`srcs` cannot be empty")
- output = ctx.actions.declare_file(ctx.label.name)
-
- ctx.actions.write(
- output = output,
- content = _SRCS_TEMPLATE.format(
- srcs = "\n ".join(["\"{}\",".format(src) for src in srcs]),
- ),
- )
-
- return DefaultInfo(
- files = depset([output]),
- )
-
-_srcs_module = rule(
- doc = "A rule for writing a list of sources to a templated file",
- implementation = _srcs_module_impl,
- attrs = {
- "srcs": attr.label(
- doc = "A filegroup of source files",
- allow_files = True,
- ),
- },
-)
-
-_INSTALLER_TEMPLATE = """\
-#!/bin/bash
-set -euo pipefail
-cp -f "{path}" "${{BUILD_WORKSPACE_DIRECTORY}}/{dest}"
-"""
-
-def _srcs_updater_impl(ctx):
- output = ctx.actions.declare_file(ctx.label.name + ".sh")
- target_file = ctx.file.input
- dest = ctx.file.dest.short_path
-
- ctx.actions.write(
- output = output,
- content = _INSTALLER_TEMPLATE.format(
- path = target_file.short_path,
- dest = dest,
- ),
- is_executable = True,
- )
-
- return DefaultInfo(
- files = depset([output]),
- runfiles = ctx.runfiles(files = [target_file]),
- executable = output,
- )
-
-_srcs_updater = rule(
- doc = "A rule for writing a `srcs.bzl` file back to the repository",
- implementation = _srcs_updater_impl,
- attrs = {
- "dest": attr.label(
- doc = "The target file to write the new `input` to.",
- allow_single_file = ["srcs.bzl"],
- mandatory = True,
- ),
- "input": attr.label(
- doc = "The file to write back to the repository",
- allow_single_file = True,
- mandatory = True,
- ),
- },
- executable = True,
-)
-
-def srcs_module(name, dest, **kwargs):
- """A helper rule to ensure `pip_repository` rules are always up to date
-
- Args:
- name (str): The name of the sources module
- dest (str): The filename the module should be written as in the current package.
- **kwargs (dict): Additional keyword arguments
- """
- tags = kwargs.pop("tags", [])
-
- _srcs_module(
- name = name,
- tags = tags,
- **kwargs
- )
-
- _srcs_updater(
- name = name + ".update",
- input = name,
- dest = dest,
- tags = tags,
- )
diff --git a/python/pip_install/private/srcs.bzl b/python/pip_install/private/srcs.bzl
deleted file mode 100644
index e92e49f..0000000
--- a/python/pip_install/private/srcs.bzl
+++ /dev/null
@@ -1,18 +0,0 @@
-"""A generated file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules
-
-This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.update` target. Please
-`bazel run` this target to apply any updates. Note that doing so will discard any local modifications.
-"""
-
-# Each source file is tracked as a target so `pip_repository` rules will know to automatically rebuild if any of the
-# sources changed.
-PIP_INSTALL_PY_SRCS = [
- "@rules_python//python/pip_install/tools/dependency_resolver:__init__.py",
- "@rules_python//python/pip_install/tools/dependency_resolver:dependency_resolver.py",
- "@rules_python//python/pip_install/tools/wheel_installer:arguments.py",
- "@rules_python//python/pip_install/tools/wheel_installer:namespace_pkgs.py",
- "@rules_python//python/pip_install/tools/wheel_installer:wheel.py",
- "@rules_python//python/pip_install/tools/wheel_installer:wheel_installer.py",
- "@rules_python//python/private:repack_whl.py",
- "@rules_python//tools:wheelmaker.py",
-]
diff --git a/python/pip_install/requirements_parser.bzl b/python/pip_install/requirements_parser.bzl
index 3b49fdf..82ec1b9 100644
--- a/python/pip_install/requirements_parser.bzl
+++ b/python/pip_install/requirements_parser.bzl
@@ -14,120 +14,6 @@
"Pip requirements parser for Starlark"
-_STATE = struct(
- # Consume extraneous whitespace
- ConsumeSpace = 0,
- # Consume a comment
- ConsumeComment = 1,
- # Parse the name of a pip package
- ParseDependency = 2,
- # Parse a full requirement line
- ParseRequirement = 3,
- # Parse a pip option
- ParseOption = 4,
-)
+load("//python/private/pypi:parse_requirements_txt.bzl", "parse_requirements_txt")
-EOF = {}
-
-def parse(content):
- """A simplistic (and incomplete) pip requirements lockfile parser.
-
- Parses package names and their full requirement lines, as well pip
- options.
-
- Args:
- content: lockfile content as a string
-
- Returns:
- Struct with fields `requirements` and `options`.
-
- requirements: List of requirements, where each requirement is a 2-element
- tuple containing the package name and the requirement line.
- E.g., [(certifi', 'certifi==2021.10.8 --hash=sha256:7888...'), ...]
-
- options: List of pip option lines
- """
- content = content.replace("\r", "")
-
- result = struct(
- requirements = [],
- options = [],
- )
- state = _STATE.ConsumeSpace
- buffer = ""
-
- inputs = content.elems()[:]
- inputs.append(EOF)
-
- for input in inputs:
- if state == _STATE.ConsumeSpace:
- (state, buffer) = _handleConsumeSpace(input)
- elif state == _STATE.ConsumeComment:
- (state, buffer) = _handleConsumeComment(input, buffer, result)
- elif state == _STATE.ParseDependency:
- (state, buffer) = _handleParseDependency(input, buffer, result)
- elif state == _STATE.ParseOption:
- (state, buffer) = _handleParseOption(input, buffer, result)
- elif state == _STATE.ParseRequirement:
- (state, buffer) = _handleParseRequirement(input, buffer, result)
- else:
- fail("Unknown state %d" % state)
-
- return result
-
-def _handleConsumeSpace(input):
- if input == EOF:
- return (_STATE.ConsumeSpace, "")
- if input.isspace():
- return (_STATE.ConsumeSpace, "")
- elif input == "#":
- return (_STATE.ConsumeComment, "")
- elif input == "-":
- return (_STATE.ParseOption, input)
-
- return (_STATE.ParseDependency, input)
-
-def _handleConsumeComment(input, buffer, result):
- if input == "\n":
- if len(result.requirements) > 0 and len(result.requirements[-1]) == 1:
- result.requirements[-1] = (result.requirements[-1][0], buffer.rstrip(" \n"))
- return (_STATE.ConsumeSpace, "")
- elif len(buffer) > 0:
- result.options.append(buffer.rstrip(" \n"))
- return (_STATE.ConsumeSpace, "")
- return (_STATE.ConsumeSpace, "")
- return (_STATE.ConsumeComment, buffer)
-
-def _handleParseDependency(input, buffer, result):
- if input == EOF:
- fail("Enountered unexpected end of file while parsing requirement")
- elif input.isspace() or input in [">", "<", "~", "=", ";", "["]:
- result.requirements.append((buffer,))
- return (_STATE.ParseRequirement, buffer + input)
-
- return (_STATE.ParseDependency, buffer + input)
-
-def _handleParseOption(input, buffer, result):
- if input == "\n" and buffer.endswith("\\"):
- return (_STATE.ParseOption, buffer[0:-1])
- elif input == " ":
- result.options.append(buffer.rstrip("\n"))
- return (_STATE.ParseOption, "")
- elif input == "\n" or input == EOF:
- result.options.append(buffer.rstrip("\n"))
- return (_STATE.ConsumeSpace, "")
- elif input == "#" and (len(buffer) == 0 or buffer[-1].isspace()):
- return (_STATE.ConsumeComment, buffer)
-
- return (_STATE.ParseOption, buffer + input)
-
-def _handleParseRequirement(input, buffer, result):
- if input == "\n" and buffer.endswith("\\"):
- return (_STATE.ParseRequirement, buffer[0:-1])
- elif input == "\n" or input == EOF:
- result.requirements[-1] = (result.requirements[-1][0], buffer.rstrip(" \n"))
- return (_STATE.ConsumeSpace, "")
- elif input == "#" and (len(buffer) == 0 or buffer[-1].isspace()):
- return (_STATE.ConsumeComment, buffer)
-
- return (_STATE.ParseRequirement, buffer + input)
+parse = parse_requirements_txt
diff --git a/python/pip_install/tools/dependency_resolver/BUILD.bazel b/python/pip_install/tools/dependency_resolver/BUILD.bazel
index c2cfb39..467b009 100644
--- a/python/pip_install/tools/dependency_resolver/BUILD.bazel
+++ b/python/pip_install/tools/dependency_resolver/BUILD.bazel
@@ -15,5 +15,5 @@
include = ["**/*.py"],
exclude = ["**/*_test.py"],
),
- visibility = ["//python/pip_install:__subpackages__"],
+ visibility = ["//:__subpackages__"],
)
diff --git a/python/pip_install/tools/wheel_installer/BUILD.bazel b/python/pip_install/tools/wheel_installer/BUILD.bazel
index a396488..0c24d5a 100644
--- a/python/pip_install/tools/wheel_installer/BUILD.bazel
+++ b/python/pip_install/tools/wheel_installer/BUILD.bazel
@@ -87,5 +87,5 @@
include = ["**/*.py"],
exclude = ["**/*_test.py"],
),
- visibility = ["//python/pip_install:__subpackages__"],
+ visibility = ["//:__subpackages__"],
)
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
index cd385e3..ccc6acd 100644
--- a/python/private/BUILD.bazel
+++ b/python/private/BUILD.bazel
@@ -31,6 +31,7 @@
"//python/private/bzlmod:distribution",
"//python/private/common:distribution",
"//python/private/proto:distribution",
+ "//python/private/pypi:distribution",
"//python/private/whl_filegroup:distribution",
"//tools/build_defs/python/private:distribution",
],
@@ -126,62 +127,6 @@
)
bzl_library(
- name = "patch_whl_bzl",
- srcs = ["patch_whl.bzl"],
- deps = [":parse_whl_name_bzl"],
-)
-
-bzl_library(
- name = "parse_requirements_bzl",
- srcs = ["parse_requirements.bzl"],
- deps = [
- ":normalize_name_bzl",
- ":pypi_index_sources_bzl",
- ":whl_target_platforms_bzl",
- "//python/pip_install:requirements_parser_bzl",
- ],
-)
-
-bzl_library(
- name = "parse_whl_name_bzl",
- srcs = ["parse_whl_name.bzl"],
-)
-
-bzl_library(
- name = "pip_flags_bzl",
- srcs = ["pip_flags.bzl"],
- deps = [
- ":enum_bzl",
- ],
-)
-
-bzl_library(
- name = "pip_repo_name_bzl",
- srcs = ["pip_repo_name.bzl"],
- deps = [
- ":normalize_name_bzl",
- ":parse_whl_name_bzl",
- ],
-)
-
-bzl_library(
- name = "pypi_index_bzl",
- srcs = ["pypi_index.bzl"],
- deps = [
- ":auth_bzl",
- ":normalize_name_bzl",
- ":text_util_bzl",
- "//python/pip_install:requirements_parser_bzl",
- "//python/private/bzlmod:bazel_features_bzl",
- ],
-)
-
-bzl_library(
- name = "pypi_index_sources_bzl",
- srcs = ["pypi_index_sources.bzl"],
-)
-
-bzl_library(
name = "py_cc_toolchain_bzl",
srcs = [
"py_cc_toolchain_macro.bzl",
@@ -286,17 +231,6 @@
)
bzl_library(
- name = "render_pkg_aliases_bzl",
- srcs = ["render_pkg_aliases.bzl"],
- deps = [
- ":normalize_name_bzl",
- ":text_util_bzl",
- ":version_label_bzl",
- "//python/pip_install/private:generate_group_library_build_bazel_bzl",
- ],
-)
-
-bzl_library(
name = "repo_utils_bzl",
srcs = ["repo_utils.bzl"],
)
@@ -340,20 +274,6 @@
srcs = ["version_label.bzl"],
)
-bzl_library(
- name = "whl_target_platforms_bzl",
- srcs = ["whl_target_platforms.bzl"],
- visibility = ["//:__subpackages__"],
- deps = [
- "parse_whl_name_bzl",
- ],
-)
-
-bzl_library(
- name = "labels_bzl",
- srcs = ["labels.bzl"],
-)
-
# @bazel_tools can't define bzl_library itself, so we just put a wrapper around it.
bzl_library(
name = "bazel_tools_bzl",
diff --git a/python/private/bzlmod/BUILD.bazel b/python/private/bzlmod/BUILD.bazel
index 3362f34..2cb35fc 100644
--- a/python/private/bzlmod/BUILD.bazel
+++ b/python/private/bzlmod/BUILD.bazel
@@ -13,7 +13,6 @@
# limitations under the License.
load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
-load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")
package(default_visibility = ["//:__subpackages__"])
@@ -29,33 +28,7 @@
name = "pip_bzl",
srcs = ["pip.bzl"],
deps = [
- ":pip_repository_bzl",
- "//python/pip_install:pip_repository_bzl",
- "//python/private:pypi_index_bzl",
- "//python/private:full_version_bzl",
- "//python/private:normalize_name_bzl",
- "//python/private:parse_requirements_bzl",
- "//python/private:parse_whl_name_bzl",
- "//python/private:pip_repo_name_bzl",
- "//python/private:version_label_bzl",
- ":bazel_features_bzl",
- ] + [
- "@pythons_hub//:interpreters_bzl",
- ] if BZLMOD_ENABLED else [],
-)
-
-bzl_library(
- name = "bazel_features_bzl",
- deps = ["@bazel_features//:features"],
-)
-
-bzl_library(
- name = "pip_repository_bzl",
- srcs = ["pip_repository.bzl"],
- visibility = ["//:__subpackages__"],
- deps = [
- "//python/private:render_pkg_aliases_bzl",
- "//python/private:text_util_bzl",
+ "//python/private/pypi:bzlmod_bzl",
],
)
@@ -66,6 +39,8 @@
":pythons_hub_bzl",
"//python:repositories_bzl",
"//python/private:toolchains_repo_bzl",
+ "//python/private:util_bzl",
+ "@bazel_features//:features",
],
)
diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl
index 2ded3a4..ecf94b6 100644
--- a/python/private/bzlmod/pip.bzl
+++ b/python/private/bzlmod/pip.bzl
@@ -12,817 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-"pip module extension for use with bzlmod"
+"pip module extensions for use with bzlmod."
-load("@bazel_features//:features.bzl", "bazel_features")
-load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS")
-load(
- "//python/pip_install:pip_repository.bzl",
- "pip_repository_attrs",
- "use_isolated",
- "whl_library",
-)
-load("//python/private:auth.bzl", "AUTH_ATTRS")
-load("//python/private:normalize_name.bzl", "normalize_name")
-load("//python/private:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
-load("//python/private:parse_whl_name.bzl", "parse_whl_name")
-load("//python/private:pip_repo_name.bzl", "pip_repo_name")
-load("//python/private:pypi_index.bzl", "simpleapi_download")
-load("//python/private:render_pkg_aliases.bzl", "whl_alias")
-load("//python/private:repo_utils.bzl", "repo_utils")
-load("//python/private:version_label.bzl", "version_label")
-load(":pip_repository.bzl", "pip_repository")
+load("//python/private/pypi:bzlmod.bzl", "pypi", "pypi_internal")
-def _parse_version(version):
- major, _, version = version.partition(".")
- minor, _, version = version.partition(".")
- patch, _, version = version.partition(".")
- build, _, version = version.partition(".")
-
- return struct(
- # use semver vocabulary here
- major = major,
- minor = minor,
- patch = patch, # this is called `micro` in the Python interpreter versioning scheme
- build = build,
- )
-
-def _major_minor_version(version):
- version = _parse_version(version)
- return "{}.{}".format(version.major, version.minor)
-
-def _whl_mods_impl(mctx):
- """Implementation of the pip.whl_mods tag class.
-
- This creates the JSON files used to modify the creation of different wheels.
-"""
- whl_mods_dict = {}
- for mod in mctx.modules:
- for whl_mod_attr in mod.tags.whl_mods:
- if whl_mod_attr.hub_name not in whl_mods_dict.keys():
- whl_mods_dict[whl_mod_attr.hub_name] = {whl_mod_attr.whl_name: whl_mod_attr}
- elif whl_mod_attr.whl_name in whl_mods_dict[whl_mod_attr.hub_name].keys():
- # We cannot have the same wheel name in the same hub, as we
- # will create the same JSON file name.
- fail("""\
-Found same whl_name '{}' in the same hub '{}', please use a different hub_name.""".format(
- whl_mod_attr.whl_name,
- whl_mod_attr.hub_name,
- ))
- else:
- whl_mods_dict[whl_mod_attr.hub_name][whl_mod_attr.whl_name] = whl_mod_attr
-
- for hub_name, whl_maps in whl_mods_dict.items():
- whl_mods = {}
-
- # create a struct that we can pass to the _whl_mods_repo rule
- # to create the different JSON files.
- for whl_name, mods in whl_maps.items():
- build_content = mods.additive_build_content
- if mods.additive_build_content_file != None and mods.additive_build_content != "":
- fail("""\
-You cannot use both the additive_build_content and additive_build_content_file arguments at the same time.
-""")
- elif mods.additive_build_content_file != None:
- build_content = mctx.read(mods.additive_build_content_file)
-
- whl_mods[whl_name] = json.encode(struct(
- additive_build_content = build_content,
- copy_files = mods.copy_files,
- copy_executables = mods.copy_executables,
- data = mods.data,
- data_exclude_glob = mods.data_exclude_glob,
- srcs_exclude_glob = mods.srcs_exclude_glob,
- ))
-
- _whl_mods_repo(
- name = hub_name,
- whl_mods = whl_mods,
- )
-
-def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, simpleapi_cache):
- logger = repo_utils.logger(module_ctx)
- python_interpreter_target = pip_attr.python_interpreter_target
- is_hub_reproducible = True
-
- # if we do not have the python_interpreter set in the attributes
- # we programmatically find it.
- hub_name = pip_attr.hub_name
- if python_interpreter_target == None and not pip_attr.python_interpreter:
- python_name = "python_{}_host".format(
- pip_attr.python_version.replace(".", "_"),
- )
- if python_name not in INTERPRETER_LABELS:
- fail((
- "Unable to find interpreter for pip hub '{hub_name}' for " +
- "python_version={version}: Make sure a corresponding " +
- '`python.toolchain(python_version="{version}")` call exists.' +
- "Expected to find {python_name} among registered versions:\n {labels}"
- ).format(
- hub_name = hub_name,
- version = pip_attr.python_version,
- python_name = python_name,
- labels = " \n".join(INTERPRETER_LABELS),
- ))
- python_interpreter_target = INTERPRETER_LABELS[python_name]
-
- pip_name = "{}_{}".format(
- hub_name,
- version_label(pip_attr.python_version),
- )
- major_minor = _major_minor_version(pip_attr.python_version)
-
- if hub_name not in whl_map:
- whl_map[hub_name] = {}
-
- whl_modifications = {}
- if pip_attr.whl_modifications != None:
- for mod, whl_name in pip_attr.whl_modifications.items():
- whl_modifications[whl_name] = mod
-
- if pip_attr.experimental_requirement_cycles:
- requirement_cycles = {
- name: [normalize_name(whl_name) for whl_name in whls]
- for name, whls in pip_attr.experimental_requirement_cycles.items()
- }
-
- whl_group_mapping = {
- whl_name: group_name
- for group_name, group_whls in requirement_cycles.items()
- for whl_name in group_whls
- }
-
- # TODO @aignas 2024-04-05: how do we support different requirement
- # cycles for different abis/oses? For now we will need the users to
- # assume the same groups across all versions/platforms until we start
- # using an alternative cycle resolution strategy.
- group_map[hub_name] = pip_attr.experimental_requirement_cycles
- else:
- whl_group_mapping = {}
- requirement_cycles = {}
-
- # Create a new wheel library for each of the different whls
-
- get_index_urls = None
- if pip_attr.experimental_index_url:
- if pip_attr.download_only:
- fail("Currently unsupported to use `download_only` and `experimental_index_url`")
-
- get_index_urls = lambda ctx, distributions: simpleapi_download(
- ctx,
- attr = struct(
- index_url = pip_attr.experimental_index_url,
- extra_index_urls = pip_attr.experimental_extra_index_urls or [],
- index_url_overrides = pip_attr.experimental_index_url_overrides or {},
- sources = distributions,
- envsubst = pip_attr.envsubst,
- # Auth related info
- netrc = pip_attr.netrc,
- auth_patterns = pip_attr.auth_patterns,
- ),
- cache = simpleapi_cache,
- parallel_download = pip_attr.parallel_download,
- )
-
- requirements_by_platform = parse_requirements(
- module_ctx,
- requirements_by_platform = pip_attr.requirements_by_platform,
- requirements_linux = pip_attr.requirements_linux,
- requirements_lock = pip_attr.requirements_lock,
- requirements_osx = pip_attr.requirements_darwin,
- requirements_windows = pip_attr.requirements_windows,
- extra_pip_args = pip_attr.extra_pip_args,
- get_index_urls = get_index_urls,
- python_version = major_minor,
- logger = logger,
- )
-
- repository_platform = host_platform(module_ctx.os)
- for whl_name, requirements in requirements_by_platform.items():
- # We are not using the "sanitized name" because the user
- # would need to guess what name we modified the whl name
- # to.
- annotation = whl_modifications.get(whl_name)
- whl_name = normalize_name(whl_name)
-
- group_name = whl_group_mapping.get(whl_name)
- group_deps = requirement_cycles.get(group_name, [])
-
- # Construct args separately so that the lock file can be smaller and does not include unused
- # attrs.
- whl_library_args = dict(
- repo = pip_name,
- dep_template = "@{}//{{name}}:{{target}}".format(hub_name),
- )
- maybe_args = dict(
- # The following values are safe to omit if they have false like values
- annotation = annotation,
- download_only = pip_attr.download_only,
- enable_implicit_namespace_pkgs = pip_attr.enable_implicit_namespace_pkgs,
- environment = pip_attr.environment,
- envsubst = pip_attr.envsubst,
- experimental_target_platforms = pip_attr.experimental_target_platforms,
- group_deps = group_deps,
- group_name = group_name,
- pip_data_exclude = pip_attr.pip_data_exclude,
- python_interpreter = pip_attr.python_interpreter,
- python_interpreter_target = python_interpreter_target,
- whl_patches = {
- p: json.encode(args)
- for p, args in whl_overrides.get(whl_name, {}).items()
- },
- )
- whl_library_args.update({k: v for k, v in maybe_args.items() if v})
- maybe_args_with_default = dict(
- # The following values have defaults next to them
- isolated = (use_isolated(module_ctx, pip_attr), True),
- quiet = (pip_attr.quiet, True),
- timeout = (pip_attr.timeout, 600),
- )
- whl_library_args.update({
- k: v
- for k, (v, default) in maybe_args_with_default.items()
- if v != default
- })
-
- if get_index_urls:
- # TODO @aignas 2024-05-26: move to a separate function
- found_something = False
- for requirement in requirements:
- for distribution in requirement.whls + [requirement.sdist]:
- if not distribution:
- # sdist may be None
- continue
-
- found_something = True
- is_hub_reproducible = False
-
- if pip_attr.netrc:
- whl_library_args["netrc"] = pip_attr.netrc
- if pip_attr.auth_patterns:
- whl_library_args["auth_patterns"] = pip_attr.auth_patterns
-
- # pip is not used to download wheels and the python `whl_library` helpers are only extracting things
- whl_library_args.pop("extra_pip_args", None)
-
- # This is no-op because pip is not used to download the wheel.
- whl_library_args.pop("download_only", None)
-
- repo_name = pip_repo_name(pip_name, distribution.filename, distribution.sha256)
- whl_library_args["requirement"] = requirement.srcs.requirement
- whl_library_args["urls"] = [distribution.url]
- whl_library_args["sha256"] = distribution.sha256
- whl_library_args["filename"] = distribution.filename
- whl_library_args["experimental_target_platforms"] = requirement.target_platforms
-
- # Pure python wheels or sdists may need to have a platform here
- target_platforms = None
- if distribution.filename.endswith("-any.whl") or not distribution.filename.endswith(".whl"):
- if len(requirements) > 1:
- target_platforms = requirement.target_platforms
-
- whl_library(name = repo_name, **dict(sorted(whl_library_args.items())))
-
- whl_map[hub_name].setdefault(whl_name, []).append(
- whl_alias(
- repo = repo_name,
- version = major_minor,
- filename = distribution.filename,
- target_platforms = target_platforms,
- ),
- )
-
- if found_something:
- continue
-
- requirement = select_requirement(
- requirements,
- platform = repository_platform,
- )
- if not requirement:
- # Sometimes the package is not present for host platform if there
- # are whls specified only in particular requirements files, in that
- # case just continue, however, if the download_only flag is set up,
- # then the user can also specify the target platform of the wheel
- # packages they want to download, in that case there will be always
- # a requirement here, so we will not be in this code branch.
- continue
- elif get_index_urls:
- logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line))
-
- whl_library_args["requirement"] = requirement.requirement_line
- if requirement.extra_pip_args:
- whl_library_args["extra_pip_args"] = requirement.extra_pip_args
-
- # We sort so that the lock-file remains the same no matter the order of how the
- # args are manipulated in the code going before.
- repo_name = "{}_{}".format(pip_name, whl_name)
- whl_library(name = repo_name, **dict(sorted(whl_library_args.items())))
- whl_map[hub_name].setdefault(whl_name, []).append(
- whl_alias(
- repo = repo_name,
- version = major_minor,
- ),
- )
-
- return is_hub_reproducible
-
-def _pip_impl(module_ctx):
- """Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories.
-
- This implementation iterates through all of the `pip.parse` calls and creates
- different pip hub repositories based on the "hub_name". Each of the
- pip calls create spoke repos that uses a specific Python interpreter.
-
- In a MODULES.bazel file we have:
-
- pip.parse(
- hub_name = "pip",
- python_version = 3.9,
- requirements_lock = "//:requirements_lock_3_9.txt",
- requirements_windows = "//:requirements_windows_3_9.txt",
- )
- pip.parse(
- hub_name = "pip",
- python_version = 3.10,
- requirements_lock = "//:requirements_lock_3_10.txt",
- requirements_windows = "//:requirements_windows_3_10.txt",
- )
-
- For instance, we have a hub with the name of "pip".
- A repository named the following is created. It is actually called last when
- all of the pip spokes are collected.
-
- - @@rules_python~override~pip~pip
-
- As shown in the example code above we have the following.
- Two different pip.parse statements exist in MODULE.bazel provide the hub_name "pip".
- These definitions create two different pip spoke repositories that are
- related to the hub "pip".
- One spoke uses Python 3.9 and the other uses Python 3.10. This code automatically
- determines the Python version and the interpreter.
- Both of these pip spokes contain requirements files that includes websocket
- and its dependencies.
-
- We also need repositories for the wheels that the different pip spokes contain.
- For each Python version a different wheel repository is created. In our example
- each pip spoke had a requirements file that contained websockets. We
- then create two different wheel repositories that are named the following.
-
- - @@rules_python~override~pip~pip_39_websockets
- - @@rules_python~override~pip~pip_310_websockets
-
- And if the wheel has any other dependencies subsequent wheels are created in the same fashion.
-
- The hub repository has aliases for `pkg`, `data`, etc, which have a select that resolves to
- a spoke repository depending on the Python version.
-
- Also we may have more than one hub as defined in a MODULES.bazel file. So we could have multiple
- hubs pointing to various different pip spokes.
-
- Some other business rules notes. A hub can only have one spoke per Python version. We cannot
- have a hub named "pip" that has two spokes that use the Python 3.9 interpreter. Second
- we cannot have the same hub name used in sub-modules. The hub name has to be globally
- unique.
-
- This implementation also handles the creation of whl_modification JSON files that are used
- during the creation of wheel libraries. These JSON files used via the annotations argument
- when calling wheel_installer.py.
-
- Args:
- module_ctx: module contents
- """
-
- # Build all of the wheel modifications if the tag class is called.
- _whl_mods_impl(module_ctx)
-
- _overriden_whl_set = {}
- whl_overrides = {}
-
- for module in module_ctx.modules:
- for attr in module.tags.override:
- if not module.is_root:
- fail("overrides are only supported in root modules")
-
- if not attr.file.endswith(".whl"):
- fail("Only whl overrides are supported at this time")
-
- whl_name = normalize_name(parse_whl_name(attr.file).distribution)
-
- if attr.file in _overriden_whl_set:
- fail("Duplicate module overrides for '{}'".format(attr.file))
- _overriden_whl_set[attr.file] = None
-
- for patch in attr.patches:
- if whl_name not in whl_overrides:
- whl_overrides[whl_name] = {}
-
- if patch not in whl_overrides[whl_name]:
- whl_overrides[whl_name][patch] = struct(
- patch_strip = attr.patch_strip,
- whls = [],
- )
-
- whl_overrides[whl_name][patch].whls.append(attr.file)
-
- # Used to track all the different pip hubs and the spoke pip Python
- # versions.
- pip_hub_map = {}
-
- # Keeps track of all the hub's whl repos across the different versions.
- # dict[hub, dict[whl, dict[version, str pip]]]
- # Where hub, whl, and pip are the repo names
- hub_whl_map = {}
- hub_group_map = {}
-
- simpleapi_cache = {}
- is_extension_reproducible = True
-
- for mod in module_ctx.modules:
- for pip_attr in mod.tags.parse:
- hub_name = pip_attr.hub_name
- if hub_name not in pip_hub_map:
- pip_hub_map[pip_attr.hub_name] = struct(
- module_name = mod.name,
- python_versions = [pip_attr.python_version],
- )
- elif pip_hub_map[hub_name].module_name != mod.name:
- # We cannot have two hubs with the same name in different
- # modules.
- fail((
- "Duplicate cross-module pip hub named '{hub}': pip hub " +
- "names must be unique across modules. First defined " +
- "by module '{first_module}', second attempted by " +
- "module '{second_module}'"
- ).format(
- hub = hub_name,
- first_module = pip_hub_map[hub_name].module_name,
- second_module = mod.name,
- ))
-
- elif pip_attr.python_version in pip_hub_map[hub_name].python_versions:
- fail((
- "Duplicate pip python version '{version}' for hub " +
- "'{hub}' in module '{module}': the Python versions " +
- "used for a hub must be unique"
- ).format(
- hub = hub_name,
- module = mod.name,
- version = pip_attr.python_version,
- ))
- else:
- pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version)
-
- is_hub_reproducible = _create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache)
- is_extension_reproducible = is_extension_reproducible and is_hub_reproducible
-
- for hub_name, whl_map in hub_whl_map.items():
- pip_repository(
- name = hub_name,
- repo_name = hub_name,
- whl_map = {
- key: json.encode(value)
- for key, value in whl_map.items()
- },
- default_version = _major_minor_version(DEFAULT_PYTHON_VERSION),
- groups = hub_group_map.get(hub_name),
- )
-
- if bazel_features.external_deps.extension_metadata_has_reproducible:
- # If we are not using the `experimental_index_url feature, the extension is fully
- # deterministic and we don't need to create a lock entry for it.
- #
- # In order to be able to dogfood the `experimental_index_url` feature before it gets
- # stabilized, we have created the `_pip_non_reproducible` function, that will result
- # in extra entries in the lock file.
- return module_ctx.extension_metadata(reproducible = is_extension_reproducible)
- else:
- return None
-
-def _pip_non_reproducible(module_ctx):
- _pip_impl(module_ctx)
-
- # We default to calling the PyPI index and that will go into the
- # MODULE.bazel.lock file, hence return nothing here.
- return None
-
-def _pip_parse_ext_attrs(**kwargs):
- """Get the attributes for the pip extension.
-
- Args:
- **kwargs: A kwarg for setting defaults for the specific attributes. The
- key is expected to be the same as the attribute key.
-
- Returns:
- A dict of attributes.
- """
- attrs = dict({
- "experimental_extra_index_urls": attr.string_list(
- doc = """\
-The extra index URLs to use for downloading wheels using bazel downloader.
-Each value is going to be subject to `envsubst` substitutions if necessary.
-
-The indexes must support Simple API as described here:
-https://packaging.python.org/en/latest/specifications/simple-repository-api/
-
-This is equivalent to `--extra-index-urls` `pip` option.
-""",
- default = [],
- ),
- "experimental_index_url": attr.string(
- default = kwargs.get("experimental_index_url", ""),
- doc = """\
-The index URL to use for downloading wheels using bazel downloader. This value is going
-to be subject to `envsubst` substitutions if necessary.
-
-The indexes must support Simple API as described here:
-https://packaging.python.org/en/latest/specifications/simple-repository-api/
-
-In the future this could be defaulted to `https://pypi.org` when this feature becomes
-stable.
-
-This is equivalent to `--index-url` `pip` option.
-""",
- ),
- "experimental_index_url_overrides": attr.string_dict(
- doc = """\
-The index URL overrides for each package to use for downloading wheels using
-bazel downloader. This value is going to be subject to `envsubst` substitutions
-if necessary.
-
-The key is the package name (will be normalized before usage) and the value is the
-index URL.
-
-This design pattern has been chosen in order to be fully deterministic about which
-packages come from which source. We want to avoid issues similar to what happened in
-https://pytorch.org/blog/compromised-nightly-dependency/.
-
-The indexes must support Simple API as described here:
-https://packaging.python.org/en/latest/specifications/simple-repository-api/
-""",
- ),
- "hub_name": attr.string(
- mandatory = True,
- doc = """
-The name of the repo pip dependencies will be accessible from.
-
-This name must be unique between modules; unless your module is guaranteed to
-always be the root module, it's highly recommended to include your module name
-in the hub name. Repo mapping, `use_repo(..., pip="my_modules_pip_deps")`, can
-be used for shorter local names within your module.
-
-Within a module, the same `hub_name` can be specified to group different Python
-versions of pip dependencies under one repository name. This allows using a
-Python version-agnostic name when referring to pip dependencies; the
-correct version will be automatically selected.
-
-Typically, a module will only have a single hub of pip dependencies, but this
-is not required. Each hub is a separate resolution of pip dependencies. This
-means if different programs need different versions of some library, separate
-hubs can be created, and each program can use its respective hub's targets.
-Targets from different hubs should not be used together.
-""",
- ),
- "parallel_download": attr.bool(
- doc = """\
-The flag allows to make use of parallel downloading feature in bazel 7.1 and above
-when the bazel downloader is used. This is by default enabled as it improves the
-performance by a lot, but in case the queries to the simple API are very expensive
-or when debugging authentication issues one may want to disable this feature.
-
-NOTE, This will download (potentially duplicate) data for multiple packages if
-there is more than one index available, but in general this should be negligible
-because the simple API calls are very cheap and the user should not notice any
-extra overhead.
-
-If we are in synchronous mode, then we will use the first result that we
-find in case extra indexes are specified.
-""",
- default = True,
- ),
- "python_version": attr.string(
- mandatory = True,
- doc = """
-The Python version the dependencies are targetting, in Major.Minor format
-(e.g., "3.11") or patch level granularity (e.g. "3.11.1").
-
-If an interpreter isn't explicitly provided (using `python_interpreter` or
-`python_interpreter_target`), then the version specified here must have
-a corresponding `python.toolchain()` configured.
-""",
- ),
- "whl_modifications": attr.label_keyed_string_dict(
- mandatory = False,
- doc = """\
-A dict of labels to wheel names that is typically generated by the whl_modifications.
-The labels are JSON config files describing the modifications.
-""",
- ),
- }, **pip_repository_attrs)
- attrs.update(AUTH_ATTRS)
-
- # Like the pip_repository rule, we end up setting this manually so
- # don't allow users to override it.
- attrs.pop("repo_prefix")
-
- # annotations has been replaced with whl_modifications in bzlmod
- attrs.pop("annotations")
-
- return attrs
-
-def _whl_mod_attrs():
- attrs = {
- "additive_build_content": attr.string(
- doc = "(str, optional): Raw text to add to the generated `BUILD` file of a package.",
- ),
- "additive_build_content_file": attr.label(
- doc = """\
-(label, optional): path to a BUILD file to add to the generated
-`BUILD` file of a package. You cannot use both additive_build_content and additive_build_content_file
-arguments at the same time.""",
- ),
- "copy_executables": attr.string_dict(
- doc = """\
-(dict, optional): A mapping of `src` and `out` files for
-[@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as
-executable.""",
- ),
- "copy_files": attr.string_dict(
- doc = """\
-(dict, optional): A mapping of `src` and `out` files for
-[@bazel_skylib//rules:copy_file.bzl][cf]""",
- ),
- "data": attr.string_list(
- doc = """\
-(list, optional): A list of labels to add as `data` dependencies to
-the generated `py_library` target.""",
- ),
- "data_exclude_glob": attr.string_list(
- doc = """\
-(list, optional): A list of exclude glob patterns to add as `data` to
-the generated `py_library` target.""",
- ),
- "hub_name": attr.string(
- doc = """\
-Name of the whl modification, hub we use this name to set the modifications for
-pip.parse. If you have different pip hubs you can use a different name,
-otherwise it is best practice to just use one.
-
-You cannot have the same `hub_name` in different modules. You can reuse the same
-name in the same module for different wheels that you put in the same hub, but you
-cannot have a child module that uses the same `hub_name`.
-""",
- mandatory = True,
- ),
- "srcs_exclude_glob": attr.string_list(
- doc = """\
-(list, optional): A list of labels to add as `srcs` to the generated
-`py_library` target.""",
- ),
- "whl_name": attr.string(
- doc = "The whl name that the modifications are used for.",
- mandatory = True,
- ),
- }
- return attrs
-
-# NOTE: the naming of 'override' is taken from the bzlmod native
-# 'archive_override', 'git_override' bzlmod functions.
-_override_tag = tag_class(
- attrs = {
- "file": attr.string(
- doc = """\
-The Python distribution file name which needs to be patched. This will be
-applied to all repositories that setup this distribution via the pip.parse tag
-class.""",
- mandatory = True,
- ),
- "patch_strip": attr.int(
- default = 0,
- doc = """\
-The number of leading path segments to be stripped from the file name in the
-patches.""",
- ),
- "patches": attr.label_list(
- doc = """\
-A list of patches to apply to the repository *after* 'whl_library' is extracted
-and BUILD.bazel file is generated.""",
- mandatory = True,
- ),
- },
- doc = """\
-Apply any overrides (e.g. patches) to a given Python distribution defined by
-other tags in this extension.""",
-)
-
-pip = module_extension(
- doc = """\
-This extension is used to make dependencies from pip available.
-
-pip.parse:
-To use, call `pip.parse()` and specify `hub_name` and your requirements file.
-Dependencies will be downloaded and made available in a repo named after the
-`hub_name` argument.
-
-Each `pip.parse()` call configures a particular Python version. Multiple calls
-can be made to configure different Python versions, and will be grouped by
-the `hub_name` argument. This allows the same logical name, e.g. `@pip//numpy`
-to automatically resolve to different, Python version-specific, libraries.
-
-pip.whl_mods:
-This tag class is used to help create JSON files to describe modifications to
-the BUILD files for wheels.
-""",
- implementation = _pip_impl,
- tag_classes = {
- "override": _override_tag,
- "parse": tag_class(
- attrs = _pip_parse_ext_attrs(),
- doc = """\
-This tag class is used to create a pip hub and all of the spokes that are part of that hub.
-This tag class reuses most of the pip attributes that are found in
-@rules_python//python/pip_install:pip_repository.bzl.
-The exception is it does not use the arg 'repo_prefix'. We set the repository
-prefix for the user and the alias arg is always True in bzlmod.
-""",
- ),
- "whl_mods": tag_class(
- attrs = _whl_mod_attrs(),
- doc = """\
-This tag class is used to create JSON file that are used when calling wheel_builder.py. These
-JSON files contain instructions on how to modify a wheel's project. Each of the attributes
-create different modifications based on the type of attribute. Previously to bzlmod these
-JSON files where referred to as annotations, and were renamed to whl_modifications in this
-extension.
-""",
- ),
- },
-)
-
-pip_internal = module_extension(
- doc = """\
-This extension is used to make dependencies from pypi available.
-
-For now this is intended to be used internally so that usage of the `pip`
-extension in `rules_python` does not affect the evaluations of the extension
-for the consumers.
-
-pip.parse:
-To use, call `pip.parse()` and specify `hub_name` and your requirements file.
-Dependencies will be downloaded and made available in a repo named after the
-`hub_name` argument.
-
-Each `pip.parse()` call configures a particular Python version. Multiple calls
-can be made to configure different Python versions, and will be grouped by
-the `hub_name` argument. This allows the same logical name, e.g. `@pypi//numpy`
-to automatically resolve to different, Python version-specific, libraries.
-
-pip.whl_mods:
-This tag class is used to help create JSON files to describe modifications to
-the BUILD files for wheels.
-""",
- implementation = _pip_non_reproducible,
- tag_classes = {
- "override": _override_tag,
- "parse": tag_class(
- attrs = _pip_parse_ext_attrs(
- experimental_index_url = "https://pypi.org/simple",
- ),
- doc = """\
-This tag class is used to create a pypi hub and all of the spokes that are part of that hub.
-This tag class reuses most of the pypi attributes that are found in
-@rules_python//python/pip_install:pip_repository.bzl.
-The exception is it does not use the arg 'repo_prefix'. We set the repository
-prefix for the user and the alias arg is always True in bzlmod.
-""",
- ),
- "whl_mods": tag_class(
- attrs = _whl_mod_attrs(),
- doc = """\
-This tag class is used to create JSON file that are used when calling wheel_builder.py. These
-JSON files contain instructions on how to modify a wheel's project. Each of the attributes
-create different modifications based on the type of attribute. Previously to bzlmod these
-JSON files where referred to as annotations, and were renamed to whl_modifications in this
-extension.
-""",
- ),
- },
-)
-
-def _whl_mods_repo_impl(rctx):
- rctx.file("BUILD.bazel", "")
- for whl_name, mods in rctx.attr.whl_mods.items():
- rctx.file("{}.json".format(whl_name), mods)
-
-_whl_mods_repo = repository_rule(
- doc = """\
-This rule creates json files based on the whl_mods attribute.
-""",
- implementation = _whl_mods_repo_impl,
- attrs = {
- "whl_mods": attr.string_dict(
- mandatory = True,
- doc = "JSON endcoded string that is provided to wheel_builder.py",
- ),
- },
-)
+pip = pypi
+pip_internal = pypi_internal
diff --git a/python/private/normalize_platform.bzl b/python/private/normalize_platform.bzl
deleted file mode 100644
index 633062f..0000000
--- a/python/private/normalize_platform.bzl
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2024 The Bazel Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
new file mode 100644
index 0000000..1530837
--- /dev/null
+++ b/python/private/pypi/BUILD.bazel
@@ -0,0 +1,238 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")
+
+package(default_visibility = ["//:__subpackages__"])
+
+licenses(["notice"])
+
+filegroup(
+ name = "distribution",
+ srcs = glob(["**"]),
+ visibility = ["//python/private:__pkg__"],
+)
+
+# Filegroup of bzl files that can be used by downstream rules for documentation generation
+filegroup(
+ name = "bzl",
+ srcs = glob(["**/*.bzl"]),
+ visibility = ["//python/private:__pkg__"],
+)
+
+# Keep sorted by library name and keep the files named by the main symbol they export
+
+bzl_library(
+ name = "attrs_bzl",
+ srcs = ["attrs.bzl"],
+)
+
+bzl_library(
+ name = "bzlmod_bzl",
+ srcs = ["bzlmod.bzl"],
+ deps = [
+ ":attrs_bzl",
+ ":hub_repository_bzl",
+ ":parse_requirements_bzl",
+ ":parse_whl_name_bzl",
+ ":pip_repository_attrs_bzl",
+ ":simpleapi_download_bzl",
+ ":whl_library_bzl",
+ ":whl_repo_name_bzl",
+ "//python/private:full_version_bzl",
+ "//python/private:normalize_name_bzl",
+ "//python/private:version_label_bzl",
+ "@bazel_features//:features",
+ ] + [
+ "@pythons_hub//:interpreters_bzl",
+ ] if BZLMOD_ENABLED else [],
+)
+
+bzl_library(
+ name = "config_settings_bzl",
+ srcs = ["config_settings.bzl"],
+ deps = ["flags_bzl"],
+)
+
+bzl_library(
+ name = "flags_bzl",
+ srcs = ["flags.bzl"],
+ deps = ["//python/private:enum_bzl"],
+)
+
+bzl_library(
+ name = "generate_whl_library_build_bazel_bzl",
+ srcs = ["generate_whl_library_build_bazel.bzl"],
+ deps = [
+ ":labels_bzl",
+ "//python/private:normalize_name_bzl",
+ ],
+)
+
+bzl_library(
+ name = "generate_group_library_build_bazel_bzl",
+ srcs = ["generate_group_library_build_bazel.bzl"],
+ deps = [
+ ":labels_bzl",
+ "//python/private:normalize_name_bzl",
+ ],
+)
+
+bzl_library(
+ name = "group_library_bzl",
+ srcs = ["group_library.bzl"],
+ deps = [
+ ":generate_group_library_build_bazel_bzl",
+ ],
+)
+
+bzl_library(
+ name = "hub_repository_bzl",
+ srcs = ["hub_repository.bzl"],
+ visibility = ["//:__subpackages__"],
+ deps = [
+ ":render_pkg_aliases_bzl",
+ "//python/private:text_util_bzl",
+ ],
+)
+
+bzl_library(
+ name = "index_sources_bzl",
+ srcs = ["index_sources.bzl"],
+)
+
+bzl_library(
+ name = "labels_bzl",
+ srcs = ["labels.bzl"],
+)
+
+bzl_library(
+ name = "package_annotation_bzl",
+ srcs = ["package_annotation.bzl"],
+)
+
+bzl_library(
+ name = "parse_requirements_bzl",
+ srcs = ["parse_requirements.bzl"],
+ deps = [
+ ":index_sources_bzl",
+ ":parse_requirements_txt_bzl",
+ ":whl_target_platforms_bzl",
+ "//python/private:normalize_name_bzl",
+ ],
+)
+
+bzl_library(
+ name = "parse_requirements_txt_bzl",
+ srcs = ["parse_requirements_txt.bzl"],
+)
+
+bzl_library(
+ name = "parse_simpleapi_html_bzl",
+ srcs = ["parse_simpleapi_html.bzl"],
+)
+
+bzl_library(
+ name = "parse_whl_name_bzl",
+ srcs = ["parse_whl_name.bzl"],
+)
+
+bzl_library(
+ name = "patch_whl_bzl",
+ srcs = ["patch_whl.bzl"],
+ deps = [
+ ":parse_whl_name_bzl",
+ "//python/private:repo_utils_bzl",
+ ],
+)
+
+bzl_library(
+ name = "pip_repository_bzl",
+ srcs = ["pip_repository.bzl"],
+ deps = [
+ ":attrs_bzl",
+ ":parse_requirements_bzl",
+ ":pip_repository_attrs_bzl",
+ ":render_pkg_aliases_bzl",
+ "//python/private:normalize_name_bzl",
+ "//python/private:repo_utils_bzl",
+ "//python/private:text_util_bzl",
+ "@bazel_skylib//lib:sets",
+ ],
+)
+
+bzl_library(
+ name = "pip_repository_attrs_bzl",
+ srcs = ["pip_repository_attrs.bzl"],
+)
+
+bzl_library(
+ name = "render_pkg_aliases_bzl",
+ srcs = ["render_pkg_aliases.bzl"],
+ deps = [
+ ":generate_group_library_build_bazel_bzl",
+ ":labels_bzl",
+ ":parse_whl_name_bzl",
+ ":whl_target_platforms_bzl",
+ "//python/private:normalize_name_bzl",
+ "//python/private:text_util_bzl",
+ ],
+)
+
+bzl_library(
+ name = "simpleapi_download_bzl",
+ srcs = ["simpleapi_download.bzl"],
+ deps = [
+ ":parse_simpleapi_html_bzl",
+ "//python/private:auth_bzl",
+ "//python/private:normalize_name_bzl",
+ "//python/private:text_util_bzl",
+ "@bazel_features//:features",
+ ],
+)
+
+bzl_library(
+ name = "whl_library_bzl",
+ srcs = ["whl_library.bzl"],
+ deps = [
+ ":attrs_bzl",
+ ":generate_whl_library_build_bazel_bzl",
+ ":parse_whl_name_bzl",
+ ":patch_whl_bzl",
+ ":whl_target_platforms_bzl",
+ "//python:repositories_bzl",
+ "//python:versions_bzl",
+ "//python/pip_install:repositories_bzl",
+ "//python/private:auth_bzl",
+ "//python/private:envsubst_bzl",
+ "//python/private:repo_utils_bzl",
+ "//python/private:toolchains_repo_bzl",
+ ],
+)
+
+bzl_library(
+ name = "whl_repo_name_bzl",
+ srcs = ["whl_repo_name.bzl"],
+ deps = [
+ ":parse_whl_name_bzl",
+ "//python/private:normalize_name_bzl",
+ ],
+)
+
+bzl_library(
+ name = "whl_target_platforms_bzl",
+ srcs = ["whl_target_platforms.bzl"],
+ deps = [":parse_whl_name_bzl"],
+)
diff --git a/python/private/pypi/README.md b/python/private/pypi/README.md
new file mode 100644
index 0000000..6be5703
--- /dev/null
+++ b/python/private/pypi/README.md
@@ -0,0 +1,9 @@
+# PyPI integration code
+
+This code is for integrating with PyPI and other compatible indexes. At the
+moment we have code for:
+* Downloading packages using `pip` or `repository_ctx.download`.
+* Interacting with PyPI compatible indexes via [SimpleAPI] spec.
+* Locking a `requirements.in` or [PEP621] compliant `pyproject.toml`.
+
+[PEP621]: https://peps.python.org/pep-0621/
diff --git a/python/private/pypi/attrs.bzl b/python/private/pypi/attrs.bzl
new file mode 100644
index 0000000..79ffea5
--- /dev/null
+++ b/python/private/pypi/attrs.bzl
@@ -0,0 +1,224 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"common attributes for whl_library and pip_repository"
+
+ATTRS = {
+ "download_only": attr.bool(
+ doc = """
+Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of
+--platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different
+platform from the host platform.
+ """,
+ ),
+ "enable_implicit_namespace_pkgs": attr.bool(
+ default = False,
+ doc = """
+If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary
+and py_test targets must specify either `legacy_create_init=False` or the global Bazel option
+`--incompatible_default_to_explicit_init_py` to prevent `__init__.py` being automatically generated in every directory.
+
+This option is required to support some packages which cannot handle the conversion to pkg-util style.
+ """,
+ ),
+ "environment": attr.string_dict(
+ doc = """
+Environment variables to set in the pip subprocess.
+Can be used to set common variables such as `http_proxy`, `https_proxy` and `no_proxy`
+Note that pip is run with "--isolated" on the CLI so `PIP_<VAR>_<NAME>`
+style env vars are ignored, but env vars that control requests and urllib3
+can be passed. If you need `PIP_<VAR>_<NAME>`, take a look at `extra_pip_args`
+and `envsubst`.
+ """,
+ default = {},
+ ),
+ "envsubst": attr.string_list(
+ mandatory = False,
+ doc = """\
+A list of environment variables to substitute (e.g. `["PIP_INDEX_URL",
+"PIP_RETRIES"]`). The corresponding variables are expanded in `extra_pip_args`
+using the syntax `$VARNAME` or `${VARNAME}` (expanding to empty string if unset)
+or `${VARNAME:-default}` (expanding to default if the variable is unset or empty
+in the environment). Note: On Bazel 6 and Bazel 7.0 changes to the variables named
+here do not cause packages to be re-fetched. Don't fetch different things based
+on the value of these variables.
+""",
+ ),
+ "experimental_requirement_cycles": attr.string_list_dict(
+ default = {},
+ doc = """\
+A mapping of dependency cycle names to a list of requirements which form that cycle.
+
+Requirements which form cycles will be installed together and taken as
+dependencies together in order to ensure that the cycle is always satisified.
+
+Example:
+ `sphinx` depends on `sphinxcontrib-serializinghtml`
+ When listing both as requirements, ala
+
+ ```
+ py_binary(
+ name = "doctool",
+ ...
+ deps = [
+ "@pypi//sphinx:pkg",
+ "@pypi//sphinxcontrib_serializinghtml",
+ ]
+ )
+ ```
+
+ Will produce a Bazel error such as
+
+ ```
+ ERROR: .../external/pypi_sphinxcontrib_serializinghtml/BUILD.bazel:44:6: in alias rule @pypi_sphinxcontrib_serializinghtml//:pkg: cycle in dependency graph:
+ //:doctool (...)
+ @pypi//sphinxcontrib_serializinghtml:pkg (...)
+ .-> @pypi_sphinxcontrib_serializinghtml//:pkg (...)
+ | @pypi_sphinxcontrib_serializinghtml//:_pkg (...)
+ | @pypi_sphinx//:pkg (...)
+ | @pypi_sphinx//:_pkg (...)
+ `-- @pypi_sphinxcontrib_serializinghtml//:pkg (...)
+ ```
+
+ Which we can resolve by configuring these two requirements to be installed together as a cycle
+
+ ```
+ pip_parse(
+ ...
+ experimental_requirement_cycles = {
+ "sphinx": [
+ "sphinx",
+ "sphinxcontrib-serializinghtml",
+ ]
+ },
+ )
+ ```
+
+Warning:
+ If a dependency participates in multiple cycles, all of those cycles must be
+ collapsed down to one. For instance `a <-> b` and `a <-> c` cannot be listed
+ as two separate cycles.
+""",
+ ),
+ "experimental_target_platforms": attr.string_list(
+ default = [],
+ doc = """\
+A list of platforms that we will generate the conditional dependency graph for
+cross platform wheels by parsing the wheel metadata. This will generate the
+correct dependencies for packages like `sphinx` or `pylint`, which include
+`colorama` when installed and used on Windows platforms.
+
+An empty list means falling back to the legacy behaviour where the host
+platform is the target platform.
+
+WARNING: It may not work as expected in cases where the python interpreter
+implementation that is being used at runtime is different between different platforms.
+This has been tested for CPython only.
+
+For specific target platforms use values of the form `<os>_<arch>` where `<os>`
+is one of `linux`, `osx`, `windows` and arch is one of `x86_64`, `x86_32`,
+`aarch64`, `s390x` and `ppc64le`.
+
+You can also target a specific Python version by using `cp3<minor_version>_<os>_<arch>`.
+If multiple python versions are specified as target platforms, then select statements
+of the `lib` and `whl` targets will include usage of version aware toolchain config
+settings like `@rules_python//python/config_settings:is_python_3.y`.
+
+Special values: `host` (for generating deps for the host platform only) and
+`<prefix>_*` values. For example, `cp39_*`, `linux_*`, `cp39_linux_*`.
+
+NOTE: this is not for cross-compiling Python wheels but rather for parsing the `whl` METADATA correctly.
+""",
+ ),
+ "extra_pip_args": attr.string_list(
+ doc = """Extra arguments to pass on to pip. Must not contain spaces.
+
+Supports environment variables using the syntax `$VARNAME` or
+`${VARNAME}` (expanding to empty string if unset) or
+`${VARNAME:-default}` (expanding to default if the variable is unset
+or empty in the environment), if `"VARNAME"` is listed in the
+`envsubst` attribute. See also `envsubst`.
+""",
+ ),
+ "isolated": attr.bool(
+ doc = """\
+Whether or not to pass the [--isolated](https://pip.pypa.io/en/stable/cli/pip/#cmdoption-isolated) flag to
+the underlying pip command. Alternatively, the `RULES_PYTHON_PIP_ISOLATED` environment variable can be used
+to control this flag.
+""",
+ default = True,
+ ),
+ "pip_data_exclude": attr.string_list(
+ doc = "Additional data exclusion parameters to add to the pip packages BUILD file.",
+ ),
+ "python_interpreter": attr.string(
+ doc = """\
+The python interpreter to use. This can either be an absolute path or the name
+of a binary found on the host's `PATH` environment variable. If no value is set
+`python3` is defaulted for Unix systems and `python.exe` for Windows.
+""",
+ # NOTE: This attribute should not have a default. See `_get_python_interpreter_attr`
+ # default = "python3"
+ ),
+ "python_interpreter_target": attr.label(
+ allow_single_file = True,
+ doc = """
+If you are using a custom python interpreter built by another repository rule,
+use this attribute to specify its BUILD target. This allows pip_repository to invoke
+pip using the same interpreter as your toolchain. If set, takes precedence over
+python_interpreter. An example value: "@python3_x86_64-unknown-linux-gnu//:python".
+""",
+ ),
+ "quiet": attr.bool(
+ default = True,
+ doc = """\
+If True, suppress printing stdout and stderr output to the terminal.
+
+If you would like to get more diagnostic output, please use:
+
+ RULES_PYTHON_REPO_DEBUG=1
+
+or
+
+ RULES_PYTHON_REPO_DEBUG_VERBOSITY=<INFO|DEBUG|TRACE>
+""",
+ ),
+ # 600 is documented as default here: https://docs.bazel.build/versions/master/skylark/lib/repository_ctx.html#execute
+ "timeout": attr.int(
+ default = 600,
+ doc = "Timeout (in seconds) on the rule's execution duration.",
+ ),
+}
+
+def use_isolated(ctx, attr):
+ """Determine whether or not to pass the pip `--isolated` flag to the pip invocation.
+
+ Args:
+ ctx: repository or module context
+ attr: attributes for the repo rule or tag extension
+
+ Returns:
+ True if --isolated should be passed
+ """
+ use_isolated = attr.isolated
+
+ # The environment variable will take precedence over the attribute
+ isolated_env = ctx.os.environ.get("RULES_PYTHON_PIP_ISOLATED", None)
+ if isolated_env != None:
+ if isolated_env.lower() in ("0", "false"):
+ use_isolated = False
+ else:
+ use_isolated = True
+
+ return use_isolated
diff --git a/python/private/pypi/bzlmod.bzl b/python/private/pypi/bzlmod.bzl
new file mode 100644
index 0000000..e98208a
--- /dev/null
+++ b/python/private/pypi/bzlmod.bzl
@@ -0,0 +1,818 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"pip module extension for use with bzlmod"
+
+load("@bazel_features//:features.bzl", "bazel_features")
+load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS")
+load("//python/private:auth.bzl", "AUTH_ATTRS")
+load("//python/private:normalize_name.bzl", "normalize_name")
+load("//python/private:repo_utils.bzl", "repo_utils")
+load("//python/private:version_label.bzl", "version_label")
+load(":attrs.bzl", "use_isolated")
+load(":hub_repository.bzl", "hub_repository")
+load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
+load(":parse_whl_name.bzl", "parse_whl_name")
+load(":pip_repository_attrs.bzl", "ATTRS")
+load(":render_pkg_aliases.bzl", "whl_alias")
+load(":simpleapi_download.bzl", "simpleapi_download")
+load(":whl_library.bzl", "whl_library")
+load(":whl_repo_name.bzl", "whl_repo_name")
+
+def _parse_version(version):
+ major, _, version = version.partition(".")
+ minor, _, version = version.partition(".")
+ patch, _, version = version.partition(".")
+ build, _, version = version.partition(".")
+
+ return struct(
+ # use semver vocabulary here
+ major = major,
+ minor = minor,
+ patch = patch, # this is called `micro` in the Python interpreter versioning scheme
+ build = build,
+ )
+
+def _major_minor_version(version):
+ version = _parse_version(version)
+ return "{}.{}".format(version.major, version.minor)
+
+def _whl_mods_impl(mctx):
+ """Implementation of the pip.whl_mods tag class.
+
+ This creates the JSON files used to modify the creation of different wheels.
+"""
+ whl_mods_dict = {}
+ for mod in mctx.modules:
+ for whl_mod_attr in mod.tags.whl_mods:
+ if whl_mod_attr.hub_name not in whl_mods_dict.keys():
+ whl_mods_dict[whl_mod_attr.hub_name] = {whl_mod_attr.whl_name: whl_mod_attr}
+ elif whl_mod_attr.whl_name in whl_mods_dict[whl_mod_attr.hub_name].keys():
+ # We cannot have the same wheel name in the same hub, as we
+ # will create the same JSON file name.
+ fail("""\
+Found same whl_name '{}' in the same hub '{}', please use a different hub_name.""".format(
+ whl_mod_attr.whl_name,
+ whl_mod_attr.hub_name,
+ ))
+ else:
+ whl_mods_dict[whl_mod_attr.hub_name][whl_mod_attr.whl_name] = whl_mod_attr
+
+ for hub_name, whl_maps in whl_mods_dict.items():
+ whl_mods = {}
+
+ # create a struct that we can pass to the _whl_mods_repo rule
+ # to create the different JSON files.
+ for whl_name, mods in whl_maps.items():
+ build_content = mods.additive_build_content
+ if mods.additive_build_content_file != None and mods.additive_build_content != "":
+ fail("""\
+You cannot use both the additive_build_content and additive_build_content_file arguments at the same time.
+""")
+ elif mods.additive_build_content_file != None:
+ build_content = mctx.read(mods.additive_build_content_file)
+
+ whl_mods[whl_name] = json.encode(struct(
+ additive_build_content = build_content,
+ copy_files = mods.copy_files,
+ copy_executables = mods.copy_executables,
+ data = mods.data,
+ data_exclude_glob = mods.data_exclude_glob,
+ srcs_exclude_glob = mods.srcs_exclude_glob,
+ ))
+
+ _whl_mods_repo(
+ name = hub_name,
+ whl_mods = whl_mods,
+ )
+
+def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, simpleapi_cache):
+ logger = repo_utils.logger(module_ctx)
+ python_interpreter_target = pip_attr.python_interpreter_target
+ is_hub_reproducible = True
+
+ # if we do not have the python_interpreter set in the attributes
+ # we programmatically find it.
+ hub_name = pip_attr.hub_name
+ if python_interpreter_target == None and not pip_attr.python_interpreter:
+ python_name = "python_{}_host".format(
+ pip_attr.python_version.replace(".", "_"),
+ )
+ if python_name not in INTERPRETER_LABELS:
+ fail((
+ "Unable to find interpreter for pip hub '{hub_name}' for " +
+ "python_version={version}: Make sure a corresponding " +
+ '`python.toolchain(python_version="{version}")` call exists.' +
+ "Expected to find {python_name} among registered versions:\n {labels}"
+ ).format(
+ hub_name = hub_name,
+ version = pip_attr.python_version,
+ python_name = python_name,
+ labels = " \n".join(INTERPRETER_LABELS),
+ ))
+ python_interpreter_target = INTERPRETER_LABELS[python_name]
+
+ pip_name = "{}_{}".format(
+ hub_name,
+ version_label(pip_attr.python_version),
+ )
+ major_minor = _major_minor_version(pip_attr.python_version)
+
+ if hub_name not in whl_map:
+ whl_map[hub_name] = {}
+
+ whl_modifications = {}
+ if pip_attr.whl_modifications != None:
+ for mod, whl_name in pip_attr.whl_modifications.items():
+ whl_modifications[whl_name] = mod
+
+ if pip_attr.experimental_requirement_cycles:
+ requirement_cycles = {
+ name: [normalize_name(whl_name) for whl_name in whls]
+ for name, whls in pip_attr.experimental_requirement_cycles.items()
+ }
+
+ whl_group_mapping = {
+ whl_name: group_name
+ for group_name, group_whls in requirement_cycles.items()
+ for whl_name in group_whls
+ }
+
+ # TODO @aignas 2024-04-05: how do we support different requirement
+ # cycles for different abis/oses? For now we will need the users to
+ # assume the same groups across all versions/platforms until we start
+ # using an alternative cycle resolution strategy.
+ group_map[hub_name] = pip_attr.experimental_requirement_cycles
+ else:
+ whl_group_mapping = {}
+ requirement_cycles = {}
+
+ # Create a new wheel library for each of the different whls
+
+ get_index_urls = None
+ if pip_attr.experimental_index_url:
+ if pip_attr.download_only:
+ fail("Currently unsupported to use `download_only` and `experimental_index_url`")
+
+ get_index_urls = lambda ctx, distributions: simpleapi_download(
+ ctx,
+ attr = struct(
+ index_url = pip_attr.experimental_index_url,
+ extra_index_urls = pip_attr.experimental_extra_index_urls or [],
+ index_url_overrides = pip_attr.experimental_index_url_overrides or {},
+ sources = distributions,
+ envsubst = pip_attr.envsubst,
+ # Auth related info
+ netrc = pip_attr.netrc,
+ auth_patterns = pip_attr.auth_patterns,
+ ),
+ cache = simpleapi_cache,
+ parallel_download = pip_attr.parallel_download,
+ )
+
+ requirements_by_platform = parse_requirements(
+ module_ctx,
+ requirements_by_platform = pip_attr.requirements_by_platform,
+ requirements_linux = pip_attr.requirements_linux,
+ requirements_lock = pip_attr.requirements_lock,
+ requirements_osx = pip_attr.requirements_darwin,
+ requirements_windows = pip_attr.requirements_windows,
+ extra_pip_args = pip_attr.extra_pip_args,
+ get_index_urls = get_index_urls,
+ python_version = major_minor,
+ logger = logger,
+ )
+
+ repository_platform = host_platform(module_ctx.os)
+ for whl_name, requirements in requirements_by_platform.items():
+ # We are not using the "sanitized name" because the user
+ # would need to guess what name we modified the whl name
+ # to.
+ annotation = whl_modifications.get(whl_name)
+ whl_name = normalize_name(whl_name)
+
+ group_name = whl_group_mapping.get(whl_name)
+ group_deps = requirement_cycles.get(group_name, [])
+
+ # Construct args separately so that the lock file can be smaller and does not include unused
+ # attrs.
+ whl_library_args = dict(
+ repo = pip_name,
+ dep_template = "@{}//{{name}}:{{target}}".format(hub_name),
+ )
+ maybe_args = dict(
+ # The following values are safe to omit if they have false like values
+ annotation = annotation,
+ download_only = pip_attr.download_only,
+ enable_implicit_namespace_pkgs = pip_attr.enable_implicit_namespace_pkgs,
+ environment = pip_attr.environment,
+ envsubst = pip_attr.envsubst,
+ experimental_target_platforms = pip_attr.experimental_target_platforms,
+ group_deps = group_deps,
+ group_name = group_name,
+ pip_data_exclude = pip_attr.pip_data_exclude,
+ python_interpreter = pip_attr.python_interpreter,
+ python_interpreter_target = python_interpreter_target,
+ whl_patches = {
+ p: json.encode(args)
+ for p, args in whl_overrides.get(whl_name, {}).items()
+ },
+ )
+ whl_library_args.update({k: v for k, v in maybe_args.items() if v})
+ maybe_args_with_default = dict(
+ # The following values have defaults next to them
+ isolated = (use_isolated(module_ctx, pip_attr), True),
+ quiet = (pip_attr.quiet, True),
+ timeout = (pip_attr.timeout, 600),
+ )
+ whl_library_args.update({
+ k: v
+ for k, (v, default) in maybe_args_with_default.items()
+ if v != default
+ })
+
+ if get_index_urls:
+ # TODO @aignas 2024-05-26: move to a separate function
+ found_something = False
+ for requirement in requirements:
+ for distribution in requirement.whls + [requirement.sdist]:
+ if not distribution:
+ # sdist may be None
+ continue
+
+ found_something = True
+ is_hub_reproducible = False
+
+ if pip_attr.netrc:
+ whl_library_args["netrc"] = pip_attr.netrc
+ if pip_attr.auth_patterns:
+ whl_library_args["auth_patterns"] = pip_attr.auth_patterns
+
+ # pip is not used to download wheels and the python `whl_library` helpers are only extracting things
+ whl_library_args.pop("extra_pip_args", None)
+
+ # This is no-op because pip is not used to download the wheel.
+ whl_library_args.pop("download_only", None)
+
+ repo_name = whl_repo_name(pip_name, distribution.filename, distribution.sha256)
+ whl_library_args["requirement"] = requirement.srcs.requirement
+ whl_library_args["urls"] = [distribution.url]
+ whl_library_args["sha256"] = distribution.sha256
+ whl_library_args["filename"] = distribution.filename
+ whl_library_args["experimental_target_platforms"] = requirement.target_platforms
+
+ # Pure python wheels or sdists may need to have a platform here
+ target_platforms = None
+ if distribution.filename.endswith("-any.whl") or not distribution.filename.endswith(".whl"):
+ if len(requirements) > 1:
+ target_platforms = requirement.target_platforms
+
+ whl_library(name = repo_name, **dict(sorted(whl_library_args.items())))
+
+ whl_map[hub_name].setdefault(whl_name, []).append(
+ whl_alias(
+ repo = repo_name,
+ version = major_minor,
+ filename = distribution.filename,
+ target_platforms = target_platforms,
+ ),
+ )
+
+ if found_something:
+ continue
+
+ requirement = select_requirement(
+ requirements,
+ platform = repository_platform,
+ )
+ if not requirement:
+ # Sometimes the package is not present for host platform if there
+ # are whls specified only in particular requirements files, in that
+ # case just continue, however, if the download_only flag is set up,
+ # then the user can also specify the target platform of the wheel
+ # packages they want to download, in that case there will be always
+ # a requirement here, so we will not be in this code branch.
+ continue
+ elif get_index_urls:
+ logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line))
+
+ whl_library_args["requirement"] = requirement.requirement_line
+ if requirement.extra_pip_args:
+ whl_library_args["extra_pip_args"] = requirement.extra_pip_args
+
+ # We sort so that the lock-file remains the same no matter the order of how the
+ # args are manipulated in the code going before.
+ repo_name = "{}_{}".format(pip_name, whl_name)
+ whl_library(name = repo_name, **dict(sorted(whl_library_args.items())))
+ whl_map[hub_name].setdefault(whl_name, []).append(
+ whl_alias(
+ repo = repo_name,
+ version = major_minor,
+ ),
+ )
+
+ return is_hub_reproducible
+
+def _pip_impl(module_ctx):
+ """Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories.
+
+ This implementation iterates through all of the `pip.parse` calls and creates
+ different pip hub repositories based on the "hub_name". Each of the
+ pip calls create spoke repos that uses a specific Python interpreter.
+
+ In a MODULES.bazel file we have:
+
+ pip.parse(
+ hub_name = "pip",
+ python_version = 3.9,
+ requirements_lock = "//:requirements_lock_3_9.txt",
+ requirements_windows = "//:requirements_windows_3_9.txt",
+ )
+ pip.parse(
+ hub_name = "pip",
+ python_version = 3.10,
+ requirements_lock = "//:requirements_lock_3_10.txt",
+ requirements_windows = "//:requirements_windows_3_10.txt",
+ )
+
+ For instance, we have a hub with the name of "pip".
+ A repository named the following is created. It is actually called last when
+ all of the pip spokes are collected.
+
+ - @@rules_python~override~pip~pip
+
+ As shown in the example code above we have the following.
+ Two different pip.parse statements exist in MODULE.bazel provide the hub_name "pip".
+ These definitions create two different pip spoke repositories that are
+ related to the hub "pip".
+ One spoke uses Python 3.9 and the other uses Python 3.10. This code automatically
+ determines the Python version and the interpreter.
+ Both of these pip spokes contain requirements files that includes websocket
+ and its dependencies.
+
+ We also need repositories for the wheels that the different pip spokes contain.
+ For each Python version a different wheel repository is created. In our example
+ each pip spoke had a requirements file that contained websockets. We
+ then create two different wheel repositories that are named the following.
+
+ - @@rules_python~override~pip~pip_39_websockets
+ - @@rules_python~override~pip~pip_310_websockets
+
+ And if the wheel has any other dependencies subsequent wheels are created in the same fashion.
+
+ The hub repository has aliases for `pkg`, `data`, etc, which have a select that resolves to
+ a spoke repository depending on the Python version.
+
+ Also we may have more than one hub as defined in a MODULES.bazel file. So we could have multiple
+ hubs pointing to various different pip spokes.
+
+ Some other business rules notes. A hub can only have one spoke per Python version. We cannot
+ have a hub named "pip" that has two spokes that use the Python 3.9 interpreter. Second
+ we cannot have the same hub name used in sub-modules. The hub name has to be globally
+ unique.
+
+ This implementation also handles the creation of whl_modification JSON files that are used
+ during the creation of wheel libraries. These JSON files used via the annotations argument
+ when calling wheel_installer.py.
+
+ Args:
+ module_ctx: module contents
+ """
+
+ # Build all of the wheel modifications if the tag class is called.
+ _whl_mods_impl(module_ctx)
+
+ _overriden_whl_set = {}
+ whl_overrides = {}
+
+ for module in module_ctx.modules:
+ for attr in module.tags.override:
+ if not module.is_root:
+ fail("overrides are only supported in root modules")
+
+ if not attr.file.endswith(".whl"):
+ fail("Only whl overrides are supported at this time")
+
+ whl_name = normalize_name(parse_whl_name(attr.file).distribution)
+
+ if attr.file in _overriden_whl_set:
+ fail("Duplicate module overrides for '{}'".format(attr.file))
+ _overriden_whl_set[attr.file] = None
+
+ for patch in attr.patches:
+ if whl_name not in whl_overrides:
+ whl_overrides[whl_name] = {}
+
+ if patch not in whl_overrides[whl_name]:
+ whl_overrides[whl_name][patch] = struct(
+ patch_strip = attr.patch_strip,
+ whls = [],
+ )
+
+ whl_overrides[whl_name][patch].whls.append(attr.file)
+
+ # Used to track all the different pip hubs and the spoke pip Python
+ # versions.
+ pip_hub_map = {}
+
+ # Keeps track of all the hub's whl repos across the different versions.
+ # dict[hub, dict[whl, dict[version, str pip]]]
+ # Where hub, whl, and pip are the repo names
+ hub_whl_map = {}
+ hub_group_map = {}
+
+ simpleapi_cache = {}
+ is_extension_reproducible = True
+
+ for mod in module_ctx.modules:
+ for pip_attr in mod.tags.parse:
+ hub_name = pip_attr.hub_name
+ if hub_name not in pip_hub_map:
+ pip_hub_map[pip_attr.hub_name] = struct(
+ module_name = mod.name,
+ python_versions = [pip_attr.python_version],
+ )
+ elif pip_hub_map[hub_name].module_name != mod.name:
+ # We cannot have two hubs with the same name in different
+ # modules.
+ fail((
+ "Duplicate cross-module pip hub named '{hub}': pip hub " +
+ "names must be unique across modules. First defined " +
+ "by module '{first_module}', second attempted by " +
+ "module '{second_module}'"
+ ).format(
+ hub = hub_name,
+ first_module = pip_hub_map[hub_name].module_name,
+ second_module = mod.name,
+ ))
+
+ elif pip_attr.python_version in pip_hub_map[hub_name].python_versions:
+ fail((
+ "Duplicate pip python version '{version}' for hub " +
+ "'{hub}' in module '{module}': the Python versions " +
+ "used for a hub must be unique"
+ ).format(
+ hub = hub_name,
+ module = mod.name,
+ version = pip_attr.python_version,
+ ))
+ else:
+ pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version)
+
+ is_hub_reproducible = _create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache)
+ is_extension_reproducible = is_extension_reproducible and is_hub_reproducible
+
+ for hub_name, whl_map in hub_whl_map.items():
+ hub_repository(
+ name = hub_name,
+ repo_name = hub_name,
+ whl_map = {
+ key: json.encode(value)
+ for key, value in whl_map.items()
+ },
+ default_version = _major_minor_version(DEFAULT_PYTHON_VERSION),
+ groups = hub_group_map.get(hub_name),
+ )
+
+ if bazel_features.external_deps.extension_metadata_has_reproducible:
+ # If we are not using the `experimental_index_url feature, the extension is fully
+ # deterministic and we don't need to create a lock entry for it.
+ #
+ # In order to be able to dogfood the `experimental_index_url` feature before it gets
+ # stabilized, we have created the `_pip_non_reproducible` function, that will result
+ # in extra entries in the lock file.
+ return module_ctx.extension_metadata(reproducible = is_extension_reproducible)
+ else:
+ return None
+
+def _pip_non_reproducible(module_ctx):
+ _pip_impl(module_ctx)
+
+ # We default to calling the PyPI index and that will go into the
+ # MODULE.bazel.lock file, hence return nothing here.
+ return None
+
+def _pip_parse_ext_attrs(**kwargs):
+ """Get the attributes for the pip extension.
+
+ Args:
+ **kwargs: A kwarg for setting defaults for the specific attributes. The
+ key is expected to be the same as the attribute key.
+
+ Returns:
+ A dict of attributes.
+ """
+ attrs = dict({
+ "experimental_extra_index_urls": attr.string_list(
+ doc = """\
+The extra index URLs to use for downloading wheels using bazel downloader.
+Each value is going to be subject to `envsubst` substitutions if necessary.
+
+The indexes must support Simple API as described here:
+https://packaging.python.org/en/latest/specifications/simple-repository-api/
+
+This is equivalent to `--extra-index-urls` `pip` option.
+""",
+ default = [],
+ ),
+ "experimental_index_url": attr.string(
+ default = kwargs.get("experimental_index_url", ""),
+ doc = """\
+The index URL to use for downloading wheels using bazel downloader. This value is going
+to be subject to `envsubst` substitutions if necessary.
+
+The indexes must support Simple API as described here:
+https://packaging.python.org/en/latest/specifications/simple-repository-api/
+
+In the future this could be defaulted to `https://pypi.org` when this feature becomes
+stable.
+
+This is equivalent to `--index-url` `pip` option.
+""",
+ ),
+ "experimental_index_url_overrides": attr.string_dict(
+ doc = """\
+The index URL overrides for each package to use for downloading wheels using
+bazel downloader. This value is going to be subject to `envsubst` substitutions
+if necessary.
+
+The key is the package name (will be normalized before usage) and the value is the
+index URL.
+
+This design pattern has been chosen in order to be fully deterministic about which
+packages come from which source. We want to avoid issues similar to what happened in
+https://pytorch.org/blog/compromised-nightly-dependency/.
+
+The indexes must support Simple API as described here:
+https://packaging.python.org/en/latest/specifications/simple-repository-api/
+""",
+ ),
+ "hub_name": attr.string(
+ mandatory = True,
+ doc = """
+The name of the repo pip dependencies will be accessible from.
+
+This name must be unique between modules; unless your module is guaranteed to
+always be the root module, it's highly recommended to include your module name
+in the hub name. Repo mapping, `use_repo(..., pip="my_modules_pip_deps")`, can
+be used for shorter local names within your module.
+
+Within a module, the same `hub_name` can be specified to group different Python
+versions of pip dependencies under one repository name. This allows using a
+Python version-agnostic name when referring to pip dependencies; the
+correct version will be automatically selected.
+
+Typically, a module will only have a single hub of pip dependencies, but this
+is not required. Each hub is a separate resolution of pip dependencies. This
+means if different programs need different versions of some library, separate
+hubs can be created, and each program can use its respective hub's targets.
+Targets from different hubs should not be used together.
+""",
+ ),
+ "parallel_download": attr.bool(
+ doc = """\
+The flag allows to make use of parallel downloading feature in bazel 7.1 and above
+when the bazel downloader is used. This is by default enabled as it improves the
+performance by a lot, but in case the queries to the simple API are very expensive
+or when debugging authentication issues one may want to disable this feature.
+
+NOTE, This will download (potentially duplicate) data for multiple packages if
+there is more than one index available, but in general this should be negligible
+because the simple API calls are very cheap and the user should not notice any
+extra overhead.
+
+If we are in synchronous mode, then we will use the first result that we
+find in case extra indexes are specified.
+""",
+ default = True,
+ ),
+ "python_version": attr.string(
+ mandatory = True,
+ doc = """
+The Python version the dependencies are targetting, in Major.Minor format
+(e.g., "3.11") or patch level granularity (e.g. "3.11.1").
+
+If an interpreter isn't explicitly provided (using `python_interpreter` or
+`python_interpreter_target`), then the version specified here must have
+a corresponding `python.toolchain()` configured.
+""",
+ ),
+ "whl_modifications": attr.label_keyed_string_dict(
+ mandatory = False,
+ doc = """\
+A dict of labels to wheel names that is typically generated by the whl_modifications.
+The labels are JSON config files describing the modifications.
+""",
+ ),
+ }, **ATTRS)
+ attrs.update(AUTH_ATTRS)
+
+ return attrs
+
+def _whl_mod_attrs():
+ attrs = {
+ "additive_build_content": attr.string(
+ doc = "(str, optional): Raw text to add to the generated `BUILD` file of a package.",
+ ),
+ "additive_build_content_file": attr.label(
+ doc = """\
+(label, optional): path to a BUILD file to add to the generated
+`BUILD` file of a package. You cannot use both additive_build_content and additive_build_content_file
+arguments at the same time.""",
+ ),
+ "copy_executables": attr.string_dict(
+ doc = """\
+(dict, optional): A mapping of `src` and `out` files for
+[@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as
+executable.""",
+ ),
+ "copy_files": attr.string_dict(
+ doc = """\
+(dict, optional): A mapping of `src` and `out` files for
+[@bazel_skylib//rules:copy_file.bzl][cf]""",
+ ),
+ "data": attr.string_list(
+ doc = """\
+(list, optional): A list of labels to add as `data` dependencies to
+the generated `py_library` target.""",
+ ),
+ "data_exclude_glob": attr.string_list(
+ doc = """\
+(list, optional): A list of exclude glob patterns to add as `data` to
+the generated `py_library` target.""",
+ ),
+ "hub_name": attr.string(
+ doc = """\
+Name of the whl modification, hub we use this name to set the modifications for
+pip.parse. If you have different pip hubs you can use a different name,
+otherwise it is best practice to just use one.
+
+You cannot have the same `hub_name` in different modules. You can reuse the same
+name in the same module for different wheels that you put in the same hub, but you
+cannot have a child module that uses the same `hub_name`.
+""",
+ mandatory = True,
+ ),
+ "srcs_exclude_glob": attr.string_list(
+ doc = """\
+(list, optional): A list of labels to add as `srcs` to the generated
+`py_library` target.""",
+ ),
+ "whl_name": attr.string(
+ doc = "The whl name that the modifications are used for.",
+ mandatory = True,
+ ),
+ }
+ return attrs
+
+# NOTE: the naming of 'override' is taken from the bzlmod native
+# 'archive_override', 'git_override' bzlmod functions.
+_override_tag = tag_class(
+ attrs = {
+ "file": attr.string(
+ doc = """\
+The Python distribution file name which needs to be patched. This will be
+applied to all repositories that setup this distribution via the pip.parse tag
+class.""",
+ mandatory = True,
+ ),
+ "patch_strip": attr.int(
+ default = 0,
+ doc = """\
+The number of leading path segments to be stripped from the file name in the
+patches.""",
+ ),
+ "patches": attr.label_list(
+ doc = """\
+A list of patches to apply to the repository *after* 'whl_library' is extracted
+and BUILD.bazel file is generated.""",
+ mandatory = True,
+ ),
+ },
+ doc = """\
+Apply any overrides (e.g. patches) to a given Python distribution defined by
+other tags in this extension.""",
+)
+
+pypi = module_extension(
+ doc = """\
+This extension is used to make dependencies from pip available.
+
+pip.parse:
+To use, call `pip.parse()` and specify `hub_name` and your requirements file.
+Dependencies will be downloaded and made available in a repo named after the
+`hub_name` argument.
+
+Each `pip.parse()` call configures a particular Python version. Multiple calls
+can be made to configure different Python versions, and will be grouped by
+the `hub_name` argument. This allows the same logical name, e.g. `@pip//numpy`
+to automatically resolve to different, Python version-specific, libraries.
+
+pip.whl_mods:
+This tag class is used to help create JSON files to describe modifications to
+the BUILD files for wheels.
+""",
+ implementation = _pip_impl,
+ tag_classes = {
+ "override": _override_tag,
+ "parse": tag_class(
+ attrs = _pip_parse_ext_attrs(),
+ doc = """\
+This tag class is used to create a pip hub and all of the spokes that are part of that hub.
+This tag class reuses most of the pip attributes that are found in
+@rules_python//python/pip_install:pip_repository.bzl.
+The exception is it does not use the arg 'repo_prefix'. We set the repository
+prefix for the user and the alias arg is always True in bzlmod.
+""",
+ ),
+ "whl_mods": tag_class(
+ attrs = _whl_mod_attrs(),
+ doc = """\
+This tag class is used to create JSON file that are used when calling wheel_builder.py. These
+JSON files contain instructions on how to modify a wheel's project. Each of the attributes
+create different modifications based on the type of attribute. Previously to bzlmod these
+JSON files where referred to as annotations, and were renamed to whl_modifications in this
+extension.
+""",
+ ),
+ },
+)
+
+pypi_internal = module_extension(
+ doc = """\
+This extension is used to make dependencies from pypi available.
+
+For now this is intended to be used internally so that usage of the `pip`
+extension in `rules_python` does not affect the evaluations of the extension
+for the consumers.
+
+pip.parse:
+To use, call `pip.parse()` and specify `hub_name` and your requirements file.
+Dependencies will be downloaded and made available in a repo named after the
+`hub_name` argument.
+
+Each `pip.parse()` call configures a particular Python version. Multiple calls
+can be made to configure different Python versions, and will be grouped by
+the `hub_name` argument. This allows the same logical name, e.g. `@pypi//numpy`
+to automatically resolve to different, Python version-specific, libraries.
+
+pip.whl_mods:
+This tag class is used to help create JSON files to describe modifications to
+the BUILD files for wheels.
+""",
+ implementation = _pip_non_reproducible,
+ tag_classes = {
+ "override": _override_tag,
+ "parse": tag_class(
+ attrs = _pip_parse_ext_attrs(
+ experimental_index_url = "https://pypi.org/simple",
+ ),
+ doc = """\
+This tag class is used to create a pypi hub and all of the spokes that are part of that hub.
+This tag class reuses most of the pypi attributes that are found in
+@rules_python//python/pip_install:pip_repository.bzl.
+The exception is it does not use the arg 'repo_prefix'. We set the repository
+prefix for the user and the alias arg is always True in bzlmod.
+""",
+ ),
+ "whl_mods": tag_class(
+ attrs = _whl_mod_attrs(),
+ doc = """\
+This tag class is used to create JSON file that are used when calling wheel_builder.py. These
+JSON files contain instructions on how to modify a wheel's project. Each of the attributes
+create different modifications based on the type of attribute. Previously to bzlmod these
+JSON files where referred to as annotations, and were renamed to whl_modifications in this
+extension.
+""",
+ ),
+ },
+)
+
+def _whl_mods_repo_impl(rctx):
+ rctx.file("BUILD.bazel", "")
+ for whl_name, mods in rctx.attr.whl_mods.items():
+ rctx.file("{}.json".format(whl_name), mods)
+
+_whl_mods_repo = repository_rule(
+ doc = """\
+This rule creates json files based on the whl_mods attribute.
+""",
+ implementation = _whl_mods_repo_impl,
+ attrs = {
+ "whl_mods": attr.string_dict(
+ mandatory = True,
+ doc = "JSON endcoded string that is provided to wheel_builder.py",
+ ),
+ },
+)
diff --git a/python/private/pip_config_settings.bzl b/python/private/pypi/config_settings.bzl
similarity index 98%
rename from python/private/pip_config_settings.bzl
rename to python/private/pypi/config_settings.bzl
index b13b39b..9741217 100644
--- a/python/private/pip_config_settings.bzl
+++ b/python/private/pypi/config_settings.bzl
@@ -39,12 +39,7 @@
order to ensure that the matching fails if the user requests for `musl` and we don't have it or vice versa.
"""
-load(
- ":pip_flags.bzl",
- "INTERNAL_FLAGS",
- "UniversalWhlFlag",
- "WhlLibcFlag",
-)
+load(":flags.bzl", "INTERNAL_FLAGS", "UniversalWhlFlag", "WhlLibcFlag")
FLAGS = struct(
**{
@@ -73,7 +68,7 @@
}
)
-def pip_config_settings(
+def config_settings(
*,
python_versions = [],
glibc_versions = [],
diff --git a/python/private/pip_flags.bzl b/python/private/pypi/flags.bzl
similarity index 97%
rename from python/private/pip_flags.bzl
rename to python/private/pypi/flags.bzl
index 1d271c7..d834be8 100644
--- a/python/private/pip_flags.bzl
+++ b/python/private/pypi/flags.bzl
@@ -18,7 +18,7 @@
unnecessary files when all that are needed are flag definitions.
"""
-load(":enum.bzl", "enum")
+load("//python/private:enum.bzl", "enum")
# Determines if we should use whls for third party
#
diff --git a/python/pip_install/private/generate_group_library_build_bazel.bzl b/python/private/pypi/generate_group_library_build_bazel.bzl
similarity index 98%
rename from python/pip_install/private/generate_group_library_build_bazel.bzl
rename to python/private/pypi/generate_group_library_build_bazel.bzl
index 5fa93e2..54da066 100644
--- a/python/pip_install/private/generate_group_library_build_bazel.bzl
+++ b/python/private/pypi/generate_group_library_build_bazel.bzl
@@ -14,15 +14,15 @@
"""Generate the BUILD.bazel contents for a repo defined by a group_library."""
+load("//python/private:normalize_name.bzl", "normalize_name")
+load("//python/private:text_util.bzl", "render")
load(
- "//python/private:labels.bzl",
+ ":labels.bzl",
"PY_LIBRARY_IMPL_LABEL",
"PY_LIBRARY_PUBLIC_LABEL",
"WHEEL_FILE_IMPL_LABEL",
"WHEEL_FILE_PUBLIC_LABEL",
)
-load("//python/private:normalize_name.bzl", "normalize_name")
-load("//python/private:text_util.bzl", "render")
_PRELUDE = """\
load("@rules_python//python:defs.bzl", "py_library")
diff --git a/python/pip_install/private/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl
similarity index 99%
rename from python/pip_install/private/generate_whl_library_build_bazel.bzl
rename to python/private/pypi/generate_whl_library_build_bazel.bzl
index f3ddd3b..d25f73a 100644
--- a/python/pip_install/private/generate_whl_library_build_bazel.bzl
+++ b/python/private/pypi/generate_whl_library_build_bazel.bzl
@@ -14,8 +14,10 @@
"""Generate the BUILD.bazel contents for a repo defined by a whl_library."""
+load("//python/private:normalize_name.bzl", "normalize_name")
+load("//python/private:text_util.bzl", "render")
load(
- "//python/private:labels.bzl",
+ ":labels.bzl",
"DATA_LABEL",
"DIST_INFO_LABEL",
"PY_LIBRARY_IMPL_LABEL",
@@ -24,8 +26,6 @@
"WHEEL_FILE_IMPL_LABEL",
"WHEEL_FILE_PUBLIC_LABEL",
)
-load("//python/private:normalize_name.bzl", "normalize_name")
-load("//python/private:text_util.bzl", "render")
_COPY_FILE_TEMPLATE = """\
copy_file(
diff --git a/python/private/pypi/group_library.bzl b/python/private/pypi/group_library.bzl
new file mode 100644
index 0000000..ff800e2
--- /dev/null
+++ b/python/private/pypi/group_library.bzl
@@ -0,0 +1,40 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""group_library implementation for WORKSPACE setups."""
+
+load(":generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel")
+
+def _group_library_impl(rctx):
+ build_file_contents = generate_group_library_build_bazel(
+ repo_prefix = rctx.attr.repo_prefix,
+ groups = rctx.attr.groups,
+ )
+ rctx.file("BUILD.bazel", build_file_contents)
+
+group_library = repository_rule(
+ attrs = {
+ "groups": attr.string_list_dict(
+ doc = "A mapping of group names to requirements within that group.",
+ ),
+ "repo_prefix": attr.string(
+ doc = "Prefix used for the whl_library created components of each group",
+ ),
+ },
+ implementation = _group_library_impl,
+ doc = """
+Create a package containing only wrapper py_library and whl_library rules for implementing dependency groups.
+This is an implementation detail of dependency groups and should not be used alone.
+ """,
+)
diff --git a/python/private/bzlmod/pip_repository.bzl b/python/private/pypi/hub_repository.bzl
similarity index 78%
rename from python/private/bzlmod/pip_repository.bzl
rename to python/private/pypi/hub_repository.bzl
index d42eb8b..5e209d6 100644
--- a/python/private/bzlmod/pip_repository.bzl
+++ b/python/private/pypi/hub_repository.bzl
@@ -14,12 +14,12 @@
""
+load("//python/private:text_util.bzl", "render")
load(
- "//python/private:render_pkg_aliases.bzl",
+ ":render_pkg_aliases.bzl",
"render_multiplatform_pkg_aliases",
"whl_alias",
)
-load("//python/private:text_util.bzl", "render")
_BUILD_FILE_CONTENTS = """\
package(default_visibility = ["//visibility:public"])
@@ -28,7 +28,7 @@
exports_files(["requirements.bzl"])
"""
-def _pip_repository_impl(rctx):
+def _impl(rctx):
bzl_packages = rctx.attr.whl_map.keys()
aliases = render_multiplatform_pkg_aliases(
aliases = {
@@ -66,35 +66,33 @@
"%%NAME%%": rctx.attr.repo_name,
})
-pip_repository_attrs = {
- "default_version": attr.string(
- mandatory = True,
- doc = """\
+hub_repository = repository_rule(
+ attrs = {
+ "default_version": attr.string(
+ mandatory = True,
+ doc = """\
This is the default python version in the format of X.Y. This should match
what is setup by the 'python' extension using the 'is_default = True'
setting.""",
- ),
- "groups": attr.string_list_dict(
- mandatory = False,
- ),
- "repo_name": attr.string(
- mandatory = True,
- doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.",
- ),
- "whl_map": attr.string_dict(
- mandatory = True,
- doc = """\
+ ),
+ "groups": attr.string_list_dict(
+ mandatory = False,
+ ),
+ "repo_name": attr.string(
+ mandatory = True,
+ doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.",
+ ),
+ "whl_map": attr.string_dict(
+ mandatory = True,
+ doc = """\
The wheel map where values are json.encoded strings of the whl_map constructed
in the pip.parse tag class.
""",
- ),
- "_template": attr.label(
- default = ":requirements.bzl.tmpl",
- ),
-}
-
-pip_repository = repository_rule(
- attrs = pip_repository_attrs,
+ ),
+ "_template": attr.label(
+ default = ":requirements.bzl.tmpl.bzlmod",
+ ),
+ },
doc = """A rule for bzlmod mulitple pip repository creation. PRIVATE USE ONLY.""",
- implementation = _pip_repository_impl,
+ implementation = _impl,
)
diff --git a/python/private/pypi_index_sources.bzl b/python/private/pypi/index_sources.bzl
similarity index 97%
rename from python/private/pypi_index_sources.bzl
rename to python/private/pypi/index_sources.bzl
index 470a8c9..2166014 100644
--- a/python/private/pypi_index_sources.bzl
+++ b/python/private/pypi/index_sources.bzl
@@ -16,7 +16,7 @@
A file that houses private functions used in the `bzlmod` extension with the same name.
"""
-def get_simpleapi_sources(line):
+def index_sources(line):
"""Get PyPI sources from a requirements.txt line.
We interpret the spec described in
diff --git a/python/private/labels.bzl b/python/private/pypi/labels.bzl
similarity index 100%
rename from python/private/labels.bzl
rename to python/private/pypi/labels.bzl
diff --git a/python/private/pypi/package_annotation.bzl b/python/private/pypi/package_annotation.bzl
new file mode 100644
index 0000000..4a54703
--- /dev/null
+++ b/python/private/pypi/package_annotation.bzl
@@ -0,0 +1,49 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Package annotation API for WORKSPACE setups."""
+
+def package_annotation(
+ additive_build_content = None,
+ copy_files = {},
+ copy_executables = {},
+ data = [],
+ data_exclude_glob = [],
+ srcs_exclude_glob = []):
+ """Annotations to apply to the BUILD file content from package generated from a `pip_repository` rule.
+
+ [cf]: https://github.com/bazelbuild/bazel-skylib/blob/main/docs/copy_file_doc.md
+
+ Args:
+ additive_build_content (str, optional): Raw text to add to the generated `BUILD` file of a package.
+ copy_files (dict, optional): A mapping of `src` and `out` files for [@bazel_skylib//rules:copy_file.bzl][cf]
+ copy_executables (dict, optional): A mapping of `src` and `out` files for
+ [@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as
+ executable.
+ data (list, optional): A list of labels to add as `data` dependencies to the generated `py_library` target.
+ data_exclude_glob (list, optional): A list of exclude glob patterns to add as `data` to the generated
+ `py_library` target.
+ srcs_exclude_glob (list, optional): A list of labels to add as `srcs` to the generated `py_library` target.
+
+ Returns:
+ str: A json encoded string of the provided content.
+ """
+ return json.encode(struct(
+ additive_build_content = additive_build_content,
+ copy_files = copy_files,
+ copy_executables = copy_executables,
+ data = data,
+ data_exclude_glob = data_exclude_glob,
+ srcs_exclude_glob = srcs_exclude_glob,
+ ))
diff --git a/python/private/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl
similarity index 98%
rename from python/private/parse_requirements.bzl
rename to python/private/pypi/parse_requirements.bzl
index 21e132b..22a6f0a 100644
--- a/python/private/parse_requirements.bzl
+++ b/python/private/pypi/parse_requirements.bzl
@@ -26,9 +26,9 @@
behavior.
"""
-load("//python/pip_install:requirements_parser.bzl", "parse")
-load(":normalize_name.bzl", "normalize_name")
-load(":pypi_index_sources.bzl", "get_simpleapi_sources")
+load("//python/private:normalize_name.bzl", "normalize_name")
+load(":index_sources.bzl", "index_sources")
+load(":parse_requirements_txt.bzl", "parse_requirements_txt")
load(":whl_target_platforms.bzl", "select_whls", "whl_target_platforms")
# This includes the vendored _translate_cpu and _translate_os from
@@ -271,7 +271,7 @@
# Parse the requirements file directly in starlark to get the information
# needed for the whl_library declarations later.
- parse_result = parse(contents)
+ parse_result = parse_requirements_txt(contents)
# Replicate a surprising behavior that WORKSPACE builds allowed:
# Defining a repo with the same name multiple times, but only the last
@@ -317,7 +317,7 @@
(requirement_line, ",".join(extra_pip_args)),
struct(
distribution = distribution,
- srcs = get_simpleapi_sources(requirement_line),
+ srcs = index_sources(requirement_line),
requirement_line = requirement_line,
target_platforms = [],
extra_pip_args = extra_pip_args,
diff --git a/python/private/pypi/parse_requirements_txt.bzl b/python/private/pypi/parse_requirements_txt.bzl
new file mode 100644
index 0000000..6f51d03
--- /dev/null
+++ b/python/private/pypi/parse_requirements_txt.bzl
@@ -0,0 +1,133 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Pip requirements parser for Starlark."""
+
+_STATE = struct(
+ # Consume extraneous whitespace
+ ConsumeSpace = 0,
+ # Consume a comment
+ ConsumeComment = 1,
+ # Parse the name of a pip package
+ ParseDependency = 2,
+ # Parse a full requirement line
+ ParseRequirement = 3,
+ # Parse a pip option
+ ParseOption = 4,
+)
+
+EOF = {}
+
+def parse_requirements_txt(content):
+ """A simplistic (and incomplete) pip requirements lockfile parser.
+
+ Parses package names and their full requirement lines, as well pip
+ options.
+
+ Args:
+ content: lockfile content as a string
+
+ Returns:
+ Struct with fields `requirements` and `options`.
+
+ requirements: List of requirements, where each requirement is a 2-element
+ tuple containing the package name and the requirement line.
+ E.g., [(certifi', 'certifi==2021.10.8 --hash=sha256:7888...'), ...]
+
+ options: List of pip option lines
+ """
+ content = content.replace("\r", "")
+
+ result = struct(
+ requirements = [],
+ options = [],
+ )
+ state = _STATE.ConsumeSpace
+ buffer = ""
+
+ inputs = content.elems()[:]
+ inputs.append(EOF)
+
+ for input in inputs:
+ if state == _STATE.ConsumeSpace:
+ (state, buffer) = _handleConsumeSpace(input)
+ elif state == _STATE.ConsumeComment:
+ (state, buffer) = _handleConsumeComment(input, buffer, result)
+ elif state == _STATE.ParseDependency:
+ (state, buffer) = _handleParseDependency(input, buffer, result)
+ elif state == _STATE.ParseOption:
+ (state, buffer) = _handleParseOption(input, buffer, result)
+ elif state == _STATE.ParseRequirement:
+ (state, buffer) = _handleParseRequirement(input, buffer, result)
+ else:
+ fail("Unknown state %d" % state)
+
+ return result
+
+def _handleConsumeSpace(input):
+ if input == EOF:
+ return (_STATE.ConsumeSpace, "")
+ if input.isspace():
+ return (_STATE.ConsumeSpace, "")
+ elif input == "#":
+ return (_STATE.ConsumeComment, "")
+ elif input == "-":
+ return (_STATE.ParseOption, input)
+
+ return (_STATE.ParseDependency, input)
+
+def _handleConsumeComment(input, buffer, result):
+ if input == "\n":
+ if len(result.requirements) > 0 and len(result.requirements[-1]) == 1:
+ result.requirements[-1] = (result.requirements[-1][0], buffer.rstrip(" \n"))
+ return (_STATE.ConsumeSpace, "")
+ elif len(buffer) > 0:
+ result.options.append(buffer.rstrip(" \n"))
+ return (_STATE.ConsumeSpace, "")
+ return (_STATE.ConsumeSpace, "")
+ return (_STATE.ConsumeComment, buffer)
+
+def _handleParseDependency(input, buffer, result):
+ if input == EOF:
+ fail("Enountered unexpected end of file while parsing requirement")
+ elif input.isspace() or input in [">", "<", "~", "=", ";", "["]:
+ result.requirements.append((buffer,))
+ return (_STATE.ParseRequirement, buffer + input)
+
+ return (_STATE.ParseDependency, buffer + input)
+
+def _handleParseOption(input, buffer, result):
+ if input == "\n" and buffer.endswith("\\"):
+ return (_STATE.ParseOption, buffer[0:-1])
+ elif input == " ":
+ result.options.append(buffer.rstrip("\n"))
+ return (_STATE.ParseOption, "")
+ elif input == "\n" or input == EOF:
+ result.options.append(buffer.rstrip("\n"))
+ return (_STATE.ConsumeSpace, "")
+ elif input == "#" and (len(buffer) == 0 or buffer[-1].isspace()):
+ return (_STATE.ConsumeComment, buffer)
+
+ return (_STATE.ParseOption, buffer + input)
+
+def _handleParseRequirement(input, buffer, result):
+ if input == "\n" and buffer.endswith("\\"):
+ return (_STATE.ParseRequirement, buffer[0:-1])
+ elif input == "\n" or input == EOF:
+ result.requirements[-1] = (result.requirements[-1][0], buffer.rstrip(" \n"))
+ return (_STATE.ConsumeSpace, "")
+ elif input == "#" and (len(buffer) == 0 or buffer[-1].isspace()):
+ return (_STATE.ConsumeComment, buffer)
+
+ return (_STATE.ParseRequirement, buffer + input)
diff --git a/python/private/pypi/parse_simpleapi_html.bzl b/python/private/pypi/parse_simpleapi_html.bzl
new file mode 100644
index 0000000..f7cd032
--- /dev/null
+++ b/python/private/pypi/parse_simpleapi_html.bzl
@@ -0,0 +1,106 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Parse SimpleAPI HTML in Starlark.
+"""
+
+def parse_simpleapi_html(*, url, content):
+ """Get the package URLs for given shas by parsing the Simple API HTML.
+
+ Args:
+ url(str): The URL that the HTML content can be downloaded from.
+ content(str): The Simple API HTML content.
+
+ Returns:
+ A list of structs with:
+ * filename: The filename of the artifact.
+ * url: The URL to download the artifact.
+ * sha256: The sha256 of the artifact.
+ * metadata_sha256: The whl METADATA sha256 if we can download it. If this is
+ present, then the 'metadata_url' is also present. Defaults to "".
+ * metadata_url: The URL for the METADATA if we can download it. Defaults to "".
+ """
+ sdists = {}
+ whls = {}
+ lines = content.split("<a href=\"")
+
+ _, _, api_version = lines[0].partition("name=\"pypi:repository-version\" content=\"")
+ api_version, _, _ = api_version.partition("\"")
+
+ # We must assume the 1.0 if it is not present
+ # See https://packaging.python.org/en/latest/specifications/simple-repository-api/#clients
+ api_version = api_version or "1.0"
+ api_version = tuple([int(i) for i in api_version.split(".")])
+
+ if api_version >= (2, 0):
+ # We don't expect to have version 2.0 here, but have this check in place just in case.
+ # https://packaging.python.org/en/latest/specifications/simple-repository-api/#versioning-pypi-s-simple-api
+ fail("Unsupported API version: {}".format(api_version))
+
+ for line in lines[1:]:
+ dist_url, _, tail = line.partition("#sha256=")
+ sha256, _, tail = tail.partition("\"")
+
+ # See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api
+ yanked = "data-yanked" in line
+
+ maybe_metadata, _, tail = tail.partition(">")
+ filename, _, tail = tail.partition("<")
+
+ metadata_sha256 = ""
+ metadata_url = ""
+ for metadata_marker in ["data-core-metadata", "data-dist-info-metadata"]:
+ metadata_marker = metadata_marker + "=\"sha256="
+ if metadata_marker in maybe_metadata:
+ # Implement https://peps.python.org/pep-0714/
+ _, _, tail = maybe_metadata.partition(metadata_marker)
+ metadata_sha256, _, _ = tail.partition("\"")
+ metadata_url = dist_url + ".metadata"
+ break
+
+ if filename.endswith(".whl"):
+ whls[sha256] = struct(
+ filename = filename,
+ url = _absolute_url(url, dist_url),
+ sha256 = sha256,
+ metadata_sha256 = metadata_sha256,
+ metadata_url = _absolute_url(url, metadata_url),
+ yanked = yanked,
+ )
+ else:
+ sdists[sha256] = struct(
+ filename = filename,
+ url = _absolute_url(url, dist_url),
+ sha256 = sha256,
+ metadata_sha256 = "",
+ metadata_url = "",
+ yanked = yanked,
+ )
+
+ return struct(
+ sdists = sdists,
+ whls = whls,
+ )
+
+def _absolute_url(index_url, candidate):
+ if not candidate.startswith(".."):
+ return candidate
+
+ candidate_parts = candidate.split("..")
+ last = candidate_parts[-1]
+ for _ in range(len(candidate_parts) - 1):
+ index_url, _, _ = index_url.rstrip("/").rpartition("/")
+
+ return "{}/{}".format(index_url, last.strip("/"))
diff --git a/python/private/parse_whl_name.bzl b/python/private/pypi/parse_whl_name.bzl
similarity index 100%
rename from python/private/parse_whl_name.bzl
rename to python/private/pypi/parse_whl_name.bzl
diff --git a/python/private/patch_whl.bzl b/python/private/pypi/patch_whl.bzl
similarity index 87%
rename from python/private/patch_whl.bzl
rename to python/private/pypi/patch_whl.bzl
index 9e3119f..c2c633d 100644
--- a/python/private/patch_whl.bzl
+++ b/python/private/pypi/patch_whl.bzl
@@ -27,7 +27,8 @@
within the wheel.
"""
-load("//python/private:parse_whl_name.bzl", "parse_whl_name")
+load("//python/private:repo_utils.bzl", "repo_utils")
+load(":parse_whl_name.bzl", "parse_whl_name")
_rules_python_root = Label("//:BUILD.bazel")
@@ -40,7 +41,7 @@
whl_path: The whl file name to be patched.
patches: a label-keyed-int dict that has the patch files as keys and
the patch_strip as the value.
- **kwargs: extras passed to rctx.execute.
+ **kwargs: extras passed to repo_utils.execute_checked.
Returns:
value of the repackaging action.
@@ -75,11 +76,12 @@
record_patch = rctx.path("RECORD.patch")
- result = rctx.execute(
- [
+ repo_utils.execute_checked(
+ rctx,
+ arguments = [
python_interpreter,
"-m",
- "python.private.repack_whl",
+ "python.private.pypi.repack_whl",
"--record-patch",
record_patch,
whl_input,
@@ -91,16 +93,6 @@
**kwargs
)
- if result.return_code:
- fail(
- "repackaging .whl {whl} failed: with exit code '{return_code}':\n{stdout}\n\nstderr:\n{stderr}".format(
- whl = whl_input.basename,
- stdout = result.stdout,
- stderr = result.stderr,
- return_code = result.return_code,
- ),
- )
-
if record_patch.exists:
record_patch_contents = rctx.read(record_patch)
warning_msg = """WARNING: the resultant RECORD file of the patch wheel is different
diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl
new file mode 100644
index 0000000..a22f4d9
--- /dev/null
+++ b/python/private/pypi/pip_repository.bzl
@@ -0,0 +1,327 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""
+
+load("@bazel_skylib//lib:sets.bzl", "sets")
+load("//python/private:normalize_name.bzl", "normalize_name")
+load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR")
+load("//python/private:text_util.bzl", "render")
+load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
+load(":pip_repository_attrs.bzl", "ATTRS")
+load(":render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias")
+
+def _get_python_interpreter_attr(rctx):
+ """A helper function for getting the `python_interpreter` attribute or it's default
+
+ Args:
+ rctx (repository_ctx): Handle to the rule repository context.
+
+ Returns:
+ str: The attribute value or it's default
+ """
+ if rctx.attr.python_interpreter:
+ return rctx.attr.python_interpreter
+
+ if "win" in rctx.os.name:
+ return "python.exe"
+ else:
+ return "python3"
+
+def use_isolated(ctx, attr):
+ """Determine whether or not to pass the pip `--isolated` flag to the pip invocation.
+
+ Args:
+ ctx: repository or module context
+ attr: attributes for the repo rule or tag extension
+
+ Returns:
+ True if --isolated should be passed
+ """
+ use_isolated = attr.isolated
+
+ # The environment variable will take precedence over the attribute
+ isolated_env = ctx.os.environ.get("RULES_PYTHON_PIP_ISOLATED", None)
+ if isolated_env != None:
+ if isolated_env.lower() in ("0", "false"):
+ use_isolated = False
+ else:
+ use_isolated = True
+
+ return use_isolated
+
+_BUILD_FILE_CONTENTS = """\
+package(default_visibility = ["//visibility:public"])
+
+# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it
+exports_files(["requirements.bzl"])
+"""
+
+def _pip_repository_impl(rctx):
+ requirements_by_platform = parse_requirements(
+ rctx,
+ requirements_by_platform = rctx.attr.requirements_by_platform,
+ requirements_linux = rctx.attr.requirements_linux,
+ requirements_lock = rctx.attr.requirements_lock,
+ requirements_osx = rctx.attr.requirements_darwin,
+ requirements_windows = rctx.attr.requirements_windows,
+ extra_pip_args = rctx.attr.extra_pip_args,
+ )
+ selected_requirements = {}
+ options = None
+ repository_platform = host_platform(rctx.os)
+ for name, requirements in requirements_by_platform.items():
+ r = select_requirement(
+ requirements,
+ platform = repository_platform,
+ )
+ if not r:
+ continue
+ options = options or r.extra_pip_args
+ selected_requirements[name] = r.requirement_line
+
+ bzl_packages = sorted(selected_requirements.keys())
+
+ # Normalize cycles first
+ requirement_cycles = {
+ name: sorted(sets.to_list(sets.make(deps)))
+ for name, deps in rctx.attr.experimental_requirement_cycles.items()
+ }
+
+ # Check for conflicts between cycles _before_ we normalize package names so
+ # that reported errors use the names the user specified
+ for i in range(len(requirement_cycles)):
+ left_group = requirement_cycles.keys()[i]
+ left_deps = requirement_cycles.values()[i]
+ for j in range(len(requirement_cycles) - (i + 1)):
+ right_deps = requirement_cycles.values()[1 + i + j]
+ right_group = requirement_cycles.keys()[1 + i + j]
+ for d in left_deps:
+ if d in right_deps:
+ fail("Error: Requirement %s cannot be repeated between cycles %s and %s; please merge the cycles." % (d, left_group, right_group))
+
+ # And normalize the names as used in the cycle specs
+ #
+ # NOTE: We must check that a listed dependency is actually in the actual
+ # requirements set for the current platform so that we can support cycles in
+ # platform-conditional requirements. Otherwise we'll blindly generate a
+ # label referencing a package which may not be installed on the current
+ # platform.
+ requirement_cycles = {
+ normalize_name(name): sorted([normalize_name(d) for d in group if normalize_name(d) in bzl_packages])
+ for name, group in requirement_cycles.items()
+ }
+
+ imports = [
+ # NOTE: Maintain the order consistent with `buildifier`
+ 'load("@rules_python//python:pip.bzl", "pip_utils")',
+ 'load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_library")',
+ ]
+
+ annotations = {}
+ for pkg, annotation in rctx.attr.annotations.items():
+ filename = "{}.annotation.json".format(normalize_name(pkg))
+ rctx.file(filename, json.encode_indent(json.decode(annotation)))
+ annotations[pkg] = "@{name}//:{filename}".format(name = rctx.attr.name, filename = filename)
+
+ config = {
+ "download_only": rctx.attr.download_only,
+ "enable_implicit_namespace_pkgs": rctx.attr.enable_implicit_namespace_pkgs,
+ "environment": rctx.attr.environment,
+ "envsubst": rctx.attr.envsubst,
+ "extra_pip_args": options,
+ "isolated": use_isolated(rctx, rctx.attr),
+ "pip_data_exclude": rctx.attr.pip_data_exclude,
+ "python_interpreter": _get_python_interpreter_attr(rctx),
+ "quiet": rctx.attr.quiet,
+ "repo": rctx.attr.name,
+ "timeout": rctx.attr.timeout,
+ }
+ if rctx.attr.use_hub_alias_dependencies:
+ config["dep_template"] = "@{}//{{name}}:{{target}}".format(rctx.attr.name)
+ else:
+ config["repo_prefix"] = "{}_".format(rctx.attr.name)
+
+ if rctx.attr.python_interpreter_target:
+ config["python_interpreter_target"] = str(rctx.attr.python_interpreter_target)
+ if rctx.attr.experimental_target_platforms:
+ config["experimental_target_platforms"] = rctx.attr.experimental_target_platforms
+
+ macro_tmpl = "@%s//{}:{}" % rctx.attr.name
+
+ aliases = render_pkg_aliases(
+ aliases = {
+ pkg: [whl_alias(repo = rctx.attr.name + "_" + pkg)]
+ for pkg in bzl_packages or []
+ },
+ )
+ for path, contents in aliases.items():
+ rctx.file(path, contents)
+
+ rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS)
+ rctx.template("requirements.bzl", rctx.attr._template, substitutions = {
+ " # %%GROUP_LIBRARY%%": """\
+ group_repo = "{name}__groups"
+ group_library(
+ name = group_repo,
+ repo_prefix = "{name}_",
+ groups = all_requirement_groups,
+ )""".format(name = rctx.attr.name) if not rctx.attr.use_hub_alias_dependencies else "",
+ "%%ALL_DATA_REQUIREMENTS%%": render.list([
+ macro_tmpl.format(p, "data")
+ for p in bzl_packages
+ ]),
+ "%%ALL_REQUIREMENTS%%": render.list([
+ macro_tmpl.format(p, "pkg")
+ for p in bzl_packages
+ ]),
+ "%%ALL_REQUIREMENT_GROUPS%%": render.dict(requirement_cycles),
+ "%%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%": render.dict({
+ p: macro_tmpl.format(p, "whl")
+ for p in bzl_packages
+ }),
+ "%%ANNOTATIONS%%": render.dict(dict(sorted(annotations.items()))),
+ "%%CONFIG%%": render.dict(dict(sorted(config.items()))),
+ "%%EXTRA_PIP_ARGS%%": json.encode(options),
+ "%%IMPORTS%%": "\n".join(imports),
+ "%%MACRO_TMPL%%": macro_tmpl,
+ "%%NAME%%": rctx.attr.name,
+ "%%PACKAGES%%": render.list(
+ [
+ ("{}_{}".format(rctx.attr.name, p), r)
+ for p, r in sorted(selected_requirements.items())
+ ],
+ ),
+ })
+
+ return
+
+pip_repository = repository_rule(
+ attrs = dict(
+ annotations = attr.string_dict(
+ doc = "Optional annotations to apply to packages",
+ ),
+ _template = attr.label(
+ default = ":requirements.bzl.tmpl.workspace",
+ ),
+ **ATTRS
+ ),
+ doc = """Accepts a locked/compiled requirements file and installs the dependencies listed within.
+
+Those dependencies become available in a generated `requirements.bzl` file.
+You can instead check this `requirements.bzl` file into your repo, see the "vendoring" section below.
+
+In your WORKSPACE file:
+
+```starlark
+load("@rules_python//python:pip.bzl", "pip_parse")
+
+pip_parse(
+ name = "pypi",
+ requirements_lock = ":requirements.txt",
+)
+
+load("@pypi//:requirements.bzl", "install_deps")
+
+install_deps()
+```
+
+You can then reference installed dependencies from a `BUILD` file with the alias targets generated in the same repo, for example, for `PyYAML` we would have the following:
+- `@pypi//pyyaml` and `@pypi//pyyaml:pkg` both point to the `py_library`
+ created after extracting the `PyYAML` package.
+- `@pypi//pyyaml:data` points to the extra data included in the package.
+- `@pypi//pyyaml:dist_info` points to the `dist-info` files in the package.
+- `@pypi//pyyaml:whl` points to the wheel file that was extracted.
+
+```starlark
+py_library(
+ name = "bar",
+ ...
+ deps = [
+ "//my/other:dep",
+ "@pypi//numpy",
+ "@pypi//requests",
+ ],
+)
+```
+
+or
+
+```starlark
+load("@pypi//:requirements.bzl", "requirement")
+
+py_library(
+ name = "bar",
+ ...
+ deps = [
+ "//my/other:dep",
+ requirement("numpy"),
+ requirement("requests"),
+ ],
+)
+```
+
+In addition to the `requirement` macro, which is used to access the generated `py_library`
+target generated from a package's wheel, The generated `requirements.bzl` file contains
+functionality for exposing [entry points][whl_ep] as `py_binary` targets as well.
+
+[whl_ep]: https://packaging.python.org/specifications/entry-points/
+
+```starlark
+load("@pypi//:requirements.bzl", "entry_point")
+
+alias(
+ name = "pip-compile",
+ actual = entry_point(
+ pkg = "pip-tools",
+ script = "pip-compile",
+ ),
+)
+```
+
+Note that for packages whose name and script are the same, only the name of the package
+is needed when calling the `entry_point` macro.
+
+```starlark
+load("@pip//:requirements.bzl", "entry_point")
+
+alias(
+ name = "flake8",
+ actual = entry_point("flake8"),
+)
+```
+
+### Vendoring the requirements.bzl file
+
+In some cases you may not want to generate the requirements.bzl file as a repository rule
+while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module
+such as a ruleset, you may want to include the requirements.bzl file rather than make your users
+install the WORKSPACE setup to generate it.
+See https://github.com/bazelbuild/rules_python/issues/608
+
+This is the same workflow as Gazelle, which creates `go_repository` rules with
+[`update-repos`](https://github.com/bazelbuild/bazel-gazelle#update-repos)
+
+To do this, use the "write to source file" pattern documented in
+https://blog.aspect.dev/bazel-can-write-to-the-source-folder
+to put a copy of the generated requirements.bzl into your project.
+Then load the requirements.bzl file directly rather than from the generated repository.
+See the example in rules_python/examples/pip_parse_vendored.
+""",
+ implementation = _pip_repository_impl,
+ environ = [
+ "RULES_PYTHON_PIP_ISOLATED",
+ REPO_DEBUG_ENV_VAR,
+ ],
+)
diff --git a/python/private/pypi/pip_repository_attrs.bzl b/python/private/pypi/pip_repository_attrs.bzl
new file mode 100644
index 0000000..2300086
--- /dev/null
+++ b/python/private/pypi/pip_repository_attrs.bzl
@@ -0,0 +1,73 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Common attributes between bzlmod pip.parse and workspace pip_parse.
+
+A common attributes shared between bzlmod and workspace implementations
+stored in a separate file to avoid unnecessary refetching of the
+repositories."""
+
+load(":attrs.bzl", COMMON_ATTRS = "ATTRS")
+
+ATTRS = {
+ "requirements_by_platform": attr.label_keyed_string_dict(
+ doc = """\
+The requirements files and the comma delimited list of target platforms as values.
+
+The keys are the requirement files and the values are comma-separated platform
+identifiers. For now we only support `<os>_<cpu>` values that are present in
+`@platforms//os` and `@platforms//cpu` packages respectively.
+""",
+ ),
+ "requirements_darwin": attr.label(
+ allow_single_file = True,
+ doc = "Override the requirements_lock attribute when the host platform is Mac OS",
+ ),
+ "requirements_linux": attr.label(
+ allow_single_file = True,
+ doc = "Override the requirements_lock attribute when the host platform is Linux",
+ ),
+ "requirements_lock": attr.label(
+ allow_single_file = True,
+ doc = """\
+A fully resolved 'requirements.txt' pip requirement file containing the
+transitive set of your dependencies. If this file is passed instead of
+'requirements' no resolve will take place and pip_repository will create
+individual repositories for each of your dependencies so that wheels are
+fetched/built only for the targets specified by 'build/run/test'. Note that if
+your lockfile is platform-dependent, you can use the `requirements_[platform]`
+attributes.
+
+Note, that in general requirements files are compiled for a specific platform,
+but sometimes they can work for multiple platforms. `rules_python` right now
+supports requirements files that are created for a particular platform without
+platform markers.
+""",
+ ),
+ "requirements_windows": attr.label(
+ allow_single_file = True,
+ doc = "Override the requirements_lock attribute when the host platform is Windows",
+ ),
+ "use_hub_alias_dependencies": attr.bool(
+ default = False,
+ doc = """\
+Controls if the hub alias dependencies are used. If set to true, then the
+group_library will be included in the hub repo.
+
+True will become default in a subsequent release.
+""",
+ ),
+}
+
+ATTRS.update(**COMMON_ATTRS)
diff --git a/python/private/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl
similarity index 97%
rename from python/private/render_pkg_aliases.bzl
rename to python/private/pypi/render_pkg_aliases.bzl
index 82ac764..eb907fe 100644
--- a/python/private/render_pkg_aliases.bzl
+++ b/python/private/pypi/render_pkg_aliases.bzl
@@ -16,8 +16,10 @@
This is used in bzlmod and non-bzlmod setups."""
+load("//python/private:normalize_name.bzl", "normalize_name")
+load("//python/private:text_util.bzl", "render")
load(
- "//python/pip_install/private:generate_group_library_build_bazel.bzl",
+ ":generate_group_library_build_bazel.bzl",
"generate_group_library_build_bazel",
) # buildifier: disable=bzl-visibility
load(
@@ -29,9 +31,7 @@
"WHEEL_FILE_IMPL_LABEL",
"WHEEL_FILE_PUBLIC_LABEL",
)
-load(":normalize_name.bzl", "normalize_name")
load(":parse_whl_name.bzl", "parse_whl_name")
-load(":text_util.bzl", "render")
load(":whl_target_platforms.bzl", "whl_target_platforms")
NO_MATCH_ERROR_MESSAGE_TEMPLATE = """\
@@ -309,7 +309,7 @@
aliases = config_setting_aliases,
**kwargs
)
- contents["_config/BUILD.bazel"] = _render_pip_config_settings(**flag_versions)
+ contents["_config/BUILD.bazel"] = _render_config_settings(**flag_versions)
return contents
def multiplatform_whl_aliases(*, aliases, default_version = None, **kwargs):
@@ -398,12 +398,12 @@
ret.extend(versioned.values())
return ret
-def _render_pip_config_settings(python_versions = [], target_platforms = [], osx_versions = [], glibc_versions = [], muslc_versions = []):
+def _render_config_settings(python_versions = [], target_platforms = [], osx_versions = [], glibc_versions = [], muslc_versions = []):
return """\
-load("@rules_python//python/private:pip_config_settings.bzl", "pip_config_settings")
+load("@rules_python//python/private/pypi:config_settings.bzl", "config_settings")
-pip_config_settings(
- name = "pip_config_settings",
+config_settings(
+ name = "config_settings",
glibc_versions = {glibc_versions},
muslc_versions = {muslc_versions},
osx_versions = {osx_versions},
diff --git a/python/private/repack_whl.py b/python/private/pypi/repack_whl.py
similarity index 100%
rename from python/private/repack_whl.py
rename to python/private/pypi/repack_whl.py
diff --git a/python/private/bzlmod/requirements.bzl.tmpl b/python/private/pypi/requirements.bzl.tmpl.bzlmod
similarity index 100%
rename from python/private/bzlmod/requirements.bzl.tmpl
rename to python/private/pypi/requirements.bzl.tmpl.bzlmod
diff --git a/python/private/pypi/requirements.bzl.tmpl.workspace b/python/private/pypi/requirements.bzl.tmpl.workspace
new file mode 100644
index 0000000..2f4bcd6
--- /dev/null
+++ b/python/private/pypi/requirements.bzl.tmpl.workspace
@@ -0,0 +1,72 @@
+"""Starlark representation of locked requirements.
+
+@generated by rules_python pip_parse repository rule.
+"""
+
+%%IMPORTS%%
+
+all_requirements = %%ALL_REQUIREMENTS%%
+
+all_whl_requirements_by_package = %%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%
+
+all_whl_requirements = all_whl_requirements_by_package.values()
+
+all_data_requirements = %%ALL_DATA_REQUIREMENTS%%
+
+_packages = %%PACKAGES%%
+_config = %%CONFIG%%
+_annotations = %%ANNOTATIONS%%
+
+def requirement(name):
+ return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "pkg")
+
+def whl_requirement(name):
+ return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "whl")
+
+def data_requirement(name):
+ return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "data")
+
+def dist_info_requirement(name):
+ return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "dist_info")
+
+def _get_annotation(requirement):
+ # This expects to parse `setuptools==58.2.0 --hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11`
+ # down to `setuptools`.
+ name = requirement.split(" ")[0].split("=")[0].split("[")[0]
+ return _annotations.get(name)
+
+def install_deps(**whl_library_kwargs):
+ """Repository rule macro. Install dependencies from `pip_parse`.
+
+ Args:
+ **whl_library_kwargs: Additional arguments which will flow to underlying
+ `whl_library` calls. See pip_repository.bzl for details.
+ """
+
+ # Set up the requirement groups
+ all_requirement_groups = %%ALL_REQUIREMENT_GROUPS%%
+
+ requirement_group_mapping = {
+ requirement: group_name
+ for group_name, group_requirements in all_requirement_groups.items()
+ for requirement in group_requirements
+ }
+
+ # %%GROUP_LIBRARY%%
+
+ # Install wheels which may be participants in a group
+ whl_config = dict(_config)
+ whl_config.update(whl_library_kwargs)
+
+ for name, requirement in _packages:
+ group_name = requirement_group_mapping.get(name.replace("%%NAME%%_", ""))
+ group_deps = all_requirement_groups.get(group_name, [])
+
+ whl_library(
+ name = name,
+ requirement = requirement,
+ group_name = group_name,
+ group_deps = group_deps,
+ annotation = _get_annotation(requirement),
+ **whl_config
+ )
diff --git a/python/private/pypi_index.bzl b/python/private/pypi/simpleapi_download.bzl
similarity index 67%
rename from python/private/pypi_index.bzl
rename to python/private/pypi/simpleapi_download.bzl
index 64d908e..b258fef 100644
--- a/python/private/pypi_index.bzl
+++ b/python/private/pypi/simpleapi_download.bzl
@@ -17,9 +17,10 @@
"""
load("@bazel_features//:features.bzl", "bazel_features")
-load(":auth.bzl", "get_auth")
-load(":envsubst.bzl", "envsubst")
-load(":normalize_name.bzl", "normalize_name")
+load("//python/private:auth.bzl", "get_auth")
+load("//python/private:envsubst.bzl", "envsubst")
+load("//python/private:normalize_name.bzl", "normalize_name")
+load(":parse_simpleapi_html.bzl", "parse_simpleapi_html")
def simpleapi_download(ctx, *, attr, cache, parallel_download = True):
"""Download Simple API HTML.
@@ -71,7 +72,7 @@
success = False
for index_url in index_urls:
- result = read_simple_api(
+ result = _read_simpleapi(
ctx = ctx,
url = "{}/{}/".format(
index_url_overrides.get(pkg_normalized, index_url).rstrip("/"),
@@ -122,7 +123,7 @@
return contents
-def read_simple_api(ctx, url, attr, cache, **download_kwargs):
+def _read_simpleapi(ctx, url, attr, cache, **download_kwargs):
"""Read SimpleAPI.
Args:
@@ -195,98 +196,9 @@
content = ctx.read(output)
- output = parse_simple_api_html(url = url, content = content)
+ output = parse_simpleapi_html(url = url, content = content)
if output:
cache.setdefault(cache_key, output)
return struct(success = True, output = output, cache_key = cache_key)
else:
return struct(success = False)
-
-def parse_simple_api_html(*, url, content):
- """Get the package URLs for given shas by parsing the Simple API HTML.
-
- Args:
- url(str): The URL that the HTML content can be downloaded from.
- content(str): The Simple API HTML content.
-
- Returns:
- A list of structs with:
- * filename: The filename of the artifact.
- * url: The URL to download the artifact.
- * sha256: The sha256 of the artifact.
- * metadata_sha256: The whl METADATA sha256 if we can download it. If this is
- present, then the 'metadata_url' is also present. Defaults to "".
- * metadata_url: The URL for the METADATA if we can download it. Defaults to "".
- """
- sdists = {}
- whls = {}
- lines = content.split("<a href=\"")
-
- _, _, api_version = lines[0].partition("name=\"pypi:repository-version\" content=\"")
- api_version, _, _ = api_version.partition("\"")
-
- # We must assume the 1.0 if it is not present
- # See https://packaging.python.org/en/latest/specifications/simple-repository-api/#clients
- api_version = api_version or "1.0"
- api_version = tuple([int(i) for i in api_version.split(".")])
-
- if api_version >= (2, 0):
- # We don't expect to have version 2.0 here, but have this check in place just in case.
- # https://packaging.python.org/en/latest/specifications/simple-repository-api/#versioning-pypi-s-simple-api
- fail("Unsupported API version: {}".format(api_version))
-
- for line in lines[1:]:
- dist_url, _, tail = line.partition("#sha256=")
- sha256, _, tail = tail.partition("\"")
-
- # See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api
- yanked = "data-yanked" in line
-
- maybe_metadata, _, tail = tail.partition(">")
- filename, _, tail = tail.partition("<")
-
- metadata_sha256 = ""
- metadata_url = ""
- for metadata_marker in ["data-core-metadata", "data-dist-info-metadata"]:
- metadata_marker = metadata_marker + "=\"sha256="
- if metadata_marker in maybe_metadata:
- # Implement https://peps.python.org/pep-0714/
- _, _, tail = maybe_metadata.partition(metadata_marker)
- metadata_sha256, _, _ = tail.partition("\"")
- metadata_url = dist_url + ".metadata"
- break
-
- if filename.endswith(".whl"):
- whls[sha256] = struct(
- filename = filename,
- url = _absolute_url(url, dist_url),
- sha256 = sha256,
- metadata_sha256 = metadata_sha256,
- metadata_url = _absolute_url(url, metadata_url),
- yanked = yanked,
- )
- else:
- sdists[sha256] = struct(
- filename = filename,
- url = _absolute_url(url, dist_url),
- sha256 = sha256,
- metadata_sha256 = "",
- metadata_url = "",
- yanked = yanked,
- )
-
- return struct(
- sdists = sdists,
- whls = whls,
- )
-
-def _absolute_url(index_url, candidate):
- if not candidate.startswith(".."):
- return candidate
-
- candidate_parts = candidate.split("..")
- last = candidate_parts[-1]
- for _ in range(len(candidate_parts) - 1):
- index_url, _, _ = index_url.rstrip("/").rpartition("/")
-
- return "{}/{}".format(index_url, last.strip("/"))
diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl
new file mode 100644
index 0000000..cae0db3
--- /dev/null
+++ b/python/private/pypi/whl_library.bzl
@@ -0,0 +1,509 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""
+
+load("//python:repositories.bzl", "is_standalone_interpreter")
+load("//python:versions.bzl", "WINDOWS_NAME")
+load("//python/pip_install:repositories.bzl", "all_requirements")
+load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth")
+load("//python/private:envsubst.bzl", "envsubst")
+load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
+load("//python/private:toolchains_repo.bzl", "get_host_os_arch")
+load(":attrs.bzl", "ATTRS", "use_isolated")
+load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel")
+load(":parse_whl_name.bzl", "parse_whl_name")
+load(":patch_whl.bzl", "patch_whl")
+load(":whl_target_platforms.bzl", "whl_target_platforms")
+
+_CPPFLAGS = "CPPFLAGS"
+_COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools"
+_WHEEL_ENTRY_POINT_PREFIX = "rules_python_wheel_entry_point"
+
+def _construct_pypath(rctx):
+ """Helper function to construct a PYTHONPATH.
+
+ Contains entries for code in this repo as well as packages downloaded from //python/pip_install:repositories.bzl.
+ This allows us to run python code inside repository rule implementations.
+
+ Args:
+ rctx: Handle to the repository_context.
+
+ Returns: String of the PYTHONPATH.
+ """
+
+ separator = ":" if not "windows" in rctx.os.name.lower() else ";"
+ pypath = separator.join([
+ str(rctx.path(entry).dirname)
+ for entry in rctx.attr._python_path_entries
+ ])
+ return pypath
+
+def _get_python_interpreter_attr(rctx):
+ """A helper function for getting the `python_interpreter` attribute or it's default
+
+ Args:
+ rctx (repository_ctx): Handle to the rule repository context.
+
+ Returns:
+ str: The attribute value or it's default
+ """
+ if rctx.attr.python_interpreter:
+ return rctx.attr.python_interpreter
+
+ if "win" in rctx.os.name:
+ return "python.exe"
+ else:
+ return "python3"
+
+def _resolve_python_interpreter(rctx):
+ """Helper function to find the python interpreter from the common attributes
+
+ Args:
+ rctx: Handle to the rule repository context.
+
+ Returns:
+ `path` object, for the resolved path to the Python interpreter.
+ """
+ python_interpreter = _get_python_interpreter_attr(rctx)
+
+ if rctx.attr.python_interpreter_target != None:
+ python_interpreter = rctx.path(rctx.attr.python_interpreter_target)
+
+ (os, _) = get_host_os_arch(rctx)
+
+ # On Windows, the symlink doesn't work because Windows attempts to find
+ # Python DLLs where the symlink is, not where the symlink points.
+ if os == WINDOWS_NAME:
+ python_interpreter = python_interpreter.realpath
+ elif "/" not in python_interpreter:
+ # It's a plain command, e.g. "python3", to look up in the environment.
+ found_python_interpreter = rctx.which(python_interpreter)
+ if not found_python_interpreter:
+ fail("python interpreter `{}` not found in PATH".format(python_interpreter))
+ python_interpreter = found_python_interpreter
+ else:
+ python_interpreter = rctx.path(python_interpreter)
+ return python_interpreter
+
+def _get_xcode_location_cflags(rctx):
+ """Query the xcode sdk location to update cflags
+
+ Figure out if this interpreter target comes from rules_python, and patch the xcode sdk location if so.
+ Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg
+ otherwise. See https://github.com/indygreg/python-build-standalone/issues/103
+ """
+
+ # Only run on MacOS hosts
+ if not rctx.os.name.lower().startswith("mac os"):
+ return []
+
+ xcode_sdk_location = repo_utils.execute_unchecked(
+ rctx,
+ op = "GetXcodeLocation",
+ arguments = [repo_utils.which_checked(rctx, "xcode-select"), "--print-path"],
+ )
+ if xcode_sdk_location.return_code != 0:
+ return []
+
+ xcode_root = xcode_sdk_location.stdout.strip()
+ if _COMMAND_LINE_TOOLS_PATH_SLUG not in xcode_root.lower():
+ # This is a full xcode installation somewhere like /Applications/Xcode13.0.app/Contents/Developer
+ # so we need to change the path to to the macos specific tools which are in a different relative
+ # path than xcode installed command line tools.
+ xcode_root = "{}/Platforms/MacOSX.platform/Developer".format(xcode_root)
+ return [
+ "-isysroot {}/SDKs/MacOSX.sdk".format(xcode_root),
+ ]
+
+def _get_toolchain_unix_cflags(rctx, python_interpreter):
+ """Gather cflags from a standalone toolchain for unix systems.
+
+ Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg
+ otherwise. See https://github.com/indygreg/python-build-standalone/issues/103
+ """
+
+ # Only run on Unix systems
+ if not rctx.os.name.lower().startswith(("mac os", "linux")):
+ return []
+
+ # Only update the location when using a standalone toolchain.
+ if not is_standalone_interpreter(rctx, python_interpreter):
+ return []
+
+ stdout = repo_utils.execute_checked_stdout(
+ rctx,
+ op = "GetPythonVersionForUnixCflags",
+ arguments = [
+ python_interpreter,
+ "-c",
+ "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}', end='')",
+ ],
+ )
+ _python_version = stdout
+ include_path = "{}/include/python{}".format(
+ python_interpreter.dirname,
+ _python_version,
+ )
+
+ return ["-isystem {}".format(include_path)]
+
+def _parse_optional_attrs(rctx, args, extra_pip_args = None):
+ """Helper function to parse common attributes of pip_repository and whl_library repository rules.
+
+ This function also serializes the structured arguments as JSON
+ so they can be passed on the command line to subprocesses.
+
+ Args:
+ rctx: Handle to the rule repository context.
+ args: A list of parsed args for the rule.
+ extra_pip_args: The pip args to pass.
+ Returns: Augmented args list.
+ """
+
+ if use_isolated(rctx, rctx.attr):
+ args.append("--isolated")
+
+ # Bazel version 7.1.0 and later (and rolling releases from version 8.0.0-pre.20240128.3)
+ # support rctx.getenv(name, default): When building incrementally, any change to the value of
+ # the variable named by name will cause this repository to be re-fetched.
+ if "getenv" in dir(rctx):
+ getenv = rctx.getenv
+ else:
+ getenv = rctx.os.environ.get
+
+ # Check for None so we use empty default types from our attrs.
+ # Some args want to be list, and some want to be dict.
+ if extra_pip_args != None:
+ args += [
+ "--extra_pip_args",
+ json.encode(struct(arg = [
+ envsubst(pip_arg, rctx.attr.envsubst, getenv)
+ for pip_arg in rctx.attr.extra_pip_args
+ ])),
+ ]
+
+ if rctx.attr.download_only:
+ args.append("--download_only")
+
+ if rctx.attr.pip_data_exclude != None:
+ args += [
+ "--pip_data_exclude",
+ json.encode(struct(arg = rctx.attr.pip_data_exclude)),
+ ]
+
+ if rctx.attr.enable_implicit_namespace_pkgs:
+ args.append("--enable_implicit_namespace_pkgs")
+
+ if rctx.attr.environment != None:
+ args += [
+ "--environment",
+ json.encode(struct(arg = rctx.attr.environment)),
+ ]
+
+ return args
+
+def _create_repository_execution_environment(rctx, python_interpreter):
+ """Create a environment dictionary for processes we spawn with rctx.execute.
+
+ Args:
+ rctx (repository_ctx): The repository context.
+ python_interpreter (path): The resolved python interpreter.
+ Returns:
+ Dictionary of environment variable suitable to pass to rctx.execute.
+ """
+
+ # Gather any available CPPFLAGS values
+ cppflags = []
+ cppflags.extend(_get_xcode_location_cflags(rctx))
+ cppflags.extend(_get_toolchain_unix_cflags(rctx, python_interpreter))
+
+ env = {
+ "PYTHONPATH": _construct_pypath(rctx),
+ _CPPFLAGS: " ".join(cppflags),
+ }
+
+ return env
+
+def _whl_library_impl(rctx):
+ python_interpreter = _resolve_python_interpreter(rctx)
+ args = [
+ python_interpreter,
+ "-m",
+ "python.pip_install.tools.wheel_installer.wheel_installer",
+ "--requirement",
+ rctx.attr.requirement,
+ ]
+ extra_pip_args = []
+ extra_pip_args.extend(rctx.attr.extra_pip_args)
+
+ # Manually construct the PYTHONPATH since we cannot use the toolchain here
+ environment = _create_repository_execution_environment(rctx, python_interpreter)
+
+ whl_path = None
+ if rctx.attr.whl_file:
+ whl_path = rctx.path(rctx.attr.whl_file)
+
+ # Simulate the behaviour where the whl is present in the current directory.
+ rctx.symlink(whl_path, whl_path.basename)
+ whl_path = rctx.path(whl_path.basename)
+ elif rctx.attr.urls:
+ filename = rctx.attr.filename
+ urls = rctx.attr.urls
+ if not filename:
+ _, _, filename = urls[0].rpartition("/")
+
+ if not (filename.endswith(".whl") or filename.endswith("tar.gz") or filename.endswith(".zip")):
+ if rctx.attr.filename:
+ msg = "got '{}'".format(filename)
+ else:
+ msg = "detected '{}' from url:\n{}".format(filename, urls[0])
+ fail("Only '.whl', '.tar.gz' or '.zip' files are supported, {}".format(msg))
+
+ result = rctx.download(
+ url = urls,
+ output = filename,
+ sha256 = rctx.attr.sha256,
+ auth = get_auth(rctx, urls),
+ )
+
+ if not result.success:
+ fail("could not download the '{}' from {}:\n{}".format(filename, urls, result))
+
+ if filename.endswith(".whl"):
+ whl_path = rctx.path(rctx.attr.filename)
+ else:
+ # It is an sdist and we need to tell PyPI to use a file in this directory
+ # and not use any indexes.
+ extra_pip_args.extend(["--no-index", "--find-links", "."])
+
+ args = _parse_optional_attrs(rctx, args, extra_pip_args)
+
+ if not whl_path:
+ repo_utils.execute_checked(
+ rctx,
+ op = "whl_library.ResolveRequirement({}, {})".format(rctx.attr.name, rctx.attr.requirement),
+ arguments = args,
+ environment = environment,
+ quiet = rctx.attr.quiet,
+ timeout = rctx.attr.timeout,
+ )
+
+ whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"])
+ if not rctx.delete("whl_file.json"):
+ fail("failed to delete the whl_file.json file")
+
+ if rctx.attr.whl_patches:
+ patches = {}
+ for patch_file, json_args in rctx.attr.whl_patches.items():
+ patch_dst = struct(**json.decode(json_args))
+ if whl_path.basename in patch_dst.whls:
+ patches[patch_file] = patch_dst.patch_strip
+
+ whl_path = patch_whl(
+ rctx,
+ op = "whl_library.PatchWhl({}, {})".format(rctx.attr.name, rctx.attr.requirement),
+ python_interpreter = python_interpreter,
+ whl_path = whl_path,
+ patches = patches,
+ quiet = rctx.attr.quiet,
+ timeout = rctx.attr.timeout,
+ )
+
+ target_platforms = rctx.attr.experimental_target_platforms
+ if target_platforms:
+ parsed_whl = parse_whl_name(whl_path.basename)
+ if parsed_whl.platform_tag != "any":
+ # NOTE @aignas 2023-12-04: if the wheel is a platform specific
+ # wheel, we only include deps for that target platform
+ target_platforms = [
+ p.target_platform
+ for p in whl_target_platforms(
+ platform_tag = parsed_whl.platform_tag,
+ abi_tag = parsed_whl.abi_tag,
+ )
+ ]
+
+ repo_utils.execute_checked(
+ rctx,
+ op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path),
+ arguments = args + [
+ "--whl-file",
+ whl_path,
+ ] + ["--platform={}".format(p) for p in target_platforms],
+ environment = environment,
+ quiet = rctx.attr.quiet,
+ timeout = rctx.attr.timeout,
+ )
+
+ metadata = json.decode(rctx.read("metadata.json"))
+ rctx.delete("metadata.json")
+
+ # NOTE @aignas 2024-06-22: this has to live on until we stop supporting
+ # passing `twine` as a `:pkg` library via the `WORKSPACE` builds.
+ #
+ # See ../../packaging.bzl line 190
+ entry_points = {}
+ for item in metadata["entry_points"]:
+ name = item["name"]
+ module = item["module"]
+ attribute = item["attribute"]
+
+ # There is an extreme edge-case with entry_points that end with `.py`
+ # See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174
+ entry_point_without_py = name[:-3] + "_py" if name.endswith(".py") else name
+ entry_point_target_name = (
+ _WHEEL_ENTRY_POINT_PREFIX + "_" + entry_point_without_py
+ )
+ entry_point_script_name = entry_point_target_name + ".py"
+
+ rctx.file(
+ entry_point_script_name,
+ _generate_entry_point_contents(module, attribute),
+ )
+ entry_points[entry_point_without_py] = entry_point_script_name
+
+ build_file_contents = generate_whl_library_build_bazel(
+ dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix),
+ whl_name = whl_path.basename,
+ dependencies = metadata["deps"],
+ dependencies_by_platform = metadata["deps_by_platform"],
+ group_name = rctx.attr.group_name,
+ group_deps = rctx.attr.group_deps,
+ data_exclude = rctx.attr.pip_data_exclude,
+ tags = [
+ "pypi_name=" + metadata["name"],
+ "pypi_version=" + metadata["version"],
+ ],
+ entry_points = entry_points,
+ annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))),
+ )
+ rctx.file("BUILD.bazel", build_file_contents)
+
+ return
+
+def _generate_entry_point_contents(
+ module,
+ attribute,
+ shebang = "#!/usr/bin/env python3"):
+ """Generate the contents of an entry point script.
+
+ Args:
+ module (str): The name of the module to use.
+ attribute (str): The name of the attribute to call.
+ shebang (str, optional): The shebang to use for the entry point python
+ file.
+
+ Returns:
+ str: A string of python code.
+ """
+ contents = """\
+{shebang}
+import sys
+from {module} import {attribute}
+if __name__ == "__main__":
+ sys.exit({attribute}())
+""".format(
+ shebang = shebang,
+ module = module,
+ attribute = attribute,
+ )
+ return contents
+
+# NOTE @aignas 2024-03-21: The usage of dict({}, **common) ensures that all args to `dict` are unique
+whl_library_attrs = dict({
+ "annotation": attr.label(
+ doc = (
+ "Optional json encoded file containing annotation to apply to the extracted wheel. " +
+ "See `package_annotation`"
+ ),
+ allow_files = True,
+ ),
+ "dep_template": attr.string(
+ doc = """
+The dep template to use for referencing the dependencies. It should have `{name}`
+and `{target}` tokens that will be replaced with the normalized distribution name
+and the target that we need respectively.
+""",
+ ),
+ "filename": attr.string(
+ doc = "Download the whl file to this filename. Only used when the `urls` is passed. If not specified, will be auto-detected from the `urls`.",
+ ),
+ "group_deps": attr.string_list(
+ doc = "List of dependencies to skip in order to break the cycles within a dependency group.",
+ default = [],
+ ),
+ "group_name": attr.string(
+ doc = "Name of the group, if any.",
+ ),
+ "repo": attr.string(
+ mandatory = True,
+ doc = "Pointer to parent repo name. Used to make these rules rerun if the parent repo changes.",
+ ),
+ "repo_prefix": attr.string(
+ doc = """
+Prefix for the generated packages will be of the form `@<prefix><sanitized-package-name>//...`
+
+DEPRECATED. Only left for people who vendor requirements.bzl.
+""",
+ ),
+ "requirement": attr.string(
+ mandatory = True,
+ doc = "Python requirement string describing the package to make available, if 'urls' or 'whl_file' is given, then this only needs to include foo[any_extras] as a bare minimum.",
+ ),
+ "sha256": attr.string(
+ doc = "The sha256 of the downloaded whl. Only used when the `urls` is passed.",
+ ),
+ "urls": attr.string_list(
+ doc = """\
+The list of urls of the whl to be downloaded using bazel downloader. Using this
+attr makes `extra_pip_args` and `download_only` ignored.""",
+ ),
+ "whl_file": attr.label(
+ doc = "The whl file that should be used instead of downloading or building the whl.",
+ ),
+ "whl_patches": attr.label_keyed_string_dict(
+ doc = """a label-keyed-string dict that has
+ json.encode(struct([whl_file], patch_strip]) as values. This
+ is to maintain flexibility and correct bzlmod extension interface
+ until we have a better way to define whl_library and move whl
+ patching to a separate place. INTERNAL USE ONLY.""",
+ ),
+ "_python_path_entries": attr.label_list(
+ # Get the root directory of these rules and keep them as a default attribute
+ # in order to avoid unnecessary repository fetching restarts.
+ #
+ # This is very similar to what was done in https://github.com/bazelbuild/rules_go/pull/3478
+ default = [
+ Label("//:BUILD.bazel"),
+ ] + [
+ # Includes all the external dependencies from repositories.bzl
+ Label("@" + repo + "//:BUILD.bazel")
+ for repo in all_requirements
+ ],
+ ),
+}, **ATTRS)
+whl_library_attrs.update(AUTH_ATTRS)
+
+whl_library = repository_rule(
+ attrs = whl_library_attrs,
+ doc = """
+Download and extracts a single wheel based into a bazel repo based on the requirement string passed in.
+Instantiated from pip_repository and inherits config options from there.""",
+ implementation = _whl_library_impl,
+ environ = [
+ "RULES_PYTHON_PIP_ISOLATED",
+ REPO_DEBUG_ENV_VAR,
+ ],
+)
diff --git a/python/private/pip_repo_name.bzl b/python/private/pypi/whl_repo_name.bzl
similarity index 93%
rename from python/private/pip_repo_name.bzl
rename to python/private/pypi/whl_repo_name.bzl
index bef4304..295f5a4 100644
--- a/python/private/pip_repo_name.bzl
+++ b/python/private/pypi/whl_repo_name.bzl
@@ -15,10 +15,10 @@
"""A function to convert a dist name to a valid bazel repo name.
"""
-load(":normalize_name.bzl", "normalize_name")
+load("//python/private:normalize_name.bzl", "normalize_name")
load(":parse_whl_name.bzl", "parse_whl_name")
-def pip_repo_name(prefix, filename, sha256):
+def whl_repo_name(prefix, filename, sha256):
"""Return a valid whl_library repo name given a distribution filename.
Args:
diff --git a/python/private/whl_target_platforms.bzl b/python/private/pypi/whl_target_platforms.bzl
similarity index 100%
rename from python/private/whl_target_platforms.bzl
rename to python/private/pypi/whl_target_platforms.bzl
diff --git a/python/private/text_util.bzl b/python/private/text_util.bzl
index dade9cb..702a08e 100644
--- a/python/private/text_util.bzl
+++ b/python/private/text_util.bzl
@@ -36,6 +36,9 @@
])
def _render_dict(d, *, key_repr = repr, value_repr = repr):
+ if not d:
+ return "{}"
+
return "\n".join([
"{",
_indent("\n".join([
