fix: load target_platforms through the hub (#2781)
This PR moves the parsing of `Requires-Dist` to the loading phase
within the `whl_library_targets_from_requires` macro. The original
`whl_library_targets` macro has been left unchanged so that I don't have
to reinvent the unit tests - it is well covered under tests.
Before this PR we had to wire the `target_platforms` via the
`experimental_target_platforms` attr in the `whl_library`, which means
that whenever this would change (e.g. the minor Python version changes),
the wheel would be re-extracted even though the final result may be the
same.
This refactor uncovered that the dependency graph creation was incorrect
if we had multiple target Python versions due to various heuristics that
this had. In hindsight I had them to make the generated `BUILD.bazel`
files more readable when the unit test coverage was not great. Now this
is unnecessary and since everything is happening in Starlark I thought
that having a simpler algorithm that does the right thing always is the
best way.
This also cleans up the code by removing left over TODO notes or code
that no longer make sense.
Work towards #260, #2319
diff --git a/CHANGELOG.md b/CHANGELOG.md
index cad074e..154b661 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -105,6 +105,13 @@
[PR #2746](https://github.com/bazel-contrib/rules_python/pull/2746).
* (rules) {attr}`py_binary.srcs` and {attr}`py_test.srcs` is no longer mandatory when
`main_module` is specified (for `--bootstrap_impl=script`)
+* (pypi) From now on the `Requires-Dist` from the wheel metadata is analysed in
+ the loading phase instead of repository rule phase giving better caching
+ performance when the target platforms are changed (e.g. target python
+ versions). This is preparatory work for stabilizing the cross-platform wheel
+ support. From now on the usage of `experimental_target_platforms` should be
+ avoided and the `requirements_by_platform` values should be instead used to
+ specify the target platforms for the given dependencies.
[20250317]: https://github.com/astral-sh/python-build-standalone/releases/tag/20250317
diff --git a/config.bzl.tmpl.bzlmod b/config.bzl.tmpl.bzlmod
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/config.bzl.tmpl.bzlmod
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
index 7297238..a758b3f 100644
--- a/python/private/pypi/BUILD.bazel
+++ b/python/private/pypi/BUILD.bazel
@@ -213,15 +213,6 @@
)
bzl_library(
- name = "pep508_bzl",
- srcs = ["pep508.bzl"],
- deps = [
- ":pep508_env_bzl",
- ":pep508_evaluate_bzl",
- ],
-)
-
-bzl_library(
name = "pep508_deps_bzl",
srcs = ["pep508_deps.bzl"],
deps = [
@@ -378,13 +369,12 @@
":attrs_bzl",
":deps_bzl",
":generate_whl_library_build_bazel_bzl",
- ":parse_whl_name_bzl",
":patch_whl_bzl",
- ":pep508_deps_bzl",
+ ":pep508_requirement_bzl",
":pypi_repo_utils_bzl",
":whl_metadata_bzl",
- ":whl_target_platforms_bzl",
"//python/private:auth_bzl",
+ "//python/private:bzlmod_enabled_bzl",
"//python/private:envsubst_bzl",
"//python/private:is_standalone_interpreter_bzl",
"//python/private:repo_utils_bzl",
diff --git a/python/private/pypi/attrs.bzl b/python/private/pypi/attrs.bzl
index 9d88c1e..fe35d8b 100644
--- a/python/private/pypi/attrs.bzl
+++ b/python/private/pypi/attrs.bzl
@@ -123,6 +123,9 @@
"experimental_target_platforms": attr.string_list(
default = [],
doc = """\
+*NOTE*: This will be removed in the next major version, so please consider migrating
+to `bzlmod` and rely on {attr}`pip.parse.requirements_by_platform` for this feature.
+
A list of platforms that we will generate the conditional dependency graph for
cross platform wheels by parsing the wheel metadata. This will generate the
correct dependencies for packages like `sphinx` or `pylint`, which include
diff --git a/python/private/pypi/config.bzl.tmpl.bzlmod b/python/private/pypi/config.bzl.tmpl.bzlmod
new file mode 100644
index 0000000..deb5363
--- /dev/null
+++ b/python/private/pypi/config.bzl.tmpl.bzlmod
@@ -0,0 +1,9 @@
+"""Extra configuration values that are exposed from the hub repository for spoke repositories to access.
+
+NOTE: This is internal `rules_python` API and if you would like to depend on it, please raise an issue
+with your usecase. This may change in between rules_python versions without any notice.
+
+@generated by rules_python pip.parse bzlmod extension.
+"""
+
+target_platforms = %%TARGET_PLATFORMS%%
diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl
index 68776e3..d1895ca 100644
--- a/python/private/pypi/extension.bzl
+++ b/python/private/pypi/extension.bzl
@@ -32,7 +32,6 @@
load(":whl_config_setting.bzl", "whl_config_setting")
load(":whl_library.bzl", "whl_library")
load(":whl_repo_name.bzl", "pypi_repo_name", "whl_repo_name")
-load(":whl_target_platforms.bzl", "whl_target_platforms")
def _major_minor_version(version):
version = semver(version)
@@ -68,7 +67,6 @@
*,
pip_attr,
whl_overrides,
- evaluate_markers = evaluate_markers,
available_interpreters = INTERPRETER_LABELS,
get_index_urls = None):
"""create all of the whl repositories
@@ -77,7 +75,6 @@
module_ctx: {type}`module_ctx`.
pip_attr: {type}`struct` - the struct that comes from the tag class iteration.
whl_overrides: {type}`dict[str, struct]` - per-wheel overrides.
- evaluate_markers: the function to use to evaluate markers.
get_index_urls: A function used to get the index URLs
available_interpreters: {type}`dict[str, Label]` The dictionary of available
interpreters that have been registered using the `python` bzlmod extension.
@@ -162,14 +159,12 @@
requirements_osx = pip_attr.requirements_darwin,
requirements_windows = pip_attr.requirements_windows,
extra_pip_args = pip_attr.extra_pip_args,
+ # TODO @aignas 2025-04-15: pass the full version into here
python_version = major_minor,
logger = logger,
),
extra_pip_args = pip_attr.extra_pip_args,
get_index_urls = get_index_urls,
- # NOTE @aignas 2025-02-24: we will use the "cp3xx_os_arch" platform labels
- # for converting to the PEP508 environment and will evaluate them in starlark
- # without involving the interpreter at all.
evaluate_markers = evaluate_markers,
logger = logger,
)
@@ -191,7 +186,6 @@
enable_implicit_namespace_pkgs = pip_attr.enable_implicit_namespace_pkgs,
environment = pip_attr.environment,
envsubst = pip_attr.envsubst,
- experimental_target_platforms = pip_attr.experimental_target_platforms,
group_deps = group_deps,
group_name = group_name,
pip_data_exclude = pip_attr.pip_data_exclude,
@@ -244,6 +238,12 @@
},
extra_aliases = extra_aliases,
whl_libraries = whl_libraries,
+ target_platforms = {
+ plat: None
+ for reqs in requirements_by_platform.values()
+ for req in reqs
+ for plat in req.target_platforms
+ },
)
def _whl_repos(*, requirement, whl_library_args, download_only, netrc, auth_patterns, multiple_requirements_for_whl = False, python_version):
@@ -274,20 +274,11 @@
args["urls"] = [distribution.url]
args["sha256"] = distribution.sha256
args["filename"] = distribution.filename
- args["experimental_target_platforms"] = requirement.target_platforms
# Pure python wheels or sdists may need to have a platform here
target_platforms = None
if distribution.filename.endswith(".whl") and not distribution.filename.endswith("-any.whl"):
- parsed_whl = parse_whl_name(distribution.filename)
- whl_platforms = whl_target_platforms(
- platform_tag = parsed_whl.platform_tag,
- )
- args["experimental_target_platforms"] = [
- p
- for p in requirement.target_platforms
- if [None for wp in whl_platforms if p.endswith(wp.target_platform)]
- ]
+ pass
elif multiple_requirements_for_whl:
target_platforms = requirement.target_platforms
@@ -416,6 +407,7 @@
hub_group_map = {}
exposed_packages = {}
extra_aliases = {}
+ target_platforms = {}
whl_libraries = {}
for mod in module_ctx.modules:
@@ -498,6 +490,7 @@
for whl_name, aliases in out.extra_aliases.items():
extra_aliases[hub_name].setdefault(whl_name, {}).update(aliases)
exposed_packages.setdefault(hub_name, {}).update(out.exposed_packages)
+ target_platforms.setdefault(hub_name, {}).update(out.target_platforms)
whl_libraries.update(out.whl_libraries)
# TODO @aignas 2024-04-05: how do we support different requirement
@@ -535,6 +528,10 @@
}
for hub_name, extra_whl_aliases in extra_aliases.items()
},
+ target_platforms = {
+ hub_name: sorted(p)
+ for hub_name, p in target_platforms.items()
+ },
whl_libraries = {
k: dict(sorted(args.items()))
for k, args in sorted(whl_libraries.items())
@@ -626,15 +623,13 @@
},
packages = mods.exposed_packages.get(hub_name, []),
groups = mods.hub_group_map.get(hub_name),
+ target_platforms = mods.target_platforms.get(hub_name, []),
)
if bazel_features.external_deps.extension_metadata_has_reproducible:
- # If we are not using the `experimental_index_url feature, the extension is fully
- # deterministic and we don't need to create a lock entry for it.
- #
- # In order to be able to dogfood the `experimental_index_url` feature before it gets
- # stabilized, we have created the `_pip_non_reproducible` function, that will result
- # in extra entries in the lock file.
+ # NOTE @aignas 2025-04-15: this is set to be reproducible, because the
+ # results after calling the PyPI index should be reproducible on each
+ # machine.
return module_ctx.extension_metadata(reproducible = True)
else:
return None
diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl
index 8050cd2..7988aca 100644
--- a/python/private/pypi/generate_whl_library_build_bazel.bzl
+++ b/python/private/pypi/generate_whl_library_build_bazel.bzl
@@ -21,23 +21,23 @@
"copy_files": render.dict,
"data": render.list,
"data_exclude": render.list,
- "dependencies": render.list,
- "dependencies_by_platform": lambda x: render.dict(x, value_repr = render.list),
"entry_points": render.dict,
+ "extras": render.list,
"group_deps": render.list,
+ "requires_dist": render.list,
"srcs_exclude": render.list,
- "tags": render.list,
+ "target_platforms": lambda x: render.list(x) if x else "target_platforms",
}
# NOTE @aignas 2024-10-25: We have to keep this so that files in
# this repository can be publicly visible without the need for
# export_files
_TEMPLATE = """\
-load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets")
+{loads}
package(default_visibility = ["//visibility:public"])
-whl_library_targets(
+whl_library_targets_from_requires(
{kwargs}
)
"""
@@ -45,11 +45,13 @@
def generate_whl_library_build_bazel(
*,
annotation = None,
+ default_python_version = None,
**kwargs):
"""Generate a BUILD file for an unzipped Wheel
Args:
annotation: The annotation for the build file.
+ default_python_version: The python version to use to parse the METADATA.
**kwargs: Extra args serialized to be passed to the
{obj}`whl_library_targets`.
@@ -57,6 +59,18 @@
A complete BUILD file as a string
"""
+ loads = [
+ """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets_from_requires")""",
+ ]
+ if not kwargs.setdefault("target_platforms", None):
+ dep_template = kwargs["dep_template"]
+ loads.append(
+ "load(\"{}\", \"{}\")".format(
+ dep_template.format(name = "", target = "config.bzl"),
+ "target_platforms",
+ ),
+ )
+
additional_content = []
if annotation:
kwargs["data"] = annotation.data
@@ -66,10 +80,13 @@
kwargs["srcs_exclude"] = annotation.srcs_exclude_glob
if annotation.additive_build_content:
additional_content.append(annotation.additive_build_content)
+ if default_python_version:
+ kwargs["default_python_version"] = default_python_version
contents = "\n".join(
[
_TEMPLATE.format(
+ loads = "\n".join(loads),
kwargs = render.indent("\n".join([
"{} = {},".format(k, _RENDER.get(k, repr)(v))
for k, v in sorted(kwargs.items())
diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl
index 48245b4..d2cbf88 100644
--- a/python/private/pypi/hub_repository.bzl
+++ b/python/private/pypi/hub_repository.bzl
@@ -45,7 +45,14 @@
macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name)
rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS)
- rctx.template("requirements.bzl", rctx.attr._template, substitutions = {
+ rctx.template(
+ "config.bzl",
+ rctx.attr._config_template,
+ substitutions = {
+ "%%TARGET_PLATFORMS%%": render.list(rctx.attr.target_platforms),
+ },
+ )
+ rctx.template("requirements.bzl", rctx.attr._requirements_bzl_template, substitutions = {
"%%ALL_DATA_REQUIREMENTS%%": render.list([
macro_tmpl.format(p, "data")
for p in bzl_packages
@@ -80,6 +87,10 @@
mandatory = True,
doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.",
),
+ "target_platforms": attr.string_list(
+ mandatory = True,
+ doc = "All of the target platforms for the hub repo",
+ ),
"whl_map": attr.string_dict(
mandatory = True,
doc = """\
@@ -87,7 +98,10 @@
in the pip.parse tag class.
""",
),
- "_template": attr.label(
+ "_config_template": attr.label(
+ default = ":config.bzl.tmpl.bzlmod",
+ ),
+ "_requirements_bzl_template": attr.label(
default = ":requirements.bzl.tmpl.bzlmod",
),
},
diff --git a/python/private/pypi/pep508.bzl b/python/private/pypi/pep508.bzl
deleted file mode 100644
index e74352d..0000000
--- a/python/private/pypi/pep508.bzl
+++ /dev/null
@@ -1,23 +0,0 @@
-# Copyright 2025 The Bazel Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""This module is for implementing PEP508 in starlark as FeatureFlagInfo
-"""
-
-load(":pep508_env.bzl", _env = "env")
-load(":pep508_evaluate.bzl", _evaluate = "evaluate", _to_string = "to_string")
-
-to_string = _to_string
-evaluate = _evaluate
-env = _env
diff --git a/python/private/pypi/pep508_deps.bzl b/python/private/pypi/pep508_deps.bzl
index af0a753..115bbd7 100644
--- a/python/private/pypi/pep508_deps.bzl
+++ b/python/private/pypi/pep508_deps.bzl
@@ -15,36 +15,24 @@
"""This module is for implementing PEP508 compliant METADATA deps parsing.
"""
+load("@pythons_hub//:versions.bzl", "DEFAULT_PYTHON_VERSION")
load("//python/private:normalize_name.bzl", "normalize_name")
load(":pep508_env.bzl", "env")
load(":pep508_evaluate.bzl", "evaluate")
load(":pep508_platform.bzl", "platform", "platform_from_str")
load(":pep508_requirement.bzl", "requirement")
-_ALL_OS_VALUES = [
- "windows",
- "osx",
- "linux",
-]
-_ALL_ARCH_VALUES = [
- "aarch64",
- "ppc64",
- "ppc64le",
- "s390x",
- "x86_32",
- "x86_64",
-]
-
-def deps(name, *, requires_dist, platforms = [], extras = [], host_python_version = None):
+def deps(name, *, requires_dist, platforms = [], extras = [], excludes = [], default_python_version = None):
"""Parse the RequiresDist from wheel METADATA
Args:
name: {type}`str` the name of the wheel.
requires_dist: {type}`list[str]` the list of RequiresDist lines from the
METADATA file.
+ excludes: {type}`list[str]` what packages should we exclude.
extras: {type}`list[str]` the requested extras to generate targets for.
platforms: {type}`list[str]` the list of target platform strings.
- host_python_version: {type}`str` the host python version.
+ default_python_version: {type}`str` the host python version.
Returns:
A struct with attributes:
@@ -62,18 +50,17 @@
want_extras = _resolve_extras(name, reqs, extras)
# drop self edges
- reqs = [r for r in reqs if r.name != name]
+ excludes = [name] + [normalize_name(x) for x in excludes]
+ default_python_version = default_python_version or DEFAULT_PYTHON_VERSION
platforms = [
- platform_from_str(p, python_version = host_python_version)
+ platform_from_str(p, python_version = default_python_version)
for p in platforms
- ] or [
- platform_from_str("", python_version = host_python_version),
]
abis = sorted({p.abi: True for p in platforms if p.abi})
- if host_python_version and len(abis) > 1:
- _, _, minor_version = host_python_version.partition(".")
+ if default_python_version and len(abis) > 1:
+ _, _, minor_version = default_python_version.partition(".")
minor_version, _, _ = minor_version.partition(".")
default_abi = "cp3" + minor_version
elif len(abis) > 1:
@@ -83,11 +70,20 @@
else:
default_abi = None
+ reqs_by_name = {}
+
for req in reqs:
- _add_req(
+ if req.name_ in excludes:
+ continue
+
+ reqs_by_name.setdefault(req.name, []).append(req)
+
+ for name, reqs in reqs_by_name.items():
+ _add_reqs(
deps,
deps_select,
- req,
+ normalize_name(name),
+ reqs,
extras = want_extras,
platforms = platforms,
default_abi = default_abi,
@@ -103,49 +99,14 @@
def _platform_str(self):
if self.abi == None:
- if not self.os and not self.arch:
- return "//conditions:default"
- elif not self.arch:
- return "@platforms//os:{}".format(self.os)
- else:
- return "{}_{}".format(self.os, self.arch)
+ return "{}_{}".format(self.os, self.arch)
- minor_version = self.abi[3:]
- if self.arch == None and self.os == None:
- return str(Label("//python/config_settings:is_python_3.{}".format(minor_version)))
-
- return "cp3{}_{}_{}".format(
- minor_version,
+ return "{}_{}_{}".format(
+ self.abi,
self.os or "anyos",
self.arch or "anyarch",
)
-def _platform_specializations(self, cpu_values = _ALL_ARCH_VALUES, os_values = _ALL_OS_VALUES):
- """Return the platform itself and all its unambiguous specializations.
-
- For more info about specializations see
- https://bazel.build/docs/configurable-attributes
- """
- specializations = []
- specializations.append(self)
- if self.arch == None:
- specializations.extend([
- platform(os = self.os, arch = arch, abi = self.abi)
- for arch in cpu_values
- ])
- if self.os == None:
- specializations.extend([
- platform(os = os, arch = self.arch, abi = self.abi)
- for os in os_values
- ])
- if self.os == None and self.arch == None:
- specializations.extend([
- platform(os = os, arch = arch, abi = self.abi)
- for os in os_values
- for arch in cpu_values
- ])
- return specializations
-
def _add(deps, deps_select, dep, platform):
dep = normalize_name(dep)
@@ -172,53 +133,7 @@
return
# Add the platform-specific branch
- deps_select.setdefault(platform, {})
-
- # Add the dep to specializations of the given platform if they
- # exist in the select statement.
- for p in _platform_specializations(platform):
- if p not in deps_select:
- continue
-
- deps_select[p][dep] = True
-
- if len(deps_select[platform]) == 1:
- # We are adding a new item to the select and we need to ensure that
- # existing dependencies from less specialized platforms are propagated
- # to the newly added dependency set.
- for p, _deps in deps_select.items():
- # Check if the existing platform overlaps with the given platform
- if p == platform or platform not in _platform_specializations(p):
- continue
-
- deps_select[platform].update(_deps)
-
-def _maybe_add_common_dep(deps, deps_select, platforms, dep):
- abis = sorted({p.abi: True for p in platforms if p.abi})
- if len(abis) < 2:
- return
-
- platforms = [platform()] + [
- platform(abi = abi)
- for abi in abis
- ]
-
- # If the dep is targeting all target python versions, lets add it to
- # the common dependency list to simplify the select statements.
- for p in platforms:
- if p not in deps_select:
- return
-
- if dep not in deps_select[p]:
- return
-
- # All of the python version-specific branches have the dep, so lets add
- # it to the common deps.
- deps[dep] = True
- for p in platforms:
- deps_select[p].pop(dep)
- if not deps_select[p]:
- deps_select.pop(p)
+ deps_select.setdefault(platform, {})[dep] = True
def _resolve_extras(self_name, reqs, extras):
"""Resolve extras which are due to depending on self[some_other_extra].
@@ -275,77 +190,37 @@
# Poor mans set
return sorted({x: None for x in extras})
-def _add_req(deps, deps_select, req, *, extras, platforms, default_abi = None):
- if not req.marker:
- _add(deps, deps_select, req.name, None)
- return
+def _add_reqs(deps, deps_select, dep, reqs, *, extras, platforms, default_abi = None):
+ for req in reqs:
+ if not req.marker:
+ _add(deps, deps_select, dep, None)
+ return
- # NOTE @aignas 2023-12-08: in order to have reasonable select statements
- # we do have to have some parsing of the markers, so it begs the question
- # if packaging should be reimplemented in Starlark to have the best solution
- # for now we will implement it in Python and see what the best parsing result
- # can be before making this decision.
- match_os = len([
- tag
- for tag in [
- "os_name",
- "sys_platform",
- "platform_system",
- ]
- if tag in req.marker
- ]) > 0
- match_arch = "platform_machine" in req.marker
- match_version = "version" in req.marker
-
- if not (match_os or match_arch or match_version):
- if [
- True
- for extra in extras
- for p in platforms
- if evaluate(
- req.marker,
- env = env(
- target_platform = p,
- extra = extra,
- ),
- )
- ]:
- _add(deps, deps_select, req.name, None)
- return
-
+ platforms_to_add = {}
for plat in platforms:
- if not [
- True
- for extra in extras
- if evaluate(
- req.marker,
- env = env(
- target_platform = plat,
- extra = extra,
- ),
- )
- ]:
+ if plat in platforms_to_add:
+ # marker evaluation is more expensive than this check
continue
- if match_arch and default_abi:
- _add(deps, deps_select, req.name, plat)
- if plat.abi == default_abi:
- _add(deps, deps_select, req.name, platform(os = plat.os, arch = plat.arch))
- elif match_arch:
- _add(deps, deps_select, req.name, platform(os = plat.os, arch = plat.arch))
- elif match_os and default_abi:
- _add(deps, deps_select, req.name, platform(os = plat.os, abi = plat.abi))
- if plat.abi == default_abi:
- _add(deps, deps_select, req.name, platform(os = plat.os))
- elif match_os:
- _add(deps, deps_select, req.name, platform(os = plat.os))
- elif match_version and default_abi:
- _add(deps, deps_select, req.name, platform(abi = plat.abi))
- if plat.abi == default_abi:
- _add(deps, deps_select, req.name, platform())
- elif match_version:
- _add(deps, deps_select, req.name, None)
- else:
- fail("BUG: {} support is not implemented".format(req.marker))
+ added = False
+ for extra in extras:
+ if added:
+ break
- _maybe_add_common_dep(deps, deps_select, platforms, req.name)
+ for req in reqs:
+ if evaluate(req.marker, env = env(target_platform = plat, extra = extra)):
+ platforms_to_add[plat] = True
+ added = True
+ break
+
+ if len(platforms_to_add) == len(platforms):
+ # the dep is in all target platforms, let's just add it to the regular
+ # list
+ _add(deps, deps_select, dep, None)
+ return
+
+ for plat in platforms_to_add:
+ if default_abi:
+ _add(deps, deps_select, dep, plat)
+ if plat.abi == default_abi or not default_abi:
+ _add(deps, deps_select, dep, platform(os = plat.os, arch = plat.arch))
diff --git a/python/private/pypi/pep508_requirement.bzl b/python/private/pypi/pep508_requirement.bzl
index ee7b5df..b5be17f 100644
--- a/python/private/pypi/pep508_requirement.bzl
+++ b/python/private/pypi/pep508_requirement.bzl
@@ -47,9 +47,11 @@
requires, _, _ = requires.partition(char)
extras = extras_unparsed.replace(" ", "").split(",")
name = requires.strip(" ")
+ name = normalize_name(name)
return struct(
- name = normalize_name(name).replace("_", "-"),
+ name = name.replace("_", "-"),
+ name_ = name,
marker = marker.strip(" "),
extras = extras,
version = version,
diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl
index 0a58001..630dc85 100644
--- a/python/private/pypi/whl_library.bzl
+++ b/python/private/pypi/whl_library.bzl
@@ -15,6 +15,7 @@
""
load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth")
+load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")
load("//python/private:envsubst.bzl", "envsubst")
load("//python/private:is_standalone_interpreter.bzl", "is_standalone_interpreter")
load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
@@ -22,13 +23,10 @@
load(":deps.bzl", "all_repo_names", "record_files")
load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel")
load(":parse_requirements.bzl", "host_platform")
-load(":parse_whl_name.bzl", "parse_whl_name")
load(":patch_whl.bzl", "patch_whl")
-load(":pep508_deps.bzl", "deps")
load(":pep508_requirement.bzl", "requirement")
load(":pypi_repo_utils.bzl", "pypi_repo_utils")
load(":whl_metadata.bzl", "whl_metadata")
-load(":whl_target_platforms.bzl", "whl_target_platforms")
_CPPFLAGS = "CPPFLAGS"
_COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools"
@@ -344,20 +342,6 @@
timeout = rctx.attr.timeout,
)
- target_platforms = rctx.attr.experimental_target_platforms
- if target_platforms:
- parsed_whl = parse_whl_name(whl_path.basename)
- if parsed_whl.platform_tag != "any":
- # NOTE @aignas 2023-12-04: if the wheel is a platform specific
- # wheel, we only include deps for that target platform
- target_platforms = [
- p.target_platform
- for p in whl_target_platforms(
- platform_tag = parsed_whl.platform_tag,
- abi_tag = parsed_whl.abi_tag.strip("tm"),
- )
- ]
-
pypi_repo_utils.execute_checked(
rctx,
op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path),
@@ -400,63 +384,45 @@
)
entry_points[entry_point_without_py] = entry_point_script_name
- # TODO @aignas 2025-04-04: move this to whl_library_targets.bzl to have
- # this in the analysis phase.
- #
- # This means that whl_library_targets will have to accept the following args:
- # * name - the name of the package in the METADATA.
- # * requires_dist - the list of METADATA Requires-Dist.
- # * platforms - the list of target platforms. The target_platforms
- # should come from the hub repo via a 'load' statement so that they don't
- # need to be passed as an argument to `whl_library`.
- # * extras - the list of required extras. This comes from the
- # `rctx.attr.requirement` for now. In the future the required extras could
- # stay in the hub repo, where we calculate the extra aliases that we need
- # to create automatically and this way expose the targets for the specific
- # extras. The first step will be to generate a target per extra for the
- # `py_library` and `filegroup`. Maybe we need to have a special provider
- # or an output group so that we can return the `whl` file from the
- # `py_library` target? filegroup can use output groups to expose files.
- # * host_python_version/versons - the list of python versions to support
- # should come from the hub, similar to how the target platforms are specified.
- #
- # Extra things that we should move at the same time:
- # * group_name, group_deps - this info can stay in the hub repository so that
- # it is piped at the analysis time and changing the requirement groups does
- # cause to re-fetch the deps.
- python_version = metadata["python_version"]
+ if BZLMOD_ENABLED:
+ # The following attributes are unset on bzlmod and we pass data through
+ # the hub via load statements.
+ default_python_version = None
+ target_platforms = []
+ else:
+ # NOTE @aignas 2025-04-16: if BZLMOD_ENABLED, we should use
+ # DEFAULT_PYTHON_VERSION since platforms always come with the actual
+ # python version otherwise we should use the version of the interpreter
+ # here. In WORKSPACE `multi_pip_parse` is using an interpreter for each
+ # `pip_parse` invocation, so we will have the host target platform
+ # only. Even if somebody would change the code to support
+ # `experimental_target_platforms`, they would be for a single python
+ # version. Hence, using the `default_python_version` that we get from the
+ # interpreter is correct. Hence, we unset the argument if we are on bzlmod.
+ default_python_version = metadata["python_version"]
+ target_platforms = rctx.attr.experimental_target_platforms or [host_platform(rctx)]
+
metadata = whl_metadata(
install_dir = rctx.path("site-packages"),
read_fn = rctx.read,
logger = logger,
)
- # TODO @aignas 2025-04-09: this will later be removed when loaded through the hub
- major_minor, _, _ = python_version.rpartition(".")
- package_deps = deps(
- name = metadata.name,
- requires_dist = metadata.requires_dist,
- platforms = target_platforms or [
- "cp{}_{}".format(major_minor.replace(".", ""), host_platform(rctx)),
- ],
- extras = requirement(rctx.attr.requirement).extras,
- host_python_version = python_version,
- )
-
build_file_contents = generate_whl_library_build_bazel(
name = whl_path.basename,
+ metadata_name = metadata.name,
+ metadata_version = metadata.version,
+ requires_dist = metadata.requires_dist,
dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix),
- dependencies = package_deps.deps,
- dependencies_by_platform = package_deps.deps_select,
- group_name = rctx.attr.group_name,
- group_deps = rctx.attr.group_deps,
- data_exclude = rctx.attr.pip_data_exclude,
- tags = [
- "pypi_name=" + metadata.name,
- "pypi_version=" + metadata.version,
- ],
entry_points = entry_points,
+ target_platforms = target_platforms,
+ default_python_version = default_python_version,
+ # TODO @aignas 2025-04-14: load through the hub:
annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))),
+ data_exclude = rctx.attr.pip_data_exclude,
+ extras = requirement(rctx.attr.requirement).extras,
+ group_deps = rctx.attr.group_deps,
+ group_name = rctx.attr.group_name,
)
rctx.file("BUILD.bazel", build_file_contents)
@@ -517,10 +483,7 @@
doc = "Name of the group, if any.",
),
"repo": attr.string(
- doc = """\
-Pointer to parent repo name. Used to make these rules rerun if the parent repo changes.
-Only used in WORKSPACE when the {attr}`dep_template` is not set.
-""",
+ doc = "Pointer to parent repo name. Used to make these rules rerun if the parent repo changes.",
),
"repo_prefix": attr.string(
doc = """
diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl
index d32746b..cf3df13 100644
--- a/python/private/pypi/whl_library_targets.bzl
+++ b/python/private/pypi/whl_library_targets.bzl
@@ -29,6 +29,89 @@
"WHEEL_FILE_IMPL_LABEL",
"WHEEL_FILE_PUBLIC_LABEL",
)
+load(":parse_whl_name.bzl", "parse_whl_name")
+load(":pep508_deps.bzl", "deps")
+load(":whl_target_platforms.bzl", "whl_target_platforms")
+
+def whl_library_targets_from_requires(
+ *,
+ name,
+ metadata_name = "",
+ metadata_version = "",
+ requires_dist = [],
+ extras = [],
+ target_platforms = [],
+ default_python_version = None,
+ group_deps = [],
+ **kwargs):
+ """The macro to create whl targets from the METADATA.
+
+ Args:
+ name: {type}`str` The wheel filename
+ metadata_name: {type}`str` The package name as written in wheel `METADATA`.
+ metadata_version: {type}`str` The package version as written in wheel `METADATA`.
+ group_deps: {type}`list[str]` names of fellow members of the group (if
+ any). These will be excluded from generated deps lists so as to avoid
+ direct cycles. These dependencies will be provided at runtime by the
+ group rules which wrap this library and its fellows together.
+ requires_dist: {type}`list[str]` The list of `Requires-Dist` values from
+ the whl `METADATA`.
+ extras: {type}`list[str]` The list of requested extras. This essentially includes extra transitive dependencies in the final targets depending on the wheel `METADATA`.
+ target_platforms: {type}`list[str]` The list of target platforms to create
+ dependency closures for.
+ default_python_version: {type}`str` The python version to assume when parsing
+ the `METADATA`. This is only used when the `target_platforms` do not
+ include the version information.
+ **kwargs: Extra args passed to the {obj}`whl_library_targets`
+ """
+ package_deps = _parse_requires_dist(
+ name = name,
+ default_python_version = default_python_version,
+ requires_dist = requires_dist,
+ excludes = group_deps,
+ extras = extras,
+ target_platforms = target_platforms,
+ )
+ whl_library_targets(
+ name = name,
+ dependencies = package_deps.deps,
+ dependencies_by_platform = package_deps.deps_select,
+ tags = [
+ "pypi_name={}".format(metadata_name),
+ "pypi_version={}".format(metadata_version),
+ ],
+ **kwargs
+ )
+
+def _parse_requires_dist(
+ *,
+ name,
+ default_python_version,
+ requires_dist,
+ excludes,
+ extras,
+ target_platforms):
+ parsed_whl = parse_whl_name(name)
+
+ # NOTE @aignas 2023-12-04: if the wheel is a platform specific wheel, we
+ # only include deps for that target platform
+ if parsed_whl.platform_tag != "any":
+ target_platforms = [
+ p.target_platform
+ for p in whl_target_platforms(
+ platform_tag = parsed_whl.platform_tag,
+ abi_tag = parsed_whl.abi_tag.strip("tm"),
+ )
+ ]
+
+ return deps(
+ name = normalize_name(parsed_whl.distribution),
+ requires_dist = requires_dist,
+ platforms = target_platforms,
+ excludes = excludes,
+ extras = extras,
+ default_python_version = default_python_version,
+ )
def whl_library_targets(
*,
diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl
index 4d86d6a..ce5474e 100644
--- a/tests/pypi/extension/extension_tests.bzl
+++ b/tests/pypi/extension/extension_tests.bzl
@@ -436,7 +436,6 @@
pypi.whl_libraries().contains_exactly({
"pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": {
"dep_template": "@pypi//{name}:{target}",
- "experimental_target_platforms": ["cp312_linux_x86_64"],
"filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl",
"python_interpreter_target": "unit_test_interpreter_target",
"requirement": "torch==2.4.1+cpu",
@@ -445,7 +444,6 @@
},
"pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": {
"dep_template": "@pypi//{name}:{target}",
- "experimental_target_platforms": ["cp312_linux_aarch64"],
"filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
"python_interpreter_target": "unit_test_interpreter_target",
"requirement": "torch==2.4.1",
@@ -454,7 +452,6 @@
},
"pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": {
"dep_template": "@pypi//{name}:{target}",
- "experimental_target_platforms": ["cp312_windows_x86_64"],
"filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl",
"python_interpreter_target": "unit_test_interpreter_target",
"requirement": "torch==2.4.1+cpu",
@@ -463,7 +460,6 @@
},
"pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": {
"dep_template": "@pypi//{name}:{target}",
- "experimental_target_platforms": ["cp312_osx_aarch64"],
"filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl",
"python_interpreter_target": "unit_test_interpreter_target",
"requirement": "torch==2.4.1",
@@ -750,7 +746,6 @@
pypi.whl_libraries().contains_exactly({
"pypi_315_any_name": {
"dep_template": "@pypi//{name}:{target}",
- "experimental_target_platforms": ["cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"],
"extra_pip_args": ["--extra-args-for-sdist-building"],
"filename": "any-name.tar.gz",
"python_interpreter_target": "unit_test_interpreter_target",
@@ -760,7 +755,6 @@
},
"pypi_315_direct_without_sha_0_0_1_py3_none_any": {
"dep_template": "@pypi//{name}:{target}",
- "experimental_target_platforms": ["cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"],
"filename": "direct_without_sha-0.0.1-py3-none-any.whl",
"python_interpreter_target": "unit_test_interpreter_target",
"requirement": "direct_without_sha==0.0.1 @ example-direct.org/direct_without_sha-0.0.1-py3-none-any.whl",
@@ -781,7 +775,6 @@
},
"pypi_315_simple_py3_none_any_deadb00f": {
"dep_template": "@pypi//{name}:{target}",
- "experimental_target_platforms": ["cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"],
"filename": "simple-0.0.1-py3-none-any.whl",
"python_interpreter_target": "unit_test_interpreter_target",
"requirement": "simple==0.0.1",
@@ -790,7 +783,6 @@
},
"pypi_315_simple_sdist_deadbeef": {
"dep_template": "@pypi//{name}:{target}",
- "experimental_target_platforms": ["cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"],
"extra_pip_args": ["--extra-args-for-sdist-building"],
"filename": "simple-0.0.1.tar.gz",
"python_interpreter_target": "unit_test_interpreter_target",
@@ -800,7 +792,6 @@
},
"pypi_315_some_pkg_py3_none_any_deadbaaf": {
"dep_template": "@pypi//{name}:{target}",
- "experimental_target_platforms": ["cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"],
"filename": "some_pkg-0.0.1-py3-none-any.whl",
"python_interpreter_target": "unit_test_interpreter_target",
"requirement": "some_pkg==0.0.1 @ example-direct.org/some_pkg-0.0.1-py3-none-any.whl --hash=sha256:deadbaaf",
@@ -809,7 +800,6 @@
},
"pypi_315_some_py3_none_any_deadb33f": {
"dep_template": "@pypi//{name}:{target}",
- "experimental_target_platforms": ["cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"],
"filename": "some-other-pkg-0.0.1-py3-none-any.whl",
"python_interpreter_target": "unit_test_interpreter_target",
"requirement": "some_other_pkg==0.0.1",
diff --git a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl
index b0d8f6d..7bd19b6 100644
--- a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl
+++ b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl
@@ -21,11 +21,11 @@
def _test_all(env):
want = """\
-load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets")
+load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets_from_requires")
package(default_visibility = ["//visibility:public"])
-whl_library_targets(
+whl_library_targets_from_requires(
copy_executables = {
"exec_src": "exec_dest",
},
@@ -38,19 +38,6 @@
"data_exclude_all",
],
dep_template = "@pypi//{name}:{target}",
- dependencies = [
- "foo",
- "bar-baz",
- "qux",
- ],
- dependencies_by_platform = {
- "linux_x86_64": [
- "box",
- "box-amd64",
- ],
- "windows_x86_64": ["fox"],
- "@platforms//os:linux": ["box"],
- },
entry_points = {
"foo": "bar.py",
},
@@ -61,11 +48,13 @@
],
group_name = "qux",
name = "foo.whl",
- srcs_exclude = ["srcs_exclude_all"],
- tags = [
- "tag2",
- "tag1",
+ requires_dist = [
+ "foo",
+ "bar-baz",
+ "qux",
],
+ srcs_exclude = ["srcs_exclude_all"],
+ target_platforms = ["foo"],
)
# SOMETHING SPECIAL AT THE END
@@ -73,13 +62,72 @@
actual = generate_whl_library_build_bazel(
dep_template = "@pypi//{name}:{target}",
name = "foo.whl",
- dependencies = ["foo", "bar-baz", "qux"],
- dependencies_by_platform = {
- "linux_x86_64": ["box", "box-amd64"],
- "windows_x86_64": ["fox"],
- "@platforms//os:linux": ["box"], # buildifier: disable=unsorted-dict-items to check that we sort inside the test
+ requires_dist = ["foo", "bar-baz", "qux"],
+ entry_points = {
+ "foo": "bar.py",
},
- tags = ["tag2", "tag1"],
+ data_exclude = ["exclude_via_attr"],
+ annotation = struct(
+ copy_files = {"file_src": "file_dest"},
+ copy_executables = {"exec_src": "exec_dest"},
+ data = ["extra_target"],
+ data_exclude_glob = ["data_exclude_all"],
+ srcs_exclude_glob = ["srcs_exclude_all"],
+ additive_build_content = """# SOMETHING SPECIAL AT THE END""",
+ ),
+ group_name = "qux",
+ target_platforms = ["foo"],
+ group_deps = ["foo", "fox", "qux"],
+ )
+ env.expect.that_str(actual.replace("@@", "@")).equals(want)
+
+_tests.append(_test_all)
+
+def _test_all_with_loads(env):
+ want = """\
+load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets_from_requires")
+load("@pypi//:config.bzl", "target_platforms")
+
+package(default_visibility = ["//visibility:public"])
+
+whl_library_targets_from_requires(
+ copy_executables = {
+ "exec_src": "exec_dest",
+ },
+ copy_files = {
+ "file_src": "file_dest",
+ },
+ data = ["extra_target"],
+ data_exclude = [
+ "exclude_via_attr",
+ "data_exclude_all",
+ ],
+ dep_template = "@pypi//{name}:{target}",
+ entry_points = {
+ "foo": "bar.py",
+ },
+ group_deps = [
+ "foo",
+ "fox",
+ "qux",
+ ],
+ group_name = "qux",
+ name = "foo.whl",
+ requires_dist = [
+ "foo",
+ "bar-baz",
+ "qux",
+ ],
+ srcs_exclude = ["srcs_exclude_all"],
+ target_platforms = target_platforms,
+)
+
+# SOMETHING SPECIAL AT THE END
+"""
+ actual = generate_whl_library_build_bazel(
+ dep_template = "@pypi//{name}:{target}",
+ name = "foo.whl",
+ requires_dist = ["foo", "bar-baz", "qux"],
entry_points = {
"foo": "bar.py",
},
@@ -97,7 +145,7 @@
)
env.expect.that_str(actual.replace("@@", "@")).equals(want)
-_tests.append(_test_all)
+_tests.append(_test_all_with_loads)
def generate_whl_library_build_bazel_test_suite(name):
"""Create the test suite.
diff --git a/tests/pypi/pep508/deps_tests.bzl b/tests/pypi/pep508/deps_tests.bzl
index 44031ab..d362925 100644
--- a/tests/pypi/pep508/deps_tests.bzl
+++ b/tests/pypi/pep508/deps_tests.bzl
@@ -29,58 +29,48 @@
_tests.append(test_simple_deps)
def test_can_add_os_specific_deps(env):
- got = deps(
- "foo",
- requires_dist = [
- "bar",
- "an_osx_dep; sys_platform=='darwin'",
- "posix_dep; os_name=='posix'",
- "win_dep; os_name=='nt'",
- ],
- platforms = [
- "linux_x86_64",
- "osx_x86_64",
- "osx_aarch64",
- "windows_x86_64",
- ],
- host_python_version = "3.3.1",
- )
+ for target in [
+ struct(
+ platforms = [
+ "linux_x86_64",
+ "osx_x86_64",
+ "osx_aarch64",
+ "windows_x86_64",
+ ],
+ python_version = "3.3.1",
+ ),
+ struct(
+ platforms = [
+ "cp33_linux_x86_64",
+ "cp33_osx_x86_64",
+ "cp33_osx_aarch64",
+ "cp33_windows_x86_64",
+ ],
+ python_version = "",
+ ),
+ ]:
+ got = deps(
+ "foo",
+ requires_dist = [
+ "bar",
+ "an_osx_dep; sys_platform=='darwin'",
+ "posix_dep; os_name=='posix'",
+ "win_dep; os_name=='nt'",
+ ],
+ platforms = target.platforms,
+ default_python_version = target.python_version,
+ )
- env.expect.that_collection(got.deps).contains_exactly(["bar"])
- env.expect.that_dict(got.deps_select).contains_exactly({
- "@platforms//os:linux": ["posix_dep"],
- "@platforms//os:osx": ["an_osx_dep", "posix_dep"],
- "@platforms//os:windows": ["win_dep"],
- })
+ env.expect.that_collection(got.deps).contains_exactly(["bar"])
+ env.expect.that_dict(got.deps_select).contains_exactly({
+ "linux_x86_64": ["posix_dep"],
+ "osx_aarch64": ["an_osx_dep", "posix_dep"],
+ "osx_x86_64": ["an_osx_dep", "posix_dep"],
+ "windows_x86_64": ["win_dep"],
+ })
_tests.append(test_can_add_os_specific_deps)
-def test_can_add_os_specific_deps_with_python_version(env):
- got = deps(
- "foo",
- requires_dist = [
- "bar",
- "an_osx_dep; sys_platform=='darwin'",
- "posix_dep; os_name=='posix'",
- "win_dep; os_name=='nt'",
- ],
- platforms = [
- "cp33_linux_x86_64",
- "cp33_osx_x86_64",
- "cp33_osx_aarch64",
- "cp33_windows_x86_64",
- ],
- )
-
- env.expect.that_collection(got.deps).contains_exactly(["bar"])
- env.expect.that_dict(got.deps_select).contains_exactly({
- "@platforms//os:linux": ["posix_dep"],
- "@platforms//os:osx": ["an_osx_dep", "posix_dep"],
- "@platforms//os:windows": ["win_dep"],
- })
-
-_tests.append(test_can_add_os_specific_deps_with_python_version)
-
def test_deps_are_added_to_more_specialized_platforms(env):
got = deps(
"foo",
@@ -92,41 +82,16 @@
"osx_x86_64",
"osx_aarch64",
],
- host_python_version = "3.8.4",
+ default_python_version = "3.8.4",
)
- env.expect.that_collection(got.deps).contains_exactly([])
+ env.expect.that_collection(got.deps).contains_exactly(["mac_dep"])
env.expect.that_dict(got.deps_select).contains_exactly({
- "@platforms//os:osx": ["mac_dep"],
- "osx_aarch64": ["m1_dep", "mac_dep"],
+ "osx_aarch64": ["m1_dep"],
})
_tests.append(test_deps_are_added_to_more_specialized_platforms)
-def test_deps_from_more_specialized_platforms_are_propagated(env):
- got = deps(
- "foo",
- requires_dist = [
- "a_mac_dep; sys_platform=='darwin'",
- "m1_dep; sys_platform=='darwin' and platform_machine=='arm64'",
- ],
- platforms = [
- "osx_x86_64",
- "osx_aarch64",
- ],
- host_python_version = "3.8.4",
- )
-
- env.expect.that_collection(got.deps).contains_exactly([])
- env.expect.that_dict(got.deps_select).contains_exactly(
- {
- "@platforms//os:osx": ["a_mac_dep"],
- "osx_aarch64": ["a_mac_dep", "m1_dep"],
- },
- )
-
-_tests.append(test_deps_from_more_specialized_platforms_are_propagated)
-
def test_non_platform_markers_are_added_to_common_deps(env):
got = deps(
"foo",
@@ -141,7 +106,7 @@
"osx_aarch64",
"windows_x86_64",
],
- host_python_version = "3.8.4",
+ default_python_version = "3.8.4",
)
env.expect.that_collection(got.deps).contains_exactly(["bar", "baz"])
@@ -204,38 +169,34 @@
platforms = ["cp37_linux_x86_64"],
)
+ # since there is a single target platform, the deps_select will be empty
env.expect.that_collection(py37.deps).contains_exactly(["bar", "baz"])
env.expect.that_dict(py37.deps_select).contains_exactly({})
- env.expect.that_collection(py38.deps).contains_exactly(["bar"])
- env.expect.that_dict(py38.deps_select).contains_exactly({"@platforms//os:linux": ["posix_dep"]})
+ env.expect.that_collection(py38.deps).contains_exactly(["bar", "posix_dep"])
+ env.expect.that_dict(py38.deps_select).contains_exactly({})
_tests.append(_test_can_get_deps_based_on_specific_python_version)
def _test_no_version_select_when_single_version(env):
- requires_dist = [
- "bar",
- "baz; python_version >= '3.8'",
- "posix_dep; os_name=='posix'",
- "posix_dep_with_version; os_name=='posix' and python_version >= '3.8'",
- "arch_dep; platform_machine=='x86_64' and python_version >= '3.8'",
- ]
- host_python_version = "3.7.5"
-
got = deps(
"foo",
- requires_dist = requires_dist,
+ requires_dist = [
+ "bar",
+ "baz; python_version >= '3.8'",
+ "posix_dep; os_name=='posix'",
+ "posix_dep_with_version; os_name=='posix' and python_version >= '3.8'",
+ "arch_dep; platform_machine=='x86_64' and python_version >= '3.8'",
+ ],
platforms = [
"cp38_linux_x86_64",
"cp38_windows_x86_64",
],
- host_python_version = host_python_version,
+ default_python_version = "",
)
- env.expect.that_collection(got.deps).contains_exactly(["bar", "baz"])
+ env.expect.that_collection(got.deps).contains_exactly(["bar", "baz", "arch_dep"])
env.expect.that_dict(got.deps_select).contains_exactly({
- "@platforms//os:linux": ["posix_dep", "posix_dep_with_version"],
- "linux_x86_64": ["arch_dep", "posix_dep", "posix_dep_with_version"],
- "windows_x86_64": ["arch_dep"],
+ "linux_x86_64": ["posix_dep", "posix_dep_with_version"],
})
_tests.append(_test_no_version_select_when_single_version)
@@ -249,7 +210,7 @@
"posix_dep_with_version; os_name=='posix' and python_version >= '3.8'",
"arch_dep; platform_machine=='x86_64' and python_version < '3.8'",
]
- host_python_version = "3.7.4"
+ default_python_version = "3.7.4"
got = deps(
"foo",
@@ -259,31 +220,19 @@
for minor in [7, 8, 9]
for os in ["linux", "windows"]
],
- host_python_version = host_python_version,
+ default_python_version = default_python_version,
)
env.expect.that_collection(got.deps).contains_exactly(["bar"])
env.expect.that_dict(got.deps_select).contains_exactly({
- str(Label("//python/config_settings:is_python_3.7")): ["baz"],
- str(Label("//python/config_settings:is_python_3.8")): ["baz_new"],
- str(Label("//python/config_settings:is_python_3.9")): ["baz_new"],
- "@platforms//os:linux": ["baz", "posix_dep"],
- "cp37_linux_anyarch": ["baz", "posix_dep"],
"cp37_linux_x86_64": ["arch_dep", "baz", "posix_dep"],
"cp37_windows_x86_64": ["arch_dep", "baz"],
- "cp38_linux_anyarch": [
- "baz_new",
- "posix_dep",
- "posix_dep_with_version",
- ],
- "cp39_linux_anyarch": [
- "baz_new",
- "posix_dep",
- "posix_dep_with_version",
- ],
+ "cp38_linux_x86_64": ["baz_new", "posix_dep", "posix_dep_with_version"],
+ "cp38_windows_x86_64": ["baz_new"],
+ "cp39_linux_x86_64": ["baz_new", "posix_dep", "posix_dep_with_version"],
+ "cp39_windows_x86_64": ["baz_new"],
"linux_x86_64": ["arch_dep", "baz", "posix_dep"],
"windows_x86_64": ["arch_dep", "baz"],
- "//conditions:default": ["baz"],
})
_tests.append(_test_can_get_version_select)
@@ -294,7 +243,7 @@
"baz (<2,>=1.11) ; python_version < '3.8'",
"baz (<2,>=1.14) ; python_version >= '3.8'",
]
- host_python_version = "3.8.4"
+ default_python_version = "3.8.4"
got = deps(
"foo",
@@ -303,7 +252,7 @@
"cp3{}_linux_x86_64".format(minor)
for minor in [7, 8, 9]
],
- host_python_version = host_python_version,
+ default_python_version = default_python_version,
)
env.expect.that_collection(got.deps).contains_exactly(["bar", "baz"])
@@ -312,7 +261,7 @@
_tests.append(_test_deps_spanning_all_target_py_versions_are_added_to_common)
def _test_deps_are_not_duplicated(env):
- host_python_version = "3.7.4"
+ default_python_version = "3.7.4"
# See an example in
# https://files.pythonhosted.org/packages/76/9e/db1c2d56c04b97981c06663384f45f28950a73d9acf840c4006d60d0a1ff/opencv_python-4.9.0.80-cp37-abi3-win32.whl.metadata
@@ -336,7 +285,7 @@
for os in ["linux", "osx", "windows"]
for arch in ["x86_64", "aarch64"]
],
- host_python_version = host_python_version,
+ default_python_version = default_python_version,
)
env.expect.that_collection(got.deps).contains_exactly(["bar"])
@@ -345,7 +294,7 @@
_tests.append(_test_deps_are_not_duplicated)
def _test_deps_are_not_duplicated_when_encountering_platform_dep_first(env):
- host_python_version = "3.7.1"
+ default_python_version = "3.7.1"
# Note, that we are sorting the incoming `requires_dist` and we need to ensure that we are not getting any
# issues even if the platform-specific line comes first.
@@ -363,15 +312,13 @@
"cp310_linux_aarch64",
"cp310_linux_x86_64",
],
- host_python_version = host_python_version,
+ default_python_version = default_python_version,
)
- # TODO @aignas 2025-02-24: this test case in the python version is passing but
- # I am not sure why. The starlark version behaviour looks more correct.
env.expect.that_collection(got.deps).contains_exactly([])
env.expect.that_dict(got.deps_select).contains_exactly({
- str(Label("//python/config_settings:is_python_3.10")): ["bar"],
"cp310_linux_aarch64": ["bar"],
+ "cp310_linux_x86_64": ["bar"],
"cp37_linux_aarch64": ["bar"],
"linux_aarch64": ["bar"],
})
diff --git a/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl b/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl
index f738e03..61e5441 100644
--- a/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl
+++ b/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl
@@ -16,7 +16,7 @@
load("@rules_testing//lib:test_suite.bzl", "test_suite")
load("//python/private:glob_excludes.bzl", "glob_excludes") # buildifier: disable=bzl-visibility
-load("//python/private/pypi:whl_library_targets.bzl", "whl_library_targets") # buildifier: disable=bzl-visibility
+load("//python/private/pypi:whl_library_targets.bzl", "whl_library_targets", "whl_library_targets_from_requires") # buildifier: disable=bzl-visibility
_tests = []
@@ -183,6 +183,71 @@
_tests.append(_test_entrypoints)
+def _test_whl_and_library_deps_from_requires(env):
+ filegroup_calls = []
+ py_library_calls = []
+
+ whl_library_targets_from_requires(
+ name = "foo-0-py3-none-any.whl",
+ metadata_name = "Foo",
+ metadata_version = "0",
+ dep_template = "@pypi_{name}//:{target}",
+ requires_dist = [
+ "foo", # this self-edge will be ignored
+ "bar-baz",
+ ],
+ target_platforms = ["cp38_linux_x86_64"],
+ default_python_version = "3.8.1",
+ data_exclude = [],
+ # Overrides for testing
+ filegroups = {},
+ native = struct(
+ filegroup = lambda **kwargs: filegroup_calls.append(kwargs),
+ config_setting = lambda **_: None,
+ glob = _glob,
+ select = _select,
+ ),
+ rules = struct(
+ py_library = lambda **kwargs: py_library_calls.append(kwargs),
+ ),
+ )
+
+ env.expect.that_collection(filegroup_calls).contains_exactly([
+ {
+ "name": "whl",
+ "srcs": ["foo-0-py3-none-any.whl"],
+ "data": ["@pypi_bar_baz//:whl"],
+ "visibility": ["//visibility:public"],
+ },
+ ]) # buildifier: @unsorted-dict-items
+ env.expect.that_collection(py_library_calls).contains_exactly([
+ {
+ "name": "pkg",
+ "srcs": _glob(
+ ["site-packages/**/*.py"],
+ exclude = [],
+ allow_empty = True,
+ ),
+ "pyi_srcs": _glob(["site-packages/**/*.pyi"], allow_empty = True),
+ "data": [] + _glob(
+ ["site-packages/**/*"],
+ exclude = [
+ "**/*.py",
+ "**/*.pyc",
+ "**/*.pyc.*",
+ "**/*.dist-info/RECORD",
+ ] + glob_excludes.version_dependent_exclusions(),
+ ),
+ "imports": ["site-packages"],
+ "deps": ["@pypi_bar_baz//:pkg"],
+ "tags": ["pypi_name=Foo", "pypi_version=0"],
+ "visibility": ["//visibility:public"],
+ "experimental_venvs_site_packages": Label("//python/config_settings:venvs_site_packages"),
+ },
+ ]) # buildifier: @unsorted-dict-items
+
+_tests.append(_test_whl_and_library_deps_from_requires)
+
def _test_whl_and_library_deps(env):
filegroup_calls = []
py_library_calls = []
