refactor(pypi): translate wheel METADATA parsing to starlark (#2629)
This PR starts using the newly introduced (#2692) PEP508 compliant
requirement marker parser in starlark and moves the dependency
generation from the Python language (`whl_installer`) to the Starlark
in the `whl_library` repository rule.
This PR is (almost) a pure refactor where no bugs are fixed, but this is
foundational work that also adds notes on how things will be moved
to macros (i.e. analysis phase) so that we can fix a few long standing
bugs and prepare for stabilizing the `experimental_index_url` (#260).
Refactor:
* I have migrated all of the unit tests from Python to starlark for deps
generation from METADATA `Requires-Dist` fields.
* Read the `METADATA` file itself in Starlark.
Work towards #260, #2319, #2241
Fixes #2423
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
index e0a2f20..7297238 100644
--- a/python/private/pypi/BUILD.bazel
+++ b/python/private/pypi/BUILD.bazel
@@ -222,6 +222,18 @@
)
bzl_library(
+ name = "pep508_deps_bzl",
+ srcs = ["pep508_deps.bzl"],
+ deps = [
+ ":pep508_env_bzl",
+ ":pep508_evaluate_bzl",
+ ":pep508_platform_bzl",
+ ":pep508_requirement_bzl",
+ "//python/private:normalize_name_bzl",
+ ],
+)
+
+bzl_library(
name = "pep508_env_bzl",
srcs = ["pep508_env.bzl"],
deps = [
@@ -368,7 +380,9 @@
":generate_whl_library_build_bazel_bzl",
":parse_whl_name_bzl",
":patch_whl_bzl",
+ ":pep508_deps_bzl",
":pypi_repo_utils_bzl",
+ ":whl_metadata_bzl",
":whl_target_platforms_bzl",
"//python/private:auth_bzl",
"//python/private:envsubst_bzl",
@@ -378,6 +392,11 @@
)
bzl_library(
+ name = "whl_metadata_bzl",
+ srcs = ["whl_metadata.bzl"],
+)
+
+bzl_library(
name = "whl_repo_name_bzl",
srcs = ["whl_repo_name.bzl"],
deps = [
diff --git a/python/private/pypi/pep508_deps.bzl b/python/private/pypi/pep508_deps.bzl
new file mode 100644
index 0000000..af0a753
--- /dev/null
+++ b/python/private/pypi/pep508_deps.bzl
@@ -0,0 +1,351 @@
+# Copyright 2025 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This module is for implementing PEP508 compliant METADATA deps parsing.
+"""
+
+load("//python/private:normalize_name.bzl", "normalize_name")
+load(":pep508_env.bzl", "env")
+load(":pep508_evaluate.bzl", "evaluate")
+load(":pep508_platform.bzl", "platform", "platform_from_str")
+load(":pep508_requirement.bzl", "requirement")
+
+_ALL_OS_VALUES = [
+ "windows",
+ "osx",
+ "linux",
+]
+_ALL_ARCH_VALUES = [
+ "aarch64",
+ "ppc64",
+ "ppc64le",
+ "s390x",
+ "x86_32",
+ "x86_64",
+]
+
+def deps(name, *, requires_dist, platforms = [], extras = [], host_python_version = None):
+ """Parse the RequiresDist from wheel METADATA
+
+ Args:
+ name: {type}`str` the name of the wheel.
+ requires_dist: {type}`list[str]` the list of RequiresDist lines from the
+ METADATA file.
+ extras: {type}`list[str]` the requested extras to generate targets for.
+ platforms: {type}`list[str]` the list of target platform strings.
+ host_python_version: {type}`str` the host python version.
+
+ Returns:
+ A struct with attributes:
+ * deps: {type}`list[str]` dependencies to include unconditionally.
+ * deps_select: {type}`dict[str, list[str]]` dependencies to include on particular
+ subset of target platforms.
+ """
+ reqs = sorted(
+ [requirement(r) for r in requires_dist],
+ key = lambda x: "{}:{}:".format(x.name, sorted(x.extras), x.marker),
+ )
+ deps = {}
+ deps_select = {}
+ name = normalize_name(name)
+ want_extras = _resolve_extras(name, reqs, extras)
+
+ # drop self edges
+ reqs = [r for r in reqs if r.name != name]
+
+ platforms = [
+ platform_from_str(p, python_version = host_python_version)
+ for p in platforms
+ ] or [
+ platform_from_str("", python_version = host_python_version),
+ ]
+
+ abis = sorted({p.abi: True for p in platforms if p.abi})
+ if host_python_version and len(abis) > 1:
+ _, _, minor_version = host_python_version.partition(".")
+ minor_version, _, _ = minor_version.partition(".")
+ default_abi = "cp3" + minor_version
+ elif len(abis) > 1:
+ fail(
+ "all python versions need to be specified explicitly, got: {}".format(platforms),
+ )
+ else:
+ default_abi = None
+
+ for req in reqs:
+ _add_req(
+ deps,
+ deps_select,
+ req,
+ extras = want_extras,
+ platforms = platforms,
+ default_abi = default_abi,
+ )
+
+ return struct(
+ deps = sorted(deps),
+ deps_select = {
+ _platform_str(p): sorted(deps)
+ for p, deps in deps_select.items()
+ },
+ )
+
+def _platform_str(self):
+ if self.abi == None:
+ if not self.os and not self.arch:
+ return "//conditions:default"
+ elif not self.arch:
+ return "@platforms//os:{}".format(self.os)
+ else:
+ return "{}_{}".format(self.os, self.arch)
+
+ minor_version = self.abi[3:]
+ if self.arch == None and self.os == None:
+ return str(Label("//python/config_settings:is_python_3.{}".format(minor_version)))
+
+ return "cp3{}_{}_{}".format(
+ minor_version,
+ self.os or "anyos",
+ self.arch or "anyarch",
+ )
+
+def _platform_specializations(self, cpu_values = _ALL_ARCH_VALUES, os_values = _ALL_OS_VALUES):
+ """Return the platform itself and all its unambiguous specializations.
+
+ For more info about specializations see
+ https://bazel.build/docs/configurable-attributes
+ """
+ specializations = []
+ specializations.append(self)
+ if self.arch == None:
+ specializations.extend([
+ platform(os = self.os, arch = arch, abi = self.abi)
+ for arch in cpu_values
+ ])
+ if self.os == None:
+ specializations.extend([
+ platform(os = os, arch = self.arch, abi = self.abi)
+ for os in os_values
+ ])
+ if self.os == None and self.arch == None:
+ specializations.extend([
+ platform(os = os, arch = arch, abi = self.abi)
+ for os in os_values
+ for arch in cpu_values
+ ])
+ return specializations
+
+def _add(deps, deps_select, dep, platform):
+ dep = normalize_name(dep)
+
+ if platform == None:
+ deps[dep] = True
+
+ # If the dep is in the platform-specific list, remove it from the select.
+ pop_keys = []
+ for p, _deps in deps_select.items():
+ if dep not in _deps:
+ continue
+
+ _deps.pop(dep)
+ if not _deps:
+ pop_keys.append(p)
+
+ for p in pop_keys:
+ deps_select.pop(p)
+ return
+
+ if dep in deps:
+ # If the dep is already in the main dependency list, no need to add it in the
+ # platform-specific dependency list.
+ return
+
+ # Add the platform-specific branch
+ deps_select.setdefault(platform, {})
+
+ # Add the dep to specializations of the given platform if they
+ # exist in the select statement.
+ for p in _platform_specializations(platform):
+ if p not in deps_select:
+ continue
+
+ deps_select[p][dep] = True
+
+ if len(deps_select[platform]) == 1:
+ # We are adding a new item to the select and we need to ensure that
+ # existing dependencies from less specialized platforms are propagated
+ # to the newly added dependency set.
+ for p, _deps in deps_select.items():
+ # Check if the existing platform overlaps with the given platform
+ if p == platform or platform not in _platform_specializations(p):
+ continue
+
+ deps_select[platform].update(_deps)
+
+def _maybe_add_common_dep(deps, deps_select, platforms, dep):
+ abis = sorted({p.abi: True for p in platforms if p.abi})
+ if len(abis) < 2:
+ return
+
+ platforms = [platform()] + [
+ platform(abi = abi)
+ for abi in abis
+ ]
+
+ # If the dep is targeting all target python versions, lets add it to
+ # the common dependency list to simplify the select statements.
+ for p in platforms:
+ if p not in deps_select:
+ return
+
+ if dep not in deps_select[p]:
+ return
+
+ # All of the python version-specific branches have the dep, so lets add
+ # it to the common deps.
+ deps[dep] = True
+ for p in platforms:
+ deps_select[p].pop(dep)
+ if not deps_select[p]:
+ deps_select.pop(p)
+
+def _resolve_extras(self_name, reqs, extras):
+ """Resolve extras which are due to depending on self[some_other_extra].
+
+ Some packages may have cyclic dependencies resulting from extras being used, one example is
+ `etils`, where we have one set of extras as aliases for other extras
+ and we have an extra called 'all' that includes all other extras.
+
+ Example: github.com/google/etils/blob/a0b71032095db14acf6b33516bca6d885fe09e35/pyproject.toml#L32.
+
+ When the `requirements.txt` is generated by `pip-tools`, then it is likely that
+ this step is not needed, but for other `requirements.txt` files this may be useful.
+
+ NOTE @aignas 2023-12-08: the extra resolution is not platform dependent,
+ but in order for it to become platform dependent we would have to have
+ separate targets for each extra in extras.
+ """
+
+ # Resolve any extra extras due to self-edges, empty string means no
+ # extras The empty string in the set is just a way to make the handling
+ # of no extras and a single extra easier and having a set of {"", "foo"}
+ # is equivalent to having {"foo"}.
+ extras = extras or [""]
+
+ self_reqs = []
+ for req in reqs:
+ if req.name != self_name:
+ continue
+
+ if req.marker == None:
+ # I am pretty sure we cannot reach this code as it does not
+ # make sense to specify packages in this way, but since it is
+ # easy to handle, lets do it.
+ #
+ # TODO @aignas 2023-12-08: add a test
+ extras = extras + req.extras
+ else:
+ # process these in a separate loop
+ self_reqs.append(req)
+
+ # A double loop is not strictly optimal, but always correct without recursion
+ for req in self_reqs:
+ if [True for extra in extras if evaluate(req.marker, env = {"extra": extra})]:
+ extras = extras + req.extras
+ else:
+ continue
+
+ # Iterate through all packages to ensure that we include all of the extras from previously
+ # visited packages.
+ for req_ in self_reqs:
+ if [True for extra in extras if evaluate(req.marker, env = {"extra": extra})]:
+ extras = extras + req_.extras
+
+ # Poor mans set
+ return sorted({x: None for x in extras})
+
+def _add_req(deps, deps_select, req, *, extras, platforms, default_abi = None):
+ if not req.marker:
+ _add(deps, deps_select, req.name, None)
+ return
+
+ # NOTE @aignas 2023-12-08: in order to have reasonable select statements
+ # we do have to have some parsing of the markers, so it begs the question
+ # if packaging should be reimplemented in Starlark to have the best solution
+ # for now we will implement it in Python and see what the best parsing result
+ # can be before making this decision.
+ match_os = len([
+ tag
+ for tag in [
+ "os_name",
+ "sys_platform",
+ "platform_system",
+ ]
+ if tag in req.marker
+ ]) > 0
+ match_arch = "platform_machine" in req.marker
+ match_version = "version" in req.marker
+
+ if not (match_os or match_arch or match_version):
+ if [
+ True
+ for extra in extras
+ for p in platforms
+ if evaluate(
+ req.marker,
+ env = env(
+ target_platform = p,
+ extra = extra,
+ ),
+ )
+ ]:
+ _add(deps, deps_select, req.name, None)
+ return
+
+ for plat in platforms:
+ if not [
+ True
+ for extra in extras
+ if evaluate(
+ req.marker,
+ env = env(
+ target_platform = plat,
+ extra = extra,
+ ),
+ )
+ ]:
+ continue
+
+ if match_arch and default_abi:
+ _add(deps, deps_select, req.name, plat)
+ if plat.abi == default_abi:
+ _add(deps, deps_select, req.name, platform(os = plat.os, arch = plat.arch))
+ elif match_arch:
+ _add(deps, deps_select, req.name, platform(os = plat.os, arch = plat.arch))
+ elif match_os and default_abi:
+ _add(deps, deps_select, req.name, platform(os = plat.os, abi = plat.abi))
+ if plat.abi == default_abi:
+ _add(deps, deps_select, req.name, platform(os = plat.os))
+ elif match_os:
+ _add(deps, deps_select, req.name, platform(os = plat.os))
+ elif match_version and default_abi:
+ _add(deps, deps_select, req.name, platform(abi = plat.abi))
+ if plat.abi == default_abi:
+ _add(deps, deps_select, req.name, platform())
+ elif match_version:
+ _add(deps, deps_select, req.name, None)
+ else:
+ fail("BUG: {} support is not implemented".format(req.marker))
+
+ _maybe_add_common_dep(deps, deps_select, platforms, req.name)
diff --git a/python/private/pypi/pep508_evaluate.bzl b/python/private/pypi/pep508_evaluate.bzl
index f45eb75..f8ef553 100644
--- a/python/private/pypi/pep508_evaluate.bzl
+++ b/python/private/pypi/pep508_evaluate.bzl
@@ -138,7 +138,7 @@
"""
tokens = tokenize(marker)
- ast = _new_expr(**kwargs)
+ ast = _new_expr(marker = marker, **kwargs)
for _ in range(len(tokens) * 2):
if not tokens:
break
@@ -219,17 +219,20 @@
return not x
def _new_expr(
+ *,
+ marker,
and_fn = _and_fn,
or_fn = _or_fn,
not_fn = _not_fn):
# buildifier: disable=uninitialized
self = struct(
+ marker = marker,
tree = [],
parse = lambda **kwargs: _parse(self, **kwargs),
value = lambda: _value(self),
# This is a way for us to have a handle to the currently constructed
# expression tree branch.
- current = lambda: self._current[0] if self._current else None,
+ current = lambda: self._current[-1] if self._current else None,
_current = [],
_and = and_fn,
_or = or_fn,
@@ -313,6 +316,7 @@
#
# The following normalizes the values
left = env.get(_ENV_ALIASES, {}).get(var_name, {}).get(left, left)
+
else:
var_name = left
left = env[left]
@@ -392,12 +396,15 @@
current.tree.append(value)
elif hasattr(current.tree[-1], "append"):
current.tree[-1].append(value)
- else:
+ elif hasattr(current.tree, "_append"):
current.tree._append(value)
+ else:
+ fail("Cannot evaluate '{}' in '{}', current: {}".format(value, self.marker, current))
def _open_parenthesis(self):
"""Add an extra node into the tree to perform evaluate inside parenthesis."""
self._current.append(_new_expr(
+ marker = self.marker,
and_fn = self._and,
or_fn = self._or,
not_fn = self._not,
diff --git a/python/private/pypi/whl_installer/BUILD.bazel b/python/private/pypi/whl_installer/BUILD.bazel
index 5fb6170..49f1a11 100644
--- a/python/private/pypi/whl_installer/BUILD.bazel
+++ b/python/private/pypi/whl_installer/BUILD.bazel
@@ -6,7 +6,6 @@
srcs = [
"arguments.py",
"namespace_pkgs.py",
- "platform.py",
"wheel.py",
"wheel_installer.py",
],
diff --git a/python/private/pypi/whl_installer/arguments.py b/python/private/pypi/whl_installer/arguments.py
index 29bea80..bb841ea 100644
--- a/python/private/pypi/whl_installer/arguments.py
+++ b/python/private/pypi/whl_installer/arguments.py
@@ -17,8 +17,6 @@
import pathlib
from typing import Any, Dict, Set
-from python.private.pypi.whl_installer.platform import Platform
-
def parser(**kwargs: Any) -> argparse.ArgumentParser:
"""Create a parser for the wheel_installer tool."""
@@ -42,12 +40,6 @@
help="Extra arguments to pass down to pip.",
)
parser.add_argument(
- "--platform",
- action="extend",
- type=Platform.from_string,
- help="Platforms to target dependencies. Can be used multiple times.",
- )
- parser.add_argument(
"--pip_data_exclude",
action="store",
help="Additional data exclusion parameters to add to the pip packages BUILD file.",
diff --git a/python/private/pypi/whl_installer/platform.py b/python/private/pypi/whl_installer/platform.py
deleted file mode 100644
index 11dd6e3..0000000
--- a/python/private/pypi/whl_installer/platform.py
+++ /dev/null
@@ -1,304 +0,0 @@
-# Copyright 2024 The Bazel Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Utility class to inspect an extracted wheel directory"""
-
-import platform
-import sys
-from dataclasses import dataclass
-from enum import Enum
-from typing import Any, Dict, Iterator, List, Optional, Union
-
-
-class OS(Enum):
- linux = 1
- osx = 2
- windows = 3
- darwin = osx
- win32 = windows
-
- @classmethod
- def interpreter(cls) -> "OS":
- "Return the interpreter operating system."
- return cls[sys.platform.lower()]
-
- def __str__(self) -> str:
- return self.name.lower()
-
-
-class Arch(Enum):
- x86_64 = 1
- x86_32 = 2
- aarch64 = 3
- ppc = 4
- ppc64le = 5
- s390x = 6
- arm = 7
- amd64 = x86_64
- arm64 = aarch64
- i386 = x86_32
- i686 = x86_32
- x86 = x86_32
-
- @classmethod
- def interpreter(cls) -> "Arch":
- "Return the currently running interpreter architecture."
- # FIXME @aignas 2023-12-13: Hermetic toolchain on Windows 3.11.6
- # is returning an empty string here, so lets default to x86_64
- return cls[platform.machine().lower() or "x86_64"]
-
- def __str__(self) -> str:
- return self.name.lower()
-
-
-def _as_int(value: Optional[Union[OS, Arch]]) -> int:
- """Convert one of the enums above to an int for easier sorting algorithms.
-
- Args:
- value: The value of an enum or None.
-
- Returns:
- -1 if we get None, otherwise, the numeric value of the given enum.
- """
- if value is None:
- return -1
-
- return int(value.value)
-
-
-def host_interpreter_minor_version() -> int:
- return sys.version_info.minor
-
-
-@dataclass(frozen=True)
-class Platform:
- os: Optional[OS] = None
- arch: Optional[Arch] = None
- minor_version: Optional[int] = None
-
- @classmethod
- def all(
- cls,
- want_os: Optional[OS] = None,
- minor_version: Optional[int] = None,
- ) -> List["Platform"]:
- return sorted(
- [
- cls(os=os, arch=arch, minor_version=minor_version)
- for os in OS
- for arch in Arch
- if not want_os or want_os == os
- ]
- )
-
- @classmethod
- def host(cls) -> List["Platform"]:
- """Use the Python interpreter to detect the platform.
-
- We extract `os` from sys.platform and `arch` from platform.machine
-
- Returns:
- A list of parsed values which makes the signature the same as
- `Platform.all` and `Platform.from_string`.
- """
- return [
- Platform(
- os=OS.interpreter(),
- arch=Arch.interpreter(),
- minor_version=host_interpreter_minor_version(),
- )
- ]
-
- def all_specializations(self) -> Iterator["Platform"]:
- """Return the platform itself and all its unambiguous specializations.
-
- For more info about specializations see
- https://bazel.build/docs/configurable-attributes
- """
- yield self
- if self.arch is None:
- for arch in Arch:
- yield Platform(os=self.os, arch=arch, minor_version=self.minor_version)
- if self.os is None:
- for os in OS:
- yield Platform(os=os, arch=self.arch, minor_version=self.minor_version)
- if self.arch is None and self.os is None:
- for os in OS:
- for arch in Arch:
- yield Platform(os=os, arch=arch, minor_version=self.minor_version)
-
- def __lt__(self, other: Any) -> bool:
- """Add a comparison method, so that `sorted` returns the most specialized platforms first."""
- if not isinstance(other, Platform) or other is None:
- raise ValueError(f"cannot compare {other} with Platform")
-
- self_arch, self_os = _as_int(self.arch), _as_int(self.os)
- other_arch, other_os = _as_int(other.arch), _as_int(other.os)
-
- if self_os == other_os:
- return self_arch < other_arch
- else:
- return self_os < other_os
-
- def __str__(self) -> str:
- if self.minor_version is None:
- if self.os is None and self.arch is None:
- return "//conditions:default"
-
- if self.arch is None:
- return f"@platforms//os:{self.os}"
- else:
- return f"{self.os}_{self.arch}"
-
- if self.arch is None and self.os is None:
- return f"@//python/config_settings:is_python_3.{self.minor_version}"
-
- if self.arch is None:
- return f"cp3{self.minor_version}_{self.os}_anyarch"
-
- if self.os is None:
- return f"cp3{self.minor_version}_anyos_{self.arch}"
-
- return f"cp3{self.minor_version}_{self.os}_{self.arch}"
-
- @classmethod
- def from_string(cls, platform: Union[str, List[str]]) -> List["Platform"]:
- """Parse a string and return a list of platforms"""
- platform = [platform] if isinstance(platform, str) else list(platform)
- ret = set()
- for p in platform:
- if p == "host":
- ret.update(cls.host())
- continue
-
- abi, _, tail = p.partition("_")
- if not abi.startswith("cp"):
- # The first item is not an abi
- tail = p
- abi = ""
- os, _, arch = tail.partition("_")
- arch = arch or "*"
-
- minor_version = int(abi[len("cp3") :]) if abi else None
-
- if arch != "*":
- ret.add(
- cls(
- os=OS[os] if os != "*" else None,
- arch=Arch[arch],
- minor_version=minor_version,
- )
- )
-
- else:
- ret.update(
- cls.all(
- want_os=OS[os] if os != "*" else None,
- minor_version=minor_version,
- )
- )
-
- return sorted(ret)
-
- # NOTE @aignas 2023-12-05: below is the minimum number of accessors that are defined in
- # https://peps.python.org/pep-0496/ to make rules_python generate dependencies.
- #
- # WARNING: It may not work in cases where the python implementation is different between
- # different platforms.
-
- # derived from OS
- @property
- def os_name(self) -> str:
- if self.os == OS.linux or self.os == OS.osx:
- return "posix"
- elif self.os == OS.windows:
- return "nt"
- else:
- return ""
-
- @property
- def sys_platform(self) -> str:
- if self.os == OS.linux:
- return "linux"
- elif self.os == OS.osx:
- return "darwin"
- elif self.os == OS.windows:
- return "win32"
- else:
- return ""
-
- @property
- def platform_system(self) -> str:
- if self.os == OS.linux:
- return "Linux"
- elif self.os == OS.osx:
- return "Darwin"
- elif self.os == OS.windows:
- return "Windows"
- else:
- return ""
-
- # derived from OS and Arch
- @property
- def platform_machine(self) -> str:
- """Guess the target 'platform_machine' marker.
-
- NOTE @aignas 2023-12-05: this may not work on really new systems, like
- Windows if they define the platform markers in a different way.
- """
- if self.arch == Arch.x86_64:
- return "x86_64"
- elif self.arch == Arch.x86_32 and self.os != OS.osx:
- return "i386"
- elif self.arch == Arch.x86_32:
- return ""
- elif self.arch == Arch.aarch64 and self.os == OS.linux:
- return "aarch64"
- elif self.arch == Arch.aarch64:
- # Assuming that OSX and Windows use this one since the precedent is set here:
- # https://github.com/cgohlke/win_arm64-wheels
- return "arm64"
- elif self.os != OS.linux:
- return ""
- elif self.arch == Arch.ppc:
- return "ppc"
- elif self.arch == Arch.ppc64le:
- return "ppc64le"
- elif self.arch == Arch.s390x:
- return "s390x"
- else:
- return ""
-
- def env_markers(self, extra: str) -> Dict[str, str]:
- # If it is None, use the host version
- minor_version = self.minor_version or host_interpreter_minor_version()
-
- return {
- "extra": extra,
- "os_name": self.os_name,
- "sys_platform": self.sys_platform,
- "platform_machine": self.platform_machine,
- "platform_system": self.platform_system,
- "platform_release": "", # unset
- "platform_version": "", # unset
- "python_version": f"3.{minor_version}",
- # FIXME @aignas 2024-01-14: is putting zero last a good idea? Maybe we should
- # use `20` or something else to avoid having weird issues where the full version is used for
- # matching and the author decides to only support 3.y.5 upwards.
- "implementation_version": f"3.{minor_version}.0",
- "python_full_version": f"3.{minor_version}.0",
- # we assume that the following are the same as the interpreter used to setup the deps:
- # "implementation_name": "cpython"
- # "platform_python_implementation: "CPython",
- }
diff --git a/python/private/pypi/whl_installer/wheel.py b/python/private/pypi/whl_installer/wheel.py
index d95b33a..da81b5e 100644
--- a/python/private/pypi/whl_installer/wheel.py
+++ b/python/private/pypi/whl_installer/wheel.py
@@ -25,275 +25,6 @@
from packaging.requirements import Requirement
from pip._vendor.packaging.utils import canonicalize_name
-from python.private.pypi.whl_installer.platform import (
- Platform,
- host_interpreter_minor_version,
-)
-
-
-@dataclass(frozen=True)
-class FrozenDeps:
- deps: List[str]
- deps_select: Dict[str, List[str]]
-
-
-class Deps:
- """Deps is a dependency builder that has a build() method to return FrozenDeps."""
-
- def __init__(
- self,
- name: str,
- requires_dist: List[str],
- *,
- extras: Optional[Set[str]] = None,
- platforms: Optional[Set[Platform]] = None,
- ):
- """Create a new instance and parse the requires_dist
-
- Args:
- name (str): The name of the whl distribution
- requires_dist (list[Str]): The Requires-Dist from the METADATA of the whl
- distribution.
- extras (set[str], optional): The list of requested extras, defaults to None.
- platforms (set[Platform], optional): The list of target platforms, defaults to
- None. If the list of platforms has multiple `minor_version` values, it
- will change the code to generate the select statements using
- `@rules_python//python/config_settings:is_python_3.y` conditions.
- """
- self.name: str = Deps._normalize(name)
- self._platforms: Set[Platform] = platforms or set()
- self._target_versions = {p.minor_version for p in platforms or {}}
- self._default_minor_version = None
- if platforms and len(self._target_versions) > 2:
- # TODO @aignas 2024-06-23: enable this to be set via a CLI arg
- # for being more explicit.
- self._default_minor_version = host_interpreter_minor_version()
-
- if None in self._target_versions and len(self._target_versions) > 2:
- raise ValueError(
- f"all python versions need to be specified explicitly, got: {platforms}"
- )
-
- # Sort so that the dictionary order in the FrozenDeps is deterministic
- # without the final sort because Python retains insertion order. That way
- # the sorting by platform is limited within the Platform class itself and
- # the unit-tests for the Deps can be simpler.
- reqs = sorted(
- (Requirement(wheel_req) for wheel_req in requires_dist),
- key=lambda x: f"{x.name}:{sorted(x.extras)}",
- )
-
- want_extras = self._resolve_extras(reqs, extras)
-
- # Then add all of the requirements in order
- self._deps: Set[str] = set()
- self._select: Dict[Platform, Set[str]] = defaultdict(set)
- for req in reqs:
- self._add_req(req, want_extras)
-
- def _add(self, dep: str, platform: Optional[Platform]):
- dep = Deps._normalize(dep)
-
- # Self-edges are processed in _resolve_extras
- if dep == self.name:
- return
-
- if not platform:
- self._deps.add(dep)
-
- # If the dep is in the platform-specific list, remove it from the select.
- pop_keys = []
- for p, deps in self._select.items():
- if dep not in deps:
- continue
-
- deps.remove(dep)
- if not deps:
- pop_keys.append(p)
-
- for p in pop_keys:
- self._select.pop(p)
- return
-
- if dep in self._deps:
- # If the dep is already in the main dependency list, no need to add it in the
- # platform-specific dependency list.
- return
-
- # Add the platform-specific dep
- self._select[platform].add(dep)
-
- # Add the dep to specializations of the given platform if they
- # exist in the select statement.
- for p in platform.all_specializations():
- if p not in self._select:
- continue
-
- self._select[p].add(dep)
-
- if len(self._select[platform]) == 1:
- # We are adding a new item to the select and we need to ensure that
- # existing dependencies from less specialized platforms are propagated
- # to the newly added dependency set.
- for p, deps in self._select.items():
- # Check if the existing platform overlaps with the given platform
- if p == platform or platform not in p.all_specializations():
- continue
-
- self._select[platform].update(self._select[p])
-
- def _maybe_add_common_dep(self, dep):
- if len(self._target_versions) < 2:
- return
-
- platforms = [Platform()] + [
- Platform(minor_version=v) for v in self._target_versions
- ]
-
- # If the dep is targeting all target python versions, lets add it to
- # the common dependency list to simplify the select statements.
- for p in platforms:
- if p not in self._select:
- return
-
- if dep not in self._select[p]:
- return
-
- # All of the python version-specific branches have the dep, so lets add
- # it to the common deps.
- self._deps.add(dep)
- for p in platforms:
- self._select[p].remove(dep)
- if not self._select[p]:
- self._select.pop(p)
-
- @staticmethod
- def _normalize(name: str) -> str:
- return re.sub(r"[-_.]+", "_", name).lower()
-
- def _resolve_extras(
- self, reqs: List[Requirement], extras: Optional[Set[str]]
- ) -> Set[str]:
- """Resolve extras which are due to depending on self[some_other_extra].
-
- Some packages may have cyclic dependencies resulting from extras being used, one example is
- `etils`, where we have one set of extras as aliases for other extras
- and we have an extra called 'all' that includes all other extras.
-
- Example: github.com/google/etils/blob/a0b71032095db14acf6b33516bca6d885fe09e35/pyproject.toml#L32.
-
- When the `requirements.txt` is generated by `pip-tools`, then it is likely that
- this step is not needed, but for other `requirements.txt` files this may be useful.
-
- NOTE @aignas 2023-12-08: the extra resolution is not platform dependent,
- but in order for it to become platform dependent we would have to have
- separate targets for each extra in extras.
- """
-
- # Resolve any extra extras due to self-edges, empty string means no
- # extras The empty string in the set is just a way to make the handling
- # of no extras and a single extra easier and having a set of {"", "foo"}
- # is equivalent to having {"foo"}.
- extras = extras or {""}
-
- self_reqs = []
- for req in reqs:
- if Deps._normalize(req.name) != self.name:
- continue
-
- if req.marker is None:
- # I am pretty sure we cannot reach this code as it does not
- # make sense to specify packages in this way, but since it is
- # easy to handle, lets do it.
- #
- # TODO @aignas 2023-12-08: add a test
- extras = extras | req.extras
- else:
- # process these in a separate loop
- self_reqs.append(req)
-
- # A double loop is not strictly optimal, but always correct without recursion
- for req in self_reqs:
- if any(req.marker.evaluate({"extra": extra}) for extra in extras):
- extras = extras | req.extras
- else:
- continue
-
- # Iterate through all packages to ensure that we include all of the extras from previously
- # visited packages.
- for req_ in self_reqs:
- if any(req_.marker.evaluate({"extra": extra}) for extra in extras):
- extras = extras | req_.extras
-
- return extras
-
- def _add_req(self, req: Requirement, extras: Set[str]) -> None:
- if req.marker is None:
- self._add(req.name, None)
- return
-
- marker_str = str(req.marker)
-
- if not self._platforms:
- if any(req.marker.evaluate({"extra": extra}) for extra in extras):
- self._add(req.name, None)
- return
-
- # NOTE @aignas 2023-12-08: in order to have reasonable select statements
- # we do have to have some parsing of the markers, so it begs the question
- # if packaging should be reimplemented in Starlark to have the best solution
- # for now we will implement it in Python and see what the best parsing result
- # can be before making this decision.
- match_os = any(
- tag in marker_str
- for tag in [
- "os_name",
- "sys_platform",
- "platform_system",
- ]
- )
- match_arch = "platform_machine" in marker_str
- match_version = "version" in marker_str
-
- if not (match_os or match_arch or match_version):
- if any(req.marker.evaluate({"extra": extra}) for extra in extras):
- self._add(req.name, None)
- return
-
- for plat in self._platforms:
- if not any(
- req.marker.evaluate(plat.env_markers(extra)) for extra in extras
- ):
- continue
-
- if match_arch and self._default_minor_version:
- self._add(req.name, plat)
- if plat.minor_version == self._default_minor_version:
- self._add(req.name, Platform(plat.os, plat.arch))
- elif match_arch:
- self._add(req.name, Platform(plat.os, plat.arch))
- elif match_os and self._default_minor_version:
- self._add(req.name, Platform(plat.os, minor_version=plat.minor_version))
- if plat.minor_version == self._default_minor_version:
- self._add(req.name, Platform(plat.os))
- elif match_os:
- self._add(req.name, Platform(plat.os))
- elif match_version and self._default_minor_version:
- self._add(req.name, Platform(minor_version=plat.minor_version))
- if plat.minor_version == self._default_minor_version:
- self._add(req.name, Platform())
- elif match_version:
- self._add(req.name, None)
-
- # Merge to common if possible after processing all platforms
- self._maybe_add_common_dep(req.name)
-
- def build(self) -> FrozenDeps:
- return FrozenDeps(
- deps=sorted(self._deps),
- deps_select={str(p): sorted(deps) for p, deps in self._select.items()},
- )
-
class Wheel:
"""Representation of the compressed .whl file"""
@@ -344,18 +75,6 @@
return entry_points_mapping
- def dependencies(
- self,
- extras_requested: Set[str] = None,
- platforms: Optional[Set[Platform]] = None,
- ) -> FrozenDeps:
- return Deps(
- self.name,
- extras=extras_requested,
- platforms=platforms,
- requires_dist=self.metadata.get_all("Requires-Dist", []),
- ).build()
-
def unzip(self, directory: str) -> None:
installation_schemes = {
"purelib": "/site-packages",
diff --git a/python/private/pypi/whl_installer/wheel_installer.py b/python/private/pypi/whl_installer/wheel_installer.py
index ef8181c..c7695d9 100644
--- a/python/private/pypi/whl_installer/wheel_installer.py
+++ b/python/private/pypi/whl_installer/wheel_installer.py
@@ -23,7 +23,7 @@
import sys
from pathlib import Path
from tempfile import NamedTemporaryFile
-from typing import Dict, List, Optional, Set, Tuple
+from typing import Dict, Optional, Set, Tuple
from pip._vendor.packaging.utils import canonicalize_name
@@ -103,9 +103,7 @@
def _extract_wheel(
wheel_file: str,
- extras: Dict[str, Set[str]],
enable_implicit_namespace_pkgs: bool,
- platforms: List[wheel.Platform],
installation_dir: Path = Path("."),
) -> None:
"""Extracts wheel into given directory and creates py_library and filegroup targets.
@@ -113,7 +111,6 @@
Args:
wheel_file: the filepath of the .whl
installation_dir: the destination directory for installation of the wheel.
- extras: a list of extras to add as dependencies for the installed wheel
enable_implicit_namespace_pkgs: if true, disables conversion of implicit namespace packages and will unzip as-is
"""
@@ -123,25 +120,19 @@
if not enable_implicit_namespace_pkgs:
_setup_namespace_pkg_compatibility(installation_dir)
- extras_requested = extras[whl.name] if whl.name in extras else set()
-
- dependencies = whl.dependencies(extras_requested, platforms)
+ metadata = {
+ "python_version": sys.version.partition(" ")[0],
+ "entry_points": [
+ {
+ "name": name,
+ "module": module,
+ "attribute": attribute,
+ }
+ for name, (module, attribute) in sorted(whl.entry_points().items())
+ ],
+ }
with open(os.path.join(installation_dir, "metadata.json"), "w") as f:
- metadata = {
- "name": whl.name,
- "version": whl.version,
- "deps": dependencies.deps,
- "deps_by_platform": dependencies.deps_select,
- "entry_points": [
- {
- "name": name,
- "module": module,
- "attribute": attribute,
- }
- for name, (module, attribute) in sorted(whl.entry_points().items())
- ],
- }
json.dump(metadata, f)
@@ -155,13 +146,9 @@
if args.whl_file:
whl = Path(args.whl_file)
- name, extras_for_pkg = _parse_requirement_for_extra(args.requirement)
- extras = {name: extras_for_pkg} if extras_for_pkg and name else dict()
_extract_wheel(
wheel_file=whl,
- extras=extras,
enable_implicit_namespace_pkgs=args.enable_implicit_namespace_pkgs,
- platforms=arguments.get_platforms(args),
)
return
diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl
index 493f113..54f9ff3 100644
--- a/python/private/pypi/whl_library.bzl
+++ b/python/private/pypi/whl_library.bzl
@@ -21,9 +21,13 @@
load(":attrs.bzl", "ATTRS", "use_isolated")
load(":deps.bzl", "all_repo_names", "record_files")
load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel")
+load(":parse_requirements.bzl", "host_platform")
load(":parse_whl_name.bzl", "parse_whl_name")
load(":patch_whl.bzl", "patch_whl")
+load(":pep508_deps.bzl", "deps")
+load(":pep508_requirement.bzl", "requirement")
load(":pypi_repo_utils.bzl", "pypi_repo_utils")
+load(":whl_metadata.bzl", "whl_metadata")
load(":whl_target_platforms.bzl", "whl_target_platforms")
_CPPFLAGS = "CPPFLAGS"
@@ -361,7 +365,7 @@
arguments = args + [
"--whl-file",
whl_path,
- ] + ["--platform={}".format(p) for p in target_platforms],
+ ],
srcs = rctx.attr._python_srcs,
environment = environment,
quiet = rctx.attr.quiet,
@@ -396,17 +400,60 @@
)
entry_points[entry_point_without_py] = entry_point_script_name
+ # TODO @aignas 2025-04-04: move this to whl_library_targets.bzl to have
+ # this in the analysis phase.
+ #
+ # This means that whl_library_targets will have to accept the following args:
+ # * name - the name of the package in the METADATA.
+ # * requires_dist - the list of METADATA Requires-Dist.
+ # * platforms - the list of target platforms. The target_platforms
+ # should come from the hub repo via a 'load' statement so that they don't
+ # need to be passed as an argument to `whl_library`.
+ # * extras - the list of required extras. This comes from the
+ # `rctx.attr.requirement` for now. In the future the required extras could
+ # stay in the hub repo, where we calculate the extra aliases that we need
+ # to create automatically and this way expose the targets for the specific
+ # extras. The first step will be to generate a target per extra for the
+ # `py_library` and `filegroup`. Maybe we need to have a special provider
+ # or an output group so that we can return the `whl` file from the
+ # `py_library` target? filegroup can use output groups to expose files.
+ # * host_python_version/versons - the list of python versions to support
+ # should come from the hub, similar to how the target platforms are specified.
+ #
+ # Extra things that we should move at the same time:
+ # * group_name, group_deps - this info can stay in the hub repository so that
+ # it is piped at the analysis time and changing the requirement groups does
+ # cause to re-fetch the deps.
+ python_version = metadata["python_version"]
+ metadata = whl_metadata(
+ install_dir = rctx.path("site-packages"),
+ read_fn = rctx.read,
+ logger = logger,
+ )
+
+ # TODO @aignas 2025-04-09: this will later be removed when loaded through the hub
+ major_minor, _, _ = python_version.rpartition(".")
+ package_deps = deps(
+ name = metadata.name,
+ requires_dist = metadata.requires_dist,
+ platforms = target_platforms or [
+ "cp{}_{}".format(major_minor.replace(".", ""), host_platform(rctx)),
+ ],
+ extras = requirement(rctx.attr.requirement).extras,
+ host_python_version = python_version,
+ )
+
build_file_contents = generate_whl_library_build_bazel(
name = whl_path.basename,
dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix),
- dependencies = metadata["deps"],
- dependencies_by_platform = metadata["deps_by_platform"],
+ dependencies = package_deps.deps,
+ dependencies_by_platform = package_deps.deps_select,
group_name = rctx.attr.group_name,
group_deps = rctx.attr.group_deps,
data_exclude = rctx.attr.pip_data_exclude,
tags = [
- "pypi_name=" + metadata["name"],
- "pypi_version=" + metadata["version"],
+ "pypi_name=" + metadata.name,
+ "pypi_version=" + metadata.version,
],
entry_points = entry_points,
annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))),
diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl
index 95031e6..d32746b 100644
--- a/python/private/pypi/whl_library_targets.bzl
+++ b/python/private/pypi/whl_library_targets.bzl
@@ -90,8 +90,6 @@
native: {type}`native` The native struct for overriding in tests.
rules: {type}`struct` A struct with references to rules for creating targets.
"""
- _ = name # buildifier: @unused
-
dependencies = sorted([normalize_name(d) for d in dependencies])
dependencies_by_platform = {
platform: sorted([normalize_name(d) for d in deps])
diff --git a/python/private/pypi/whl_metadata.bzl b/python/private/pypi/whl_metadata.bzl
new file mode 100644
index 0000000..8a86ffb
--- /dev/null
+++ b/python/private/pypi/whl_metadata.bzl
@@ -0,0 +1,108 @@
+"""A simple function to find the METADATA file and parse it"""
+
+_NAME = "Name: "
+_PROVIDES_EXTRA = "Provides-Extra: "
+_REQUIRES_DIST = "Requires-Dist: "
+_VERSION = "Version: "
+
+def whl_metadata(*, install_dir, read_fn, logger):
+ """Find and parse the METADATA file in the extracted whl contents dir.
+
+ Args:
+ install_dir: {type}`path` location where the wheel has been extracted.
+ read_fn: the function used to read files.
+ logger: the function used to log failures.
+
+ Returns:
+ A struct with parsed values:
+ * `name`: {type}`str` the name of the wheel.
+ * `version`: {type}`str` the version of the wheel.
+ * `requires_dist`: {type}`list[str]` the list of requirements.
+ * `provides_extra`: {type}`list[str]` the list of extras that this package
+ provides.
+ """
+ metadata_file = find_whl_metadata(install_dir = install_dir, logger = logger)
+ contents = read_fn(metadata_file)
+ result = parse_whl_metadata(contents)
+
+ if not (result.name and result.version):
+ logger.fail("Failed to parsed the wheel METADATA file:\n{}".format(contents))
+ return None
+
+ return result
+
+def parse_whl_metadata(contents):
+ """Parse .whl METADATA file
+
+ Args:
+ contents: {type}`str` the contents of the file.
+
+ Returns:
+ A struct with parsed values:
+ * `name`: {type}`str` the name of the wheel.
+ * `version`: {type}`str` the version of the wheel.
+ * `requires_dist`: {type}`list[str]` the list of requirements.
+ * `provides_extra`: {type}`list[str]` the list of extras that this package
+ provides.
+ """
+ parsed = {
+ "name": "",
+ "provides_extra": [],
+ "requires_dist": [],
+ "version": "",
+ }
+ for line in contents.strip().split("\n"):
+ if not line.strip():
+ # Stop parsing on first empty line, which marks the end of the
+ # headers containing the metadata.
+ break
+
+ if line.startswith(_NAME):
+ _, _, value = line.partition(_NAME)
+ parsed["name"] = value.strip()
+ elif line.startswith(_VERSION):
+ _, _, value = line.partition(_VERSION)
+ parsed["version"] = value.strip()
+ elif line.startswith(_REQUIRES_DIST):
+ _, _, value = line.partition(_REQUIRES_DIST)
+ parsed["requires_dist"].append(value.strip(" "))
+ elif line.startswith(_PROVIDES_EXTRA):
+ _, _, value = line.partition(_PROVIDES_EXTRA)
+ parsed["provides_extra"].append(value.strip(" "))
+
+ return struct(
+ name = parsed["name"],
+ provides_extra = parsed["provides_extra"],
+ requires_dist = parsed["requires_dist"],
+ version = parsed["version"],
+ )
+
+def find_whl_metadata(*, install_dir, logger):
+ """Find the whl METADATA file in the install_dir.
+
+ Args:
+ install_dir: {type}`path` location where the wheel has been extracted.
+ logger: the function used to log failures.
+
+ Returns:
+ {type}`path` The path to the METADATA file.
+ """
+ dist_info = None
+ for maybe_dist_info in install_dir.readdir():
+ # first find the ".dist-info" folder
+ if not (maybe_dist_info.is_dir and maybe_dist_info.basename.endswith(".dist-info")):
+ continue
+
+ dist_info = maybe_dist_info
+ metadata_file = dist_info.get_child("METADATA")
+
+ if metadata_file.exists:
+ return metadata_file
+
+ break
+
+ if dist_info:
+ logger.fail("The METADATA file for the wheel could not be found in '{}/{}'".format(install_dir.basename, dist_info.basename))
+ else:
+ logger.fail("The '*.dist-info' directory could not be found in '{}'".format(install_dir.basename))
+ return None
diff --git a/tests/pypi/pep508/BUILD.bazel b/tests/pypi/pep508/BUILD.bazel
index 575f28a..7eab2e0 100644
--- a/tests/pypi/pep508/BUILD.bazel
+++ b/tests/pypi/pep508/BUILD.bazel
@@ -1,6 +1,11 @@
+load(":deps_tests.bzl", "deps_test_suite")
load(":evaluate_tests.bzl", "evaluate_test_suite")
load(":requirement_tests.bzl", "requirement_test_suite")
+deps_test_suite(
+ name = "deps_tests",
+)
+
evaluate_test_suite(
name = "evaluate_tests",
)
diff --git a/tests/pypi/pep508/deps_tests.bzl b/tests/pypi/pep508/deps_tests.bzl
new file mode 100644
index 0000000..44031ab
--- /dev/null
+++ b/tests/pypi/pep508/deps_tests.bzl
@@ -0,0 +1,385 @@
+# Copyright 2025 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for construction of Python version matching config settings."""
+
+load("@rules_testing//lib:test_suite.bzl", "test_suite")
+load("//python/private/pypi:pep508_deps.bzl", "deps") # buildifier: disable=bzl-visibility
+
+_tests = []
+
+def test_simple_deps(env):
+ got = deps(
+ "foo",
+ requires_dist = ["bar-Bar"],
+ )
+ env.expect.that_collection(got.deps).contains_exactly(["bar_bar"])
+ env.expect.that_dict(got.deps_select).contains_exactly({})
+
+_tests.append(test_simple_deps)
+
+def test_can_add_os_specific_deps(env):
+ got = deps(
+ "foo",
+ requires_dist = [
+ "bar",
+ "an_osx_dep; sys_platform=='darwin'",
+ "posix_dep; os_name=='posix'",
+ "win_dep; os_name=='nt'",
+ ],
+ platforms = [
+ "linux_x86_64",
+ "osx_x86_64",
+ "osx_aarch64",
+ "windows_x86_64",
+ ],
+ host_python_version = "3.3.1",
+ )
+
+ env.expect.that_collection(got.deps).contains_exactly(["bar"])
+ env.expect.that_dict(got.deps_select).contains_exactly({
+ "@platforms//os:linux": ["posix_dep"],
+ "@platforms//os:osx": ["an_osx_dep", "posix_dep"],
+ "@platforms//os:windows": ["win_dep"],
+ })
+
+_tests.append(test_can_add_os_specific_deps)
+
+def test_can_add_os_specific_deps_with_python_version(env):
+ got = deps(
+ "foo",
+ requires_dist = [
+ "bar",
+ "an_osx_dep; sys_platform=='darwin'",
+ "posix_dep; os_name=='posix'",
+ "win_dep; os_name=='nt'",
+ ],
+ platforms = [
+ "cp33_linux_x86_64",
+ "cp33_osx_x86_64",
+ "cp33_osx_aarch64",
+ "cp33_windows_x86_64",
+ ],
+ )
+
+ env.expect.that_collection(got.deps).contains_exactly(["bar"])
+ env.expect.that_dict(got.deps_select).contains_exactly({
+ "@platforms//os:linux": ["posix_dep"],
+ "@platforms//os:osx": ["an_osx_dep", "posix_dep"],
+ "@platforms//os:windows": ["win_dep"],
+ })
+
+_tests.append(test_can_add_os_specific_deps_with_python_version)
+
+def test_deps_are_added_to_more_specialized_platforms(env):
+ got = deps(
+ "foo",
+ requires_dist = [
+ "m1_dep; sys_platform=='darwin' and platform_machine=='arm64'",
+ "mac_dep; sys_platform=='darwin'",
+ ],
+ platforms = [
+ "osx_x86_64",
+ "osx_aarch64",
+ ],
+ host_python_version = "3.8.4",
+ )
+
+ env.expect.that_collection(got.deps).contains_exactly([])
+ env.expect.that_dict(got.deps_select).contains_exactly({
+ "@platforms//os:osx": ["mac_dep"],
+ "osx_aarch64": ["m1_dep", "mac_dep"],
+ })
+
+_tests.append(test_deps_are_added_to_more_specialized_platforms)
+
+def test_deps_from_more_specialized_platforms_are_propagated(env):
+ got = deps(
+ "foo",
+ requires_dist = [
+ "a_mac_dep; sys_platform=='darwin'",
+ "m1_dep; sys_platform=='darwin' and platform_machine=='arm64'",
+ ],
+ platforms = [
+ "osx_x86_64",
+ "osx_aarch64",
+ ],
+ host_python_version = "3.8.4",
+ )
+
+ env.expect.that_collection(got.deps).contains_exactly([])
+ env.expect.that_dict(got.deps_select).contains_exactly(
+ {
+ "@platforms//os:osx": ["a_mac_dep"],
+ "osx_aarch64": ["a_mac_dep", "m1_dep"],
+ },
+ )
+
+_tests.append(test_deps_from_more_specialized_platforms_are_propagated)
+
+def test_non_platform_markers_are_added_to_common_deps(env):
+ got = deps(
+ "foo",
+ requires_dist = [
+ "bar",
+ "baz; implementation_name=='cpython'",
+ "m1_dep; sys_platform=='darwin' and platform_machine=='arm64'",
+ ],
+ platforms = [
+ "linux_x86_64",
+ "osx_x86_64",
+ "osx_aarch64",
+ "windows_x86_64",
+ ],
+ host_python_version = "3.8.4",
+ )
+
+ env.expect.that_collection(got.deps).contains_exactly(["bar", "baz"])
+ env.expect.that_dict(got.deps_select).contains_exactly({
+ "osx_aarch64": ["m1_dep"],
+ })
+
+_tests.append(test_non_platform_markers_are_added_to_common_deps)
+
+def test_self_is_ignored(env):
+ got = deps(
+ "foo",
+ requires_dist = [
+ "bar",
+ "req_dep; extra == 'requests'",
+ "foo[requests]; extra == 'ssl'",
+ "ssl_lib; extra == 'ssl'",
+ ],
+ extras = ["ssl"],
+ )
+
+ env.expect.that_collection(got.deps).contains_exactly(["bar", "req_dep", "ssl_lib"])
+ env.expect.that_dict(got.deps_select).contains_exactly({})
+
+_tests.append(test_self_is_ignored)
+
+def test_self_dependencies_can_come_in_any_order(env):
+ got = deps(
+ "foo",
+ requires_dist = [
+ "bar",
+ "baz; extra == 'feat'",
+ "foo[feat2]; extra == 'all'",
+ "foo[feat]; extra == 'feat2'",
+ "zdep; extra == 'all'",
+ ],
+ extras = ["all"],
+ )
+
+ env.expect.that_collection(got.deps).contains_exactly(["bar", "baz", "zdep"])
+ env.expect.that_dict(got.deps_select).contains_exactly({})
+
+_tests.append(test_self_dependencies_can_come_in_any_order)
+
+def _test_can_get_deps_based_on_specific_python_version(env):
+ requires_dist = [
+ "bar",
+ "baz; python_version < '3.8'",
+ "posix_dep; os_name=='posix' and python_version >= '3.8'",
+ ]
+
+ py38 = deps(
+ "foo",
+ requires_dist = requires_dist,
+ platforms = ["cp38_linux_x86_64"],
+ )
+ py37 = deps(
+ "foo",
+ requires_dist = requires_dist,
+ platforms = ["cp37_linux_x86_64"],
+ )
+
+ env.expect.that_collection(py37.deps).contains_exactly(["bar", "baz"])
+ env.expect.that_dict(py37.deps_select).contains_exactly({})
+ env.expect.that_collection(py38.deps).contains_exactly(["bar"])
+ env.expect.that_dict(py38.deps_select).contains_exactly({"@platforms//os:linux": ["posix_dep"]})
+
+_tests.append(_test_can_get_deps_based_on_specific_python_version)
+
+def _test_no_version_select_when_single_version(env):
+ requires_dist = [
+ "bar",
+ "baz; python_version >= '3.8'",
+ "posix_dep; os_name=='posix'",
+ "posix_dep_with_version; os_name=='posix' and python_version >= '3.8'",
+ "arch_dep; platform_machine=='x86_64' and python_version >= '3.8'",
+ ]
+ host_python_version = "3.7.5"
+
+ got = deps(
+ "foo",
+ requires_dist = requires_dist,
+ platforms = [
+ "cp38_linux_x86_64",
+ "cp38_windows_x86_64",
+ ],
+ host_python_version = host_python_version,
+ )
+
+ env.expect.that_collection(got.deps).contains_exactly(["bar", "baz"])
+ env.expect.that_dict(got.deps_select).contains_exactly({
+ "@platforms//os:linux": ["posix_dep", "posix_dep_with_version"],
+ "linux_x86_64": ["arch_dep", "posix_dep", "posix_dep_with_version"],
+ "windows_x86_64": ["arch_dep"],
+ })
+
+_tests.append(_test_no_version_select_when_single_version)
+
+def _test_can_get_version_select(env):
+ requires_dist = [
+ "bar",
+ "baz; python_version < '3.8'",
+ "baz_new; python_version >= '3.8'",
+ "posix_dep; os_name=='posix'",
+ "posix_dep_with_version; os_name=='posix' and python_version >= '3.8'",
+ "arch_dep; platform_machine=='x86_64' and python_version < '3.8'",
+ ]
+ host_python_version = "3.7.4"
+
+ got = deps(
+ "foo",
+ requires_dist = requires_dist,
+ platforms = [
+ "cp3{}_{}_x86_64".format(minor, os)
+ for minor in [7, 8, 9]
+ for os in ["linux", "windows"]
+ ],
+ host_python_version = host_python_version,
+ )
+
+ env.expect.that_collection(got.deps).contains_exactly(["bar"])
+ env.expect.that_dict(got.deps_select).contains_exactly({
+ str(Label("//python/config_settings:is_python_3.7")): ["baz"],
+ str(Label("//python/config_settings:is_python_3.8")): ["baz_new"],
+ str(Label("//python/config_settings:is_python_3.9")): ["baz_new"],
+ "@platforms//os:linux": ["baz", "posix_dep"],
+ "cp37_linux_anyarch": ["baz", "posix_dep"],
+ "cp37_linux_x86_64": ["arch_dep", "baz", "posix_dep"],
+ "cp37_windows_x86_64": ["arch_dep", "baz"],
+ "cp38_linux_anyarch": [
+ "baz_new",
+ "posix_dep",
+ "posix_dep_with_version",
+ ],
+ "cp39_linux_anyarch": [
+ "baz_new",
+ "posix_dep",
+ "posix_dep_with_version",
+ ],
+ "linux_x86_64": ["arch_dep", "baz", "posix_dep"],
+ "windows_x86_64": ["arch_dep", "baz"],
+ "//conditions:default": ["baz"],
+ })
+
+_tests.append(_test_can_get_version_select)
+
+def _test_deps_spanning_all_target_py_versions_are_added_to_common(env):
+ requires_dist = [
+ "bar",
+ "baz (<2,>=1.11) ; python_version < '3.8'",
+ "baz (<2,>=1.14) ; python_version >= '3.8'",
+ ]
+ host_python_version = "3.8.4"
+
+ got = deps(
+ "foo",
+ requires_dist = requires_dist,
+ platforms = [
+ "cp3{}_linux_x86_64".format(minor)
+ for minor in [7, 8, 9]
+ ],
+ host_python_version = host_python_version,
+ )
+
+ env.expect.that_collection(got.deps).contains_exactly(["bar", "baz"])
+ env.expect.that_dict(got.deps_select).contains_exactly({})
+
+_tests.append(_test_deps_spanning_all_target_py_versions_are_added_to_common)
+
+def _test_deps_are_not_duplicated(env):
+ host_python_version = "3.7.4"
+
+ # See an example in
+ # https://files.pythonhosted.org/packages/76/9e/db1c2d56c04b97981c06663384f45f28950a73d9acf840c4006d60d0a1ff/opencv_python-4.9.0.80-cp37-abi3-win32.whl.metadata
+ requires_dist = [
+ "bar >=0.1.0 ; python_version < '3.7'",
+ "bar >=0.2.0 ; python_version >= '3.7'",
+ "bar >=0.4.0 ; python_version >= '3.6' and platform_system == 'Linux' and platform_machine == 'aarch64'",
+ "bar >=0.4.0 ; python_version >= '3.9'",
+ "bar >=0.5.0 ; python_version <= '3.9' and platform_system == 'Darwin' and platform_machine == 'arm64'",
+ "bar >=0.5.0 ; python_version >= '3.10' and platform_system == 'Darwin'",
+ "bar >=0.5.0 ; python_version >= '3.10'",
+ "bar >=0.6.0 ; python_version >= '3.11'",
+ ]
+
+ got = deps(
+ "foo",
+ requires_dist = requires_dist,
+ platforms = [
+ "cp3{}_{}_{}".format(minor, os, arch)
+ for minor in [7, 10]
+ for os in ["linux", "osx", "windows"]
+ for arch in ["x86_64", "aarch64"]
+ ],
+ host_python_version = host_python_version,
+ )
+
+ env.expect.that_collection(got.deps).contains_exactly(["bar"])
+ env.expect.that_dict(got.deps_select).contains_exactly({})
+
+_tests.append(_test_deps_are_not_duplicated)
+
+def _test_deps_are_not_duplicated_when_encountering_platform_dep_first(env):
+ host_python_version = "3.7.1"
+
+ # Note, that we are sorting the incoming `requires_dist` and we need to ensure that we are not getting any
+ # issues even if the platform-specific line comes first.
+ requires_dist = [
+ "bar >=0.4.0 ; python_version >= '3.6' and platform_system == 'Linux' and platform_machine == 'aarch64'",
+ "bar >=0.5.0 ; python_version >= '3.9'",
+ ]
+
+ got = deps(
+ "foo",
+ requires_dist = requires_dist,
+ platforms = [
+ "cp37_linux_aarch64",
+ "cp37_linux_x86_64",
+ "cp310_linux_aarch64",
+ "cp310_linux_x86_64",
+ ],
+ host_python_version = host_python_version,
+ )
+
+ # TODO @aignas 2025-02-24: this test case in the python version is passing but
+ # I am not sure why. The starlark version behaviour looks more correct.
+ env.expect.that_collection(got.deps).contains_exactly([])
+ env.expect.that_dict(got.deps_select).contains_exactly({
+ str(Label("//python/config_settings:is_python_3.10")): ["bar"],
+ "cp310_linux_aarch64": ["bar"],
+ "cp37_linux_aarch64": ["bar"],
+ "linux_aarch64": ["bar"],
+ })
+
+_tests.append(_test_deps_are_not_duplicated_when_encountering_platform_dep_first)
+
+def deps_test_suite(name): # buildifier: disable=function-docstring
+ test_suite(
+ name = name,
+ basic_tests = _tests,
+ )
diff --git a/tests/pypi/pep508/evaluate_tests.bzl b/tests/pypi/pep508/evaluate_tests.bzl
index 80b70f4..14e5e40 100644
--- a/tests/pypi/pep508/evaluate_tests.bzl
+++ b/tests/pypi/pep508/evaluate_tests.bzl
@@ -148,6 +148,8 @@
# expr
"os_name == 'fo'": False,
"(os_name == 'fo')": False,
+ "((os_name == 'fo'))": False,
+ "((os_name == 'foo'))": True,
"not (os_name == 'fo')": True,
# and
diff --git a/tests/pypi/whl_installer/BUILD.bazel b/tests/pypi/whl_installer/BUILD.bazel
index 040e4d7..fea6a46 100644
--- a/tests/pypi/whl_installer/BUILD.bazel
+++ b/tests/pypi/whl_installer/BUILD.bazel
@@ -28,18 +28,6 @@
)
py_test(
- name = "platform_test",
- size = "small",
- srcs = [
- "platform_test.py",
- ],
- data = ["//examples/wheel:minimal_with_py_package"],
- deps = [
- ":lib",
- ],
-)
-
-py_test(
name = "wheel_installer_test",
size = "small",
srcs = [
@@ -50,15 +38,3 @@
":lib",
],
)
-
-py_test(
- name = "wheel_test",
- size = "small",
- srcs = [
- "wheel_test.py",
- ],
- data = ["//examples/wheel:minimal_with_py_package"],
- deps = [
- ":lib",
- ],
-)
diff --git a/tests/pypi/whl_installer/arguments_test.py b/tests/pypi/whl_installer/arguments_test.py
index 5538054..9f73ae9 100644
--- a/tests/pypi/whl_installer/arguments_test.py
+++ b/tests/pypi/whl_installer/arguments_test.py
@@ -15,7 +15,7 @@
import json
import unittest
-from python.private.pypi.whl_installer import arguments, wheel
+from python.private.pypi.whl_installer import arguments
class ArgumentsTestCase(unittest.TestCase):
@@ -49,18 +49,6 @@
self.assertEqual(args["environment"], {"PIP_DO_SOMETHING": "True"})
self.assertEqual(args["extra_pip_args"], [])
- def test_platform_aggregation(self) -> None:
- parser = arguments.parser()
- args = parser.parse_args(
- args=[
- "--platform=linux_*",
- "--platform=osx_*",
- "--platform=windows_*",
- "--requirement=foo",
- ]
- )
- self.assertEqual(set(wheel.Platform.all()), arguments.get_platforms(args))
-
if __name__ == "__main__":
unittest.main()
diff --git a/tests/pypi/whl_installer/platform_test.py b/tests/pypi/whl_installer/platform_test.py
deleted file mode 100644
index 2aeb4ca..0000000
--- a/tests/pypi/whl_installer/platform_test.py
+++ /dev/null
@@ -1,154 +0,0 @@
-import unittest
-from random import shuffle
-
-from python.private.pypi.whl_installer.platform import (
- OS,
- Arch,
- Platform,
- host_interpreter_minor_version,
-)
-
-
-class MinorVersionTest(unittest.TestCase):
- def test_host(self):
- host = host_interpreter_minor_version()
- self.assertIsNotNone(host)
-
-
-class PlatformTest(unittest.TestCase):
- def test_can_get_host(self):
- host = Platform.host()
- self.assertIsNotNone(host)
- self.assertEqual(1, len(Platform.from_string("host")))
- self.assertEqual(host, Platform.from_string("host"))
-
- def test_can_get_linux_x86_64_without_py_version(self):
- got = Platform.from_string("linux_x86_64")
- want = Platform(os=OS.linux, arch=Arch.x86_64)
- self.assertEqual(want, got[0])
-
- def test_can_get_specific_from_string(self):
- got = Platform.from_string("cp33_linux_x86_64")
- want = Platform(os=OS.linux, arch=Arch.x86_64, minor_version=3)
- self.assertEqual(want, got[0])
-
- def test_can_get_all_for_py_version(self):
- cp39 = Platform.all(minor_version=9)
- self.assertEqual(21, len(cp39), f"Got {cp39}")
- self.assertEqual(cp39, Platform.from_string("cp39_*"))
-
- def test_can_get_all_for_os(self):
- linuxes = Platform.all(OS.linux, minor_version=9)
- self.assertEqual(7, len(linuxes))
- self.assertEqual(linuxes, Platform.from_string("cp39_linux_*"))
-
- def test_can_get_all_for_os_for_host_python(self):
- linuxes = Platform.all(OS.linux)
- self.assertEqual(7, len(linuxes))
- self.assertEqual(linuxes, Platform.from_string("linux_*"))
-
- def test_specific_version_specializations(self):
- any_py33 = Platform(minor_version=3)
-
- # When
- all_specializations = list(any_py33.all_specializations())
-
- want = (
- [any_py33]
- + [
- Platform(arch=arch, minor_version=any_py33.minor_version)
- for arch in Arch
- ]
- + [Platform(os=os, minor_version=any_py33.minor_version) for os in OS]
- + Platform.all(minor_version=any_py33.minor_version)
- )
- self.assertEqual(want, all_specializations)
-
- def test_aarch64_specializations(self):
- any_aarch64 = Platform(arch=Arch.aarch64)
- all_specializations = list(any_aarch64.all_specializations())
- want = [
- Platform(os=None, arch=Arch.aarch64),
- Platform(os=OS.linux, arch=Arch.aarch64),
- Platform(os=OS.osx, arch=Arch.aarch64),
- Platform(os=OS.windows, arch=Arch.aarch64),
- ]
- self.assertEqual(want, all_specializations)
-
- def test_linux_specializations(self):
- any_linux = Platform(os=OS.linux)
- all_specializations = list(any_linux.all_specializations())
- want = [
- Platform(os=OS.linux, arch=None),
- Platform(os=OS.linux, arch=Arch.x86_64),
- Platform(os=OS.linux, arch=Arch.x86_32),
- Platform(os=OS.linux, arch=Arch.aarch64),
- Platform(os=OS.linux, arch=Arch.ppc),
- Platform(os=OS.linux, arch=Arch.ppc64le),
- Platform(os=OS.linux, arch=Arch.s390x),
- Platform(os=OS.linux, arch=Arch.arm),
- ]
- self.assertEqual(want, all_specializations)
-
- def test_osx_specializations(self):
- any_osx = Platform(os=OS.osx)
- all_specializations = list(any_osx.all_specializations())
- # NOTE @aignas 2024-01-14: even though in practice we would only have
- # Python on osx aarch64 and osx x86_64, we return all arch posibilities
- # to make the code simpler.
- want = [
- Platform(os=OS.osx, arch=None),
- Platform(os=OS.osx, arch=Arch.x86_64),
- Platform(os=OS.osx, arch=Arch.x86_32),
- Platform(os=OS.osx, arch=Arch.aarch64),
- Platform(os=OS.osx, arch=Arch.ppc),
- Platform(os=OS.osx, arch=Arch.ppc64le),
- Platform(os=OS.osx, arch=Arch.s390x),
- Platform(os=OS.osx, arch=Arch.arm),
- ]
- self.assertEqual(want, all_specializations)
-
- def test_platform_sort(self):
- platforms = [
- Platform(os=OS.linux, arch=None),
- Platform(os=OS.linux, arch=Arch.x86_64),
- Platform(os=OS.osx, arch=None),
- Platform(os=OS.osx, arch=Arch.x86_64),
- Platform(os=OS.osx, arch=Arch.aarch64),
- ]
- shuffle(platforms)
- platforms.sort()
- want = [
- Platform(os=OS.linux, arch=None),
- Platform(os=OS.linux, arch=Arch.x86_64),
- Platform(os=OS.osx, arch=None),
- Platform(os=OS.osx, arch=Arch.x86_64),
- Platform(os=OS.osx, arch=Arch.aarch64),
- ]
-
- self.assertEqual(want, platforms)
-
- def test_wheel_os_alias(self):
- self.assertEqual("osx", str(OS.osx))
- self.assertEqual(str(OS.darwin), str(OS.osx))
-
- def test_wheel_arch_alias(self):
- self.assertEqual("x86_64", str(Arch.x86_64))
- self.assertEqual(str(Arch.amd64), str(Arch.x86_64))
-
- def test_wheel_platform_alias(self):
- give = Platform(
- os=OS.darwin,
- arch=Arch.amd64,
- )
- alias = Platform(
- os=OS.osx,
- arch=Arch.x86_64,
- )
-
- self.assertEqual("osx_x86_64", str(give))
- self.assertEqual(str(alias), str(give))
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/pypi/whl_installer/wheel_installer_test.py b/tests/pypi/whl_installer/wheel_installer_test.py
index 7139779..3c118af 100644
--- a/tests/pypi/whl_installer/wheel_installer_test.py
+++ b/tests/pypi/whl_installer/wheel_installer_test.py
@@ -22,39 +22,6 @@
from python.private.pypi.whl_installer import wheel_installer
-class TestRequirementExtrasParsing(unittest.TestCase):
- def test_parses_requirement_for_extra(self) -> None:
- cases = [
- ("name[foo]", ("name", frozenset(["foo"]))),
- ("name[ Foo123 ]", ("name", frozenset(["Foo123"]))),
- (" name1[ foo ] ", ("name1", frozenset(["foo"]))),
- ("Name[foo]", ("name", frozenset(["foo"]))),
- ("name_foo[bar]", ("name-foo", frozenset(["bar"]))),
- (
- "name [fred,bar] @ http://foo.com ; python_version=='2.7'",
- ("name", frozenset(["fred", "bar"])),
- ),
- (
- "name[quux, strange];python_version<'2.7' and platform_version=='2'",
- ("name", frozenset(["quux", "strange"])),
- ),
- (
- "name; (os_name=='a' or os_name=='b') and os_name=='c'",
- (None, None),
- ),
- (
- "name@http://foo.com",
- (None, None),
- ),
- ]
-
- for case, expected in cases:
- with self.subTest():
- self.assertTupleEqual(
- wheel_installer._parse_requirement_for_extra(case), expected
- )
-
-
class TestWhlFilegroup(unittest.TestCase):
def setUp(self) -> None:
self.wheel_name = "example_minimal_package-0.0.1-py3-none-any.whl"
@@ -68,10 +35,8 @@
def test_wheel_exists(self) -> None:
wheel_installer._extract_wheel(
Path(self.wheel_path),
- installation_dir=Path(self.wheel_dir),
- extras={},
enable_implicit_namespace_pkgs=False,
- platforms=[],
+ installation_dir=Path(self.wheel_dir),
)
want_files = [
@@ -92,11 +57,8 @@
metadata_file_content = json.load(metadata_file)
want = dict(
- version="0.0.1",
- name="example-minimal-package",
- deps=[],
- deps_by_platform={},
entry_points=[],
+ python_version="3.11.11",
)
self.assertEqual(want, metadata_file_content)
diff --git a/tests/pypi/whl_installer/wheel_test.py b/tests/pypi/whl_installer/wheel_test.py
deleted file mode 100644
index 404218e..0000000
--- a/tests/pypi/whl_installer/wheel_test.py
+++ /dev/null
@@ -1,371 +0,0 @@
-import unittest
-from unittest import mock
-
-from python.private.pypi.whl_installer import wheel
-from python.private.pypi.whl_installer.platform import OS, Arch, Platform
-
-_HOST_INTERPRETER_FN = (
- "python.private.pypi.whl_installer.wheel.host_interpreter_minor_version"
-)
-
-
-class DepsTest(unittest.TestCase):
- def test_simple(self):
- deps = wheel.Deps("foo", requires_dist=["bar"])
-
- got = deps.build()
-
- self.assertIsInstance(got, wheel.FrozenDeps)
- self.assertEqual(["bar"], got.deps)
- self.assertEqual({}, got.deps_select)
-
- def test_can_add_os_specific_deps(self):
- deps = wheel.Deps(
- "foo",
- requires_dist=[
- "bar",
- "an_osx_dep; sys_platform=='darwin'",
- "posix_dep; os_name=='posix'",
- "win_dep; os_name=='nt'",
- ],
- platforms={
- Platform(os=OS.linux, arch=Arch.x86_64),
- Platform(os=OS.osx, arch=Arch.x86_64),
- Platform(os=OS.osx, arch=Arch.aarch64),
- Platform(os=OS.windows, arch=Arch.x86_64),
- },
- )
-
- got = deps.build()
-
- self.assertEqual(["bar"], got.deps)
- self.assertEqual(
- {
- "@platforms//os:linux": ["posix_dep"],
- "@platforms//os:osx": ["an_osx_dep", "posix_dep"],
- "@platforms//os:windows": ["win_dep"],
- },
- got.deps_select,
- )
-
- def test_can_add_os_specific_deps_with_specific_python_version(self):
- deps = wheel.Deps(
- "foo",
- requires_dist=[
- "bar",
- "an_osx_dep; sys_platform=='darwin'",
- "posix_dep; os_name=='posix'",
- "win_dep; os_name=='nt'",
- ],
- platforms={
- Platform(os=OS.linux, arch=Arch.x86_64, minor_version=8),
- Platform(os=OS.osx, arch=Arch.x86_64, minor_version=8),
- Platform(os=OS.osx, arch=Arch.aarch64, minor_version=8),
- Platform(os=OS.windows, arch=Arch.x86_64, minor_version=8),
- },
- )
-
- got = deps.build()
-
- self.assertEqual(["bar"], got.deps)
- self.assertEqual(
- {
- "@platforms//os:linux": ["posix_dep"],
- "@platforms//os:osx": ["an_osx_dep", "posix_dep"],
- "@platforms//os:windows": ["win_dep"],
- },
- got.deps_select,
- )
-
- def test_deps_are_added_to_more_specialized_platforms(self):
- got = wheel.Deps(
- "foo",
- requires_dist=[
- "m1_dep; sys_platform=='darwin' and platform_machine=='arm64'",
- "mac_dep; sys_platform=='darwin'",
- ],
- platforms={
- Platform(os=OS.osx, arch=Arch.x86_64),
- Platform(os=OS.osx, arch=Arch.aarch64),
- },
- ).build()
-
- self.assertEqual(
- wheel.FrozenDeps(
- deps=[],
- deps_select={
- "osx_aarch64": ["m1_dep", "mac_dep"],
- "@platforms//os:osx": ["mac_dep"],
- },
- ),
- got,
- )
-
- def test_deps_from_more_specialized_platforms_are_propagated(self):
- got = wheel.Deps(
- "foo",
- requires_dist=[
- "a_mac_dep; sys_platform=='darwin'",
- "m1_dep; sys_platform=='darwin' and platform_machine=='arm64'",
- ],
- platforms={
- Platform(os=OS.osx, arch=Arch.x86_64),
- Platform(os=OS.osx, arch=Arch.aarch64),
- },
- ).build()
-
- self.assertEqual([], got.deps)
- self.assertEqual(
- {
- "osx_aarch64": ["a_mac_dep", "m1_dep"],
- "@platforms//os:osx": ["a_mac_dep"],
- },
- got.deps_select,
- )
-
- def test_non_platform_markers_are_added_to_common_deps(self):
- got = wheel.Deps(
- "foo",
- requires_dist=[
- "bar",
- "baz; implementation_name=='cpython'",
- "m1_dep; sys_platform=='darwin' and platform_machine=='arm64'",
- ],
- platforms={
- Platform(os=OS.linux, arch=Arch.x86_64),
- Platform(os=OS.osx, arch=Arch.x86_64),
- Platform(os=OS.osx, arch=Arch.aarch64),
- Platform(os=OS.windows, arch=Arch.x86_64),
- },
- ).build()
-
- self.assertEqual(["bar", "baz"], got.deps)
- self.assertEqual(
- {
- "osx_aarch64": ["m1_dep"],
- },
- got.deps_select,
- )
-
- def test_self_is_ignored(self):
- deps = wheel.Deps(
- "foo",
- requires_dist=[
- "bar",
- "req_dep; extra == 'requests'",
- "foo[requests]; extra == 'ssl'",
- "ssl_lib; extra == 'ssl'",
- ],
- extras={"ssl"},
- )
-
- got = deps.build()
-
- self.assertEqual(["bar", "req_dep", "ssl_lib"], got.deps)
- self.assertEqual({}, got.deps_select)
-
- def test_self_dependencies_can_come_in_any_order(self):
- deps = wheel.Deps(
- "foo",
- requires_dist=[
- "bar",
- "baz; extra == 'feat'",
- "foo[feat2]; extra == 'all'",
- "foo[feat]; extra == 'feat2'",
- "zdep; extra == 'all'",
- ],
- extras={"all"},
- )
-
- got = deps.build()
-
- self.assertEqual(["bar", "baz", "zdep"], got.deps)
- self.assertEqual({}, got.deps_select)
-
- def test_can_get_deps_based_on_specific_python_version(self):
- requires_dist = [
- "bar",
- "baz; python_version < '3.8'",
- "posix_dep; os_name=='posix' and python_version >= '3.8'",
- ]
-
- py38_deps = wheel.Deps(
- "foo",
- requires_dist=requires_dist,
- platforms=[
- Platform(os=OS.linux, arch=Arch.x86_64, minor_version=8),
- ],
- ).build()
- py37_deps = wheel.Deps(
- "foo",
- requires_dist=requires_dist,
- platforms=[
- Platform(os=OS.linux, arch=Arch.x86_64, minor_version=7),
- ],
- ).build()
-
- self.assertEqual(["bar", "baz"], py37_deps.deps)
- self.assertEqual({}, py37_deps.deps_select)
- self.assertEqual(["bar"], py38_deps.deps)
- self.assertEqual({"@platforms//os:linux": ["posix_dep"]}, py38_deps.deps_select)
-
- @mock.patch(_HOST_INTERPRETER_FN)
- def test_no_version_select_when_single_version(self, mock_host_interpreter_version):
- requires_dist = [
- "bar",
- "baz; python_version >= '3.8'",
- "posix_dep; os_name=='posix'",
- "posix_dep_with_version; os_name=='posix' and python_version >= '3.8'",
- "arch_dep; platform_machine=='x86_64' and python_version >= '3.8'",
- ]
- mock_host_interpreter_version.return_value = 7
-
- self.maxDiff = None
-
- deps = wheel.Deps(
- "foo",
- requires_dist=requires_dist,
- platforms=[
- Platform(os=os, arch=Arch.x86_64, minor_version=minor)
- for minor in [8]
- for os in [OS.linux, OS.windows]
- ],
- )
- got = deps.build()
-
- self.assertEqual(["bar", "baz"], got.deps)
- self.assertEqual(
- {
- "@platforms//os:linux": ["posix_dep", "posix_dep_with_version"],
- "linux_x86_64": ["arch_dep", "posix_dep", "posix_dep_with_version"],
- "windows_x86_64": ["arch_dep"],
- },
- got.deps_select,
- )
-
- @mock.patch(_HOST_INTERPRETER_FN)
- def test_can_get_version_select(self, mock_host_interpreter_version):
- requires_dist = [
- "bar",
- "baz; python_version < '3.8'",
- "baz_new; python_version >= '3.8'",
- "posix_dep; os_name=='posix'",
- "posix_dep_with_version; os_name=='posix' and python_version >= '3.8'",
- "arch_dep; platform_machine=='x86_64' and python_version < '3.8'",
- ]
- mock_host_interpreter_version.return_value = 7
-
- self.maxDiff = None
-
- deps = wheel.Deps(
- "foo",
- requires_dist=requires_dist,
- platforms=[
- Platform(os=os, arch=Arch.x86_64, minor_version=minor)
- for minor in [7, 8, 9]
- for os in [OS.linux, OS.windows]
- ],
- )
- got = deps.build()
-
- self.assertEqual(["bar"], got.deps)
- self.assertEqual(
- {
- "//conditions:default": ["baz"],
- "@//python/config_settings:is_python_3.7": ["baz"],
- "@//python/config_settings:is_python_3.8": ["baz_new"],
- "@//python/config_settings:is_python_3.9": ["baz_new"],
- "@platforms//os:linux": ["baz", "posix_dep"],
- "cp37_linux_x86_64": ["arch_dep", "baz", "posix_dep"],
- "cp37_windows_x86_64": ["arch_dep", "baz"],
- "cp37_linux_anyarch": ["baz", "posix_dep"],
- "cp38_linux_anyarch": [
- "baz_new",
- "posix_dep",
- "posix_dep_with_version",
- ],
- "cp39_linux_anyarch": [
- "baz_new",
- "posix_dep",
- "posix_dep_with_version",
- ],
- "linux_x86_64": ["arch_dep", "baz", "posix_dep"],
- "windows_x86_64": ["arch_dep", "baz"],
- },
- got.deps_select,
- )
-
- @mock.patch(_HOST_INTERPRETER_FN)
- def test_deps_spanning_all_target_py_versions_are_added_to_common(
- self, mock_host_version
- ):
- requires_dist = [
- "bar",
- "baz (<2,>=1.11) ; python_version < '3.8'",
- "baz (<2,>=1.14) ; python_version >= '3.8'",
- ]
- mock_host_version.return_value = 8
-
- deps = wheel.Deps(
- "foo",
- requires_dist=requires_dist,
- platforms=Platform.from_string(["cp37_*", "cp38_*", "cp39_*"]),
- )
- got = deps.build()
-
- self.assertEqual(["bar", "baz"], got.deps)
- self.assertEqual({}, got.deps_select)
-
- @mock.patch(_HOST_INTERPRETER_FN)
- def test_deps_are_not_duplicated(self, mock_host_version):
- mock_host_version.return_value = 7
-
- # See an example in
- # https://files.pythonhosted.org/packages/76/9e/db1c2d56c04b97981c06663384f45f28950a73d9acf840c4006d60d0a1ff/opencv_python-4.9.0.80-cp37-abi3-win32.whl.metadata
- requires_dist = [
- "bar >=0.1.0 ; python_version < '3.7'",
- "bar >=0.2.0 ; python_version >= '3.7'",
- "bar >=0.4.0 ; python_version >= '3.6' and platform_system == 'Linux' and platform_machine == 'aarch64'",
- "bar >=0.4.0 ; python_version >= '3.9'",
- "bar >=0.5.0 ; python_version <= '3.9' and platform_system == 'Darwin' and platform_machine == 'arm64'",
- "bar >=0.5.0 ; python_version >= '3.10' and platform_system == 'Darwin'",
- "bar >=0.5.0 ; python_version >= '3.10'",
- "bar >=0.6.0 ; python_version >= '3.11'",
- ]
-
- deps = wheel.Deps(
- "foo",
- requires_dist=requires_dist,
- platforms=Platform.from_string(["cp37_*", "cp310_*"]),
- )
- got = deps.build()
-
- self.assertEqual(["bar"], got.deps)
- self.assertEqual({}, got.deps_select)
-
- @mock.patch(_HOST_INTERPRETER_FN)
- def test_deps_are_not_duplicated_when_encountering_platform_dep_first(
- self, mock_host_version
- ):
- mock_host_version.return_value = 7
-
- # Note, that we are sorting the incoming `requires_dist` and we need to ensure that we are not getting any
- # issues even if the platform-specific line comes first.
- requires_dist = [
- "bar >=0.4.0 ; python_version >= '3.6' and platform_system == 'Linux' and platform_machine == 'aarch64'",
- "bar >=0.5.0 ; python_version >= '3.9'",
- ]
-
- deps = wheel.Deps(
- "foo",
- requires_dist=requires_dist,
- platforms=Platform.from_string(["cp37_*", "cp310_*"]),
- )
- got = deps.build()
-
- self.assertEqual(["bar"], got.deps)
- self.assertEqual({}, got.deps_select)
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/pypi/whl_metadata/BUILD.bazel b/tests/pypi/whl_metadata/BUILD.bazel
new file mode 100644
index 0000000..3f1d665
--- /dev/null
+++ b/tests/pypi/whl_metadata/BUILD.bazel
@@ -0,0 +1,5 @@
+load(":whl_metadata_tests.bzl", "whl_metadata_test_suite")
+
+whl_metadata_test_suite(
+ name = "whl_metadata_tests",
+)
diff --git a/tests/pypi/whl_metadata/whl_metadata_tests.bzl b/tests/pypi/whl_metadata/whl_metadata_tests.bzl
new file mode 100644
index 0000000..4acbc92
--- /dev/null
+++ b/tests/pypi/whl_metadata/whl_metadata_tests.bzl
@@ -0,0 +1,147 @@
+""
+
+load("@rules_testing//lib:test_suite.bzl", "test_suite")
+load("@rules_testing//lib:truth.bzl", "subjects")
+load(
+ "//python/private/pypi:whl_metadata.bzl",
+ "find_whl_metadata",
+ "parse_whl_metadata",
+) # buildifier: disable=bzl-visibility
+
+_tests = []
+
+def _test_empty(env):
+ fake_path = struct(
+ basename = "site-packages",
+ readdir = lambda watch = None: [],
+ )
+ fail_messages = []
+ find_whl_metadata(install_dir = fake_path, logger = struct(
+ fail = fail_messages.append,
+ ))
+ env.expect.that_collection(fail_messages).contains_exactly([
+ "The '*.dist-info' directory could not be found in 'site-packages'",
+ ])
+
+_tests.append(_test_empty)
+
+def _test_contains_dist_info_but_no_metadata(env):
+ fake_path = struct(
+ basename = "site-packages",
+ readdir = lambda watch = None: [
+ struct(
+ basename = "something.dist-info",
+ is_dir = True,
+ get_child = lambda basename: struct(
+ basename = basename,
+ exists = False,
+ ),
+ ),
+ ],
+ )
+ fail_messages = []
+ find_whl_metadata(install_dir = fake_path, logger = struct(
+ fail = fail_messages.append,
+ ))
+ env.expect.that_collection(fail_messages).contains_exactly([
+ "The METADATA file for the wheel could not be found in 'site-packages/something.dist-info'",
+ ])
+
+_tests.append(_test_contains_dist_info_but_no_metadata)
+
+def _test_contains_metadata(env):
+ fake_path = struct(
+ basename = "site-packages",
+ readdir = lambda watch = None: [
+ struct(
+ basename = "something.dist-info",
+ is_dir = True,
+ get_child = lambda basename: struct(
+ basename = basename,
+ exists = True,
+ ),
+ ),
+ ],
+ )
+ fail_messages = []
+ got = find_whl_metadata(install_dir = fake_path, logger = struct(
+ fail = fail_messages.append,
+ ))
+ env.expect.that_collection(fail_messages).contains_exactly([])
+ env.expect.that_str(got.basename).equals("METADATA")
+
+_tests.append(_test_contains_metadata)
+
+def _parse_whl_metadata(env, **kwargs):
+ result = parse_whl_metadata(**kwargs)
+
+ return env.expect.that_struct(
+ struct(
+ name = result.name,
+ version = result.version,
+ requires_dist = result.requires_dist,
+ provides_extra = result.provides_extra,
+ ),
+ attrs = dict(
+ name = subjects.str,
+ version = subjects.str,
+ requires_dist = subjects.collection,
+ provides_extra = subjects.collection,
+ ),
+ )
+
+def _test_parse_metadata_invalid(env):
+ got = _parse_whl_metadata(
+ env,
+ contents = "",
+ )
+ got.name().equals("")
+ got.version().equals("")
+ got.requires_dist().contains_exactly([])
+ got.provides_extra().contains_exactly([])
+
+_tests.append(_test_parse_metadata_invalid)
+
+def _test_parse_metadata_basic(env):
+ got = _parse_whl_metadata(
+ env,
+ contents = """\
+Name: foo
+Version: 0.0.1
+""",
+ )
+ got.name().equals("foo")
+ got.version().equals("0.0.1")
+ got.requires_dist().contains_exactly([])
+ got.provides_extra().contains_exactly([])
+
+_tests.append(_test_parse_metadata_basic)
+
+def _test_parse_metadata_all(env):
+ got = _parse_whl_metadata(
+ env,
+ contents = """\
+Name: foo
+Version: 0.0.1
+Requires-Dist: bar; extra == "all"
+Provides-Extra: all
+
+Requires-Dist: this will be ignored
+""",
+ )
+ got.name().equals("foo")
+ got.version().equals("0.0.1")
+ got.requires_dist().contains_exactly([
+ "bar; extra == \"all\"",
+ ])
+ got.provides_extra().contains_exactly([
+ "all",
+ ])
+
+_tests.append(_test_parse_metadata_all)
+
+def whl_metadata_test_suite(name): # buildifier: disable=function-docstring
+ test_suite(
+ name = name,
+ basic_tests = _tests,
+ )
