refactor/fix: store dists in parse_requirements output (#1917)
This moves some of the code out of the `pip.bzl` extension and changes
the layout of the code to prepare for multi-platform whl support.
Summary:
* parse_requirements: add whls and sdists attribute, so that we can use
a function to populate the lists. Not sure if there is a better way to
do this.
* parse_requirements: add an extra code to ensure that we are handling
the target platform filtering correctly.
* select_whl: split the `select_whl` into `select_whls`, which filters
the whls (this can be used later in multi-platform selects) and
select_whl , which just is used get the most appropriate whl for the
host platform.
* Additionally fix the logic in `select_whl`, which would result in
Python 3.12 wheels being selected on Python 3.11 interpreters because
we were not taking into account the interpreter tag when doing the
filtering.
Fixes #1930
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e15be3b..3fdd039 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -49,6 +49,8 @@
"panic: runtime error: invalid memory address or nil pointer dereference"
* (bzlmod) remove `pip.parse(annotations)` attribute as it is unused and has been
replaced by whl_modifications.
+* (pip) Correctly select wheels when the python tag includes minor versions.
+ See ([#1930](https://github.com/bazelbuild/rules_python/issues/1930))
### Added
* (rules) Precompiling Python source at build time is available. but is
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl
index 17d8083..d6c8d91 100644
--- a/python/pip_install/pip_repository.bzl
+++ b/python/pip_install/pip_repository.bzl
@@ -584,7 +584,17 @@
),
"quiet": attr.bool(
default = True,
- doc = "If True, suppress printing stdout and stderr output to the terminal.",
+ doc = """\
+If True, suppress printing stdout and stderr output to the terminal.
+
+If you would like to get more diagnostic output, please use:
+
+ RULES_PYTHON_REPO_DEBUG=1
+
+or
+
+ RULES_PYTHON_REPO_DEBUG_VERBOSITY=<INFO|DEBUG|TRACE>
+""",
),
"repo_prefix": attr.string(
doc = """
diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl
index aa70810..9e29877 100644
--- a/python/private/bzlmod/pip.bzl
+++ b/python/private/bzlmod/pip.bzl
@@ -28,6 +28,7 @@
load("//python/private:parse_whl_name.bzl", "parse_whl_name")
load("//python/private:pypi_index.bzl", "simpleapi_download")
load("//python/private:render_pkg_aliases.bzl", "whl_alias")
+load("//python/private:repo_utils.bzl", "repo_utils")
load("//python/private:version_label.bzl", "version_label")
load("//python/private:whl_target_platforms.bzl", "select_whl")
load(":pip_repository.bzl", "pip_repository")
@@ -100,6 +101,7 @@
)
def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, simpleapi_cache):
+ logger = repo_utils.logger(module_ctx)
python_interpreter_target = pip_attr.python_interpreter_target
# if we do not have the python_interpreter set in the attributes
@@ -160,32 +162,18 @@
# Create a new wheel library for each of the different whls
- requirements_by_platform = parse_requirements(
- module_ctx,
- requirements_by_platform = pip_attr.requirements_by_platform,
- requirements_linux = pip_attr.requirements_linux,
- requirements_lock = pip_attr.requirements_lock,
- requirements_osx = pip_attr.requirements_darwin,
- requirements_windows = pip_attr.requirements_windows,
- extra_pip_args = pip_attr.extra_pip_args,
- )
-
- index_urls = {}
+ get_index_urls = None
if pip_attr.experimental_index_url:
if pip_attr.download_only:
fail("Currently unsupported to use `download_only` and `experimental_index_url`")
- index_urls = simpleapi_download(
- module_ctx,
+ get_index_urls = lambda ctx, distributions: simpleapi_download(
+ ctx,
attr = struct(
index_url = pip_attr.experimental_index_url,
extra_index_urls = pip_attr.experimental_extra_index_urls or [],
index_url_overrides = pip_attr.experimental_index_url_overrides or {},
- sources = list({
- req.distribution: None
- for reqs in requirements_by_platform.values()
- for req in reqs
- }),
+ sources = distributions,
envsubst = pip_attr.envsubst,
# Auth related info
netrc = pip_attr.netrc,
@@ -195,6 +183,19 @@
parallel_download = pip_attr.parallel_download,
)
+ requirements_by_platform = parse_requirements(
+ module_ctx,
+ requirements_by_platform = pip_attr.requirements_by_platform,
+ requirements_linux = pip_attr.requirements_linux,
+ requirements_lock = pip_attr.requirements_lock,
+ requirements_osx = pip_attr.requirements_darwin,
+ requirements_windows = pip_attr.requirements_windows,
+ extra_pip_args = pip_attr.extra_pip_args,
+ get_index_urls = get_index_urls,
+ python_version = major_minor,
+ logger = logger,
+ )
+
repository_platform = host_platform(module_ctx.os)
for whl_name, requirements in requirements_by_platform.items():
requirement = select_requirement(
@@ -255,37 +256,22 @@
)
whl_library_args.update({k: v for k, (v, default) in maybe_args_with_default.items() if v == default})
- if index_urls:
- whls = []
- sdist = None
- for sha256 in requirement.srcs.shas:
- # For now if the artifact is marked as yanked we just ignore it.
- #
- # See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api
-
- maybe_whl = index_urls[whl_name].whls.get(sha256)
- if maybe_whl and not maybe_whl.yanked:
- whls.append(maybe_whl)
- continue
-
- maybe_sdist = index_urls[whl_name].sdists.get(sha256)
- if maybe_sdist and not maybe_sdist.yanked:
- sdist = maybe_sdist
- continue
-
- print("WARNING: Could not find a whl or an sdist with sha256={}".format(sha256)) # buildifier: disable=print
-
+ if requirement.whls or requirement.sdist:
+ logger.debug(lambda: "Selecting a compatible dist for {} from dists:\n{}".format(
+ repository_platform,
+ json.encode(
+ struct(
+ whls = requirement.whls,
+ sdist = requirement.sdist,
+ ),
+ ),
+ ))
distribution = select_whl(
- whls = whls,
- want_abis = [
- "none",
- "abi3",
- "cp" + major_minor.replace(".", ""),
- # Older python versions have wheels for the `*m` ABI.
- "cp" + major_minor.replace(".", "") + "m",
- ],
+ whls = requirement.whls,
want_platform = repository_platform,
- ) or sdist
+ ) or requirement.sdist
+
+ logger.debug(lambda: "Selected: {}".format(distribution))
if distribution:
whl_library_args["requirement"] = requirement.srcs.requirement
@@ -303,7 +289,7 @@
# This is no-op because pip is not used to download the wheel.
whl_library_args.pop("download_only", None)
else:
- print("WARNING: falling back to pip for installing the right file for {}".format(requirement.requirement_line)) # buildifier: disable=print
+ logger.warn("falling back to pip for installing the right file for {}".format(requirement.requirement_line))
# We sort so that the lock-file remains the same no matter the order of how the
# args are manipulated in the code going before.
diff --git a/python/private/parse_requirements.bzl b/python/private/parse_requirements.bzl
index f9d7a05..c6a4985 100644
--- a/python/private/parse_requirements.bzl
+++ b/python/private/parse_requirements.bzl
@@ -29,7 +29,7 @@
load("//python/pip_install:requirements_parser.bzl", "parse")
load(":normalize_name.bzl", "normalize_name")
load(":pypi_index_sources.bzl", "get_simpleapi_sources")
-load(":whl_target_platforms.bzl", "whl_target_platforms")
+load(":whl_target_platforms.bzl", "select_whls", "whl_target_platforms")
# This includes the vendored _translate_cpu and _translate_os from
# @platforms//host:extension.bzl at version 0.0.9 so that we don't
@@ -84,6 +84,11 @@
if not filter:
fail("Must specific a filter string, got: {}".format(filter))
+ if filter.startswith("cp3"):
+ # TODO @aignas 2024-05-23: properly handle python versions in the filter.
+ # For now we are just dropping it to ensure that we don't fail.
+ _, _, filter = filter.partition("_")
+
sanitized = filter.replace("*", "").replace("_", "")
if sanitized and not sanitized.isalnum():
fail("The platform filter can only contain '*', '_' and alphanumerics")
@@ -142,6 +147,9 @@
requirements_lock = None,
requirements_windows = None,
extra_pip_args = [],
+ get_index_urls = None,
+ python_version = None,
+ logger = None,
fail_fn = fail):
"""Get the requirements with platforms that the requirements apply to.
@@ -156,6 +164,12 @@
requirements_windows (label): The requirements file for windows OS.
extra_pip_args (string list): Extra pip arguments to perform extra validations and to
be joined with args fined in files.
+ get_index_urls: Callable[[ctx, list[str]], dict], a callable to get all
+ of the distribution URLs from a PyPI index. Accepts ctx and
+ distribution names to query.
+ python_version: str or None. This is needed when the get_index_urls is
+ specified. It should be of the form "3.x.x",
+ logger: repo_utils.logger or None, a simple struct to log diagnostic messages.
fail_fn (Callable[[str], None]): A failure function used in testing failure cases.
Returns:
@@ -312,20 +326,46 @@
)
for_req.target_platforms.append(target_platform)
- return {
- whl_name: [
- struct(
- distribution = r.distribution,
- srcs = r.srcs,
- requirement_line = r.requirement_line,
- target_platforms = sorted(r.target_platforms),
- extra_pip_args = r.extra_pip_args,
- download = r.download,
+ index_urls = {}
+ if get_index_urls:
+ if not python_version:
+ fail_fn("'python_version' must be provided")
+ return None
+
+ index_urls = get_index_urls(
+ ctx,
+ # Use list({}) as a way to have a set
+ list({
+ req.distribution: None
+ for reqs in requirements_by_platform.values()
+ for req in reqs.values()
+ }),
+ )
+
+ ret = {}
+ for whl_name, reqs in requirements_by_platform.items():
+ for r in sorted(reqs.values(), key = lambda r: r.requirement_line):
+ whls, sdist = _add_dists(
+ r,
+ index_urls.get(whl_name),
+ python_version = python_version,
+ logger = logger,
)
- for r in sorted(reqs.values(), key = lambda r: r.requirement_line)
- ]
- for whl_name, reqs in requirements_by_platform.items()
- }
+
+ ret.setdefault(whl_name, []).append(
+ struct(
+ distribution = r.distribution,
+ srcs = r.srcs,
+ requirement_line = r.requirement_line,
+ target_platforms = sorted(r.target_platforms),
+ extra_pip_args = r.extra_pip_args,
+ download = r.download,
+ whls = whls,
+ sdist = sdist,
+ ),
+ )
+
+ return ret
def select_requirement(requirements, *, platform):
"""A simple function to get a requirement for a particular platform.
@@ -372,3 +412,58 @@
_translate_os(repository_os.name.lower()),
_translate_cpu(repository_os.arch.lower()),
)
+
+def _add_dists(requirement, index_urls, python_version, logger = None):
+ """Populate dists based on the information from the PyPI index.
+
+ This function will modify the given requirements_by_platform data structure.
+
+ Args:
+ requirement: The result of parse_requirements function.
+ index_urls: The result of simpleapi_download.
+ python_version: The version of the python interpreter.
+ logger: A logger for printing diagnostic info.
+ """
+ if not index_urls:
+ return [], None
+
+ whls = []
+ sdist = None
+
+ # TODO @aignas 2024-05-22: it is in theory possible to add all
+ # requirements by version instead of by sha256. This may be useful
+ # for some projects.
+ for sha256 in requirement.srcs.shas:
+ # For now if the artifact is marked as yanked we just ignore it.
+ #
+ # See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api
+
+ maybe_whl = index_urls.whls.get(sha256)
+ if maybe_whl and not maybe_whl.yanked:
+ whls.append(maybe_whl)
+ continue
+
+ maybe_sdist = index_urls.sdists.get(sha256)
+ if maybe_sdist and not maybe_sdist.yanked:
+ sdist = maybe_sdist
+ continue
+
+ if logger:
+ logger.warn("Could not find a whl or an sdist with sha256={}".format(sha256))
+
+ # Filter out the wheels that are incompatible with the target_platforms.
+ whls = select_whls(
+ whls = whls,
+ want_abis = [
+ "none",
+ "abi3",
+ "cp" + python_version.replace(".", ""),
+ # Older python versions have wheels for the `*m` ABI.
+ "cp" + python_version.replace(".", "") + "m",
+ ],
+ want_platforms = requirement.target_platforms,
+ want_python_version = python_version,
+ logger = logger,
+ )
+
+ return whls, sdist
diff --git a/python/private/parse_whl_name.bzl b/python/private/parse_whl_name.bzl
index 9c7866e..063ac84 100644
--- a/python/private/parse_whl_name.bzl
+++ b/python/private/parse_whl_name.bzl
@@ -16,6 +16,30 @@
A starlark implementation of a Wheel filename parsing.
"""
+# Taken from https://peps.python.org/pep-0600/
+_LEGACY_ALIASES = {
+ "manylinux1_i686": "manylinux_2_5_i686",
+ "manylinux1_x86_64": "manylinux_2_5_x86_64",
+ "manylinux2010_i686": "manylinux_2_12_i686",
+ "manylinux2010_x86_64": "manylinux_2_12_x86_64",
+ "manylinux2014_aarch64": "manylinux_2_17_aarch64",
+ "manylinux2014_armv7l": "manylinux_2_17_armv7l",
+ "manylinux2014_i686": "manylinux_2_17_i686",
+ "manylinux2014_ppc64": "manylinux_2_17_ppc64",
+ "manylinux2014_ppc64le": "manylinux_2_17_ppc64le",
+ "manylinux2014_s390x": "manylinux_2_17_s390x",
+ "manylinux2014_x86_64": "manylinux_2_17_x86_64",
+}
+
+def normalize_platform_tag(tag):
+ """Resolve legacy aliases to modern equivalents for easier parsing elsewhere."""
+ return ".".join(list({
+ # The `list({})` usage here is to use it as a string set, where we will
+ # deduplicate, but otherwise retain the order of the tags.
+ _LEGACY_ALIASES.get(p, p): None
+ for p in tag.split(".")
+ }))
+
def parse_whl_name(file):
"""Parse whl file name into a struct of constituents.
@@ -68,5 +92,5 @@
build_tag = build_tag,
python_tag = python_tag,
abi_tag = abi_tag,
- platform_tag = platform_tag,
+ platform_tag = normalize_platform_tag(platform_tag),
)
diff --git a/python/private/repo_utils.bzl b/python/private/repo_utils.bzl
index 7a59217..5267758 100644
--- a/python/private/repo_utils.bzl
+++ b/python/private/repo_utils.bzl
@@ -18,6 +18,7 @@
"""
REPO_DEBUG_ENV_VAR = "RULES_PYTHON_REPO_DEBUG"
+REPO_VERBOSITY_ENV_VAR = "RULES_PYTHON_REPO_DEBUG_VERBOSITY"
def _is_repo_debug_enabled(rctx):
"""Tells if debbugging output is requested during repo operatiosn.
@@ -41,6 +42,42 @@
if _is_repo_debug_enabled(rctx):
print(message_cb()) # buildifier: disable=print
+def _logger(rctx):
+ """Creates a logger instance for printing messages.
+
+ Args:
+ rctx: repository_ctx object.
+
+ Returns:
+ A struct with attributes logging: trace, debug, info, warn, fail.
+ """
+ if _is_repo_debug_enabled(rctx):
+ verbosity_level = "DEBUG"
+ else:
+ verbosity_level = "WARN"
+
+ env_var_verbosity = rctx.os.environ.get(REPO_VERBOSITY_ENV_VAR)
+ verbosity_level = env_var_verbosity or verbosity_level
+
+ verbosity = {
+ "DEBUG": 2,
+ "INFO": 1,
+ "TRACE": 3,
+ }.get(verbosity_level, 0)
+
+ def _log(enabled_on_verbosity, level, message_cb):
+ if verbosity < enabled_on_verbosity:
+ return
+
+ print("\nrules_python: {}: ".format(level.upper()), message_cb()) # buildifier: disable=print
+
+ return struct(
+ trace = lambda message_cb: _log(3, "TRACE", message_cb),
+ debug = lambda message_cb: _log(2, "DEBUG", message_cb),
+ info = lambda message_cb: _log(1, "INFO", message_cb),
+ warn = lambda message_cb: _log(0, "WARNING", message_cb),
+ )
+
def _execute_internal(
rctx,
*,
@@ -232,4 +269,5 @@
is_repo_debug_enabled = _is_repo_debug_enabled,
debug_print = _debug_print,
which_checked = _which_checked,
+ logger = _logger,
)
diff --git a/python/private/whl_target_platforms.bzl b/python/private/whl_target_platforms.bzl
index 14e178a..678c841 100644
--- a/python/private/whl_target_platforms.bzl
+++ b/python/private/whl_target_platforms.bzl
@@ -18,21 +18,6 @@
load(":parse_whl_name.bzl", "parse_whl_name")
-# Taken from https://peps.python.org/pep-0600/
-_LEGACY_ALIASES = {
- "manylinux1_i686": "manylinux_2_5_i686",
- "manylinux1_x86_64": "manylinux_2_5_x86_64",
- "manylinux2010_i686": "manylinux_2_12_i686",
- "manylinux2010_x86_64": "manylinux_2_12_x86_64",
- "manylinux2014_aarch64": "manylinux_2_17_aarch64",
- "manylinux2014_armv7l": "manylinux_2_17_armv7l",
- "manylinux2014_i686": "manylinux_2_17_i686",
- "manylinux2014_ppc64": "manylinux_2_17_ppc64",
- "manylinux2014_ppc64le": "manylinux_2_17_ppc64le",
- "manylinux2014_s390x": "manylinux_2_17_s390x",
- "manylinux2014_x86_64": "manylinux_2_17_x86_64",
-}
-
# The order of the dictionaries is to keep definitions with their aliases next to each
# other
_CPU_ALIASES = {
@@ -48,6 +33,7 @@
"ppc64": "ppc",
"ppc64le": "ppc",
"s390x": "s390x",
+ "arm": "arm",
"armv6l": "arm",
"armv7l": "arm",
} # buildifier: disable=unsorted-dict-items
@@ -90,7 +76,6 @@
value, _, _ = value.partition(".")
if "any" == value:
- # This is just a big value that should be larger than any other value returned by this function
return (True, False, 0, 0)
if "linux" in value:
@@ -118,12 +103,111 @@
# Windows does not have multiple wheels for the same target platform
return (False, False, 0, 0)
-def select_whl(*, whls, want_abis, want_platform):
+def select_whls(*, whls, want_python_version = "3.0", want_abis = [], want_platforms = [], logger = None):
+ """Select a subset of wheels suitable for target platforms from a list.
+
+ Args:
+ whls(list[struct]): A list of candidates which have a `filename`
+ attribute containing the `whl` filename.
+ want_python_version(str): An optional parameter to filter whls by python version. Defaults to '3.0'.
+ want_abis(list[str]): A list of ABIs that are supported.
+ want_platforms(str): The platforms
+ logger: A logger for printing diagnostic messages.
+
+ Returns:
+ A filtered list of items from the `whls` arg where `filename` matches
+ the selected criteria. If no match is found, an empty list is returned.
+ """
+ if not whls:
+ return []
+
+ version_limit = -1
+ if want_python_version:
+ version_limit = int(want_python_version.split(".")[1])
+
+ candidates = {}
+ for whl in whls:
+ parsed = parse_whl_name(whl.filename)
+
+ if logger:
+ logger.trace(lambda: "Deciding whether to use '{}'".format(whl.filename))
+
+ supported_implementations = {}
+ whl_version_min = 0
+ for tag in parsed.python_tag.split("."):
+ supported_implementations[tag[:2]] = None
+
+ if tag.startswith("cp3") or tag.startswith("py3"):
+ version = int(tag[len("..3"):] or 0)
+ else:
+ # In this case it should be eithor "cp2" or "py2" and we will default
+ # to `whl_version_min` = 0
+ continue
+
+ if whl_version_min == 0 or version < whl_version_min:
+ whl_version_min = version
+
+ if not ("cp" in supported_implementations or "py" in supported_implementations):
+ if logger:
+ logger.trace(lambda: "Discarding the whl because the whl does not support CPython, whl supported implementations are: {}".format(supported_implementations))
+ continue
+
+ if want_abis and parsed.abi_tag not in want_abis:
+ # Filter out incompatible ABIs
+ if logger:
+ logger.trace(lambda: "Discarding the whl because the whl abi did not match")
+ continue
+
+ if version_limit != -1 and whl_version_min > version_limit:
+ if logger:
+ logger.trace(lambda: "Discarding the whl because the whl supported python version is too high")
+ continue
+
+ compatible = False
+ if parsed.platform_tag == "any":
+ compatible = True
+ else:
+ for p in whl_target_platforms(parsed.platform_tag):
+ if p.target_platform in want_platforms:
+ compatible = True
+ break
+
+ if not compatible:
+ if logger:
+ logger.trace(lambda: "Discarding the whl because the whl does not support the desired platforms: {}".format(want_platforms))
+ continue
+
+ for implementation in supported_implementations:
+ candidates.setdefault(
+ (
+ parsed.abi_tag,
+ parsed.platform_tag,
+ ),
+ {},
+ ).setdefault(
+ (
+ # prefer cp implementation
+ implementation == "cp",
+ # prefer higher versions
+ whl_version_min,
+ # prefer abi3 over none
+ parsed.abi_tag != "none",
+ # prefer cpx abi over abi3
+ parsed.abi_tag != "abi3",
+ ),
+ [],
+ ).append(whl)
+
+ return [
+ candidates[key][sorted(v)[-1]][-1]
+ for key, v in candidates.items()
+ ]
+
+def select_whl(*, whls, want_platform):
"""Select a suitable wheel from a list.
Args:
whls(list[struct]): A list of candidates.
- want_abis(list[str]): A list of ABIs that are supported.
want_platform(str): The target platform.
Returns:
@@ -133,53 +217,31 @@
if not whls:
return None
- candidates = {}
- for whl in whls:
- parsed = parse_whl_name(whl.filename)
- if parsed.abi_tag not in want_abis:
- # Filter out incompatible ABIs
- continue
+ # TODO @aignas 2024-05-23: once we do the selection in the hub repo using
+ # an actual select, then this function will be the one that is used within
+ # the repository context instead of `select_whl`.
+ whls = select_whls(
+ whls = whls,
+ want_python_version = "",
+ want_platforms = [want_platform],
+ )
- platform_tags = list({_LEGACY_ALIASES.get(p, p): True for p in parsed.platform_tag.split(".")})
+ candidates = {
+ parse_whl_name(w.filename).platform_tag: w
+ for w in whls
+ # TODO @aignas 2024-06-01: to be addressed in #1837, where we add the necessary
+ # config settings.
+ if "musllinux_" not in w.filename
+ }
- for tag in platform_tags:
- candidates[tag] = whl
+ target_whl_platform = sorted(
+ candidates.keys(),
+ key = _whl_priority,
+ )
+ if not target_whl_platform:
+ return None
- # For most packages - if they supply 'any' wheel and there are no other
- # compatible wheels with the selected abis, we can just return the value.
- if len(candidates) == 1 and "any" in candidates:
- return struct(
- url = candidates["any"].url,
- sha256 = candidates["any"].sha256,
- filename = candidates["any"].filename,
- )
-
- target_plats = {}
- has_any = "any" in candidates
- for platform_tag, whl in candidates.items():
- if platform_tag == "any":
- continue
-
- if "musl" in platform_tag:
- # Ignore musl wheels for now
- continue
-
- platform_tag = ".".join({_LEGACY_ALIASES.get(p, p): True for p in platform_tag.split(".")})
- platforms = whl_target_platforms(platform_tag)
- for p in platforms:
- target_plats.setdefault("{}_{}".format(p.os, p.cpu), []).append(platform_tag)
-
- for p, platform_tags in target_plats.items():
- if has_any:
- platform_tags.append("any")
-
- target_plats[p] = sorted(platform_tags, key = _whl_priority)
-
- want = target_plats.get(want_platform)
- if not want:
- return want
-
- return candidates[want[0]]
+ return candidates[target_whl_platform[0]]
def whl_target_platforms(platform_tag, abi_tag = ""):
"""Parse the wheel abi and platform tags and return (os, cpu) tuples.
diff --git a/tests/private/parse_requirements/parse_requirements_tests.bzl b/tests/private/parse_requirements/parse_requirements_tests.bzl
index 0d6cd4e..81cf523 100644
--- a/tests/private/parse_requirements/parse_requirements_tests.bzl
+++ b/tests/private/parse_requirements/parse_requirements_tests.bzl
@@ -96,6 +96,8 @@
"osx_x86_64",
"windows_x86_64",
],
+ whls = [],
+ sdist = None,
),
],
})
@@ -109,6 +111,47 @@
_tests.append(_test_simple)
+def _test_platform_markers_with_python_version(env):
+ got = parse_requirements(
+ ctx = _mock_ctx(),
+ requirements_by_platform = {
+ "requirements_lock": "cp39_linux_*",
+ },
+ )
+ got_alternative = parse_requirements(
+ ctx = _mock_ctx(),
+ requirements_by_platform = {
+ "requirements_lock": "linux_*",
+ },
+ )
+ env.expect.that_dict(got).contains_exactly({
+ "foo": [
+ struct(
+ distribution = "foo",
+ download = False,
+ extra_pip_args = [],
+ requirement_line = "foo[extra]==0.0.1 --hash=sha256:deadbeef",
+ srcs = struct(
+ requirement = "foo[extra]==0.0.1",
+ shas = ["deadbeef"],
+ version = "0.0.1",
+ ),
+ target_platforms = [
+ "linux_aarch64",
+ "linux_arm",
+ "linux_ppc",
+ "linux_s390x",
+ "linux_x86_64",
+ ],
+ whls = [],
+ sdist = None,
+ ),
+ ],
+ })
+ env.expect.that_dict(got).contains_exactly(got_alternative)
+
+_tests.append(_test_platform_markers_with_python_version)
+
def _test_dupe_requirements(env):
got = parse_requirements(
ctx = _mock_ctx(),
@@ -136,6 +179,8 @@
"osx_x86_64",
"windows_x86_64",
],
+ whls = [],
+ sdist = None,
),
],
})
@@ -173,6 +218,8 @@
version = "0.0.1",
),
target_platforms = ["windows_x86_64"],
+ whls = [],
+ sdist = None,
),
],
"foo": [
@@ -195,6 +242,8 @@
"osx_aarch64",
"osx_x86_64",
],
+ whls = [],
+ sdist = None,
),
struct(
distribution = "foo",
@@ -207,6 +256,8 @@
version = "0.0.2",
),
target_platforms = ["windows_x86_64"],
+ whls = [],
+ sdist = None,
),
],
})
@@ -264,6 +315,8 @@
version = "0.0.3",
),
target_platforms = ["linux_x86_64"],
+ whls = [],
+ sdist = None,
),
],
})
@@ -316,6 +369,8 @@
version = "0.0.3",
),
target_platforms = ["linux_aarch64", "linux_x86_64"],
+ whls = [],
+ sdist = None,
),
struct(
distribution = "foo",
@@ -328,6 +383,8 @@
version = "",
),
target_platforms = ["linux_super_exotic"],
+ whls = [],
+ sdist = None,
),
struct(
distribution = "foo",
@@ -347,6 +404,8 @@
"osx_x86_64",
"windows_x86_64",
],
+ whls = [],
+ sdist = None,
),
],
})
@@ -365,6 +424,18 @@
_tests.append(_test_os_arch_requirements_with_default)
+def _test_fail_no_python_version(env):
+ errors = []
+ parse_requirements(
+ ctx = _mock_ctx(),
+ requirements_lock = "requirements_lock",
+ get_index_urls = lambda _, __: {},
+ fail_fn = errors.append,
+ )
+ env.expect.that_str(errors[0]).equals("'python_version' must be provided")
+
+_tests.append(_test_fail_no_python_version)
+
def parse_requirements_test_suite(name):
"""Create the test suite.
diff --git a/tests/private/whl_target_platforms/select_whl_tests.bzl b/tests/private/whl_target_platforms/select_whl_tests.bzl
index bed6d66..ebd2b26 100644
--- a/tests/private/whl_target_platforms/select_whl_tests.bzl
+++ b/tests/private/whl_target_platforms/select_whl_tests.bzl
@@ -15,108 +15,281 @@
""
load("@rules_testing//lib:test_suite.bzl", "test_suite")
-load("//python/private:whl_target_platforms.bzl", "select_whl") # buildifier: disable=bzl-visibility
+load("//python/private:whl_target_platforms.bzl", "select_whl", "select_whls") # buildifier: disable=bzl-visibility
WHL_LIST = [
- struct(
- filename = f,
- url = "https://" + f,
- sha256 = "sha256://" + f,
- )
- for f in [
- "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl",
- "pkg-0.0.1-cp311-cp311-macosx_10_9_x86_64.whl",
- "pkg-0.0.1-cp311-cp311-macosx_11_0_arm64.whl",
- "pkg-0.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
- "pkg-0.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
- "pkg-0.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl",
- "pkg-0.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
- "pkg-0.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl",
- "pkg-0.0.1-cp311-cp311-musllinux_1_1_aarch64.whl",
- "pkg-0.0.1-cp311-cp311-musllinux_1_1_i686.whl",
- "pkg-0.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl",
- "pkg-0.0.1-cp311-cp311-musllinux_1_1_s390x.whl",
- "pkg-0.0.1-cp311-cp311-musllinux_1_1_x86_64.whl",
- "pkg-0.0.1-cp311-cp311-win32.whl",
- "pkg-0.0.1-cp311-cp311-win_amd64.whl",
- "pkg-0.0.1-cp37-cp37m-macosx_10_9_x86_64.whl",
- "pkg-0.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
- "pkg-0.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
- "pkg-0.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl",
- "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
- "pkg-0.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl",
- "pkg-0.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl",
- "pkg-0.0.1-cp37-cp37m-musllinux_1_1_i686.whl",
- "pkg-0.0.1-cp37-cp37m-musllinux_1_1_ppc64le.whl",
- "pkg-0.0.1-cp37-cp37m-musllinux_1_1_s390x.whl",
- "pkg-0.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl",
- "pkg-0.0.1-cp37-cp37m-win32.whl",
- "pkg-0.0.1-cp37-cp37m-win_amd64.whl",
- "pkg-0.0.1-cp39-cp39-macosx_10_9_universal2.whl",
- "pkg-0.0.1-cp39-cp39-macosx_10_9_x86_64.whl",
- "pkg-0.0.1-cp39-cp39-macosx_11_0_arm64.whl",
- "pkg-0.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
- "pkg-0.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
- "pkg-0.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl",
- "pkg-0.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
- "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl",
- "pkg-0.0.1-cp39-cp39-musllinux_1_1_aarch64.whl",
- "pkg-0.0.1-cp39-cp39-musllinux_1_1_i686.whl",
- "pkg-0.0.1-cp39-cp39-musllinux_1_1_ppc64le.whl",
- "pkg-0.0.1-cp39-cp39-musllinux_1_1_s390x.whl",
- "pkg-0.0.1-cp39-cp39-musllinux_1_1_x86_64.whl",
- "pkg-0.0.1-cp39-cp39-win32.whl",
- "pkg-0.0.1-cp39-cp39-win_amd64.whl",
- "pkg-0.0.1-py3-abi3-any.whl",
- "pkg-0.0.1-py3-none-any.whl",
- ]
+ "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl",
+ "pkg-0.0.1-cp311-cp311-macosx_10_9_x86_64.whl",
+ "pkg-0.0.1-cp311-cp311-macosx_11_0_arm64.whl",
+ "pkg-0.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
+ "pkg-0.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
+ "pkg-0.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl",
+ "pkg-0.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+ "pkg-0.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl",
+ "pkg-0.0.1-cp311-cp311-musllinux_1_1_aarch64.whl",
+ "pkg-0.0.1-cp311-cp311-musllinux_1_1_i686.whl",
+ "pkg-0.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl",
+ "pkg-0.0.1-cp311-cp311-musllinux_1_1_s390x.whl",
+ "pkg-0.0.1-cp311-cp311-musllinux_1_1_x86_64.whl",
+ "pkg-0.0.1-cp311-cp311-win32.whl",
+ "pkg-0.0.1-cp311-cp311-win_amd64.whl",
+ "pkg-0.0.1-cp37-cp37m-macosx_10_9_x86_64.whl",
+ "pkg-0.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
+ "pkg-0.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
+ "pkg-0.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl",
+ "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+ "pkg-0.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl",
+ "pkg-0.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl",
+ "pkg-0.0.1-cp37-cp37m-musllinux_1_1_i686.whl",
+ "pkg-0.0.1-cp37-cp37m-musllinux_1_1_ppc64le.whl",
+ "pkg-0.0.1-cp37-cp37m-musllinux_1_1_s390x.whl",
+ "pkg-0.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl",
+ "pkg-0.0.1-cp37-cp37m-win32.whl",
+ "pkg-0.0.1-cp37-cp37m-win_amd64.whl",
+ "pkg-0.0.1-cp39-cp39-macosx_10_9_universal2.whl",
+ "pkg-0.0.1-cp39-cp39-macosx_10_9_x86_64.whl",
+ "pkg-0.0.1-cp39-cp39-macosx_11_0_arm64.whl",
+ "pkg-0.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
+ "pkg-0.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
+ "pkg-0.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl",
+ "pkg-0.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+ "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl",
+ "pkg-0.0.1-cp39-cp39-musllinux_1_1_aarch64.whl",
+ "pkg-0.0.1-cp39-cp39-musllinux_1_1_i686.whl",
+ "pkg-0.0.1-cp39-cp39-musllinux_1_1_ppc64le.whl",
+ "pkg-0.0.1-cp39-cp39-musllinux_1_1_s390x.whl",
+ "pkg-0.0.1-cp39-cp39-musllinux_1_1_x86_64.whl",
+ "pkg-0.0.1-cp39-cp39-win32.whl",
+ "pkg-0.0.1-cp39-cp39-win_amd64.whl",
+ "pkg-0.0.1-cp39-abi3-any.whl",
+ "pkg-0.0.1-py310-abi3-any.whl",
+ "pkg-0.0.1-py3-abi3-any.whl",
+ "pkg-0.0.1-py3-none-any.whl",
]
-def _match(env, got, want_filename):
- if want_filename:
- env.expect.that_str(got.filename).equals(want_filename)
- env.expect.that_str(got.sha256).equals("sha256://" + want_filename)
- env.expect.that_str(got.url).equals("https://" + want_filename)
- else:
- env.expect.that_int(got).equals(None)
+def _match(env, got, *want_filenames):
+ if not want_filenames:
+ env.expect.that_collection(got).has_size(len(want_filenames))
+ return
+
+ got_filenames = [g.filename for g in got]
+ env.expect.that_collection(got_filenames).contains_exactly(want_filenames)
+
+ if got:
+ # Check that we pass the original structs
+ env.expect.that_str(got[0].other).equals("dummy")
+
+def _select_whl(**kwargs):
+ """A small wrapper to make the tests more DRY."""
+ got_single = select_whl(**kwargs)
+ return [got_single] if got_single else []
+
+def _select_whls(whls, **kwargs):
+ return select_whls(
+ whls = [
+ struct(
+ filename = f,
+ other = "dummy",
+ )
+ for f in whls
+ ],
+ **kwargs
+ )
_tests = []
-def _test_selecting(env):
- got = select_whl(whls = WHL_LIST, want_abis = ["none"], want_platform = "ignored")
- _match(env, got, "pkg-0.0.1-py3-none-any.whl")
+def _test_simplest(env):
+ got = _select_whls(
+ whls = [
+ "pkg-0.0.1-py2.py3-abi3-any.whl",
+ "pkg-0.0.1-py3-abi3-any.whl",
+ "pkg-0.0.1-py3-none-any.whl",
+ ],
+ want_abis = ["none"],
+ want_platforms = ["ignored"],
+ )
+ _match(
+ env,
+ got,
+ "pkg-0.0.1-py3-none-any.whl",
+ )
- got = select_whl(whls = WHL_LIST, want_abis = ["abi3"], want_platform = "ignored")
- _match(env, got, "pkg-0.0.1-py3-abi3-any.whl")
+_tests.append(_test_simplest)
- # Check the selection failure
- got = select_whl(whls = WHL_LIST, want_abis = ["cp39"], want_platform = "fancy_exotic")
- _match(env, got, None)
+def _test_select_abi3(env):
+ got = _select_whls(
+ whls = [
+ "pkg-0.0.1-py2.py3-abi3-any.whl",
+ "pkg-0.0.1-py3-abi3-any.whl",
+ "pkg-0.0.1-py3-none-any.whl",
+ ],
+ want_abis = ["abi3"],
+ want_platforms = ["ignored"],
+ )
+ _match(
+ env,
+ got,
+ "pkg-0.0.1-py3-abi3-any.whl",
+ )
+_tests.append(_test_select_abi3)
+
+def _test_select_by_supported_py_version(env):
+ for want_python_version, match in {
+ "3.11": "pkg-0.0.1-py311-abi3-any.whl",
+ "3.8": "pkg-0.0.1-py3-abi3-any.whl",
+ }.items():
+ got = _select_whls(
+ whls = [
+ "pkg-0.0.1-py2.py3-abi3-any.whl",
+ "pkg-0.0.1-py3-abi3-any.whl",
+ "pkg-0.0.1-py311-abi3-any.whl",
+ ],
+ want_abis = ["abi3"],
+ want_platforms = ["ignored"],
+ want_python_version = want_python_version,
+ )
+ _match(env, got, match)
+
+_tests.append(_test_select_by_supported_py_version)
+
+def _test_select_by_supported_cp_version(env):
+ for want_python_version, match in {
+ "3.11": "pkg-0.0.1-cp311-abi3-any.whl",
+ "3.8": "pkg-0.0.1-py3-abi3-any.whl",
+ }.items():
+ got = _select_whls(
+ whls = [
+ "pkg-0.0.1-py2.py3-abi3-any.whl",
+ "pkg-0.0.1-py3-abi3-any.whl",
+ "pkg-0.0.1-py311-abi3-any.whl",
+ "pkg-0.0.1-cp311-abi3-any.whl",
+ ],
+ want_abis = ["abi3"],
+ want_platforms = ["ignored"],
+ want_python_version = want_python_version,
+ )
+ _match(env, got, match)
+
+_tests.append(_test_select_by_supported_cp_version)
+
+def _test_supported_cp_version_manylinux(env):
+ for want_python_version, match in {
+ "3.11": "pkg-0.0.1-cp311-none-manylinux_x86_64.whl",
+ "3.8": "pkg-0.0.1-py3-none-manylinux_x86_64.whl",
+ }.items():
+ got = _select_whls(
+ whls = [
+ "pkg-0.0.1-py2.py3-none-manylinux_x86_64.whl",
+ "pkg-0.0.1-py3-none-manylinux_x86_64.whl",
+ "pkg-0.0.1-py311-none-manylinux_x86_64.whl",
+ "pkg-0.0.1-cp311-none-manylinux_x86_64.whl",
+ ],
+ want_abis = ["none"],
+ want_platforms = ["linux_x86_64"],
+ want_python_version = want_python_version,
+ )
+ _match(env, got, match)
+
+_tests.append(_test_supported_cp_version_manylinux)
+
+def _test_ignore_unsupported(env):
+ got = _select_whls(
+ whls = [
+ "pkg-0.0.1-xx3-abi3-any.whl",
+ ],
+ want_abis = ["abi3"],
+ want_platforms = ["ignored"],
+ )
+ _match(env, got)
+
+_tests.append(_test_ignore_unsupported)
+
+def _test_match_abi_and_not_py_version(env):
# Check we match the ABI and not the py version
- got = select_whl(whls = WHL_LIST, want_abis = ["cp37m"], want_platform = "linux_x86_64")
+ got = _select_whls(whls = WHL_LIST, want_abis = ["cp37m"], want_platforms = ["linux_x86_64"], want_python_version = "3.7")
+ _match(
+ env,
+ got,
+ "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+ "pkg-0.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl",
+ )
+ got = _select_whl(whls = got, want_platform = "linux_x86_64")
_match(env, got, "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl")
+_tests.append(_test_match_abi_and_not_py_version)
+
+def _test_select_filename_with_many_tags(env):
# Check we can select a filename with many platform tags
- got = select_whl(whls = WHL_LIST, want_abis = ["cp39"], want_platform = "linux_x86_32")
+ got = _select_whls(whls = WHL_LIST, want_abis = ["cp39"], want_platforms = ["linux_x86_32"], want_python_version = "3.9")
+ _match(
+ env,
+ got,
+ "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl",
+ "pkg-0.0.1-cp39-cp39-musllinux_1_1_i686.whl",
+ )
+ got = _select_whl(whls = got, want_platform = "linux_x86_32")
_match(env, got, "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl")
+_tests.append(_test_select_filename_with_many_tags)
+
+def _test_osx_prefer_arch_specific(env):
# Check that we prefer the specific wheel
- got = select_whl(whls = WHL_LIST, want_abis = ["cp311"], want_platform = "osx_x86_64")
+ got = _select_whls(
+ whls = WHL_LIST,
+ want_abis = ["cp311"],
+ want_platforms = ["osx_x86_64", "osx_x86_32"],
+ want_python_version = "3.11",
+ )
+ _match(
+ env,
+ got,
+ "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl",
+ "pkg-0.0.1-cp311-cp311-macosx_10_9_x86_64.whl",
+ )
+ got = _select_whl(whls = got, want_platform = "osx_x86_64")
_match(env, got, "pkg-0.0.1-cp311-cp311-macosx_10_9_x86_64.whl")
- got = select_whl(whls = WHL_LIST, want_abis = ["cp311"], want_platform = "osx_aarch64")
+ got = _select_whls(whls = WHL_LIST, want_abis = ["cp311"], want_platforms = ["osx_aarch64"], want_python_version = "3.11")
+ _match(
+ env,
+ got,
+ "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl",
+ "pkg-0.0.1-cp311-cp311-macosx_11_0_arm64.whl",
+ )
+ got = _select_whl(whls = got, want_platform = "osx_aarch64")
_match(env, got, "pkg-0.0.1-cp311-cp311-macosx_11_0_arm64.whl")
+_tests.append(_test_osx_prefer_arch_specific)
+
+def _test_osx_fallback_to_universal2(env):
# Check that we can use the universal2 if the arm wheel is not available
- got = select_whl(whls = [w for w in WHL_LIST if "arm64" not in w.filename], want_abis = ["cp311"], want_platform = "osx_aarch64")
+ got = _select_whls(whls = [w for w in WHL_LIST if "arm64" not in w], want_abis = ["cp311"], want_platforms = ["osx_aarch64"], want_python_version = "3.11")
+ _match(
+ env,
+ got,
+ "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl",
+ )
+ got = _select_whl(whls = got, want_platform = "osx_aarch64")
_match(env, got, "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl")
+_tests.append(_test_osx_fallback_to_universal2)
+
+def _test_prefer_manylinux_wheels(env):
# Check we prefer platform specific wheels
- got = select_whl(whls = WHL_LIST, want_abis = ["none", "abi3", "cp39"], want_platform = "linux_x86_64")
+ got = _select_whls(whls = WHL_LIST, want_abis = ["none", "abi3", "cp39"], want_platforms = ["linux_x86_64"], want_python_version = "3.9")
+ _match(
+ env,
+ got,
+ "pkg-0.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+ "pkg-0.0.1-cp39-cp39-musllinux_1_1_x86_64.whl",
+ "pkg-0.0.1-cp39-abi3-any.whl",
+ "pkg-0.0.1-py3-none-any.whl",
+ )
+ got = _select_whl(whls = got, want_platform = "linux_x86_64")
_match(env, got, "pkg-0.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl")
-_tests.append(_test_selecting)
+_tests.append(_test_prefer_manylinux_wheels)
def select_whl_test_suite(name):
"""Create the test suite.
