feat(bzlmod): cross-platform builds without experimental_index_url (#2325)
With this change we finally generate the same lock file within the
legacy code `pip.parse` code path and it allows to slowly transition to
using the new code path as much as possible without user doing anything.
This moves the selection of the host-compatible lock file from the
extension evaluation to the build phase - note, we will generate extra
repositories here which might not work on the host platform, however, if
the users are consuming the `whl_library` repos through the hub repo
only, then everything should work. A known issue is that it may break
`bazel query` and in these usecases it is advisable to use `cquery`
until we have `sdist` cross-building from source fully working.
Summary:
- feat: reuse the `render_pkg_aliases` for when filename is not known
but platform is known
- feat: support generating the extra config settings required
- feat: `get_whl_flag_versions` now generates extra args for the rules
- feat: make lock file generation the same irrespective of the host
platform
- test: add an extra test with multiple requirements files
- feat: support cross-platform builds using `download_only = True` in
legacy setups
Note, that users depending on the naming of the whl libraries will need
to start using `extra_hub_aliases` attribute instead to keep their
setups not relying on this implementation detail.
Fixes #2268
Work towards #260
---------
Co-authored-by: Richard Levasseur <richardlev@gmail.com>
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d5b757e..8eb269c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -39,6 +39,12 @@
[`pip.parse#extra_pip_args`](https://rules-python.readthedocs.io/en/latest/api/rules_python/python/extensions/pip.html#pip.parse.extra_pip_args)
* (pip.parse) {attr}`pip.parse.whl_modifications` now normalizes the given whl names
and now `pyyaml` and `PyYAML` will both work.
+* (bzlmod) `pip.parse` spoke repository naming will be changed in an upcoming
+ release in places where the users specify different package versions per
+ platform in the same hub repository. The naming of the spoke repos is considered
+ an implementation detail and we advise the users to use the `hub` repository
+ directly to avoid such breakage in the future. If `rules_python` is missing
+ features to allow one to do that, please raise tickets.
{#v0-0-0-fixed}
### Fixed
@@ -51,6 +57,12 @@
pass the `extra_pip_args` value when building an `sdist`.
* (pypi) The patched wheel filenames from now on are using local version specifiers
which fixes usage of the said wheels using standard package managers.
+* (bzlmod) The extension evaluation has been adjusted to always generate the
+ same lock file irrespective if `experimental_index_url` is set by any module
+ or not. Fixes
+ [#2268](https://github.com/bazelbuild/rules_python/issues/2268). A known
+ issue is that it may break `bazel query` and in these use cases it is
+ advisable to use `cquery` or switch to `download_only = True`
{#v0-0-0-added}
### Added
@@ -63,6 +75,11 @@
* (pip.parse) {attr}`pip.parse.extra_hub_aliases` can now be used to expose extra
targets created by annotations in whl repositories.
Fixes [#2187](https://github.com/bazelbuild/rules_python/issues/2187).
+* (bzlmod) `pip.parse` now supports `whl-only` setup using
+ `download_only = True` where users can specify multiple requirements files
+ and use the `pip` backend to do the downloading. This was only available for
+ users setting {bzl:obj}`pip.parse.experimental_index_url`, but now users have
+ more options whilst we continue to work on stabilizing the experimental feature.
{#v0-0-0-removed}
### Removed
diff --git a/MODULE.bazel b/MODULE.bazel
index de14b86..50f1376 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -56,6 +56,10 @@
pip = use_extension("//python/private/pypi:pip.bzl", "pip_internal")
pip.parse(
+ # NOTE @aignas 2024-10-26: We have an integration test that depends on us
+ # being able to build sdists for this hub, so explicitly set this to False.
+ download_only = False,
+ experimental_index_url = "https://pypi.org/simple",
hub_name = "rules_python_publish_deps",
python_version = "3.11",
requirements_by_platform = {
@@ -90,17 +94,20 @@
)
dev_pip = use_extension(
- "//python/private/pypi:pip.bzl",
- "pip_internal",
+ "//python/extensions:pip.bzl",
+ "pip",
dev_dependency = True,
)
dev_pip.parse(
- download_only = True, # this will not add the `sdist` values to the transitive closures at all.
+ download_only = True,
+ experimental_index_url = "https://pypi.org/simple",
hub_name = "dev_pip",
python_version = "3.11",
requirements_lock = "//docs:requirements.txt",
)
dev_pip.parse(
+ download_only = True,
+ experimental_index_url = "https://pypi.org/simple",
hub_name = "pypiserver",
python_version = "3.11",
requirements_lock = "//examples/wheel:requirements_server.txt",
diff --git a/docs/pypi-dependencies.md b/docs/pypi-dependencies.md
index 636fefb..28e630c 100644
--- a/docs/pypi-dependencies.md
+++ b/docs/pypi-dependencies.md
@@ -308,6 +308,59 @@
(bazel-downloader)=
+### Multi-platform support
+
+Multi-platform support of cross-building the wheels can be done in two ways - either
+using {bzl:attr}`experimental_index_url` for the {bzl:obj}`pip.parse` bzlmod tag class
+or by using the {bzl:attr}`pip.parse.download_only` setting. In this section we
+are going to outline quickly how one can use the latter option.
+
+Let's say you have 2 requirements files:
+```
+# requirements.linux_x86_64.txt
+--platform=manylinux_2_17_x86_64
+--python-version=39
+--implementation=cp
+--abi=cp39
+
+foo==0.0.1 --hash=sha256:deadbeef
+bar==0.0.1 --hash=sha256:deadb00f
+```
+
+```
+# requirements.osx_aarch64.txt contents
+--platform=macosx_10_9_arm64
+--python-version=39
+--implementation=cp
+--abi=cp39
+
+foo==0.0.3 --hash=sha256:deadbaaf
+```
+
+With these 2 files your {bzl:obj}`pip.parse` could look like:
+```
+pip.parse(
+ hub_name = "pip",
+ python_version = "3.9",
+ # Tell `pip` to ignore sdists
+ download_only = True,
+ requirements_by_platform = {
+ "requirements.linux_x86_64.txt": "linux_x86_64",
+ "requirements.osx_aarch64.txt": "osx_aarch64",
+ },
+)
+```
+
+With this, the `pip.parse` will create a hub repository that is going to
+support only two platforms - `cp39_osx_aarch64` and `cp39_linux_x86_64` and it
+will only use `wheels` and ignore any sdists that it may find on the PyPI
+compatible indexes.
+
+```{note}
+This is only supported on `bzlmd`.
+```
+
+(bazel-downloader)=
### Bazel downloader and multi-platform wheel hub repository.
The `bzlmod` `pip.parse` call supports pulling information from `PyPI` (or a
diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel
index 4381cb0..1b8bbbf 100644
--- a/examples/bzlmod/MODULE.bazel
+++ b/examples/bzlmod/MODULE.bazel
@@ -221,6 +221,9 @@
"host",
],
hub_name = "pip",
+ # Parse all requirements files for the same lock file on all OSes, this will
+ # become the default with 1.0 release
+ parse_all_requirements_files = True,
python_version = "3.10",
# The requirements files for each platform that we want to support.
requirements_by_platform = {
diff --git a/examples/bzlmod/MODULE.bazel.lock b/examples/bzlmod/MODULE.bazel.lock
index c115ef9..eb578f6 100644
--- a/examples/bzlmod/MODULE.bazel.lock
+++ b/examples/bzlmod/MODULE.bazel.lock
@@ -1392,8 +1392,8 @@
},
"@@rules_python~//python/extensions:pip.bzl%pip": {
"general": {
- "bzlTransitiveDigest": "E5Yr6AjquyIy5ae3c7URmvtPPOm2j+7XOr58GOHp8vw=",
- "usagesDigest": "iVxh/vcpGrSKpO8rafQwAe7uq+pHhasSXC7Pg4o/1dw=",
+ "bzlTransitiveDigest": "0Qn7Q9FuTxYCxMKm2DsW7mbXYcxL71sS/l1baXvY1vA=",
+ "usagesDigest": "GGeczTmsfE4YHAy32dV/jfOfbYmpyu/QGe35drFuZ5E=",
"recordedFileInputs": {
"@@other_module~//requirements_lock_3_11.txt": "a7d0061366569043d5efcf80e34a32c732679367cb3c831c4cdc606adc36d314",
"@@rules_python~//python/private/pypi/whl_installer/platform.py": "b944b908b25a2f97d6d9f491504ad5d2507402d7e37c802ee878783f87f2aa11",
@@ -6587,8 +6587,8 @@
},
"@@rules_python~//python/private/pypi:pip.bzl%pip_internal": {
"general": {
- "bzlTransitiveDigest": "wz5L+/+R6gOtD681pNVgPUUipqqPH0bP/b0e22JbSOI=",
- "usagesDigest": "LYtSAPzhPjmfD9vF39mCED1UQSvHEo2Hv+aK5Z4ZWWc=",
+ "bzlTransitiveDigest": "SnuwsgZv1SGZz4jVPvwaEUwPTnea18fXIueD9vSR3sQ=",
+ "usagesDigest": "O2O2oBIbKEglN2K3FECsRxUKVS/zg/6a86F3MO1ZtmY=",
"recordedFileInputs": {
"@@rules_python~//tools/publish/requirements_linux.txt": "8175b4c8df50ae2f22d1706961884beeb54e7da27bd2447018314a175981997d",
"@@rules_python~//tools/publish/requirements_windows.txt": "7673adc71dc1a81d3661b90924d7a7c0fc998cd508b3cb4174337cef3f2de556",
diff --git a/python/private/pypi/config_settings.bzl b/python/private/pypi/config_settings.bzl
index 492acf1..9f3f4d4 100644
--- a/python/private/pypi/config_settings.bzl
+++ b/python/private/pypi/config_settings.bzl
@@ -148,6 +148,13 @@
)
def _dist_config_settings(*, suffix, plat_flag_values, **kwargs):
+ if kwargs.get("constraint_values"):
+ # Add python version + platform config settings
+ _dist_config_setting(
+ name = suffix.strip("_"),
+ **kwargs
+ )
+
flag_values = {_flags.dist: ""}
# First create an sdist, we will be building upon the flag values, which
@@ -277,7 +284,7 @@
return ret
-def _dist_config_setting(*, name, is_pip_whl, is_python, python_version, native = native, **kwargs):
+def _dist_config_setting(*, name, is_python, python_version, is_pip_whl = None, native = native, **kwargs):
"""A macro to create a target that matches is_pip_whl_auto and one more value.
Args:
@@ -310,6 +317,10 @@
# `python_version` setting.
return
+ if not is_pip_whl:
+ native.config_setting(name = _name, **kwargs)
+ return
+
config_setting_name = _name + "_setting"
native.config_setting(name = config_setting_name, **kwargs)
diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl
index c566027..7b31d0d 100644
--- a/python/private/pypi/extension.bzl
+++ b/python/private/pypi/extension.bzl
@@ -31,7 +31,7 @@
load(":requirements_files_by_platform.bzl", "requirements_files_by_platform")
load(":simpleapi_download.bzl", "simpleapi_download")
load(":whl_library.bzl", "whl_library")
-load(":whl_repo_name.bzl", "whl_repo_name")
+load(":whl_repo_name.bzl", "pypi_repo_name", "whl_repo_name")
def _major_minor_version(version):
version = semver(version)
@@ -260,10 +260,10 @@
if v != default
})
+ is_exposed = False
if get_index_urls:
# TODO @aignas 2024-05-26: move to a separate function
found_something = False
- is_exposed = False
for requirement in requirements:
is_exposed = is_exposed or requirement.is_exposed
dists = requirement.whls
@@ -319,35 +319,69 @@
exposed_packages[whl_name] = None
continue
- requirement = select_requirement(
- requirements,
- platform = None if pip_attr.download_only else repository_platform,
- )
- if not requirement:
- # Sometimes the package is not present for host platform if there
- # are whls specified only in particular requirements files, in that
- # case just continue, however, if the download_only flag is set up,
- # then the user can also specify the target platform of the wheel
- # packages they want to download, in that case there will be always
- # a requirement here, so we will not be in this code branch.
+ if not pip_attr.parse_all_requirements_files:
+ requirement = select_requirement(
+ requirements,
+ platform = None if pip_attr.download_only else repository_platform,
+ )
+ if not requirement:
+ # Sometimes the package is not present for host platform if there
+ # are whls specified only in particular requirements files, in that
+ # case just continue, however, if the download_only flag is set up,
+ # then the user can also specify the target platform of the wheel
+ # packages they want to download, in that case there will be always
+ # a requirement here, so we will not be in this code branch.
+ continue
+ elif get_index_urls:
+ logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line))
+
+ whl_library_args["requirement"] = requirement.requirement_line
+ if requirement.extra_pip_args:
+ whl_library_args["extra_pip_args"] = requirement.extra_pip_args
+
+ # We sort so that the lock-file remains the same no matter the order of how the
+ # args are manipulated in the code going before.
+ repo_name = "{}_{}".format(pip_name, whl_name)
+ whl_libraries[repo_name] = dict(whl_library_args.items())
+ whl_map.setdefault(whl_name, []).append(
+ whl_alias(
+ repo = repo_name,
+ version = major_minor,
+ ),
+ )
continue
- elif get_index_urls:
- logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line))
- whl_library_args["requirement"] = requirement.requirement_line
- if requirement.extra_pip_args:
- whl_library_args["extra_pip_args"] = requirement.extra_pip_args
+ is_exposed = False
+ for requirement in requirements:
+ is_exposed = is_exposed or requirement.is_exposed
+ if get_index_urls:
+ logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line))
- # We sort so that the lock-file remains the same no matter the order of how the
- # args are manipulated in the code going before.
- repo_name = "{}_{}".format(pip_name, whl_name)
- whl_libraries[repo_name] = dict(whl_library_args.items())
- whl_map.setdefault(whl_name, []).append(
- whl_alias(
- repo = repo_name,
- version = major_minor,
- ),
- )
+ args = dict(whl_library_args) # make a copy
+ args["requirement"] = requirement.requirement_line
+ if requirement.extra_pip_args:
+ args["extra_pip_args"] = requirement.extra_pip_args
+
+ if pip_attr.download_only:
+ args.setdefault("experimental_target_platforms", requirement.target_platforms)
+
+ target_platforms = requirement.target_platforms if len(requirements) > 1 else []
+ repo_name = pypi_repo_name(
+ pip_name,
+ whl_name,
+ *target_platforms
+ )
+ whl_libraries[repo_name] = args
+ whl_map.setdefault(whl_name, []).append(
+ whl_alias(
+ repo = repo_name,
+ version = major_minor,
+ target_platforms = target_platforms or None,
+ ),
+ )
+
+ if is_exposed:
+ exposed_packages[whl_name] = None
return struct(
is_reproducible = is_reproducible,
@@ -502,7 +536,8 @@
hub_group_map[hub_name] = pip_attr.experimental_requirement_cycles
return struct(
- # We sort the output here so that the lock file is sorted
+ # We sort so that the lock-file remains the same no matter the order of how the
+ # args are manipulated in the code going before.
whl_mods = dict(sorted(whl_mods.items())),
hub_whl_map = {
hub_name: {
@@ -518,7 +553,10 @@
}
for hub_name, group_map in sorted(hub_group_map.items())
},
- exposed_packages = {k: sorted(v) for k, v in sorted(exposed_packages.items())},
+ exposed_packages = {
+ k: sorted(v)
+ for k, v in sorted(exposed_packages.items())
+ },
extra_aliases = {
hub_name: {
whl_name: sorted(aliases)
@@ -526,7 +564,10 @@
}
for hub_name, extra_whl_aliases in extra_aliases.items()
},
- whl_libraries = dict(sorted(whl_libraries.items())),
+ whl_libraries = {
+ k: dict(sorted(args.items()))
+ for k, args in sorted(whl_libraries.items())
+ },
is_reproducible = is_reproducible,
)
@@ -601,10 +642,8 @@
# Build all of the wheel modifications if the tag class is called.
_whl_mods_impl(mods.whl_mods)
- for name, args in sorted(mods.whl_libraries.items()):
- # We sort so that the lock-file remains the same no matter the order of how the
- # args are manipulated in the code going before.
- whl_library(name = name, **dict(sorted(args.items())))
+ for name, args in mods.whl_libraries.items():
+ whl_library(name = name, **args)
for hub_name, whl_map in mods.hub_whl_map.items():
hub_repository(
@@ -746,6 +785,20 @@
""",
default = True,
),
+ "parse_all_requirements_files": attr.bool(
+ default = False,
+ doc = """\
+A temporary flag to enable users to make `pip` extension result always
+the same independent of the whether transitive dependencies use {bzl:attr}`experimental_index_url` or not.
+
+This enables users to migrate to a solution that fixes
+[#2268](https://github.com/bazelbuild/rules_python/issues/2268).
+
+::::{deprecated} 0.38.0
+This is a transition flag and will be removed in a subsequent release.
+::::
+""",
+ ),
"python_version": attr.string(
mandatory = True,
doc = """
@@ -907,24 +960,22 @@
pypi_internal = module_extension(
doc = """\
This extension is used to make dependencies from pypi available.
-
For now this is intended to be used internally so that usage of the `pip`
extension in `rules_python` does not affect the evaluations of the extension
for the consumers.
-
pip.parse:
To use, call `pip.parse()` and specify `hub_name` and your requirements file.
Dependencies will be downloaded and made available in a repo named after the
`hub_name` argument.
-
Each `pip.parse()` call configures a particular Python version. Multiple calls
can be made to configure different Python versions, and will be grouped by
the `hub_name` argument. This allows the same logical name, e.g. `@pypi//numpy`
to automatically resolve to different, Python version-specific, libraries.
-
pip.whl_mods:
This tag class is used to help create JSON files to describe modifications to
the BUILD files for wheels.
+
+TODO: will be removed before 1.0.
""",
implementation = _pip_non_reproducible,
tag_classes = {
diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl
index aacc8bd..a43217d 100644
--- a/python/private/pypi/parse_requirements.bzl
+++ b/python/private/pypi/parse_requirements.bzl
@@ -168,7 +168,7 @@
)
ret = {}
- for whl_name, reqs in requirements_by_platform.items():
+ for whl_name, reqs in sorted(requirements_by_platform.items()):
requirement_target_platforms = {}
for r in reqs.values():
target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms)
@@ -212,6 +212,8 @@
def select_requirement(requirements, *, platform):
"""A simple function to get a requirement for a particular platform.
+ Only used in WORKSPACE.
+
Args:
requirements (list[struct]): The list of requirements as returned by
the `parse_requirements` function above.
@@ -243,6 +245,8 @@
def host_platform(ctx):
"""Return a string representation of the repository OS.
+ Only used in WORKSPACE.
+
Args:
ctx (struct): The `module_ctx` or `repository_ctx` attribute.
diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl
index 60f4b54..7a75979 100644
--- a/python/private/pypi/render_pkg_aliases.bzl
+++ b/python/private/pypi/render_pkg_aliases.bzl
@@ -326,16 +326,19 @@
ret = []
versioned_additions = {}
for alias in aliases:
- if not alias.filename:
+ if not alias.filename and not alias.target_platforms:
ret.append(alias)
continue
config_settings, all_versioned_settings = get_filename_config_settings(
# TODO @aignas 2024-05-27: pass the parsed whl to reduce the
# number of duplicate operations.
- filename = alias.filename,
+ filename = alias.filename or "",
target_platforms = alias.target_platforms,
python_version = alias.version,
+ # If we have multiple platforms but no wheel filename, lets use different
+ # config settings.
+ non_whl_prefix = "sdist" if alias.filename else "",
**kwargs
)
@@ -437,10 +440,7 @@
if a.version:
python_versions[a.version] = None
- if not a.filename:
- continue
-
- if a.filename.endswith(".whl") and not a.filename.endswith("-any.whl"):
+ if a.filename and a.filename.endswith(".whl") and not a.filename.endswith("-any.whl"):
parsed = parse_whl_name(a.filename)
else:
for plat in a.target_platforms or []:
@@ -499,10 +499,11 @@
*,
filename,
target_platforms,
- glibc_versions,
- muslc_versions,
- osx_versions,
- python_version):
+ python_version,
+ glibc_versions = None,
+ muslc_versions = None,
+ osx_versions = None,
+ non_whl_prefix = "sdist"):
"""Get the filename config settings.
Args:
@@ -512,6 +513,8 @@
muslc_versions: list[tuple[int, int]], list of versions.
osx_versions: list[tuple[int, int]], list of versions.
python_version: the python version to generate the config_settings for.
+ non_whl_prefix: the prefix of the config setting when the whl we don't have
+ a filename ending with ".whl".
Returns:
A tuple:
@@ -520,19 +523,20 @@
"""
prefixes = []
suffixes = []
- if (0, 0) in glibc_versions:
- fail("Invalid version in 'glibc_versions': cannot specify (0, 0) as a value")
- if (0, 0) in muslc_versions:
- fail("Invalid version in 'muslc_versions': cannot specify (0, 0) as a value")
- if (0, 0) in osx_versions:
- fail("Invalid version in 'osx_versions': cannot specify (0, 0) as a value")
-
- glibc_versions = sorted(glibc_versions)
- muslc_versions = sorted(muslc_versions)
- osx_versions = sorted(osx_versions)
setting_supported_versions = {}
if filename.endswith(".whl"):
+ if (0, 0) in glibc_versions:
+ fail("Invalid version in 'glibc_versions': cannot specify (0, 0) as a value")
+ if (0, 0) in muslc_versions:
+ fail("Invalid version in 'muslc_versions': cannot specify (0, 0) as a value")
+ if (0, 0) in osx_versions:
+ fail("Invalid version in 'osx_versions': cannot specify (0, 0) as a value")
+
+ glibc_versions = sorted(glibc_versions)
+ muslc_versions = sorted(muslc_versions)
+ osx_versions = sorted(osx_versions)
+
parsed = parse_whl_name(filename)
if parsed.python_tag == "py2.py3":
py = "py"
@@ -547,10 +551,10 @@
abi = parsed.abi_tag
if parsed.platform_tag == "any":
- prefixes = ["{}_{}_any".format(py, abi)]
+ prefixes = ["_{}_{}_any".format(py, abi)]
suffixes = [_non_versioned_platform(p) for p in target_platforms or []]
else:
- prefixes = ["{}_{}".format(py, abi)]
+ prefixes = ["_{}_{}".format(py, abi)]
suffixes = _whl_config_setting_suffixes(
platform_tag = parsed.platform_tag,
glibc_versions = glibc_versions,
@@ -559,12 +563,12 @@
setting_supported_versions = setting_supported_versions,
)
else:
- prefixes = ["sdist"]
+ prefixes = [""] if not non_whl_prefix else ["_" + non_whl_prefix]
suffixes = [_non_versioned_platform(p) for p in target_platforms or []]
versioned = {
- ":is_cp{}_{}_{}".format(python_version, p, suffix): {
- version: ":is_cp{}_{}_{}".format(python_version, p, setting)
+ ":is_cp{}{}_{}".format(python_version, p, suffix): {
+ version: ":is_cp{}{}_{}".format(python_version, p, setting)
for version, setting in versions.items()
}
for p in prefixes
@@ -572,9 +576,9 @@
}
if suffixes or versioned:
- return [":is_cp{}_{}_{}".format(python_version, p, s) for p in prefixes for s in suffixes], versioned
+ return [":is_cp{}{}_{}".format(python_version, p, s) for p in prefixes for s in suffixes], versioned
else:
- return [":is_cp{}_{}".format(python_version, p) for p in prefixes], setting_supported_versions
+ return [":is_cp{}{}".format(python_version, p) for p in prefixes], setting_supported_versions
def _whl_config_setting_suffixes(
platform_tag,
diff --git a/python/private/pypi/whl_repo_name.bzl b/python/private/pypi/whl_repo_name.bzl
index 295f5a4..38ed600 100644
--- a/python/private/pypi/whl_repo_name.bzl
+++ b/python/private/pypi/whl_repo_name.bzl
@@ -22,12 +22,12 @@
"""Return a valid whl_library repo name given a distribution filename.
Args:
- prefix: str, the prefix of the whl_library.
- filename: str, the filename of the distribution.
- sha256: str, the sha256 of the distribution.
+ prefix: {type}`str` the prefix of the whl_library.
+ filename: {type}`str` the filename of the distribution.
+ sha256: {type}`str` the sha256 of the distribution.
Returns:
- a string that can be used in `whl_library`.
+ a string that can be used in {obj}`whl_library`.
"""
parts = [prefix]
@@ -50,3 +50,22 @@
parts.append(sha256[:8])
return "_".join(parts)
+
+def pypi_repo_name(prefix, whl_name, *target_platforms):
+ """Return a valid whl_library given a requirement line.
+
+ Args:
+ prefix: {type}`str` the prefix of the whl_library.
+ whl_name: {type}`str` the whl_name to use.
+ *target_platforms: {type}`list[str]` the target platforms to use in the name.
+
+ Returns:
+ {type}`str` that can be used in {obj}`whl_library`.
+ """
+ parts = [
+ prefix,
+ normalize_name(whl_name),
+ ]
+ parts.extend([p.partition("_")[-1] for p in target_platforms])
+
+ return "_".join(parts)
diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl
index aa120af..0405bad 100644
--- a/tests/pypi/extension/extension_tests.bzl
+++ b/tests/pypi/extension/extension_tests.bzl
@@ -20,14 +20,14 @@
_tests = []
-def _mock_mctx(*modules, environ = {}, read = None, os_name = "unittest", os_arch = "exotic"):
+def _mock_mctx(*modules, environ = {}, read = None):
return struct(
os = struct(
environ = environ,
- name = os_name,
- arch = os_arch,
+ name = "unittest",
+ arch = "exotic",
),
- read = read or (lambda _: "simple==0.0.1 --hash=sha256:deadbeef"),
+ read = read or (lambda _: "simple==0.0.1 --hash=sha256:deadbeef --hash=sha256:deadbaaf"),
modules = [
struct(
name = modules[0].name,
@@ -61,7 +61,6 @@
attrs = dict(
is_reproducible = subjects.bool,
exposed_packages = subjects.dict,
- extra_aliases = subjects.dict,
hub_group_map = subjects.dict,
hub_whl_map = subjects.dict,
whl_libraries = subjects.dict,
@@ -69,29 +68,6 @@
),
)
-def _whl_mods(
- *,
- whl_name,
- hub_name,
- additive_build_content = None,
- additive_build_content_file = None,
- copy_executables = {},
- copy_files = {},
- data = [],
- data_exclude_glob = [],
- srcs_exclude_glob = []):
- return struct(
- additive_build_content = additive_build_content,
- additive_build_content_file = additive_build_content_file,
- copy_executables = copy_executables,
- copy_files = copy_files,
- data = data,
- data_exclude_glob = data_exclude_glob,
- hub_name = hub_name,
- srcs_exclude_glob = srcs_exclude_glob,
- whl_name = whl_name,
- )
-
def _parse(
*,
hub_name,
@@ -109,6 +85,7 @@
extra_pip_args = [],
isolated = True,
netrc = None,
+ parse_all_requirements_files = True,
pip_data_exclude = None,
python_interpreter = None,
python_interpreter_target = None,
@@ -136,6 +113,7 @@
hub_name = hub_name,
isolated = isolated,
netrc = netrc,
+ parse_all_requirements_files = parse_all_requirements_files,
pip_data_exclude = pip_data_exclude,
python_interpreter = python_interpreter,
python_interpreter_target = python_interpreter_target,
@@ -176,58 +154,7 @@
)
pypi.is_reproducible().equals(True)
- pypi.exposed_packages().contains_exactly({"pypi": []})
- pypi.hub_group_map().contains_exactly({"pypi": {}})
- pypi.hub_whl_map().contains_exactly({"pypi": {}})
- pypi.whl_libraries().contains_exactly({})
- pypi.whl_mods().contains_exactly({})
-
-_tests.append(_test_simple)
-
-def _test_simple_with_whl_mods(env):
- pypi = _parse_modules(
- env,
- module_ctx = _mock_mctx(
- _mod(
- name = "rules_python",
- whl_mods = [
- _whl_mods(
- additive_build_content = """\
-filegroup(
- name = "foo",
- srcs = ["all"],
-)""",
- hub_name = "whl_mods_hub",
- whl_name = "simple",
- ),
- ],
- parse = [
- _parse(
- hub_name = "pypi",
- python_version = "3.15",
- requirements_lock = "requirements.txt",
- extra_hub_aliases = {
- "simple": ["foo"],
- },
- whl_modifications = {
- "@whl_mods_hub//:simple.json": "simple",
- },
- ),
- ],
- ),
- os_name = "linux",
- os_arch = "aarch64",
- ),
- available_interpreters = {
- "python_3_15_host": "unit_test_interpreter_target",
- },
- )
-
- pypi.is_reproducible().equals(True)
- pypi.exposed_packages().contains_exactly({"pypi": []})
- pypi.extra_aliases().contains_exactly({
- "pypi": {"simple": ["foo"]},
- })
+ pypi.exposed_packages().contains_exactly({"pypi": ["simple"]})
pypi.hub_group_map().contains_exactly({"pypi": {}})
pypi.hub_whl_map().contains_exactly({"pypi": {
"simple": [
@@ -242,27 +169,190 @@
}})
pypi.whl_libraries().contains_exactly({
"pypi_315_simple": {
- "annotation": "@whl_mods_hub//:simple.json",
+ "dep_template": "@pypi//{name}:{target}",
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "repo": "pypi_315",
+ "requirement": "simple==0.0.1 --hash=sha256:deadbeef --hash=sha256:deadbaaf",
+ },
+ })
+ pypi.whl_mods().contains_exactly({})
+
+_tests.append(_test_simple)
+
+def _test_simple_multiple_requirements(env):
+ pypi = _parse_modules(
+ env,
+ module_ctx = _mock_mctx(
+ _mod(
+ name = "rules_python",
+ parse = [
+ _parse(
+ hub_name = "pypi",
+ python_version = "3.15",
+ requirements_darwin = "darwin.txt",
+ requirements_windows = "win.txt",
+ ),
+ ],
+ ),
+ read = lambda x: {
+ "darwin.txt": "simple==0.0.2 --hash=sha256:deadb00f",
+ "win.txt": "simple==0.0.1 --hash=sha256:deadbeef",
+ }[x],
+ ),
+ available_interpreters = {
+ "python_3_15_host": "unit_test_interpreter_target",
+ },
+ )
+
+ pypi.is_reproducible().equals(True)
+ pypi.exposed_packages().contains_exactly({"pypi": ["simple"]})
+ pypi.hub_group_map().contains_exactly({"pypi": {}})
+ pypi.hub_whl_map().contains_exactly({"pypi": {
+ "simple": [
+ struct(
+ config_setting = "//_config:is_python_3.15",
+ filename = None,
+ repo = "pypi_315_simple_windows_x86_64",
+ target_platforms = [
+ "cp315_windows_x86_64",
+ ],
+ version = "3.15",
+ ),
+ struct(
+ config_setting = "//_config:is_python_3.15",
+ filename = None,
+ repo = "pypi_315_simple_osx_aarch64_osx_x86_64",
+ target_platforms = [
+ "cp315_osx_aarch64",
+ "cp315_osx_x86_64",
+ ],
+ version = "3.15",
+ ),
+ ],
+ }})
+ pypi.whl_libraries().contains_exactly({
+ "pypi_315_simple_osx_aarch64_osx_x86_64": {
+ "dep_template": "@pypi//{name}:{target}",
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "repo": "pypi_315",
+ "requirement": "simple==0.0.2 --hash=sha256:deadb00f",
+ },
+ "pypi_315_simple_windows_x86_64": {
"dep_template": "@pypi//{name}:{target}",
"python_interpreter_target": "unit_test_interpreter_target",
"repo": "pypi_315",
"requirement": "simple==0.0.1 --hash=sha256:deadbeef",
},
})
- pypi.whl_mods().contains_exactly({
- "whl_mods_hub": {
- "simple": struct(
- build_content = "filegroup(\n name = \"foo\",\n srcs = [\"all\"],\n)",
- copy_executables = {},
- copy_files = {},
- data = [],
- data_exclude_glob = [],
- srcs_exclude_glob = [],
+ pypi.whl_mods().contains_exactly({})
+
+_tests.append(_test_simple_multiple_requirements)
+
+def _test_download_only_multiple(env):
+ pypi = _parse_modules(
+ env,
+ module_ctx = _mock_mctx(
+ _mod(
+ name = "rules_python",
+ parse = [
+ _parse(
+ hub_name = "pypi",
+ python_version = "3.15",
+ download_only = True,
+ requirements_by_platform = {
+ "requirements.linux_x86_64.txt": "linux_x86_64",
+ "requirements.osx_aarch64.txt": "osx_aarch64",
+ },
+ ),
+ ],
),
+ read = lambda x: {
+ "requirements.linux_x86_64.txt": """\
+--platform=manylinux_2_17_x86_64
+--python-version=315
+--implementation=cp
+--abi=cp315
+
+simple==0.0.1 --hash=sha256:deadbeef
+extra==0.0.1 --hash=sha256:deadb00f
+""",
+ "requirements.osx_aarch64.txt": """\
+--platform=macosx_10_9_arm64
+--python-version=315
+--implementation=cp
+--abi=cp315
+
+simple==0.0.3 --hash=sha256:deadbaaf
+""",
+ }[x],
+ ),
+ available_interpreters = {
+ "python_3_15_host": "unit_test_interpreter_target",
+ },
+ )
+
+ pypi.is_reproducible().equals(True)
+ pypi.exposed_packages().contains_exactly({"pypi": ["simple"]})
+ pypi.hub_group_map().contains_exactly({"pypi": {}})
+ pypi.hub_whl_map().contains_exactly({"pypi": {
+ "extra": [
+ struct(
+ config_setting = "//_config:is_python_3.15",
+ filename = None,
+ repo = "pypi_315_extra",
+ target_platforms = None,
+ version = "3.15",
+ ),
+ ],
+ "simple": [
+ struct(
+ config_setting = "//_config:is_python_3.15",
+ filename = None,
+ repo = "pypi_315_simple_linux_x86_64",
+ target_platforms = ["cp315_linux_x86_64"],
+ version = "3.15",
+ ),
+ struct(
+ config_setting = "//_config:is_python_3.15",
+ filename = None,
+ repo = "pypi_315_simple_osx_aarch64",
+ target_platforms = ["cp315_osx_aarch64"],
+ version = "3.15",
+ ),
+ ],
+ }})
+ pypi.whl_libraries().contains_exactly({
+ "pypi_315_extra": {
+ "dep_template": "@pypi//{name}:{target}",
+ "download_only": True,
+ "experimental_target_platforms": ["cp315_linux_x86_64"],
+ "extra_pip_args": ["--platform=manylinux_2_17_x86_64", "--python-version=315", "--implementation=cp", "--abi=cp315"],
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "repo": "pypi_315",
+ "requirement": "extra==0.0.1 --hash=sha256:deadb00f",
+ },
+ "pypi_315_simple_linux_x86_64": {
+ "dep_template": "@pypi//{name}:{target}",
+ "download_only": True,
+ "experimental_target_platforms": ["cp315_linux_x86_64"],
+ "extra_pip_args": ["--platform=manylinux_2_17_x86_64", "--python-version=315", "--implementation=cp", "--abi=cp315"],
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "repo": "pypi_315",
+ "requirement": "simple==0.0.1 --hash=sha256:deadbeef",
+ },
+ "pypi_315_simple_osx_aarch64": {
+ "dep_template": "@pypi//{name}:{target}",
+ "download_only": True,
+ "experimental_target_platforms": ["cp315_osx_aarch64"],
+ "extra_pip_args": ["--platform=macosx_10_9_arm64", "--python-version=315", "--implementation=cp", "--abi=cp315"],
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "repo": "pypi_315",
+ "requirement": "simple==0.0.3 --hash=sha256:deadbaaf",
},
})
+ pypi.whl_mods().contains_exactly({})
-_tests.append(_test_simple_with_whl_mods)
+_tests.append(_test_download_only_multiple)
def _test_simple_get_index(env):
got_simpleapi_download_args = []
@@ -310,7 +400,10 @@
],
),
read = lambda x: {
- "requirements.txt": "simple==0.0.1 --hash=sha256:deadbeef --hash=sha256:deadb00f",
+ "requirements.txt": """
+simple==0.0.1 --hash=sha256:deadbeef --hash=sha256:deadb00f
+some_pkg==0.0.1
+""",
}[x],
),
available_interpreters = {
@@ -320,26 +413,37 @@
)
pypi.is_reproducible().equals(False)
- pypi.exposed_packages().contains_exactly({"pypi": ["simple"]})
+ pypi.exposed_packages().contains_exactly({"pypi": ["simple", "some_pkg"]})
pypi.hub_group_map().contains_exactly({"pypi": {}})
- pypi.hub_whl_map().contains_exactly({"pypi": {
- "simple": [
- struct(
- config_setting = "//_config:is_python_3.15",
- filename = "simple-0.0.1-py3-none-any.whl",
- repo = "pypi_315_simple_py3_none_any_deadb00f",
- target_platforms = None,
- version = "3.15",
- ),
- struct(
- config_setting = "//_config:is_python_3.15",
- filename = "simple-0.0.1.tar.gz",
- repo = "pypi_315_simple_sdist_deadbeef",
- target_platforms = None,
- version = "3.15",
- ),
- ],
- }})
+ pypi.hub_whl_map().contains_exactly({
+ "pypi": {
+ "simple": [
+ struct(
+ config_setting = "//_config:is_python_3.15",
+ filename = "simple-0.0.1-py3-none-any.whl",
+ repo = "pypi_315_simple_py3_none_any_deadb00f",
+ target_platforms = None,
+ version = "3.15",
+ ),
+ struct(
+ config_setting = "//_config:is_python_3.15",
+ filename = "simple-0.0.1.tar.gz",
+ repo = "pypi_315_simple_sdist_deadbeef",
+ target_platforms = None,
+ version = "3.15",
+ ),
+ ],
+ "some_pkg": [
+ struct(
+ config_setting = "//_config:is_python_3.15",
+ filename = None,
+ repo = "pypi_315_some_pkg",
+ target_platforms = None,
+ version = "3.15",
+ ),
+ ],
+ },
+ })
pypi.whl_libraries().contains_exactly({
"pypi_315_simple_py3_none_any_deadb00f": {
"dep_template": "@pypi//{name}:{target}",
@@ -372,9 +476,7 @@
"cp315_osx_x86_64",
"cp315_windows_x86_64",
],
- "extra_pip_args": [
- "--extra-args-for-sdist-building",
- ],
+ "extra_pip_args": ["--extra-args-for-sdist-building"],
"filename": "simple-0.0.1.tar.gz",
"python_interpreter_target": "unit_test_interpreter_target",
"repo": "pypi_315",
@@ -382,8 +484,29 @@
"sha256": "deadbeef",
"urls": ["example.org"],
},
+ # We are falling back to regular `pip`
+ "pypi_315_some_pkg": {
+ "dep_template": "@pypi//{name}:{target}",
+ "extra_pip_args": ["--extra-args-for-sdist-building"],
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "repo": "pypi_315",
+ "requirement": "some_pkg==0.0.1",
+ },
})
pypi.whl_mods().contains_exactly({})
+ env.expect.that_dict(got_simpleapi_download_kwargs).contains_exactly({
+ "attr": struct(
+ auth_patterns = {},
+ envsubst = {},
+ extra_index_urls = [],
+ index_url = "pypi.org",
+ index_url_overrides = {},
+ netrc = None,
+ sources = ["simple", "some_pkg"],
+ ),
+ "cache": {},
+ "parallel_download": False,
+ })
_tests.append(_test_simple_get_index)
diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl
index c719ad6..a6e17be 100644
--- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl
+++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl
@@ -30,6 +30,16 @@
"requirements_linux": """\
foo==0.0.3 --hash=sha256:deadbaaf
""",
+ # download_only = True
+ "requirements_linux_download_only": """\
+--platform=manylinux_2_17_x86_64
+--python-version=39
+--implementation=cp
+--abi=cp39
+
+foo==0.0.1 --hash=sha256:deadbeef
+bar==0.0.1 --hash=sha256:deadb00f
+""",
"requirements_lock": """\
foo[extra]==0.0.1 --hash=sha256:deadbeef
""",
@@ -45,6 +55,14 @@
"requirements_osx": """\
foo==0.0.3 --hash=sha256:deadbaaf
""",
+ "requirements_osx_download_only": """\
+--platform=macosx_10_9_arm64
+--python-version=39
+--implementation=cp
+--abi=cp39
+
+foo==0.0.3 --hash=sha256:deadbaaf
+""",
"requirements_windows": """\
foo[extra]==0.0.2 --hash=sha256:deadbeef
bar==0.0.1 --hash=sha256:deadb00f
@@ -229,6 +247,66 @@
_tests.append(_test_multi_os)
+def _test_multi_os_legacy(env):
+ got = parse_requirements(
+ ctx = _mock_ctx(),
+ requirements_by_platform = {
+ "requirements_linux_download_only": ["cp39_linux_x86_64"],
+ "requirements_osx_download_only": ["cp39_osx_aarch64"],
+ },
+ )
+
+ env.expect.that_dict(got).contains_exactly({
+ "bar": [
+ struct(
+ distribution = "bar",
+ extra_pip_args = ["--platform=manylinux_2_17_x86_64", "--python-version=39", "--implementation=cp", "--abi=cp39"],
+ is_exposed = False,
+ requirement_line = "bar==0.0.1 --hash=sha256:deadb00f",
+ sdist = None,
+ srcs = struct(
+ requirement = "bar==0.0.1",
+ shas = ["deadb00f"],
+ version = "0.0.1",
+ ),
+ target_platforms = ["cp39_linux_x86_64"],
+ whls = [],
+ ),
+ ],
+ "foo": [
+ struct(
+ distribution = "foo",
+ extra_pip_args = ["--platform=manylinux_2_17_x86_64", "--python-version=39", "--implementation=cp", "--abi=cp39"],
+ is_exposed = True,
+ requirement_line = "foo==0.0.1 --hash=sha256:deadbeef",
+ sdist = None,
+ srcs = struct(
+ requirement = "foo==0.0.1",
+ shas = ["deadbeef"],
+ version = "0.0.1",
+ ),
+ target_platforms = ["cp39_linux_x86_64"],
+ whls = [],
+ ),
+ struct(
+ distribution = "foo",
+ extra_pip_args = ["--platform=macosx_10_9_arm64", "--python-version=39", "--implementation=cp", "--abi=cp39"],
+ is_exposed = True,
+ requirement_line = "foo==0.0.3 --hash=sha256:deadbaaf",
+ sdist = None,
+ srcs = struct(
+ requirement = "foo==0.0.3",
+ shas = ["deadbaaf"],
+ version = "0.0.3",
+ ),
+ target_platforms = ["cp39_osx_aarch64"],
+ whls = [],
+ ),
+ ],
+ })
+
+_tests.append(_test_multi_os_legacy)
+
def _test_select_requirement_none_platform(env):
got = select_requirement(
[
diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl
index 9de309b..f518778 100644
--- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl
+++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl
@@ -387,6 +387,24 @@
_tests.append(_test_get_python_versions)
+def _test_get_python_versions_with_target_platforms(env):
+ got = get_whl_flag_versions(
+ aliases = [
+ whl_alias(repo = "foo", version = "3.3", target_platforms = ["cp33_linux_x86_64"]),
+ whl_alias(repo = "foo", version = "3.2", target_platforms = ["cp32_linux_x86_64", "cp32_osx_aarch64"]),
+ ],
+ )
+ want = {
+ "python_versions": ["3.2", "3.3"],
+ "target_platforms": [
+ "linux_x86_64",
+ "osx_aarch64",
+ ],
+ }
+ env.expect.that_dict(got).contains_exactly(want)
+
+_tests.append(_test_get_python_versions_with_target_platforms)
+
def _test_get_python_versions_from_filenames(env):
got = get_whl_flag_versions(
aliases = [
@@ -660,6 +678,29 @@
_tests.append(_test_multiplatform_whl_aliases_nofilename)
+def _test_multiplatform_whl_aliases_nofilename_target_platforms(env):
+ aliases = [
+ whl_alias(
+ repo = "foo",
+ config_setting = "//:ignored",
+ version = "3.1",
+ target_platforms = [
+ "cp31_linux_x86_64",
+ "cp31_linux_aarch64",
+ ],
+ ),
+ ]
+
+ got = multiplatform_whl_aliases(aliases = aliases)
+
+ want = [
+ whl_alias(config_setting = "//_config:is_cp3.1_linux_x86_64", repo = "foo", version = "3.1"),
+ whl_alias(config_setting = "//_config:is_cp3.1_linux_aarch64", repo = "foo", version = "3.1"),
+ ]
+ env.expect.that_collection(got).contains_exactly(want)
+
+_tests.append(_test_multiplatform_whl_aliases_nofilename_target_platforms)
+
def _test_multiplatform_whl_aliases_filename(env):
aliases = [
whl_alias(
@@ -734,6 +775,52 @@
_tests.append(_test_multiplatform_whl_aliases_filename_versioned)
+def _mock_alias(container):
+ return lambda name, **kwargs: container.append(name)
+
+def _mock_config_setting(container):
+ def _inner(name, flag_values = None, constraint_values = None, **_):
+ if flag_values or constraint_values:
+ container.append(name)
+ return
+
+ fail("At least one of 'flag_values' or 'constraint_values' needs to be set")
+
+ return _inner
+
+def _test_config_settings_exist_legacy(env):
+ aliases = [
+ whl_alias(
+ repo = "repo",
+ version = "3.11",
+ target_platforms = [
+ "cp311_linux_aarch64",
+ "cp311_linux_x86_64",
+ ],
+ ),
+ ]
+ available_config_settings = []
+ config_settings(
+ python_versions = ["3.11"],
+ native = struct(
+ alias = _mock_alias(available_config_settings),
+ config_setting = _mock_config_setting(available_config_settings),
+ ),
+ target_platforms = [
+ "linux_aarch64",
+ "linux_x86_64",
+ ],
+ )
+
+ got_aliases = multiplatform_whl_aliases(
+ aliases = aliases,
+ )
+ got = [a.config_setting.partition(":")[-1] for a in got_aliases]
+
+ env.expect.that_collection(available_config_settings).contains_at_least(got)
+
+_tests.append(_test_config_settings_exist_legacy)
+
def _test_config_settings_exist(env):
for py_tag in ["py2.py3", "py3", "py311", "cp311"]:
if py_tag == "py2.py3":
@@ -771,12 +858,11 @@
),
]
available_config_settings = []
- mock_rule = lambda name, **kwargs: available_config_settings.append(name)
config_settings(
python_versions = ["3.11"],
native = struct(
- alias = mock_rule,
- config_setting = mock_rule,
+ alias = _mock_alias(available_config_settings),
+ config_setting = _mock_config_setting(available_config_settings),
),
**kwargs
)
