feat(pypi/parse_requirements): get dists by version when no hash provied (#2695)
This pull request modifies the SimpleAPI HTML parsing to add a new
field where we can get the `sha256` values by package version. This
allows us to very easily fallback to all packages of a particular
version when using `experimental_index_url` if the hashes are not
specified.
The code deciding which packages to query the SimpleAPI for has been
also modified to only omit queries for packages that are included via
direct URL references.
If we fail to get the data from the SimpleAPI, we will fallback to
`pip` and try to install it via the legacy behaviour.
Fixes #2023
Work towards #260
Work towards #1357
Work towards #2363
---------
Co-authored-by: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5974a65..bbcf256 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -81,6 +81,14 @@
{#v0-0-0-added}
### Added
+* (pypi) From now on `sha256` values in the `requirements.txt` is no longer
+ mandatory when enabling {attr}`pip.parse.experimental_index_url` feature.
+ This means that `rules_python` will attempt to fetch metadata for all
+ packages through SimpleAPI unless they are pulled through direct URL
+ references. Fixes [#2023](https://github.com/bazel-contrib/rules_python/issues/2023).
+ In case you see issues with `rules_python` being too eager to fetch the SimpleAPI
+ metadata, you can use the newly added {attr}`pip.parse.experimental_skip_sources`
+ to skip metadata fetching for those packages.
* (uv) A {obj}`lock` rule that is the replacement for the
{obj}`compile_pip_requirements`. This may still have rough corners
so please report issues with it in the
diff --git a/docs/pypi-dependencies.md b/docs/pypi-dependencies.md
index 039200d..6cc0da6 100644
--- a/docs/pypi-dependencies.md
+++ b/docs/pypi-dependencies.md
@@ -386,11 +386,13 @@
rather means that it is calling the PyPI server to get the Simple API response
to get the list of all available source and wheel distributions. Once it has
got all of the available distributions, it will select the right ones depending
-on the `sha256` values in your `requirements_lock.txt` file. The compatible
-distribution URLs will be then written to the `MODULE.bazel.lock` file. Currently
-users wishing to use the lock file with `rules_python` with this feature have
-to set an environment variable `RULES_PYTHON_OS_ARCH_LOCK_FILE=0` which will
-become default in the next release.
+on the `sha256` values in your `requirements_lock.txt` file. If `sha256` hashes
+are not present in the requirements file, we will fallback to matching by version
+specified in the lock file. The compatible distribution URLs will be then
+written to the `MODULE.bazel.lock` file. Currently users wishing to use the
+lock file with `rules_python` with this feature have to set an environment
+variable `RULES_PYTHON_OS_ARCH_LOCK_FILE=0` which will become default in the
+next release.
Fetching the distribution information from the PyPI allows `rules_python` to
know which `whl` should be used on which target platform and it will determine
diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl
index 490bd05..f782e69 100644
--- a/python/private/pypi/extension.bzl
+++ b/python/private/pypi/extension.bzl
@@ -459,13 +459,21 @@
get_index_urls = None
if pip_attr.experimental_index_url:
is_reproducible = False
+ skip_sources = [
+ normalize_name(s)
+ for s in pip_attr.simpleapi_skip
+ ]
get_index_urls = lambda ctx, distributions: simpleapi_download(
ctx,
attr = struct(
index_url = pip_attr.experimental_index_url,
extra_index_urls = pip_attr.experimental_extra_index_urls or [],
index_url_overrides = pip_attr.experimental_index_url_overrides or {},
- sources = distributions,
+ sources = [
+ d
+ for d in distributions
+ if normalize_name(d) not in skip_sources
+ ],
envsubst = pip_attr.envsubst,
# Auth related info
netrc = pip_attr.netrc,
@@ -682,6 +690,11 @@
If {attr}`download_only` is set, then `sdist` archives will be discarded and `pip.parse` will
operate in wheel-only mode.
:::
+
+:::{versionchanged} VERSION_NEXT_FEATURE
+Index metadata will be used to deduct `sha256` values for packages even if the
+`sha256` values are not present in the requirements.txt lock file.
+:::
""",
),
"experimental_index_url_overrides": attr.string_dict(
@@ -751,6 +764,18 @@
a corresponding `python.toolchain()` configured.
""",
),
+ "simpleapi_skip": attr.string_list(
+ doc = """\
+The list of packages to skip fetching metadata for from SimpleAPI index. You should
+normally not need this attribute, but in case you do, please report this as a bug
+to `rules_python` and use this attribute until the bug is fixed.
+
+EXPERIMENTAL: this may be removed without notice.
+
+:::{versionadded} VERSION_NEXT_FEATURE
+:::
+""",
+ ),
"whl_modifications": attr.label_keyed_string_dict(
mandatory = False,
doc = """\
diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl
index 7aadc15..3280ce8 100644
--- a/python/private/pypi/parse_requirements.bzl
+++ b/python/private/pypi/parse_requirements.bzl
@@ -184,7 +184,7 @@
req.distribution: None
for reqs in requirements_by_platform.values()
for req in reqs.values()
- if req.srcs.shas
+ if not req.srcs.url
}),
)
@@ -315,10 +315,15 @@
whls = []
sdist = None
- # TODO @aignas 2024-05-22: it is in theory possible to add all
- # requirements by version instead of by sha256. This may be useful
- # for some projects.
- for sha256 in requirement.srcs.shas:
+ # First try to find distributions by SHA256 if provided
+ shas_to_use = requirement.srcs.shas
+ if not shas_to_use:
+ version = requirement.srcs.version
+ shas_to_use = index_urls.sha256s_by_version.get(version, [])
+ if logger:
+ logger.warn(lambda: "requirement file has been generated without hashes, will use all hashes for the given version {} that could find on the index:\n {}".format(version, shas_to_use))
+
+ for sha256 in shas_to_use:
# For now if the artifact is marked as yanked we just ignore it.
#
# See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api
diff --git a/python/private/pypi/parse_simpleapi_html.bzl b/python/private/pypi/parse_simpleapi_html.bzl
index e549e76..8c6f739 100644
--- a/python/private/pypi/parse_simpleapi_html.bzl
+++ b/python/private/pypi/parse_simpleapi_html.bzl
@@ -26,6 +26,7 @@
Returns:
A list of structs with:
* filename: The filename of the artifact.
+ * version: The version of the artifact.
* url: The URL to download the artifact.
* sha256: The sha256 of the artifact.
* metadata_sha256: The whl METADATA sha256 if we can download it. If this is
@@ -51,8 +52,11 @@
# Each line follows the following pattern
# <a href="https://...#sha256=..." attribute1="foo" ... attributeN="bar">filename</a><br />
+ sha256_by_version = {}
for line in lines[1:]:
dist_url, _, tail = line.partition("#sha256=")
+ dist_url = _absolute_url(url, dist_url)
+
sha256, _, tail = tail.partition("\"")
# See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api
@@ -60,6 +64,8 @@
head, _, _ = tail.rpartition("</a>")
maybe_metadata, _, filename = head.rpartition(">")
+ version = _version(filename)
+ sha256_by_version.setdefault(version, []).append(sha256)
metadata_sha256 = ""
metadata_url = ""
@@ -75,7 +81,8 @@
if filename.endswith(".whl"):
whls[sha256] = struct(
filename = filename,
- url = _absolute_url(url, dist_url),
+ version = version,
+ url = dist_url,
sha256 = sha256,
metadata_sha256 = metadata_sha256,
metadata_url = _absolute_url(url, metadata_url) if metadata_url else "",
@@ -84,7 +91,8 @@
else:
sdists[sha256] = struct(
filename = filename,
- url = _absolute_url(url, dist_url),
+ version = version,
+ url = dist_url,
sha256 = sha256,
metadata_sha256 = "",
metadata_url = "",
@@ -94,8 +102,31 @@
return struct(
sdists = sdists,
whls = whls,
+ sha256_by_version = sha256_by_version,
)
+_SDIST_EXTS = [
+ ".tar", # handles any compression
+ ".zip",
+]
+
+def _version(filename):
+ # See https://packaging.python.org/en/latest/specifications/binary-distribution-format/#binary-distribution-format
+
+ _, _, tail = filename.partition("-")
+ version, _, _ = tail.partition("-")
+ if version != tail:
+ # The format is {name}-{version}-{whl_specifiers}.whl
+ return version
+
+ # NOTE @aignas 2025-03-29: most of the files are wheels, so this is not the common path
+
+ # {name}-{version}.{ext}
+ for ext in _SDIST_EXTS:
+ version, _, _ = version.partition(ext) # build or name
+
+ return version
+
def _get_root_directory(url):
scheme_end = url.find("://")
if scheme_end == -1:
diff --git a/python/private/pypi/simpleapi_download.bzl b/python/private/pypi/simpleapi_download.bzl
index ef39fb8..e8d7d09 100644
--- a/python/private/pypi/simpleapi_download.bzl
+++ b/python/private/pypi/simpleapi_download.bzl
@@ -127,10 +127,17 @@
failed_sources = [pkg for pkg in attr.sources if pkg not in found_on_index]
if failed_sources:
- _fail("Failed to download metadata for {} for from urls: {}".format(
- failed_sources,
- index_urls,
- ))
+ _fail(
+ "\n".join([
+ "Failed to download metadata for {} for from urls: {}.".format(
+ failed_sources,
+ index_urls,
+ ),
+ "If you would like to skip downloading metadata for these packages please add 'simpleapi_skip={}' to your 'pip.parse' call.".format(
+ render.list(failed_sources),
+ ),
+ ]),
+ )
return None
if warn_overrides:
diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl
index 38ac9dc..2904f85 100644
--- a/python/private/pypi/whl_library.bzl
+++ b/python/private/pypi/whl_library.bzl
@@ -270,6 +270,12 @@
sha256 = rctx.attr.sha256,
auth = get_auth(rctx, urls),
)
+ if not rctx.attr.sha256:
+ # this is only seen when there is a direct URL reference without sha256
+ logger.warn("Please update the requirement line to include the hash:\n{} \\\n --hash=sha256:{}".format(
+ rctx.attr.requirement,
+ result.sha256,
+ ))
if not result.success:
fail("could not download the '{}' from {}:\n{}".format(filename, urls, result))
diff --git a/python/private/pypi/whl_repo_name.bzl b/python/private/pypi/whl_repo_name.bzl
index 48bbd1a..02a7c81 100644
--- a/python/private/pypi/whl_repo_name.bzl
+++ b/python/private/pypi/whl_repo_name.bzl
@@ -32,11 +32,19 @@
if not filename.endswith(".whl"):
# Then the filename is basically foo-3.2.1.<ext>
- parts.append(normalize_name(filename.rpartition("-")[0]))
- parts.append("sdist")
+ name, _, tail = filename.rpartition("-")
+ parts.append(normalize_name(name))
+ if sha256:
+ parts.append("sdist")
+ version = ""
+ else:
+ for ext in [".tar", ".zip"]:
+ tail, _, _ = tail.partition(ext)
+ version = tail.replace(".", "_").replace("!", "_")
else:
parsed = parse_whl_name(filename)
name = normalize_name(parsed.distribution)
+ version = parsed.version.replace(".", "_").replace("!", "_")
python_tag, _, _ = parsed.python_tag.partition(".")
abi_tag, _, _ = parsed.abi_tag.partition(".")
platform_tag, _, _ = parsed.platform_tag.partition(".")
@@ -46,7 +54,10 @@
parts.append(abi_tag)
parts.append(platform_tag)
- parts.append(sha256[:8])
+ if sha256:
+ parts.append(sha256[:8])
+ elif version:
+ parts.insert(1, version)
return "_".join(parts)
diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl
index 858c026..3a91c7b 100644
--- a/tests/pypi/extension/extension_tests.bzl
+++ b/tests/pypi/extension/extension_tests.bzl
@@ -100,6 +100,7 @@
requirements_linux = None,
requirements_lock = None,
requirements_windows = None,
+ simpleapi_skip = [],
timeout = 600,
whl_modifications = {},
**kwargs):
@@ -135,6 +136,7 @@
experimental_extra_index_urls = [],
parallel_download = False,
experimental_index_url_overrides = {},
+ simpleapi_skip = simpleapi_skip,
**kwargs
)
@@ -616,6 +618,21 @@
),
},
),
+ "some_other_pkg": struct(
+ whls = {
+ "deadb33f": struct(
+ yanked = False,
+ filename = "some-other-pkg-0.0.1-py3-none-any.whl",
+ sha256 = "deadb33f",
+ url = "example2.org/index/some_other_pkg/",
+ ),
+ },
+ sdists = {},
+ sha256s_by_version = {
+ "0.0.1": ["deadb33f"],
+ "0.0.3": ["deadbeef"],
+ },
+ ),
}
pypi = _parse_modules(
@@ -640,7 +657,11 @@
simple==0.0.1 \
--hash=sha256:deadbeef \
--hash=sha256:deadb00f
-some_pkg==0.0.1
+some_pkg==0.0.1 @ example-direct.org/some_pkg-0.0.1-py3-none-any.whl \
+ --hash=sha256:deadbaaf
+direct_without_sha==0.0.1 @ example-direct.org/direct_without_sha-0.0.1-py3-none-any.whl
+some_other_pkg==0.0.1
+pip_fallback==0.0.1
""",
}[x],
),
@@ -651,42 +672,91 @@
)
pypi.is_reproducible().equals(False)
- pypi.exposed_packages().contains_exactly({"pypi": ["simple", "some_pkg"]})
+ pypi.exposed_packages().contains_exactly({"pypi": ["direct_without_sha", "pip_fallback", "simple", "some_other_pkg", "some_pkg"]})
pypi.hub_group_map().contains_exactly({"pypi": {}})
pypi.hub_whl_map().contains_exactly({
"pypi": {
+ "direct_without_sha": {
+ "pypi_315_direct_without_sha_0_0_1_py3_none_any": [
+ struct(
+ config_setting = None,
+ filename = "direct_without_sha-0.0.1-py3-none-any.whl",
+ target_platforms = None,
+ version = "3.15",
+ ),
+ ],
+ },
+ "pip_fallback": {
+ "pypi_315_pip_fallback": [
+ struct(
+ config_setting = None,
+ filename = None,
+ target_platforms = None,
+ version = "3.15",
+ ),
+ ],
+ },
"simple": {
"pypi_315_simple_py3_none_any_deadb00f": [
- whl_config_setting(
+ struct(
+ config_setting = None,
filename = "simple-0.0.1-py3-none-any.whl",
+ target_platforms = None,
version = "3.15",
),
],
"pypi_315_simple_sdist_deadbeef": [
- whl_config_setting(
+ struct(
+ config_setting = None,
filename = "simple-0.0.1.tar.gz",
+ target_platforms = None,
+ version = "3.15",
+ ),
+ ],
+ },
+ "some_other_pkg": {
+ "pypi_315_some_py3_none_any_deadb33f": [
+ struct(
+ config_setting = None,
+ filename = "some-other-pkg-0.0.1-py3-none-any.whl",
+ target_platforms = None,
version = "3.15",
),
],
},
"some_pkg": {
- "pypi_315_some_pkg": [whl_config_setting(version = "3.15")],
+ "pypi_315_some_pkg_py3_none_any_deadbaaf": [
+ struct(
+ config_setting = None,
+ filename = "some_pkg-0.0.1-py3-none-any.whl",
+ target_platforms = None,
+ version = "3.15",
+ ),
+ ],
},
},
})
pypi.whl_libraries().contains_exactly({
+ "pypi_315_direct_without_sha_0_0_1_py3_none_any": {
+ "dep_template": "@pypi//{name}:{target}",
+ "experimental_target_platforms": ["cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"],
+ "filename": "direct_without_sha-0.0.1-py3-none-any.whl",
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "repo": "pypi_315",
+ "requirement": "direct_without_sha==0.0.1 @ example-direct.org/direct_without_sha-0.0.1-py3-none-any.whl",
+ "sha256": "",
+ "urls": ["example-direct.org/direct_without_sha-0.0.1-py3-none-any.whl"],
+ },
+ "pypi_315_pip_fallback": {
+ "dep_template": "@pypi//{name}:{target}",
+ "extra_pip_args": ["--extra-args-for-sdist-building"],
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "repo": "pypi_315",
+ "requirement": "pip_fallback==0.0.1",
+ },
"pypi_315_simple_py3_none_any_deadb00f": {
"dep_template": "@pypi//{name}:{target}",
- "experimental_target_platforms": [
- "cp315_linux_aarch64",
- "cp315_linux_arm",
- "cp315_linux_ppc",
- "cp315_linux_s390x",
- "cp315_linux_x86_64",
- "cp315_osx_aarch64",
- "cp315_osx_x86_64",
- "cp315_windows_x86_64",
- ],
+ "experimental_target_platforms": ["cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"],
"filename": "simple-0.0.1-py3-none-any.whl",
"python_interpreter_target": "unit_test_interpreter_target",
"repo": "pypi_315",
@@ -696,16 +766,7 @@
},
"pypi_315_simple_sdist_deadbeef": {
"dep_template": "@pypi//{name}:{target}",
- "experimental_target_platforms": [
- "cp315_linux_aarch64",
- "cp315_linux_arm",
- "cp315_linux_ppc",
- "cp315_linux_s390x",
- "cp315_linux_x86_64",
- "cp315_osx_aarch64",
- "cp315_osx_x86_64",
- "cp315_windows_x86_64",
- ],
+ "experimental_target_platforms": ["cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"],
"extra_pip_args": ["--extra-args-for-sdist-building"],
"filename": "simple-0.0.1.tar.gz",
"python_interpreter_target": "unit_test_interpreter_target",
@@ -714,29 +775,43 @@
"sha256": "deadbeef",
"urls": ["example.org"],
},
- # We are falling back to regular `pip`
- "pypi_315_some_pkg": {
+ "pypi_315_some_pkg_py3_none_any_deadbaaf": {
"dep_template": "@pypi//{name}:{target}",
- "extra_pip_args": ["--extra-args-for-sdist-building"],
+ "experimental_target_platforms": ["cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"],
+ "filename": "some_pkg-0.0.1-py3-none-any.whl",
"python_interpreter_target": "unit_test_interpreter_target",
"repo": "pypi_315",
- "requirement": "some_pkg==0.0.1",
+ "requirement": "some_pkg==0.0.1 @ example-direct.org/some_pkg-0.0.1-py3-none-any.whl --hash=sha256:deadbaaf",
+ "sha256": "deadbaaf",
+ "urls": ["example-direct.org/some_pkg-0.0.1-py3-none-any.whl"],
+ },
+ "pypi_315_some_py3_none_any_deadb33f": {
+ "dep_template": "@pypi//{name}:{target}",
+ "experimental_target_platforms": ["cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"],
+ "filename": "some-other-pkg-0.0.1-py3-none-any.whl",
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "repo": "pypi_315",
+ "requirement": "some_other_pkg==0.0.1",
+ "sha256": "deadb33f",
+ "urls": ["example2.org/index/some_other_pkg/"],
},
})
pypi.whl_mods().contains_exactly({})
- env.expect.that_dict(got_simpleapi_download_kwargs).contains_exactly({
- "attr": struct(
- auth_patterns = {},
- envsubst = {},
- extra_index_urls = [],
- index_url = "pypi.org",
- index_url_overrides = {},
- netrc = None,
- sources = ["simple"],
- ),
- "cache": {},
- "parallel_download": False,
- })
+ env.expect.that_dict(got_simpleapi_download_kwargs).contains_exactly(
+ {
+ "attr": struct(
+ auth_patterns = {},
+ envsubst = {},
+ extra_index_urls = [],
+ index_url = "pypi.org",
+ index_url_overrides = {},
+ netrc = None,
+ sources = ["simple", "pip_fallback", "some_other_pkg"],
+ ),
+ "cache": {},
+ "parallel_download": False,
+ },
+ )
_tests.append(_test_simple_get_index)
diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl
index 7bbd696..c504821 100644
--- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl
+++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl
@@ -62,6 +62,10 @@
foo[extra]==0.0.1 ;marker --hash=sha256:deadbeef
bar==0.0.1 --hash=sha256:deadbeef
""",
+ "requirements_optional_hash": """
+foo==0.0.4 @ https://example.org/foo-0.0.4.whl
+foo==0.0.5 @ https://example.org/foo-0.0.5.whl --hash=sha256:deadbeef
+""",
"requirements_osx": """\
foo==0.0.3 --hash=sha256:deadbaaf
""",
@@ -563,6 +567,62 @@
_tests.append(_test_different_package_version)
+def _test_optional_hash(env):
+ got = parse_requirements(
+ ctx = _mock_ctx(),
+ requirements_by_platform = {
+ "requirements_optional_hash": ["linux_x86_64"],
+ },
+ )
+ env.expect.that_dict(got).contains_exactly({
+ "foo": [
+ struct(
+ distribution = "foo",
+ extra_pip_args = [],
+ sdist = None,
+ is_exposed = True,
+ srcs = struct(
+ marker = "",
+ requirement = "foo==0.0.4 @ https://example.org/foo-0.0.4.whl",
+ requirement_line = "foo==0.0.4 @ https://example.org/foo-0.0.4.whl",
+ shas = [],
+ version = "0.0.4",
+ url = "https://example.org/foo-0.0.4.whl",
+ ),
+ target_platforms = ["linux_x86_64"],
+ whls = [struct(
+ url = "https://example.org/foo-0.0.4.whl",
+ filename = "foo-0.0.4.whl",
+ sha256 = "",
+ yanked = False,
+ )],
+ ),
+ struct(
+ distribution = "foo",
+ extra_pip_args = [],
+ sdist = None,
+ is_exposed = True,
+ srcs = struct(
+ marker = "",
+ requirement = "foo==0.0.5 @ https://example.org/foo-0.0.5.whl --hash=sha256:deadbeef",
+ requirement_line = "foo==0.0.5 @ https://example.org/foo-0.0.5.whl --hash=sha256:deadbeef",
+ shas = ["deadbeef"],
+ version = "0.0.5",
+ url = "https://example.org/foo-0.0.5.whl",
+ ),
+ target_platforms = ["linux_x86_64"],
+ whls = [struct(
+ url = "https://example.org/foo-0.0.5.whl",
+ filename = "foo-0.0.5.whl",
+ sha256 = "deadbeef",
+ yanked = False,
+ )],
+ ),
+ ],
+ })
+
+_tests.append(_test_optional_hash)
+
def parse_requirements_test_suite(name):
"""Create the test suite.
diff --git a/tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl b/tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl
index d3c42a8..abaa7a6 100644
--- a/tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl
+++ b/tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl
@@ -52,13 +52,14 @@
'data-requires-python=">=3.7"',
],
filename = "foo-0.0.1.tar.gz",
- url = "ignored",
+ url = "foo",
),
struct(
filename = "foo-0.0.1.tar.gz",
sha256 = "deadbeefasource",
url = "https://example.org/full-url/foo-0.0.1.tar.gz",
yanked = False,
+ version = "0.0.1",
),
),
(
@@ -68,12 +69,13 @@
'data-requires-python=">=3.7"',
],
filename = "foo-0.0.1.tar.gz",
- url = "ignored",
+ url = "foo",
),
struct(
filename = "foo-0.0.1.tar.gz",
sha256 = "deadbeefasource",
url = "https://example.org/full-url/foo-0.0.1.tar.gz",
+ version = "0.0.1",
yanked = False,
),
),
@@ -94,12 +96,14 @@
sha256 = subjects.str,
url = subjects.str,
yanked = subjects.bool,
+ version = subjects.str,
),
)
actual.filename().equals(want.filename)
actual.sha256().equals(want.sha256)
actual.url().equals(want.url)
actual.yanked().equals(want.yanked)
+ actual.version().equals(want.version)
_tests.append(_test_sdist)
@@ -115,7 +119,7 @@
'data-core-metadata="sha256=deadb00f"',
],
filename = "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
- url = "ignored",
+ url = "foo",
),
struct(
filename = "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
@@ -123,6 +127,7 @@
metadata_url = "https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata",
sha256 = "deadbeef",
url = "https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+ version = "0.0.2",
yanked = False,
),
),
@@ -135,7 +140,7 @@
'data-core-metadata="sha256=deadb00f"',
],
filename = "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
- url = "ignored",
+ url = "foo",
),
struct(
filename = "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
@@ -143,6 +148,7 @@
metadata_url = "https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata",
sha256 = "deadbeef",
url = "https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+ version = "0.0.2",
yanked = False,
),
),
@@ -154,13 +160,14 @@
'data-core-metadata="sha256=deadb00f"',
],
filename = "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
- url = "ignored",
+ url = "foo",
),
struct(
filename = "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
metadata_sha256 = "deadb00f",
metadata_url = "https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata",
sha256 = "deadbeef",
+ version = "0.0.2",
url = "https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
yanked = False,
),
@@ -173,13 +180,14 @@
'data-dist-info-metadata="sha256=deadb00f"',
],
filename = "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
- url = "ignored",
+ url = "foo",
),
struct(
filename = "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
metadata_sha256 = "deadb00f",
metadata_url = "https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata",
sha256 = "deadbeef",
+ version = "0.0.2",
url = "https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
yanked = False,
),
@@ -191,7 +199,7 @@
'data-requires-python=">=3.7"',
],
filename = "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
- url = "ignored",
+ url = "foo",
),
struct(
filename = "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
@@ -199,6 +207,7 @@
metadata_url = "",
sha256 = "deadbeef",
url = "https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+ version = "0.0.2",
yanked = False,
),
),
@@ -217,6 +226,7 @@
metadata_sha256 = "deadb00f",
metadata_url = "https://example.org/python-wheels/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata",
sha256 = "deadbeef",
+ version = "0.0.2",
url = "https://example.org/python-wheels/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
yanked = False,
),
@@ -235,6 +245,7 @@
metadata_url = "",
sha256 = "deadbeef",
url = "https://download.pytorch.org/whl/torch-2.0.0-cp38-cp38-manylinux2014_aarch64.whl",
+ version = "2.0.0",
yanked = False,
),
),
@@ -252,6 +263,7 @@
metadata_url = "",
sha256 = "notdeadbeef",
url = "http://download.pytorch.org/whl/torch-2.0.0-cp38-cp38-manylinux2014_aarch64.whl",
+ version = "2.0.0",
yanked = False,
),
),
@@ -267,6 +279,7 @@
filename = "mypy_extensions-1.0.0-py3-none-any.whl",
metadata_sha256 = "",
metadata_url = "",
+ version = "1.0.0",
sha256 = "deadbeef",
url = "https://example.org/simple/mypy_extensions/1.0.0/mypy_extensions-1.0.0-py3-none-any.whl",
yanked = False,
@@ -285,6 +298,7 @@
metadata_sha256 = "",
metadata_url = "",
sha256 = "deadbeef",
+ version = "1.0.0",
url = "https://example.org/simple/mypy_extensions/unknown://example.com/mypy_extensions-1.0.0-py3-none-any.whl",
yanked = False,
),
@@ -308,6 +322,7 @@
sha256 = subjects.str,
url = subjects.str,
yanked = subjects.bool,
+ version = subjects.str,
),
)
actual.filename().equals(want.filename)
@@ -316,6 +331,7 @@
actual.sha256().equals(want.sha256)
actual.url().equals(want.url)
actual.yanked().equals(want.yanked)
+ actual.version().equals(want.version)
_tests.append(_test_whls)
diff --git a/tests/pypi/simpleapi_download/simpleapi_download_tests.bzl b/tests/pypi/simpleapi_download/simpleapi_download_tests.bzl
index 964d3e2..ce214d6 100644
--- a/tests/pypi/simpleapi_download/simpleapi_download_tests.bzl
+++ b/tests/pypi/simpleapi_download/simpleapi_download_tests.bzl
@@ -110,7 +110,10 @@
)
env.expect.that_collection(fails).contains_exactly([
- """Failed to download metadata for ["foo"] for from urls: ["main", "extra"]""",
+ """\
+Failed to download metadata for ["foo"] for from urls: ["main", "extra"].
+If you would like to skip downloading metadata for these packages please add 'simpleapi_skip=["foo"]' to your 'pip.parse' call.\
+""",
])
env.expect.that_collection(calls).contains_exactly([
"extra/foo/",
diff --git a/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl b/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl
index 000941b..f0d1d05 100644
--- a/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl
+++ b/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl
@@ -25,12 +25,24 @@
_tests.append(_test_simple)
+def _test_simple_no_sha(env):
+ got = whl_repo_name("foo-1.2.3-py3-none-any.whl", "")
+ env.expect.that_str(got).equals("foo_1_2_3_py3_none_any")
+
+_tests.append(_test_simple_no_sha)
+
def _test_sdist(env):
got = whl_repo_name("foo-1.2.3.tar.gz", "deadbeef000deadbeef")
env.expect.that_str(got).equals("foo_sdist_deadbeef")
_tests.append(_test_sdist)
+def _test_sdist_no_sha(env):
+ got = whl_repo_name("foo-1.2.3.tar.gz", "")
+ env.expect.that_str(got).equals("foo_1_2_3")
+
+_tests.append(_test_sdist_no_sha)
+
def _test_platform_whl(env):
got = whl_repo_name(
"foo-1.2.3-cp39.cp310-abi3-manylinux1_x86_64.manylinux_2_17_x86_64.whl",
