feat(pypi): support direct urls for wheels in bazel downloader (#2655)
This PR adds support for installing wheels via direct urls in the
requirements lock file:
```
foo==0.0.1 @ https://someurl.org/package.whl
bar==0.0.1 @ https://someurl.org/package.tar.gz
```
This is to improve parity between bazel downloader and pip behavior.
Before this change, direct urls used fallback to pip install.
Partially addresses #2363 as it does not add support for git urls.
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 403dbaf..9029794 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -88,6 +88,9 @@
GitHub releases page metadata published by the `uv` project.
* (pypi) An extra argument to add the interpreter lib dir to `LDFLAGS` when
building wheels from `sdist`.
+* (pypi) Direct HTTP urls for wheels and sdists are now supported when using
+ {obj}`experimental_index_url` (bazel downloader).
+ Partially fixes [#2363](https://github.com/bazelbuild/rules_python/issues/2363).
{#v0-0-0-removed}
### Removed
diff --git a/python/private/pypi/index_sources.bzl b/python/private/pypi/index_sources.bzl
index 8b3c300..e3762d2 100644
--- a/python/private/pypi/index_sources.bzl
+++ b/python/private/pypi/index_sources.bzl
@@ -32,6 +32,7 @@
* `marker` - str; the marker expression, as per PEP508 spec.
* `requirement` - str; a requirement line without the marker. This can
be given to `pip` to install a package.
+ * `url` - str; URL if the requirement specifies a direct URL, empty string otherwise.
"""
line = line.replace("\\", " ")
head, _, maybe_hashes = line.partition(";")
@@ -55,9 +56,12 @@
requirement,
" ".join(["--hash=sha256:{}".format(sha) for sha in shas]),
).strip()
+
+ url = ""
if "@" in head:
requirement = requirement_line
- shas = []
+ _, _, url_and_rest = requirement.partition("@")
+ url = url_and_rest.strip().partition(" ")[0].strip()
return struct(
requirement = requirement,
@@ -65,4 +69,5 @@
version = version,
shas = sorted(shas),
marker = marker,
+ url = url,
)
diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl
index 2bca8d8..dbff44e 100644
--- a/python/private/pypi/parse_requirements.bzl
+++ b/python/private/pypi/parse_requirements.bzl
@@ -292,6 +292,23 @@
index_urls: The result of simpleapi_download.
logger: A logger for printing diagnostic info.
"""
+
+ # Handle direct URLs in requirements
+ if requirement.srcs.url:
+ url = requirement.srcs.url
+ _, _, filename = url.rpartition("/")
+ direct_url_dist = struct(
+ url = url,
+ filename = filename,
+ sha256 = requirement.srcs.shas[0] if requirement.srcs.shas else "",
+ yanked = False,
+ )
+
+ if filename.endswith(".whl"):
+ return [direct_url_dist], None
+ else:
+ return [], direct_url_dist
+
if not index_urls:
return [], None
diff --git a/tests/pypi/index_sources/index_sources_tests.bzl b/tests/pypi/index_sources/index_sources_tests.bzl
index 440957e..ffeed87 100644
--- a/tests/pypi/index_sources/index_sources_tests.bzl
+++ b/tests/pypi/index_sources/index_sources_tests.bzl
@@ -24,27 +24,39 @@
"foo==0.0.1": struct(
requirement = "foo==0.0.1",
marker = "",
+ url = "",
),
"foo==0.0.1 @ https://someurl.org": struct(
requirement = "foo==0.0.1 @ https://someurl.org",
marker = "",
+ url = "https://someurl.org",
),
- "foo==0.0.1 @ https://someurl.org --hash=sha256:deadbeef": struct(
- requirement = "foo==0.0.1 @ https://someurl.org --hash=sha256:deadbeef",
+ "foo==0.0.1 @ https://someurl.org/package.whl": struct(
+ requirement = "foo==0.0.1 @ https://someurl.org/package.whl",
marker = "",
+ url = "https://someurl.org/package.whl",
),
- "foo==0.0.1 @ https://someurl.org; python_version < \"2.7\"\\ --hash=sha256:deadbeef": struct(
- requirement = "foo==0.0.1 @ https://someurl.org --hash=sha256:deadbeef",
+ "foo==0.0.1 @ https://someurl.org/package.whl --hash=sha256:deadbeef": struct(
+ requirement = "foo==0.0.1 @ https://someurl.org/package.whl --hash=sha256:deadbeef",
+ marker = "",
+ url = "https://someurl.org/package.whl",
+ shas = ["deadbeef"],
+ ),
+ "foo==0.0.1 @ https://someurl.org/package.whl; python_version < \"2.7\"\\ --hash=sha256:deadbeef": struct(
+ requirement = "foo==0.0.1 @ https://someurl.org/package.whl --hash=sha256:deadbeef",
marker = "python_version < \"2.7\"",
+ url = "https://someurl.org/package.whl",
+ shas = ["deadbeef"],
),
}
for input, want in inputs.items():
got = index_sources(input)
- env.expect.that_collection(got.shas).contains_exactly([])
+ env.expect.that_collection(got.shas).contains_exactly(want.shas if hasattr(want, "shas") else [])
env.expect.that_str(got.version).equals("0.0.1")
env.expect.that_str(got.requirement).equals(want.requirement)
env.expect.that_str(got.requirement_line).equals(got.requirement)
env.expect.that_str(got.marker).equals(want.marker)
+ env.expect.that_str(got.url).equals(want.url)
_tests.append(_test_no_simple_api_sources)
@@ -58,6 +70,7 @@
marker = "",
requirement = "foo==0.0.2",
requirement_line = "foo==0.0.2 --hash=sha256:deafbeef --hash=sha256:deadbeef",
+ url = "",
),
"foo[extra]==0.0.2; (python_version < 2.7 or extra == \"@\") --hash=sha256:deafbeef --hash=sha256:deadbeef": struct(
shas = [
@@ -67,6 +80,7 @@
marker = "(python_version < 2.7 or extra == \"@\")",
requirement = "foo[extra]==0.0.2",
requirement_line = "foo[extra]==0.0.2 --hash=sha256:deafbeef --hash=sha256:deadbeef",
+ url = "",
),
}
for input, want in tests.items():
@@ -76,6 +90,7 @@
env.expect.that_str(got.requirement).equals(want.requirement)
env.expect.that_str(got.requirement_line).equals(want.requirement_line)
env.expect.that_str(got.marker).equals(want.marker)
+ env.expect.that_str(got.url).equals(want.url)
_tests.append(_test_simple_api_sources)
diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl
index 77e22b8..8edc268 100644
--- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl
+++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl
@@ -26,7 +26,10 @@
--hash=sha256:deadb00f
""",
"requirements_direct": """\
-foo[extra] @ https://some-url
+foo[extra] @ https://some-url/package.whl
+bar @ https://example.org/bar-1.0.whl --hash=sha256:deadbeef
+baz @ https://test.com/baz-2.0.whl; python_version < "3.8" --hash=sha256:deadb00f
+qux @ https://example.org/qux-1.0.tar.gz --hash=sha256:deadbe0f
""",
"requirements_extra_args": """\
--index-url=example.org
@@ -106,6 +109,7 @@
requirement_line = "foo[extra]==0.0.1 --hash=sha256:deadbeef",
shas = ["deadbeef"],
version = "0.0.1",
+ url = "",
),
target_platforms = [
"linux_x86_64",
@@ -124,6 +128,110 @@
_tests.append(_test_simple)
+def _test_direct_urls(env):
+ got = parse_requirements(
+ ctx = _mock_ctx(),
+ requirements_by_platform = {
+ "requirements_direct": ["linux_x86_64"],
+ },
+ )
+ env.expect.that_dict(got).contains_exactly({
+ "bar": [
+ struct(
+ distribution = "bar",
+ extra_pip_args = [],
+ sdist = None,
+ is_exposed = True,
+ srcs = struct(
+ marker = "",
+ requirement = "bar @ https://example.org/bar-1.0.whl --hash=sha256:deadbeef",
+ requirement_line = "bar @ https://example.org/bar-1.0.whl --hash=sha256:deadbeef",
+ shas = ["deadbeef"],
+ version = "",
+ url = "https://example.org/bar-1.0.whl",
+ ),
+ target_platforms = ["linux_x86_64"],
+ whls = [struct(
+ url = "https://example.org/bar-1.0.whl",
+ filename = "bar-1.0.whl",
+ sha256 = "deadbeef",
+ yanked = False,
+ )],
+ ),
+ ],
+ "baz": [
+ struct(
+ distribution = "baz",
+ extra_pip_args = [],
+ sdist = None,
+ is_exposed = True,
+ srcs = struct(
+ marker = "python_version < \"3.8\"",
+ requirement = "baz @ https://test.com/baz-2.0.whl --hash=sha256:deadb00f",
+ requirement_line = "baz @ https://test.com/baz-2.0.whl --hash=sha256:deadb00f",
+ shas = ["deadb00f"],
+ version = "",
+ url = "https://test.com/baz-2.0.whl",
+ ),
+ target_platforms = ["linux_x86_64"],
+ whls = [struct(
+ url = "https://test.com/baz-2.0.whl",
+ filename = "baz-2.0.whl",
+ sha256 = "deadb00f",
+ yanked = False,
+ )],
+ ),
+ ],
+ "foo": [
+ struct(
+ distribution = "foo",
+ extra_pip_args = [],
+ sdist = None,
+ is_exposed = True,
+ srcs = struct(
+ marker = "",
+ requirement = "foo[extra] @ https://some-url/package.whl",
+ requirement_line = "foo[extra] @ https://some-url/package.whl",
+ shas = [],
+ version = "",
+ url = "https://some-url/package.whl",
+ ),
+ target_platforms = ["linux_x86_64"],
+ whls = [struct(
+ url = "https://some-url/package.whl",
+ filename = "package.whl",
+ sha256 = "",
+ yanked = False,
+ )],
+ ),
+ ],
+ "qux": [
+ struct(
+ distribution = "qux",
+ extra_pip_args = [],
+ sdist = struct(
+ url = "https://example.org/qux-1.0.tar.gz",
+ filename = "qux-1.0.tar.gz",
+ sha256 = "deadbe0f",
+ yanked = False,
+ ),
+ is_exposed = True,
+ srcs = struct(
+ marker = "",
+ requirement = "qux @ https://example.org/qux-1.0.tar.gz --hash=sha256:deadbe0f",
+ requirement_line = "qux @ https://example.org/qux-1.0.tar.gz --hash=sha256:deadbe0f",
+ shas = ["deadbe0f"],
+ version = "",
+ url = "https://example.org/qux-1.0.tar.gz",
+ ),
+ target_platforms = ["linux_x86_64"],
+ whls = [],
+ ),
+ ],
+ })
+
+_tests.append(_test_direct_urls)
+
def _test_extra_pip_args(env):
got = parse_requirements(
ctx = _mock_ctx(),
@@ -145,6 +253,7 @@
requirement_line = "foo[extra]==0.0.1 --hash=sha256:deadbeef",
shas = ["deadbeef"],
version = "0.0.1",
+ url = "",
),
target_platforms = [
"linux_x86_64",
@@ -182,6 +291,7 @@
requirement_line = "foo[extra,extra_2]==0.0.1 --hash=sha256:deadbeef",
shas = ["deadbeef"],
version = "0.0.1",
+ url = "",
),
target_platforms = ["linux_x86_64"],
whls = [],
@@ -211,6 +321,7 @@
requirement_line = "bar==0.0.1 --hash=sha256:deadb00f",
shas = ["deadb00f"],
version = "0.0.1",
+ url = "",
),
target_platforms = ["windows_x86_64"],
whls = [],
@@ -228,6 +339,7 @@
requirement_line = "foo==0.0.3 --hash=sha256:deadbaaf",
shas = ["deadbaaf"],
version = "0.0.3",
+ url = "",
),
target_platforms = ["linux_x86_64"],
whls = [],
@@ -243,6 +355,7 @@
requirement_line = "foo[extra]==0.0.2 --hash=sha256:deadbeef",
shas = ["deadbeef"],
version = "0.0.2",
+ url = "",
),
target_platforms = ["windows_x86_64"],
whls = [],
@@ -282,6 +395,7 @@
requirement_line = "bar==0.0.1 --hash=sha256:deadb00f",
shas = ["deadb00f"],
version = "0.0.1",
+ url = "",
),
target_platforms = ["cp39_linux_x86_64"],
whls = [],
@@ -299,6 +413,7 @@
requirement_line = "foo==0.0.1 --hash=sha256:deadbeef",
shas = ["deadbeef"],
version = "0.0.1",
+ url = "",
),
target_platforms = ["cp39_linux_x86_64"],
whls = [],
@@ -314,6 +429,7 @@
requirement = "foo==0.0.3",
shas = ["deadbaaf"],
version = "0.0.3",
+ url = "",
),
target_platforms = ["cp39_osx_aarch64"],
whls = [],
@@ -367,6 +483,7 @@
requirement_line = "bar==0.0.1 --hash=sha256:deadbeef",
shas = ["deadbeef"],
version = "0.0.1",
+ url = "",
),
target_platforms = ["cp311_linux_super_exotic", "cp311_windows_x86_64"],
whls = [],
@@ -384,6 +501,7 @@
requirement_line = "foo[extra]==0.0.1 --hash=sha256:deadbeef",
shas = ["deadbeef"],
version = "0.0.1",
+ url = "",
),
target_platforms = ["cp311_windows_x86_64"],
whls = [],
@@ -419,6 +537,7 @@
requirement_line = "foo==0.0.1 --hash=sha256:deadb00f",
shas = ["deadb00f"],
version = "0.0.1",
+ url = "",
),
target_platforms = ["linux_x86_64"],
whls = [],
@@ -434,6 +553,7 @@
requirement_line = "foo==0.0.1+local --hash=sha256:deadbeef",
shas = ["deadbeef"],
version = "0.0.1+local",
+ url = "",
),
target_platforms = ["linux_x86_64"],
whls = [],
