feat(pypi): support direct urls for wheels in bazel downloader (#2655) This PR adds support for installing wheels via direct urls in the requirements lock file: ``` foo==0.0.1 @ https://someurl.org/package.whl bar==0.0.1 @ https://someurl.org/package.tar.gz ``` This is to improve parity between bazel downloader and pip behavior. Before this change, direct urls used fallback to pip install. Partially addresses #2363 as it does not add support for git urls.
diff --git a/CHANGELOG.md b/CHANGELOG.md index 403dbaf..9029794 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md
@@ -88,6 +88,9 @@ GitHub releases page metadata published by the `uv` project. * (pypi) An extra argument to add the interpreter lib dir to `LDFLAGS` when building wheels from `sdist`. +* (pypi) Direct HTTP urls for wheels and sdists are now supported when using + {obj}`experimental_index_url` (bazel downloader). + Partially fixes [#2363](https://github.com/bazelbuild/rules_python/issues/2363). {#v0-0-0-removed} ### Removed
diff --git a/python/private/pypi/index_sources.bzl b/python/private/pypi/index_sources.bzl index 8b3c300..e3762d2 100644 --- a/python/private/pypi/index_sources.bzl +++ b/python/private/pypi/index_sources.bzl
@@ -32,6 +32,7 @@ * `marker` - str; the marker expression, as per PEP508 spec. * `requirement` - str; a requirement line without the marker. This can be given to `pip` to install a package. + * `url` - str; URL if the requirement specifies a direct URL, empty string otherwise. """ line = line.replace("\\", " ") head, _, maybe_hashes = line.partition(";") @@ -55,9 +56,12 @@ requirement, " ".join(["--hash=sha256:{}".format(sha) for sha in shas]), ).strip() + + url = "" if "@" in head: requirement = requirement_line - shas = [] + _, _, url_and_rest = requirement.partition("@") + url = url_and_rest.strip().partition(" ")[0].strip() return struct( requirement = requirement, @@ -65,4 +69,5 @@ version = version, shas = sorted(shas), marker = marker, + url = url, )
diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index 2bca8d8..dbff44e 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl
@@ -292,6 +292,23 @@ index_urls: The result of simpleapi_download. logger: A logger for printing diagnostic info. """ + + # Handle direct URLs in requirements + if requirement.srcs.url: + url = requirement.srcs.url + _, _, filename = url.rpartition("/") + direct_url_dist = struct( + url = url, + filename = filename, + sha256 = requirement.srcs.shas[0] if requirement.srcs.shas else "", + yanked = False, + ) + + if filename.endswith(".whl"): + return [direct_url_dist], None + else: + return [], direct_url_dist + if not index_urls: return [], None
diff --git a/tests/pypi/index_sources/index_sources_tests.bzl b/tests/pypi/index_sources/index_sources_tests.bzl index 440957e..ffeed87 100644 --- a/tests/pypi/index_sources/index_sources_tests.bzl +++ b/tests/pypi/index_sources/index_sources_tests.bzl
@@ -24,27 +24,39 @@ "foo==0.0.1": struct( requirement = "foo==0.0.1", marker = "", + url = "", ), "foo==0.0.1 @ https://someurl.org": struct( requirement = "foo==0.0.1 @ https://someurl.org", marker = "", + url = "https://someurl.org", ), - "foo==0.0.1 @ https://someurl.org --hash=sha256:deadbeef": struct( - requirement = "foo==0.0.1 @ https://someurl.org --hash=sha256:deadbeef", + "foo==0.0.1 @ https://someurl.org/package.whl": struct( + requirement = "foo==0.0.1 @ https://someurl.org/package.whl", marker = "", + url = "https://someurl.org/package.whl", ), - "foo==0.0.1 @ https://someurl.org; python_version < \"2.7\"\\ --hash=sha256:deadbeef": struct( - requirement = "foo==0.0.1 @ https://someurl.org --hash=sha256:deadbeef", + "foo==0.0.1 @ https://someurl.org/package.whl --hash=sha256:deadbeef": struct( + requirement = "foo==0.0.1 @ https://someurl.org/package.whl --hash=sha256:deadbeef", + marker = "", + url = "https://someurl.org/package.whl", + shas = ["deadbeef"], + ), + "foo==0.0.1 @ https://someurl.org/package.whl; python_version < \"2.7\"\\ --hash=sha256:deadbeef": struct( + requirement = "foo==0.0.1 @ https://someurl.org/package.whl --hash=sha256:deadbeef", marker = "python_version < \"2.7\"", + url = "https://someurl.org/package.whl", + shas = ["deadbeef"], ), } for input, want in inputs.items(): got = index_sources(input) - env.expect.that_collection(got.shas).contains_exactly([]) + env.expect.that_collection(got.shas).contains_exactly(want.shas if hasattr(want, "shas") else []) env.expect.that_str(got.version).equals("0.0.1") env.expect.that_str(got.requirement).equals(want.requirement) env.expect.that_str(got.requirement_line).equals(got.requirement) env.expect.that_str(got.marker).equals(want.marker) + env.expect.that_str(got.url).equals(want.url) _tests.append(_test_no_simple_api_sources) @@ -58,6 +70,7 @@ marker = "", requirement = "foo==0.0.2", requirement_line = "foo==0.0.2 --hash=sha256:deafbeef --hash=sha256:deadbeef", + url = "", ), "foo[extra]==0.0.2; (python_version < 2.7 or extra == \"@\") --hash=sha256:deafbeef --hash=sha256:deadbeef": struct( shas = [ @@ -67,6 +80,7 @@ marker = "(python_version < 2.7 or extra == \"@\")", requirement = "foo[extra]==0.0.2", requirement_line = "foo[extra]==0.0.2 --hash=sha256:deafbeef --hash=sha256:deadbeef", + url = "", ), } for input, want in tests.items(): @@ -76,6 +90,7 @@ env.expect.that_str(got.requirement).equals(want.requirement) env.expect.that_str(got.requirement_line).equals(want.requirement_line) env.expect.that_str(got.marker).equals(want.marker) + env.expect.that_str(got.url).equals(want.url) _tests.append(_test_simple_api_sources)
diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index 77e22b8..8edc268 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl
@@ -26,7 +26,10 @@ --hash=sha256:deadb00f """, "requirements_direct": """\ -foo[extra] @ https://some-url +foo[extra] @ https://some-url/package.whl +bar @ https://example.org/bar-1.0.whl --hash=sha256:deadbeef +baz @ https://test.com/baz-2.0.whl; python_version < "3.8" --hash=sha256:deadb00f +qux @ https://example.org/qux-1.0.tar.gz --hash=sha256:deadbe0f """, "requirements_extra_args": """\ --index-url=example.org @@ -106,6 +109,7 @@ requirement_line = "foo[extra]==0.0.1 --hash=sha256:deadbeef", shas = ["deadbeef"], version = "0.0.1", + url = "", ), target_platforms = [ "linux_x86_64", @@ -124,6 +128,110 @@ _tests.append(_test_simple) +def _test_direct_urls(env): + got = parse_requirements( + ctx = _mock_ctx(), + requirements_by_platform = { + "requirements_direct": ["linux_x86_64"], + }, + ) + env.expect.that_dict(got).contains_exactly({ + "bar": [ + struct( + distribution = "bar", + extra_pip_args = [], + sdist = None, + is_exposed = True, + srcs = struct( + marker = "", + requirement = "bar @ https://example.org/bar-1.0.whl --hash=sha256:deadbeef", + requirement_line = "bar @ https://example.org/bar-1.0.whl --hash=sha256:deadbeef", + shas = ["deadbeef"], + version = "", + url = "https://example.org/bar-1.0.whl", + ), + target_platforms = ["linux_x86_64"], + whls = [struct( + url = "https://example.org/bar-1.0.whl", + filename = "bar-1.0.whl", + sha256 = "deadbeef", + yanked = False, + )], + ), + ], + "baz": [ + struct( + distribution = "baz", + extra_pip_args = [], + sdist = None, + is_exposed = True, + srcs = struct( + marker = "python_version < \"3.8\"", + requirement = "baz @ https://test.com/baz-2.0.whl --hash=sha256:deadb00f", + requirement_line = "baz @ https://test.com/baz-2.0.whl --hash=sha256:deadb00f", + shas = ["deadb00f"], + version = "", + url = "https://test.com/baz-2.0.whl", + ), + target_platforms = ["linux_x86_64"], + whls = [struct( + url = "https://test.com/baz-2.0.whl", + filename = "baz-2.0.whl", + sha256 = "deadb00f", + yanked = False, + )], + ), + ], + "foo": [ + struct( + distribution = "foo", + extra_pip_args = [], + sdist = None, + is_exposed = True, + srcs = struct( + marker = "", + requirement = "foo[extra] @ https://some-url/package.whl", + requirement_line = "foo[extra] @ https://some-url/package.whl", + shas = [], + version = "", + url = "https://some-url/package.whl", + ), + target_platforms = ["linux_x86_64"], + whls = [struct( + url = "https://some-url/package.whl", + filename = "package.whl", + sha256 = "", + yanked = False, + )], + ), + ], + "qux": [ + struct( + distribution = "qux", + extra_pip_args = [], + sdist = struct( + url = "https://example.org/qux-1.0.tar.gz", + filename = "qux-1.0.tar.gz", + sha256 = "deadbe0f", + yanked = False, + ), + is_exposed = True, + srcs = struct( + marker = "", + requirement = "qux @ https://example.org/qux-1.0.tar.gz --hash=sha256:deadbe0f", + requirement_line = "qux @ https://example.org/qux-1.0.tar.gz --hash=sha256:deadbe0f", + shas = ["deadbe0f"], + version = "", + url = "https://example.org/qux-1.0.tar.gz", + ), + target_platforms = ["linux_x86_64"], + whls = [], + ), + ], + }) + +_tests.append(_test_direct_urls) + def _test_extra_pip_args(env): got = parse_requirements( ctx = _mock_ctx(), @@ -145,6 +253,7 @@ requirement_line = "foo[extra]==0.0.1 --hash=sha256:deadbeef", shas = ["deadbeef"], version = "0.0.1", + url = "", ), target_platforms = [ "linux_x86_64", @@ -182,6 +291,7 @@ requirement_line = "foo[extra,extra_2]==0.0.1 --hash=sha256:deadbeef", shas = ["deadbeef"], version = "0.0.1", + url = "", ), target_platforms = ["linux_x86_64"], whls = [], @@ -211,6 +321,7 @@ requirement_line = "bar==0.0.1 --hash=sha256:deadb00f", shas = ["deadb00f"], version = "0.0.1", + url = "", ), target_platforms = ["windows_x86_64"], whls = [], @@ -228,6 +339,7 @@ requirement_line = "foo==0.0.3 --hash=sha256:deadbaaf", shas = ["deadbaaf"], version = "0.0.3", + url = "", ), target_platforms = ["linux_x86_64"], whls = [], @@ -243,6 +355,7 @@ requirement_line = "foo[extra]==0.0.2 --hash=sha256:deadbeef", shas = ["deadbeef"], version = "0.0.2", + url = "", ), target_platforms = ["windows_x86_64"], whls = [], @@ -282,6 +395,7 @@ requirement_line = "bar==0.0.1 --hash=sha256:deadb00f", shas = ["deadb00f"], version = "0.0.1", + url = "", ), target_platforms = ["cp39_linux_x86_64"], whls = [], @@ -299,6 +413,7 @@ requirement_line = "foo==0.0.1 --hash=sha256:deadbeef", shas = ["deadbeef"], version = "0.0.1", + url = "", ), target_platforms = ["cp39_linux_x86_64"], whls = [], @@ -314,6 +429,7 @@ requirement = "foo==0.0.3", shas = ["deadbaaf"], version = "0.0.3", + url = "", ), target_platforms = ["cp39_osx_aarch64"], whls = [], @@ -367,6 +483,7 @@ requirement_line = "bar==0.0.1 --hash=sha256:deadbeef", shas = ["deadbeef"], version = "0.0.1", + url = "", ), target_platforms = ["cp311_linux_super_exotic", "cp311_windows_x86_64"], whls = [], @@ -384,6 +501,7 @@ requirement_line = "foo[extra]==0.0.1 --hash=sha256:deadbeef", shas = ["deadbeef"], version = "0.0.1", + url = "", ), target_platforms = ["cp311_windows_x86_64"], whls = [], @@ -419,6 +537,7 @@ requirement_line = "foo==0.0.1 --hash=sha256:deadb00f", shas = ["deadb00f"], version = "0.0.1", + url = "", ), target_platforms = ["linux_x86_64"], whls = [], @@ -434,6 +553,7 @@ requirement_line = "foo==0.0.1+local --hash=sha256:deadbeef", shas = ["deadbeef"], version = "0.0.1+local", + url = "", ), target_platforms = ["linux_x86_64"], whls = [],