fix(pypi): build uv.lock git sources with pip (#4209) `pip.parse(uv_lock = ...)` passes a git-sourced package's raw uv source string (`https://host/repo?rev=<ref>#<commit>`) to the Bazel downloader as a URL, which cannot fetch a git repository (#4139). #4086 fixed the invalid repo name for these sources but kept the URL, so they still fail at fetch time. This makes uv.lock git sources match how a `foo @ git+...` line from a requirements file is already handled: - The src gets an empty `url` and `filename`, so `_whl_repo` takes the existing pip path instead of the downloader. - The requirement line becomes a pip direct reference, `foo @ git+<repo url>@<commit>`, pinned to the commit uv resolved rather than the requested `rev`/`tag`/`branch`. Before, it was `foo==<version>`, which pip cannot satisfy for a VCS-only version such as `0.12.0a0+39be1c6`. - A `subdirectory` query parameter carries over as `#subdirectory=<dir>`. uv percent-encodes it (`subdirectory=python%2Ffoo`) and pip reads the fragment as a literal path, so it is decoded. I checked both against uv 0.9.30 and pip 26.2.1: pip fails on the encoded form with "does not appear to be a Python project" and builds the decoded one. Tests: updated `test_uv_lock_vcs_entry` for the new src shape and added `test_uv_lock_vcs_entry_subdirectory`. `//tests/pypi/parse_requirements/...` and `//tests/pypi/hub_builder/...` pass. End to end, a large monorepo using `pip.parse(uv_lock = ...)` builds its git-sourced `torchdata` dependency against this branch via `--override_module`; we have been carrying the same change as a patch on 2.3.1. Fixes #4139
diff --git a/news/4139.fixed.md b/news/4139.fixed.md new file mode 100644 index 0000000..589812e --- /dev/null +++ b/news/4139.fixed.md
@@ -0,0 +1,5 @@ +(pypi) `pip.parse(uv_lock = ...)` now builds packages with a git source with +`pip`, from a `name @ git+<url>@<commit>` direct reference pinned to the commit +`uv` resolved, instead of passing the uv.lock source string to the Bazel +downloader as a URL. +([#4139](https://github.com/bazel-contrib/rules_python/issues/4139))
diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index 580989e..2ac8b37 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl
@@ -226,18 +226,16 @@ git_struct = None if pkg.get("source", {}).get("git"): - url = pkg["source"]["git"] - - # Keep the revision in the URL, but exclude it from the repository filename. - url_path, _, _ = url.partition("?") - url_path, _, _ = url_path.partition("#") - _, _, filename = url_path.rpartition("/") + # A git source is not a downloadable artifact. Leave `url` and + # `filename` empty, the same as a `foo @ git+...` line from a + # requirements file, so that `pip` builds it from the requirement + # line instead of the Bazel downloader trying to fetch the URL. git_struct = struct( - filename = filename, - url = url, + filename = "", + url = "", digest = "", kind = "git", - source = pkg["source"], + requirement = _uv_lock_git_requirement(pkg["source"]["git"]), ) plat_to_src = {} @@ -275,11 +273,18 @@ for key, val in src_to_plats.items(): src = val.src plats = sorted(val.plats) - requirement_line = "{name}{extras}=={version}".format( - name = name, - extras = extra_str, - version = version, - ) + if src.kind == "git": + requirement_line = "{name}{extras} @ {requirement}".format( + name = name, + extras = extra_str, + requirement = src.requirement, + ) + else: + requirement_line = "{name}{extras}=={version}".format( + name = name, + extras = extra_str, + version = version, + ) entry["resolved_srcs"].append(struct( distribution = name, extra_pip_args = extra_pip_args or [], @@ -313,6 +318,55 @@ logger.debug(lambda: "Parsed {} packages from uv.lock".format(len(ret))) return ret +def _uv_lock_git_requirement(git_source): + """Turn a uv.lock git source into a pip direct reference. + + uv records `<repo url>?<query>#<resolved commit>`, where the query carries + `rev`, `tag` or `branch` and optionally `subdirectory`. pip expects + `git+<repo url>@<commit>[#subdirectory=<dir>]`. Pinning to the resolved + commit rather than the requested ref keeps the build reproducible. + + Args: + git_source: {type}`str` the `source.git` value from uv.lock. + + Returns: + {type}`str` the pip direct reference, without the `<name> @ ` prefix. + """ + head, _, commit = git_source.partition("#") + repo_url, _, query = head.partition("?") + subdirectory = "" + for param in query.split("&"): + key, _, value = param.partition("=") + if key == "subdirectory": + # uv percent-encodes the value (`python%2Ffoo`), but pip reads the + # `#subdirectory=` fragment as a literal path. + subdirectory = _percent_decode(value) + requirement = "git+{}@{}".format(repo_url, commit) + if subdirectory: + requirement += "#subdirectory={}".format(subdirectory) + return requirement + +# Printable ASCII, indexed by `code point - 0x20`, as Starlark has no `chr()`. +_PRINTABLE_ASCII = " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" + +def _percent_decode(value): + """Decode `%XX` escapes of printable ASCII; leave any other escape as-is.""" + parts = value.split("%") + out = [parts[0]] + for part in parts[1:]: + code = int(part[:2], 16) if len(part) >= 2 and _is_hex(part[:2]) else -1 + if code >= 0x20 and code <= 0x7e: + out.append(_PRINTABLE_ASCII[code - 0x20] + part[2:]) + else: + out.append("%" + part) + return "".join(out) + +def _is_hex(s): + for c in s.elems(): + if c not in "0123456789abcdefABCDEF": + return False + return True + def _parse_uv_lock_hash(hash_str): """Parse a uv.lock `hash` value of the form `<algo>:<digest>`.
diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index f6b9d7e..733abe4 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl
@@ -115,6 +115,7 @@ "uv_lock_foo_virtual": """{"package":[{"name":"foo","source":{"registry":"https://pypi.org/simple"},"version":"0.0.1","wheels":[{"hash":"sha256:deadbeef","url":"https://files.pythonhosted.org/packages/foo-0.0.1-py3-none-any.whl"}]},{"name":"virtual-pkg","source":{"virtual":true},"version":"0.0.0"}]}""", "uv_lock_foo_with_extras": """{"package":[{"name":"foo","provides-extras":["extra"],"source":{"registry":"https://pypi.org/simple"},"version":"0.0.1","wheels":[{"hash":"sha256:deadbeef","url":"https://files.pythonhosted.org/packages/foo-0.0.1-py3-none-any.whl"}]}]}""", "uv_lock_git_vcs": """{"package":[{"name":"foo","source":{"git":"https://github.com/org/foo?rev=deadbeef#deadbeef"},"version":"0.1.0"}]}""", + "uv_lock_git_vcs_subdirectory": """{"package":[{"name":"foo","source":{"git":"https://github.com/org/mono.git?subdirectory=python%2Ffoo&branch=main#0123abcd"},"version":"0.1.0+g0123abcd"}]}""", "uv_lock_rules_python_pkg": """{"package":[{"name":"rules_python","source":{"registry":"https://pypi.org/simple"},"version":"0.0.1","wheels":[{"hash":"sha256:deadbeef","url":"https://files.pythonhosted.org/packages/rules_python-0.0.1-py3-none-any.whl"}]}]}""", } @@ -1321,7 +1322,7 @@ _tests.append(_test_uv_lock_cross_consistent) def _test_uv_lock_vcs_entry(env): - """Test that VCS entry filenames exclude URL query and fragment components.""" + """Test that a uv.lock git source is built by pip from a direct reference.""" got = parse_requirements( uv_lock = "uv_lock_git_vcs", ) @@ -1335,11 +1336,11 @@ struct( distribution = "foo", extra_pip_args = [], - requirement_line = "foo==0.1.0", + requirement_line = "foo @ git+https://github.com/org/foo@deadbeef", target_platforms = ["linux_x86_64"], - filename = "foo", + filename = "", digest = "", - url = "https://github.com/org/foo?rev=deadbeef#deadbeef", + url = "", yanked = None, ), ], @@ -1348,6 +1349,34 @@ _tests.append(_test_uv_lock_vcs_entry) +def _test_uv_lock_vcs_entry_subdirectory(env): + """Test that a uv.lock git source pins the resolved commit and keeps its subdirectory.""" + got = parse_requirements( + uv_lock = "uv_lock_git_vcs_subdirectory", + ) + env.expect.that_collection(got).contains_exactly([ + struct( + name = "foo", + index_url = "", + is_exposed = True, + is_multiple_versions = False, + srcs = [ + struct( + distribution = "foo", + extra_pip_args = [], + requirement_line = "foo @ git+https://github.com/org/mono.git@0123abcd#subdirectory=python/foo", + target_platforms = ["linux_x86_64"], + filename = "", + digest = "", + url = "", + yanked = None, + ), + ], + ), + ]) + +_tests.append(_test_uv_lock_vcs_entry_subdirectory) + def _test_uv_lock_rules_python_pkg_not_skipped(env): """Test that 'rules_python' package is not skipped from uv.lock.""" got = parse_requirements(