fix(pypi): normalize extras in requirement strings per PEP 685 (#3588) ## Summary Extras parsed from requirement strings (e.g., from `requirements.txt`) were not being normalized, causing mismatches when evaluating PEP 508 marker expressions. For example, `sqlalchemy[postgresql-psycopg2binary]` would fail to resolve `psycopg2-binary` as a transitive dependency because the wheel METADATA marker expression `extra == "postgresql_psycopg2binary"` uses the underscore-normalized form (per PEP 685), while the extras set retained the original hyphenated form from the requirement string. ## Before ``` # requirements.txt sqlalchemy[postgresql-psycopg2binary]==2.0.36 # Parsed extras: ["postgresql-psycopg2binary"] # Marker evaluation: "postgresql-psycopg2binary" != "postgresql_psycopg2binary" -> MISS # Result: psycopg2-binary NOT included as a dependency ``` ## After ``` # requirements.txt sqlalchemy[postgresql-psycopg2binary]==2.0.36 # Parsed extras: ["postgresql_psycopg2binary"] (normalized) # Marker evaluation: "postgresql_psycopg2binary" == "postgresql_psycopg2binary" -> MATCH # Result: psycopg2-binary correctly included as a dependency ``` ## Changes - **`python/private/pypi/pep508_requirement.bzl`**: Apply `normalize_name()` to each extra during requirement parsing, consistent with how the package name is already normalized. - **`tests/pypi/pep508/requirement_tests.bzl`**: Updated existing test expectation for case normalization and added test case for hyphenated extras (`sqlalchemy[asyncio,postgresql-psycopg2binary,postgresql-asyncpg]`). - **`tests/pypi/pep508/deps_tests.bzl`**: Added `test_extras_with_hyphens_are_normalized` integration test confirming that dependencies gated behind hyphenated extras are correctly resolved. - **`CHANGELOG.md`**: Added entry under Unreleased > Fixed. Fixes #3587 --------- Co-authored-by: Ignas Anikevicius <240938+aignas@users.noreply.github.com> (cherry picked from commit 9fe42b1f0badfc258b159dbc0a05a8392c0234e5)
diff --git a/CHANGELOG.md b/CHANGELOG.md index 81ca109..a8c298e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md
@@ -60,6 +60,10 @@ * (pypi) `pip_parse` no longer silently drops PEP 508 URL-based requirements (`pkg @ https://...`) when `extract_url_srcs=False` (the default for `pip_repository`). +* (pypi) Extras in requirement strings are now normalized per PEP 685, + fixing missing transitive dependencies when extras contain hyphens + (e.g., `sqlalchemy[postgresql-psycopg2binary]`). + ([#3587](https://github.com/bazel-contrib/rules_python/issues/3587)) {#v1-8-4} ## [1.8.4] - 2026-02-10
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 8194bb5..e51148a 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel
@@ -255,6 +255,7 @@ name = "pep508_deps_bzl", srcs = ["pep508_deps.bzl"], deps = [ + ":pep508_env_bzl", ":pep508_evaluate_bzl", ":pep508_requirement_bzl", "//python/private:normalize_name_bzl", @@ -265,6 +266,7 @@ name = "pep508_env_bzl", srcs = ["pep508_env.bzl"], deps = [ + "//python/private:normalize_name_bzl", "//python/private:version_bzl", ], )
diff --git a/python/private/pypi/pep508_deps.bzl b/python/private/pypi/pep508_deps.bzl index ad6589c..c004334 100644 --- a/python/private/pypi/pep508_deps.bzl +++ b/python/private/pypi/pep508_deps.bzl
@@ -16,6 +16,7 @@ """ load("//python/private:normalize_name.bzl", "normalize_name") +load(":pep508_env.bzl", "create_env") load(":pep508_evaluate.bzl", "evaluate") load(":pep508_requirement.bzl", "requirement") @@ -155,8 +156,9 @@ return sorted(extras) def _evaluate_any(req, extras): + env = create_env() for extra in extras: - if evaluate(req.marker, env = {"extra": extra}): + if evaluate(req.marker, env = env | {"extra": extra}): return True return False @@ -167,11 +169,12 @@ _add(deps, deps_select, dep) return + env = create_env() markers = {} found_unconditional = False for req in reqs: for x in extras: - m = evaluate(req.marker, env = {"extra": x}, strict = False) + m = evaluate(req.marker, env = env | {"extra": x}, strict = False) if m == False: continue elif m == True:
diff --git a/python/private/pypi/pep508_env.bzl b/python/private/pypi/pep508_env.bzl index 5031eba..9fe9dca 100644 --- a/python/private/pypi/pep508_env.bzl +++ b/python/private/pypi/pep508_env.bzl
@@ -15,6 +15,7 @@ """This module is for implementing PEP508 environment definition. """ +load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:version.bzl", "version") _DEFAULT = "//conditions:default" @@ -215,9 +216,11 @@ def create_env(): return { - # This is split by topic + # Per-variable normalization functions. Each entry maps a marker + # variable name to a function (value) -> normalized_value. "_aliases": { - "platform_machine": platform_machine_aliases, + "extra": normalize_name, + "platform_machine": lambda x: platform_machine_aliases.get(x, x), }, }
diff --git a/python/private/pypi/pep508_evaluate.bzl b/python/private/pypi/pep508_evaluate.bzl index fe2cac9..61e461a 100644 --- a/python/private/pypi/pep508_evaluate.bzl +++ b/python/private/pypi/pep508_evaluate.bzl
@@ -300,12 +300,10 @@ left = left.strip("\"") if _ENV_ALIASES in env: - # On Windows, Linux, OSX different values may mean the same hardware, - # e.g. Python on Windows returns arm64, but on Linux returns aarch64. - # e.g. Python on Windows returns amd64, but on Linux returns x86_64. - # - # The following normalizes the values - left = env.get(_ENV_ALIASES, {}).get(var_name, {}).get(left, left) + # Normalize the literal value using per-variable normalization + # functions. This handles platform aliases (e.g. arm64 -> aarch64) + # and PEP 685 extra name normalization (e.g. db-backend -> db_backend). + left = env.get(_ENV_ALIASES, {}).get(var_name, lambda x: x)(left) else: var_name = left @@ -314,7 +312,7 @@ if _ENV_ALIASES in env: # See the note above on normalization - right = env.get(_ENV_ALIASES, {}).get(var_name, {}).get(right, right) + right = env.get(_ENV_ALIASES, {}).get(var_name, lambda x: x)(right) if var_name in _NON_VERSION_VAR_NAMES: return _env_expr(left, op, right)
diff --git a/python/private/pypi/pep508_requirement.bzl b/python/private/pypi/pep508_requirement.bzl index b5be17f..7552642 100644 --- a/python/private/pypi/pep508_requirement.bzl +++ b/python/private/pypi/pep508_requirement.bzl
@@ -45,7 +45,7 @@ extras_unparsed, _, _ = extras_unparsed.partition("]") for char in _STRIP: requires, _, _ = requires.partition(char) - extras = extras_unparsed.replace(" ", "").split(",") + extras = [normalize_name(e) for e in extras_unparsed.replace(" ", "").split(",") if e] name = requires.strip(" ") name = normalize_name(name)
diff --git a/tests/pypi/pep508/deps_tests.bzl b/tests/pypi/pep508/deps_tests.bzl index 1404ad6..e88acb8 100644 --- a/tests/pypi/pep508/deps_tests.bzl +++ b/tests/pypi/pep508/deps_tests.bzl
@@ -218,6 +218,38 @@ _tests.append(test_span_all_python_versions) +def test_extras_with_hyphens_are_normalized(env): + """Test that extras with hyphens in marker expressions are normalized. + + When wheel METADATA uses hyphens in marker expressions + (e.g., extra == "db-backend") but the extras from requirement parsing + are already normalized (e.g., "db_backend"), the deps should still + resolve because marker evaluation normalizes per PEP 685. + + Args: + env: the test environment. + """ + requires_dist = [ + "bar", + 'baz-lib; extra == "db-backend"', + 'qux-async; extra == "async-driver"', + ] + + got = deps( + "foo", + extras = ["db_backend", "async_driver"], + requires_dist = requires_dist, + ) + + env.expect.that_collection(got.deps).contains_exactly([ + "bar", + "baz_lib", + "qux_async", + ]) + env.expect.that_dict(got.deps_select).contains_exactly({}) + +_tests.append(test_extras_with_hyphens_are_normalized) + def deps_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name,
diff --git a/tests/pypi/pep508/requirement_tests.bzl b/tests/pypi/pep508/requirement_tests.bzl index 9afb43a..2ce4592 100644 --- a/tests/pypi/pep508/requirement_tests.bzl +++ b/tests/pypi/pep508/requirement_tests.bzl
@@ -23,9 +23,10 @@ " name1[ foo ] ": ("name1", ["foo"], None, ""), "Name[foo]": ("name", ["foo"], None, ""), "name [fred,bar] @ http://foo.com ; python_version=='2.7'": ("name", ["fred", "bar"], None, "python_version=='2.7'"), - "name; (os_name=='a' or os_name=='b') and os_name=='c'": ("name", [""], None, "(os_name=='a' or os_name=='b') and os_name=='c'"), - "name@http://foo.com": ("name", [""], None, ""), - "name[ Foo123 ]": ("name", ["Foo123"], None, ""), + "name; (os_name=='a' or os_name=='b') and os_name=='c'": ("name", [], None, "(os_name=='a' or os_name=='b') and os_name=='c'"), + "name@http://foo.com": ("name", [], None, ""), + "name[ Foo123 ]": ("name", ["foo123"], None, ""), + "name[extra-one,extra-two.three]==1.0": ("name", ["extra_one", "extra_two_three"], "1.0", ""), "name[extra]@http://foo.com": ("name", ["extra"], None, ""), "name[foo]": ("name", ["foo"], None, ""), "name[quux, strange];python_version<'2.7' and platform_version=='2'": ("name", ["quux", "strange"], None, "python_version<'2.7' and platform_version=='2'"),