fix(pypi): correctly override default indexes as documented in our docs (#4124) This fixes how defaults for `pip.default(index_url)` is interpreted if the `--index-url` is used in a `requirements.txt` file arguments. From now on the default set in `pip.default` takes precedence if it is set to something else than `https://pypi.org/simple`. Now behaviour is as such: * default - `https://pypi.org/simple` is used or if there is a `requirements.txt` value, it will take precedence. * user specifies `https://pypi-internal.mirror` - `https://pypi-internal.mirror` is used at all times. * user specifies `https://pypi.org/simple` - same as default Closes #4123
diff --git a/python/private/pypi/argparse.bzl b/python/private/pypi/argparse.bzl index 8964411..0a852af 100644 --- a/python/private/pypi/argparse.bzl +++ b/python/private/pypi/argparse.bzl
@@ -27,9 +27,9 @@ if repeated: if val not in value: - value.append(val) + value.append(val.strip("\"\'")) else: - value = val + value = val.strip("\"\'") return value
diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index d1b583d..cbf4877 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl
@@ -393,7 +393,7 @@ self._get_index_urls[python_version] = lambda ctx, distributions, *, index_url = None, extra_index_urls = None: self._simpleapi_download_fn( ctx, attr = struct( - index_url = (index_url or default_index_url).rstrip("/"), + index_url = _select_default_index(index_url, default_index_url).rstrip("/"), extra_index_urls = [ x.rstrip("/") for x in (extra_index_urls or default_extra_index_urls) @@ -414,6 +414,14 @@ ) return True +def _select_default_index(index, default): + # We will use the value from the requirements file by default + index = (index or default).rstrip("/") + default = default.rstrip("/") + + # If the default is set to something else than the public mirror, use that + return index if default == "https://pypi.org/simple" else default + def _host_interpreter_name(python_version): return "python_{}_host".format(python_version.replace(".", "_"))
diff --git a/tests/pypi/argparse/argparse_tests.bzl b/tests/pypi/argparse/argparse_tests.bzl index f8fd1c9..4899f35 100644 --- a/tests/pypi/argparse/argparse_tests.bzl +++ b/tests/pypi/argparse/argparse_tests.bzl
@@ -12,6 +12,8 @@ env.expect.that_str(argparse.index_url(["-i", "https://example.com/simple"], "default")).equals("https://example.com/simple") env.expect.that_str(argparse.index_url(["--index-url", "https://example.com/simple"], "default")).equals("https://example.com/simple") env.expect.that_str(argparse.index_url(["--index-url=https://example.com/simple"], "default")).equals("https://example.com/simple") + env.expect.that_str(argparse.index_url(["--index-url=\"https://example.com/simple\""], "default")).equals("https://example.com/simple") + env.expect.that_str(argparse.index_url(["--index-url='https://example.com/simple'"], "default")).equals("https://example.com/simple") env.expect.that_str(argparse.index_url(["--extra-index-url", "https://extra.com", "-i", "https://index.com"], "default")).equals("https://index.com") @@ -23,6 +25,8 @@ env.expect.that_collection(argparse.extra_index_url(["--extra-index-url", "https://extra.com/simple"], [])).contains_exactly(["https://extra.com/simple"]) env.expect.that_collection(argparse.extra_index_url(["--extra-index-url=https://extra.com/simple"], [])).contains_exactly(["https://extra.com/simple"]) + env.expect.that_collection(argparse.extra_index_url(["--extra-index-url=\"https://extra.com/simple\""], [])).contains_exactly(["https://extra.com/simple"]) + env.expect.that_collection(argparse.extra_index_url(["--extra-index-url='https://extra.com/simple'"], [])).contains_exactly(["https://extra.com/simple"]) env.expect.that_collection(argparse.extra_index_url(["--extra-index-url", "https://first.com", "--extra-index-url", "https://second.com"], [])).contains_exactly(["https://first.com", "https://second.com"])
diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 6d7e971..b3a2ebf 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl
@@ -643,6 +643,124 @@ _tests.append(_test_extension_dep_coexists_with_concrete_hub) +def _test_default_index_setting(env): + def mock_simpleapi_download(*_, attr, **_kwargs): + digest_in_lock_file = "sha256:deadbeef" + return { + "simple": struct( + whls = { + digest_in_lock_file: struct( + yanked = None, + filename = "simple-0.0.1-py3-none-any.whl", + digest = digest_in_lock_file, + url = "file://simple-0.0.1-py3-none-any.whl", + ), + }, + sdists = {}, + hashes_by_version = {}, + index_url = attr.index_url, + ), + } + + for test in [ + struct( + modules = [ + _mod( + name = "my_module", + parse = [ + _parse( + hub_name = "pypi_a", + python_version = "3.15", + requirements_lock = "requirements.txt", + ), + ], + ), + ], + want_index_url = "https://pypi.org/simple", + ), + struct( + modules = [ + _mod( + name = "my_module", + default = [ + _default( + index_url = "https://pypi.internal.org/simple", + ), + ], + parse = [ + _parse( + hub_name = "pypi_a", + python_version = "3.15", + requirements_lock = "requirements.txt", + ), + ], + ), + ], + want_index_url = "https://pypi.internal.org/simple", + ), + struct( + modules = [ + _mod( + name = "my_module", + default = [ + _default( + index_url = "https://pypi.internal.org/simple", + ), + ], + parse = [ + _parse( + hub_name = "pypi_a", + python_version = "3.15", + requirements_lock = "requirements_with_index_url.txt", + ), + ], + ), + ], + # We want the file to have lower precedence than our default if the + # `my_module` is setting the default. + want_index_url = "https://pypi.internal.org/simple", + ), + ]: + pypi = _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + os_name = "linux", + arch_name = "x86_64", + mock_files = { + "requirements.txt": """\ +simple==0.0.1 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf""", + "requirements_with_index_url.txt": """\ +--index-url="https://pypi.org/simple" + +simple==0.0.1 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf""", + }, + *test.modules + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.15": "3.15.19"}, + simpleapi_download = mock_simpleapi_download, + ) + + pypi.whl_libraries().contains_exactly({ + "pypi_a_315_simple_py3_none_any_deadbeef": { + "config_load": "@pypi_a//:config.bzl", + "dep_template": "@pypi_a//{name}:{target}", + "filename": "simple-0.0.1-py3-none-any.whl", + "index_url": test.want_index_url, + "integrity": "sha256-3q2+7w==", + "requirement": "simple==0.0.1", + "urls": ["file://simple-0.0.1-py3-none-any.whl"], + }, + }) + +_tests.append(_test_default_index_setting) + def extension_test_suite(name): """Create the test suite.
diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 1d4a602..48f6c44 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl
@@ -865,9 +865,9 @@ """, experimental_index_url = "https://experimental.example.com/simple", experimental_extra_index_urls = [], - expect_index_url = "https://file.example.com/simple", + expect_index_url = "https://experimental.example.com/simple", expect_extra_index_urls = [], - expect_url = "file.example.com/simple/", + expect_url = "experimental.example.com/simple/", ), struct( requirements_txt = "simple==0.0.1 --hash=sha256:deadb00f",