fix(pypi): expose pypi packages only common to all python versions (#3107)
Closes #2921
For a single pypi repo with multiple python versions, `all_requirements`
fail when a pypi package supports Python version A but not version B. In
this case, the pypi package would be included only in requirements lock
file for version A, not in one for version B. However, the failure
occurs since the package is included in `all_requirements` even for
Python version B.
(Minimal reproduction:
https://github.com/dotoleeoak/rules-python-2921-repro)
This happens since `packages` parameter for `hub_repository` targets are
including all packages across all requirement lock files. Instead of
union of packages, intersection of packages for requirement files should
be passed to `packages` and exposed to `all_requirements` macro, so that
those packages are compatible with all Python versions.
---------
Co-authored-by: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d54d3a8..74a4409 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -91,6 +91,8 @@
* Multi-line python imports (e.g. with escaped newlines) are now correctly processed by Gazelle.
* (toolchains) `local_runtime_repo` works with multiarch Debian with Python 3.8
([#3099](https://github.com/bazel-contrib/rules_python/issues/3099)).
+* (pypi) Expose pypi packages only common to all Python versions in `all_requirements`
+ ([#2921](https://github.com/bazel-contrib/rules_python/issues/2921)).
{#v0-0-0-added}
### Added
diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl
index 2c1528e..096256e 100644
--- a/python/private/pypi/extension.bzl
+++ b/python/private/pypi/extension.bzl
@@ -601,7 +601,15 @@
extra_aliases.setdefault(hub_name, {})
for whl_name, aliases in out.extra_aliases.items():
extra_aliases[hub_name].setdefault(whl_name, {}).update(aliases)
- exposed_packages.setdefault(hub_name, {}).update(out.exposed_packages)
+ if hub_name not in exposed_packages:
+ exposed_packages[hub_name] = out.exposed_packages
+ else:
+ intersection = {}
+ for pkg in out.exposed_packages:
+ if pkg not in exposed_packages[hub_name]:
+ continue
+ intersection[pkg] = None
+ exposed_packages[hub_name] = intersection
whl_libraries.update(out.whl_libraries)
# TODO @aignas 2024-04-05: how do we support different requirement
diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl
index 0303843..52e0e29 100644
--- a/tests/pypi/extension/extension_tests.bzl
+++ b/tests/pypi/extension/extension_tests.bzl
@@ -285,6 +285,104 @@
_tests.append(_test_simple_multiple_requirements)
+def _test_simple_multiple_python_versions(env):
+ pypi = _parse_modules(
+ env,
+ module_ctx = _mock_mctx(
+ _mod(
+ name = "rules_python",
+ parse = [
+ _parse(
+ hub_name = "pypi",
+ python_version = "3.15",
+ requirements_lock = "requirements_3_15.txt",
+ ),
+ _parse(
+ hub_name = "pypi",
+ python_version = "3.16",
+ requirements_lock = "requirements_3_16.txt",
+ ),
+ ],
+ ),
+ read = lambda x: {
+ "requirements_3_15.txt": """
+simple==0.0.1 --hash=sha256:deadbeef
+old-package==0.0.1 --hash=sha256:deadbaaf
+""",
+ "requirements_3_16.txt": """
+simple==0.0.2 --hash=sha256:deadb00f
+new-package==0.0.1 --hash=sha256:deadb00f2
+""",
+ }[x],
+ ),
+ available_interpreters = {
+ "python_3_15_host": "unit_test_interpreter_target",
+ "python_3_16_host": "unit_test_interpreter_target",
+ },
+ minor_mapping = {
+ "3.15": "3.15.19",
+ "3.16": "3.16.9",
+ },
+ )
+
+ pypi.exposed_packages().contains_exactly({"pypi": ["simple"]})
+ pypi.hub_group_map().contains_exactly({"pypi": {}})
+ pypi.hub_whl_map().contains_exactly({
+ "pypi": {
+ "new_package": {
+ "pypi_316_new_package": [
+ whl_config_setting(
+ version = "3.16",
+ ),
+ ],
+ },
+ "old_package": {
+ "pypi_315_old_package": [
+ whl_config_setting(
+ version = "3.15",
+ ),
+ ],
+ },
+ "simple": {
+ "pypi_315_simple": [
+ whl_config_setting(
+ version = "3.15",
+ ),
+ ],
+ "pypi_316_simple": [
+ whl_config_setting(
+ version = "3.16",
+ ),
+ ],
+ },
+ },
+ })
+ pypi.whl_libraries().contains_exactly({
+ "pypi_315_old_package": {
+ "dep_template": "@pypi//{name}:{target}",
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "requirement": "old-package==0.0.1 --hash=sha256:deadbaaf",
+ },
+ "pypi_315_simple": {
+ "dep_template": "@pypi//{name}:{target}",
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "requirement": "simple==0.0.1 --hash=sha256:deadbeef",
+ },
+ "pypi_316_new_package": {
+ "dep_template": "@pypi//{name}:{target}",
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "requirement": "new-package==0.0.1 --hash=sha256:deadb00f2",
+ },
+ "pypi_316_simple": {
+ "dep_template": "@pypi//{name}:{target}",
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "requirement": "simple==0.0.2 --hash=sha256:deadb00f",
+ },
+ })
+ pypi.whl_mods().contains_exactly({})
+
+_tests.append(_test_simple_multiple_python_versions)
+
def _test_simple_with_markers(env):
pypi = _parse_modules(
env,
