feat(pypi): support freethreaded in experimental_index_url (#2460)
With this we:
* Fix the previous behaviour where `abi3` wheels would be selected when
freethreaded builds are selected. Whilst this may work in practise
sometimes, I am not sure it has been supported by reading PEP703.
* Start selecting `cp313t` wheels when we scan what is available on
PyPI.
* Ensure that the `whl_library` repository rule handles `cp313t` wheel
extraction.
* Generate `cp313t` config_settings so that we can use them in
`pkg_aliases`.
* Generate `cp313t` references in `pkg_aliases` macro.
* Add the 3.13 deps to dev_pip for testing.
Also tested by manually running:
```
$ bazel cquery --//python/config_settings:python_version=3.13 --//python/config_settings:py_freethreaded=yes 'kind("py_library rule", deps(@dev_pip//markupsafe))'
INFO: Analyzed target @@_main~pip~dev_pip//markupsafe:markupsafe (3 packages loaded, 4091 targets configured).
INFO: Found 1 target...
@@_main~pip~dev_pip_313_markupsafe_cp313_cp313t_manylinux_2_17_x86_64_c0ef13ea//:pkg (008c5a5)
$bazel build --//python/config_settings:python_version=3.13 --//python/config_settings:py_freethreaded=yes @dev_pip//markupsafe
```
Fixes #2386diff --git a/MODULE.bazel b/MODULE.bazel
index 2ae3173..e4b113e 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -124,6 +124,13 @@
dev_pip.parse(
download_only = True,
experimental_index_url = "https://pypi.org/simple",
+ hub_name = "dev_pip",
+ python_version = "3.13.0",
+ requirements_lock = "//docs:requirements.txt",
+)
+dev_pip.parse(
+ download_only = True,
+ experimental_index_url = "https://pypi.org/simple",
hub_name = "pypiserver",
python_version = "3.11",
requirements_lock = "//examples/wheel:requirements_server.txt",
diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel
index aa26e6e..5455f5a 100644
--- a/python/config_settings/BUILD.bazel
+++ b/python/config_settings/BUILD.bazel
@@ -106,6 +106,12 @@
visibility = ["//visibility:public"],
)
+config_setting(
+ name = "is_py_non_freethreaded",
+ flag_values = {":py_freethreaded": FreeThreadedFlag.NO},
+ visibility = ["//visibility:public"],
+)
+
# pip.parse related flags
string_flag(
diff --git a/python/private/pypi/config_settings.bzl b/python/private/pypi/config_settings.bzl
index 6f927f2..620e50e 100644
--- a/python/private/pypi/config_settings.bzl
+++ b/python/private/pypi/config_settings.bzl
@@ -20,20 +20,21 @@
most specialized wheels are used by default with the users being able to
configure string_flag values to select the less specialized ones.
-The list of specialization of the dists goes like follows:
+The list of specialization of the dists goes like follows (cpxyt stands for freethreaded
+environments):
* sdist
* py*-none-any.whl
* py*-abi3-any.whl
-* py*-cpxy-any.whl
+* py*-cpxy-any.whl or py*-cpxyt-any.whl
* cp*-none-any.whl
* cp*-abi3-any.whl
-* cp*-cpxy-plat.whl
+* cp*-cpxy-any.whl or cp*-cpxyt-any.whl
* py*-none-plat.whl
* py*-abi3-plat.whl
-* py*-cpxy-plat.whl
+* py*-cpxy-plat.whl or py*-cpxyt-plat.whl
* cp*-none-plat.whl
* cp*-abi3-plat.whl
-* cp*-cpxy-plat.whl
+* cp*-cpxy-plat.whl or cp*-cpxyt-plat.whl
Note, that here the specialization of musl vs manylinux wheels is the same in
order to ensure that the matching fails if the user requests for `musl` and we don't have it or vice versa.
@@ -46,19 +47,24 @@
**{
f: str(Label("//python/config_settings:" + f))
for f in [
- "python_version",
+ "is_pip_whl_auto",
+ "is_pip_whl_no",
+ "is_pip_whl_only",
+ "is_py_freethreaded",
+ "is_py_non_freethreaded",
"pip_whl_glibc_version",
"pip_whl_muslc_version",
"pip_whl_osx_arch",
"pip_whl_osx_version",
"py_linux_libc",
- "is_pip_whl_no",
- "is_pip_whl_only",
- "is_pip_whl_auto",
+ "python_version",
]
}
)
+_DEFAULT = "//conditions:default"
+_INCOMPATIBLE = "@platforms//:incompatible"
+
# Here we create extra string flags that are just to work with the select
# selecting the most specialized match. We don't allow the user to change
# them.
@@ -170,52 +176,70 @@
**kwargs
)
- for name, f in [
- ("py_none", _flags.whl_py2_py3),
- ("py3_none", _flags.whl_py3),
- ("py3_abi3", _flags.whl_py3_abi3),
- ("cp3x_none", _flags.whl_pycp3x),
- ("cp3x_abi3", _flags.whl_pycp3x_abi3),
- ("cp3x_cp", _flags.whl_pycp3x_abicp),
+ used_flags = {}
+
+ # NOTE @aignas 2024-12-01: the abi3 is not compatible with freethreaded
+ # builds as per PEP703 (https://peps.python.org/pep-0703/#backwards-compatibility)
+ #
+ # The discussion here also reinforces this notion:
+ # https://discuss.python.org/t/pep-703-making-the-global-interpreter-lock-optional-3-12-updates/26503/99
+
+ for name, f, abi in [
+ ("py_none", _flags.whl_py2_py3, None),
+ ("py3_none", _flags.whl_py3, None),
+ ("py3_abi3", _flags.whl_py3_abi3, (FLAGS.is_py_non_freethreaded,)),
+ ("cp3x_none", _flags.whl_pycp3x, None),
+ ("cp3x_abi3", _flags.whl_pycp3x_abi3, (FLAGS.is_py_non_freethreaded,)),
+ # The below are not specializations of one another, they are variants
+ ("cp3x_cp", _flags.whl_pycp3x_abicp, (FLAGS.is_py_non_freethreaded,)),
+ ("cp3x_cpt", _flags.whl_pycp3x_abicp, (FLAGS.is_py_freethreaded,)),
]:
- if f in flag_values:
+ if (f, abi) in used_flags:
# This should never happen as all of the different whls should have
- # unique flag values.
+ # unique flag values
fail("BUG: the flag {} is attempted to be added twice to the list".format(f))
else:
flag_values[f] = ""
+ used_flags[(f, abi)] = True
_dist_config_setting(
name = "{}_any{}".format(name, suffix),
flag_values = flag_values,
is_pip_whl = FLAGS.is_pip_whl_only,
+ abi = abi,
**kwargs
)
generic_flag_values = flag_values
+ generic_used_flags = used_flags
for (suffix, flag_values) in plat_flag_values:
+ used_flags = {(f, None): True for f in flag_values} | generic_used_flags
flag_values = flag_values | generic_flag_values
- for name, f in [
- ("py_none", _flags.whl_plat),
- ("py3_none", _flags.whl_plat_py3),
- ("py3_abi3", _flags.whl_plat_py3_abi3),
- ("cp3x_none", _flags.whl_plat_pycp3x),
- ("cp3x_abi3", _flags.whl_plat_pycp3x_abi3),
- ("cp3x_cp", _flags.whl_plat_pycp3x_abicp),
+ for name, f, abi in [
+ ("py_none", _flags.whl_plat, None),
+ ("py3_none", _flags.whl_plat_py3, None),
+ ("py3_abi3", _flags.whl_plat_py3_abi3, (FLAGS.is_py_non_freethreaded,)),
+ ("cp3x_none", _flags.whl_plat_pycp3x, None),
+ ("cp3x_abi3", _flags.whl_plat_pycp3x_abi3, (FLAGS.is_py_non_freethreaded,)),
+ # The below are not specializations of one another, they are variants
+ ("cp3x_cp", _flags.whl_plat_pycp3x_abicp, (FLAGS.is_py_non_freethreaded,)),
+ ("cp3x_cpt", _flags.whl_plat_pycp3x_abicp, (FLAGS.is_py_freethreaded,)),
]:
- if f in flag_values:
+ if (f, abi) in used_flags:
# This should never happen as all of the different whls should have
# unique flag values.
fail("BUG: the flag {} is attempted to be added twice to the list".format(f))
else:
flag_values[f] = ""
+ used_flags[(f, abi)] = True
_dist_config_setting(
name = "{}_{}".format(name, suffix),
flag_values = flag_values,
is_pip_whl = FLAGS.is_pip_whl_only,
+ abi = abi,
**kwargs
)
@@ -285,7 +309,7 @@
return ret
-def _dist_config_setting(*, name, is_python, python_version, is_pip_whl = None, native = native, **kwargs):
+def _dist_config_setting(*, name, is_python, python_version, is_pip_whl = None, abi = None, native = native, **kwargs):
"""A macro to create a target that matches is_pip_whl_auto and one more value.
Args:
@@ -294,6 +318,10 @@
`is_pip_whl_auto` when evaluating the config setting.
is_python: The python version config_setting to match.
python_version: The python version name.
+ abi: {type}`tuple[Label]` A collection of ABI config settings that are
+ compatible with the given dist config setting. For example, if only
+ non-freethreaded python builds are allowed, add
+ FLAGS.is_py_non_freethreaded here.
native (struct): The struct containing alias and config_setting rules
to use for creating the objects. Can be overridden for unit tests
reasons.
@@ -306,9 +334,9 @@
native.alias(
name = "is_cp{}_{}".format(python_version, name) if python_version else "is_{}".format(name),
actual = select({
- # First match by the python version
- is_python: _name,
- "//conditions:default": is_python,
+ # First match by the python version and then by ABI
+ is_python: _name + ("_abi" if abi else ""),
+ _DEFAULT: _INCOMPATIBLE,
}),
visibility = visibility,
)
@@ -325,12 +353,23 @@
config_setting_name = _name + "_setting"
native.config_setting(name = config_setting_name, **kwargs)
+ if abi:
+ native.alias(
+ name = _name + "_abi",
+ actual = select(
+ {k: _name for k in abi} | {
+ _DEFAULT: _INCOMPATIBLE,
+ },
+ ),
+ visibility = visibility,
+ )
+
# Next match by the `pip_whl` flag value and then match by the flags that
# are intrinsic to the distribution.
native.alias(
name = _name,
actual = select({
- "//conditions:default": FLAGS.is_pip_whl_auto,
+ _DEFAULT: _INCOMPATIBLE,
FLAGS.is_pip_whl_auto: config_setting_name,
is_pip_whl: config_setting_name,
}),
diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl
index 5a3f841..a6872fd 100644
--- a/python/private/pypi/pkg_aliases.bzl
+++ b/python/private/pypi/pkg_aliases.bzl
@@ -308,7 +308,9 @@
else:
py = "py3"
- if parsed.abi_tag.startswith("cp"):
+ if parsed.abi_tag.startswith("cp") and parsed.abi_tag.endswith("t"):
+ abi = "cpt"
+ elif parsed.abi_tag.startswith("cp"):
abi = "cp"
else:
abi = parsed.abi_tag
diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl
index 612ca2c..79a58a8 100644
--- a/python/private/pypi/whl_library.bzl
+++ b/python/private/pypi/whl_library.bzl
@@ -287,7 +287,7 @@
p.target_platform
for p in whl_target_platforms(
platform_tag = parsed_whl.platform_tag,
- abi_tag = parsed_whl.abi_tag,
+ abi_tag = parsed_whl.abi_tag.strip("tm"),
)
]
diff --git a/python/private/pypi/whl_target_platforms.bzl b/python/private/pypi/whl_target_platforms.bzl
index bdc44c6..6823199 100644
--- a/python/private/pypi/whl_target_platforms.bzl
+++ b/python/private/pypi/whl_target_platforms.bzl
@@ -89,6 +89,10 @@
want_abis[abi] = None
want_abis[abi + "m"] = None
+ # Also add freethreaded wheels if we find them since we started supporting them
+ _want_platforms["{}t_{}".format(abi, os_cpu)] = None
+ want_abis[abi + "t"] = None
+
want_platforms = sorted(_want_platforms)
candidates = {}
diff --git a/tests/pypi/config_settings/config_settings_tests.bzl b/tests/pypi/config_settings/config_settings_tests.bzl
index a77fa5b..049556a 100644
--- a/tests/pypi/config_settings/config_settings_tests.bzl
+++ b/tests/pypi/config_settings/config_settings_tests.bzl
@@ -39,6 +39,7 @@
pip_whl_osx_arch = lambda x: (str(Label("//python/config_settings:pip_whl_osx_arch")), str(x)),
py_linux_libc = lambda x: (str(Label("//python/config_settings:py_linux_libc")), str(x)),
python_version = lambda x: (str(Label("//python/config_settings:python_version")), str(x)),
+ py_freethreaded = lambda x: (str(Label("//python/config_settings:py_freethreaded")), str(x)),
)
def _analysis_test(*, name, dist, want, config_settings = [_flag.platform("linux_aarch64")]):
@@ -286,6 +287,38 @@
_tests.append(_test_py_none_any_versioned)
+def _test_cp_whl_is_not_prefered_over_py3_non_freethreaded(name):
+ _analysis_test(
+ name = name,
+ dist = {
+ "is_cp3.7_cp3x_abi3_any": "py3_abi3",
+ "is_cp3.7_cp3x_cpt_any": "cp",
+ "is_cp3.7_cp3x_none_any": "py3",
+ },
+ want = "py3_abi3",
+ config_settings = [
+ _flag.py_freethreaded("no"),
+ ],
+ )
+
+_tests.append(_test_cp_whl_is_not_prefered_over_py3_non_freethreaded)
+
+def _test_cp_whl_is_not_prefered_over_py3_freethreaded(name):
+ _analysis_test(
+ name = name,
+ dist = {
+ "is_cp3.7_cp3x_abi3_any": "py3_abi3",
+ "is_cp3.7_cp3x_cp_any": "cp",
+ "is_cp3.7_cp3x_none_any": "py3",
+ },
+ want = "py3",
+ config_settings = [
+ _flag.py_freethreaded("yes"),
+ ],
+ )
+
+_tests.append(_test_cp_whl_is_not_prefered_over_py3_freethreaded)
+
def _test_cp_cp_whl(name):
_analysis_test(
name = name,
@@ -412,6 +445,7 @@
name = name,
dist = {
"is_cp3.7_cp3x_cp_windows_x86_64": "whl",
+ "is_cp3.7_cp3x_cpt_windows_x86_64": "whl_freethreaded",
},
want = "whl",
config_settings = [
@@ -421,6 +455,22 @@
_tests.append(_test_windows)
+def _test_windows_freethreaded(name):
+ _analysis_test(
+ name = name,
+ dist = {
+ "is_cp3.7_cp3x_cp_windows_x86_64": "whl",
+ "is_cp3.7_cp3x_cpt_windows_x86_64": "whl_freethreaded",
+ },
+ want = "whl_freethreaded",
+ config_settings = [
+ _flag.platform("windows_x86_64"),
+ _flag.py_freethreaded("yes"),
+ ],
+ )
+
+_tests.append(_test_windows_freethreaded)
+
def _test_osx(name):
_analysis_test(
name = name,
diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl
index 0fa66d0..23a0f01 100644
--- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl
+++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl
@@ -288,6 +288,14 @@
version = "3.1",
): "foo-py3-0.0.1",
whl_config_setting(
+ filename = "foo-0.0.1-cp313-cp313-any.whl",
+ version = "3.1",
+ ): "foo-cp-0.0.1",
+ whl_config_setting(
+ filename = "foo-0.0.1-cp313-cp313t-any.whl",
+ version = "3.1",
+ ): "foo-cpt-0.0.1",
+ whl_config_setting(
filename = "foo-0.0.2-py3-none-any.whl",
version = "3.1",
target_platforms = [
@@ -303,6 +311,8 @@
osx_versions = [],
)
want = {
+ "//_config:is_cp3.1_cp3x_cp_any": "foo-cp-0.0.1",
+ "//_config:is_cp3.1_cp3x_cpt_any": "foo-cpt-0.0.1",
"//_config:is_cp3.1_py3_none_any": "foo-py3-0.0.1",
"//_config:is_cp3.1_py3_none_any_linux_aarch64": "foo-0.0.2",
"//_config:is_cp3.1_py3_none_any_linux_x86_64": "foo-0.0.2",
diff --git a/tests/pypi/whl_target_platforms/select_whl_tests.bzl b/tests/pypi/whl_target_platforms/select_whl_tests.bzl
index 2994bd5..8ab2413 100644
--- a/tests/pypi/whl_target_platforms/select_whl_tests.bzl
+++ b/tests/pypi/whl_target_platforms/select_whl_tests.bzl
@@ -27,6 +27,10 @@
"pkg-0.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl",
"pkg-0.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
"pkg-0.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl",
+ "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl",
+ "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl",
+ "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl",
+ "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl",
"pkg-0.0.1-cp311-cp311-musllinux_1_1_aarch64.whl",
"pkg-0.0.1-cp311-cp311-musllinux_1_1_i686.whl",
"pkg-0.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl",
@@ -269,6 +273,22 @@
_tests.append(_test_prefer_manylinux_wheels)
+def _test_freethreaded_wheels(env):
+ # Check we prefer platform specific wheels
+ got = _select_whls(whls = WHL_LIST, want_platforms = ["cp313_linux_x86_64"])
+ _match(
+ env,
+ got,
+ "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl",
+ "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl",
+ "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl",
+ "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl",
+ "pkg-0.0.1-cp39-abi3-any.whl",
+ "pkg-0.0.1-py3-none-any.whl",
+ )
+
+_tests.append(_test_freethreaded_wheels)
+
def select_whl_test_suite(name):
"""Create the test suite.
