feat(toolchains): support debug runtimes on Linux and macOS (#4471) Follow-up to #4442. python-build-standalone debug (`Py_DEBUG`) archives for Linux and macOS add a `d` ABI flag to some names: `include/python3.14d` and `lib/libpython3.14d.so` (`.dylib` on macOS), or `td` for free-threaded debug builds. The runtime repo only knows the release names, so a debug runtime registered with `python.single_version_platform_override` does not work for C code: `current_py_cc_headers` fails with `Python.h: No such file or directory` and `current_py_cc_libs` with `missing input file '...//:lib/libpython3.14.so'`. On a macOS host, the repo rule's `install_name_tool` step also looks for the release dylib. To fix, the repo rule checks the extracted archive for the `d` include dir and passes that to the runtime macro, which uses the `d` names. The macro also sets `abi_flags` on `py_runtime` and `py_cc_toolchain`, because `<AUTO>` only derives `t`. Extensions built against the debug headers need a debug interpreter, so they get the debug SOABI (for example `cpython-314d-x86_64-linux-gnu`) and wheel ABI tag (`cp314d`). Windows debug archives use other names (`python_d.exe`, `python314_d.dll`) and are not handled here. Will be addressed separately. * Also registers a debug runtime for tests in `MODULE.bazel` and adds `//tests/toolchains:debug_runtime_test`, which builds an extension against it and imports it. --------- Co-authored-by: Claude Fable 5.2 <noreply@anthropic.com>
diff --git a/MODULE.bazel b/MODULE.bazel index 3227fba..b7301c7 100644 --- a/MODULE.bazel +++ b/MODULE.bazel
@@ -399,6 +399,25 @@ urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20250115/cpython-3.13.1+20250115-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz"], ) +# For testing a debug (`Py_DEBUG`) runtime. +# See //tests/toolchains:debug_runtime_test +# +# NOTE: 3.14.10, the newest 3.14, would break //tests/toolchains:build_test. +dev_python.single_version_platform_override( + platform = "linux-x86-debug", + python_version = "3.14.4", + sha256 = "65db1c71f78ac00553e6353f730ff2f942cc18a5758f5d99c77fd4be5d7719de", + strip_prefix = "python/install", + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + target_settings = [ + "@@//tests/support:is_custom_runtime_linux-x86-debug", + ], + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260414/cpython-3.14.4+20260414-x86_64-unknown-linux-gnu-debug-full.tar.zst"], +) + dev_pip = use_extension( "//python/extensions:pip.bzl", "pip",
diff --git a/news/debug-runtime-unix-layout.added.md b/news/debug-runtime-unix-layout.added.md new file mode 100644 index 0000000..607cf0d --- /dev/null +++ b/news/debug-runtime-unix-layout.added.md
@@ -0,0 +1,4 @@ +(toolchains) python-build-standalone debug (`Py_DEBUG`) runtimes registered +with {obj}`python.single_version_platform_override` now provide their headers +and `libpython` on Linux and macOS, and report the `d` ABI flag +([#4471](https://github.com/bazel-contrib/rules_python/issues/4471)).
diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl index d61a040..516b1ab 100644 --- a/python/private/hermetic_runtime_repo_setup.bzl +++ b/python/private/hermetic_runtime_repo_setup.bzl
@@ -30,6 +30,7 @@ extra_files_glob_include, extra_files_glob_exclude, interpreter_has_static_libpython = False, + debug = False, python_version, python_bin, coverage_tool): @@ -48,6 +49,9 @@ patterns for the target runtime files. interpreter_has_static_libpython: {type}`bool` whether the interpreter includes libpython statically. + debug: {type}`bool` whether the runtime is a debug (`Py_DEBUG`) + build. On Linux and macOS, its include dir and libpython names + have a `d` ABI flag. python_version: {type}`str` The Python version, in `major.minor.micro` format. python_bin: {type}`str` The path to the Python binary within the @@ -58,6 +62,7 @@ _ = name # @unused version_info = version.parse(python_version) version_dict = { + "d": "d" if debug else "", "major": version_info.release[0], "minor": version_info.release[1], } @@ -126,10 +131,10 @@ "include", ] + select({ _IS_FREETHREADED_YES: [ - "include/python{major}.{minor}t".format(**version_dict), + "include/python{major}.{minor}t{d}".format(**version_dict), ], _IS_FREETHREADED_NO: [ - "include/python{major}.{minor}".format(**version_dict), + "include/python{major}.{minor}{d}".format(**version_dict), "include/python{major}.{minor}m".format(**version_dict), ], }), @@ -178,11 +183,11 @@ hdrs = [":includes"], srcs = select({ ":is_freethreaded_linux": [ - "lib/libpython{major}.{minor}t.so".format(**version_dict), - "lib/libpython{major}.{minor}t.so.1.0".format(**version_dict), + "lib/libpython{major}.{minor}t{d}.so".format(**version_dict), + "lib/libpython{major}.{minor}t{d}.so.1.0".format(**version_dict), ], ":is_freethreaded_osx": [ - "lib/libpython{major}.{minor}t.dylib".format(**version_dict), + "lib/libpython{major}.{minor}t{d}.dylib".format(**version_dict), ], ":is_freethreaded_windows": [ "python3t.dll", @@ -190,7 +195,9 @@ "libs/python{major}{minor}t.lib".format(**version_dict), "libs/python3t.lib", ], - "@platforms//os:macos": ["lib/libpython{major}.{minor}.dylib".format(**version_dict)], + "@platforms//os:macos": [ + "lib/libpython{major}.{minor}{d}.dylib".format(**version_dict), + ], "@platforms//os:windows": [ "python3.dll", "python{major}{minor}.dll".format(**version_dict), @@ -198,12 +205,20 @@ "libs/python3.lib", ], "//conditions:default": [ - "lib/libpython{major}.{minor}.so".format(**version_dict), - "lib/libpython{major}.{minor}.so.1.0".format(**version_dict), + "lib/libpython{major}.{minor}{d}.so".format(**version_dict), + "lib/libpython{major}.{minor}{d}.so.1.0".format(**version_dict), ], }), ) + # `<AUTO>` derives only the free-threaded "t" flag. + abi_flags = None + if debug: + abi_flags = select({ + _IS_FREETHREADED_YES: "td", + _IS_FREETHREADED_NO: "d", + }) + native.exports_files(["python", python_bin]) # Used to only download coverage toolchain when the coverage is collected by @@ -240,6 +255,7 @@ }), python_version = "PY3", implementation_name = "cpython", + abi_flags = abi_flags, # See https://peps.python.org/pep-3147/ for pyc tag infix format pyc_tag = select({ _IS_FREETHREADED_YES: "cpython-{major}{minor}t".format(**version_dict), @@ -267,6 +283,7 @@ py_cc_toolchain( name = "py_cc_toolchain", + abi_flags = abi_flags, headers = ":python_headers", headers_abi3 = ":python_headers_abi3", # TODO #3155: add libctl, libtk
diff --git a/python/private/python_repository.bzl b/python/private/python_repository.bzl index 5a89553..3ada5dd 100644 --- a/python/private/python_repository.bzl +++ b/python/private/python_repository.bzl
@@ -206,6 +206,10 @@ for patch in patches: rctx.patch(patch, strip = rctx.attr.patch_strip) + # Debug (`Py_DEBUG`) builds add a "d" ABI flag to the include dir and + # libpython names, but not to the stdlib dir. + debug = rctx.path("include/python{}d".format(python_short_version)).exists + # Write distutils.cfg to the Python installation. if "windows" in platform: distutils_path = "Lib/distutils/distutils.cfg" @@ -223,7 +227,10 @@ # dyld lookup errors. To fix, set the full path to the dylib as # it appears in the Bazel workspace as its LC_ID_DYLIB using # the `install_name_tool` bundled with macOS. - dylib = "libpython{}.dylib".format(python_short_version) + dylib = "libpython{}{}.dylib".format( + python_short_version, + "d" if debug else "", + ) repo_utils.execute_checked( rctx, op = "python_repository.FixUpDyldIdPath", @@ -300,6 +307,7 @@ extra_files_glob_include = {extra_files_glob_include}, extra_files_glob_exclude = {extra_files_glob_exclude}, interpreter_has_static_libpython = {interpreter_has_static_libpython}, + debug = {debug}, python_version = {python_version}, python_bin = {python_bin}, coverage_tool = {coverage_tool}, @@ -308,6 +316,7 @@ extra_files_glob_exclude = render.list(glob_exclude), extra_files_glob_include = render.list(glob_include), interpreter_has_static_libpython = str(interpreter_has_static_libpython), + debug = str(debug), python_bin = render.str(python_bin), python_version = render.str(rctx.attr.python_version), coverage_tool = render.str(coverage_tool),
diff --git a/tests/support/BUILD.bazel b/tests/support/BUILD.bazel index 45f43c8..5060c56 100644 --- a/tests/support/BUILD.bazel +++ b/tests/support/BUILD.bazel
@@ -34,3 +34,10 @@ ":custom_runtime": "linux-x86-install-only-stripped", }, ) + +config_setting( + name = "is_custom_runtime_linux-x86-debug", + flag_values = { + ":custom_runtime": "linux-x86-debug", + }, +)
diff --git a/tests/toolchains/BUILD.bazel b/tests/toolchains/BUILD.bazel index fbf2fe8..67796d0 100644 --- a/tests/toolchains/BUILD.bazel +++ b/tests/toolchains/BUILD.bazel
@@ -14,6 +14,7 @@ load("@bazel_skylib//rules:build_test.bzl", "build_test") load("//python:pytest_test.bzl", "pytest_test") +load("//python/cc:py_extension.bzl", "py_extension") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility load("//tests/support:py_reconfig.bzl", "py_reconfig_test") load(":defs.bzl", "define_toolchain_tests") @@ -33,6 +34,25 @@ ] if BZLMOD_ENABLED else ["@platforms//:incompatible"], ) +# Manual: it only compiles with the headers of the debug runtime. +py_extension( + name = "debug_ext", + srcs = ["debug_ext.c"], + tags = ["manual"], +) + +py_reconfig_test( + name = "debug_runtime_test", + srcs = ["debug_runtime_test.py"], + config_settings = {"//tests/support:custom_runtime": "linux-x86-debug"}, + python_version = "3.14.4", + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ] if BZLMOD_ENABLED else ["@platforms//:incompatible"], + deps = [":debug_ext"], +) + build_test( name = "build_test", targets = [
diff --git a/tests/toolchains/debug_ext.c b/tests/toolchains/debug_ext.c new file mode 100644 index 0000000..459aeda --- /dev/null +++ b/tests/toolchains/debug_ext.c
@@ -0,0 +1,23 @@ +#include <Python.h> + +// The headers must be the ones of the debug runtime. +#ifndef Py_DEBUG +#error "Py_DEBUG is not defined" +#endif + +static PyObject* answer(PyObject* self, PyObject* args) { + return PyLong_FromLong(42); +} + +static PyMethodDef methods[] = { + {"answer", answer, METH_NOARGS, NULL}, + {NULL, NULL, 0, NULL}, +}; + +static struct PyModuleDef module = { + PyModuleDef_HEAD_INIT, "debug_ext", NULL, -1, methods, +}; + +PyMODINIT_FUNC PyInit_debug_ext(void) { + return PyModule_Create(&module); +}
diff --git a/tests/toolchains/debug_runtime_test.py b/tests/toolchains/debug_runtime_test.py new file mode 100644 index 0000000..37b743a --- /dev/null +++ b/tests/toolchains/debug_runtime_test.py
@@ -0,0 +1,23 @@ +import sys +import sysconfig +import unittest + +from tests.toolchains import ( + debug_ext, # pyrefly: ignore[missing-module-attribute] +) + + +class DebugRuntimeTest(unittest.TestCase): + def test_interpreter_is_a_debug_build(self): + self.assertEqual("d", sys.abiflags) + + def test_extension_uses_the_debug_abi(self): + self.assertEqual(42, debug_ext.answer()) + self.assertTrue( + debug_ext.__file__.endswith(sysconfig.get_config_var("EXT_SUFFIX")), + debug_ext.__file__, + ) + + +if __name__ == "__main__": + unittest.main()