fix(local) Add api3 targets and additional defines. (#3408)
Propagate defines and additional dll requirements for local python
installs.
In get_local_runtime_info.py:
* detect abi3 vs. full abi libraries.
* Ensure that returned libraries are unique.
* Add additional dlls required by pythonXY.dll / pythonX.dll on windows.
* Add default defines for Py_GIL_DISABLED when the local python is a
freethreaded install.
* Add defines (windows) for Py_NO_LINK_LIB to avoid #pragma comment(lib
...) macros
In local_runtime_repo_setup.bzl
* More closely match hermetic_runtime_repo_setup
* Add abi3 header targets.
In local_runtime_repo.bzl
* rework linking to local repository directories to handl abi3 and extra
dlls.
* Update parameters passed into local_runtime_repo_setup.bzl
Before these changes, some bazel builds using local Python fail to link
properly.
This happens due to a mismatch in the interpreter and the python GIL
DISABLED mode, or (on Windows), where both freethreaded and
non-freethreaded libraries may attempt to be linked at the same time.
---------
Co-authored-by: Richard Levasseur <richardlev@gmail.com>
Co-authored-by: Richard Levasseur <rlevasseur@google.com>
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4f1911f..6709a9d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -78,6 +78,9 @@
([#3085](https://github.com/bazel-contrib/rules_python/issues/3085)).
* (toolchains) local toolchains now tell the `sys.abiflags` value of the
underlying runtime.
+* (toolchains) various local toolchain fixes: add abi3 header targets,
+ fixes to linking, Windows DLL detection, and defines for free threaded
+ runtimes.
* (toolchains) The `python_headers` target is now compatible with
layering_check.
* (performance) 90% reduction in py_binary/py_test analysis phase cost.
diff --git a/python/private/get_local_runtime_info.py b/python/private/get_local_runtime_info.py
index b20c159..a59e17a 100644
--- a/python/private/get_local_runtime_info.py
+++ b/python/private/get_local_runtime_info.py
@@ -13,16 +13,27 @@
# limitations under the License.
"""Returns information about the local Python runtime as JSON."""
+import glob
import json
import os
import sys
import sysconfig
+from typing import Any
_IS_WINDOWS = sys.platform == "win32"
_IS_DARWIN = sys.platform == "darwin"
-def _search_directories(get_config, base_executable):
+def _get_abi_flags(get_config) -> str:
+ """Returns the ABI flags for the Python runtime."""
+ # sys.abiflags may not exist, but it still may be set in the config.
+ abi_flags = getattr(sys, "abiflags", None)
+ if abi_flags is None:
+ abi_flags = get_config("ABIFLAGS") or get_config("abiflags") or ""
+ return abi_flags
+
+
+def _search_directories(get_config, base_executable) -> list[str]:
"""Returns a list of library directories to search for shared libraries."""
# There's several types of libraries with different names and a plethora
# of settings, and many different config variables to check:
@@ -73,23 +84,31 @@
lib_dirs.append(os.path.join(os.path.dirname(exec_dir), "lib"))
# Dedup and remove empty values, keeping the order.
- lib_dirs = [v for v in lib_dirs if v]
- return {k: None for k in lib_dirs}.keys()
+ return list(dict.fromkeys(d for d in lib_dirs if d))
-def _get_shlib_suffix(get_config) -> str:
- """Returns the suffix for shared libraries."""
- if _IS_DARWIN:
- return ".dylib"
+def _default_library_names(version, abi_flags) -> tuple[str, ...]:
+ """Returns a list of default library files to search for shared libraries."""
if _IS_WINDOWS:
- return ".dll"
- suffix = get_config("SHLIB_SUFFIX")
- if not suffix:
- suffix = ".so"
- return suffix
+ return (
+ f"python{version}{abi_flags}.dll",
+ f"python{version}.dll",
+ )
+ elif _IS_DARWIN:
+ return (
+ f"libpython{version}{abi_flags}.dylib",
+ f"libpython{version}.dylib",
+ )
+ else:
+ return (
+ f"libpython{version}{abi_flags}.so",
+ f"libpython{version}.so",
+ f"libpython{version}{abi_flags}.so.1.0",
+ f"libpython{version}.so.1.0",
+ )
-def _search_library_names(get_config, shlib_suffix):
+def _search_library_names(get_config, version, abi_flags) -> list[str]:
"""Returns a list of library files to search for shared libraries."""
# Quoting configure.ac in the cpython code base:
# "INSTSONAME is the name of the shared library that will be use to install
@@ -112,71 +131,75 @@
)
]
- # Set the prefix and suffix to construct the library name used for linking.
- # The suffix and version are set here to the default values for the OS,
- # since they are used below to construct "default" library names.
- if _IS_DARWIN:
- prefix = "lib"
- elif _IS_WINDOWS:
- prefix = ""
- else:
- prefix = "lib"
+ # Include the default libraries for the system.
+ lib_names.extend(_default_library_names(version, abi_flags))
- version = get_config("VERSION")
+ # Also include the abi3 libraries for the system.
+ lib_names.extend(_default_library_names(sys.version_info.major, abi_flags))
- # Ensure that the pythonXY.dll files are included in the search.
- lib_names.append(f"{prefix}python{version}{shlib_suffix}")
-
- # If there are ABIFLAGS, also add them to the python version lib search.
- abiflags = get_config("ABIFLAGS") or get_config("abiflags") or ""
- if abiflags:
- lib_names.append(f"{prefix}python{version}{abiflags}{shlib_suffix}")
-
- # Add the abi-version includes to the search list.
- lib_names.append(f"{prefix}python{sys.version_info.major}{shlib_suffix}")
-
- # Dedup and remove empty values, keeping the order.
- lib_names = [v for v in lib_names if v]
- return {k: None for k in lib_names}.keys()
+ # Uniqify, preserving order.
+ return list(dict.fromkeys(k for k in lib_names if k))
-def _get_python_library_info(base_executable):
+def _get_python_library_info(base_executable) -> dict[str, Any]:
"""Returns a dictionary with the static and dynamic python libraries."""
config_vars = sysconfig.get_config_vars()
# VERSION is X.Y in Linux/macOS and XY in Windows. This is used to
# construct library paths such as python3.12, so ensure it exists.
- if not config_vars.get("VERSION"):
- if sys.platform == "win32":
- config_vars["VERSION"] = (
- f"{sys.version_info.major}{sys.version_info.minor}")
+ version = config_vars.get("VERSION")
+ if not version:
+ if _IS_WINDOWS:
+ version = f"{sys.version_info.major}{sys.version_info.minor}"
else:
- config_vars["VERSION"] = (
- f"{sys.version_info.major}.{sys.version_info.minor}")
+ version = f"{sys.version_info.major}.{sys.version_info.minor}"
- shlib_suffix = _get_shlib_suffix(config_vars.get)
+ defines = []
+ if config_vars.get("Py_GIL_DISABLED", "0") == "1":
+ defines.append("Py_GIL_DISABLED")
+
+ # Avoid automatically linking the libraries on windows via pydefine.h
+ # pragma comment(lib ...)
+ if _IS_WINDOWS:
+ defines.append("Py_NO_LINK_LIB")
+
+ # sys.abiflags may not exist, but it still may be set in the config.
+ abi_flags = _get_abi_flags(config_vars.get)
+
search_directories = _search_directories(config_vars.get, base_executable)
- search_libnames = _search_library_names(config_vars.get, shlib_suffix)
+ search_libnames = _search_library_names(config_vars.get, version,
+ abi_flags)
- interface_libraries = {}
- dynamic_libraries = {}
- static_libraries = {}
+ # Used to test whether the library is an abi3 library or a full api library.
+ abi3_libraries = _default_library_names(sys.version_info.major, abi_flags)
+
+ # Found libraries
+ static_libraries: dict[str, None] = {}
+ dynamic_libraries: dict[str, None] = {}
+ interface_libraries: dict[str, None] = {}
+ abi_dynamic_libraries: dict[str, None] = {}
+ abi_interface_libraries: dict[str, None] = {}
for root_dir in search_directories:
for libname in search_libnames:
- # Check whether the library exists.
composed_path = os.path.join(root_dir, libname)
+ is_abi3_file = os.path.basename(composed_path) in abi3_libraries
+
+ # Check whether the library exists and add it to the appropriate list.
if os.path.exists(composed_path) or os.path.isdir(composed_path):
- if libname.endswith(".a"):
+ if is_abi3_file:
+ if not libname.endswith(".a"):
+ abi_dynamic_libraries[composed_path] = None
+ elif libname.endswith(".a"):
static_libraries[composed_path] = None
else:
dynamic_libraries[composed_path] = None
interface_path = None
if libname.endswith(".dll"):
- # On windows a .lib file may be an "import library" or a static library.
- # The file could be inspected to determine which it is; typically python
- # is used as a shared library.
+ # On windows a .lib file may be an "import library" or a static
+ # library. The file could be inspected to determine which it is;
+ # typically python is used as a shared library.
#
# On Windows, extensions should link with the pythonXY.lib interface
# libraries.
@@ -190,39 +213,51 @@
# Check whether an interface library exists.
if interface_path and os.path.exists(interface_path):
- interface_libraries[interface_path] = None
+ if is_abi3_file:
+ abi_interface_libraries[interface_path] = None
+ else:
+ interface_libraries[interface_path] = None
- # Non-windows typically has abiflags.
- if hasattr(sys, "abiflags"):
- abiflags = sys.abiflags
- else:
- abiflags = ""
+ # Additional DLLs are needed on Windows to link properly.
+ dlls = []
+ if _IS_WINDOWS:
+ dlls.extend(
+ glob.glob(os.path.join(os.path.dirname(base_executable), "*.dll")))
+ dlls = [
+ x for x in dlls
+ if x not in dynamic_libraries and x not in abi_dynamic_libraries
+ ]
+
+ def _unique_basenames(inputs: dict[str, None]) -> list[str]:
+ """Returns a list of paths, keeping only the first path for each basename."""
+ result = []
+ seen = set()
+ for k in inputs:
+ b = os.path.basename(k)
+ if b not in seen:
+ seen.add(b)
+ result.append(k)
+ return result
# When no libraries are found it's likely that the python interpreter is not
# configured to use shared or static libraries (minilinux). If this seems
# suspicious try running `uv tool run find_libpython --list-all -v`
return {
- "dynamic_libraries": list(dynamic_libraries.keys()),
- "static_libraries": list(static_libraries.keys()),
- "interface_libraries": list(interface_libraries.keys()),
- "shlib_suffix": "" if _IS_WINDOWS else shlib_suffix,
- "abi_flags": abiflags,
+ "dynamic_libraries": _unique_basenames(dynamic_libraries),
+ "static_libraries": _unique_basenames(static_libraries),
+ "interface_libraries": _unique_basenames(interface_libraries),
+ "abi_dynamic_libraries": _unique_basenames(abi_dynamic_libraries),
+ "abi_interface_libraries": _unique_basenames(abi_interface_libraries),
+ "abi_flags": abi_flags,
+ "shlib_suffix": ".dylib" if _IS_DARWIN else "",
+ "additional_dlls": dlls,
+ "defines": defines,
}
-def _get_base_executable():
+def _get_base_executable() -> str:
"""Returns the base executable path."""
- try:
- if sys._base_executable: # pylint: disable=protected-access
- return sys._base_executable # pylint: disable=protected-access
- except AttributeError:
- # Bug reports indicate sys._base_executable doesn't exist in some cases,
- # but it's not clear why.
- # See https://github.com/bazel-contrib/rules_python/issues/3172
- pass
- # The normal sys.executable is the next-best guess if sys._base_executable
- # is missing.
- return sys.executable
+ return getattr(sys, "_base_executable", None) or sys.executable
data = {
diff --git a/python/private/local_runtime_repo.bzl b/python/private/local_runtime_repo.bzl
index 024f7c5..df27c74 100644
--- a/python/private/local_runtime_repo.bzl
+++ b/python/private/local_runtime_repo.bzl
@@ -34,15 +34,36 @@
major = "{major}",
minor = "{minor}",
micro = "{micro}",
+ abi_flags = "{abi_flags}",
+ os = "{os}",
+ implementation_name = "{implementation_name}",
interpreter_path = "{interpreter_path}",
interface_library = {interface_library},
libraries = {libraries},
- implementation_name = "{implementation_name}",
- os = "{os}",
- abi_flags = "{abi_flags}",
+ defines = {defines},
+ abi3_interface_library = {abi3_interface_library},
+ abi3_libraries = {abi3_libraries},
+ additional_dlls = {additional_dlls},
)
"""
+def _expand_incompatible_template():
+ return _TOOLCHAIN_IMPL_TEMPLATE.format(
+ major = "0",
+ minor = "0",
+ micro = "0",
+ abi_flags = "",
+ os = "@platforms//:incompatible",
+ implementation_name = "incompatible",
+ interpreter_path = "/incompatible",
+ interface_library = "None",
+ libraries = "[]",
+ defines = "[]",
+ abi3_interface_library = "None",
+ abi3_libraries = "[]",
+ additional_dlls = "[]",
+ )
+
def _norm_path(path):
"""Returns a path using '/' separators and no trailing slash."""
path = path.replace("\\", "/")
@@ -50,33 +71,39 @@
path = path[:-1]
return path
-def _symlink_first_library(rctx, logger, libraries, shlib_suffix):
+def _symlink_libraries(rctx, logger, libraries, shlib_suffix):
"""Symlinks the shared libraries into the lib/ directory.
Args:
rctx: A repository_ctx object
logger: A repo_utils.logger object
- libraries: A list of static library paths to potentially symlink.
- shlib_suffix: A suffix only provided for shared libraries to ensure
- that the srcs restriction of cc_library targets are met.
+ libraries: paths to libraries to attempt to symlink.
+ shlib_suffix: Optional. Ensure that the generated symlinks end with this suffix.
Returns:
- A single library path linked by the action.
+ A list of library paths (under lib/) linked by the action.
+
+ Individual files are symlinked instead of the whole directory because
+ shared_lib_dirs contains multiple search paths for the shared libraries,
+ and the python files may be missing from any of those directories, and
+ any of those directories may include non-python runtime libraries,
+ as would be the case if LIBDIR were, for example, /usr/lib.
"""
- for target in libraries:
- origin = rctx.path(target)
+ result = []
+ for source in libraries:
+ origin = rctx.path(source)
if not origin.exists:
# The reported names don't always exist; it depends on the particulars
# of the runtime installation.
continue
- if shlib_suffix and not target.endswith(shlib_suffix):
- linked = "lib/{}{}".format(origin.basename, shlib_suffix)
+ if shlib_suffix and not origin.basename.endswith(shlib_suffix):
+ target = "lib/{}{}".format(origin.basename, shlib_suffix)
else:
- linked = "lib/{}".format(origin.basename)
- logger.debug("Symlinking {} to {}".format(origin, linked))
+ target = "lib/{}".format(origin.basename)
+ logger.debug(lambda: "Symlinking {} to {}".format(origin, target))
rctx.watch(origin)
- rctx.symlink(origin, linked)
- return linked
- return None
+ rctx.symlink(origin, target)
+ result.append(target)
+ return result
def _local_runtime_repo_impl(rctx):
logger = repo_utils.logger(rctx)
@@ -150,31 +177,48 @@
# The cc_library.includes values have to be non-absolute paths, otherwise
# the toolchain will give an error. Work around this error by making them
# appear as part of this repo.
+ logger.debug(lambda: "Symlinking {} to include".format(include_path))
rctx.symlink(include_path, "include")
rctx.report_progress("Symlinking external Python shared libraries")
- interface_library = _symlink_first_library(rctx, logger, info["interface_libraries"], None)
- shared_library = _symlink_first_library(rctx, logger, info["dynamic_libraries"], info["shlib_suffix"])
- static_library = _symlink_first_library(rctx, logger, info["static_libraries"], None)
- libraries = []
- if shared_library:
- libraries.append(shared_library)
- elif static_library:
- libraries.append(static_library)
+ interface_library = None
+ if info["dynamic_libraries"]:
+ libraries = _symlink_libraries(rctx, logger, info["dynamic_libraries"][:1], info["shlib_suffix"])
+ symlinked = _symlink_libraries(rctx, logger, info["interface_libraries"][:1], None)
+ if symlinked:
+ interface_library = symlinked[0]
else:
- logger.warn("No external python libraries found.")
+ libraries = _symlink_libraries(rctx, logger, info["static_libraries"], None)
+ if not libraries:
+ logger.info("No python libraries found.")
+
+ abi3_interface_library = None
+ if info["abi_dynamic_libraries"]:
+ abi3_libraries = _symlink_libraries(rctx, logger, info["abi_dynamic_libraries"][:1], info["shlib_suffix"])
+ symlinked = _symlink_libraries(rctx, logger, info["abi_interface_libraries"][:1], None)
+ if symlinked:
+ abi3_interface_library = symlinked[0]
+ else:
+ abi3_libraries = []
+ logger.info("No abi3 python libraries found.")
+
+ additional_dlls = _symlink_libraries(rctx, logger, info["additional_dlls"], None)
build_bazel = _TOOLCHAIN_IMPL_TEMPLATE.format(
major = info["major"],
minor = info["minor"],
micro = info["micro"],
+ abi_flags = info["abi_flags"],
+ os = "@platforms//os:{}".format(repo_utils.get_platforms_os_name(rctx)),
+ implementation_name = info["implementation_name"],
interpreter_path = _norm_path(interpreter_path),
interface_library = repr(interface_library),
libraries = repr(libraries),
- implementation_name = info["implementation_name"],
- os = "@platforms//os:{}".format(repo_utils.get_platforms_os_name(rctx)),
- abi_flags = info["abi_flags"],
+ defines = repr(info["defines"]),
+ abi3_interface_library = repr(abi3_interface_library),
+ abi3_libraries = repr(abi3_libraries),
+ additional_dlls = repr(additional_dlls),
)
logger.debug(lambda: "BUILD.bazel\n{}".format(build_bazel))
@@ -261,19 +305,6 @@
environ = ["PATH", REPO_DEBUG_ENV_VAR, "DEVELOPER_DIR", "XCODE_VERSION"],
)
-def _expand_incompatible_template():
- return _TOOLCHAIN_IMPL_TEMPLATE.format(
- interpreter_path = "/incompatible",
- implementation_name = "incompatible",
- interface_library = "None",
- libraries = "[]",
- major = "0",
- minor = "0",
- micro = "0",
- os = "@platforms//:incompatible",
- abi_flags = "",
- )
-
def _find_python_exe_from_target(rctx):
base_path = rctx.path(rctx.attr.interpreter_target)
if base_path.exists:
diff --git a/python/private/local_runtime_repo_setup.bzl b/python/private/local_runtime_repo_setup.bzl
index 0ce1d4d..0922181 100644
--- a/python/private/local_runtime_repo_setup.bzl
+++ b/python/private/local_runtime_repo_setup.bzl
@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
"""Setup code called by the code generated by `local_runtime_repo`."""
load("@bazel_skylib//lib:selects.bzl", "selects")
@@ -29,12 +28,16 @@
major,
minor,
micro,
+ abi_flags,
+ os,
+ implementation_name,
interpreter_path,
interface_library,
libraries,
- implementation_name,
- os,
- abi_flags):
+ defines,
+ abi3_interface_library,
+ abi3_libraries,
+ additional_dlls):
"""Defines a toolchain implementation for a local Python runtime.
Generates public targets:
@@ -51,16 +54,23 @@
major: `str` The major Python version, e.g. `3` of `3.9.1`.
minor: `str` The minor Python version, e.g. `9` of `3.9.1`.
micro: `str` The micro Python version, e.g. "1" of `3.9.1`.
+ abi_flags: `str` The abi flags, as returned by `sys.abiflags`.
+ os: `str` A label to the OS constraint (e.g. `@platforms//os:linux`) for
+ this runtime.
+ implementation_name: `str` The implementation name, as returned by
+ `sys.implementation.name`.
interpreter_path: `str` Absolute path to the interpreter.
interface_library: `str` Path to the interface library.
e.g. "lib/python312.lib"
libraries: `list[str]` Path[s] to the python libraries.
e.g. ["lib/python312.dll"] or ["lib/python312.so"]
- implementation_name: `str` The implementation name, as returned by
- `sys.implementation.name`.
- os: `str` A label to the OS constraint (e.g. `@platforms//os:linux`) for
- this runtime.
- abi_flags: `str` Str. Flags provided by sys.abiflags for the runtime.
+ defines: `list[str]` List of additional defines.
+ abi3_interface_library: `str` Path to the interface library.
+ e.g. "lib/python3.lib"
+ abi3_libraries: `list[str]` Path[s] to the python libraries.
+ e.g. ["lib/python3.dll"] or ["lib/python3.so"]
+ additional_dlls: `list[str]` Path[s] to additional DLLs.
+ e.g. ["lib/msvcrt123.dll"]
"""
major_minor = "{}.{}".format(major, minor)
major_minor_micro = "{}.{}".format(major_minor, micro)
@@ -69,44 +79,70 @@
# See https://docs.python.org/3/extending/windows.html
# However not all python installations (such as manylinux) include shared or static libraries,
# so only create the import library when interface_library is set.
- full_abi_deps = []
- abi3_deps = []
if interface_library:
cc_import(
- name = "_python_interface_library",
+ name = "interface",
interface_library = interface_library,
- system_provided = 1,
+ system_provided = True,
)
- if interface_library.endswith("{}.lib".format(major)):
- abi3_deps = [":_python_interface_library"]
- else:
- full_abi_deps = [":_python_interface_library"]
- cc_library(
- name = "_python_headers_abi3",
- # NOTE: Keep in sync with watch_tree() called in local_runtime_repo
+ if abi3_interface_library:
+ cc_import(
+ name = "abi3_interface",
+ interface_library = abi3_interface_library,
+ system_provided = True,
+ )
+
+ native.filegroup(
+ name = "includes",
srcs = native.glob(
include = ["include/**/*.h"],
exclude = ["include/numpy/**"], # numpy headers are handled separately
allow_empty = True, # A Python install may not have C headers
),
- deps = abi3_deps,
+ )
+
+ # header libraries.
+ cc_library(
+ name = "python_headers_abi3",
+ hdrs = [":includes"],
includes = ["include"],
- )
- cc_library(
- name = "_python_headers",
- deps = [":_python_headers_abi3"] + full_abi_deps,
+ defines = defines, # NOTE: Users should define Py_LIMITED_API=3
+ deps = select({
+ "@bazel_tools//src/conditions:windows": [":abi3_interface"],
+ "//conditions:default": [],
+ }),
)
cc_library(
- name = "_libpython",
- hdrs = [":_python_headers"],
- srcs = libraries,
- deps = [],
+ name = "python_headers",
+ hdrs = [":includes"],
+ includes = ["include"],
+ defines = defines,
+ deps = select({
+ "@bazel_tools//src/conditions:windows": [":interface"],
+ "//conditions:default": [],
+ }),
)
+ # python libraries
+ cc_library(
+ name = "libpython_abi3",
+ hdrs = [":includes"],
+ defines = defines, # NOTE: Users should define Py_LIMITED_API=3
+ srcs = abi3_libraries + additional_dlls,
+ )
+
+ cc_library(
+ name = "libpython",
+ hdrs = [":includes"],
+ defines = defines,
+ srcs = libraries + additional_dlls,
+ )
+
+ # runtime configuration
py_runtime(
- name = "_py3_runtime",
+ name = "py3_runtime",
interpreter_path = interpreter_path,
python_version = "PY3",
interpreter_version_info = {
@@ -116,12 +152,13 @@
},
implementation_name = implementation_name,
abi_flags = abi_flags,
+ pyc_tag = "{}-{}{}{}".format(implementation_name, major, minor, abi_flags),
)
py_runtime_pair(
name = "python_runtimes",
py2_runtime = None,
- py3_runtime = ":_py3_runtime",
+ py3_runtime = ":py3_runtime",
visibility = ["//visibility:public"],
)
@@ -133,9 +170,9 @@
py_cc_toolchain(
name = "py_cc_toolchain",
- headers = ":_python_headers",
- headers_abi3 = ":_python_headers_abi3",
- libs = ":_libpython",
+ headers = ":python_headers",
+ headers_abi3 = ":python_headers_abi3",
+ libs = ":libpython",
python_version = major_minor_micro,
visibility = ["//visibility:public"],
)
