fix(local_runtime): Improve local_runtime usability in macos / windows (#3148)
local_runtime fails to handle many variations of python install on
Windows and
MacOS, such as:
* LDLIBRARY on MacOS may refer to a file under PYTHONFRAMEWORKPREFIX,
not LIBDIR
* LDLIBRARY on Windows refers to pythonXY.dll, not the linkable
pythonXY.lib
* LIBDIR may not be correctly set on Windows.
* On windows, interpreter_path needs to be normalized. Other paths also
require this.
* SHLIB_SUFFIX does not indicate the correct suffix.
For examples, see: https://docs.python.org/3/extending/windows.html
In order to resolve this the shared library resolution has been moved
into get_local_runtime_info.py, which now does the following:
* Constructs a list of paths to search based on LIBDIR, LIBPL,
PYTHONFRAMEWORKPREFIX,
and the executable directory.
* Constructs a list of libraries to search based on INSTSONAME,
LDLIBRARY,
pythonXY.lib, etc.
* Checks to see which files exist, partitioning the result into a list
of
"dynamic_libraries" and "static_libraries"
On Windows and macOS, since SHLIB_SUFFIX does not always indicate the
filenames
needed searching, this has been removed from local_runtime_repo_setup
and
replaced with an explicit file.
On Windows the interpreter_path and other search paths are now
normalized
(`\` converted to `/`).
Additional logging added to local_runtime_repo.
Fixes https://github.com/bazel-contrib/rules_python/issues/3055
Work towards https://github.com/bazel-contrib/rules_python/issues/824
---------
Co-authored-by: Richard Levasseur <richardlev@gmail.com>
diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml
index 6457363..5889823 100644
--- a/.bazelci/presubmit.yml
+++ b/.bazelci/presubmit.yml
@@ -57,6 +57,7 @@
- "--enable_workspace"
- "--build_tag_filters=-integration-test"
bazel: 7.x
+# NOTE: The Mac and Windows bazelinbazel jobs override parts of this config.
.common_bazelinbazel_config: &common_bazelinbazel_config
build_flags:
- "--build_tag_filters=integration-test"
@@ -503,6 +504,24 @@
<<: *common_bazelinbazel_config
name: "tests/integration bazel-in-bazel: Debian"
platform: debian11
+ # The bazelinbazel tests were disabled on Mac to save CI jobs slots, and
+ # have bitrotted a bit. For now, just run a subset of what we're most
+ # interested in.
+ integration_test_bazelinbazel_macos:
+ <<: *common_bazelinbazel_config
+ name: "tests/integration bazel-in-bazel: macOS (subset)"
+ platform: macos
+ build_targets: ["//tests/integration:local_toolchains_test_bazel_self"]
+ test_targets: ["//tests/integration:local_toolchains_test_bazel_self"]
+ # The bazelinbazel tests were disabled on Windows to save CI jobs slots, and
+ # have bitrotted a bit. For now, just run a subset of what we're most
+ # interested in.
+ integration_test_bazelinbazel_windows:
+ <<: *common_bazelinbazel_config
+ name: "tests/integration bazel-in-bazel: Windows (subset)"
+ platform: windows
+ build_targets: ["//tests/integration:local_toolchains_test_bazel_self"]
+ test_targets: ["//tests/integration:local_toolchains_test_bazel_self"]
integration_test_compile_pip_requirements_ubuntu:
<<: *reusable_build_test_all
diff --git a/.gitignore b/.gitignore
index 863b0e9..fb1b17e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -37,6 +37,8 @@
/bazel-genfiles
/bazel-out
/bazel-testlogs
+**/bazel-*
+
user.bazelrc
# vim swap files
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 37a8c1d..2dc235f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -106,6 +106,12 @@
* (toolchains) use "command -v" to find interpreter in `$PATH`
([#3150](https://github.com/bazel-contrib/rules_python/pull/3150)).
* (pypi) `bazel vendor` now works in `bzlmod` ({gh-issue}`3079`).
+* (toolchains) `local_runtime_repo` now works on Windows
+ ([#3055](https://github.com/bazel-contrib/rules_python/issues/3055)).
+* (toolchains) `local_runtime_repo` supports more types of Python
+ installations (Mac frameworks, missing dynamic libraries, and other
+ esoteric cases, see
+ [#3148](https://github.com/bazel-contrib/rules_python/pull/3148) for details).
{#v0-0-0-added}
### Added
diff --git a/python/private/get_local_runtime_info.py b/python/private/get_local_runtime_info.py
index c837135..ff3b0ae 100644
--- a/python/private/get_local_runtime_info.py
+++ b/python/private/get_local_runtime_info.py
@@ -12,10 +12,181 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""Returns information about the local Python runtime as JSON."""
+
import json
+import os
import sys
import sysconfig
+_IS_WINDOWS = sys.platform == "win32"
+_IS_DARWIN = sys.platform == "darwin"
+
+
+def _search_directories(get_config):
+ """Returns a list of library directories to search for shared libraries."""
+ # There's several types of libraries with different names and a plethora
+ # of settings, and many different config variables to check:
+ #
+ # LIBPL is used in python-config when shared library is not enabled:
+ # https://github.com/python/cpython/blob/v3.12.0/Misc/python-config.in#L63
+ #
+ # LIBDIR may also be the python directory with library files.
+ # https://stackoverflow.com/questions/47423246/get-pythons-lib-path
+ # See also: MULTIARCH
+ #
+ # On MacOS, the LDLIBRARY may be a relative path under /Library/Frameworks,
+ # such as "Python.framework/Versions/3.12/Python", not a file under the
+ # LIBDIR/LIBPL directory, so include PYTHONFRAMEWORKPREFIX.
+ lib_dirs = [get_config(x) for x in ("PYTHONFRAMEWORKPREFIX", "LIBPL", "LIBDIR")]
+
+ # On Debian, with multiarch enabled, prior to Python 3.10, `LIBDIR` didn't
+ # tell the location of the libs, just the base directory. The `MULTIARCH`
+ # sysconfig variable tells the subdirectory within it with the libs.
+ # See:
+ # https://wiki.debian.org/Python/MultiArch
+ # https://git.launchpad.net/ubuntu/+source/python3.12/tree/debian/changelog#n842
+ multiarch = get_config("MULTIARCH")
+ if multiarch:
+ for x in ("LIBPL", "LIBDIR"):
+ config_value = get_config(x)
+ if config_value and not config_value.endswith(multiarch):
+ lib_dirs.append(os.path.join(config_value, multiarch))
+
+ if _IS_WINDOWS:
+ # On Windows DLLs go in the same directory as the executable, while .lib
+ # files live in the lib/ or libs/ subdirectory.
+ lib_dirs.append(get_config("BINDIR"))
+ lib_dirs.append(os.path.join(os.path.dirname(sys.executable)))
+ lib_dirs.append(os.path.join(os.path.dirname(sys.executable), "lib"))
+ lib_dirs.append(os.path.join(os.path.dirname(sys.executable), "libs"))
+ elif not _IS_DARWIN:
+ # On most systems the executable is in a bin/ directory and the libraries
+ # are in a sibling lib/ directory.
+ lib_dirs.append(
+ os.path.join(os.path.dirname(os.path.dirname(sys.executable)), "lib")
+ )
+
+ # Dedup and remove empty values, keeping the order.
+ lib_dirs = [v for v in lib_dirs if v]
+ return {k: None for k in lib_dirs}.keys()
+
+
+def _search_library_names(get_config):
+ """Returns a list of library files to search for shared libraries."""
+ # Quoting configure.ac in the cpython code base:
+ # "INSTSONAME is the name of the shared library that will be use to install
+ # on the system - some systems like version suffix, others don't.""
+ #
+ # A typical INSTSONAME is 'libpython3.8.so.1.0' on Linux, or
+ # 'Python.framework/Versions/3.9/Python' on MacOS.
+ #
+ # A typical LDLIBRARY is 'libpythonX.Y.so' on Linux, or 'pythonXY.dll' on
+ # Windows, or 'Python.framework/Versions/3.9/Python' on MacOS.
+ #
+ # A typical LIBRARY is 'libpythonX.Y.a' on Linux.
+ lib_names = [
+ get_config(x)
+ for x in (
+ "LDLIBRARY",
+ "INSTSONAME",
+ "PY3LIBRARY",
+ "LIBRARY",
+ "DLLLIBRARY",
+ )
+ ]
+
+ # Set the prefix and suffix to construct the library name used for linking.
+ # The suffix and version are set here to the default values for the OS,
+ # since they are used below to construct "default" library names.
+ if _IS_DARWIN:
+ suffix = ".dylib"
+ prefix = "lib"
+ elif _IS_WINDOWS:
+ suffix = ".dll"
+ prefix = ""
+ else:
+ suffix = get_config("SHLIB_SUFFIX")
+ prefix = "lib"
+ if not suffix:
+ suffix = ".so"
+
+ version = get_config("VERSION")
+
+ # Ensure that the pythonXY.dll files are included in the search.
+ lib_names.append(f"{prefix}python{version}{suffix}")
+
+ # If there are ABIFLAGS, also add them to the python version lib search.
+ abiflags = get_config("ABIFLAGS") or get_config("abiflags") or ""
+ if abiflags:
+ lib_names.append(f"{prefix}python{version}{abiflags}{suffix}")
+
+ # Dedup and remove empty values, keeping the order.
+ lib_names = [v for v in lib_names if v]
+ return {k: None for k in lib_names}.keys()
+
+
+def _get_python_library_info():
+ """Returns a dictionary with the static and dynamic python libraries."""
+ config_vars = sysconfig.get_config_vars()
+
+ # VERSION is X.Y in Linux/macOS and XY in Windows. This is used to
+ # construct library paths such as python3.12, so ensure it exists.
+ if not config_vars.get("VERSION"):
+ if sys.platform == "win32":
+ config_vars["VERSION"] = f"{sys.version_info.major}{sys.version_info.minor}"
+ else:
+ config_vars["VERSION"] = (
+ f"{sys.version_info.major}.{sys.version_info.minor}"
+ )
+
+ search_directories = _search_directories(config_vars.get)
+ search_libnames = _search_library_names(config_vars.get)
+
+ def _add_if_exists(target, path):
+ if os.path.exists(path) or os.path.isdir(path):
+ target[path] = None
+
+ interface_libraries = {}
+ dynamic_libraries = {}
+ static_libraries = {}
+ for root_dir in search_directories:
+ for libname in search_libnames:
+ composed_path = os.path.join(root_dir, libname)
+ if libname.endswith(".a"):
+ _add_if_exists(static_libraries, composed_path)
+ continue
+
+ _add_if_exists(dynamic_libraries, composed_path)
+ if libname.endswith(".dll"):
+ # On windows a .lib file may be an "import library" or a static library.
+ # The file could be inspected to determine which it is; typically python
+ # is used as a shared library.
+ #
+ # On Windows, extensions should link with the pythonXY.lib interface
+ # libraries.
+ #
+ # See: https://docs.python.org/3/extending/windows.html
+ # https://learn.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-creation
+ _add_if_exists(
+ interface_libraries, os.path.join(root_dir, libname[:-3] + "lib")
+ )
+ elif libname.endswith(".so"):
+ # It's possible, though unlikely, that interface stubs (.ifso) exist.
+ _add_if_exists(
+ interface_libraries, os.path.join(root_dir, libname[:-2] + "ifso")
+ )
+
+ # When no libraries are found it's likely that the python interpreter is not
+ # configured to use shared or static libraries (minilinux). If this seems
+ # suspicious try running `uv tool run find_libpython --list-all -v`
+ return {
+ "dynamic_libraries": list(dynamic_libraries.keys()),
+ "static_libraries": list(static_libraries.keys()),
+ "interface_libraries": list(interface_libraries.keys()),
+ }
+
+
data = {
"major": sys.version_info.major,
"minor": sys.version_info.minor,
@@ -24,35 +195,5 @@
"implementation_name": sys.implementation.name,
"base_executable": sys._base_executable,
}
-
-config_vars = [
- # The libpythonX.Y.so file. Usually?
- # It might be a static archive (.a) file instead.
- "LDLIBRARY",
- # The directory with library files. Supposedly.
- # It's not entirely clear how to get the directory with libraries.
- # There's several types of libraries with different names and a plethora
- # of settings.
- # https://stackoverflow.com/questions/47423246/get-pythons-lib-path
- # For now, it seems LIBDIR has what is needed, so just use that.
- # See also: MULTIARCH
- "LIBDIR",
- # On Debian, with multiarch enabled, prior to Python 3.10, `LIBDIR` didn't
- # tell the location of the libs, just the base directory. The `MULTIARCH`
- # sysconfig variable tells the subdirectory within it with the libs.
- # See:
- # https://wiki.debian.org/Python/MultiArch
- # https://git.launchpad.net/ubuntu/+source/python3.12/tree/debian/changelog#n842
- "MULTIARCH",
- # The versioned libpythonX.Y.so.N file. Usually?
- # It might be a static archive (.a) file instead.
- "INSTSONAME",
- # The libpythonX.so file. Usually?
- # It might be a static archive (a.) file instead.
- "PY3LIBRARY",
- # The platform-specific filename suffix for library files.
- # Includes the dot, e.g. `.so`
- "SHLIB_SUFFIX",
-]
-data.update(zip(config_vars, sysconfig.get_config_vars(*config_vars)))
+data.update(_get_python_library_info())
print(json.dumps(data))
diff --git a/python/private/local_runtime_repo.bzl b/python/private/local_runtime_repo.bzl
index b8b7164..21bdfa6 100644
--- a/python/private/local_runtime_repo.bzl
+++ b/python/private/local_runtime_repo.bzl
@@ -31,27 +31,67 @@
define_local_runtime_toolchain_impl(
name = "local_runtime",
- lib_ext = "{lib_ext}",
major = "{major}",
minor = "{minor}",
micro = "{micro}",
interpreter_path = "{interpreter_path}",
+ interface_library = {interface_library},
+ libraries = {libraries},
implementation_name = "{implementation_name}",
os = "{os}",
)
"""
+def _norm_path(path):
+ """Returns a path using '/' separators and no trailing slash."""
+ path = path.replace("\\", "/")
+ if path[-1] == "/":
+ path = path[:-1]
+ return path
+
+def _symlink_first_library(rctx, logger, libraries):
+ """Symlinks the shared libraries into the lib/ directory.
+
+ Args:
+ rctx: A repository_ctx object
+ logger: A repo_utils.logger object
+ libraries: A list of static library paths to potentially symlink.
+ Returns:
+ A single library path linked by the action.
+ """
+ linked = None
+ for target in libraries:
+ origin = rctx.path(target)
+ if not origin.exists:
+ # The reported names don't always exist; it depends on the particulars
+ # of the runtime installation.
+ continue
+ if target.endswith("/Python"):
+ linked = "lib/{}.dylib".format(origin.basename)
+ else:
+ linked = "lib/{}".format(origin.basename)
+ logger.debug("Symlinking {} to {}".format(origin, linked))
+ repo_utils.watch(rctx, origin)
+ rctx.symlink(origin, linked)
+ break
+
+ return linked
+
def _local_runtime_repo_impl(rctx):
logger = repo_utils.logger(rctx)
on_failure = rctx.attr.on_failure
+ def _emit_log(msg):
+ if on_failure == "fail":
+ logger.fail(msg)
+ elif on_failure == "warn":
+ logger.warn(msg)
+ else:
+ logger.debug(msg)
+
result = _resolve_interpreter_path(rctx)
if not result.resolved_path:
- if on_failure == "fail":
- fail("interpreter not found: {}".format(result.describe_failure()))
-
- if on_failure == "warn":
- logger.warn(lambda: "interpreter not found: {}".format(result.describe_failure()))
+ _emit_log(lambda: "interpreter not found: {}".format(result.describe_failure()))
# else, on_failure must be skip
rctx.file("BUILD.bazel", _expand_incompatible_template())
@@ -72,10 +112,7 @@
logger = logger,
)
if exec_result.return_code != 0:
- if on_failure == "fail":
- fail("GetPythonInfo failed: {}".format(exec_result.describe_failure()))
- if on_failure == "warn":
- logger.warn(lambda: "GetPythonInfo failed: {}".format(exec_result.describe_failure()))
+ _emit_log(lambda: "GetPythonInfo failed: {}".format(exec_result.describe_failure()))
# else, on_failure must be skip
rctx.file("BUILD.bazel", _expand_incompatible_template())
@@ -112,53 +149,37 @@
# The cc_library.includes values have to be non-absolute paths, otherwise
# the toolchain will give an error. Work around this error by making them
# appear as part of this repo.
- rctx.symlink(info["include"], "include")
+ rctx.symlink(include_path, "include")
- shared_lib_names = [
- info["PY3LIBRARY"],
- info["LDLIBRARY"],
- info["INSTSONAME"],
- ]
-
- # In some cases, the value may be empty. Not clear why.
- shared_lib_names = [v for v in shared_lib_names if v]
-
- # In some cases, the same value is returned for multiple keys. Not clear why.
- shared_lib_names = {v: None for v in shared_lib_names}.keys()
- shared_lib_dir = info["LIBDIR"]
- multiarch = info["MULTIARCH"]
-
- # The specific files are symlinked instead of the whole directory
- # because it can point to a directory that has more than just
- # the Python runtime shared libraries, e.g. /usr/lib, or a Python
- # specific directory with pip-installed shared libraries.
rctx.report_progress("Symlinking external Python shared libraries")
- for name in shared_lib_names:
- origin = rctx.path("{}/{}".format(shared_lib_dir, name))
+ interface_library = _symlink_first_library(rctx, logger, info["interface_libraries"])
+ shared_library = _symlink_first_library(rctx, logger, info["dynamic_libraries"])
+ static_library = _symlink_first_library(rctx, logger, info["static_libraries"])
- # If the origin doesn't exist, try the multiarch location, in case
- # it's an older Python / Debian release.
- if not origin.exists and multiarch:
- origin = rctx.path("{}/{}/{}".format(shared_lib_dir, multiarch, name))
+ libraries = []
+ if shared_library:
+ libraries.append(shared_library)
+ elif static_library:
+ libraries.append(static_library)
+ else:
+ logger.warn("No external python libraries found.")
- # The reported names don't always exist; it depends on the particulars
- # of the runtime installation.
- if origin.exists:
- repo_utils.watch(rctx, origin)
- rctx.symlink(origin, "lib/" + name)
+ build_bazel = _TOOLCHAIN_IMPL_TEMPLATE.format(
+ major = info["major"],
+ minor = info["minor"],
+ micro = info["micro"],
+ interpreter_path = _norm_path(interpreter_path),
+ interface_library = repr(interface_library),
+ libraries = repr(libraries),
+ implementation_name = info["implementation_name"],
+ os = "@platforms//os:{}".format(repo_utils.get_platforms_os_name(rctx)),
+ )
+ logger.debug(lambda: "BUILD.bazel\n{}".format(build_bazel))
rctx.file("WORKSPACE", "")
rctx.file("MODULE.bazel", "")
rctx.file("REPO.bazel", "")
- rctx.file("BUILD.bazel", _TOOLCHAIN_IMPL_TEMPLATE.format(
- major = info["major"],
- minor = info["minor"],
- micro = info["micro"],
- interpreter_path = interpreter_path,
- lib_ext = info["SHLIB_SUFFIX"],
- implementation_name = info["implementation_name"],
- os = "@platforms//os:{}".format(repo_utils.get_platforms_os_name(rctx)),
- ))
+ rctx.file("BUILD.bazel", build_bazel)
local_runtime_repo = repository_rule(
implementation = _local_runtime_repo_impl,
@@ -218,7 +239,8 @@
return _TOOLCHAIN_IMPL_TEMPLATE.format(
interpreter_path = "/incompatible",
implementation_name = "incompatible",
- lib_ext = "incompatible",
+ interface_library = "None",
+ libraries = "[]",
major = "0",
minor = "0",
micro = "0",
diff --git a/python/private/local_runtime_repo_setup.bzl b/python/private/local_runtime_repo_setup.bzl
index 37eab59..5d3a781 100644
--- a/python/private/local_runtime_repo_setup.bzl
+++ b/python/private/local_runtime_repo_setup.bzl
@@ -15,6 +15,7 @@
"""Setup code called by the code generated by `local_runtime_repo`."""
load("@bazel_skylib//lib:selects.bzl", "selects")
+load("@rules_cc//cc:cc_import.bzl", "cc_import")
load("@rules_cc//cc:cc_library.bzl", "cc_library")
load("@rules_python//python:py_runtime.bzl", "py_runtime")
load("@rules_python//python:py_runtime_pair.bzl", "py_runtime_pair")
@@ -25,11 +26,12 @@
def define_local_runtime_toolchain_impl(
name,
- lib_ext,
major,
minor,
micro,
interpreter_path,
+ interface_library,
+ libraries,
implementation_name,
os):
"""Defines a toolchain implementation for a local Python runtime.
@@ -45,11 +47,14 @@
Args:
name: `str` Only present to satisfy tooling
- lib_ext: `str` The file extension for the `libpython` shared libraries
major: `str` The major Python version, e.g. `3` of `3.9.1`.
minor: `str` The minor Python version, e.g. `9` of `3.9.1`.
micro: `str` The micro Python version, e.g. "1" of `3.9.1`.
interpreter_path: `str` Absolute path to the interpreter.
+ interface_library: `str` Path to the interface library.
+ e.g. "lib/python312.lib"
+ libraries: `list[str]` Path[s] to the python libraries.
+ e.g. ["lib/python312.dll"] or ["lib/python312.so"]
implementation_name: `str` The implementation name, as returned by
`sys.implementation.name`.
os: `str` A label to the OS constraint (e.g. `@platforms//os:linux`) for
@@ -58,30 +63,36 @@
major_minor = "{}.{}".format(major, minor)
major_minor_micro = "{}.{}".format(major_minor, micro)
+ # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib
+ # See https://docs.python.org/3/extending/windows.html
+ # However not all python installations (such as manylinux) include shared or static libraries,
+ # so only create the import library when interface_library is set.
+ import_deps = []
+ if interface_library:
+ cc_import(
+ name = "_python_interface_library",
+ interface_library = interface_library,
+ system_provided = 1,
+ )
+ import_deps = [":_python_interface_library"]
+
cc_library(
name = "_python_headers",
# NOTE: Keep in sync with watch_tree() called in local_runtime_repo
srcs = native.glob(
- ["include/**/*.h"],
- # A Python install may not have C headers
- allow_empty = True,
+ include = ["include/**/*.h"],
+ exclude = ["include/numpy/**"], # numpy headers are handled separately
+ allow_empty = True, # A Python install may not have C headers
),
+ deps = import_deps,
includes = ["include"],
)
cc_library(
name = "_libpython",
- # Don't use a recursive glob because the lib/ directory usually contains
- # a subdirectory of the stdlib -- lots of unrelated files
- srcs = native.glob(
- [
- "lib/*{}".format(lib_ext), # Match libpython*.so
- "lib/*{}*".format(lib_ext), # Also match libpython*.so.1.0
- ],
- # A Python install may not have shared libraries.
- allow_empty = True,
- ),
hdrs = [":_python_headers"],
+ srcs = libraries,
+ deps = [],
)
py_runtime(
diff --git a/tests/integration/BUILD.bazel b/tests/integration/BUILD.bazel
index d178e0f..df7fe15 100644
--- a/tests/integration/BUILD.bazel
+++ b/tests/integration/BUILD.bazel
@@ -96,6 +96,17 @@
)
rules_python_integration_test(
+ name = "local_toolchains_workspace_test",
+ bazel_versions = [
+ version
+ for version in bazel_binaries.versions.all
+ if not version.startswith("6.")
+ ],
+ bzlmod = False,
+ workspace_path = "local_toolchains",
+)
+
+rules_python_integration_test(
name = "pip_parse_test",
)
diff --git a/tests/integration/local_toolchains/BUILD.bazel b/tests/integration/local_toolchains/BUILD.bazel
index 6b73118..a0cb2b1 100644
--- a/tests/integration/local_toolchains/BUILD.bazel
+++ b/tests/integration/local_toolchains/BUILD.bazel
@@ -13,7 +13,9 @@
# limitations under the License.
load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
load("@rules_python//python:py_test.bzl", "py_test")
+load(":py_extension.bzl", "py_extension")
py_test(
name = "test",
@@ -35,3 +37,30 @@
name = "py",
build_setting_default = "",
)
+
+# Build rules to generate a python extension.
+cc_library(
+ name = "echo_ext_cc",
+ testonly = True,
+ srcs = ["echo_ext.cc"],
+ deps = [
+ "@rules_python//python/cc:current_py_cc_headers",
+ ],
+ alwayslink = True,
+)
+
+py_extension(
+ name = "echo_ext",
+ testonly = True,
+ copts = select({
+ "@rules_cc//cc/compiler:msvc-cl": [],
+ "//conditions:default": ["-fvisibility=hidden"],
+ }),
+ deps = [":echo_ext_cc"],
+)
+
+py_test(
+ name = "echo_test",
+ srcs = ["echo_test.py"],
+ deps = [":echo_ext"],
+)
diff --git a/tests/integration/local_toolchains/MODULE.bazel b/tests/integration/local_toolchains/MODULE.bazel
index 6c06909..45afaaf 100644
--- a/tests/integration/local_toolchains/MODULE.bazel
+++ b/tests/integration/local_toolchains/MODULE.bazel
@@ -16,6 +16,7 @@
bazel_dep(name = "rules_python", version = "0.0.0")
bazel_dep(name = "bazel_skylib", version = "1.7.1")
bazel_dep(name = "platforms", version = "0.0.11")
+bazel_dep(name = "rules_cc", version = "0.0.16")
local_path_override(
module_name = "rules_python",
diff --git a/tests/integration/local_toolchains/WORKSPACE b/tests/integration/local_toolchains/WORKSPACE
index e69de29..480cd27 100644
--- a/tests/integration/local_toolchains/WORKSPACE
+++ b/tests/integration/local_toolchains/WORKSPACE
@@ -0,0 +1,31 @@
+workspace(
+ name = "module_under_test",
+)
+
+local_repository(
+ name = "rules_python",
+ path = "../../..",
+)
+
+load("@rules_python//python:repositories.bzl", "py_repositories")
+
+py_repositories()
+
+load("@rules_python//python/local_toolchains:repos.bzl", "local_runtime_repo", "local_runtime_toolchains_repo")
+
+# Step 1: Define the python runtime.
+local_runtime_repo(
+ name = "local_python3",
+ interpreter_path = "python3",
+ on_failure = "fail",
+ # or interpreter_path = "C:\\path\\to\\python.exe"
+)
+
+# Step 2: Create toolchains for the runtimes
+local_runtime_toolchains_repo(
+ name = "local_toolchains",
+ runtimes = ["local_python3"],
+)
+
+# Step 3: Register the toolchains
+register_toolchains("@local_toolchains//:all")
diff --git a/tests/integration/local_toolchains/echo_ext.cc b/tests/integration/local_toolchains/echo_ext.cc
new file mode 100644
index 0000000..367d1a1
--- /dev/null
+++ b/tests/integration/local_toolchains/echo_ext.cc
@@ -0,0 +1,21 @@
+#include <Python.h>
+
+static PyObject *echoArgs(PyObject *self, PyObject *args) { return args; }
+
+static PyMethodDef echo_methods[] = {
+ { "echo", echoArgs, METH_VARARGS, "Returns a tuple of the input args" },
+ { NULL, NULL, 0, NULL },
+};
+
+extern "C" {
+
+PyMODINIT_FUNC PyInit_echo_ext(void) {
+ static struct PyModuleDef echo_module_def = {
+ // Module definition
+ PyModuleDef_HEAD_INIT, "echo_ext", "'echo_ext' module", -1, echo_methods
+ };
+
+ return PyModule_Create(&echo_module_def);
+}
+
+} // extern "C"
diff --git a/tests/integration/local_toolchains/echo_test.py b/tests/integration/local_toolchains/echo_test.py
new file mode 100644
index 0000000..4cc31ff
--- /dev/null
+++ b/tests/integration/local_toolchains/echo_test.py
@@ -0,0 +1,9 @@
+import unittest
+
+import echo_ext
+
+
+class ExtensionTest(unittest.TestCase):
+
+ def test_echo_extension(self):
+ self.assertEqual(echo_ext.echo(42, "str"), tuple(42, "str"))
diff --git a/tests/integration/local_toolchains/py_extension.bzl b/tests/integration/local_toolchains/py_extension.bzl
new file mode 100644
index 0000000..5d37fd7
--- /dev/null
+++ b/tests/integration/local_toolchains/py_extension.bzl
@@ -0,0 +1,154 @@
+# Copyright 2025 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Macro to build a python C/C++ extension.
+
+There are variants of py_extension in many other projects, such as:
+* https://github.com/protocolbuffers/protobuf/tree/main/python/py_extension.bzl
+* https://github.com/google/riegeli/blob/master/python/riegeli/py_extension.bzl
+* https://github.com/pybind/pybind11_bazel/blob/master/build_defs.bzl
+
+The issue for a generic verion is:
+* https://github.com/bazel-contrib/rules_python/issues/824
+"""
+
+load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
+load("@rules_cc//cc:cc_binary.bzl", "cc_binary")
+load("@rules_python//python:defs.bzl", "py_library")
+
+def py_extension(
+ *,
+ name,
+ deps = None,
+ linkopts = None,
+ imports = None,
+ visibility = None,
+ **kwargs):
+ """Creates a Python module implemented in C++.
+
+ A Python extension has 2 essential parts:
+ 1. An internal shared object / pyd package for the extension, `name.pyd`/`name.so`
+ 2. The py_library target for the extension.`
+
+ Python modules can depend on a py_extension.
+
+ Args:
+ name: `str`. Name for this target. This is typically the module name.
+ deps: `list`. Required. C++ libraries to link into the module.
+ linkopts: `list`. Linking options for the shared library.
+ imports: `list`. Additional imports for the py_library rule.
+ visibility: `str`. Visibility for target.
+ **kwargs: Additional options for the cc_library rule.
+ """
+ if not name:
+ fail("py_extension requires a name")
+ if not deps:
+ fail("py_extension requires a non-empty deps attribute")
+ if "linkshared" in kwargs:
+ fail("py_extension attribute linkshared not allowed")
+
+ if not linkopts:
+ linkopts = []
+
+ testonly = kwargs.get("testonly")
+ tags = kwargs.pop("tags", [])
+
+ cc_binary_so_name = name + ".so"
+ cc_binary_dll_name = name + ".dll"
+ cc_binary_pyd_name = name + ".pyd"
+ linker_script_name = name + ".lds"
+ linker_script_name_rule = name + "_lds"
+ shared_objects_name = name + "__shared_objects"
+
+ # On Unix, restrict symbol visibility.
+ exported_symbol = "PyInit_" + name
+
+ # Generate linker script used on non-macOS unix platforms.
+ native.genrule(
+ name = linker_script_name_rule,
+ outs = [linker_script_name],
+ cmd = "\n".join([
+ "cat <<'EOF' >$@",
+ "{",
+ " global: " + exported_symbol + ";",
+ " local: *;",
+ "};",
+ "EOF",
+ ]),
+ )
+
+ for cc_binary_name in [cc_binary_dll_name, cc_binary_so_name]:
+ cur_linkopts = linkopts
+ cur_deps = deps
+ if cc_binary_name == cc_binary_so_name:
+ cur_linkopts = linkopts + select({
+ "@platforms//os:macos": [
+ # Avoid undefined symbol errors for CPython symbols that
+ # will be resolved at runtime.
+ "-undefined",
+ "dynamic_lookup",
+ # On macOS, the linker does not support version scripts. Use
+ # the `-exported_symbol` option instead to restrict symbol
+ # visibility.
+ "-Wl,-exported_symbol",
+ # On macOS, the symbol starts with an underscore.
+ "-Wl,_" + exported_symbol,
+ ],
+ # On non-macOS unix, use a version script to restrict symbol
+ # visibility.
+ "//conditions:default": [
+ "-Wl,--version-script",
+ "-Wl,$(location :" + linker_script_name + ")",
+ ],
+ })
+ cur_deps = cur_deps + select({
+ "@platforms//os:macos": [],
+ "//conditions:default": [linker_script_name],
+ })
+
+ cc_binary(
+ name = cc_binary_name,
+ linkshared = True,
+ visibility = ["//visibility:private"],
+ deps = cur_deps,
+ tags = tags + ["manual"],
+ linkopts = cur_linkopts,
+ **kwargs
+ )
+
+ copy_file(
+ name = cc_binary_pyd_name + "__pyd_copy",
+ src = ":" + cc_binary_dll_name,
+ out = cc_binary_pyd_name,
+ visibility = visibility,
+ tags = ["manual"],
+ testonly = testonly,
+ )
+
+ native.filegroup(
+ name = shared_objects_name,
+ data = select({
+ "@platforms//os:windows": [":" + cc_binary_pyd_name],
+ "//conditions:default": [":" + cc_binary_so_name],
+ }),
+ testonly = testonly,
+ )
+ py_library(
+ name = name,
+ data = [":" + shared_objects_name],
+ imports = imports,
+ tags = tags,
+ testonly = testonly,
+ visibility = visibility,
+ )
diff --git a/tests/integration/local_toolchains/test.py b/tests/integration/local_toolchains/test.py
index 8e37fff..0a0d6be 100644
--- a/tests/integration/local_toolchains/test.py
+++ b/tests/integration/local_toolchains/test.py
@@ -20,18 +20,20 @@
# things like pyenv: they install a shim that re-execs python.
# The shim is e.g. /home/user/.pyenv/shims/python3, which then
# runs e.g. /usr/bin/python3
- with tempfile.NamedTemporaryFile(suffix="_info.py", mode="w+") as f:
- f.write(
+ with tempfile.TemporaryDirectory() as temp_dir:
+ file_path = os.path.join(temp_dir, "info.py")
+ with open(file_path, 'w') as f:
+ f.write(
"""
import sys
print(sys.executable)
print(sys._base_executable)
"""
- )
- f.flush()
+ )
+ f.flush()
output_lines = (
subprocess.check_output(
- [shell_path, f.name],
+ [shell_path, file_path],
text=True,
)
.strip()
