Initial commit to add embedding support via pybind11_embed target (#15)
* Initial commit to add embedding support via pybind11_embed target
Currently, only MacOS 10.13.3 tested and Python 3 supported
//python_configure.bzl:
_find_python_config() function to find the local Python's python-config script
_get_embed_flags() function to identify the copts and linkopts flags required to embed the Python interpreter.
//py/BUILD.tpl: cc_library(name= "python_embed") combines python headers with embed flags.
//pybind11.BUILD: pybind11_embed target that depends on python_embed.
To embed the python interpreter, add "@pybind11//:pybind11_embed" as a dependency
to your cc_binary target that embeds Python.
* Removed unused found_config variable
* Added pybind11_embed instructions
* Updated README to reflect Linux support
* readme formatting fixes
* Improved python_configure.bzl organizationdiff --git a/README.md b/README.md
index 0091692..9ee536a 100644
--- a/README.md
+++ b/README.md
@@ -14,6 +14,23 @@
To test a `pybind_extension`, the most common approach is to write the test in
python and use the standard `py_test` build rule.
+Provided targets:
+
+ - `@pybind11//:pybind11_embed`: Automatically adds required build flags to
+ embed Python.
+ Add as a dependency to your `cc_binary`.
+
+ `@pybind11//:pybind11_embed` currently supports Python 3 MacOS/Ubuntu/Debian
+ environments:
+ - pyenv
+ - pipenv
+ - virtualenv
+
+ If `pybind11_embed` doesn't work with your embedded Python project, add
+ `@pybind11` as a dependency to your `cc_binary` and [follow the instructions
+ for manually retrieving the build flags](https://docs.python.org/3/extending/embedding.html#embedding-python-in-c).
+
+
## Installation
In your `WORKSPACE` file:
diff --git a/py/BUILD.tpl b/py/BUILD.tpl
index 8bd1374..44fbfcd 100644
--- a/py/BUILD.tpl
+++ b/py/BUILD.tpl
@@ -53,6 +53,18 @@
)
cc_library(
+ name = "python_embed",
+ hdrs = [":python_include"],
+ deps = select({
+ ":windows": [":python_lib"],
+ "//conditions:default": [],
+ }),
+ includes = ["python_include"],
+ linkopts = ["%{PYTHON_EMBED_LINKOPTS}"],
+ copts = ["%{PYTHON_EMBED_COPTS}"],
+)
+
+cc_library(
name = "numpy_headers",
hdrs = [":numpy_include"],
includes = ["numpy_include"],
diff --git a/pybind11.BUILD b/pybind11.BUILD
index aa46e91..4d7d77a 100644
--- a/pybind11.BUILD
+++ b/pybind11.BUILD
@@ -36,6 +36,17 @@
deps = ["@local_config_python//:python_headers"],
)
+cc_library(
+ name = "pybind11_embed",
+ hdrs = glob(
+ INCLUDES,
+ exclude = EXCLUDES,
+ ),
+ copts = OPTIONS,
+ includes = ["include"],
+ deps = ["@local_config_python//:python_embed"],
+)
+
config_setting(
name = "osx",
constraint_values = ["@platforms//os:osx"],
diff --git a/python_configure.bzl b/python_configure.bzl
index ac41ef8..795fa71 100644
--- a/python_configure.bzl
+++ b/python_configure.bzl
@@ -270,6 +270,50 @@
)
return result.stdout.splitlines()[0]
+def _find_python_config(repository_ctx, python_bin):
+ """Searches for python-config in the Python bin directory through the Python environment symlinks
+
+ Returns a string path to python-config, or None if not found
+ """
+ bin_dir = repository_ctx.path(python_bin).dirname
+
+ for i in bin_dir.readdir():
+ if "python-config" == i.basename:
+ return str(bin_dir.get_child("python-config"))
+
+ symlink_base_dir = repository_ctx.path(python_bin).realpath.dirname
+ for i in symlink_base_dir.readdir():
+ if "python-config" == i.basename:
+ return str(symlink_base_dir.get_child("python-config"))
+
+ return None
+
+def _get_embed_flags(repository_ctx, python_config):
+ """Identifies compiler and linker flags output by python-config required to embed Python
+
+ Returns a tuple containing the copts and linkopts, or empty strings if unsuccessful.
+ """
+ err_cmd = python_config + " --help"
+ comp_cmd = python_config + " --cflags"
+
+ # --embed is an undocumented python >=3.8 flag.
+ # See https://github.com/python/cpython/pull/13500
+ link_cmd = python_config + " --ldflags --embed"
+
+ err = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", err_cmd]).stdout.strip("\n")
+ compiler_flags = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", comp_cmd]).stdout.strip("\n")
+ linker_flags = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", link_cmd]).stdout.strip("\n")
+
+ if linker_flags == err:
+ # Try again without --embed
+ link_cmd = python_config + " --ldflags"
+ linker_flags = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", link_cmd]).stdout.strip("\n")
+
+ if linker_flags == err or compiler_flags == err:
+ return "", ""
+
+ return compiler_flags, linker_flags
+
def _create_local_python_repository(repository_ctx):
"""Creates the repository containing files set up to build with Python."""
python_bin = _get_python_bin(repository_ctx)
@@ -283,6 +327,15 @@
"python_include",
"python_include",
)
+
+ # To embed python in C++, we need the linker and compiler flags required to embed the Python interpreter
+ # See https://docs.python.org/3/extending/embedding.html#embedding-python-in-c
+ python_config = _find_python_config(repository_ctx, python_bin)
+ python_embed_copts = ""
+ python_embed_linkopts = ""
+ if python_config:
+ python_embed_copts, python_embed_linkopts = _get_embed_flags(repository_ctx, python_config)
+
python_import_lib_genrule = ""
# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib
@@ -303,6 +356,8 @@
"%{PYTHON_BIN_PATH}": python_bin,
"%{PYTHON_INCLUDE_GENRULE}": python_include_rule,
"%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule,
+ "%{PYTHON_EMBED_COPTS}": python_embed_copts,
+ "%{PYTHON_EMBED_LINKOPTS}": python_embed_linkopts,
})
def _create_remote_python_repository(repository_ctx, remote_config_repo):
@@ -314,6 +369,7 @@
"""Implementation of the python_autoconf repository rule."""
_create_local_python_repository(repository_ctx)
+# Configure Activated Python Environment
python_configure = repository_rule(
implementation = _python_autoconf_impl,
environ = [
