fix(venv): symlink shared libraries directly (#3331)
It seems `$ORIGIN` resolves prior to symlink resolution. This makes it
resolve
differently depending on if the directory or file itself is symlinked.
To fix, special case shared libraries and have them symlinked directly.
Since an
explicit file is the target, `VenvSymlinkEntry.link_to_file` is added to
hold the
File object that will be linked to.
An unfortunate side-effect of this logic is any package with `lib*.so`
files
will be more expensive to build (depset flattened at analysis time, more
files
symlinked), but it beats not working at all. Optimizing that can be done
in
another change.
Tests added to generate libraries that look like what something from
PyPI
does. Manually verified a case using jax and jax plugins.
Fixes https://github.com/bazel-contrib/rules_python/issues/3228
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 04e01a9..7782454 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -102,6 +102,9 @@
* (venvs) {obj}`--venvs_site_packages=yes` no longer errors when packages with
overlapping files or directories are used together.
([#3204](https://github.com/bazel-contrib/rules_python/issues/3204)).
+* (venvs) {obj}`--venvs_site_packages=yes` works for packages that dynamically
+ link to shared libraries
+ ([#3228](https://github.com/bazel-contrib/rules_python/issues/3228)).
* (uv) {obj}`//python/uv:lock.bzl%lock` now works with a local platform
runtime.
* (toolchains) WORKSPACE builds now correctly register musl and freethreaded
diff --git a/docs/pyproject.toml b/docs/pyproject.toml
index 9a089df..f4bbbaf 100644
--- a/docs/pyproject.toml
+++ b/docs/pyproject.toml
@@ -13,5 +13,7 @@
"absl-py",
"typing-extensions",
"sphinx-reredirects",
- "pefile"
+ "pefile",
+ "pyelftools",
+ "macholib",
]
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 290113c..c5a5fea 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -14,6 +14,10 @@
--hash=sha256:c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e \
--hash=sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b
# via sphinx
+altgraph==0.17.4 \
+ --hash=sha256:1b5afbb98f6c4dcadb2e2ae6ab9fa994bbb8c1d75f4fa96d340f9437ae454406 \
+ --hash=sha256:642743b4750de17e655e6711601b077bc6598dbfa3ba5fa2b2a35ce12b508dff
+ # via macholib
astroid==3.3.11 \
--hash=sha256:1e5a5011af2920c7c67a53f65d536d65bfa7116feeaf2354d8b94f29573bb0ce \
--hash=sha256:54c760ae8322ece1abd213057c4b5bba7c49818853fc901ef09719a60dbf9dec
@@ -111,9 +115,9 @@
--hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \
--hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6
# via sphinx
-docutils==0.22.2 \
- --hash=sha256:9fdb771707c8784c8f2728b67cb2c691305933d68137ef95a75db5f4dfbc213d \
- --hash=sha256:b0e98d679283fc3bb0ead8a5da7f501baa632654e7056e9c5846842213d674d8
+docutils==0.21.2 \
+ --hash=sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f \
+ --hash=sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2
# via
# myst-parser
# sphinx
@@ -137,9 +141,13 @@
# myst-parser
# readthedocs-sphinx-ext
# sphinx
-markdown-it-py==4.0.0 \
- --hash=sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147 \
- --hash=sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3
+macholib==1.16.3 \
+ --hash=sha256:07ae9e15e8e4cd9a788013d81f5908b3609aa76f9b1421bae9c4d7606ec86a30 \
+ --hash=sha256:0e315d7583d38b8c77e815b1ecbdbf504a8258d8b3e17b61165c6feb60d18f2c
+ # via rules-python-docs (docs/pyproject.toml)
+markdown-it-py==3.0.0 \
+ --hash=sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1 \
+ --hash=sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb
# via
# mdit-py-plugins
# myst-parser
@@ -264,6 +272,10 @@
--hash=sha256:3ff6c5d8b43e8c37bb6e6dd5085658d658a7a0bdcd20b6a07b1fcfc1c4e9d632 \
--hash=sha256:76f8b485dcd3b1bb8166f1128d395fa3d87af26360c2358fb75b80019b957c6f
# via rules-python-docs (docs/pyproject.toml)
+pyelftools==0.32 \
+ --hash=sha256:013df952a006db5e138b1edf6d8a68ecc50630adbd0d83a2d41e7f846163d738 \
+ --hash=sha256:6de90ee7b8263e740c8715a925382d4099b354f29ac48ea40d840cf7aa14ace5
+ # via rules-python-docs (docs/pyproject.toml)
pygments==2.19.2 \
--hash=sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887 \
--hash=sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b
@@ -432,39 +444,49 @@
--hash=sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331 \
--hash=sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d
# via sphinx
-tomli==2.2.1 ; python_full_version < '3.11' \
- --hash=sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6 \
- --hash=sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd \
- --hash=sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c \
- --hash=sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b \
- --hash=sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8 \
- --hash=sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6 \
- --hash=sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77 \
- --hash=sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff \
- --hash=sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea \
- --hash=sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192 \
- --hash=sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249 \
- --hash=sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee \
- --hash=sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4 \
- --hash=sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98 \
- --hash=sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8 \
- --hash=sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4 \
- --hash=sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281 \
- --hash=sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744 \
- --hash=sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69 \
- --hash=sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13 \
- --hash=sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140 \
- --hash=sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e \
- --hash=sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e \
- --hash=sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc \
- --hash=sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff \
- --hash=sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec \
- --hash=sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2 \
- --hash=sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222 \
- --hash=sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106 \
- --hash=sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272 \
- --hash=sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a \
- --hash=sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7
+tomli==2.3.0 ; python_full_version < '3.11' \
+ --hash=sha256:00b5f5d95bbfc7d12f91ad8c593a1659b6387b43f054104cda404be6bda62456 \
+ --hash=sha256:0a154a9ae14bfcf5d8917a59b51ffd5a3ac1fd149b71b47a3a104ca4edcfa845 \
+ --hash=sha256:0c95ca56fbe89e065c6ead5b593ee64b84a26fca063b5d71a1122bf26e533999 \
+ --hash=sha256:0eea8cc5c5e9f89c9b90c4896a8deefc74f518db5927d0e0e8d4a80953d774d0 \
+ --hash=sha256:1cb4ed918939151a03f33d4242ccd0aa5f11b3547d0cf30f7c74a408a5b99878 \
+ --hash=sha256:4021923f97266babc6ccab9f5068642a0095faa0a51a246a6a02fccbb3514eaf \
+ --hash=sha256:4c2ef0244c75aba9355561272009d934953817c49f47d768070c3c94355c2aa3 \
+ --hash=sha256:4dc4ce8483a5d429ab602f111a93a6ab1ed425eae3122032db7e9acf449451be \
+ --hash=sha256:4f195fe57ecceac95a66a75ac24d9d5fbc98ef0962e09b2eddec5d39375aae52 \
+ --hash=sha256:5192f562738228945d7b13d4930baffda67b69425a7f0da96d360b0a3888136b \
+ --hash=sha256:5e01decd096b1530d97d5d85cb4dff4af2d8347bd35686654a004f8dea20fc67 \
+ --hash=sha256:64be704a875d2a59753d80ee8a533c3fe183e3f06807ff7dc2232938ccb01549 \
+ --hash=sha256:70a251f8d4ba2d9ac2542eecf008b3c8a9fc5c3f9f02c56a9d7952612be2fdba \
+ --hash=sha256:73ee0b47d4dad1c5e996e3cd33b8a76a50167ae5f96a2607cbe8cc773506ab22 \
+ --hash=sha256:74bf8464ff93e413514fefd2be591c3b0b23231a77f901db1eb30d6f712fc42c \
+ --hash=sha256:792262b94d5d0a466afb5bc63c7daa9d75520110971ee269152083270998316f \
+ --hash=sha256:7b0882799624980785240ab732537fcfc372601015c00f7fc367c55308c186f6 \
+ --hash=sha256:883b1c0d6398a6a9d29b508c331fa56adbcdff647f6ace4dfca0f50e90dfd0ba \
+ --hash=sha256:88bd15eb972f3664f5ed4b57c1634a97153b4bac4479dcb6a495f41921eb7f45 \
+ --hash=sha256:8a35dd0e643bb2610f156cca8db95d213a90015c11fee76c946aa62b7ae7e02f \
+ --hash=sha256:940d56ee0410fa17ee1f12b817b37a4d4e4dc4d27340863cc67236c74f582e77 \
+ --hash=sha256:97d5eec30149fd3294270e889b4234023f2c69747e555a27bd708828353ab606 \
+ --hash=sha256:a0e285d2649b78c0d9027570d4da3425bdb49830a6156121360b3f8511ea3441 \
+ --hash=sha256:a1f7f282fe248311650081faafa5f4732bdbfef5d45fe3f2e702fbc6f2d496e0 \
+ --hash=sha256:a4ea38c40145a357d513bffad0ed869f13c1773716cf71ccaa83b0fa0cc4e42f \
+ --hash=sha256:a56212bdcce682e56b0aaf79e869ba5d15a6163f88d5451cbde388d48b13f530 \
+ --hash=sha256:ad805ea85eda330dbad64c7ea7a4556259665bdf9d2672f5dccc740eb9d3ca05 \
+ --hash=sha256:b273fcbd7fc64dc3600c098e39136522650c49bca95df2d11cf3b626422392c8 \
+ --hash=sha256:b5870b50c9db823c595983571d1296a6ff3e1b88f734a4c8f6fc6188397de005 \
+ --hash=sha256:b74a0e59ec5d15127acdabd75ea17726ac4c5178ae51b85bfe39c4f8a278e879 \
+ --hash=sha256:be71c93a63d738597996be9528f4abe628d1adf5e6eb11607bc8fe1a510b5dae \
+ --hash=sha256:c22a8bf253bacc0cf11f35ad9808b6cb75ada2631c2d97c971122583b129afbc \
+ --hash=sha256:c4665508bcbac83a31ff8ab08f424b665200c0e1e645d2bd9ab3d3e557b6185b \
+ --hash=sha256:c5f3ffd1e098dfc032d4d3af5c0ac64f6d286d98bc148698356847b80fa4de1b \
+ --hash=sha256:cebc6fe843e0733ee827a282aca4999b596241195f43b4cc371d64fc6639da9e \
+ --hash=sha256:d1381caf13ab9f300e30dd8feadb3de072aeb86f1d34a8569453ff32a7dea4bf \
+ --hash=sha256:d7d86942e56ded512a594786a5ba0a5e521d02529b3826e7761a05138341a2ac \
+ --hash=sha256:e31d432427dcbf4d86958c184b9bfd1e96b5b71f8eb17e6d02531f434fd335b8 \
+ --hash=sha256:e95b1af3c5b07d9e643909b5abbec77cd9f1217e6d0bca72b0234736b9fb1f1b \
+ --hash=sha256:f85209946d1fe94416debbb88d00eb92ce9cd5266775424ff81bc959e001acaf \
+ --hash=sha256:feb0dacc61170ed7ab602d3d972a58f14ee3ee60494292d384649a3dc38ef463 \
+ --hash=sha256:ff72b71b5d10d22ecb084d345fc26f42b5143c5533db5e2eaba7d2d335358876
# via
# sphinx
# sphinx-autodoc2
diff --git a/python/private/py_info.bzl b/python/private/py_info.bzl
index 9318347..95b739d 100644
--- a/python/private/py_info.bzl
+++ b/python/private/py_info.bzl
@@ -47,12 +47,17 @@
INCLUDE = "INCLUDE",
)
+def _VenvSymlinkEntry_init(**kwargs):
+ kwargs.setdefault("link_to_file", None)
+ return kwargs
+
# A provider is used for memory efficiency.
# buildifier: disable=name-conventions
-VenvSymlinkEntry = provider(
+VenvSymlinkEntry, _ = provider(
doc = """
An entry in `PyInfo.venv_symlinks`
""",
+ init = _VenvSymlinkEntry_init,
fields = {
"files": """
:type: depset[File]
@@ -68,11 +73,20 @@
One of the {obj}`VenvSymlinkKind` values. It represents which directory within
the venv to create the path under.
""",
+ "link_to_file": """
+:type: File | None
+
+A file that `venv_path` should point to. The file to link to should also be in
+`files`.
+
+:::{versionadded} VERSION_NEXT_FEATURE
+:::
+""",
"link_to_path": """
:type: str | None
-A runfiles-root relative path that `venv_path` will symlink to. If `None`,
-it means to not create a symlink.
+A runfiles-root relative path that `venv_path` will symlink to (if
+`link_to_file` is `None`). If `None`, it means to not create it in the venv.
""",
"package": """
:type: str | None
diff --git a/python/private/venv_runfiles.bzl b/python/private/venv_runfiles.bzl
index 9fbe97a..9bdacf8 100644
--- a/python/private/venv_runfiles.bzl
+++ b/python/private/venv_runfiles.bzl
@@ -81,7 +81,7 @@
Returns:
{type}`dict[str, dict[str, str|File]]` Mappings of venv paths to their
backing files. The first key is a `VenvSymlinkKind` value.
- The inner dict keys are venv paths relative to the kind's diretory. The
+ The inner dict keys are venv paths relative to the kind's directory. The
inner dict values are strings or Files to link to.
"""
@@ -116,7 +116,10 @@
# If there's just one group, we can symlink to the directory
if len(group) == 1:
entry = group[0]
- keep_kind_link_map[entry.venv_path] = entry.link_to_path
+ if entry.link_to_file:
+ keep_kind_link_map[entry.venv_path] = entry.link_to_file
+ else:
+ keep_kind_link_map[entry.venv_path] = entry.link_to_path
else:
# Merge a group of overlapping prefixes
_merge_venv_path_group(ctx, group, keep_kind_link_map)
@@ -172,7 +175,9 @@
# TODO: Compute the minimum number of entries to create. This can't avoid
# flattening the files depset, but can lower the number of materialized
# files significantly. Usually overlaps are limited to a small number
- # of directories.
+ # of directories. Note that, when doing so, shared libraries need to
+ # be symlinked directly, not the directory containing them, due to
+ # dynamic linker symlink resolution semantics on Linux.
for entry in group:
prefix = entry.venv_path
for file in entry.files.to_list():
@@ -249,13 +254,26 @@
continue
path = path.removeprefix(site_packages_root)
dir_name, _, filename = path.rpartition("/")
+ runfiles_dir_name, _, _ = runfiles_root_path(ctx, src.short_path).partition("/")
+
+ if _is_linker_loaded_library(filename):
+ entry = VenvSymlinkEntry(
+ kind = VenvSymlinkKind.LIB,
+ link_to_path = paths.join(runfiles_dir_name, site_packages_root, filename),
+ link_to_file = src,
+ package = package,
+ version = version_str,
+ venv_path = path,
+ files = depset([src]),
+ )
+ venv_symlinks.append(entry)
+ continue
if dir_name in dir_symlinks:
# we already have this dir, this allows us to short-circuit since most of the
# ctx.files.data might share the same directories as ctx.files.srcs
continue
- runfiles_dir_name, _, _ = runfiles_root_path(ctx, src.short_path).partition("/")
if dir_name:
# This can be either:
# * a directory with libs (e.g. numpy.libs, created by auditwheel)
@@ -312,6 +330,24 @@
return venv_symlinks
+def _is_linker_loaded_library(filename):
+ """Tells if a filename is one that `dlopen()` or the runtime linker handles.
+
+ This should return true for regular C libraries, but false for Python
+ C extension modules.
+
+ Python extensions: .so (linux, mac), .pyd (windows)
+
+ C libraries: lib*.so (linux), lib*.so.* (linux), lib*.dylib (mac), .dll (windows)
+ """
+ if filename.endswith(".dll"):
+ return True
+ if filename.startswith("lib") and (
+ filename.endswith((".so", ".dylib")) or ".so." in filename
+ ):
+ return True
+ return False
+
def _repo_relative_short_path(short_path):
# Convert `../+pypi+foo/some/file.py` to `some/file.py`
if short_path.startswith("../"):
diff --git a/tests/support/copy_file.bzl b/tests/support/copy_file.bzl
new file mode 100644
index 0000000..bd9bb21
--- /dev/null
+++ b/tests/support/copy_file.bzl
@@ -0,0 +1,33 @@
+"""Copies a file to a directory."""
+
+def _copy_file_to_dir_impl(ctx):
+ out_file = ctx.actions.declare_file(
+ "{}/{}".format(ctx.attr.out_dir, ctx.file.src.basename),
+ )
+ ctx.actions.run_shell(
+ inputs = [ctx.file.src],
+ outputs = [out_file],
+ arguments = [ctx.file.src.path, out_file.path],
+ # Perform a copy to better match how a file install from
+ # a repo-phase (e.g. whl extraction) looks.
+ command = 'cp -f "$1" "$2"',
+ progress_message = "Copying %{input} to %{output}",
+ )
+ return [DefaultInfo(files = depset([out_file]))]
+
+copy_file_to_dir = rule(
+ implementation = _copy_file_to_dir_impl,
+ doc = """
+This allows copying a file whose name is platform-dependent to a directory.
+
+While bazel_skylib has a copy_file rule, you must statically specify the
+output file name.
+""",
+ attrs = {
+ "out_dir": attr.string(mandatory = True),
+ "src": attr.label(
+ allow_single_file = True,
+ mandatory = True,
+ ),
+ },
+)
diff --git a/tests/venv_site_packages_libs/BUILD.bazel b/tests/venv_site_packages_libs/BUILD.bazel
index 92d5dec..2ce4ad9 100644
--- a/tests/venv_site_packages_libs/BUILD.bazel
+++ b/tests/venv_site_packages_libs/BUILD.bazel
@@ -1,6 +1,10 @@
load("//python:py_library.bzl", "py_library")
load("//tests/support:py_reconfig.bzl", "py_reconfig_test")
-load("//tests/support:support.bzl", "SUPPORTS_BOOTSTRAP_SCRIPT")
+load(
+ "//tests/support:support.bzl",
+ "NOT_WINDOWS",
+ "SUPPORTS_BOOTSTRAP_SCRIPT",
+)
py_library(
name = "user_lib",
@@ -34,3 +38,17 @@
"@other//with_external_data",
],
)
+
+py_reconfig_test(
+ name = "shared_lib_loading_test",
+ srcs = ["shared_lib_loading_test.py"],
+ bootstrap_impl = "script",
+ main = "shared_lib_loading_test.py",
+ target_compatible_with = NOT_WINDOWS,
+ venvs_site_packages = "yes",
+ deps = [
+ "//tests/venv_site_packages_libs/ext_with_libs",
+ "@dev_pip//macholib",
+ "@dev_pip//pyelftools",
+ ],
+)
diff --git a/tests/venv_site_packages_libs/app_files_building/app_files_building_tests.bzl b/tests/venv_site_packages_libs/app_files_building/app_files_building_tests.bzl
index 68e1716..31c720a 100644
--- a/tests/venv_site_packages_libs/app_files_building/app_files_building_tests.bzl
+++ b/tests/venv_site_packages_libs/app_files_building/app_files_building_tests.bzl
@@ -4,7 +4,25 @@
load("@rules_testing//lib:analysis_test.bzl", "analysis_test")
load("@rules_testing//lib:test_suite.bzl", "test_suite")
load("//python/private:py_info.bzl", "VenvSymlinkEntry", "VenvSymlinkKind") # buildifier: disable=bzl-visibility
-load("//python/private:venv_runfiles.bzl", "build_link_map") # buildifier: disable=bzl-visibility
+load("//python/private:venv_runfiles.bzl", "build_link_map", "get_venv_symlinks") # buildifier: disable=bzl-visibility
+
+def _empty_files_impl(ctx):
+ files = []
+ for p in ctx.attr.paths:
+ f = ctx.actions.declare_file(p)
+ ctx.actions.write(output = f, content = "")
+ files.append(f)
+ return [DefaultInfo(files = depset(files))]
+
+empty_files = rule(
+ implementation = _empty_files_impl,
+ attrs = {
+ "paths": attr.string_list(
+ doc = "A list of paths to create as files.",
+ mandatory = True,
+ ),
+ },
+)
_tests = []
@@ -242,6 +260,63 @@
VenvSymlinkKind.INCLUDE,
])
+def _test_shared_library_symlinking(name):
+ empty_files(
+ name = name + "_files",
+ # NOTE: Test relies upon order
+ paths = [
+ "site-packages/bar/libs/liby.so",
+ "site-packages/bar/x.py",
+ "site-packages/bar/y.so",
+ "site-packages/foo.libs/libx.so",
+ "site-packages/foo/a.py",
+ "site-packages/foo/b.so",
+ ],
+ )
+ analysis_test(
+ name = name,
+ impl = _test_shared_library_symlinking_impl,
+ target = name + "_files",
+ )
+
+_tests.append(_test_shared_library_symlinking)
+
+def _test_shared_library_symlinking_impl(env, target):
+ srcs = target.files.to_list()
+ actual_entries = get_venv_symlinks(
+ _ctx(),
+ srcs,
+ package = "foo",
+ version_str = "1.0",
+ site_packages_root = env.ctx.label.package + "/site-packages",
+ )
+
+ actual = [e for e in actual_entries if e.venv_path == "foo.libs/libx.so"]
+ if not actual:
+ fail("Did not find VenvSymlinkEntry with venv_path equal to foo.libs/libx.so. " +
+ "Found: {}".format(actual_entries))
+ elif len(actual) > 1:
+ fail("Found multiple entries with venv_path=foo.libs/libx.so. " +
+ "Found: {}".format(actual_entries))
+ actual = actual[0]
+
+ actual_files = actual.files.to_list()
+ expected_lib_dso = [f for f in srcs if f.basename == "libx.so"]
+ env.expect.that_collection(actual_files).contains_exactly(expected_lib_dso)
+
+ entries = actual_entries
+ actual = build_link_map(_ctx(), entries)
+
+ # The important condition is that each lib*.so file is linked directly.
+ expected_libs = {
+ "bar/libs/liby.so": srcs[0],
+ "bar/x.py": srcs[1],
+ "bar/y.so": srcs[2],
+ "foo": "_main/tests/venv_site_packages_libs/app_files_building/site-packages/foo",
+ "foo.libs/libx.so": srcs[3],
+ }
+ env.expect.that_dict(actual[VenvSymlinkKind.LIB]).contains_exactly(expected_libs)
+
def app_files_building_test_suite(name):
test_suite(
name = name,
diff --git a/tests/venv_site_packages_libs/ext_with_libs/BUILD.bazel b/tests/venv_site_packages_libs/ext_with_libs/BUILD.bazel
new file mode 100644
index 0000000..8f161ee
--- /dev/null
+++ b/tests/venv_site_packages_libs/ext_with_libs/BUILD.bazel
@@ -0,0 +1,94 @@
+load("@rules_cc//cc:cc_library.bzl", "cc_library")
+load("@rules_cc//cc:cc_shared_library.bzl", "cc_shared_library")
+load("//python:py_library.bzl", "py_library")
+load("//tests/support:copy_file.bzl", "copy_file_to_dir")
+
+package(
+ default_visibility = ["//visibility:public"],
+)
+
+cc_library(
+ name = "increment_impl",
+ srcs = ["increment.c"],
+ deps = [":increment_headers"],
+)
+
+cc_library(
+ name = "increment_headers",
+ hdrs = ["increment.h"],
+)
+
+cc_shared_library(
+ name = "increment",
+ user_link_flags = select({
+ "@platforms//os:osx": [
+ # Needed so that DT_NEEDED=libincrement.dylib can find
+ # this shared library
+ "-Wl,-install_name,@rpath/libincrement.dylib",
+ ],
+ "//conditions:default": [],
+ }),
+ deps = [":increment_impl"],
+)
+
+cc_library(
+ name = "adder_impl",
+ srcs = ["adder.c"],
+ deps = [
+ ":increment_headers",
+ "@rules_python//python/cc:current_py_cc_headers",
+ ],
+)
+
+cc_shared_library(
+ name = "adder",
+ # Necessary for several reasons:
+ # 1. Ensures the output doesn't include increment itself (avoids ODRs)
+ # 2. Adds -lincrement (DT_NEEDED for libincrement.so)
+ # 3. Ensures libincrement.so is available at link time to satisfy (2)
+ dynamic_deps = [":increment"],
+ shared_lib_name = "adder.so",
+ tags = ["manual"],
+ # NOTE: cc_shared_library adds Bazelized rpath entries, too.
+ user_link_flags = [
+ ] + select({
+ "@platforms//os:osx": [
+ "-Wl,-rpath,@loader_path/libs",
+ "-undefined",
+ "dynamic_lookup",
+ "-Wl,-exported_symbol",
+ "-Wl,_PyInit_adder",
+ ],
+ # Assume linux default
+ "//conditions:default": [
+ "-Wl,-rpath,$ORIGIN/libs",
+ ],
+ }),
+ deps = [":adder_impl"],
+)
+
+copy_file_to_dir(
+ name = "relocate_adder",
+ src = ":adder",
+ out_dir = "site-packages/ext_with_libs",
+ tags = ["manual"],
+)
+
+copy_file_to_dir(
+ name = "relocate_increment",
+ src = ":increment",
+ out_dir = "site-packages/ext_with_libs/libs",
+ tags = ["manual"],
+)
+
+py_library(
+ name = "ext_with_libs",
+ srcs = glob(["site-packages/**/*.py"]),
+ data = [
+ ":relocate_adder",
+ ":relocate_increment",
+ ],
+ experimental_venvs_site_packages = "//python/config_settings:venvs_site_packages",
+ imports = [package_name() + "/site-packages"],
+ tags = ["manual"],
+)
diff --git a/tests/venv_site_packages_libs/ext_with_libs/adder.c b/tests/venv_site_packages_libs/ext_with_libs/adder.c
new file mode 100644
index 0000000..8b04b17
--- /dev/null
+++ b/tests/venv_site_packages_libs/ext_with_libs/adder.c
@@ -0,0 +1,15 @@
+#include <Python.h>
+
+#include "increment.h"
+
+static PyObject *do_add(PyObject *self, PyObject *Py_UNUSED(args)) {
+ return PyLong_FromLong(increment(1));
+}
+
+static PyMethodDef AdderMethods[] = {
+ {"do_add", do_add, METH_NOARGS, "Add one"}, {NULL, NULL, 0, NULL}};
+
+static struct PyModuleDef addermodule = {PyModuleDef_HEAD_INIT, "adder", NULL,
+ -1, AdderMethods};
+
+PyMODINIT_FUNC PyInit_adder(void) { return PyModule_Create(&addermodule); }
diff --git a/tests/venv_site_packages_libs/ext_with_libs/increment.c b/tests/venv_site_packages_libs/ext_with_libs/increment.c
new file mode 100644
index 0000000..b194325
--- /dev/null
+++ b/tests/venv_site_packages_libs/ext_with_libs/increment.c
@@ -0,0 +1,3 @@
+#include "increment.h"
+
+int increment(int val) { return val + 1; }
diff --git a/tests/venv_site_packages_libs/ext_with_libs/increment.h b/tests/venv_site_packages_libs/ext_with_libs/increment.h
new file mode 100644
index 0000000..8a13bf5
--- /dev/null
+++ b/tests/venv_site_packages_libs/ext_with_libs/increment.h
@@ -0,0 +1,6 @@
+#ifndef TESTS_VENV_SITE_PACKAGES_LIBS_EXT_WITH_LIBS_INCREMENT_H_
+#define TESTS_VENV_SITE_PACKAGES_LIBS_EXT_WITH_LIBS_INCREMENT_H_
+
+int increment(int);
+
+#endif // TESTS_VENV_SITE_PACKAGES_LIBS_EXT_WITH_LIBS_INCREMENT_H_
diff --git a/tests/venv_site_packages_libs/ext_with_libs/site-packages/ext_with_libs/__init__.py b/tests/venv_site_packages_libs/ext_with_libs/site-packages/ext_with_libs/__init__.py
new file mode 100644
index 0000000..ea0485c
--- /dev/null
+++ b/tests/venv_site_packages_libs/ext_with_libs/site-packages/ext_with_libs/__init__.py
@@ -0,0 +1,2 @@
+# This just marks the directory as a Pyton package. Python C extension modules
+# and C libraries are populated in this directory at build time.
diff --git a/tests/venv_site_packages_libs/shared_lib_loading_test.py b/tests/venv_site_packages_libs/shared_lib_loading_test.py
new file mode 100644
index 0000000..2b58f85
--- /dev/null
+++ b/tests/venv_site_packages_libs/shared_lib_loading_test.py
@@ -0,0 +1,117 @@
+import importlib.util
+import os
+import unittest
+
+from elftools.elf.elffile import ELFFile
+from macholib import mach_o
+from macholib.MachO import MachO
+
+ELF_MAGIC = b"\x7fELF"
+MACHO_MAGICS = (
+ b"\xce\xfa\xed\xfe", # 32-bit big-endian
+ b"\xcf\xfa\xed\xfe", # 64-bit big-endian
+ b"\xfe\xed\xfa\xce", # 32-bit little-endian
+ b"\xfe\xed\xfa\xcf", # 64-bit little-endian
+)
+
+
+class SharedLibLoadingTest(unittest.TestCase):
+ def test_shared_library_linking(self):
+ try:
+ import ext_with_libs.adder
+ except ImportError as e:
+ spec = importlib.util.find_spec("ext_with_libs.adder")
+ if not spec or not spec.origin:
+ self.fail(f"Import failed and could not find module spec: {e}")
+
+ info = self._get_linking_info(spec.origin)
+
+ # Give a useful error message for debugging.
+ self.fail(
+ f"Failed to import adder extension.\n"
+ f"Original error: {e}\n"
+ f"Linking info for {spec.origin}:\n"
+ f" RPATHs: {info.get('rpaths', 'N/A')}\n"
+ f" Needed libs: {info.get('needed', 'N/A')}"
+ )
+
+ # Check that the module was loaded from the venv.
+ self.assertIn(".venv/", ext_with_libs.adder.__file__)
+
+ adder_path = os.path.realpath(ext_with_libs.adder.__file__)
+
+ with open(adder_path, "rb") as f:
+ magic_bytes = f.read(4)
+
+ if magic_bytes == ELF_MAGIC:
+ self._assert_elf_linking(adder_path)
+ elif magic_bytes in MACHO_MAGICS:
+ self._assert_macho_linking(adder_path)
+ else:
+ self.fail(f"Unsupported file format for adder: magic bytes {magic_bytes!r}")
+
+ # Check the function works regardless of format.
+ self.assertEqual(ext_with_libs.adder.do_add(), 2)
+
+ def _get_linking_info(self, path):
+ """Parses a shared library and returns its rpaths and dependencies."""
+ path = os.path.realpath(path)
+ with open(path, "rb") as f:
+ magic_bytes = f.read(4)
+
+ if magic_bytes == ELF_MAGIC:
+ return self._get_elf_info(path)
+ elif magic_bytes in MACHO_MAGICS:
+ return self._get_macho_info(path)
+ return {}
+
+ def _get_elf_info(self, path):
+ """Extracts linking information from an ELF file."""
+ info = {"rpaths": [], "needed": [], "undefined_symbols": []}
+ with open(path, "rb") as f:
+ elf = ELFFile(f)
+ dynamic = elf.get_section_by_name(".dynamic")
+ if dynamic:
+ for tag in dynamic.iter_tags():
+ if tag.entry.d_tag == "DT_NEEDED":
+ info["needed"].append(tag.needed)
+ elif tag.entry.d_tag == "DT_RPATH":
+ info["rpaths"].append(tag.rpath)
+ elif tag.entry.d_tag == "DT_RUNPATH":
+ info["rpaths"].append(tag.runpath)
+
+ dynsym = elf.get_section_by_name(".dynsym")
+ if dynsym:
+ info["undefined_symbols"] = [
+ s.name
+ for s in dynsym.iter_symbols()
+ if s.entry["st_shndx"] == "SHN_UNDEF"
+ ]
+ return info
+
+ def _get_macho_info(self, path):
+ """Extracts linking information from a Mach-O file."""
+ info = {"rpaths": [], "needed": []}
+ macho = MachO(path)
+ for header in macho.headers:
+ for cmd_load, cmd, data in header.commands:
+ if cmd_load.cmd == mach_o.LC_LOAD_DYLIB:
+ info["needed"].append(data.decode().strip("\x00"))
+ elif cmd_load.cmd == mach_o.LC_RPATH:
+ info["rpaths"].append(data.decode().strip("\x00"))
+ return info
+
+ def _assert_elf_linking(self, path):
+ """Asserts dynamic linking properties for an ELF file."""
+ info = self._get_elf_info(path)
+ self.assertIn("libincrement.so", info["needed"])
+ self.assertIn("increment", info["undefined_symbols"])
+
+ def _assert_macho_linking(self, path):
+ """Asserts dynamic linking properties for a Mach-O file."""
+ info = self._get_macho_info(path)
+ self.assertIn("@rpath/libincrement.dylib", info["needed"])
+
+
+if __name__ == "__main__":
+ unittest.main()
