refactor: change site_packages_symlinks to venv_symlinks (#2939)
This generalizes the ability to populate the venv directory by adding
and additional field,
`kind`, which tells which directory of the venv to populate. A symbolic
constant is used
to indicate which directory so that users don't have to re-derive the
platform and version
specific paths that make up the venv directory names.
This follows the design described by
https://github.com/bazel-contrib/rules_python/issues/2156#issuecomment-2855580026
This also changes it to a depset of structs to make it more forward
compatible. A provider
is used because they're slightly more memory efficient than regular
structs.
Work towards https://github.com/bazel-contrib/rules_python/issues/2156
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9655b90..4a6bdf0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -70,6 +70,8 @@
`_test` target is deprecated and will be removed in the next major release.
([#2794](https://github.com/bazel-contrib/rules_python/issues/2794)
* (py_wheel) py_wheel always creates zip64-capable wheel zips
+* (providers) (experimental) {obj}`PyInfo.venv_symlinks` replaces
+ `PyInfo.site_packages_symlinks`
{#v0-0-0-fixed}
### Fixed
@@ -203,7 +205,7 @@
please check the {obj}`uv.configure` tag class.
* Add support for riscv64 linux platform.
* (toolchains) Add python 3.13.2 and 3.12.9 toolchains
-* (providers) (experimental) {obj}`PyInfo.site_packages_symlinks` field added to
+* (providers) (experimental) `PyInfo.site_packages_symlinks` field added to
allow specifying links to create within the venv site packages (only
applicable with {obj}`--bootstrap_impl=script`)
([#2156](https://github.com/bazelbuild/rules_python/issues/2156)).
diff --git a/python/features.bzl b/python/features.bzl
index 917bd38..b678a45 100644
--- a/python/features.bzl
+++ b/python/features.bzl
@@ -31,11 +31,11 @@
:::
::::
- ::::{field} py_info_site_packages_symlinks
+ ::::{field} py_info_venv_symlinks
- True if the `PyInfo.site_packages_symlinks` field is available.
+ True if the `PyInfo.venv_symlinks` field is available.
- :::{versionadded} 1.4.0
+ :::{versionadded} VERSION_NEXT_FEATURE
:::
::::
@@ -61,7 +61,7 @@
TYPEDEF = _features_typedef,
# keep sorted
precompile = True,
- py_info_site_packages_symlinks = True,
+ py_info_venv_symlinks = True,
uses_builtin_rules = not config.enable_pystar,
version = _VERSION_PRIVATE if "$Format" not in _VERSION_PRIVATE else "",
)
diff --git a/python/private/attributes.bzl b/python/private/attributes.bzl
index 98aba4e..ad8cba2 100644
--- a/python/private/attributes.bzl
+++ b/python/private/attributes.bzl
@@ -260,7 +260,7 @@
from dependencies is merged in, which can be relevant depending on the ordering
mode of depsets that are merged.
-* {obj}`PyInfo.site_packages_symlinks` uses topological ordering.
+* {obj}`PyInfo.venv_symlinks` uses topological ordering.
See {obj}`PyInfo` for more information about the ordering of its depsets and
how its fields are merged.
diff --git a/python/private/common.bzl b/python/private/common.bzl
index a58a9c0..e49dbad 100644
--- a/python/private/common.bzl
+++ b/python/private/common.bzl
@@ -378,7 +378,7 @@
implicit_pyc_files,
implicit_pyc_source_files,
imports,
- site_packages_symlinks = []):
+ venv_symlinks = []):
"""Create PyInfo provider.
Args:
@@ -396,7 +396,7 @@
implicit_pyc_files: {type}`depset[File]` Implicitly generated pyc files
that a binary can choose to include.
imports: depset of strings; the import path values to propagate.
- site_packages_symlinks: {type}`list[tuple[str, str]]` tuples of
+ venv_symlinks: {type}`list[tuple[str, str]]` tuples of
`(runfiles_path, site_packages_path)` for symlinks to create
in the consuming binary's venv site packages.
@@ -406,7 +406,7 @@
necessary for deprecated extra actions support).
"""
py_info = PyInfoBuilder.new()
- py_info.site_packages_symlinks.add(site_packages_symlinks)
+ py_info.venv_symlinks.add(venv_symlinks)
py_info.direct_original_sources.add(original_sources)
py_info.direct_pyc_files.add(required_pyc_files)
py_info.direct_pyi_files.add(ctx.files.pyi_srcs)
diff --git a/python/private/flags.bzl b/python/private/flags.bzl
index 40ce63b..710402b 100644
--- a/python/private/flags.bzl
+++ b/python/private/flags.bzl
@@ -154,12 +154,12 @@
flag_value = ctx.attr.experimental_venvs_site_packages[BuildSettingInfo].value
return flag_value == VenvsSitePackages.YES
-# Decides if libraries try to use a site-packages layout using site_packages_symlinks
+# Decides if libraries try to use a site-packages layout using venv_symlinks
# buildifier: disable=name-conventions
VenvsSitePackages = FlagEnum(
- # Use site_packages_symlinks
+ # Use venv_symlinks
YES = "yes",
- # Don't use site_packages_symlinks
+ # Don't use venv_symlinks
NO = "no",
is_enabled = _venvs_site_packages_is_enabled,
)
diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl
index 24be8dd..7c3e0cb 100644
--- a/python/private/py_executable.bzl
+++ b/python/private/py_executable.bzl
@@ -54,7 +54,7 @@
load(":precompile.bzl", "maybe_precompile")
load(":py_cc_link_params_info.bzl", "PyCcLinkParamsInfo")
load(":py_executable_info.bzl", "PyExecutableInfo")
-load(":py_info.bzl", "PyInfo")
+load(":py_info.bzl", "PyInfo", "VenvSymlinkKind")
load(":py_internal.bzl", "py_internal")
load(":py_runtime_info.bzl", "DEFAULT_STUB_SHEBANG", "PyRuntimeInfo")
load(":reexports.bzl", "BuiltinPyInfo", "BuiltinPyRuntimeInfo")
@@ -543,6 +543,7 @@
VenvsUseDeclareSymlinkFlag.get_value(ctx) == VenvsUseDeclareSymlinkFlag.YES
)
recreate_venv_at_runtime = False
+ bin_dir = "{}/bin".format(venv)
if not venvs_use_declare_symlink_enabled or not runtime.supports_build_time_venv:
recreate_venv_at_runtime = True
@@ -556,7 +557,7 @@
# When the venv symlinks are disabled, the $venv/bin/python3 file isn't
# needed or used at runtime. However, the zip code uses the interpreter
# File object to figure out some paths.
- interpreter = ctx.actions.declare_file("{}/bin/{}".format(venv, py_exe_basename))
+ interpreter = ctx.actions.declare_file("{}/{}".format(bin_dir, py_exe_basename))
ctx.actions.write(interpreter, "actual:{}".format(interpreter_actual_path))
elif runtime.interpreter:
@@ -568,7 +569,7 @@
# declare_symlink() is required to ensure that the resulting file
# in runfiles is always a symlink. An RBE implementation, for example,
# may choose to write what symlink() points to instead.
- interpreter = ctx.actions.declare_symlink("{}/bin/{}".format(venv, py_exe_basename))
+ interpreter = ctx.actions.declare_symlink("{}/{}".format(bin_dir, py_exe_basename))
interpreter_actual_path = runfiles_root_path(ctx, runtime.interpreter.short_path)
rel_path = relative_path(
@@ -581,7 +582,7 @@
ctx.actions.symlink(output = interpreter, target_path = rel_path)
else:
py_exe_basename = paths.basename(runtime.interpreter_path)
- interpreter = ctx.actions.declare_symlink("{}/bin/{}".format(venv, py_exe_basename))
+ interpreter = ctx.actions.declare_symlink("{}/{}".format(bin_dir, py_exe_basename))
ctx.actions.symlink(output = interpreter, target_path = runtime.interpreter_path)
interpreter_actual_path = runtime.interpreter_path
@@ -618,89 +619,104 @@
},
computed_substitutions = computed_subs,
)
- site_packages_symlinks = _create_site_packages_symlinks(ctx, site_packages)
+
+ venv_dir_map = {
+ VenvSymlinkKind.BIN: bin_dir,
+ VenvSymlinkKind.LIB: site_packages,
+ }
+ venv_symlinks = _create_venv_symlinks(ctx, venv_dir_map)
return struct(
interpreter = interpreter,
recreate_venv_at_runtime = recreate_venv_at_runtime,
# Runfiles root relative path or absolute path
interpreter_actual_path = interpreter_actual_path,
- files_without_interpreter = [pyvenv_cfg, pth, site_init] + site_packages_symlinks,
+ files_without_interpreter = [pyvenv_cfg, pth, site_init] + venv_symlinks,
# string; venv-relative path to the site-packages directory.
venv_site_packages = venv_site_packages,
)
-def _create_site_packages_symlinks(ctx, site_packages):
- """Creates symlinks within site-packages.
+def _create_venv_symlinks(ctx, venv_dir_map):
+ """Creates symlinks within the venv.
Args:
ctx: current rule ctx
- site_packages: runfiles-root-relative path to the site-packages directory
+ venv_dir_map: mapping of VenvSymlinkKind constants to the
+ venv path.
Returns:
{type}`list[File]` list of the File symlink objects created.
"""
- # maps site-package symlink to the runfiles path it should point to
+ # maps venv-relative path to the runfiles path it should point to
entries = depset(
# NOTE: Topological ordering is used so that dependencies closer to the
# binary have precedence in creating their symlinks. This allows the
# binary a modicum of control over the result.
order = "topological",
transitive = [
- dep[PyInfo].site_packages_symlinks
+ dep[PyInfo].venv_symlinks
for dep in ctx.attr.deps
if PyInfo in dep
],
).to_list()
- link_map = _build_link_map(entries)
- sp_files = []
- for sp_dir_path, link_to in link_map.items():
- sp_link = ctx.actions.declare_symlink(paths.join(site_packages, sp_dir_path))
- sp_link_rf_path = runfiles_root_path(ctx, sp_link.short_path)
- rel_path = relative_path(
- # dirname is necessary because a relative symlink is relative to
- # the directory the symlink resides within.
- from_ = paths.dirname(sp_link_rf_path),
- to = link_to,
- )
- ctx.actions.symlink(output = sp_link, target_path = rel_path)
- sp_files.append(sp_link)
- return sp_files
+ link_map = _build_link_map(entries)
+ venv_files = []
+ for kind, kind_map in link_map.items():
+ base = venv_dir_map[kind]
+ for venv_path, link_to in kind_map.items():
+ venv_link = ctx.actions.declare_symlink(paths.join(base, venv_path))
+ venv_link_rf_path = runfiles_root_path(ctx, venv_link.short_path)
+ rel_path = relative_path(
+ # dirname is necessary because a relative symlink is relative to
+ # the directory the symlink resides within.
+ from_ = paths.dirname(venv_link_rf_path),
+ to = link_to,
+ )
+ ctx.actions.symlink(output = venv_link, target_path = rel_path)
+ venv_files.append(venv_link)
+
+ return venv_files
def _build_link_map(entries):
+ # dict[str kind, dict[str rel_path, str link_to_path]]
link_map = {}
- for link_to_runfiles_path, site_packages_path in entries:
- if site_packages_path in link_map:
+ for entry in entries:
+ kind = entry.kind
+ kind_map = link_map.setdefault(kind, {})
+ if entry.venv_path in kind_map:
# We ignore duplicates by design. The dependency closer to the
# binary gets precedence due to the topological ordering.
continue
else:
- link_map[site_packages_path] = link_to_runfiles_path
+ kind_map[entry.venv_path] = entry.link_to_path
# An empty link_to value means to not create the site package symlink.
# Because of the topological ordering, this allows binaries to remove
# entries by having an earlier dependency produce empty link_to values.
- for sp_dir_path, link_to in link_map.items():
- if not link_to:
- link_map.pop(sp_dir_path)
+ for kind, kind_map in link_map.items():
+ for dir_path, link_to in kind_map.items():
+ if not link_to:
+ kind_map.pop(dir_path)
+
+ # dict[str kind, dict[str rel_path, str link_to_path]]
+ keep_link_map = {}
# Remove entries that would be a child path of a created symlink.
# Earlier entries have precedence to match how exact matches are handled.
- keep_link_map = {}
- for _ in range(len(link_map)):
- if not link_map:
- break
- dirname, value = link_map.popitem()
- keep_link_map[dirname] = value
-
- prefix = dirname + "/" # Add slash to prevent /X matching /XY
- for maybe_suffix in link_map.keys():
- maybe_suffix += "/" # Add slash to prevent /X matching /XY
- if maybe_suffix.startswith(prefix) or prefix.startswith(maybe_suffix):
- link_map.pop(maybe_suffix)
-
+ for kind, kind_map in link_map.items():
+ keep_kind_map = keep_link_map.setdefault(kind, {})
+ for _ in range(len(kind_map)):
+ if not kind_map:
+ break
+ dirname, value = kind_map.popitem()
+ keep_kind_map[dirname] = value
+ prefix = dirname + "/" # Add slash to prevent /X matching /XY
+ for maybe_suffix in kind_map.keys():
+ maybe_suffix += "/" # Add slash to prevent /X matching /XY
+ if maybe_suffix.startswith(prefix) or prefix.startswith(maybe_suffix):
+ kind_map.pop(maybe_suffix)
return keep_link_map
def _map_each_identity(v):
diff --git a/python/private/py_info.bzl b/python/private/py_info.bzl
index d175eef..2a2f455 100644
--- a/python/private/py_info.bzl
+++ b/python/private/py_info.bzl
@@ -18,6 +18,64 @@
load(":reexports.bzl", "BuiltinPyInfo")
load(":util.bzl", "define_bazel_6_provider")
+def _VenvSymlinkKind_typedef():
+ """An enum of types of venv directories.
+
+ :::{field} BIN
+ :type: object
+
+ Indicates to create paths under the directory that has binaries
+ within the venv.
+ :::
+
+ :::{field} LIB
+ :type: object
+
+ Indicates to create paths under the venv's site-packages directory.
+ :::
+
+ :::{field} INCLUDE
+ :type: object
+
+ Indicates to create paths under the venv's include directory.
+ :::
+ """
+
+# buildifier: disable=name-conventions
+VenvSymlinkKind = struct(
+ TYPEDEF = _VenvSymlinkKind_typedef,
+ BIN = "BIN",
+ LIB = "LIB",
+ INCLUDE = "INCLUDE",
+)
+
+# A provider is used for memory efficiency.
+# buildifier: disable=name-conventions
+VenvSymlinkEntry = provider(
+ doc = """
+An entry in `PyInfo.venv_symlinks`
+""",
+ fields = {
+ "kind": """
+:type: str
+
+One of the {obj}`VenvSymlinkKind` values. It represents which directory within
+the venv to create the path under.
+""",
+ "link_to_path": """
+:type: str | None
+
+A runfiles-root relative path that `venv_path` will symlink to. If `None`,
+it means to not create a symlink.
+""",
+ "venv_path": """
+:type: str
+
+A path relative to the `kind` directory within the venv.
+""",
+ },
+)
+
def _check_arg_type(name, required_type, value):
"""Check that a value is of an expected type."""
value_type = type(value)
@@ -43,7 +101,7 @@
transitive_original_sources = depset(),
direct_pyi_files = depset(),
transitive_pyi_files = depset(),
- site_packages_symlinks = depset()):
+ venv_symlinks = depset()):
_check_arg_type("transitive_sources", "depset", transitive_sources)
# Verify it's postorder compatible, but retain is original ordering.
@@ -71,7 +129,6 @@
"has_py2_only_sources": has_py2_only_sources,
"has_py3_only_sources": has_py2_only_sources,
"imports": imports,
- "site_packages_symlinks": site_packages_symlinks,
"transitive_implicit_pyc_files": transitive_implicit_pyc_files,
"transitive_implicit_pyc_source_files": transitive_implicit_pyc_source_files,
"transitive_original_sources": transitive_original_sources,
@@ -79,6 +136,7 @@
"transitive_pyi_files": transitive_pyi_files,
"transitive_sources": transitive_sources,
"uses_shared_libraries": uses_shared_libraries,
+ "venv_symlinks": venv_symlinks,
}
PyInfo, _unused_raw_py_info_ctor = define_bazel_6_provider(
@@ -147,34 +205,6 @@
The order of the depset is not guaranteed and may be changed in the future. It
is recommended to use `default` order (the default).
""",
- "site_packages_symlinks": """
-:type: depset[tuple[str | None, str]]
-
-A depset with `topological` ordering.
-
-Tuples of `(runfiles_path, site_packages_path)`. Where
-* `runfiles_path` is a runfiles-root relative path. It is the path that
- has the code to make importable. If `None` or empty string, then it means
- to not create a site packages directory with the `site_packages_path`
- name.
-* `site_packages_path` is a path relative to the site-packages directory of
- the venv for whatever creates the venv (typically py_binary). It makes
- the code in `runfiles_path` available for import. Note that this
- is created as a "raw" symlink (via `declare_symlink`).
-
-:::{include} /_includes/experimental_api.md
-:::
-
-:::{tip}
-The topological ordering means dependencies earlier and closer to the consumer
-have precedence. This allows e.g. a binary to add dependencies that override
-values from further way dependencies, such as forcing symlinks to point to
-specific paths or preventing symlinks from being created.
-:::
-
-:::{versionadded} 1.4.0
-:::
-""",
"transitive_implicit_pyc_files": """
:type: depset[File]
@@ -263,6 +293,35 @@
This field is currently unused in Bazel and may go away in the future.
""",
+ "venv_symlinks": """
+:type: depset[VenvSymlinkEntry]
+
+A depset with `topological` ordering.
+
+
+Tuples of `(runfiles_path, site_packages_path)`. Where
+* `runfiles_path` is a runfiles-root relative path. It is the path that
+ has the code to make importable. If `None` or empty string, then it means
+ to not create a site packages directory with the `site_packages_path`
+ name.
+* `site_packages_path` is a path relative to the site-packages directory of
+ the venv for whatever creates the venv (typically py_binary). It makes
+ the code in `runfiles_path` available for import. Note that this
+ is created as a "raw" symlink (via `declare_symlink`).
+
+:::{include} /_includes/experimental_api.md
+:::
+
+:::{tip}
+The topological ordering means dependencies earlier and closer to the consumer
+have precedence. This allows e.g. a binary to add dependencies that override
+values from further way dependencies, such as forcing symlinks to point to
+specific paths or preventing symlinks from being created.
+:::
+
+:::{versionadded} VERSION_NEXT_FEATURE
+:::
+""",
},
)
@@ -314,7 +373,7 @@
:type: DepsetBuilder[File]
:::
- :::{field} site_packages_symlinks
+ :::{field} venv_symlinks
:type: DepsetBuilder[tuple[str | None, str]]
NOTE: This depset has `topological` order
@@ -358,7 +417,7 @@
transitive_pyc_files = builders.DepsetBuilder(),
transitive_pyi_files = builders.DepsetBuilder(),
transitive_sources = builders.DepsetBuilder(),
- site_packages_symlinks = builders.DepsetBuilder(order = "topological"),
+ venv_symlinks = builders.DepsetBuilder(order = "topological"),
)
return self
@@ -525,7 +584,7 @@
self.transitive_original_sources.add(info.transitive_original_sources)
self.transitive_pyc_files.add(info.transitive_pyc_files)
self.transitive_pyi_files.add(info.transitive_pyi_files)
- self.site_packages_symlinks.add(info.site_packages_symlinks)
+ self.venv_symlinks.add(info.venv_symlinks)
return self
@@ -583,7 +642,7 @@
transitive_original_sources = self.transitive_original_sources.build(),
transitive_pyc_files = self.transitive_pyc_files.build(),
transitive_pyi_files = self.transitive_pyi_files.build(),
- site_packages_symlinks = self.site_packages_symlinks.build(),
+ venv_symlinks = self.venv_symlinks.build(),
)
else:
kwargs = {}
diff --git a/python/private/py_library.bzl b/python/private/py_library.bzl
index fd9dad9..fabc880 100644
--- a/python/private/py_library.bzl
+++ b/python/private/py_library.bzl
@@ -43,7 +43,7 @@
load(":flags.bzl", "AddSrcsToRunfilesFlag", "PrecompileFlag", "VenvsSitePackages")
load(":precompile.bzl", "maybe_precompile")
load(":py_cc_link_params_info.bzl", "PyCcLinkParamsInfo")
-load(":py_info.bzl", "PyInfo")
+load(":py_info.bzl", "PyInfo", "VenvSymlinkEntry", "VenvSymlinkKind")
load(":py_internal.bzl", "py_internal")
load(":reexports.bzl", "BuiltinPyInfo")
load(":rule_builders.bzl", "ruleb")
@@ -90,9 +90,9 @@
likely lead to conflicts with other targets that contribute to the namespace.
:::{tip}
-This attributes populates {obj}`PyInfo.site_packages_symlinks`, which is
+This attributes populates {obj}`PyInfo.venv_symlinks`, which is
a topologically ordered depset. This means dependencies closer and earlier
-to a consumer have precedence. See {obj}`PyInfo.site_packages_symlinks` for
+to a consumer have precedence. See {obj}`PyInfo.venv_symlinks` for
more information.
:::
@@ -155,9 +155,9 @@
runfiles = runfiles.build(ctx)
imports = []
- site_packages_symlinks = []
+ venv_symlinks = []
- imports, site_packages_symlinks = _get_imports_and_site_packages_symlinks(ctx, semantics)
+ imports, venv_symlinks = _get_imports_and_venv_symlinks(ctx, semantics)
cc_info = semantics.get_cc_info_for_library(ctx)
py_info, deps_transitive_sources, builtins_py_info = create_py_info(
@@ -168,7 +168,7 @@
implicit_pyc_files = implicit_pyc_files,
implicit_pyc_source_files = implicit_pyc_source_files,
imports = imports,
- site_packages_symlinks = site_packages_symlinks,
+ venv_symlinks = venv_symlinks,
)
# TODO(b/253059598): Remove support for extra actions; https://github.com/bazelbuild/bazel/issues/16455
@@ -206,16 +206,16 @@
:::
"""
-def _get_imports_and_site_packages_symlinks(ctx, semantics):
+def _get_imports_and_venv_symlinks(ctx, semantics):
imports = depset()
- site_packages_symlinks = depset()
+ venv_symlinks = depset()
if VenvsSitePackages.is_enabled(ctx):
- site_packages_symlinks = _get_site_packages_symlinks(ctx)
+ venv_symlinks = _get_venv_symlinks(ctx)
else:
imports = collect_imports(ctx, semantics)
- return imports, site_packages_symlinks
+ return imports, venv_symlinks
-def _get_site_packages_symlinks(ctx):
+def _get_venv_symlinks(ctx):
imports = ctx.attr.imports
if len(imports) == 0:
fail("When venvs_site_packages is enabled, exactly one `imports` " +
@@ -253,7 +253,7 @@
repo_runfiles_dirname = None
dirs_with_init = {} # dirname -> runfile path
- site_packages_symlinks = []
+ venv_symlinks = []
for src in ctx.files.srcs:
if src.extension not in PYTHON_FILE_EXTENSIONS:
continue
@@ -271,9 +271,10 @@
# This would be files that do not have directories and we just need to add
# direct symlinks to them as is:
- site_packages_symlinks.append((
- paths.join(repo_runfiles_dirname, site_packages_root, filename),
- filename,
+ venv_symlinks.append(VenvSymlinkEntry(
+ kind = VenvSymlinkKind.LIB,
+ link_to_path = paths.join(repo_runfiles_dirname, site_packages_root, filename),
+ venv_path = filename,
))
# Sort so that we encounter `foo` before `foo/bar`. This ensures we
@@ -291,11 +292,12 @@
first_level_explicit_packages.append(d)
for dirname in first_level_explicit_packages:
- site_packages_symlinks.append((
- paths.join(repo_runfiles_dirname, site_packages_root, dirname),
- dirname,
+ venv_symlinks.append(VenvSymlinkEntry(
+ kind = VenvSymlinkKind.LIB,
+ link_to_path = paths.join(repo_runfiles_dirname, site_packages_root, dirname),
+ venv_path = dirname,
))
- return site_packages_symlinks
+ return venv_symlinks
def _repo_relative_short_path(short_path):
# Convert `../+pypi+foo/some/file.py` to `some/file.py`
