fix: use runfiles symlinks for venv symlink creation to reduce action count (#3402)
When the venv files are materialized, it can result in many symlink
actions
being created. Rather than register them as regular symlink actions,
batch them
into the runfiles object, which can probably handle large numbers of
them more
efficiently.
Work towards https://github.com/bazel-contrib/rules_python/issues/3401
diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl
index d5c0fa5..669951e 100644
--- a/python/private/py_executable.bzl
+++ b/python/private/py_executable.bzl
@@ -356,7 +356,7 @@
[stage2_bootstrap] + (
venv.files_without_interpreter if venv else []
),
- )
+ ).merge(venv.lib_runfiles)
zip_main = _create_zip_main(
ctx,
stage2_bootstrap = stage2_bootstrap,
@@ -606,7 +606,7 @@
}
venv_app_files = create_venv_app_files(ctx, ctx.attr.deps, venv_dir_map)
- files_without_interpreter = [pth, site_init] + venv_app_files
+ files_without_interpreter = [pth, site_init] + venv_app_files.venv_files
if pyvenv_cfg:
files_without_interpreter.append(pyvenv_cfg)
@@ -629,6 +629,11 @@
venv,
),
),
+ # venv files for user library dependencies (files that are specific
+ # to the executable bootstrap and python runtime aren't here).
+ lib_runfiles = ctx.runfiles(
+ symlinks = venv_app_files.runfiles_symlinks,
+ ),
)
def _map_each_identity(v):
diff --git a/python/private/venv_runfiles.bzl b/python/private/venv_runfiles.bzl
index 05dc296..eeedda4 100644
--- a/python/private/venv_runfiles.bzl
+++ b/python/private/venv_runfiles.bzl
@@ -30,7 +30,12 @@
paths within the current ctx's venv (e.g. `_foo.venv/bin`).
Returns:
- {type}`list[File]` of the files that were created.
+ {type}`struct` with the following attributes:
+ * {type}`list[File]` `venv_files` additional files created for
+ the venv.
+ * {type}`dict[str, File]` `runfiles_symlinks` map intended for
+ the `runfiles.symlinks` argument. A map of main-repo
+ relative paths to File.
"""
# maps venv-relative path to the runfiles path it should point to
@@ -44,16 +49,16 @@
link_map = build_link_map(ctx, entries)
venv_files = []
+ runfiles_symlinks = {}
+
for kind, kind_map in link_map.items():
base = venv_dir_map[kind]
for venv_path, link_to in kind_map.items():
bin_venv_path = paths.join(base, venv_path)
if is_file(link_to):
- if link_to.is_directory:
- venv_link = ctx.actions.declare_directory(bin_venv_path)
- else:
- venv_link = ctx.actions.declare_file(bin_venv_path)
- ctx.actions.symlink(output = venv_link, target_file = link_to)
+ symlink_from = "{}/{}".format(ctx.label.package, bin_venv_path)
+ runfiles_symlinks[symlink_from] = link_to
+
else:
venv_link = ctx.actions.declare_symlink(bin_venv_path)
venv_link_rf_path = runfiles_root_path(ctx, venv_link.short_path)
@@ -64,9 +69,12 @@
to = link_to,
)
ctx.actions.symlink(output = venv_link, target_path = rel_path)
- venv_files.append(venv_link)
+ venv_files.append(venv_link)
- return venv_files
+ return struct(
+ venv_files = venv_files,
+ runfiles_symlinks = runfiles_symlinks,
+ )
# Visible for testing
def build_link_map(ctx, entries):
