fix(toolchain): symlink all toolchain files for the host toolchain (#1745)
Previously we would only symlink the interpreter binary itself
when creating the hermetic host toolchain used in setting up the
`whl_library` repositories. This works on UNIX platforms and Windows
if they have the following in their `.bazelrc`:
```
startup --windows_enable_symlinks
```
In our CI we had the same lines but the users did not need to add them
until the `0.29.0` forced them to have them because we actually started
using symlinks on Windows. If the symlinks are not enabled on the host
platform `bazel` tries to be helpful and copies the files over instead
of making the links. We are leveraging this to just symlink all of the
contents of the python interpreter repository for the host platform to
the `_host` toolchain repository.
Fixes #1723
diff --git a/.bazelrc b/.bazelrc
index c911ea5..27e89fa 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -4,8 +4,8 @@
# (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
# To update these lines, execute
# `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
test --test_output=errors
@@ -19,7 +19,6 @@
# Windows makes use of runfiles for some rules
build --enable_runfiles
-startup --windows_enable_symlinks
# Make Bazel 6 use bzlmod by default
common --enable_bzlmod
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d1e116e..4bb076d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -36,13 +36,18 @@
### Fixed
* (bzlmod) pip.parse now does not fail with an empty `requirements.txt`.
+
* (py_wheel) Wheels generated by `py_wheel` now preserve executable bits when
being extracted by `installer` and/or `pip`.
+
* (coverage) During the running of lcov, the stdout/stderr was causing test
failures. By default, suppress output when generating lcov. This can be
overridden by setting 'VERBOSE_COVERAGE'. This change only affect bazel
7.x.x and above.
+* (toolchain) Changed the `host_toolchain` to symlink all files to support
+ Windows host environments without symlink support.
+
### Added
* (py_wheel) Added `requires_file` and `extra_requires_files` attributes.
diff --git a/docs/sphinx/pypi-dependencies.md b/docs/sphinx/pypi-dependencies.md
index 880945f..f08f7fb 100644
--- a/docs/sphinx/pypi-dependencies.md
+++ b/docs/sphinx/pypi-dependencies.md
@@ -27,6 +27,18 @@
For more documentation, including how the rules can update/create a requirements
file, see the bzlmod examples under the {gh-path}`examples` folder.
+We are using a host-platform compatible toolchain by default to setup pip dependencies.
+During the setup phase, we create some symlinks, which may be inefficient on Windows
+by default. In that case use the following `.bazelrc` options to improve performance if
+you have admin privileges:
+```
+startup --windows_enable_symlinks
+```
+
+This will enable symlinks on Windows and help with bootstrap performance of setting up the
+hermetic host python interpreter on this platform. Linux and OSX users should see no
+difference.
+
### Using a WORKSPACE file
To add pip dependencies to your `WORKSPACE`, load the `pip_parse` function and
diff --git a/examples/build_file_generation/.bazelrc b/examples/build_file_generation/.bazelrc
index 7e6911f..e0b1984 100644
--- a/examples/build_file_generation/.bazelrc
+++ b/examples/build_file_generation/.bazelrc
@@ -2,7 +2,6 @@
# Windows requires these for multi-python support:
build --enable_runfiles
-startup --windows_enable_symlinks
# The bzlmod version of this example is in examples/bzlmod_build_file_generation
# Once WORKSPACE support is dropped, this example can be entirely deleted.
diff --git a/examples/bzlmod/.bazelrc b/examples/bzlmod/.bazelrc
index e9a73c5..578342d 100644
--- a/examples/bzlmod/.bazelrc
+++ b/examples/bzlmod/.bazelrc
@@ -6,5 +6,3 @@
# Windows requires these for multi-python support:
build --enable_runfiles
-
-startup --windows_enable_symlinks
diff --git a/examples/bzlmod_build_file_generation/.bazelrc b/examples/bzlmod_build_file_generation/.bazelrc
index 1fbada7..acc7102 100644
--- a/examples/bzlmod_build_file_generation/.bazelrc
+++ b/examples/bzlmod_build_file_generation/.bazelrc
@@ -2,7 +2,6 @@
# Windows requires these for multi-python support:
build --enable_runfiles
-startup --windows_enable_symlinks
common --experimental_enable_bzlmod
diff --git a/examples/multi_python_versions/.bazelrc b/examples/multi_python_versions/.bazelrc
index 3fd6365..58080ab 100644
--- a/examples/multi_python_versions/.bazelrc
+++ b/examples/multi_python_versions/.bazelrc
@@ -2,6 +2,5 @@
# Windows requires these for multi-python support:
build --enable_runfiles
-startup --windows_enable_symlinks
coverage --java_runtime_version=remotejdk_11
diff --git a/examples/pip_parse_vendored/.bazelrc b/examples/pip_parse_vendored/.bazelrc
index b90bf8f..3818a03 100644
--- a/examples/pip_parse_vendored/.bazelrc
+++ b/examples/pip_parse_vendored/.bazelrc
@@ -2,7 +2,6 @@
# Windows requires these for multi-python support:
build --enable_runfiles
-startup --windows_enable_symlinks
# Vendoring requirements.bzl files isn't necessary under bzlmod
# When workspace support is dropped, this example can be removed.
diff --git a/gazelle/.bazelrc b/gazelle/.bazelrc
index 7a67d3e..e10cd78 100644
--- a/gazelle/.bazelrc
+++ b/gazelle/.bazelrc
@@ -10,7 +10,6 @@
# Windows makes use of runfiles for some rules
build --enable_runfiles
-startup --windows_enable_symlinks
# Do NOT implicitly create empty __init__.py files in the runfiles tree.
# By default, these are created in every directory containing Python source code
diff --git a/python/private/repo_utils.bzl b/python/private/repo_utils.bzl
index 4d8b408..d42a67a 100644
--- a/python/private/repo_utils.bzl
+++ b/python/private/repo_utils.bzl
@@ -223,7 +223,7 @@
repo_utils = struct(
execute_checked = _execute_checked,
execute_unchecked = _execute_unchecked,
- execute_check_stdout = _execute_checked_stdout,
+ execute_checked_stdout = _execute_checked_stdout,
is_repo_debug_enabled = _is_repo_debug_enabled,
debug_print = _debug_print,
which_checked = _which_checked,
diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl
index c586208..018c443 100644
--- a/python/private/toolchains_repo.bzl
+++ b/python/private/toolchains_repo.bzl
@@ -238,15 +238,64 @@
(os_name, arch) = get_host_os_arch(rctx)
host_platform = get_host_platform(os_name, arch)
- host_python = rctx.path(
- Label(
- "@@{py_repository}_{host_platform}//:python".format(
- py_repository = rctx.attr.name[:-len("_host")],
- host_platform = host_platform,
- ),
- ),
+ repo = "@@{py_repository}_{host_platform}".format(
+ py_repository = rctx.attr.name[:-len("_host")],
+ host_platform = host_platform,
)
- rctx.symlink(host_python, "python")
+
+ rctx.report_progress("Symlinking interpreter files to the target platform")
+ host_python_repo = rctx.path(Label("{repo}//:BUILD.bazel".format(repo = repo)))
+
+ # The interpreter might not work on platfroms that don't have symlink support if
+ # we just symlink the interpreter itself. rctx.symlink does a copy in such cases
+ # so we can just attempt to symlink all of the directories in the host interpreter
+ # repo, which should be faster than re-downloading it.
+ for p in host_python_repo.dirname.readdir():
+ if p.basename in [
+ # ignore special files created by the repo rule automatically
+ "BUILD.bazel",
+ "MODULE.bazel",
+ "REPO.bazel",
+ "WORKSPACE",
+ "WORKSPACE.bazel",
+ "WORKSPACE.bzlmod",
+ ]:
+ continue
+
+ # symlink works on all platforms that bazel supports, so it should work on
+ # UNIX and Windows with and without symlink support. For better performance
+ # users should enable the symlink startup option, however that requires admin
+ # privileges.
+ rctx.symlink(p, p.basename)
+
+ is_windows = (os_name == WINDOWS_NAME)
+ python_binary = "python.exe" if is_windows else "python"
+
+ # Ensure that we can run the interpreter and check that we are not
+ # using the host interpreter.
+ python_tester_contents = """\
+from pathlib import Path
+import sys
+
+python = Path(sys.executable)
+want_python = str(Path("{python}").resolve())
+got_python = str(Path(sys.executable).resolve())
+
+assert want_python == got_python, \
+ "Expected to use a different interpreter:\\nwant: '{{}}'\\n got: '{{}}'".format(
+ want_python,
+ got_python,
+ )
+""".format(repo = repo.strip("@"), python = python_binary)
+ python_tester = rctx.path("python_tester.py")
+ rctx.file(python_tester, python_tester_contents)
+ repo_utils.execute_checked(
+ rctx,
+ op = "CheckHostInterpreter",
+ arguments = [rctx.path(python_binary), python_tester],
+ )
+ if not rctx.delete(python_tester):
+ fail("Failed to delete the python tester")
host_toolchain = repository_rule(
_host_toolchain_impl,
diff --git a/tests/integration/compile_pip_requirements/.bazelrc b/tests/integration/compile_pip_requirements/.bazelrc
index f23315a..8a42e64 100644
--- a/tests/integration/compile_pip_requirements/.bazelrc
+++ b/tests/integration/compile_pip_requirements/.bazelrc
@@ -2,4 +2,3 @@
# Windows requires these for multi-python support:
build --enable_runfiles
-startup --windows_enable_symlinks
diff --git a/tests/integration/ignore_root_user_error/.bazelrc b/tests/integration/ignore_root_user_error/.bazelrc
index fde94c7..27d7d13 100644
--- a/tests/integration/ignore_root_user_error/.bazelrc
+++ b/tests/integration/ignore_root_user_error/.bazelrc
@@ -4,4 +4,3 @@
# Windows requires these for multi-python support:
build --enable_runfiles
-startup --windows_enable_symlinks
diff --git a/tests/integration/pip_parse/.bazelrc b/tests/integration/pip_parse/.bazelrc
index 76f7da1..efeccbe 100644
--- a/tests/integration/pip_parse/.bazelrc
+++ b/tests/integration/pip_parse/.bazelrc
@@ -1,7 +1,6 @@
# Bazel configuration flags
build --enable_runfiles
-startup --windows_enable_symlinks
# https://docs.bazel.build/versions/main/best-practices.html#using-the-bazelrc-file
try-import %workspace%/user.bazelrc
diff --git a/tests/integration/pip_repository_entry_points/.bazelrc b/tests/integration/pip_repository_entry_points/.bazelrc
index 936806d..55ee08f 100644
--- a/tests/integration/pip_repository_entry_points/.bazelrc
+++ b/tests/integration/pip_repository_entry_points/.bazelrc
@@ -1,7 +1,6 @@
# Bazel configuration flags
build --enable_runfiles
-startup --windows_enable_symlinks
# https://docs.bazel.build/versions/main/best-practices.html#using-the-bazelrc-file
try-import %workspace%/user.bazelrc
