tests: add integration test for build_python_zip (#2045)
This is a more comprehensive regression test for verifying
`--build_python_zip` is
actually working
(https://github.com/bazelbuild/rules_python/issues/1840)
This also creates a small framework to make it easier to write
integration tests that
need to customize the environment bazel runs in and check the output of
bazel itself.
I figure this will be helpful for writing simple verification tests for
repository/bzlmod
phase logic (i.e. set the debug env vars and grep the output). While we
should avoid heavy
usage of these bazel-in-bazel tests, a bit of grepping logs would go a
long way for covering
edge cases that examples don't cover.
diff --git a/.bazelrc b/.bazelrc
index 07188a9..bf13baf 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -4,8 +4,8 @@
# (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
# To update these lines, execute
# `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
test --test_output=errors
diff --git a/tests/integration/BUILD.bazel b/tests/integration/BUILD.bazel
index f1c427f..ac475da 100644
--- a/tests/integration/BUILD.bazel
+++ b/tests/integration/BUILD.bazel
@@ -13,6 +13,7 @@
# limitations under the License.
load("@rules_bazel_integration_test//bazel_integration_test:defs.bzl", "default_test_runner")
+load("//python:py_library.bzl", "py_library")
load(":integration_test.bzl", "rules_python_integration_test")
licenses(["notice"])
@@ -102,3 +103,14 @@
bzlmod = False,
workspace_path = "py_cc_toolchain_registered",
)
+
+rules_python_integration_test(
+ name = "custom_commands_test",
+ py_main = "custom_commands_test.py",
+)
+
+py_library(
+ name = "runner_lib",
+ srcs = ["runner.py"],
+ imports = ["../../"],
+)
diff --git a/tests/integration/custom_commands/BUILD.bazel b/tests/integration/custom_commands/BUILD.bazel
new file mode 100644
index 0000000..b0fafff
--- /dev/null
+++ b/tests/integration/custom_commands/BUILD.bazel
@@ -0,0 +1,20 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("@rules_python//python:py_binary.bzl", "py_binary")
+
+py_binary(
+ name = "bin",
+ srcs = ["bin.py"],
+)
diff --git a/tests/integration/custom_commands/MODULE.bazel b/tests/integration/custom_commands/MODULE.bazel
new file mode 100644
index 0000000..5bea812
--- /dev/null
+++ b/tests/integration/custom_commands/MODULE.bazel
@@ -0,0 +1,21 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module(name = "module_under_test")
+
+bazel_dep(name = "rules_python", version = "0.0.0")
+local_path_override(
+ module_name = "rules_python",
+ path = "../../..",
+)
diff --git a/tests/integration/custom_commands/WORKSPACE b/tests/integration/custom_commands/WORKSPACE
new file mode 100644
index 0000000..de90854
--- /dev/null
+++ b/tests/integration/custom_commands/WORKSPACE
@@ -0,0 +1,13 @@
+local_repository(
+ name = "rules_python",
+ path = "../../..",
+)
+
+load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains")
+
+py_repositories()
+
+python_register_toolchains(
+ name = "python_3_11",
+ python_version = "3.11",
+)
diff --git a/tests/integration/custom_commands/WORKSPACE.bzlmod b/tests/integration/custom_commands/WORKSPACE.bzlmod
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/integration/custom_commands/WORKSPACE.bzlmod
diff --git a/tests/integration/custom_commands/bin.py b/tests/integration/custom_commands/bin.py
new file mode 100644
index 0000000..62487b5
--- /dev/null
+++ b/tests/integration/custom_commands/bin.py
@@ -0,0 +1,16 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+print("Hello, world")
+print(__file__)
diff --git a/tests/integration/custom_commands_test.py b/tests/integration/custom_commands_test.py
new file mode 100644
index 0000000..f78ee46
--- /dev/null
+++ b/tests/integration/custom_commands_test.py
@@ -0,0 +1,31 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import unittest
+
+from tests.integration import runner
+
+
+class CustomCommandsTest(runner.TestCase):
+ # Regression test for https://github.com/bazelbuild/rules_python/issues/1840
+ def test_run_build_python_zip_false(self):
+ result = self.run_bazel("run", "--build_python_zip=false", "//:bin")
+ self.assert_result_matches(result, "bazel-out")
+
+
+if __name__ == "__main__":
+ # Enabling this makes the runner log subprocesses as the test goes along.
+ # logging.basicConfig(level = "INFO")
+ unittest.main()
diff --git a/tests/integration/integration_test.bzl b/tests/integration/integration_test.bzl
index 16d6a5a..7a8070a 100644
--- a/tests/integration/integration_test.bzl
+++ b/tests/integration/integration_test.bzl
@@ -19,6 +19,7 @@
"bazel_integration_tests",
"integration_test_utils",
)
+load("//python:py_test.bzl", "py_test")
def rules_python_integration_test(
name,
@@ -26,6 +27,7 @@
bzlmod = True,
gazelle_plugin = False,
tags = None,
+ py_main = None,
**kwargs):
"""Runs a bazel-in-bazel integration test.
@@ -37,10 +39,24 @@
disable bzlmod.
gazelle_plugin: Whether the test uses the gazelle plugin.
tags: Test tags.
+ py_main: Optional `.py` file to run tests using. When specified, a
+ python based test runner is used, and this source file is the main
+ entry point and responsible for executing tests.
**kwargs: Passed to the upstream `bazel_integration_tests` rule.
"""
workspace_path = workspace_path or name.removesuffix("_test")
- if bzlmod:
+ if py_main:
+ test_runner = name + "_py_runner"
+ py_test(
+ name = test_runner,
+ srcs = [py_main],
+ main = py_main,
+ deps = [":runner_lib"],
+ # Hide from ... patterns; should only be run as part
+ # of the bazel integration test
+ tags = ["manual"],
+ )
+ elif bzlmod:
if gazelle_plugin:
test_runner = "//tests/integration:test_runner_gazelle_plugin"
else:
diff --git a/tests/integration/runner.py b/tests/integration/runner.py
new file mode 100644
index 0000000..9414a86
--- /dev/null
+++ b/tests/integration/runner.py
@@ -0,0 +1,131 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import os.path
+import pathlib
+import re
+import shlex
+import subprocess
+import unittest
+
+_logger = logging.getLogger(__name__)
+
+class ExecuteError(Exception):
+ def __init__(self, result):
+ self.result = result
+ def __str__(self):
+ return self.result.describe()
+
+class ExecuteResult:
+ def __init__(
+ self,
+ args: list[str],
+ env: dict[str, str],
+ cwd: pathlib.Path,
+ proc_result: subprocess.CompletedProcess,
+ ):
+ self.args = args
+ self.env = env
+ self.cwd = cwd
+ self.exit_code = proc_result.returncode
+ self.stdout = proc_result.stdout
+ self.stderr = proc_result.stderr
+
+ def describe(self) -> str:
+ env_lines = [
+ " " + shlex.quote(f"{key}={value}")
+ for key, value in sorted(self.env.items())
+ ]
+ env = " \\\n".join(env_lines)
+ args = shlex.join(self.args)
+ maybe_stdout_nl = "" if self.stdout.endswith("\n") else "\n"
+ maybe_stderr_nl = "" if self.stderr.endswith("\n") else "\n"
+ return f"""\
+COMMAND:
+cd {self.cwd} && \\
+env \\
+{env} \\
+ {args}
+RESULT: exit_code: {self.exit_code}
+===== STDOUT START =====
+{self.stdout}{maybe_stdout_nl}===== STDOUT END =====
+===== STDERR START =====
+{self.stderr}{maybe_stderr_nl}===== STDERR END =====
+"""
+
+
+class TestCase(unittest.TestCase):
+ def setUp(self):
+ super().setUp()
+ self.repo_root = pathlib.Path(os.environ["BIT_WORKSPACE_DIR"])
+ self.bazel = pathlib.Path(os.environ["BIT_BAZEL_BINARY"])
+ outer_test_tmpdir = pathlib.Path(os.environ["TEST_TMPDIR"])
+ self.test_tmp_dir = outer_test_tmpdir / "bit_test_tmp"
+ # Put the global tmp not under the test tmp to better match how a real
+ # execution has entirely different directories for these.
+ self.tmp_dir = outer_test_tmpdir / "bit_tmp"
+ self.bazel_env = {
+ "PATH": os.environ["PATH"],
+ "TEST_TMPDIR": str(self.test_tmp_dir),
+ "TMP": str(self.tmp_dir),
+ # For some reason, this is necessary for Bazel 6.4 to work.
+ # If not present, it can't find some bash helpers in @bazel_tools
+ "RUNFILES_DIR": os.environ["TEST_SRCDIR"]
+ }
+
+ def run_bazel(self, *args: str, check: bool = True) -> ExecuteResult:
+ """Run a bazel invocation.
+
+ Args:
+ *args: The args to pass to bazel; the leading `bazel` command is
+ added automatically
+ check: True if the execution must succeed, False if failure
+ should raise an error.
+ Returns:
+ An `ExecuteResult` from running Bazel
+ """
+ args = [str(self.bazel), *args]
+ env = self.bazel_env
+ _logger.info("executing: %s", shlex.join(args))
+ cwd = self.repo_root
+ proc_result = subprocess.run(
+ args=args,
+ text=True,
+ capture_output=True,
+ cwd=cwd,
+ env=env,
+ check=False,
+ )
+ exec_result = ExecuteResult(args, env, cwd, proc_result)
+ if check and exec_result.exit_code:
+ raise ExecuteError(exec_result)
+ else:
+ return exec_result
+
+ def assert_result_matches(self, result: ExecuteResult, regex: str) -> None:
+ """Assert stdout/stderr of an invocation matches a regex.
+
+ Args:
+ result: ExecuteResult from `run_bazel` whose stdout/stderr will
+ be checked.
+ regex: Pattern to match, using `re.search` semantics.
+ """
+ if not re.search(regex, result.stdout + result.stderr):
+ self.fail(
+ "Bazel output did not match expected pattern\n"
+ + f"expected pattern: {regex}\n"
+ + f"invocation details:\n{result.describe()}"
+ )
