Update test regeneration script to support multiple Bazel versions (#232)
Refactor stardoc_test() to create regenerator binaries for each golden test, and have update-stardoc-tests.sh query for those regenerator binaries and run them with the appropriate version of Bazel. As a side effect, this makes virtually all of Stardoc's tests auto-updateable, reducing maintenance burden.
Also take the opportunity to switch manual legacy tests from Bazel 7.1 to 7.2
diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml
index c3ef4e5..a6ba4cc 100644
--- a/.bazelci/presubmit.yml
+++ b/.bazelci/presubmit.yml
@@ -37,13 +37,13 @@
test_targets:
# Some tests are expected to fail in legacy WORKSPACE mode due to repo name change
- "//test/..."
- - "-//test:attribute_defaults_test_e2e_test"
- - "-//test:function_wrap_multiple_lines_test_e2e_test"
- - "-//test:misc_apis_test_e2e_test"
- - "-//test:module_extension_test_e2e_test"
- - "-//test:proto_format_test_e2e_test"
+ - "-//test:attribute_defaults_test"
+ - "-//test:function_wrap_multiple_lines_test"
+ - "-//test:misc_apis_test"
+ - "-//test:module_extension_test"
+ - "-//test:proto_format_test"
- "-//test:stardoc_self_gen_test"
- - "-//test:table_of_contents_test_e2e_test"
+ - "-//test:table_of_contents_test"
.windows_task_config: &windows_task_config
<<: *common_task_config
@@ -90,15 +90,15 @@
platform: windows
working_directory: test/bzlmod
- bazel_7_1_tests:
- name: Stardoc golden tests requiring Bazel 7.1
+ bazel_7_2_tests:
+ name: Stardoc golden tests requiring Bazel 7.2
platform: ubuntu2004
- bazel: 7.1.2
+ bazel: 7.2.0
build_flags: *common_flags
test_flags: *common_flags
test_targets:
- - "//test:proto_format_test_e2e_test"
- - "//test:macro_kwargs_legacy_test_e2e_test"
+ - "//test:proto_format_test"
+ - "//test:macro_kwargs_legacy_test"
bazel_8_tests:
name: Stardoc golden tests requiring Bazel HEAD
@@ -107,6 +107,6 @@
build_flags: *common_flags
test_flags: *common_flags
test_targets:
- - "//test:macro_kwargs_test_e2e_test"
+ - "//test:macro_kwargs_test"
buildifier: latest
diff --git a/test/BUILD b/test/BUILD
index 3c5bc9e..971b9d9 100644
--- a/test/BUILD
+++ b/test/BUILD
@@ -44,9 +44,9 @@
format = "proto",
golden_file = "testdata/proto_format_test/golden.binaryproto",
input_file = "testdata/proto_format_test/input.bzl",
- # Golden output was generated with Bazel 7.1 and may differ in other versions
+ # Golden output was generated with Bazel 7.2 and may differ in other versions
tags = [
- "bazel_7_1",
+ "bazel_7_2",
"manual",
],
)
@@ -208,9 +208,9 @@
name = "macro_kwargs_legacy_test",
golden_file = "testdata/macro_kwargs_test/legacy_golden.md",
input_file = "testdata/macro_kwargs_test/input.bzl",
- # Golden output was generated with Bazel 7.1 and may differ in other versions
+ # Golden output was generated with Bazel 7.2 and may differ in other versions
tags = [
- "bazel_7_1",
+ "bazel_7_2",
"manual",
],
)
diff --git a/test/stardoc_test.bzl b/test/stardoc_test.bzl
index cf1e7b0..085dc5b 100644
--- a/test/stardoc_test.bzl
+++ b/test/stardoc_test.bzl
@@ -29,11 +29,12 @@
Each invocation creates multiple targets:
1. A `stardoc` target which will generate a new golden file given an input
- file. This target should be used to regenerate the golden file when
- updating Stardoc, named "regenerate_{name}_golden".
+ file, named "{name}_stardoc".
2. An `sh_test` target which verifies that the output of the `stardoc`
target above matches a golden file.
- 3. A bzl_library target for convenient wrapping of input bzl files, named "{name}_lib".
+ 3. A shell script which can be executed via `bazel run` to update the golden
+ file from the `stardoc` target's output, named "{name}_regenerate"
+ 4. A bzl_library target for convenient wrapping of input bzl files, named "{name}_lib".
Args:
name: A unique name to qualify the created targets.
@@ -52,8 +53,9 @@
)
_create_test_targets(
- test_name = "%s_e2e_test" % name,
- genrule_name = "regenerate_%s_golden" % name,
+ test_name = name,
+ stardoc_name = "%s_stardoc" % name,
+ regenerate_name = "%s_regenerate" % name,
lib_name = "%s_lib" % name,
input_file = input_file,
golden_file = golden_file,
@@ -63,13 +65,15 @@
def _create_test_targets(
test_name,
- genrule_name,
+ stardoc_name,
+ regenerate_name,
lib_name,
input_file,
golden_file,
test,
**kwargs):
- actual_generated_doc = "%s.out" % genrule_name
+ actual_generated_doc = "%s.md" % stardoc_name
+ tags = kwargs.get("tags", [])
native.sh_test(
name = test_name,
@@ -82,12 +86,32 @@
actual_generated_doc,
golden_file,
],
- tags = kwargs.get("tags", []),
+ tags = tags,
+ )
+
+ regenerate_sh = "%s.sh" % regenerate_name
+ native.genrule(
+ name = "%s_sh" % regenerate_name,
+ cmd = """cat > $(location %s) <<EOF
+#!/usr/bin/env bash
+cd \\$${BUILD_WORKSPACE_DIRECTORY}
+cp -fv $(location %s) $(location %s)
+EOF""" % (regenerate_sh, actual_generated_doc, golden_file),
+ outs = [regenerate_sh],
+ srcs = [actual_generated_doc, golden_file],
+ tags = tags,
+ )
+
+ native.sh_binary(
+ name = regenerate_name,
+ srcs = [regenerate_sh],
+ data = [actual_generated_doc],
+ tags = tags,
)
if test == "default":
stardoc(
- name = genrule_name,
+ name = stardoc_name,
out = actual_generated_doc,
input = input_file,
deps = [lib_name],
@@ -95,7 +119,7 @@
)
elif test == "html_tables":
html_tables_stardoc(
- name = genrule_name,
+ name = stardoc_name,
out = actual_generated_doc,
input = input_file,
deps = [lib_name],
diff --git a/update-stardoc-docs.sh b/update-stardoc-docs.sh
index d1559bb..41b167f 100755
--- a/update-stardoc-docs.sh
+++ b/update-stardoc-docs.sh
@@ -18,10 +18,11 @@
set -eu
# Allow users to override the bazel command with e.g. bazelisk.
-: "${BAZEL:=bazel}"
+: "${USE_BAZEL_VERSION:=8.0.0-pre.20240603.2}"
+: "${BAZEL:=bazelisk}"
echo "** Generating Stardoc documentation..."
-${BAZEL} build //stardoc:stardoc_doc.md
+USE_BAZEL_VERSION="${USE_BAZEL_VERSION}" ${BAZEL} build //stardoc:stardoc_doc.md
echo "** Copying result to docs/stardoc_rule.md ..."
cp bazel-bin/stardoc/stardoc_doc.md docs/stardoc_rule.md
diff --git a/update-stardoc-tests.sh b/update-stardoc-tests.sh
index c79f50b..9d45698 100755
--- a/update-stardoc-tests.sh
+++ b/update-stardoc-tests.sh
@@ -32,32 +32,35 @@
fi
}
+function update_non_manual_tests () {
+ echo "** Querying for non-manual tests..."
+ regenerate $(${BAZEL} query "kind(sh_binary, //test:all) - attr(tags, manual, //test:all)" | grep _regenerate)
+}
+
+function update_manual_tests_with_tag () {
+ local manual_tag="$1"; shift
+ echo "** Querying for tests tagged \"${manual_tag}\", \"manual\" using 'USE_BAZEL_VERSION=${USE_BAZEL_VERSION} ${BAZEL}' ..."
+ regenerate $(${BAZEL} query "attr(tags, ${manual_tag}, attr(tags, manual, kind(sh_binary, //test:all)))" | grep _regenerate)
+}
+
+function regenerate () {
+ echo "** Regenerating and copying goldens..."
+ for regen_target in $@; do
+ if [[ -z ${USE_BAZEL_VERSION+x} ]]; then
+ echo "** Running '${BAZEL} run ${regen_target}' ..."
+ else
+ echo "** Running 'USE_BAZEL_VERSION=${USE_BAZEL_VERSION} ${BAZEL} run ${regen_target}' ..."
+ fi
+ ${BAZEL} run "${regen_target}"
+ done
+}
+
# Allow users to override the bazel command with e.g. bazelisk.
-: "${BAZEL:=bazel}"
+: "${BAZEL:=bazelisk}"
-# Some tests cannot be automatically regenerated using this script, as they don't fall under the normal
-# golden test pattern or require a specific Bazel version
-EXCLUDED_TESTS="namespace_test_with_allowlist|multi_level_namespace_test_with_allowlist|local_repository_test|stamping_with_stamping_off|macro_kwargs"
-echo "** Querying for tests..."
-regen_targets=$(${BAZEL} query //test:all | grep regenerate_ | grep -vE "_golden\.extract|$EXCLUDED_TESTS")
-
-echo "** Building goldens..."
-${BAZEL} build $regen_targets
-
-echo "** Copying goldens..."
-for regen_target in $regen_targets; do
- base_target_name=$(echo $regen_target | sed 's/\/\/test://g')
- testdata_pkg_name=$(echo $base_target_name | sed 's/regenerate_//g' | sed 's/_golden//g')
- out_file="bazel-bin/test/${base_target_name}.out"
- if [[ $regen_target == *"proto_format"* ]]; then
- ext="binaryproto"
- else
- ext="md"
- fi
- golden="test/testdata/${testdata_pkg_name}/golden.${ext}"
- cp "${out_file}" "${golden}"
- chmod 644 "${golden}"
-done
+update_non_manual_tests
+USE_BAZEL_VERSION="7.2.0" update_manual_tests_with_tag "bazel_7_2"
+USE_BAZEL_VERSION="8.0.0-pre.20240603.2" update_manual_tests_with_tag "bazel_8"
echo "** Files copied."
echo "Please note that not all golden files are correctly copied by this script."
