Shard integration tests
diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index e8b3451..8e4692a 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml
@@ -6,6 +6,10 @@ integration_platform: - ubuntu2404 - ubuntu2204 + integration_shard_flags: + - ["--test_tag_filters=shard_0", "--config=rbe"] + - ["--test_tag_filters=shard_1", "--config=rbe"] + - ["--test_tag_filters=shard_2", "--config=rbe"] # - macos # - windows re-enable when rules_bazel_integration_test can support custom test runner on windows. test_flags: @@ -30,8 +34,7 @@ integration_tests: name: "Integration Tests" platform: ${{ integration_platform }} - test_flags: - - "--config=rbe" + test_flags: ${{ integration_shard_flags }} test_targets: - //examples:all rbe_ubuntu1604:
diff --git a/examples/BUILD b/examples/BUILD index 6c71273..ab56ed3 100644 --- a/examples/BUILD +++ b/examples/BUILD
@@ -1,8 +1,4 @@ -load("@bazel_binaries//:defs.bzl", "bazel_binaries") -load( - "@rules_bazel_integration_test//bazel_integration_test:defs.bzl", - "bazel_integration_tests", -) +load(":integration.bzl", "derive_metadata", "example_integration_test_suite") genrule( name = "update_bit_ignore", @@ -21,54 +17,23 @@ executable = True, ) +SHARD_COUNT = 3 + [ - bazel_integration_tests( - name = "%s_test" % example, - timeout = "eternal", - additional_env_inherit = [ - "ANDROID_HOME", - "ANDROID_SDK_ROOT", - "ANDROID_NDK_HOME", - ], - bazel_versions = [ - version - for version in bazel_binaries.versions.all - if version in metadata["only"] or (not metadata["only"] and version not in metadata["exclude"]) - ], - tags = [], - test_runner = "//src/main/kotlin/io/bazel/kotlin/test:BazelIntegrationTestRunner", - workspace_files = glob( - ["%s/**/**" % example], - # exclude any bazel directories if existing - exclude = ["%s/bazel-*/**" % example], - ), - workspace_path = example, + example_integration_test_suite( + name = example, + metadata = metadata, + tags = ["shard_%s" % (idx % SHARD_COUNT)], ) - for (example, metadata) in { - example: { - "exclude": [ - # Cut to the file name, and use it as an excluded bazel version. For exclusion to work - # the file name in the `exclude` directory must match the bazel version in `bazel_binaries.versions.all`. - # This is done as a secondary loop for readability and avoiding over-globbing. - version.rpartition("/")[2] - for version in glob( - ["%s/exclude/*" % example], - allow_empty = True, - ) - ], - "only": [ - # Cut to the file name, and use it as an only bazel version. For exclusion to work - # the file name in the `only` directory must match the bazel version in `bazel_binaries.versions.all`. - # This is done as a secondary loop for readability and avoiding over-globbing. - version.rpartition("/")[2] - for version in glob( - ["%s/only/*" % example], - allow_empty = True, - ) - ], - } + for ( + idx, + (example, metadata), + ) in enumerate({ + example: derive_metadata( + directory = example, + ) for example in { - # Cut to the directory. + # Cut to the directory, de-duplicate via dict. file.partition("/")[0]: True for file in glob( ["**/*"], @@ -77,10 +42,10 @@ "*", # Node is currently broken. "node/**", - # Anvil is broken by a verison upgrade. + # Anvil is broken by a version upgrade. "anvil/**", ], ) } - }.items() + }.items()) ]
diff --git a/examples/integration.bzl b/examples/integration.bzl new file mode 100644 index 0000000..86c33af --- /dev/null +++ b/examples/integration.bzl
@@ -0,0 +1,65 @@ +"""Macros for managing the integration test framework.""" + +load("@bazel_binaries//:defs.bzl", "bazel_binaries") +load( + "@rules_bazel_integration_test//bazel_integration_test:defs.bzl", + "bazel_integration_test", +) + +def derive_metadata(directory): + return struct( + directory = directory, + workspace_files = native.glob( + ["%s/**/**" % directory], + # exclude any bazel directories if existing + exclude = ["%s/bazel-*/**" % directory], + ), + exclude = [ + # Cut to the file name, and use it as an excluded bazel version. For exclusion to work + # the file name in the `exclude` directory must match the bazel version in `bazel_binaries.versions.all`. + # This is done as a secondary loop for readability and avoiding over-globbing. + version.rpartition("/")[2] + for version in native.glob( + ["%s/exclude/*" % directory], + allow_empty = True, + ) + ], + only = [ + # Cut to the file name, and use it as an only bazel version. For exclusion to work + # the file name in the `only` directory must match the bazel version in `bazel_binaries.versions.all`. + # This is done as a secondary loop for readability and avoiding over-globbing. + version.rpartition("/")[2] + for version in native.glob( + ["%s/only/*" % directory], + allow_empty = True, + ) + ], + ) + +def example_integration_test_suite( + name, + metadata, + tags): + for version in bazel_binaries.versions.all: + if version in metadata.only or (not metadata.only and version not in metadata.exclude): + clean_bazel_version = Label(version).name + test_name = "%s_%s_test" % (name, clean_bazel_version) + bazel_integration_test( + name = test_name, + timeout = "eternal", + additional_env_inherit = [ + "ANDROID_HOME", + "ANDROID_SDK_ROOT", + "ANDROID_NDK_HOME", + ], + bazel_version = version, + tags = tags + [clean_bazel_version, name], + test_runner = "//src/main/kotlin/io/bazel/kotlin/test:BazelIntegrationTestRunner", + workspace_files = metadata.workspace_files, + workspace_path = metadata.directory, + ) + + native.test_suite( + name = name, + tags = [name], + )