dedupe_triggers: Initial commit Change-Id: Ie4e21e810ac9bad115e6910ee12780d11d3ec4a8 Bug: b/543993774 Reviewed-on: https://pigweed-review.googlesource.com/c/infra/recipes/+/451092
diff --git a/recipes/dedupe_triggers.proto b/recipes/dedupe_triggers.proto new file mode 100644 index 0000000..12a6a95 --- /dev/null +++ b/recipes/dedupe_triggers.proto
@@ -0,0 +1,25 @@ +// Copyright 2026 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +syntax = "proto3"; + +package recipes.pigweed.dedupe_triggers; + +message InputProperties { + // Builder in the same bucket to trigger. + string builder = 1; + + // Whether deduplication is enabled. If false, always launches subbuild. + bool enabled = 2; +}
diff --git a/recipes/dedupe_triggers.py b/recipes/dedupe_triggers.py new file mode 100644 index 0000000..4fa6db3 --- /dev/null +++ b/recipes/dedupe_triggers.py
@@ -0,0 +1,623 @@ +# Copyright 2026 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Deduplicates builder triggers by commit. + +This recipe looks for recent builds over the past week with an input commit that +matches its triggering commit. Only the commit has to match, not the branch or +repository. + +If enabled: +* If there are completed matching builds, it creates an empty step with a link + to the most recent completed build and duplicates its output properties and + build summary. +* If there are matching builds still running that have higher Buildbucket IDs + than the current build (meaning they were created earlier), it waits until + the build completes and then creates an empty step with a link to the + completed build and duplicates its output properties and build summary. +* If there are no other matching builds (or any running matching builds have + lower Buildbucket IDs and were created later), it triggers the target builder + in the same bucket, waits until it completes, and duplicates its output + properties and build summary. + +If not enabled: +* It searches for matching builds and lists what it would have done if enabled, + but always launches the subbuild and uses its output. +""" + +from __future__ import annotations + +import datetime +from collections.abc import Iterator + +from google.protobuf import json_format + +from recipe_engine import post_process, recipe_api, recipe_test_api + +from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb +from PB.go.chromium.org.luci.buildbucket.proto import ( + builds_service as builds_service_pb, +) +from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb +from PB.recipe_engine import result as result_pb +from PB.recipes.pigweed.dedupe_triggers import InputProperties + +from RECIPE_MODULES.pigweed.raw_result import api as raw_result_api + +DEPS = [ + 'pigweed/raw_result', + 'recipe_engine/buildbucket', + 'recipe_engine/properties', + 'recipe_engine/step', + 'recipe_engine/time', +] + +PROPERTIES = InputProperties + + +def _duplicate_build_output( + api: recipe_api.RecipeScriptApi, + build: build_pb.Build, + step_name: str, +) -> result_pb.RawResult: + pres = api.step.empty(step_name).presentation + pres.links[str(build.id)] = api.buildbucket.build_url(build_id=build.id) + output_props = json_format.MessageToDict(build.output.properties) + for key, value in output_props.items(): + pres.properties[key] = value + + return result_pb.RawResult( + summary_markdown=build.summary_markdown, + status=build.status or common_pb.SUCCESS, + ) + + +def _trigger_and_collect( + api: recipe_api.RecipeScriptApi, + builder: str, +) -> result_pb.RawResult: + req = api.buildbucket.schedule_request( + builder=builder, + ) + scheduled = api.buildbucket.schedule([req]) + subbuild = scheduled[0] + completed = api.buildbucket.collect_build( + subbuild.id, + fields=api.buildbucket.DEFAULT_FIELDS | {'summary_markdown'}, + ) + return _duplicate_build_output(api, completed, f'subbuild {completed.id}') + + +@raw_result_api.wrap_run_steps +def RunSteps( + api: recipe_api.RecipeScriptApi, + props: InputProperties, +) -> result_pb.RawResult | None: + if not props.builder: + raise api.step.StepFailure('Target builder property not specified') + + commit = api.buildbucket.gitiles_commit + commit_id = commit.id if commit else '' + + predicate = builds_service_pb.BuildPredicate() + predicate.builder.project = api.buildbucket.build.builder.project + predicate.builder.bucket = api.buildbucket.build.builder.bucket + predicate.builder.builder = props.builder + + now = api.time.utcnow() + start_time = now - datetime.timedelta(days=7) + predicate.create_time.start_time.FromDatetime(start_time) + + builds = api.buildbucket.search( + predicate, + fields=api.buildbucket.DEFAULT_FIELDS | {'summary_markdown'}, + ) + + matching_builds = [ + b + for b in builds + if b.id != api.buildbucket.build.id + and commit_id + and b.input.gitiles_commit.id == commit_id + ] + + completed_builds = [ + b for b in matching_builds if b.status & common_pb.ENDED_MASK + ] + completed_builds.sort( + key=lambda b: (b.create_time.seconds, b.create_time.nanos, -b.id), + reverse=True, + ) + running_earlier_builds = [ + b + for b in matching_builds + if not (b.status & common_pb.ENDED_MASK) + and b.id > api.buildbucket.build.id + ] + running_earlier_builds.sort( + key=lambda b: (b.create_time.seconds, b.create_time.nanos, -b.id), + reverse=True, + ) + + if props.enabled: + if completed_builds: + target = completed_builds[0] + return _duplicate_build_output( + api, target, f'reused completed build {target.id}' + ) + + if running_earlier_builds: + target = running_earlier_builds[0] + completed = api.buildbucket.collect_build( + target.id, + fields=api.buildbucket.DEFAULT_FIELDS | {'summary_markdown'}, + ) + return _duplicate_build_output( + api, completed, f'reused completed build {completed.id}' + ) + + return _trigger_and_collect(api, props.builder) + + if completed_builds: + target = completed_builds[0] + pres = api.step.empty( + f'would have reused completed build {target.id}' + ).presentation + pres.links[str(target.id)] = api.buildbucket.build_url( + build_id=target.id + ) + + elif running_earlier_builds: + target = running_earlier_builds[0] + pres = api.step.empty( + f'would have waited for running build {target.id}' + ).presentation + pres.links[str(target.id)] = api.buildbucket.build_url( + build_id=target.id + ) + + else: + api.step.empty( + 'no matching builds to reuse, would have triggered ' + f'{props.builder} anyway' + ) + + return _trigger_and_collect(api, props.builder) + + +def GenTests( + api: recipe_test_api.RecipeTestApi, +) -> Iterator[recipe_test_api.TestData]: + target_builder = 'target-builder' + test_commit = '1111111111111111111111111111111111111111' + current_build_id = 5000000000000000000 + + def current_build(**kwargs) -> recipe_test_api.TestData: + return api.buildbucket.ci_build( + project='pigweed', + bucket='ci', + builder='dedupe-builder', + git_repo='https://pigweed.googlesource.com/pigweed/pigweed', + revision=test_commit, + build_id=current_build_id, + **kwargs, + ) + + def search_result(*builds: build_pb.Build) -> recipe_test_api.TestData: + return api.buildbucket.simulated_search_results(list(builds)) + + # Test 1: Completed build found, enabled=True -> Reuses completed build. + completed_build = api.buildbucket.ci_build_message( + project='pigweed', + bucket='ci', + builder=target_builder, + git_repo='https://pigweed.googlesource.com/pigweed/pigweed', + revision=test_commit, + build_id=6000000000000000000, + status='SUCCESS', + summary_markdown='Target build succeeded', + ) + completed_build.output.properties['output_prop'] = 'val1' + + yield ( + api.test('completed_build_enabled') + + current_build() + + api.properties( + InputProperties( + builder=target_builder, + enabled=True, + ) + ) + + search_result(completed_build) + + api.post_check( + post_process.MustRun, + 'reused completed build 6000000000000000000', + ) + + api.post_check( + post_process.SummaryMarkdown, + 'Target build succeeded', + ) + + api.post_check( + post_process.PropertyEquals, + 'output_prop', + 'val1', + ) + + api.post_check( + post_process.DoesNotRun, + 'buildbucket.schedule', + ) + + api.post_process(post_process.DropExpectation) + ) + + # Test 2: Completed build found with different repo/branch, enabled=True + # Only commit has to match. + diff_repo_build = api.buildbucket.ci_build_message( + project='other_proj', + bucket='ci', + builder=target_builder, + git_repo='https://pigweed.googlesource.com/other/repo', + git_ref='refs/heads/other-branch', + revision=test_commit, + build_id=6000000000000000001, + status='SUCCESS', + summary_markdown='Different repo build succeeded', + ) + diff_repo_build.output.properties['diff_prop'] = 'diff_val' + + yield ( + api.test('completed_build_different_repo_branch') + + current_build() + + api.properties( + InputProperties( + builder=target_builder, + enabled=True, + ) + ) + + search_result(diff_repo_build) + + api.post_check( + post_process.MustRun, + 'reused completed build 6000000000000000001', + ) + + api.post_check( + post_process.PropertyEquals, + 'diff_prop', + 'diff_val', + ) + + api.post_process(post_process.DropExpectation) + ) + + # Test 3: Running build created earlier (higher ID), enabled=True -> Waits + # and reuses. + running_earlier_build = api.buildbucket.ci_build_message( + project='pigweed', + bucket='ci', + builder=target_builder, + git_repo='https://pigweed.googlesource.com/pigweed/pigweed', + revision=test_commit, + build_id=current_build_id + 1000, + status='STARTED', + ) + collected_running_build = api.buildbucket.ci_build_message( + project='pigweed', + bucket='ci', + builder=target_builder, + git_repo='https://pigweed.googlesource.com/pigweed/pigweed', + revision=test_commit, + build_id=current_build_id + 1000, + status='SUCCESS', + summary_markdown='Waited build finished successfully', + ) + collected_running_build.output.properties['running_prop'] = 'running_val' + + yield ( + api.test('running_build_created_earlier_enabled') + + current_build() + + api.properties( + InputProperties( + builder=target_builder, + enabled=True, + ) + ) + + search_result(running_earlier_build) + + api.buildbucket.simulated_collect_output([collected_running_build]) + + api.post_check( + post_process.MustRun, + f'reused completed build {current_build_id + 1000}', + ) + + api.post_check( + post_process.SummaryMarkdown, + 'Waited build finished successfully', + ) + + api.post_check( + post_process.PropertyEquals, + 'running_prop', + 'running_val', + ) + + api.post_check( + post_process.DoesNotRun, + 'buildbucket.schedule', + ) + + api.post_process(post_process.DropExpectation) + ) + + subbuild_id = 8945511751514863184 + subbuild_collected = api.buildbucket.ci_build_message( + project='pigweed', + bucket='ci', + builder=target_builder, + git_repo='https://pigweed.googlesource.com/pigweed/pigweed', + revision=test_commit, + build_id=subbuild_id, + status='SUCCESS', + summary_markdown='New subbuild completed', + ) + subbuild_collected.output.properties['sub_prop'] = 'sub_val' + + def simulate_subbuild(build: build_pb.Build) -> recipe_test_api.TestData: + return api.buildbucket.simulated_schedule_output( + batch_response=builds_service_pb.BatchResponse( + responses=[ + dict( + schedule_build=dict( + id=build.id, + builder=build.builder, + ) + ) + ] + ) + ) + api.buildbucket.simulated_collect_output([build]) + + # Test 4: Running build created later (lower ID), enabled=True -> Triggers + # new subbuild. + running_later_build = api.buildbucket.ci_build_message( + project='pigweed', + bucket='ci', + builder=target_builder, + git_repo='https://pigweed.googlesource.com/pigweed/pigweed', + revision=test_commit, + build_id=current_build_id - 1000, + status='STARTED', + ) + + yield ( + api.test('running_build_created_later_enabled') + + current_build() + + api.properties( + InputProperties( + builder=target_builder, + enabled=True, + ) + ) + + search_result(running_later_build) + + simulate_subbuild(subbuild_collected) + + api.post_check( + post_process.MustRun, + 'buildbucket.schedule', + ) + + api.post_check( + post_process.MustRun, + f'subbuild {subbuild_id}', + ) + + api.post_check( + post_process.PropertyEquals, + 'sub_prop', + 'sub_val', + ) + + api.post_process(post_process.DropExpectation) + ) + + # Test 5: No matching builds found, enabled=True -> Triggers subbuild. + other_commit_build = api.buildbucket.ci_build_message( + project='pigweed', + bucket='ci', + builder=target_builder, + git_repo='https://pigweed.googlesource.com/pigweed/pigweed', + revision='2222222222222222222222222222222222222222', + build_id=6000000000000000005, + status='SUCCESS', + ) + + yield ( + api.test('no_other_builds_enabled') + + current_build() + + api.properties( + InputProperties( + builder=target_builder, + enabled=True, + ) + ) + + search_result(other_commit_build) + + simulate_subbuild(subbuild_collected) + + api.post_check( + post_process.MustRun, + 'buildbucket.schedule', + ) + + api.post_check( + post_process.MustRun, + f'subbuild {subbuild_id}', + ) + + api.post_process(post_process.DropExpectation) + ) + + # Test 6: Disabled, completed build exists -> Lists would have reused, + # triggers subbuild. + yield ( + api.test('disabled_with_completed_build') + + current_build() + + api.properties( + InputProperties( + builder=target_builder, + enabled=False, + ) + ) + + search_result(completed_build) + + simulate_subbuild(subbuild_collected) + + api.post_check( + post_process.MustRun, + 'would have reused completed build 6000000000000000000', + ) + + api.post_check( + post_process.MustRun, + 'buildbucket.schedule', + ) + + api.post_check( + post_process.MustRun, + f'subbuild {subbuild_id}', + ) + + api.post_process(post_process.DropExpectation) + ) + + # Test 7: Disabled, running earlier build exists -> Lists would have + # waited, triggers subbuild. + yield ( + api.test('disabled_with_running_earlier_build') + + current_build() + + api.properties( + InputProperties( + builder=target_builder, + enabled=False, + ) + ) + + search_result(running_earlier_build) + + simulate_subbuild(subbuild_collected) + + api.post_check( + post_process.MustRun, + f'would have waited for running build {current_build_id + 1000}', + ) + + api.post_check( + post_process.MustRun, + 'buildbucket.schedule', + ) + + api.post_process(post_process.DropExpectation) + ) + + # Test 8: Disabled, no matching builds -> Lists would have triggered, + # triggers subbuild. + yield ( + api.test('disabled_with_no_builds') + + current_build() + + api.properties( + InputProperties( + builder=target_builder, + enabled=False, + ) + ) + + search_result() + + simulate_subbuild(subbuild_collected) + + api.post_check( + post_process.MustRun, + f'no matching builds to reuse, would have triggered {target_builder} anyway', + ) + + api.post_check( + post_process.MustRun, + 'buildbucket.schedule', + ) + + api.post_process(post_process.DropExpectation) + ) + + # Test 9: No builder specified -> StepFailure + yield ( + api.test('no_builder_specified', status='FAILURE') + + current_build() + + api.properties( + InputProperties( + builder='', + enabled=True, + ) + ) + + api.post_process(post_process.StatusFailure) + + api.post_process(post_process.DropExpectation) + ) + + # Test 10: Reused build failure status and summary propagation. + failed_build = api.buildbucket.ci_build_message( + project='pigweed', + bucket='ci', + builder=target_builder, + git_repo='https://pigweed.googlesource.com/pigweed/pigweed', + revision=test_commit, + build_id=6000000000000000010, + status='FAILURE', + summary_markdown='Compilation error on target', + ) + + yield ( + api.test('reused_build_failure', status='FAILURE') + + current_build() + + api.properties( + InputProperties( + builder=target_builder, + enabled=True, + ) + ) + + search_result(failed_build) + + api.post_process(post_process.StatusFailure) + + api.post_check( + post_process.SummaryMarkdown, + 'Compilation error on target', + ) + + api.post_process(post_process.DropExpectation) + ) + + # Test 11: Multiple completed matching builds -> Most recent one is used. + older_completed_build = api.buildbucket.ci_build_message( + project='pigweed', + bucket='ci', + builder=target_builder, + git_repo='https://pigweed.googlesource.com/pigweed/pigweed', + revision=test_commit, + build_id=7000000000000000000, + status='SUCCESS', + summary_markdown='Older build summary', + ) + older_completed_build.output.properties['which_build'] = 'older' + older_completed_build.create_time.seconds = 1500000000 + + newer_completed_build = api.buildbucket.ci_build_message( + project='pigweed', + bucket='ci', + builder=target_builder, + git_repo='https://pigweed.googlesource.com/pigweed/pigweed', + revision=test_commit, + build_id=6000000000000000000, + status='SUCCESS', + summary_markdown='Newer build summary', + ) + newer_completed_build.output.properties['which_build'] = 'newer' + newer_completed_build.create_time.seconds = 1600000000 + + yield ( + api.test('multiple_completed_builds_uses_most_recent') + + current_build() + + api.properties( + InputProperties( + builder=target_builder, + enabled=True, + ) + ) + + search_result(older_completed_build, newer_completed_build) + + api.post_check( + post_process.MustRun, + 'reused completed build 6000000000000000000', + ) + + api.post_check( + post_process.SummaryMarkdown, + 'Newer build summary', + ) + + api.post_check( + post_process.PropertyEquals, + 'which_build', + 'newer', + ) + + api.post_process(post_process.DropExpectation) + )