rerunner: Initial commit
Bug: b/288464961
Change-Id: I53aecce872bf0fe59ac14bf234867de16cf84343
Reviewed-on: https://pigweed-review.googlesource.com/c/infra/recipes/+/152853
Reviewed-by: Marc-Antoine Ruel <maruel@google.com>
Commit-Queue: Rob Mohr <mohrr@google.com>
diff --git a/recipes/rerunner.proto b/recipes/rerunner.proto
new file mode 100644
index 0000000..d7bb790
--- /dev/null
+++ b/recipes/rerunner.proto
@@ -0,0 +1,29 @@
+// Copyright 2023 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+syntax = "proto3";
+
+package recipes.pigweed.rerunner;
+
+message InputProperties {
+ // Buckets to include from rerunning. Uses glob-style matching.
+ repeated string included_buckets = 1;
+
+ // Buckets to exclude from rerunning. Uses glob-style matching. Overrides
+ // included_buckets.
+ repeated string excluded_buckets = 2;
+
+ // Don't do anything that has external effects.
+ bool dry_run = 3;
+}
diff --git a/recipes/rerunner.py b/recipes/rerunner.py
new file mode 100644
index 0000000..3a543d6
--- /dev/null
+++ b/recipes/rerunner.py
@@ -0,0 +1,326 @@
+# Copyright 2023 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Retrigger builds where the latest build failed.
+
+If the most recent build of a builder failed, retrigger it. Exceptions:
+
+* A build of the builder is currently scheduled or started (e.g., about to run,
+ or already running)
+* No recent builds of the builder passed (e.g., this is a true failure and not
+ a flake)
+
+This will allow other tooling to check the latest build in ci and evaluate
+whether it's passing. There are three cases: it's passing, it's failing because
+of a flake, and it's failing because it's broken. This should reduce the impact
+of the second case because the builder will be retried several times, until
+there's a passing build, or there are 10 failing builds in a row.
+
+Tools should check to see if any of the most recent 10 builds passed, and if so
+assume the builder is passing. If the builder is broken, the passing build will
+be bumped from the 10 most recent builds before long.
+"""
+
+import fnmatch
+from typing import Sequence, Tuple
+
+from PB.go.chromium.org.luci.buildbucket.proto import project_config as bb_cfg
+from PB.recipes.pigweed.rerunner import InputProperties
+from PB.recipe_engine import result
+from recipe_engine import post_process
+
+DEPS = [
+ 'fuchsia/buildbucket_util',
+ 'fuchsia/luci_config',
+ 'pigweed/builder_status',
+ 'recipe_engine/buildbucket',
+ 'recipe_engine/properties',
+ 'recipe_engine/step',
+ 'recipe_engine/time',
+]
+
+PROPERTIES = InputProperties
+
+
+def include_bucket(props, bucket):
+ if not props.excluded_buckets and not props.included_buckets:
+ props.included_buckets.append('*.ci')
+ props.included_buckets.append('ci')
+
+ for excluded_bucket in props.excluded_buckets:
+ if fnmatch.fnmatch(bucket, excluded_bucket):
+ return False
+
+ for included_bucket in props.included_buckets:
+ if fnmatch.fnmatch(bucket, included_bucket):
+ return True
+
+ return False
+
+
+def RunSteps(api, props): # pylint: disable=invalid-name
+ cfg = api.luci_config.buildbucket()
+
+ schedule_requests = []
+
+ for bucket in cfg.buckets:
+ if not include_bucket(props, bucket.name):
+ api.step(
+ f'excluding {len(bucket.swarming.builders)} builders in '
+ f'bucket {bucket.name}', None)
+ continue
+
+ with api.step.nest(bucket.name):
+ for builder in bucket.swarming.builders:
+ with api.step.nest(builder.name):
+ # Don't DoS buildbucket. (And there's no need for this
+ # builder to run quickly.)
+ api.time.sleep(1)
+
+ status = api.builder_status.retrieve(
+ bucket=bucket.name,
+ builder=builder.name,
+ )
+
+ # If the builder is currently running, don't start a new
+ # build.
+ if api.builder_status.is_incomplete(status):
+ api.step('is incomplete', None)
+ continue
+
+ # If the builder is currently passing, don't start a new
+ # build.
+ if api.builder_status.is_passing(status):
+ api.step('is passing', None)
+ continue
+
+ # If the builder hasn't recently passed, this probably isn't
+ # a flake and we should not start a new build.
+ if not api.builder_status.has_recently_passed(status):
+ api.step('no recent passes', None)
+ continue
+
+ api.step('scheduling', None)
+
+ schedule_requests.append(
+ api.buildbucket.schedule_request(
+ bucket=bucket.name,
+ builder=builder.name,
+ )
+ )
+
+ if not schedule_requests:
+ api.step('nothing to launch', None)
+ return
+
+ if props.dry_run:
+ api.step('dry-run, not launching builds', None)
+ return
+
+ with api.step.nest('launch') as pres:
+ builds = api.buildbucket.schedule(schedule_requests)
+ for build in builds:
+ pres.links[f'{build.builder.bucket}/{build.builder.builder}'] = (
+ api.buildbucket.build_url(build_id=build.id)
+ )
+
+
+def GenTests(api): # pylint: disable=invalid-name
+ def properties(
+ *,
+ included_buckets: Sequence[str] = (),
+ excluded_buckets: Sequence[str] = (),
+ dry_run: bool = False,
+ ):
+ props = InputProperties(dry_run=dry_run)
+ props.included_buckets.extend(included_buckets)
+ props.excluded_buckets.extend(excluded_buckets)
+ return api.properties(props)
+
+ def test(name):
+ return api.test(name) + api.buildbucket.ci_build(project='pigweed')
+
+ def buildbucket_config(buckets: bb_cfg.Bucket):
+ cfg = bb_cfg.BuildbucketCfg()
+ cfg.buckets.extend(buckets)
+ return cfg
+
+ def bucket_config(
+ name: str,
+ builders: Sequence[bb_cfg.BuilderConfig],
+ ):
+ cfg = bb_cfg.Bucket(name=name)
+ cfg.swarming.builders.extend(builders)
+ return cfg
+
+ def builder_config(name: str):
+ return bb_cfg.BuilderConfig(name=name)
+
+ def mock_config(*buckets_builders: Sequence[Tuple[str, Sequence[str]]]):
+ buckets: List[bb_cfg.Bucket] = []
+ for bucket_name, builder_names in buckets_builders:
+ builders: List[bb_cfg.BuilderConfig] = []
+ for builder in builder_names:
+ builders.append(builder_config(builder))
+ buckets.append(bucket_config(bucket_name, builders))
+ return api.luci_config.mock_config(
+ project='pigweed',
+ config_name='cr-buildbucket.cfg',
+ data=buildbucket_config(buckets),
+ )
+
+ def excluding(bucket, num):
+ return api.post_process(
+ post_process.MustRun,
+ f'excluding {num} builders in bucket {bucket}',
+ )
+
+ def including(bucket):
+ return api.post_process(post_process.MustRun, bucket)
+
+ def build_status(*statuses: str, prefix: str = ''):
+ step_name = None
+ if prefix:
+ step_name = f'{prefix}.buildbucket.search'
+ return api.buildbucket.simulated_search_results(
+ [getattr(api.builder_status, x)() for x in statuses],
+ step_name=step_name,
+ )
+
+ def assert_skip_is_incomplete(prefix):
+ return api.post_process(post_process.MustRun, f'{prefix}.is incomplete')
+
+ def assert_skip_is_passing(prefix):
+ return api.post_process(post_process.MustRun, f'{prefix}.is passing')
+
+ def assert_skip_no_recent_passes(prefix):
+ return api.post_process(
+ post_process.MustRun,
+ f'{prefix}.no recent passes',
+ )
+
+ def assert_scheduling(prefix):
+ return api.post_process(post_process.MustRun, f'{prefix}.scheduling')
+
+ def assert_launched():
+ return api.post_process(
+ post_process.MustRun,
+ f'launch.buildbucket.schedule',
+ )
+
+ def assert_nothing_to_launch():
+ return api.post_process(
+ post_process.MustRun,
+ f'nothing to launch',
+ )
+
+ def assert_dry_run():
+ return api.post_process(
+ post_process.MustRun,
+ f'dry-run, not launching builds',
+ )
+
+ def drop_expectations_must_be_last():
+ # No need for expectation files, everything of note here is tested by
+ # assertions. This must be the last thing added to the test.
+ return api.post_process(post_process.DropExpectation)
+
+ yield (
+ test('default-ci-only')
+ + mock_config(('try', ('foo', 'bar', 'baz')))
+ + excluding('try', 3)
+ + assert_nothing_to_launch()
+ + drop_expectations_must_be_last()
+ )
+
+ yield (
+ test('ci-only')
+ + properties(included_buckets=("*.ci", "ci"))
+ + mock_config(('try', ('foo', 'bar', 'baz')))
+ + excluding('try', 3)
+ + assert_nothing_to_launch()
+ + drop_expectations_must_be_last()
+ )
+
+ yield (
+ test('ignore-abc')
+ + properties(
+ included_buckets=("*.ci", "ci"),
+ excluded_buckets=("abc.*"),
+ )
+ + mock_config(('abc.ci', ('foo', 'bar', 'baz')))
+ + excluding('abc.ci', 3)
+ + assert_nothing_to_launch()
+ + drop_expectations_must_be_last()
+ )
+
+ yield (
+ test('scheduled')
+ + mock_config(('abc.ci', ('foo',)))
+ + including('abc.ci')
+ + build_status('scheduled', prefix='abc.ci.foo')
+ + assert_skip_is_incomplete('abc.ci.foo')
+ + assert_nothing_to_launch()
+ + drop_expectations_must_be_last()
+ )
+
+ yield (
+ test('running')
+ + mock_config(('abc.ci', ('foo',)))
+ + including('abc.ci')
+ + build_status('running', prefix='abc.ci.foo')
+ + assert_skip_is_incomplete('abc.ci.foo')
+ + assert_nothing_to_launch()
+ + drop_expectations_must_be_last()
+ )
+
+ yield (
+ test('passed')
+ + mock_config(('abc.ci', ('foo',)))
+ + including('abc.ci')
+ + build_status('passed', prefix='abc.ci.foo')
+ + assert_skip_is_passing('abc.ci.foo')
+ + assert_nothing_to_launch()
+ + drop_expectations_must_be_last()
+ )
+
+ yield (
+ test('no_recent_passes')
+ + mock_config(('abc.ci', ('foo',)))
+ + including('abc.ci')
+ + build_status('failure', 'failure', 'failure', prefix='abc.ci.foo')
+ + assert_skip_no_recent_passes('abc.ci.foo')
+ + assert_nothing_to_launch()
+ + drop_expectations_must_be_last()
+ )
+
+ yield (
+ test('recent_passes')
+ + mock_config(('abc.ci', ('foo',)))
+ + including('abc.ci')
+ + build_status('failure', 'failure', 'passed', prefix='abc.ci.foo')
+ + assert_scheduling('abc.ci.foo')
+ + assert_launched()
+ + drop_expectations_must_be_last()
+ )
+
+ yield (
+ test('dry_run')
+ + properties(dry_run=True)
+ + mock_config(('abc.ci', ('foo',)))
+ + including('abc.ci')
+ + build_status('failure', 'failure', 'passed', prefix='abc.ci.foo')
+ + assert_scheduling('abc.ci.foo')
+ + assert_dry_run()
+ + drop_expectations_must_be_last()
+ )
