blob: 3eccab78d4003488a6aee8138e89ea3ee7e63805 [file] [log] [blame]
# Copyright 2023 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""Retrigger builds where the latest build failed.
If the most recent build of a builder failed, retrigger it. Exceptions:
* A build of the builder is currently scheduled or started (e.g., about to run,
or already running)
* No recent builds of the builder passed (e.g., this is a true failure and not
a flake)
This will allow other tooling to check the latest build in ci and evaluate
whether it's passing. There are three cases: it's passing, it's failing because
of a flake, and it's failing because it's broken. This should reduce the impact
of the second case because the builder will be retried several times, until
there's a passing build, or there are 10 failing builds in a row.
Tools should check to see if any of the most recent 10 builds passed, and if so
assume the builder is passing. If the builder is broken, the passing build will
be bumped from the 10 most recent builds before long.
"""
from __future__ import annotations
import collections
import fnmatch
import json
from typing import TYPE_CHECKING
from PB.recipe_engine import result as result_pb2
from PB.go.chromium.org.luci.buildbucket.proto import (
build as build_pb2,
builds_service as builds_service_pb2,
common as common_pb2,
project_config as bb_pb2,
)
from PB.go.chromium.org.luci.scheduler.appengine.messages import (
config as scheduler_pb2,
)
from PB.recipes.pigweed.rerunner import InputProperties
from PB.recipe_engine import result
from recipe_engine import post_process
if TYPE_CHECKING: # pragma: no cover
from typing import Generator, Sequence
from recipe_engine import recipe_api, recipe_test_api
DEPS = [
'fuchsia/builder_status',
'fuchsia/luci_config',
'recipe_engine/buildbucket',
'recipe_engine/properties',
'recipe_engine/scheduler',
'recipe_engine/step',
'recipe_engine/time',
]
PROPERTIES = InputProperties
def include_bucket(props: InputProperties, bucket: str) -> bool:
if not props.excluded_buckets and not props.included_buckets:
props.included_buckets.append('*.ci')
props.included_buckets.append('ci')
for excluded_bucket in props.excluded_buckets:
if fnmatch.fnmatch(bucket, excluded_bucket):
return False
for included_bucket in props.included_buckets:
if fnmatch.fnmatch(bucket, included_bucket):
return True
return False
def RunSteps(api: recipe_api.RecipeApi, props: InputProperties):
bb_cfg: bb_pb2.BuildbucketCfg = api.luci_config.buildbucket()
builds_to_launch: list[tuple[str, str]] = []
sched_cfg: scheduler_pb2.ProjectConfig = api.luci_config.scheduler()
jobs: dict[tuple[str, str], list[schedule_pb2.Job]] = (
collections.defaultdict(list)
)
for job in sched_cfg.job:
bucket_builder = (job.buildbucket.bucket, job.buildbucket.builder)
jobs[bucket_builder].append(job)
for bucket in bb_cfg.buckets:
if not include_bucket(props, bucket.name):
api.step(
f'excluding {len(bucket.swarming.builders)} builders in '
f'bucket {bucket.name}',
None,
)
continue
with api.step.nest(bucket.name) as pres:
included = excluded = 0
for builder in bucket.swarming.builders:
num_builds: int = props.num_builds or 10
with api.step.nest(builder.name):
if builder.properties:
if json.loads(builder.properties).get('do_not_rerun'):
api.step.empty('do not rerun')
excluded += 1
continue
if len(jobs[bucket.name, builder.name]) == 1:
job = jobs[bucket.name, builder.name][0]
if job.schedule == "triggered":
api.step.empty('manually triggered')
excluded += 1
continue
included += 1
# Don't DoS buildbucket. (And there's no need for this
# builder to run quickly.)
api.time.sleep(0.1)
status: BuilderStatus = api.builder_status.retrieve(
bucket=bucket.name, builder=builder.name, n=num_builds
)
# If the builder is currently running, don't start a new
# build.
if api.builder_status.is_incomplete(status):
api.step.empty('is incomplete')
continue
# If the builder is currently passing, don't start a new
# build.
if api.builder_status.is_passing(status):
api.step.empty('is passing')
continue
# If the builder hasn't recently passed, this probably isn't
# a flake and we should not start a new build.
if not api.builder_status.has_recently_passed(status):
api.step.empty('no recent passes')
continue
api.step.empty('scheduling')
builds_to_launch.append((bucket.name, builder.name))
pres.step_summary_text = f'included {included}, excluded {excluded}'
# These don't help users much but are useful for testing.
api.step.empty(f'included {included}')
api.step.empty(f'excluded {excluded}')
if not builds_to_launch:
api.step('nothing to launch', None)
return result_pb2.RawResult(
summary_markdown='nothing to launch',
status=common_pb2.SUCCESS,
)
bb_requests: list[builds_service_pb2.ScheduleBuildRequest] = []
sched_ids: list[str] = []
for bucket_builder in builds_to_launch:
if bucket_builder in jobs and len(jobs[bucket_builder]) == 1:
sched_ids.append(jobs[bucket_builder][0].id)
else:
bb_requests.append(
api.buildbucket.schedule_request(
bucket=bucket_builder[0],
builder=bucket_builder[1],
)
)
def scheduler_link(job_id):
project: str = api.buildbucket.build.builder.project
return f'https://luci-scheduler.appspot.com/jobs/{project}/{job_id}'
if props.dry_run:
with api.step.nest('dry-run, not launching builds'):
links: list[tuple[str, str]] = [
(x, scheduler_link(x)) for x in sched_ids
]
for req in bb_requests:
bucket_builder: str = (
f'{req.builder.bucket}/{req.builder.builder}'
)
api.step(bucket_builder, None)
links.append(
(
bucket_builder,
f'https://ci.chromium.org/ui/p/{req.builder.project}/'
f'builders/{bucket_builder}',
)
)
links_combined: str = ''.join(
f'<br/>[{name}]({link})' for name, link in links
)
return result_pb2.RawResult(
summary_markdown=f'dry-run, would have launched: {links_combined}',
status=common_pb2.SUCCESS,
)
with api.step.nest('launch') as pres:
links: list[tuple[str, str]] = []
if sched_ids:
api.scheduler.emit_trigger(
api.scheduler.BuildbucketTrigger(),
api.buildbucket.build.builder.project,
sched_ids,
)
links.extend((x, scheduler_link(x)) for x in sched_ids)
if bb_requests:
builds: list[build_pb2.Build] = api.buildbucket.schedule(
bb_requests
)
for build in builds:
bucket_builder: str = (
f'{build.builder.bucket}/{build.builder.builder}'
)
link: str = api.buildbucket.build_url(build_id=build.id)
pres.links[bucket_builder] = link
links.append((bucket_builder, link))
links_combined: str = ''.join(
f'<br/>[{name}]({link})' for name, link in links
)
return result_pb2.RawResult(
summary_markdown=f'launched: {links_combined}',
status=common_pb2.SUCCESS,
)
def GenTests(api) -> Generator[recipe_test_api.TestData, None, None]:
def properties(
*,
included_buckets: Sequence[str] = (),
excluded_buckets: Sequence[str] = (),
dry_run: bool = False,
ignore_triggered_only_builders: bool = False,
):
props = InputProperties(
dry_run=dry_run,
ignore_triggered_only_builders=ignore_triggered_only_builders,
)
props.included_buckets.extend(included_buckets)
props.excluded_buckets.extend(excluded_buckets)
return api.properties(props)
def test(name, *args, **kwargs):
return api.test(
name,
api.buildbucket.ci_build(project='pigweed'),
*args,
**kwargs,
)
def buildbucket_config(buckets: bb_pb2.Bucket):
cfg = bb_pb2.BuildbucketCfg()
cfg.buckets.extend(buckets)
return cfg
def bucket_config(
name: str,
builders: Sequence[bb_pb2.BuilderConfig],
):
cfg = bb_pb2.Bucket(name=name)
cfg.swarming.builders.extend(builders)
return cfg
def builder_config(name: str):
return bb_pb2.BuilderConfig(
name=name,
properties=json.dumps({'do_not_rerun': 'norerun' in name}),
)
def mock_buildbucket_config(
*buckets_builders: Sequence[tuple[str, Sequence[str]]],
):
buckets: List[bb_pb2.Bucket] = []
for bucket_name, builder_names in buckets_builders:
builders: List[bb_pb2.BuilderConfig] = []
for builder in builder_names:
builders.append(builder_config(builder))
buckets.append(bucket_config(bucket_name, builders))
return api.luci_config.mock_config(
project='pigweed',
config_name='cr-buildbucket.cfg',
data=buildbucket_config(buckets),
)
def mock_scheduler_config(
*buckets_builders: Sequence[tuple[str, Sequence[str]]],
manually_triggered: Sequence[tuple[str, str]] = (),
):
cfg = scheduler_pb2.ProjectConfig()
for bucket_name, builder_names in buckets_builders:
for builder_name in builder_names:
job = scheduler_pb2.Job(
id=f'{bucket_name}-{builder_name}',
buildbucket=scheduler_pb2.BuildbucketTask(
bucket=bucket_name,
builder=builder_name,
),
)
if (bucket_name, builder_name) in manually_triggered:
job.schedule = "triggered"
cfg.job.append(job)
return api.luci_config.mock_config(
project='pigweed',
config_name='luci-scheduler.cfg',
data=cfg,
)
def excluding_bucket(bucket, num):
return api.post_process(
post_process.MustRun,
f'excluding {num} builders in bucket {bucket}',
)
def including_bucket(bucket):
return api.post_process(post_process.MustRun, bucket)
def excluding_builder(bucket, num):
return api.post_process(
post_process.MustRun,
f'{bucket}.excluded {num}',
)
def including_builder(bucket, num):
return api.post_process(
post_process.MustRun,
f'{bucket}.included {num}',
)
def build_status(*statuses: str, prefix: str = ''):
step_name = None
if prefix:
step_name = f'{prefix}.buildbucket.search'
return api.buildbucket.simulated_search_results(
[getattr(api.builder_status, x)() for x in statuses],
step_name=step_name,
)
def assert_do_not_rerun(prefix):
return api.post_process(post_process.MustRun, f'{prefix}.do not rerun')
def assert_manually_triggered(prefix):
return api.post_process(
post_process.MustRun,
f'{prefix}.manually triggered',
)
def assert_skip_is_incomplete(prefix):
return api.post_process(post_process.MustRun, f'{prefix}.is incomplete')
def assert_skip_is_passing(prefix):
return api.post_process(post_process.MustRun, f'{prefix}.is passing')
def assert_skip_no_recent_passes(prefix):
return api.post_process(
post_process.MustRun,
f'{prefix}.no recent passes',
)
def assert_scheduling(prefix):
return api.post_process(post_process.MustRun, f'{prefix}.scheduling')
def assert_buildbucket_scheduled():
return api.post_process(
post_process.MustRun,
f'launch.buildbucket.schedule',
)
def assert_scheduler_triggered():
return api.post_process(
post_process.MustRun,
'launch.luci-scheduler.EmitTriggers',
)
def assert_nothing_to_launch():
return api.post_process(
post_process.MustRun,
f'nothing to launch',
)
def assert_dry_run():
return api.post_process(
post_process.MustRun,
f'dry-run, not launching builds',
)
def drop_expectations_must_be_last():
# No need for expectation files, everything of note here is tested by
# assertions. This must be the last thing added to the test.
return api.post_process(post_process.DropExpectation)
yield test(
'default-ci-only',
mock_buildbucket_config(('try', ('foo', 'bar', 'baz'))),
excluding_bucket('try', 3),
assert_nothing_to_launch(),
drop_expectations_must_be_last(),
)
yield test(
'ci-only',
properties(included_buckets=("*.ci", "ci")),
mock_buildbucket_config(('try', ('foo', 'bar', 'baz'))),
excluding_bucket('try', 3),
assert_nothing_to_launch(),
drop_expectations_must_be_last(),
)
yield test(
'ignore-abc',
properties(
included_buckets=("*.ci", "ci"),
excluded_buckets=("abc.*"),
),
mock_buildbucket_config(('abc.ci', ('foo', 'bar', 'baz'))),
excluding_bucket('abc.ci', 3),
assert_nothing_to_launch(),
drop_expectations_must_be_last(),
)
yield test(
'donotrerun',
mock_buildbucket_config(('abc.ci', ('foo-norerun', 'bar', 'baz'))),
including_bucket('abc.ci'),
including_builder('abc.ci', 2),
excluding_builder('abc.ci', 1),
assert_do_not_rerun('abc.ci.foo-norerun'),
drop_expectations_must_be_last(),
)
yield test(
'scheduled',
mock_buildbucket_config(('abc.ci', ('foo',))),
including_bucket('abc.ci'),
including_builder('abc.ci', 1),
build_status('scheduled', prefix='abc.ci.foo'),
assert_skip_is_incomplete('abc.ci.foo'),
assert_nothing_to_launch(),
drop_expectations_must_be_last(),
)
yield test(
'running',
mock_buildbucket_config(('abc.ci', ('foo',))),
including_bucket('abc.ci'),
including_builder('abc.ci', 1),
build_status('running', prefix='abc.ci.foo'),
assert_skip_is_incomplete('abc.ci.foo'),
assert_nothing_to_launch(),
drop_expectations_must_be_last(),
)
yield test(
'passed',
mock_buildbucket_config(('abc.ci', ('foo',))),
including_bucket('abc.ci'),
including_builder('abc.ci', 1),
build_status('passed', prefix='abc.ci.foo'),
assert_skip_is_passing('abc.ci.foo'),
assert_nothing_to_launch(),
drop_expectations_must_be_last(),
)
yield test(
'no_recent_passes',
mock_buildbucket_config(('abc.ci', ('foo',))),
including_bucket('abc.ci'),
including_builder('abc.ci', 1),
build_status('failure', 'failure', 'failure', prefix='abc.ci.foo'),
assert_skip_no_recent_passes('abc.ci.foo'),
assert_nothing_to_launch(),
drop_expectations_must_be_last(),
)
yield test(
'recent_passes',
mock_buildbucket_config(('abc.ci', ('foo', 'bar'))),
mock_scheduler_config(('abc.ci', ('foo',))),
including_bucket('abc.ci'),
including_builder('abc.ci', 2),
build_status('failure', 'failure', 'passed', prefix='abc.ci.foo'),
build_status('failure', 'failure', 'passed', prefix='abc.ci.bar'),
assert_scheduling('abc.ci.foo'),
assert_scheduling('abc.ci.bar'),
assert_buildbucket_scheduled(),
assert_scheduler_triggered(),
drop_expectations_must_be_last(),
)
yield test(
'manual_only',
properties(ignore_triggered_only_builders=True),
mock_buildbucket_config(('abc.ci', ('foo',))),
mock_scheduler_config(
('abc.ci', ('foo',)),
manually_triggered=(('abc.ci', 'foo'),),
),
including_bucket('abc.ci'),
including_builder('abc.ci', 0),
excluding_builder('abc.ci', 1),
assert_manually_triggered('abc.ci.foo'),
assert_nothing_to_launch(),
drop_expectations_must_be_last(),
)
yield test(
'dry_run',
properties(dry_run=True),
mock_buildbucket_config(('abc.ci', ('foo', 'bar'))),
mock_scheduler_config(('abc.ci', ('foo',))),
including_bucket('abc.ci'),
including_builder('abc.ci', 2),
build_status('failure', 'failure', 'passed', prefix='abc.ci.foo'),
build_status('failure', 'failure', 'passed', prefix='abc.ci.bar'),
assert_scheduling('abc.ci.foo'),
assert_scheduling('abc.ci.bar'),
assert_dry_run(),
drop_expectations_must_be_last(),
)