recipes/rerunner.py

# Copyright 2023 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""Retrigger builds where the latest build failed.

If the most recent build of a builder failed, retrigger it. Exceptions:

* A build of the builder is currently scheduled or started (e.g., about to run,
  or already running)
* No recent builds of the builder passed (e.g., this is a true failure and not
  a flake)

This will allow other tooling to check the latest build in ci and evaluate
whether it's passing. There are three cases: it's passing, it's failing because
of a flake, and it's failing because it's broken. This should reduce the impact
of the second case because the builder will be retried several times, until
there's a passing build, or there are 10 failing builds in a row.

Tools should check to see if any of the most recent 10 builds passed, and if so
assume the builder is passing. If the builder is broken, the passing build will
be bumped from the 10 most recent builds before long.
"""

import fnmatch
from typing import Sequence, Tuple

from PB.go.chromium.org.luci.buildbucket.proto import project_config as bb_cfg
from PB.recipes.pigweed.rerunner import InputProperties
from PB.recipe_engine import result
from recipe_engine import post_process

DEPS = [
    'fuchsia/buildbucket_util',
    'fuchsia/luci_config',
    'pigweed/builder_status',
    'recipe_engine/buildbucket',
    'recipe_engine/properties',
    'recipe_engine/step',
    'recipe_engine/time',
]

PROPERTIES = InputProperties


def include_bucket(props, bucket):
    if not props.excluded_buckets and not props.included_buckets:
        props.included_buckets.append('*.ci')
        props.included_buckets.append('ci')

    for excluded_bucket in props.excluded_buckets:
        if fnmatch.fnmatch(bucket, excluded_bucket):
            return False

    for included_bucket in props.included_buckets:
        if fnmatch.fnmatch(bucket, included_bucket):
            return True

    return False


def RunSteps(api, props):  # pylint: disable=invalid-name
    cfg = api.luci_config.buildbucket()

    schedule_requests = []

    for bucket in cfg.buckets:
        if not include_bucket(props, bucket.name):
            api.step(
                f'excluding {len(bucket.swarming.builders)} builders in '
                f'bucket {bucket.name}', None)
            continue

        with api.step.nest(bucket.name):
            for builder in bucket.swarming.builders:
                with api.step.nest(builder.name):
                    # Don't DoS buildbucket. (And there's no need for this
                    # builder to run quickly.)
                    api.time.sleep(1)

                    status = api.builder_status.retrieve(
                        bucket=bucket.name,
                        builder=builder.name,
                    )

                    # If the builder is currently running, don't start a new
                    # build.
                    if api.builder_status.is_incomplete(status):
                        api.step('is incomplete', None)
                        continue

                    # If the builder is currently passing, don't start a new
                    # build.
                    if api.builder_status.is_passing(status):
                        api.step('is passing', None)
                        continue

                    # If the builder hasn't recently passed, this probably isn't
                    # a flake and we should not start a new build.
                    if not api.builder_status.has_recently_passed(status):
                        api.step('no recent passes', None)
                        continue

                    api.step('scheduling', None)

                    schedule_requests.append(
                        api.buildbucket.schedule_request(
                            bucket=bucket.name,
                            builder=builder.name,
                        )
                    )

    if not schedule_requests:
        api.step('nothing to launch', None)
        return

    if props.dry_run:
        api.step('dry-run, not launching builds', None)
        return

    with api.step.nest('launch') as pres:
        builds = api.buildbucket.schedule(schedule_requests)
        for build in builds:
            pres.links[f'{build.builder.bucket}/{build.builder.builder}'] = (
                api.buildbucket.build_url(build_id=build.id)
            )


def GenTests(api):  # pylint: disable=invalid-name
    def properties(
        *,
        included_buckets: Sequence[str] = (),
        excluded_buckets: Sequence[str] = (),
        dry_run: bool = False,
    ):
        props = InputProperties(dry_run=dry_run)
        props.included_buckets.extend(included_buckets)
        props.excluded_buckets.extend(excluded_buckets)
        return api.properties(props)

    def test(name):
        return api.test(name) + api.buildbucket.ci_build(project='pigweed')

    def buildbucket_config(buckets: bb_cfg.Bucket):
        cfg = bb_cfg.BuildbucketCfg()
        cfg.buckets.extend(buckets)
        return cfg

    def bucket_config(
        name: str,
        builders: Sequence[bb_cfg.BuilderConfig],
    ):
        cfg = bb_cfg.Bucket(name=name)
        cfg.swarming.builders.extend(builders)
        return cfg

    def builder_config(name: str):
        return bb_cfg.BuilderConfig(name=name)

    def mock_config(*buckets_builders: Sequence[Tuple[str, Sequence[str]]]):
        buckets: List[bb_cfg.Bucket] = []
        for bucket_name, builder_names in buckets_builders:
            builders: List[bb_cfg.BuilderConfig] = []
            for builder in builder_names:
                builders.append(builder_config(builder))
            buckets.append(bucket_config(bucket_name, builders))
        return api.luci_config.mock_config(
            project='pigweed',
            config_name='cr-buildbucket.cfg',
            data=buildbucket_config(buckets),
        )

    def excluding(bucket, num):
        return api.post_process(
            post_process.MustRun,
            f'excluding {num} builders in bucket {bucket}',
        )

    def including(bucket):
        return api.post_process(post_process.MustRun, bucket)

    def build_status(*statuses: str, prefix: str = ''):
        step_name = None
        if prefix:
            step_name = f'{prefix}.buildbucket.search'
        return api.buildbucket.simulated_search_results(
            [getattr(api.builder_status, x)() for x in statuses],
            step_name=step_name,
        )

    def assert_skip_is_incomplete(prefix):
        return api.post_process(post_process.MustRun, f'{prefix}.is incomplete')

    def assert_skip_is_passing(prefix):
        return api.post_process(post_process.MustRun, f'{prefix}.is passing')

    def assert_skip_no_recent_passes(prefix):
        return api.post_process(
            post_process.MustRun,
            f'{prefix}.no recent passes',
        )

    def assert_scheduling(prefix):
        return api.post_process(post_process.MustRun, f'{prefix}.scheduling')

    def assert_launched():
        return api.post_process(
            post_process.MustRun,
            f'launch.buildbucket.schedule',
        )

    def assert_nothing_to_launch():
        return api.post_process(
            post_process.MustRun,
            f'nothing to launch',
        )

    def assert_dry_run():
        return api.post_process(
            post_process.MustRun,
            f'dry-run, not launching builds',
        )

    def drop_expectations_must_be_last():
        # No need for expectation files, everything of note here is tested by
        # assertions. This must be the last thing added to the test.
        return api.post_process(post_process.DropExpectation)

    yield (
        test('default-ci-only')
        + mock_config(('try', ('foo', 'bar', 'baz')))
        + excluding('try', 3)
        + assert_nothing_to_launch()
        + drop_expectations_must_be_last()
    )

    yield (
        test('ci-only')
        + properties(included_buckets=("*.ci", "ci"))
        + mock_config(('try', ('foo', 'bar', 'baz')))
        + excluding('try', 3)
        + assert_nothing_to_launch()
        + drop_expectations_must_be_last()
    )

    yield (
        test('ignore-abc')
        + properties(
            included_buckets=("*.ci", "ci"),
            excluded_buckets=("abc.*"),
        )
        + mock_config(('abc.ci', ('foo', 'bar', 'baz')))
        + excluding('abc.ci', 3)
        + assert_nothing_to_launch()
        + drop_expectations_must_be_last()
    )

    yield (
        test('scheduled')
        + mock_config(('abc.ci', ('foo',)))
        + including('abc.ci')
        + build_status('scheduled', prefix='abc.ci.foo')
        + assert_skip_is_incomplete('abc.ci.foo')
        + assert_nothing_to_launch()
        + drop_expectations_must_be_last()
    )

    yield (
        test('running')
        + mock_config(('abc.ci', ('foo',)))
        + including('abc.ci')
        + build_status('running', prefix='abc.ci.foo')
        + assert_skip_is_incomplete('abc.ci.foo')
        + assert_nothing_to_launch()
        + drop_expectations_must_be_last()
    )

    yield (
        test('passed')
        + mock_config(('abc.ci', ('foo',)))
        + including('abc.ci')
        + build_status('passed', prefix='abc.ci.foo')
        + assert_skip_is_passing('abc.ci.foo')
        + assert_nothing_to_launch()
        + drop_expectations_must_be_last()
    )

    yield (
        test('no_recent_passes')
        + mock_config(('abc.ci', ('foo',)))
        + including('abc.ci')
        + build_status('failure', 'failure', 'failure', prefix='abc.ci.foo')
        + assert_skip_no_recent_passes('abc.ci.foo')
        + assert_nothing_to_launch()
        + drop_expectations_must_be_last()
    )

    yield (
        test('recent_passes')
        + mock_config(('abc.ci', ('foo',)))
        + including('abc.ci')
        + build_status('failure', 'failure', 'passed', prefix='abc.ci.foo')
        + assert_scheduling('abc.ci.foo')
        + assert_launched()
        + drop_expectations_must_be_last()
    )

    yield (
        test('dry_run')
        + properties(dry_run=True)
        + mock_config(('abc.ci', ('foo',)))
        + including('abc.ci')
        + build_status('failure', 'failure', 'passed', prefix='abc.ci.foo')
        + assert_scheduling('abc.ci.foo')
        + assert_dry_run()
        + drop_expectations_must_be_last()
    )