| # Copyright 2025 The Pigweed Authors |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may not |
| # use this file except in compliance with the License. You may obtain a copy of |
| # the License at |
| # |
| # https://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations under |
| # the License. |
| |
| from __future__ import annotations |
| |
| import collections |
| import dataclasses |
| import datetime |
| import itertools |
| import fnmatch |
| import json |
| from typing import TYPE_CHECKING |
| import urllib |
| |
| from PB.recipes.pigweed.bisector import InputProperties |
| from PB.recipe_engine import result as result_pb |
| from PB.go.chromium.org.luci.buildbucket.proto import ( |
| build as build_pb, |
| builds_service as builds_service_pb, |
| common as common_pb, |
| project_config as bb_pb, |
| ) |
| from PB.recipe_modules.pigweed.bisector.options import ( |
| Options as BisectorOptions, |
| ) |
| from PB.recipe_modules.pigweed.checkout.options import ( |
| Options as CheckoutOptions, |
| ) |
| from recipe_engine import post_process |
| |
| if TYPE_CHECKING: # pragma: no cover |
| from typing import Generator, Sequence |
| from recipe_engine import recipe_api, recipe_test_api |
| |
| _PIGWEED = 'https://pigweed.googlesource.com/pigweed/pigweed' |
| PROPERTIES = InputProperties |
| |
| DEPS = [ |
| 'fuchsia/builder_state', |
| 'fuchsia/builder_status', |
| 'pigweed/bisector', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/luci_config', |
| 'recipe_engine/properties', |
| ] |
| |
| |
| def RunSteps(api: recipe_api.RecipeApi, props: InputProperties): |
| return api.bisector(props.bisector_options) |
| |
| |
| def GenTests(api) -> Generator[recipe_test_api.TestData, None, None]: |
| def test(name, *args, **kwargs): |
| return api.test( |
| name, |
| api.buildbucket.ci_build(project='pigweed'), |
| *args, |
| **kwargs, |
| ) |
| |
| def buildbucket_config(buckets: bb_pb.Bucket): |
| cfg = bb_pb.BuildbucketCfg() |
| cfg.buckets.extend(buckets) |
| return cfg |
| |
| def bucket_config( |
| name: str, |
| builders: Sequence[bb_pb.BuilderConfig], |
| ) -> list[bb_pb.Bucket]: |
| cfg = bb_pb.Bucket(name=name) |
| cfg.swarming.builders.extend(builders) |
| return cfg |
| |
| def builder_config( |
| name: str, |
| bisect: bool = True, |
| properties: bool = True, |
| ) -> bb_pb.BuilderConfig: |
| props = {'do_not_bisect': not bisect} |
| |
| kwargs = {'name': name} |
| if properties: |
| kwargs['properties'] = json.dumps(props) |
| |
| return bb_pb.BuilderConfig(**kwargs) |
| |
| def mock_buildbucket_config(bucket: str, *builders: bb_pb.BuilderConfig): |
| return api.luci_config.mock_config( |
| project='pigweed', |
| config_name='cr-buildbucket.cfg', |
| data=buildbucket_config([bucket_config(bucket, builders)]), |
| ) |
| |
| def excluding_bucket(bucket, num): |
| return api.post_process( |
| post_process.MustRun, |
| f'excluding {num} builders in bucket {bucket}', |
| ) |
| |
| def including_bucket(bucket): |
| return api.post_process(post_process.MustRun, bucket) |
| |
| def excluding_builder(bucket, num): |
| return api.post_process( |
| post_process.MustRun, |
| f'{bucket}.excluded {num}', |
| ) |
| |
| def including_builder(bucket, num): |
| return api.post_process( |
| post_process.MustRun, |
| f'{bucket}.included {num}', |
| ) |
| |
| def build_status( |
| *statuses: str, |
| spacing: int = 5, |
| prefix: str = '', |
| remote: str | None | Sequence[str | None] = _PIGWEED, |
| ): |
| step_name = None |
| if prefix: |
| step_name = f'{prefix}.buildbucket.search' |
| |
| remotes: list[str | None] |
| if isinstance(remote, (str, type(None))): |
| remotes = itertools.cycle([remote]) |
| else: |
| remotes = itertools.cycle(remote) |
| |
| builds = [] |
| for i, (status, rem) in enumerate(zip(statuses, remotes)): |
| commit = api.bisector.commit40(spacing * i) |
| if '-' in status: |
| commit = api.bisector.commit40(int(status.split('-')[1])) |
| status = status.split('-')[0] |
| |
| build = getattr(api.builder_status, status)() |
| |
| if rem: |
| parsed = urllib.parse.urlparse(rem) |
| build.input.gitiles_commit.id = commit |
| build.input.gitiles_commit.ref = 'refs/heads/main' |
| build.input.gitiles_commit.host = parsed.netloc |
| build.input.gitiles_commit.project = parsed.path.lstrip('/') |
| |
| builds.append(build) |
| |
| return api.buildbucket.simulated_search_results( |
| builds, |
| step_name=step_name, |
| ) |
| |
| def assert_count_too_low(prefix): |
| return api.post_process(post_process.MustRun, f'{prefix}.count too low') |
| |
| def assert_do_not_bisect(prefix): |
| return api.post_process(post_process.MustRun, f'{prefix}.do not bisect') |
| |
| def assert_no_properties(prefix): |
| return api.post_process(post_process.MustRun, f'{prefix}.no properties') |
| |
| def assert_no_remote(prefix): |
| return api.post_process(post_process.MustRun, f'{prefix}.no remote') |
| |
| def assert_skip_no_recent_passes(prefix): |
| return api.post_process( |
| post_process.MustRun, |
| f'{prefix}.no recent passes', |
| ) |
| |
| def assert_too_many_infra_failures(prefix): |
| return api.post_process( |
| post_process.MustRun, |
| f'{prefix}.too many infra failures', |
| ) |
| |
| def assert_no_need_to_bisect(prefix): |
| return api.post_process( |
| post_process.MustRun, |
| f'{prefix}.no need to bisect', |
| ) |
| |
| def assert_already_attributed(prefix): |
| return api.post_process( |
| post_process.MustRun, |
| f'{prefix}.already attributed', |
| ) |
| |
| def assert_picked(prefix, i): |
| return api.post_process(post_process.MustRun, f'{prefix}.picked.{i}') |
| |
| def assert_already_scheduled(prefix): |
| return api.post_process( |
| post_process.MustRun, |
| f'{prefix}.already scheduled', |
| ) |
| |
| def assert_scheduling(prefix): |
| return api.post_process(post_process.MustRun, f'{prefix}.scheduling') |
| |
| def assert_launching(n: int, dry_run: bool = False): |
| launch = 'dry-run, not launching builds' if dry_run else 'launch' |
| return api.post_process(post_process.MustRun, f'{launch}.{n} requests') |
| |
| def assert_buildbucket_scheduled(): |
| return api.post_process( |
| post_process.MustRun, |
| f'launch.buildbucket.schedule', |
| ) |
| |
| def assert_nothing_to_launch(): |
| return api.post_process( |
| post_process.MustRun, |
| f'nothing to launch', |
| ) |
| |
| def assert_dry_run(): |
| return api.post_process( |
| post_process.MustRun, |
| f'dry-run, not launching builds', |
| ) |
| |
| def drop_expectations_must_be_last(): |
| # No need for expectation files, everything of note here is tested by |
| # assertions. This must be the last thing added to the test. |
| return api.post_process(post_process.DropExpectation) |
| |
| def properties(**kwargs): |
| return api.properties( |
| bisector_options=api.bisector.options(**kwargs), |
| ) |
| |
| yield test( |
| 'default-ci-only', |
| properties(), |
| mock_buildbucket_config( |
| 'try', |
| builder_config('foo'), |
| builder_config('bar'), |
| builder_config('baz'), |
| ), |
| excluding_bucket('try', 3), |
| assert_nothing_to_launch(), |
| drop_expectations_must_be_last(), |
| ) |
| |
| yield test( |
| 'ci-only', |
| properties(included_buckets=('*.ci', 'ci')), |
| mock_buildbucket_config( |
| 'try', |
| builder_config('foo'), |
| builder_config('bar'), |
| builder_config('baz'), |
| ), |
| excluding_bucket('try', 3), |
| assert_nothing_to_launch(), |
| drop_expectations_must_be_last(), |
| ) |
| |
| yield test( |
| 'ignore-abc', |
| properties( |
| included_buckets=('*.ci', 'ci'), |
| excluded_buckets=('abc.*'), |
| ), |
| mock_buildbucket_config( |
| 'abc.ci', |
| builder_config('foo'), |
| builder_config('bar'), |
| builder_config('baz'), |
| ), |
| excluding_bucket('abc.ci', 3), |
| assert_nothing_to_launch(), |
| drop_expectations_must_be_last(), |
| ) |
| |
| yield test( |
| 'donotbisect', |
| properties(), |
| mock_buildbucket_config( |
| 'abc.ci', |
| builder_config('foo', bisect=False), |
| builder_config('bar'), |
| builder_config('baz'), |
| ), |
| including_bucket('abc.ci'), |
| build_status('passed', prefix='abc.ci.bar', spacing=5), |
| build_status('passed', prefix='abc.ci.baz', spacing=5), |
| including_builder('abc.ci', 2), |
| excluding_builder('abc.ci', 1), |
| assert_do_not_bisect('abc.ci.foo'), |
| drop_expectations_must_be_last(), |
| ) |
| |
| yield test( |
| 'repo_noprops', |
| mock_buildbucket_config( |
| 'abc.ci', |
| builder_config('repo'), |
| builder_config('noprops', properties=False), |
| ), |
| including_bucket('abc.ci'), |
| assert_no_properties('abc.ci.noprops'), |
| excluding_builder('abc.ci', 2), |
| assert_nothing_to_launch(), |
| drop_expectations_must_be_last(), |
| ) |
| |
| yield test( |
| 'noremote', |
| mock_buildbucket_config('abc.ci', builder_config('foo')), |
| including_bucket('abc.ci'), |
| build_status('passed', prefix='abc.ci.foo', remote=None), |
| assert_no_remote('abc.ci.foo'), |
| excluding_builder('abc.ci', 1), |
| assert_nothing_to_launch(), |
| drop_expectations_must_be_last(), |
| ) |
| |
| yield test( |
| 'count_too_low', |
| mock_buildbucket_config('abc.ci', builder_config('foo')), |
| including_bucket('abc.ci'), |
| build_status( |
| 'failure', |
| 'failure', |
| 'failure', |
| 'failure', |
| 'failure', |
| 'failure', |
| 'passed', |
| prefix='abc.ci.foo', |
| remote=[None] * 6 + [_PIGWEED], |
| ), |
| assert_count_too_low('abc.ci.foo'), |
| excluding_builder('abc.ci', 1), |
| assert_nothing_to_launch(), |
| drop_expectations_must_be_last(), |
| ) |
| |
| yield test( |
| 'no_recent_passes', |
| properties(), |
| mock_buildbucket_config( |
| 'abc.ci', |
| builder_config('foo'), |
| ), |
| including_bucket('abc.ci'), |
| including_builder('abc.ci', 1), |
| build_status( |
| 'failure', |
| 'failure', |
| 'failure', |
| prefix='abc.ci.foo', |
| spacing=5, |
| ), |
| assert_skip_no_recent_passes('abc.ci.foo'), |
| assert_nothing_to_launch(), |
| drop_expectations_must_be_last(), |
| ) |
| |
| yield test( |
| 'recent_passes_gaps', |
| mock_buildbucket_config( |
| 'abc.ci', |
| builder_config('foo'), |
| builder_config('bar'), |
| ), |
| including_bucket('abc.ci'), |
| including_builder('abc.ci', 2), |
| build_status( |
| 'failure', |
| 'failure', |
| 'passed', |
| prefix='abc.ci.foo', |
| spacing=5, |
| ), |
| build_status( |
| 'failure', |
| 'failure', |
| 'passed', |
| prefix='abc.ci.bar', |
| spacing=10, |
| ), |
| assert_picked('abc.ci.foo', 7), |
| assert_scheduling('abc.ci.foo'), |
| assert_picked('abc.ci.bar', 15), |
| assert_scheduling('abc.ci.bar'), |
| assert_launching(2), |
| assert_buildbucket_scheduled(), |
| drop_expectations_must_be_last(), |
| ) |
| |
| yield test( |
| 'recent_passes_already_scheduled', |
| mock_buildbucket_config( |
| 'abc.ci', |
| builder_config('foo'), |
| builder_config('bar'), |
| ), |
| including_bucket('abc.ci'), |
| including_builder('abc.ci', 2), |
| build_status( |
| 'scheduled-7', |
| 'failure', |
| 'passed', |
| prefix='abc.ci.foo', |
| spacing=5, |
| ), |
| build_status( |
| 'running-15', |
| 'failure', |
| 'passed', |
| prefix='abc.ci.bar', |
| spacing=10, |
| ), |
| assert_picked('abc.ci.foo', 7), |
| assert_already_scheduled('abc.ci.foo'), |
| assert_picked('abc.ci.bar', 15), |
| assert_already_scheduled('abc.ci.bar'), |
| assert_nothing_to_launch(), |
| drop_expectations_must_be_last(), |
| ) |
| |
| yield test( |
| 'recent_passes_nogaps', |
| mock_buildbucket_config( |
| 'abc.ci', |
| builder_config('foo'), |
| builder_config('bar'), |
| ), |
| including_bucket('abc.ci'), |
| including_builder('abc.ci', 2), |
| build_status( |
| 'failure', |
| 'failure', |
| 'passed', |
| prefix='abc.ci.foo', |
| spacing=1, |
| ), |
| build_status( |
| 'failure', |
| 'failure', |
| 'passed', |
| prefix='abc.ci.bar', |
| spacing=1, |
| ), |
| assert_already_attributed('abc.ci.foo'), |
| assert_already_attributed('abc.ci.bar'), |
| assert_nothing_to_launch(), |
| drop_expectations_must_be_last(), |
| ) |
| |
| yield test( |
| 'dry_run', |
| properties(dry_run=True), |
| mock_buildbucket_config( |
| 'abc.ci', |
| builder_config('foo'), |
| builder_config('bar'), |
| ), |
| including_bucket('abc.ci'), |
| including_builder('abc.ci', 2), |
| build_status( |
| 'failure', |
| 'failure', |
| 'passed', |
| prefix='abc.ci.foo', |
| spacing=40, |
| ), |
| build_status( |
| 'failure', |
| 'failure', |
| 'passed', |
| prefix='abc.ci.bar', |
| spacing=20, |
| ), |
| assert_picked('abc.ci.foo', 60), |
| assert_scheduling('abc.ci.foo'), |
| assert_picked('abc.ci.bar', 30), |
| assert_scheduling('abc.ci.bar'), |
| assert_launching(2, dry_run=True), |
| assert_dry_run(), |
| drop_expectations_must_be_last(), |
| ) |
| |
| yield test( |
| 'infra_failures', |
| mock_buildbucket_config( |
| 'abc.ci', |
| builder_config('foo'), |
| builder_config('bar'), |
| ), |
| including_bucket('abc.ci'), |
| including_builder('abc.ci', 2), |
| build_status( |
| 'infra_failure', |
| 'infra_failure', |
| 'infra_failure', |
| 'infra_failure', |
| 'infra_failure', |
| 'infra_failure', |
| 'passed', |
| prefix='abc.ci.foo', |
| spacing=5, |
| ), |
| build_status( |
| 'infra_failure', |
| 'infra_failure', |
| 'passed', |
| prefix='abc.ci.bar', |
| spacing=5, |
| ), |
| assert_too_many_infra_failures('abc.ci.foo'), |
| assert_no_need_to_bisect('abc.ci.bar'), |
| drop_expectations_must_be_last(), |
| ) |
| |
| def builder_state(running: int, completed: int): |
| running_ids = range(10000, 10000 + running) |
| completed_ids = range(20000, 20000 + completed) |
| |
| result = api.builder_state( |
| { |
| 'triggered_builds': list(running_ids) + list(completed_ids), |
| }, |
| ) |
| |
| builds: list[build_pb.Build] = [] |
| for i, bbid in enumerate(running_ids): |
| build = build_pb.Build(id=bbid) |
| if i % 2: |
| build.output.status = common_pb.SCHEDULED |
| else: |
| build.output.status = common_pb.STARTED |
| builds.append(build) |
| |
| for i, bbid in enumerate(completed_ids): |
| build = build_pb.Build(id=bbid) |
| if i % 4 == 0: |
| build.output.status = common_pb.SUCCESS |
| elif i % 4 == 1: |
| build.output.status = common_pb.FAILURE |
| elif i % 4 == 2: |
| build.output.status = common_pb.INFRA_FAILURE |
| else: |
| build.output.status = common_pb.CANCELED |
| builds.append(build) |
| |
| result += api.buildbucket.simulated_get_multi(builds) |
| return result |
| |
| def assert_max_triggered_builds(): |
| return api.post_process(post_process.MustRun, 'max triggered builds') |
| |
| def assert_truncated_requests(n): |
| return api.post_process(post_process.MustRun, f'truncated {n} requests') |
| |
| yield test( |
| 'too_many_running', |
| properties(max_triggered_builds=10), |
| builder_state(running=10, completed=10), |
| assert_max_triggered_builds(), |
| drop_expectations_must_be_last(), |
| ) |
| |
| yield test( |
| 'too_many_to_launch', |
| properties(max_triggered_builds=10), |
| builder_state(running=9, completed=1), |
| mock_buildbucket_config( |
| 'abc.ci', |
| builder_config('foo'), |
| builder_config('bar'), |
| ), |
| including_bucket('abc.ci'), |
| including_builder('abc.ci', 2), |
| build_status( |
| 'failure', |
| 'failure', |
| 'passed', |
| prefix='abc.ci.foo', |
| spacing=5, |
| ), |
| build_status( |
| 'failure', |
| 'failure', |
| 'passed', |
| prefix='abc.ci.bar', |
| spacing=10, |
| ), |
| assert_scheduling('abc.ci.foo'), |
| assert_scheduling('abc.ci.bar'), |
| assert_truncated_requests(1), |
| assert_launching(1), |
| assert_buildbucket_scheduled(), |
| drop_expectations_must_be_last(), |
| ) |
