| # Copyright 2024 The Pigweed Authors |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may not |
| # use this file except in compliance with the License. You may obtain a copy of |
| # the License at |
| # |
| # https://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations under |
| # the License. |
| """This recipe compares the status of dev and prod builders. |
| |
| This recipe iterates through all the builders in a project and checks whether |
| the status of each dev builder matches the status of its corresponding prod |
| builder. It reports any discrepancies, providing links to the builders with |
| disagreeing statuses. |
| |
| The recipe performs the following steps: |
| 1. Fetches all buckets and builders from the project's buildbucket config. |
| 2. For each bucket that has both a 'dev' and a 'prod' version, it retrieves |
| the status of each builder within those buckets. |
| 3. It compares the recent pass/fail status of each dev builder with its prod |
| counterpart. |
| 4. If the statuses disagree, it reports a failure and provides links to the |
| respective builders. |
| 5. If all statuses agree, it reports success. |
| |
| The behavior of this recipe can be configured through the `InputProperties` |
| proto, which allows setting the minimum number of builds to evaluate, the |
| number of builds to request from buildbucket, and the maximum age of builds to |
| consider. |
| """ |
| |
| from __future__ import annotations |
| |
| import datetime |
| from collections.abc import Iterator, Sequence |
| |
| from recipe_engine import post_process, recipe_api, recipe_test_api |
| |
| from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb |
| from PB.go.chromium.org.luci.buildbucket.proto import project_config as bb_pb |
| from PB.recipe_engine import result as result_pb |
| from PB.recipes.pigweed.dev_status import InputProperties |
| |
| DEPS = [ |
| 'fuchsia/builder_status', |
| 'fuchsia/recipe_testing', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/luci_config', |
| 'recipe_engine/properties', |
| 'recipe_engine/step', |
| 'recipe_engine/time', |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| DEFAULT_REQUESTED_BUILDS = 10 |
| DEFAULT_MIN_BUILDS = 4 |
| DEFAULT_MAX_AGE = datetime.timedelta(days=5) |
| |
| |
| def _make_dev(bucket_name): |
| """Add "dev" to bucket_name, if not present.""" |
| parts = bucket_name.split('.') |
| if 'dev' in parts: |
| return bucket_name |
| return '.'.join((*parts[0:-1], 'dev', parts[-1])) |
| |
| |
| def _make_prod(bucket_name): |
| """Remove "dev" from bucket_name, if present.""" |
| parts = bucket_name.split('.') |
| if 'dev' not in parts: |
| return bucket_name |
| return '.'.join(x for x in parts if x != 'dev') |
| |
| |
| def RunSteps(api: recipe_api.RecipeApi, props: InputProperties): |
| min_builds = props.min_builds or DEFAULT_MIN_BUILDS |
| requested_builds = props.requested_builds or DEFAULT_REQUESTED_BUILDS |
| max_age = props.max_age.ToTimedelta() or DEFAULT_MAX_AGE |
| |
| bb_cfg: bb_pb.BuildbucketCfg = api.luci_config.buildbucket() |
| |
| status: dict[str, dict[str, api.builder_status.BuilderStatus]] = {} |
| |
| bucket_names = set(x.name for x in bb_cfg.buckets if x.swarming.builders) |
| |
| with api.step.nest('status'): |
| for bucket in bb_cfg.buckets: |
| # If there's not a prod version and a dev version of this bucket |
| # then don't retrieve status for builders in the bucket. This will |
| # mostly exclude rolls. |
| if ( |
| _make_dev(bucket.name) not in bucket_names |
| or _make_prod(bucket.name) not in bucket_names |
| ): |
| step = api.step.empty(f'skipping {bucket.name}') |
| continue |
| |
| with api.step.nest(bucket.name) as pres: |
| for builder in bucket.swarming.builders: |
| with api.step.nest(builder.name): |
| if builder.name.startswith('dev-status'): |
| api.step.empty('ignoring') |
| continue |
| |
| # Don't DoS buildbucket. (And there's no need for this |
| # builder to run quickly.) |
| api.time.sleep(0.1) |
| |
| builder_status = api.builder_status.retrieve( |
| bucket=bucket.name, |
| builder=builder.name, |
| n=requested_builds, |
| max_age=max_age, |
| ) |
| |
| # If there's not much history then ignore this build. |
| if len(builder_status.builds) < min_builds: |
| no_builds = api.step.empty('not enough builds') |
| no_builds.presentation.step_summary_text = ( |
| f'{len(builder_status.builds)} builds, ' |
| f'needed {min_builds}' |
| ) |
| continue |
| |
| # If we're adding data to a dev or prod bucket it's |
| # easiest later if we just always add both buckets to |
| # the data structure. |
| status.setdefault(_make_dev(bucket.name), {}) |
| status.setdefault(_make_prod(bucket.name), {}) |
| status[bucket.name][builder.name] = builder_status |
| |
| failures: list[str] = [] |
| |
| for dev_bucket in sorted(status): |
| parts = dev_bucket.split('.') |
| if 'dev' not in parts: |
| continue |
| |
| prod_bucket = _make_prod(dev_bucket) |
| |
| with api.step.nest(dev_bucket) as pres: |
| for builder in sorted(status[dev_bucket]): |
| dev = status[dev_bucket][builder] |
| if builder not in status[prod_bucket]: |
| continue |
| prod = status[prod_bucket][builder] |
| |
| dev_status = api.builder_status.has_recently_passed(dev) |
| prod_status = api.builder_status.has_recently_passed(prod) |
| if dev_status != prod_status: |
| prod_url = api.buildbucket.builder_url( |
| bucket=prod_bucket, builder=builder |
| ) |
| dev_url = api.buildbucket.builder_url( |
| bucket=dev_bucket, builder=builder |
| ) |
| |
| def indicator(status: bool) -> str: |
| if status: |
| return '\u2705' # Green check mark emoji. |
| return '\u274c' # Cross mark emoji. |
| |
| dev_indicator = indicator(dev_status) |
| prod_indicator = indicator(prod_status) |
| |
| failures.append( |
| f'{builder}: ' |
| f'([{prod_bucket} {prod_indicator}]({prod_url}), ' |
| f'[{dev_bucket} {dev_indicator}]({dev_url}))' |
| ) |
| |
| step = api.step.empty( |
| builder, |
| status='FAILURE', |
| raise_on_failure=False, |
| ) |
| step.presentation.links['dev'] = dev.link |
| step.presentation.links['prod'] = prod.link |
| pres.status = 'FAILURE' |
| |
| # In recipe testing only fail if there was a crash. This recipe regularly |
| # fails for various reasons and that shouldn't block recipe changes. |
| failure_mode = common_pb.FAILURE |
| if api.recipe_testing.enabled: |
| failure_mode = common_pb.SUCCESS # pragma: no cover |
| |
| if failures: |
| lines = ['Disagreeing dev builds:', ''] |
| for failure in failures: |
| lines.append(f'* {failure}') |
| |
| return result_pb.RawResult( |
| summary_markdown='\n'.join(lines), |
| status=failure_mode, |
| ) |
| |
| return result_pb.RawResult( |
| summary_markdown='All dev builds agree with prod builds', |
| status=common_pb.SUCCESS, |
| ) |
| |
| |
| def GenTests( |
| api: recipe_test_api.RecipeTestApi, |
| ) -> Iterator[recipe_test_api.TestData]: |
| def test(name, *args, **kwargs): |
| return api.test( |
| name, |
| api.buildbucket.ci_build(project='pigweed'), |
| *args, |
| **kwargs, |
| ) |
| |
| def props(min_builds=0, max_age=datetime.timedelta(days=7)): |
| input_props = InputProperties(min_builds=min_builds) |
| input_props.max_age.FromTimedelta(max_age) |
| return api.properties(input_props) |
| |
| def buildbucket_config(buckets: Sequence[bb_pb.Bucket]): |
| cfg = bb_pb.BuildbucketCfg() |
| cfg.buckets.extend(buckets) |
| return cfg |
| |
| def bucket_config( |
| name: str, |
| builders: Sequence[bb_pb.BuilderConfig], |
| ): |
| cfg = bb_pb.Bucket(name=name) |
| cfg.swarming.builders.extend(builders) |
| return cfg |
| |
| def builder_config(name: str): |
| return bb_pb.BuilderConfig(name=name) |
| |
| def mock_buildbucket_config( |
| *buckets_builders: Sequence[tuple[str, Sequence[str]]], |
| ): |
| buckets: list[bb_pb.Bucket] = [] |
| for bucket_name, builder_names in buckets_builders: |
| builders: list[bb_pb.BuilderConfig] = [] |
| for builder in builder_names: |
| builders.append(builder_config(builder)) |
| buckets.append(bucket_config(bucket_name, builders)) |
| return api.luci_config.mock_config( |
| project='pigweed', |
| config_name='cr-buildbucket.cfg', |
| data=buildbucket_config(buckets), |
| ) |
| |
| def build_status(name: str, *statuses: str): |
| return api.buildbucket.simulated_search_results( |
| [getattr(api.builder_status, x)() for x in statuses], |
| step_name=f'status.{name}.buildbucket.search', |
| ) |
| |
| def skipped(bucket): |
| return api.post_check(post_process.MustRun, f'status.skipping {bucket}') |
| |
| def not_enough_builds(bucket, builder): |
| return api.post_check( |
| post_process.MustRun, |
| f'status.{bucket}.{builder}.not enough builds', |
| ) |
| |
| def enough_builds(bucket, builder): |
| return api.post_check( |
| post_process.DoesNotRun, |
| f'status.{bucket}.{builder}.not enough builds', |
| ) |
| |
| def passed(bucket): |
| return api.post_check(post_process.StepSuccess, bucket) |
| |
| def failed(bucket): |
| return api.post_check(post_process.StepFailure, bucket) |
| |
| def not_run(bucket): |
| return api.post_check(post_process.DoesNotRun, bucket) |
| |
| def drop_expectations_must_be_last(): |
| # No need for expectation files, everything of note here is tested by |
| # assertions. This must be the last thing added to the test. |
| return api.post_process(post_process.DropExpectation) |
| |
| yield test( |
| 'no-runs', |
| props(min_builds=1), |
| mock_buildbucket_config( |
| ('try', ('foo', 'bar', 'baz')), |
| ('dev.try', ('foo', 'bar', 'baz')), |
| ('roll', ('foo-roller', 'bar-roller')), |
| ), |
| not_enough_builds('try', 'foo'), |
| not_enough_builds('try', 'bar'), |
| not_enough_builds('try', 'baz'), |
| not_enough_builds('dev.try', 'foo'), |
| not_enough_builds('dev.try', 'bar'), |
| not_enough_builds('dev.try', 'baz'), |
| skipped('roll'), |
| not_run('dev.try'), |
| drop_expectations_must_be_last(), |
| ) |
| |
| yield test( |
| 'mixed-run-numbers', |
| mock_buildbucket_config( |
| ('ci', ('foo', 'bar', 'dev-status')), |
| ('dev.ci', ('foo', 'bar', 'dev-status')), |
| ), |
| build_status('ci.foo', 'passed', 'passed'), |
| not_enough_builds('ci', 'foo'), |
| build_status('ci.bar', *['passed'] * (DEFAULT_MIN_BUILDS + 1)), |
| enough_builds('ci', 'bar'), |
| build_status('dev.ci.foo', 'failure', 'failure'), |
| not_enough_builds('dev.ci', 'foo'), |
| build_status('dev.ci.bar', *['failure'] * (DEFAULT_MIN_BUILDS + 1)), |
| enough_builds('dev.ci', 'bar'), |
| failed('dev.ci'), |
| not_run('dev.ci.foo'), |
| failed('dev.ci.bar'), |
| drop_expectations_must_be_last(), |
| status='FAILURE', |
| ) |
| |
| yield test( |
| 'all-passing', |
| props(min_builds=1), |
| mock_buildbucket_config( |
| ('ci', ('foo', 'bar', 'baz')), |
| ('dev.ci', ('foo', 'bar', 'baz')), |
| ), |
| build_status('ci.foo', 'passed', 'passed'), |
| build_status('ci.bar', 'passed', 'passed'), |
| build_status('ci.baz', 'passed', 'passed'), |
| build_status('dev.ci.foo', 'passed', 'passed'), |
| build_status('dev.ci.bar', 'passed', 'passed'), |
| build_status('dev.ci.baz', 'passed', 'passed'), |
| passed('dev.ci'), |
| drop_expectations_must_be_last(), |
| ) |
| |
| yield test( |
| 'all-failing', |
| props(min_builds=1), |
| mock_buildbucket_config( |
| ('ci', ('foo', 'bar', 'baz')), |
| ('dev.ci', ('foo', 'bar', 'baz')), |
| ), |
| build_status('ci.foo', 'failure', 'failure'), |
| build_status('ci.bar', 'failure', 'failure'), |
| build_status('ci.baz', 'failure', 'failure'), |
| build_status('dev.ci.foo', 'failure', 'failure'), |
| build_status('dev.ci.bar', 'failure', 'failure'), |
| build_status('dev.ci.baz', 'failure', 'failure'), |
| passed('dev.ci'), |
| drop_expectations_must_be_last(), |
| ) |
| |
| yield test( |
| 'mismatch-cifail-devpass', |
| props(min_builds=1), |
| mock_buildbucket_config( |
| ('ci', ('foo',)), |
| ('dev.ci', ('foo',)), |
| ), |
| build_status('ci.foo', 'failure'), |
| build_status('dev.ci.foo', 'passed'), |
| failed('dev.ci'), |
| failed('dev.ci.foo'), |
| drop_expectations_must_be_last(), |
| status='FAILURE', |
| ) |
| |
| yield test( |
| 'mismatch-cipass-devfail', |
| props(min_builds=1), |
| mock_buildbucket_config( |
| ('ci', ('foo',)), |
| ('dev.ci', ('foo',)), |
| ), |
| build_status('ci.foo', 'passed'), |
| build_status('dev.ci.foo', 'failure'), |
| failed('dev.ci'), |
| failed('dev.ci.foo'), |
| drop_expectations_must_be_last(), |
| status='FAILURE', |
| ) |
| |
| yield test( |
| 'prod-dev-only', |
| props(min_builds=1), |
| mock_buildbucket_config( |
| ('try', ('foo',)), |
| ('dev.try', ('bar',)), |
| ), |
| build_status('dev.try.bar', 'failure'), |
| not_run('dev.try.bar'), |
| drop_expectations_must_be_last(), |
| ) |
