recipes/dev_status.py - infra/recipes - Git at Google

 # Copyright 2024 The Pigweed Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not
 # use this file except in compliance with the License. You may obtain a copy of
 # the License at
 #
 #     https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations under
 # the License.
 """Compare status of dev and prod builders.

 Look at all the builders in a project and check whether dev builder status
 matches prod builder status, and provide links when the status disagrees.
 """

 from __future__ import annotations

 import collections
 import datetime
 from typing import TYPE_CHECKING

 from PB.recipe_engine import result as result_pb2
 from PB.go.chromium.org.luci.buildbucket.proto import (
     common as common_pb2,
     project_config as bb_pb2,
 )
 from PB.recipes.pigweed.dev_status import InputProperties
 from PB.recipe_engine import result
 from recipe_engine import post_process

 if TYPE_CHECKING:  # pragma: no cover
     from typing import Generator, Sequence, Tuple
     from recipe_engine import recipe_api, recipe_test_api

 DEPS = [
     'fuchsia/builder_status',
     'fuchsia/luci_config',
     'recipe_engine/buildbucket',
     'recipe_engine/properties',
     'recipe_engine/step',
     'recipe_engine/time',
 ]

 PROPERTIES = InputProperties

 DEFAULT_REQUESTED_BUILDS = 10
 DEFAULT_MIN_BUILDS = 4
 DEFAULT_MAX_AGE = datetime.timedelta(days=5)


 def _make_dev(bucket_name):
     """Add "dev" to bucket_name, if not present."""
     parts = bucket_name.split('.')
     if 'dev' in parts:
         return bucket_name
     return '.'.join((*parts[0:-1], 'dev', parts[-1]))


 def _make_prod(bucket_name):
     """Remove "dev" from bucket_name, if present."""
     parts = bucket_name.split('.')
     if 'dev' not in parts:
         return bucket_name
     return '.'.join(x for x in parts if x != 'dev')


 def RunSteps(api: recipe_api.RecipeApi, props: InputProperties):
     min_builds = props.min_builds or DEFAULT_MIN_BUILDS
     requested_builds = props.requested_builds or DEFAULT_REQUESTED_BUILDS
     max_age = props.max_age.ToTimedelta() or DEFAULT_MAX_AGE

     bb_cfg: bb_pb2.BuildbucketCfg = api.luci_config.buildbucket()

     status: dict[str, dict[str, api.builder_status.BuilderStatus]] = {}

     bucket_names = set(x.name for x in bb_cfg.buckets if x.swarming.builders)

     with api.step.nest('status'):
         for bucket in bb_cfg.buckets:
             # If there's not a prod version and a dev version of this bucket
             # then don't retrieve status for builders in the bucket. This will
             # mostly exclude rolls.
             if (
                 _make_dev(bucket.name) not in bucket_names
                 or _make_prod(bucket.name) not in bucket_names
             ):
                 step = api.step.empty(f'skipping {bucket.name}')
                 continue

             with api.step.nest(bucket.name) as pres:
                 for builder in bucket.swarming.builders:
                     with api.step.nest(builder.name):
                         # Don't DoS buildbucket. (And there's no need for this
                         # builder to run quickly.)
                         api.time.sleep(0.1)

                         builder_status = api.builder_status.retrieve(
                             bucket=bucket.name,
                             builder=builder.name,
                             n=requested_builds,
                             max_age=max_age,
                         )

                         # If there's not much history then ignore this build.
                         if len(builder_status.builds) < min_builds:
                             no_builds = api.step.empty('not enough builds')
                             no_builds.presentation.step_summary_text = (
                                 f'{len(builder_status.builds)} builds, '
                                 f'needed {min_builds}'
                             )
                             continue

                         # If we're adding data to a dev or prod bucket it's
                         # easiest later if we just always add both buckets to
                         # the data structure.
                         status.setdefault(_make_dev(bucket.name), {})
                         status.setdefault(_make_prod(bucket.name), {})
                         status[bucket.name][builder.name] = builder_status

     failures: list[str] = []

     for dev_bucket in sorted(status):
         parts = dev_bucket.split('.')
         if 'dev' not in parts:
             continue

         prod_bucket = _make_prod(dev_bucket)

         with api.step.nest(dev_bucket) as pres:
             for builder in sorted(status[dev_bucket]):
                 dev = status[dev_bucket][builder]
                 if builder not in status[prod_bucket]:
                     continue
                 prod = status[prod_bucket][builder]

                 has_recently_passed = api.builder_status.has_recently_passed
                 if has_recently_passed(dev) != has_recently_passed(prod):
                     prod_url = api.buildbucket.builder_url(
                         bucket=prod_bucket, builder=builder
                     )
                     dev_url = api.buildbucket.builder_url(
                         bucket=dev_bucket, builder=builder
                     )

                     failures.append(
                         f'{builder}: '
                         f'([{prod_bucket}]({prod_url}), '
                         f'[{dev_bucket}]({dev_url}))'
                     )

                     step = api.step.empty(
                         builder,
                         status='FAILURE',
                         raise_on_failure=False,
                     )
                     step.presentation.links['dev'] = dev.link
                     step.presentation.links['prod'] = prod.link
                     pres.status = 'FAILURE'

     if failures:
         lines = ['Disagreeing dev builds:', '']
         for failure in failures:
             lines.append(f'* {failure}')

         return result_pb2.RawResult(
             summary_markdown='\n'.join(lines),
             status=common_pb2.FAILURE,
         )

     return result_pb2.RawResult(
         summary_markdown='All dev builds agree with prod builds',
         status=common_pb2.SUCCESS,
     )


 def GenTests(api) -> Generator[recipe_test_api.TestData, None, None]:

     def test(name, *args, **kwargs):
         return api.test(
             name,
             api.buildbucket.ci_build(project='pigweed'),
             *args,
             **kwargs,
         )

     def props(min_builds=0, max_age=datetime.timedelta(days=7)):
         input_props = InputProperties(min_builds=min_builds)
         input_props.max_age.FromTimedelta(max_age)
         return api.properties(input_props)

     def buildbucket_config(buckets: bb_pb2.Bucket):
         cfg = bb_pb2.BuildbucketCfg()
         cfg.buckets.extend(buckets)
         return cfg

     def bucket_config(
         name: str,
         builders: Sequence[bb_pb2.BuilderConfig],
     ):
         cfg = bb_pb2.Bucket(name=name)
         cfg.swarming.builders.extend(builders)
         return cfg

     def builder_config(name: str):
         return bb_pb2.BuilderConfig(name=name)

     def mock_buildbucket_config(
         *buckets_builders: Sequence[Tuple[str, Sequence[str]]],
     ):
         buckets: List[bb_pb2.Bucket] = []
         for bucket_name, builder_names in buckets_builders:
             builders: List[bb_pb2.BuilderConfig] = []
             for builder in builder_names:
                 builders.append(builder_config(builder))
             buckets.append(bucket_config(bucket_name, builders))
         return api.luci_config.mock_config(
             project='pigweed',
             config_name='cr-buildbucket.cfg',
             data=buildbucket_config(buckets),
         )

     def build_status(name: str, *statuses: str):
         return api.buildbucket.simulated_search_results(
             [getattr(api.builder_status, x)() for x in statuses],
             step_name=f'status.{name}.buildbucket.search',
         )

     def skipped(bucket):
         return api.post_process(
             post_process.MustRun, f'status.skipping {bucket}'
         )

     def not_enough_builds(bucket, builder):
         return api.post_process(
             post_process.MustRun,
             f'status.{bucket}.{builder}.not enough builds',
         )

     def enough_builds(bucket, builder):
         return api.post_process(
             post_process.DoesNotRun,
             f'status.{bucket}.{builder}.not enough builds',
         )

     def passed(bucket):
         return api.post_process(post_process.StepSuccess, bucket)

     def failed(bucket):
         return api.post_process(post_process.StepFailure, bucket)

     def not_run(bucket):
         return api.post_process(post_process.DoesNotRun, bucket)

     def drop_expectations_must_be_last():
         # No need for expectation files, everything of note here is tested by
         # assertions. This must be the last thing added to the test.
         return api.post_process(post_process.DropExpectation)

     yield test(
         'no-runs',
         props(min_builds=1),
         mock_buildbucket_config(
             ('try', ('foo', 'bar', 'baz')),
             ('dev.try', ('foo', 'bar', 'baz')),
             ('roll', ('foo-roller', 'bar-roller')),
         ),
         not_enough_builds('try', 'foo'),
         not_enough_builds('try', 'bar'),
         not_enough_builds('try', 'baz'),
         not_enough_builds('dev.try', 'foo'),
         not_enough_builds('dev.try', 'bar'),
         not_enough_builds('dev.try', 'baz'),
         skipped('roll'),
         not_run('dev.try'),
         drop_expectations_must_be_last(),
     )

     yield test(
         'mixed-run-numbers',
         mock_buildbucket_config(
             ('ci', ('foo', 'bar')),
             ('dev.ci', ('foo', 'bar')),
         ),
         build_status('ci.foo', 'passed', 'passed'),
         not_enough_builds('ci', 'foo'),
         build_status('ci.bar', *['passed'] * (DEFAULT_MIN_BUILDS + 1)),
         enough_builds('ci', 'bar'),
         build_status('dev.ci.foo', 'failure', 'failure'),
         not_enough_builds('dev.ci', 'foo'),
         build_status('dev.ci.bar', *['failure'] * (DEFAULT_MIN_BUILDS + 1)),
         enough_builds('dev.ci', 'bar'),
         failed('dev.ci'),
         not_run('dev.ci.foo'),
         failed('dev.ci.bar'),
         drop_expectations_must_be_last(),
         status='FAILURE',
     )

     yield test(
         'all-passing',
         props(min_builds=1),
         mock_buildbucket_config(
             ('ci', ('foo', 'bar', 'baz')),
             ('dev.ci', ('foo', 'bar', 'baz')),
         ),
         build_status('ci.foo', 'passed', 'passed'),
         build_status('ci.bar', 'passed', 'passed'),
         build_status('ci.baz', 'passed', 'passed'),
         build_status('dev.ci.foo', 'passed', 'passed'),
         build_status('dev.ci.bar', 'passed', 'passed'),
         build_status('dev.ci.baz', 'passed', 'passed'),
         passed('dev.ci'),
         drop_expectations_must_be_last(),
     )

     yield test(
         'all-failing',
         props(min_builds=1),
         mock_buildbucket_config(
             ('ci', ('foo', 'bar', 'baz')),
             ('dev.ci', ('foo', 'bar', 'baz')),
         ),
         build_status('ci.foo', 'failure', 'failure'),
         build_status('ci.bar', 'failure', 'failure'),
         build_status('ci.baz', 'failure', 'failure'),
         build_status('dev.ci.foo', 'failure', 'failure'),
         build_status('dev.ci.bar', 'failure', 'failure'),
         build_status('dev.ci.baz', 'failure', 'failure'),
         passed('dev.ci'),
         drop_expectations_must_be_last(),
     )

     yield test(
         'mismatch-cifail-devpass',
         props(min_builds=1),
         mock_buildbucket_config(
             ('ci', ('foo',)),
             ('dev.ci', ('foo',)),
         ),
         build_status('ci.foo', 'failure'),
         build_status('dev.ci.foo', 'passed'),
         failed('dev.ci'),
         failed('dev.ci.foo'),
         drop_expectations_must_be_last(),
         status='FAILURE',
     )

     yield test(
         'mismatch-cipass-devfail',
         props(min_builds=1),
         mock_buildbucket_config(
             ('ci', ('foo',)),
             ('dev.ci', ('foo',)),
         ),
         build_status('ci.foo', 'passed'),
         build_status('dev.ci.foo', 'failure'),
         failed('dev.ci'),
         failed('dev.ci.foo'),
         drop_expectations_must_be_last(),
         status='FAILURE',
     )

     yield test(
         'prod-dev-only',
         props(min_builds=1),
         mock_buildbucket_config(
             ('try', ('foo',)),
             ('dev.try', ('bar',)),
         ),
         build_status('dev.try.bar', 'failure'),
         not_run('dev.try.bar'),
         drop_expectations_must_be_last(),
     )
	# Copyright 2024 The Pigweed Authors
	#
	# Licensed under the Apache License, Version 2.0 (the "License"); you may not
	# use this file except in compliance with the License. You may obtain a copy of
	# the License at
	#
	# https://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
	# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
	# License for the specific language governing permissions and limitations under
	# the License.
	"""Compare status of dev and prod builders.

	Look at all the builders in a project and check whether dev builder status
	matches prod builder status, and provide links when the status disagrees.
	"""

	from __future__ import annotations

	import collections
	import datetime
	from typing import TYPE_CHECKING

	from PB.recipe_engine import result as result_pb2
	from PB.go.chromium.org.luci.buildbucket.proto import (
	common as common_pb2,
	project_config as bb_pb2,
	)
	from PB.recipes.pigweed.dev_status import InputProperties
	from PB.recipe_engine import result
	from recipe_engine import post_process

	if TYPE_CHECKING: # pragma: no cover
	from typing import Generator, Sequence, Tuple
	from recipe_engine import recipe_api, recipe_test_api

	DEPS = [
	'fuchsia/builder_status',
	'fuchsia/luci_config',
	'recipe_engine/buildbucket',
	'recipe_engine/properties',
	'recipe_engine/step',
	'recipe_engine/time',
	]

	PROPERTIES = InputProperties

	DEFAULT_REQUESTED_BUILDS = 10
	DEFAULT_MIN_BUILDS = 4
	DEFAULT_MAX_AGE = datetime.timedelta(days=5)


	def _make_dev(bucket_name):
	"""Add "dev" to bucket_name, if not present."""
	parts = bucket_name.split('.')
	if 'dev' in parts:
	return bucket_name
	return '.'.join((*parts[0:-1], 'dev', parts[-1]))


	def _make_prod(bucket_name):
	"""Remove "dev" from bucket_name, if present."""
	parts = bucket_name.split('.')
	if 'dev' not in parts:
	return bucket_name
	return '.'.join(x for x in parts if x != 'dev')


	def RunSteps(api: recipe_api.RecipeApi, props: InputProperties):
	min_builds = props.min_builds or DEFAULT_MIN_BUILDS
	requested_builds = props.requested_builds or DEFAULT_REQUESTED_BUILDS
	max_age = props.max_age.ToTimedelta() or DEFAULT_MAX_AGE

	bb_cfg: bb_pb2.BuildbucketCfg = api.luci_config.buildbucket()

	status: dict[str, dict[str, api.builder_status.BuilderStatus]] = {}

	bucket_names = set(x.name for x in bb_cfg.buckets if x.swarming.builders)

	with api.step.nest('status'):
	for bucket in bb_cfg.buckets:
	# If there's not a prod version and a dev version of this bucket
	# then don't retrieve status for builders in the bucket. This will
	# mostly exclude rolls.
	if (
	_make_dev(bucket.name) not in bucket_names
	or _make_prod(bucket.name) not in bucket_names
	):
	step = api.step.empty(f'skipping {bucket.name}')
	continue

	with api.step.nest(bucket.name) as pres:
	for builder in bucket.swarming.builders:
	with api.step.nest(builder.name):
	# Don't DoS buildbucket. (And there's no need for this
	# builder to run quickly.)
	api.time.sleep(0.1)

	builder_status = api.builder_status.retrieve(
	bucket=bucket.name,
	builder=builder.name,
	n=requested_builds,
	max_age=max_age,
	)

	# If there's not much history then ignore this build.
	if len(builder_status.builds) < min_builds:
	no_builds = api.step.empty('not enough builds')
	no_builds.presentation.step_summary_text = (
	f'{len(builder_status.builds)} builds, '
	f'needed {min_builds}'
	)
	continue

	# If we're adding data to a dev or prod bucket it's
	# easiest later if we just always add both buckets to
	# the data structure.
	status.setdefault(_make_dev(bucket.name), {})
	status.setdefault(_make_prod(bucket.name), {})
	status[bucket.name][builder.name] = builder_status

	failures: list[str] = []

	for dev_bucket in sorted(status):
	parts = dev_bucket.split('.')
	if 'dev' not in parts:
	continue

	prod_bucket = _make_prod(dev_bucket)

	with api.step.nest(dev_bucket) as pres:
	for builder in sorted(status[dev_bucket]):
	dev = status[dev_bucket][builder]
	if builder not in status[prod_bucket]:
	continue
	prod = status[prod_bucket][builder]

	has_recently_passed = api.builder_status.has_recently_passed
	if has_recently_passed(dev) != has_recently_passed(prod):
	prod_url = api.buildbucket.builder_url(
	bucket=prod_bucket, builder=builder
	)
	dev_url = api.buildbucket.builder_url(
	bucket=dev_bucket, builder=builder
	)

	failures.append(
	f'{builder}: '
	f'([{prod_bucket}]({prod_url}), '
	f'[{dev_bucket}]({dev_url}))'
	)

	step = api.step.empty(
	builder,
	status='FAILURE',
	raise_on_failure=False,
	)
	step.presentation.links['dev'] = dev.link
	step.presentation.links['prod'] = prod.link
	pres.status = 'FAILURE'

	if failures:
	lines = ['Disagreeing dev builds:', '']
	for failure in failures:
	lines.append(f'* {failure}')

	return result_pb2.RawResult(
	summary_markdown='\n'.join(lines),
	status=common_pb2.FAILURE,
	)

	return result_pb2.RawResult(
	summary_markdown='All dev builds agree with prod builds',
	status=common_pb2.SUCCESS,
	)


	def GenTests(api) -> Generator[recipe_test_api.TestData, None, None]:

	def test(name, args, *kwargs):
	return api.test(
	name,
	api.buildbucket.ci_build(project='pigweed'),
	*args,
	**kwargs,
	)

	def props(min_builds=0, max_age=datetime.timedelta(days=7)):
	input_props = InputProperties(min_builds=min_builds)
	input_props.max_age.FromTimedelta(max_age)
	return api.properties(input_props)

	def buildbucket_config(buckets: bb_pb2.Bucket):
	cfg = bb_pb2.BuildbucketCfg()
	cfg.buckets.extend(buckets)
	return cfg

	def bucket_config(
	name: str,
	builders: Sequence[bb_pb2.BuilderConfig],
	):
	cfg = bb_pb2.Bucket(name=name)
	cfg.swarming.builders.extend(builders)
	return cfg

	def builder_config(name: str):
	return bb_pb2.BuilderConfig(name=name)

	def mock_buildbucket_config(
	*buckets_builders: Sequence[Tuple[str, Sequence[str]]],
	):
	buckets: List[bb_pb2.Bucket] = []
	for bucket_name, builder_names in buckets_builders:
	builders: List[bb_pb2.BuilderConfig] = []
	for builder in builder_names:
	builders.append(builder_config(builder))
	buckets.append(bucket_config(bucket_name, builders))
	return api.luci_config.mock_config(
	project='pigweed',
	config_name='cr-buildbucket.cfg',
	data=buildbucket_config(buckets),
	)

	def build_status(name: str, *statuses: str):
	return api.buildbucket.simulated_search_results(
	[getattr(api.builder_status, x)() for x in statuses],
	step_name=f'status.{name}.buildbucket.search',
	)

	def skipped(bucket):
	return api.post_process(
	post_process.MustRun, f'status.skipping {bucket}'
	)

	def not_enough_builds(bucket, builder):
	return api.post_process(
	post_process.MustRun,
	f'status.{bucket}.{builder}.not enough builds',
	)

	def enough_builds(bucket, builder):
	return api.post_process(
	post_process.DoesNotRun,
	f'status.{bucket}.{builder}.not enough builds',
	)

	def passed(bucket):
	return api.post_process(post_process.StepSuccess, bucket)

	def failed(bucket):
	return api.post_process(post_process.StepFailure, bucket)

	def not_run(bucket):
	return api.post_process(post_process.DoesNotRun, bucket)

	def drop_expectations_must_be_last():
	# No need for expectation files, everything of note here is tested by
	# assertions. This must be the last thing added to the test.
	return api.post_process(post_process.DropExpectation)

	yield test(
	'no-runs',
	props(min_builds=1),
	mock_buildbucket_config(
	('try', ('foo', 'bar', 'baz')),
	('dev.try', ('foo', 'bar', 'baz')),
	('roll', ('foo-roller', 'bar-roller')),
	),
	not_enough_builds('try', 'foo'),
	not_enough_builds('try', 'bar'),
	not_enough_builds('try', 'baz'),
	not_enough_builds('dev.try', 'foo'),
	not_enough_builds('dev.try', 'bar'),
	not_enough_builds('dev.try', 'baz'),
	skipped('roll'),
	not_run('dev.try'),
	drop_expectations_must_be_last(),
	)

	yield test(
	'mixed-run-numbers',
	mock_buildbucket_config(
	('ci', ('foo', 'bar')),
	('dev.ci', ('foo', 'bar')),
	),
	build_status('ci.foo', 'passed', 'passed'),
	not_enough_builds('ci', 'foo'),
	build_status('ci.bar', ['passed'] (DEFAULT_MIN_BUILDS + 1)),
	enough_builds('ci', 'bar'),
	build_status('dev.ci.foo', 'failure', 'failure'),
	not_enough_builds('dev.ci', 'foo'),
	build_status('dev.ci.bar', ['failure'] (DEFAULT_MIN_BUILDS + 1)),
	enough_builds('dev.ci', 'bar'),
	failed('dev.ci'),
	not_run('dev.ci.foo'),
	failed('dev.ci.bar'),
	drop_expectations_must_be_last(),
	status='FAILURE',
	)

	yield test(
	'all-passing',
	props(min_builds=1),
	mock_buildbucket_config(
	('ci', ('foo', 'bar', 'baz')),
	('dev.ci', ('foo', 'bar', 'baz')),
	),
	build_status('ci.foo', 'passed', 'passed'),
	build_status('ci.bar', 'passed', 'passed'),
	build_status('ci.baz', 'passed', 'passed'),
	build_status('dev.ci.foo', 'passed', 'passed'),
	build_status('dev.ci.bar', 'passed', 'passed'),
	build_status('dev.ci.baz', 'passed', 'passed'),
	passed('dev.ci'),
	drop_expectations_must_be_last(),
	)

	yield test(
	'all-failing',
	props(min_builds=1),
	mock_buildbucket_config(
	('ci', ('foo', 'bar', 'baz')),
	('dev.ci', ('foo', 'bar', 'baz')),
	),
	build_status('ci.foo', 'failure', 'failure'),
	build_status('ci.bar', 'failure', 'failure'),
	build_status('ci.baz', 'failure', 'failure'),
	build_status('dev.ci.foo', 'failure', 'failure'),
	build_status('dev.ci.bar', 'failure', 'failure'),
	build_status('dev.ci.baz', 'failure', 'failure'),
	passed('dev.ci'),
	drop_expectations_must_be_last(),
	)

	yield test(
	'mismatch-cifail-devpass',
	props(min_builds=1),
	mock_buildbucket_config(
	('ci', ('foo',)),
	('dev.ci', ('foo',)),
	),
	build_status('ci.foo', 'failure'),
	build_status('dev.ci.foo', 'passed'),
	failed('dev.ci'),
	failed('dev.ci.foo'),
	drop_expectations_must_be_last(),
	status='FAILURE',
	)

	yield test(
	'mismatch-cipass-devfail',
	props(min_builds=1),
	mock_buildbucket_config(
	('ci', ('foo',)),
	('dev.ci', ('foo',)),
	),
	build_status('ci.foo', 'passed'),
	build_status('dev.ci.foo', 'failure'),
	failed('dev.ci'),
	failed('dev.ci.foo'),
	drop_expectations_must_be_last(),
	status='FAILURE',
	)

	yield test(
	'prod-dev-only',
	props(min_builds=1),
	mock_buildbucket_config(
	('try', ('foo',)),
	('dev.try', ('bar',)),
	),
	build_status('dev.try.bar', 'failure'),
	not_run('dev.try.bar'),
	drop_expectations_must_be_last(),
	)