recipes/post_attribution_roller_retrigger.py - infra/recipes - Git at Google

 # Copyright 2025 The Pigweed Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not
 # use this file except in compliance with the License. You may obtain a copy of
 # the License at
 #
 #     https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations under
 # the License.
 """Rerun rollers after the bisector determines responsible commit.

 Rollers run in dry-run mode during bisection. Once a new failure has been
 attributed to a specific commit, rerun the roller on the previous commit, but
 not in dry-run mode.

 This recipe is typically triggered after a bisector run completes. It inspects
 the output properties of the bisector build to identify any "roller" builders
 where a failure was successfully attributed to a specific commit.

 For each such attributed failure in a roller:
 1. It identifies the "newest_passing" commit reported by the bisector.
 2. It checks an internal state to see if a build for this specific roller and
    commit has already been re-triggered recently (within the last 14 days).
 3. If not already re-triggered, it schedules a new build for that roller using
    the "newest_passing" commit. This new build is *not* run in dry-run mode,
    allowing the roller to attempt the actual roll.

 The recipe uses the `InputProperties` message, which defines:
 - `bucket`: The bucket of the bisector builder to query. Defaults to the
             bucket of the current build if not specified.
 - `builder`: The name of the bisector builder to query. Defaults to "bisector"
              if not specified.
 - `dry_run`: A boolean flag. If true, the recipe will identify and report the
              roller builds it would re-trigger but will not actually schedule
              them. This defaults to true for development or try builds.

 The recipe outputs a summary of the builds it launched (or would have launched
 in dry-run mode).

 If all bisector-launched rolls fail, this recipe will likely rerun a roll that
 already ran in a non-dry-run mode. Since this roll will be a no-op it will
 complete quickly.
 """

 from __future__ import annotations

 import datetime
 from collections.abc import Iterator

 from google.protobuf import json_format

 from recipe_engine import post_process, recipe_api, recipe_test_api

 from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb
 from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb
 from PB.recipe_engine import result as result_pb
 from PB.recipes.pigweed.post_attribution_roller_retrigger import InputProperties

 from RECIPE_MODULES.pigweed.raw_result import api as raw_result_api

 DEPS = [
     'fuchsia/buildbucket_util',
     'fuchsia/builder_state',
     'fuchsia/gerrit',
     'pigweed/raw_result',
     'recipe_engine/buildbucket',
     'recipe_engine/properties',
     'recipe_engine/step',
     'recipe_engine/time',
 ]

 PROPERTIES = InputProperties


 @raw_result_api.wrap_run_steps
 def RunSteps(api: recipe_api.RecipeApi, props: InputProperties):
     props.dry_run = props.dry_run or api.buildbucket_util.is_dev_or_try

     now = api.time.utcnow().timestamp()
     bucket = props.bucket or api.buildbucket.build.builder.bucket
     builder = props.builder or 'bisector'

     with api.step.nest('state'):
         api.step.empty('now').presentation.step_summary_text = str(now)
         old_state = api.builder_state.fetch_previous_state()
         state = {}
         for bucket_builder_commit, timestamp in old_state.items():
             if now - timestamp < 14 * 24 * 60 * 60:
                 state[bucket_builder_commit] = timestamp
         api.builder_state.save(state)

     max_age = datetime.timedelta(hours=2)
     if api.buildbucket_util.is_dev_or_try:
         max_age = datetime.timedelta(days=7)

     with api.step.nest('bisector'):
         build = api.buildbucket_util.last_build(
             bucket=bucket,
             builder=builder,
             status=common_pb.ENDED_MASK,
             max_age=max_age,
         )

         if not build:
             api.step.empty(  # pragma: no cover
                 'no bisector build found',
                 status='FAILURE',
             )

     bisector_props = json_format.MessageToDict(build.output.properties)
     bisector_status = bisector_props.get('builders', {})
     requests = []
     for bucket, builders in bisector_status.items():
         with api.step.nest(bucket):
             if not bucket.split('.')[-1] == 'roll':
                 api.step.empty('not a roll bucket, ignoring')
                 continue

             for builder, status in builders.items():
                 with api.step.nest(builder):
                     pres = api.step.empty('data').presentation
                     pres.step_summary_text = repr(status)

                     if not status['attributed']:
                         api.step.empty('not attributed')
                         continue

                     commit = status['newest_passing']['commit']['hash']

                     key = '/'.join((bucket, builder, commit))
                     api.step.empty('key').presentation.step_summary_text = key
                     if key in state:
                         api.step.empty('already triggered')
                         continue

                     state[key] = now

                     host = api.gerrit.host_from_remote_url(status['remote'])
                     host = host.replace('-review.', '.')

                     requests.append(
                         api.buildbucket.schedule_request(
                             bucket=bucket,
                             builder=builder,
                             gitiles_commit=common_pb.GitilesCommit(
                                 host=host,
                                 project=api.gerrit.project_from_remote_url(
                                     status['remote'],
                                 ),
                                 id=commit,
                                 ref=f'refs/heads/{status["branch"]}',
                             ),
                             tags=[
                                 common_pb.StringPair(
                                     key='user_agent',
                                     value='post_attribution_roller_retrigger',
                                 ),
                             ],
                         ),
                     )

     if not requests:
         api.step.empty('nothing to launch')
         return

     builds_str = f'{len(requests)} build{"" if len(requests) == 1 else "s"}'
     with api.step.nest(f'launching {builds_str}') as pres:
         for request in requests:
             pres = api.step.empty(
                 f'{build.builder.bucket}/{build.builder.builder}'
             ).presentation
             pres.links['builder'] = api.buildbucket.builder_url(
                 bucket=request.builder.bucket,
                 builder=request.builder.builder,
             )
             commit = request.gitiles_commit
             pres.links['commit'] = (
                 f'https://{commit.host}/{commit.project}/+/{commit.id}'
             )
             pres.step_summary_text = f'branch {commit.ref}'

     if props.dry_run:
         api.step.empty('dry-run, exiting')
         return result_pb.RawResult(
             summary_markdown=f'dry-run, would have launched {builds_str}',
             status=common_pb.SUCCESS,
         )

     builds = api.buildbucket.schedule(requests)
     links: list[tuple[str, str]] = []
     for build in builds:
         bucket_builder = f'{build.builder.bucket}/{build.builder.builder}'
         link = api.buildbucket.build_url(build_id=build.id)
         links.append((bucket_builder, link))

     api.builder_state.save(state)

     summary = [f'launched {builds_str}', '']
     summary.extend(f'* [{text}]({link})' for text, link in links)

     return result_pb.RawResult(
         summary_markdown='\n'.join(summary),
         status=common_pb.SUCCESS,
     )


 def GenTests(
     api: recipe_test_api.RecipeTestApi,
 ) -> Iterator[recipe_test_api.TestData]:
     def state(
         *,
         bucket: str = 'roll',
         builder: str = 'roller',
         commit: str = '1234123412341234123412341234123412341234',
         timestamp: int = 1337025000,
     ) -> recipe_test_api.TestData:
         return api.builder_state(
             {f'{bucket}/{builder}/{commit}': timestamp},
             prefix='state',
         )

     def bisector(
         *,
         attributed: bool,
         bucket: str = 'roll',
         builder: str = 'roller',
         remote: str = 'https://pigweed.googlesource.com/pigweed/pigweed',
         branch: str = 'main',
         newest_passing: str = '1234123412341234123412341234123412341234',
         oldest_failure: str = '8765876587658765876587658765876587658765',
     ) -> recipe_test_api.TestData:
         props = {
             'builders': {
                 bucket: {
                     builder: {
                         'attributed': attributed,
                         'newest_passing': {
                             'age': 4 if attributed else 8,
                             'commit': {
                                 'firstLine': 'pw_modname: stuff',
                                 'author': 'nobody@example.com',
                                 'hash': newest_passing,
                             },
                         },
                         'oldest_failure': {
                             'age': 3,
                             'commit': {
                                 'firstLine': 'pw_modname: stuff',
                                 'author': 'nobody@example.com',
                                 'hash': oldest_failure,
                             },
                         },
                         'scheduling': False,
                         'remote': remote,
                         'branch': branch,
                     },
                 },
             },
         }

         build = build_pb.Build()
         build.output.properties.update(props)
         return api.buildbucket.simulated_search_results(
             [build],
             step_name='bisector.buildbucket.search',
         )

     def assert_not_roller(*, bucket: str) -> recipe_test_api.TestData:
         return api.post_check(
             post_process.MustRun,
             f'{bucket}.not a roll bucket, ignoring',
         )

     def assert_not_attributed(
         *, bucket: str = 'roll', builder: str = 'roller'
     ) -> recipe_test_api.TestData:
         return api.post_check(
             post_process.MustRun,
             f'{bucket}.{builder}.not attributed',
         )

     def assert_already_triggered(
         *, bucket: str = 'roll', builder: str = 'roller'
     ) -> recipe_test_api.TestData:
         return api.post_check(
             post_process.MustRun,
             f'{bucket}.{builder}.already triggered',
         )

     def assert_nothing_to_launch() -> recipe_test_api.TestData:
         return api.post_check(post_process.MustRun, 'nothing to launch')

     def assert_launching(n: int) -> recipe_test_api.TestData:
         return api.post_check(
             post_process.MustRunRE,
             rf'^launching {n} builds?$',
         )

     def assert_dry_run() -> recipe_test_api.TestData:
         return api.post_check(post_process.MustRun, 'dry-run, exiting')

     def drop_expectation() -> recipe_test_api.TestData:
         return api.post_process(post_process.DropExpectation)

     yield api.test(
         'not-roller',
         bisector(bucket='ci', attributed=True),
         assert_not_roller(bucket='ci'),
         assert_nothing_to_launch(),
         drop_expectation(),
     )

     yield api.test(
         'not-attributed',
         bisector(attributed=False),
         assert_not_attributed(),
         assert_nothing_to_launch(),
         drop_expectation(),
     )

     yield api.test(
         'attributed',
         bisector(attributed=True),
         assert_launching(1),
         drop_expectation(),
     )

     yield api.test(
         'already-triggered',
         state(),
         bisector(attributed=True),
         assert_already_triggered(),
         drop_expectation(),
     )

     yield api.test(
         'dry-run',
         api.properties(dry_run=True),
         bisector(attributed=True),
         assert_launching(1),
         assert_dry_run(),
         drop_expectation(),
     )

     yield api.test(
         'try',
         api.buildbucket.try_build(),
         api.properties(),
         bisector(attributed=True),
         assert_launching(1),
         assert_dry_run(),
         drop_expectation(),
     )
	# Copyright 2025 The Pigweed Authors
	#
	# Licensed under the Apache License, Version 2.0 (the "License"); you may not
	# use this file except in compliance with the License. You may obtain a copy of
	# the License at
	#
	# https://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
	# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
	# License for the specific language governing permissions and limitations under
	# the License.
	"""Rerun rollers after the bisector determines responsible commit.

	Rollers run in dry-run mode during bisection. Once a new failure has been
	attributed to a specific commit, rerun the roller on the previous commit, but
	not in dry-run mode.

	This recipe is typically triggered after a bisector run completes. It inspects
	the output properties of the bisector build to identify any "roller" builders
	where a failure was successfully attributed to a specific commit.

	For each such attributed failure in a roller:
	1. It identifies the "newest_passing" commit reported by the bisector.
	2. It checks an internal state to see if a build for this specific roller and
	commit has already been re-triggered recently (within the last 14 days).
	3. If not already re-triggered, it schedules a new build for that roller using
	the "newest_passing" commit. This new build is not run in dry-run mode,
	allowing the roller to attempt the actual roll.

	The recipe uses the `InputProperties` message, which defines:
	- `bucket`: The bucket of the bisector builder to query. Defaults to the
	bucket of the current build if not specified.
	- `builder`: The name of the bisector builder to query. Defaults to "bisector"
	if not specified.
	- `dry_run`: A boolean flag. If true, the recipe will identify and report the
	roller builds it would re-trigger but will not actually schedule
	them. This defaults to true for development or try builds.

	The recipe outputs a summary of the builds it launched (or would have launched
	in dry-run mode).

	If all bisector-launched rolls fail, this recipe will likely rerun a roll that
	already ran in a non-dry-run mode. Since this roll will be a no-op it will
	complete quickly.
	"""

	from __future__ import annotations

	import datetime
	from collections.abc import Iterator

	from google.protobuf import json_format

	from recipe_engine import post_process, recipe_api, recipe_test_api

	from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb
	from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb
	from PB.recipe_engine import result as result_pb
	from PB.recipes.pigweed.post_attribution_roller_retrigger import InputProperties

	from RECIPE_MODULES.pigweed.raw_result import api as raw_result_api

	DEPS = [
	'fuchsia/buildbucket_util',
	'fuchsia/builder_state',
	'fuchsia/gerrit',
	'pigweed/raw_result',
	'recipe_engine/buildbucket',
	'recipe_engine/properties',
	'recipe_engine/step',
	'recipe_engine/time',
	]

	PROPERTIES = InputProperties


	@raw_result_api.wrap_run_steps
	def RunSteps(api: recipe_api.RecipeApi, props: InputProperties):
	props.dry_run = props.dry_run or api.buildbucket_util.is_dev_or_try

	now = api.time.utcnow().timestamp()
	bucket = props.bucket or api.buildbucket.build.builder.bucket
	builder = props.builder or 'bisector'

	with api.step.nest('state'):
	api.step.empty('now').presentation.step_summary_text = str(now)
	old_state = api.builder_state.fetch_previous_state()
	state = {}
	for bucket_builder_commit, timestamp in old_state.items():
	if now - timestamp < 14 * 24 * 60 * 60:
	state[bucket_builder_commit] = timestamp
	api.builder_state.save(state)

	max_age = datetime.timedelta(hours=2)
	if api.buildbucket_util.is_dev_or_try:
	max_age = datetime.timedelta(days=7)

	with api.step.nest('bisector'):
	build = api.buildbucket_util.last_build(
	bucket=bucket,
	builder=builder,
	status=common_pb.ENDED_MASK,
	max_age=max_age,
	)

	if not build:
	api.step.empty( # pragma: no cover
	'no bisector build found',
	status='FAILURE',
	)

	bisector_props = json_format.MessageToDict(build.output.properties)
	bisector_status = bisector_props.get('builders', {})
	requests = []
	for bucket, builders in bisector_status.items():
	with api.step.nest(bucket):
	if not bucket.split('.')[-1] == 'roll':
	api.step.empty('not a roll bucket, ignoring')
	continue

	for builder, status in builders.items():
	with api.step.nest(builder):
	pres = api.step.empty('data').presentation
	pres.step_summary_text = repr(status)

	if not status['attributed']:
	api.step.empty('not attributed')
	continue

	commit = status['newest_passing']['commit']['hash']

	key = '/'.join((bucket, builder, commit))
	api.step.empty('key').presentation.step_summary_text = key
	if key in state:
	api.step.empty('already triggered')
	continue

	state[key] = now

	host = api.gerrit.host_from_remote_url(status['remote'])
	host = host.replace('-review.', '.')

	requests.append(
	api.buildbucket.schedule_request(
	bucket=bucket,
	builder=builder,
	gitiles_commit=common_pb.GitilesCommit(
	host=host,
	project=api.gerrit.project_from_remote_url(
	status['remote'],
	),
	id=commit,
	ref=f'refs/heads/{status["branch"]}',
	),
	tags=[
	common_pb.StringPair(
	key='user_agent',
	value='post_attribution_roller_retrigger',
	),
	],
	),
	)

	if not requests:
	api.step.empty('nothing to launch')
	return

	builds_str = f'{len(requests)} build{"" if len(requests) == 1 else "s"}'
	with api.step.nest(f'launching {builds_str}') as pres:
	for request in requests:
	pres = api.step.empty(
	f'{build.builder.bucket}/{build.builder.builder}'
	).presentation
	pres.links['builder'] = api.buildbucket.builder_url(
	bucket=request.builder.bucket,
	builder=request.builder.builder,
	)
	commit = request.gitiles_commit
	pres.links['commit'] = (
	f'https://{commit.host}/{commit.project}/+/{commit.id}'
	)
	pres.step_summary_text = f'branch {commit.ref}'

	if props.dry_run:
	api.step.empty('dry-run, exiting')
	return result_pb.RawResult(
	summary_markdown=f'dry-run, would have launched {builds_str}',
	status=common_pb.SUCCESS,
	)

	builds = api.buildbucket.schedule(requests)
	links: list[tuple[str, str]] = []
	for build in builds:
	bucket_builder = f'{build.builder.bucket}/{build.builder.builder}'
	link = api.buildbucket.build_url(build_id=build.id)
	links.append((bucket_builder, link))

	api.builder_state.save(state)

	summary = [f'launched {builds_str}', '']
	summary.extend(f'* [{text}]({link})' for text, link in links)

	return result_pb.RawResult(
	summary_markdown='\n'.join(summary),
	status=common_pb.SUCCESS,
	)


	def GenTests(
	api: recipe_test_api.RecipeTestApi,
	) -> Iterator[recipe_test_api.TestData]:
	def state(
	*,
	bucket: str = 'roll',
	builder: str = 'roller',
	commit: str = '1234123412341234123412341234123412341234',
	timestamp: int = 1337025000,
	) -> recipe_test_api.TestData:
	return api.builder_state(
	{f'{bucket}/{builder}/{commit}': timestamp},
	prefix='state',
	)

	def bisector(
	*,
	attributed: bool,
	bucket: str = 'roll',
	builder: str = 'roller',
	remote: str = 'https://pigweed.googlesource.com/pigweed/pigweed',
	branch: str = 'main',
	newest_passing: str = '1234123412341234123412341234123412341234',
	oldest_failure: str = '8765876587658765876587658765876587658765',
	) -> recipe_test_api.TestData:
	props = {
	'builders': {
	bucket: {
	builder: {
	'attributed': attributed,
	'newest_passing': {
	'age': 4 if attributed else 8,
	'commit': {
	'firstLine': 'pw_modname: stuff',
	'author': 'nobody@example.com',
	'hash': newest_passing,
	},
	},
	'oldest_failure': {
	'age': 3,
	'commit': {
	'firstLine': 'pw_modname: stuff',
	'author': 'nobody@example.com',
	'hash': oldest_failure,
	},
	},
	'scheduling': False,
	'remote': remote,
	'branch': branch,
	},
	},
	},
	}

	build = build_pb.Build()
	build.output.properties.update(props)
	return api.buildbucket.simulated_search_results(
	[build],
	step_name='bisector.buildbucket.search',
	)

	def assert_not_roller(*, bucket: str) -> recipe_test_api.TestData:
	return api.post_check(
	post_process.MustRun,
	f'{bucket}.not a roll bucket, ignoring',
	)

	def assert_not_attributed(
	*, bucket: str = 'roll', builder: str = 'roller'
	) -> recipe_test_api.TestData:
	return api.post_check(
	post_process.MustRun,
	f'{bucket}.{builder}.not attributed',
	)

	def assert_already_triggered(
	*, bucket: str = 'roll', builder: str = 'roller'
	) -> recipe_test_api.TestData:
	return api.post_check(
	post_process.MustRun,
	f'{bucket}.{builder}.already triggered',
	)

	def assert_nothing_to_launch() -> recipe_test_api.TestData:
	return api.post_check(post_process.MustRun, 'nothing to launch')

	def assert_launching(n: int) -> recipe_test_api.TestData:
	return api.post_check(
	post_process.MustRunRE,
	rf'^launching {n} builds?$',
	)

	def assert_dry_run() -> recipe_test_api.TestData:
	return api.post_check(post_process.MustRun, 'dry-run, exiting')

	def drop_expectation() -> recipe_test_api.TestData:
	return api.post_process(post_process.DropExpectation)

	yield api.test(
	'not-roller',
	bisector(bucket='ci', attributed=True),
	assert_not_roller(bucket='ci'),
	assert_nothing_to_launch(),
	drop_expectation(),
	)

	yield api.test(
	'not-attributed',
	bisector(attributed=False),
	assert_not_attributed(),
	assert_nothing_to_launch(),
	drop_expectation(),
	)

	yield api.test(
	'attributed',
	bisector(attributed=True),
	assert_launching(1),
	drop_expectation(),
	)

	yield api.test(
	'already-triggered',
	state(),
	bisector(attributed=True),
	assert_already_triggered(),
	drop_expectation(),
	)

	yield api.test(
	'dry-run',
	api.properties(dry_run=True),
	bisector(attributed=True),
	assert_launching(1),
	assert_dry_run(),
	drop_expectation(),
	)

	yield api.test(
	'try',
	api.buildbucket.try_build(),
	api.properties(),
	bisector(attributed=True),
	assert_launching(1),
	assert_dry_run(),
	drop_expectation(),
	)