bisector: Defer failures
Don't fail immediately if there's trouble processing one builder.
Continue to process other builders and launch builds for them and only
then fail.
Bug: b/401921575
Change-Id: I5338e788db441660238a1f6e6c185b89b1d8e23d
Reviewed-on: https://pigweed-review.googlesource.com/c/infra/recipes/+/277773
Lint: Lint 🤖 <android-build-ayeaye@system.gserviceaccount.com>
Reviewed-by: Ted Pudlik <tpudlik@google.com>
Commit-Queue: Rob Mohr <mohrr@google.com>
Presubmit-Verified: CQ Bot Account <pigweed-scoped@luci-project-accounts.iam.gserviceaccount.com>
diff --git a/recipes/bisector.py b/recipes/bisector.py
index 4f10025..5a60be0 100644
--- a/recipes/bisector.py
+++ b/recipes/bisector.py
@@ -68,6 +68,7 @@
'fuchsia/gerrit',
'fuchsia/gitiles',
'recipe_engine/buildbucket',
+ 'recipe_engine/defer',
'recipe_engine/luci_config',
'recipe_engine/properties',
'recipe_engine/step',
@@ -300,121 +301,134 @@
repos: dict[Remote, tuple[str, ...]] = {}
- for bucket in bb_cfg.buckets:
- if not include_bucket(props, bucket.name):
- if bucket.swarming.builders:
- api.step(
- f'excluding {len(bucket.swarming.builders)} builders in '
- f'bucket {bucket.name}',
- None,
- )
- continue
-
- with api.step.nest(bucket.name) as pres:
- included = excluded = 0
-
- for builder in bucket.swarming.builders:
- with api.step.nest(builder.name):
- result = process_builder(
- api=api,
- bucket_name=bucket.name,
- builder=builder,
- repos=repos,
- max_age_days=props.max_age_days,
- num_builds=props.num_builds,
+ with api.defer.context() as defer:
+ for bucket in bb_cfg.buckets:
+ if not include_bucket(props, bucket.name):
+ if bucket.swarming.builders:
+ api.step(
+ f'excluding {len(bucket.swarming.builders)} builders in '
+ f'bucket {bucket.name}',
+ None,
)
- if result.included:
- included += 1
- else:
- excluded += 1
- builds_to_launch.extend(result.triggers)
+ continue
- pres.step_summary_text = f'included {included}, excluded {excluded}'
+ with api.step.nest(bucket.name) as pres:
+ included = excluded = 0
- # These don't help users much but are useful for testing.
- api.step.empty(f'included {included}')
- api.step.empty(f'excluded {excluded}')
+ for builder in bucket.swarming.builders:
+ with api.step.nest(builder.name):
+ deferred_result = defer(
+ process_builder,
+ api=api,
+ bucket_name=bucket.name,
+ builder=builder,
+ repos=repos,
+ max_age_days=props.max_age_days,
+ num_builds=props.num_builds,
+ )
+ if deferred_result.is_ok():
+ result = deferred_result.result()
+ if result.included:
+ included += 1
+ else:
+ excluded += 1
+ builds_to_launch.extend(result.triggers)
- if not builds_to_launch:
- api.step('nothing to launch', None)
- return result_pb.RawResult(
- summary_markdown='nothing to launch',
- status=common_pb.SUCCESS,
- )
+ pres.step_summary_text = (
+ f'included {included}, excluded {excluded}'
+ )
- bb_requests: list[builds_service_pb.ScheduleBuildRequest] = []
+ # These don't help users much but are useful for testing.
+ api.step.empty(f'included {included}')
+ api.step.empty(f'excluded {excluded}')
- for trigger in builds_to_launch:
- host = api.gerrit.host_from_remote_url(trigger.remote)
- host = host.replace('-review.', '.')
+ if not builds_to_launch:
+ api.step('nothing to launch', None)
+ return result_pb.RawResult(
+ summary_markdown='nothing to launch',
+ status=common_pb.SUCCESS,
+ )
- bb_requests.append(
- api.buildbucket.schedule_request(
- bucket=trigger.bucket,
- builder=trigger.builder,
- gitiles_commit=common_pb.GitilesCommit(
- host=host,
- project=api.gerrit.project_from_remote_url(trigger.remote),
- id=trigger.commit,
- ref=f'refs/heads/{trigger.ref}',
+ bb_requests: list[builds_service_pb.ScheduleBuildRequest] = []
+
+ for trigger in builds_to_launch:
+ host = api.gerrit.host_from_remote_url(trigger.remote)
+ host = host.replace('-review.', '.')
+
+ bb_requests.append(
+ api.buildbucket.schedule_request(
+ bucket=trigger.bucket,
+ builder=trigger.builder,
+ gitiles_commit=common_pb.GitilesCommit(
+ host=host,
+ project=api.gerrit.project_from_remote_url(
+ trigger.remote,
+ ),
+ id=trigger.commit,
+ ref=f'refs/heads/{trigger.ref}',
+ ),
+ tags=[
+ common_pb.StringPair(
+ key='user_agent',
+ value='bisector',
+ ),
+ ],
),
- tags=[
- common_pb.StringPair(key='user_agent', value='bisector'),
- ],
- ),
- )
+ )
- if props.dry_run:
- with api.step.nest('dry-run, not launching builds'):
+ if props.dry_run:
+ with api.step.nest('dry-run, not launching builds'):
+ links: list[tuple[str, str]] = []
+
+ for req in bb_requests:
+ bucket_builder: str = (
+ f'{req.builder.bucket}/{req.builder.builder}'
+ )
+ pres = api.step.empty(bucket_builder).presentation
+ builder_link = (
+ f'https://ci.chromium.org/ui/p/{req.builder.project}/'
+ f'builders/{bucket_builder}'
+ )
+ pres.links['builder'] = builder_link
+ links.append((bucket_builder, builder_link))
+
+ commit = req.gitiles_commit
+ pres.links['commit'] = (
+ f'https://{commit.host}/{commit.project}/+/{commit.id}'
+ )
+
+ links_combined: str = ''.join(
+ f'<br/>[{name}]({link})' for name, link in links
+ )
+
+ return result_pb.RawResult(
+ summary_markdown=f'dry-run, would have launched: {links_combined}',
+ status=common_pb.SUCCESS,
+ )
+
+ with api.step.nest('launch') as pres:
links: list[tuple[str, str]] = []
- for req in bb_requests:
- bucket_builder: str = (
- f'{req.builder.bucket}/{req.builder.builder}'
- )
- pres = api.step.empty(bucket_builder).presentation
- builder_link = (
- f'https://ci.chromium.org/ui/p/{req.builder.project}/'
- f'builders/{bucket_builder}'
- )
- pres.links['builder'] = builder_link
- links.append((bucket_builder, builder_link))
+ if bb_requests:
+ deferred_result = defer(api.buildbucket.schedule, bb_requests)
+ if deferred_result.is_ok():
+ builds: list[build_pb.Build] = deferred_result.result()
+ for build in builds:
+ bucket_builder: str = (
+ f'{build.builder.bucket}/{build.builder.builder}'
+ )
+ link: str = api.buildbucket.build_url(build_id=build.id)
+ pres.links[bucket_builder] = link
+ links.append((bucket_builder, link))
- commit = req.gitiles_commit
- pres.links['commit'] = (
- f'https://{commit.host}/{commit.project}/+/{commit.id}'
- )
+ links_combined: str = ''.join(
+ f'<br/>[{name}]({link})' for name, link in links
+ )
- links_combined: str = ''.join(
- f'<br/>[{name}]({link})' for name, link in links
- )
-
- return result_pb.RawResult(
- summary_markdown=f'dry-run, would have launched: {links_combined}',
- status=common_pb.SUCCESS,
- )
-
- with api.step.nest('launch') as pres:
- links: list[tuple[str, str]] = []
-
- if bb_requests:
- builds: list[build_pb.Build] = api.buildbucket.schedule(bb_requests)
- for build in builds:
- bucket_builder: str = (
- f'{build.builder.bucket}/{build.builder.builder}'
- )
- link: str = api.buildbucket.build_url(build_id=build.id)
- pres.links[bucket_builder] = link
- links.append((bucket_builder, link))
-
- links_combined: str = ''.join(
- f'<br/>[{name}]({link})' for name, link in links
- )
-
- return result_pb.RawResult(
- summary_markdown=f'launched: {links_combined}',
- status=common_pb.SUCCESS,
- )
+ return result_pb.RawResult(
+ summary_markdown=f'launched: {links_combined}',
+ status=common_pb.SUCCESS,
+ )
def GenTests(api) -> Generator[recipe_test_api.TestData, None, None]:
