pw_presubmit: Allow exiting early if failing in CI
Bug: b/295020927
Change-Id: I1416ffd168042c3c73a74a5c588cbe184614fd06
Reviewed-on: https://pigweed-review.googlesource.com/c/infra/recipes/+/164371
Reviewed-by: Taylor Cramer <cramertj@google.com>
Pigweed-Auto-Submit: Rob Mohr <mohrr@google.com>
Commit-Queue: Auto-Submit <auto-submit@pigweed-service-accounts.iam.gserviceaccount.com>
Presubmit-Verified: CQ Bot Account <pigweed-scoped@luci-project-accounts.iam.gserviceaccount.com>
diff --git a/recipes/pw_presubmit.expected/one_step_exit_early.json b/recipes/pw_presubmit.expected/one_step_exit_early.json
new file mode 100644
index 0000000..a298564
--- /dev/null
+++ b/recipes/pw_presubmit.expected/one_step_exit_early.json
@@ -0,0 +1,101 @@
+[
+ {
+ "cmd": [],
+ "name": "fetch project cr-buildbucket.cfg"
+ },
+ {
+ "cmd": [
+ "luci-auth",
+ "token",
+ "-lifetime",
+ "3m"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "project:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "fetch project cr-buildbucket.cfg.get access token for default account",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::url]/resources/pycurl.py",
+ "--url",
+ "https://luci-config.appspot.com/_ah/api/config/v1/config_sets/projects/project/config/cr-buildbucket.cfg",
+ "--status-json",
+ "/path/to/tmp/json",
+ "--outfile",
+ "/path/to/tmp/json",
+ "--headers-json",
+ "{\"Authorization\": \"Bearer extra.secret.token.should.not.be.logged\"}"
+ ],
+ "luci_context": {
+ "realm": {
+ "name": "project:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "fetch project cr-buildbucket.cfg.get",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "bb",
+ "ls",
+ "-host",
+ "cr-buildbucket.appspot.com",
+ "-json",
+ "-nopage",
+ "-n",
+ "10",
+ "-fields",
+ "status",
+ "-predicate",
+ "{\"builder\": {\"bucket\": \"ci\", \"builder\": \"builder\", \"project\": \"project\"}}"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "project:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "buildbucket.search",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@raw_io.output_text@{\"status\": \"FAILURE\"}@@@",
+ "@@@STEP_LOG_END@raw_io.output_text@@@",
+ "@@@STEP_LINK@0@https://cr-buildbucket.appspot.com/build/8945511751514863184@@@"
+ ]
+ },
+ {
+ "name": "$result",
+ "summaryMarkdown": "Exiting early since [project/ci/builder](https://ci.chromium.org/ui/p/project/builders/ci/builder) has not recently passed."
+ }
+]
\ No newline at end of file
diff --git a/recipes/pw_presubmit.expected/one_step.json b/recipes/pw_presubmit.expected/one_step_no_exit.json
similarity index 96%
rename from recipes/pw_presubmit.expected/one_step.json
rename to recipes/pw_presubmit.expected/one_step_no_exit.json
index b7d0266..794a2ff 100644
--- a/recipes/pw_presubmit.expected/one_step.json
+++ b/recipes/pw_presubmit.expected/one_step_no_exit.json
@@ -1,6 +1,101 @@
[
{
"cmd": [],
+ "name": "fetch project cr-buildbucket.cfg"
+ },
+ {
+ "cmd": [
+ "luci-auth",
+ "token",
+ "-lifetime",
+ "3m"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "project:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "fetch project cr-buildbucket.cfg.get access token for default account",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::url]/resources/pycurl.py",
+ "--url",
+ "https://luci-config.appspot.com/_ah/api/config/v1/config_sets/projects/project/config/cr-buildbucket.cfg",
+ "--status-json",
+ "/path/to/tmp/json",
+ "--outfile",
+ "/path/to/tmp/json",
+ "--headers-json",
+ "{\"Authorization\": \"Bearer extra.secret.token.should.not.be.logged\"}"
+ ],
+ "luci_context": {
+ "realm": {
+ "name": "project:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "fetch project cr-buildbucket.cfg.get",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "bb",
+ "ls",
+ "-host",
+ "cr-buildbucket.appspot.com",
+ "-json",
+ "-nopage",
+ "-n",
+ "10",
+ "-fields",
+ "status",
+ "-predicate",
+ "{\"builder\": {\"bucket\": \"ci\", \"builder\": \"builder\", \"project\": \"project\"}}"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "project:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "buildbucket.search",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@raw_io.output_text@{\"status\": \"SUCCESS\"}@@@",
+ "@@@STEP_LOG_END@raw_io.output_text@@@",
+ "@@@STEP_LINK@0@https://cr-buildbucket.appspot.com/build/8945511751514863184@@@"
+ ]
+ },
+ {
+ "cmd": [],
"name": "checkout pigweed",
"~followup_annotations": [
"@@@STEP_LINK@applied pigweed:123456@https://pigweed-review.googlesource.com/c/123456@@@"
diff --git a/recipes/pw_presubmit.proto b/recipes/pw_presubmit.proto
index d3efcf2..1124848 100644
--- a/recipes/pw_presubmit.proto
+++ b/recipes/pw_presubmit.proto
@@ -51,4 +51,10 @@
// then STEP_NAME_DEFAULT is interpreted as WITH_WITHOUT_STEP_NAME. When there
// are multiple steps it's interpreted as ONLY_WITH_STEP_NAME.
StepName metadata_step_name_usage = 6;
+
+ // If the corresponding builder is persistently failing in CI, then have this
+ // builder exit immediately with a passing status. This way failing builders
+ // don't block continuing development. Only applies to TRY builders launched
+ // by CV.
+ bool exit_tryjob_early_if_failing_in_ci = 7;
}
diff --git a/recipes/pw_presubmit.py b/recipes/pw_presubmit.py
index 62ced87..4701158 100644
--- a/recipes/pw_presubmit.py
+++ b/recipes/pw_presubmit.py
@@ -14,17 +14,22 @@
"""Recipe for testing Pigweed using presubmit_checks.py script."""
import datetime
+import re
from PB.go.chromium.org.luci.buildbucket.proto import common
from PB.recipes.pigweed.pw_presubmit import InputProperties, StepName
from PB.recipe_engine import result
DEPS = [
+ 'fuchsia/buildbucket_util',
+ 'fuchsia/builder_status',
'fuchsia/gsutil',
'pigweed/checkout',
'pigweed/environment',
'pigweed/pw_presubmit',
'pigweed/util',
+ 'recipe_engine/buildbucket',
+ 'recipe_engine/cq',
'recipe_engine/file',
'recipe_engine/futures',
'recipe_engine/json',
@@ -65,6 +70,23 @@
"""Run Pigweed presubmit checks."""
gcs_bucket = props.gcs_bucket
+ if (
+ api.buildbucket_util.is_tryjob and
+ api.cq.active and
+ props.exit_tryjob_early_if_failing_in_ci
+ ):
+ bucket = re.sub(r'\btry\b', 'ci', api.buildbucket.build.builder.bucket)
+ status = api.builder_status.retrieve(bucket=bucket)
+ if not api.builder_status.has_recently_passed(status):
+ full_name = '/'.join((status.project, bucket, status.builder))
+ return result.RawResult(
+ summary_markdown=(
+ f'Exiting early since [{full_name}]({status.link}) has not '
+ 'recently passed.'
+ ),
+ status=common.SUCCESS,
+ )
+
checkout = api.checkout(props.checkout_options)
env = api.environment.init(checkout, props.environment_options)
@@ -273,6 +295,7 @@
def properties(
*,
+ exit_tryjob_early_if_failing_in_ci=False,
extensions_to_sign=('.out',),
gcs_bucket=None,
metadata_step_name_usage=None,
@@ -286,14 +309,31 @@
metadata_step_name_usage
)
props.extensions_to_sign.extend(extensions_to_sign)
+ props.exit_tryjob_early_if_failing_in_ci = (
+ exit_tryjob_early_if_failing_in_ci
+ )
if gcs_bucket:
props.gcs_bucket = gcs_bucket
return api.properties(props)
yield (
- api.test('one_step')
- + properties(step=['step1'])
+ api.test('one_step_no_exit')
+ + properties(step=['step1'], exit_tryjob_early_if_failing_in_ci=True)
+ api.checkout.try_test_data()
+ + api.cq(run_mode=api.cq.DRY_RUN)
+ + api.buildbucket.simulated_search_results(
+ [api.builder_status.passed()]
+ )
+ )
+
+ yield (
+ api.test('one_step_exit_early')
+ + properties(step=['step1'], exit_tryjob_early_if_failing_in_ci=True)
+ + api.checkout.try_test_data()
+ + api.cq(run_mode=api.cq.DRY_RUN)
+ + api.buildbucket.simulated_search_results(
+ [api.builder_status.failure()]
+ )
)
yield (
