pw_presubmit: Add timeouts for steps
If there are x minutes left in the build, run the current presubmit step
with a timeout of (x-1) minutes. This ensures that if a presubmit step
is stuck we kill it and have sufficient time to upload results to logdog
and/or GCS, hopefully giving enough data to determine why it was stuck.
If the timeout would be negative or very small set it to 30 seconds.
Change-Id: Ic1ca01432b3db9e57661e36290421b8e6068d1cc
Reviewed-on: https://pigweed-review.googlesource.com/c/infra/recipes/+/63760
Reviewed-by: Ted Pudlik <tpudlik@google.com>
Commit-Queue: Rob Mohr <mohrr@google.com>
diff --git a/recipes/pw_presubmit.expected/pigweed.json b/recipes/pw_presubmit.expected/pigweed.json
index 3af3be5..431ce37 100644
--- a/recipes/pw_presubmit.expected/pigweed.json
+++ b/recipes/pw_presubmit.expected/pigweed.json
@@ -1216,6 +1216,7 @@
}
},
"name": "full_0.run",
+ "timeout": 30,
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@"
]
@@ -1400,6 +1401,7 @@
}
},
"name": "full_1.run",
+ "timeout": 30,
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@"
]
diff --git a/recipes/pw_presubmit.expected/repo.json b/recipes/pw_presubmit.expected/repo.json
index af21298..7684296 100644
--- a/recipes/pw_presubmit.expected/repo.json
+++ b/recipes/pw_presubmit.expected/repo.json
@@ -1041,6 +1041,7 @@
}
},
"name": "step.run",
+ "timeout": 30,
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@"
]
diff --git a/recipes/pw_presubmit.expected/sign-nobuildid.json b/recipes/pw_presubmit.expected/sign-nobuildid.json
index 259a949..f2c7246 100644
--- a/recipes/pw_presubmit.expected/sign-nobuildid.json
+++ b/recipes/pw_presubmit.expected/sign-nobuildid.json
@@ -1168,6 +1168,7 @@
}
},
"name": "release.run",
+ "timeout": 30,
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@"
]
@@ -1896,7 +1897,7 @@
"-u",
"[CACHE]/cipd/path/to/gsutil/version%3Apinned-version/gsutil",
"-h",
- "Custom-Time:2012-05-14T12:53:23.000000Z",
+ "Custom-Time:2012-05-14T12:53:24.500000Z",
"-o",
"GSUtil:software_update_check_period=0",
"cp",
@@ -1931,7 +1932,7 @@
"-u",
"[CACHE]/cipd/path/to/gsutil/version%3Apinned-version/gsutil",
"-h",
- "Custom-Time:2012-05-14T12:53:24.500000Z",
+ "Custom-Time:2012-05-14T12:53:26.000000Z",
"-o",
"GSUtil:software_update_check_period=0",
"cp",
@@ -1966,7 +1967,7 @@
"-u",
"[CACHE]/cipd/path/to/gsutil/version%3Apinned-version/gsutil",
"-h",
- "Custom-Time:2012-05-14T12:53:26.000000Z",
+ "Custom-Time:2012-05-14T12:53:27.500000Z",
"-h",
"x-goog-meta-signature:John Hancock",
"-o",
@@ -2003,7 +2004,7 @@
"-u",
"[CACHE]/cipd/path/to/gsutil/version%3Apinned-version/gsutil",
"-h",
- "Custom-Time:2012-05-14T12:53:27.500000Z",
+ "Custom-Time:2012-05-14T12:53:29.000000Z",
"-o",
"GSUtil:software_update_check_period=0",
"cp",
@@ -2038,7 +2039,7 @@
"-u",
"[CACHE]/cipd/path/to/gsutil/version%3Apinned-version/gsutil",
"-h",
- "Custom-Time:2012-05-14T12:53:21.500000Z",
+ "Custom-Time:2012-05-14T12:53:23.000000Z",
"-o",
"GSUtil:software_update_check_period=0",
"-m",
@@ -2075,7 +2076,7 @@
"-u",
"[CACHE]/cipd/path/to/gsutil/version%3Apinned-version/gsutil",
"-h",
- "Custom-Time:2012-05-14T12:53:29.000000Z",
+ "Custom-Time:2012-05-14T12:53:30.500000Z",
"-o",
"GSUtil:software_update_check_period=0",
"cp",
diff --git a/recipes/pw_presubmit.expected/sign.json b/recipes/pw_presubmit.expected/sign.json
index 664e64b..3101d37 100644
--- a/recipes/pw_presubmit.expected/sign.json
+++ b/recipes/pw_presubmit.expected/sign.json
@@ -1168,6 +1168,7 @@
}
},
"name": "release.run",
+ "timeout": 30,
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@"
]
@@ -1896,7 +1897,7 @@
"-u",
"[CACHE]/cipd/path/to/gsutil/version%3Apinned-version/gsutil",
"-h",
- "Custom-Time:2012-05-14T12:53:23.000000Z",
+ "Custom-Time:2012-05-14T12:53:24.500000Z",
"-o",
"GSUtil:software_update_check_period=0",
"cp",
@@ -1931,7 +1932,7 @@
"-u",
"[CACHE]/cipd/path/to/gsutil/version%3Apinned-version/gsutil",
"-h",
- "Custom-Time:2012-05-14T12:53:24.500000Z",
+ "Custom-Time:2012-05-14T12:53:26.000000Z",
"-o",
"GSUtil:software_update_check_period=0",
"cp",
@@ -1966,7 +1967,7 @@
"-u",
"[CACHE]/cipd/path/to/gsutil/version%3Apinned-version/gsutil",
"-h",
- "Custom-Time:2012-05-14T12:53:26.000000Z",
+ "Custom-Time:2012-05-14T12:53:27.500000Z",
"-h",
"x-goog-meta-signature:John Hancock",
"-o",
@@ -2003,7 +2004,7 @@
"-u",
"[CACHE]/cipd/path/to/gsutil/version%3Apinned-version/gsutil",
"-h",
- "Custom-Time:2012-05-14T12:53:27.500000Z",
+ "Custom-Time:2012-05-14T12:53:29.000000Z",
"-o",
"GSUtil:software_update_check_period=0",
"cp",
@@ -2038,7 +2039,7 @@
"-u",
"[CACHE]/cipd/path/to/gsutil/version%3Apinned-version/gsutil",
"-h",
- "Custom-Time:2012-05-14T12:53:21.500000Z",
+ "Custom-Time:2012-05-14T12:53:23.000000Z",
"-o",
"GSUtil:software_update_check_period=0",
"-m",
@@ -2075,7 +2076,7 @@
"-u",
"[CACHE]/cipd/path/to/gsutil/version%3Apinned-version/gsutil",
"-h",
- "Custom-Time:2012-05-14T12:53:29.000000Z",
+ "Custom-Time:2012-05-14T12:53:30.500000Z",
"-o",
"GSUtil:software_update_check_period=0",
"cp",
diff --git a/recipes/pw_presubmit.expected/step.json b/recipes/pw_presubmit.expected/step.json
index b2caae4..1931696 100644
--- a/recipes/pw_presubmit.expected/step.json
+++ b/recipes/pw_presubmit.expected/step.json
@@ -1492,6 +1492,7 @@
}
},
"name": "step1.run",
+ "timeout": 40.0,
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@"
]
@@ -1678,6 +1679,7 @@
}
},
"name": "step2.run",
+ "timeout": 30,
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@"
]
@@ -2365,7 +2367,7 @@
"-u",
"[CACHE]/cipd/path/to/gsutil/version%3Apinned-version/gsutil",
"-h",
- "Custom-Time:2012-05-14T12:53:23.000000Z",
+ "Custom-Time:2020-09-13T12:28:00.000000Z",
"-o",
"GSUtil:software_update_check_period=0",
"cp",
@@ -2400,7 +2402,7 @@
"-u",
"[CACHE]/cipd/path/to/gsutil/version%3Apinned-version/gsutil",
"-h",
- "Custom-Time:2012-05-14T12:53:24.500000Z",
+ "Custom-Time:2020-09-13T12:28:20.000000Z",
"-o",
"GSUtil:software_update_check_period=0",
"cp",
@@ -2435,7 +2437,7 @@
"-u",
"[CACHE]/cipd/path/to/gsutil/version%3Apinned-version/gsutil",
"-h",
- "Custom-Time:2012-05-14T12:53:21.500000Z",
+ "Custom-Time:2020-09-13T12:27:40.000000Z",
"-o",
"GSUtil:software_update_check_period=0",
"-m",
@@ -2472,7 +2474,7 @@
"-u",
"[CACHE]/cipd/path/to/gsutil/version%3Apinned-version/gsutil",
"-h",
- "Custom-Time:2012-05-14T12:53:26.000000Z",
+ "Custom-Time:2020-09-13T12:28:40.000000Z",
"-o",
"GSUtil:software_update_check_period=0",
"cp",
diff --git a/recipes/pw_presubmit.py b/recipes/pw_presubmit.py
index 7162696..4a622f1 100644
--- a/recipes/pw_presubmit.py
+++ b/recipes/pw_presubmit.py
@@ -13,6 +13,8 @@
# the License.
"""Recipe for testing Pigweed using presubmit_checks.py script."""
+import datetime
+
from PB.go.chromium.org.luci.buildbucket.proto import common
from PB.recipes.pigweed.pw_presubmit import InputProperties
from PB.recipe_engine import result
@@ -23,6 +25,7 @@
'pigweed/checkout',
'pigweed/environment',
'pigweed/util',
+ 'recipe_engine/buildbucket',
'recipe_engine/file',
'recipe_engine/futures',
'recipe_engine/json',
@@ -31,6 +34,7 @@
'recipe_engine/python',
'recipe_engine/raw_io',
'recipe_engine/step',
+ 'recipe_engine/time',
]
PROPERTIES = InputProperties
@@ -45,6 +49,30 @@
RELEASE_PUBKEY_FILENAME = 'publickey.pem'
+def _step_timeout(api):
+ # Amount of time elapsed in the run.
+ elapsed_time = api.time.time() - api.buildbucket.build.start_time.seconds
+
+ # Amount of time before build times out.
+ time_remaining = (
+ api.buildbucket.build.execution_timeout.seconds - elapsed_time
+ )
+
+ # Give a buffer before build times out and kill this step then. This should
+ # give enough time to read any logfiles and maybe upload to logdog/GCS
+ # before the build times out.
+ step_timeout = time_remaining - 60
+
+ # If the timeout would be negative or very small set it to 30 seconds. We
+ # likely won't have enough information to debug these steps, but in case
+ # they're fast there's no reason to kill them much before the build is
+ # terminated.
+ if step_timeout < 30:
+ step_timeout = 30
+
+ return step_timeout
+
+
def _try_sign_archive(api, archive_path, name):
args = [
'--archive-file',
@@ -123,7 +151,11 @@
with api.step.defer_results():
for step in steps:
with api.step.nest(step) as pres:
- api.step('run', prefix + ['--step', step])
+ api.step(
+ 'run',
+ prefix + ['--step', step],
+ timeout=_step_timeout(api),
+ )
build_dir = presubmit_dir.join(step)
files = [
@@ -346,9 +378,14 @@
yield (
api.status_check.test('step')
+ properties(step=['step1', 'step2'], gcs_bucket='bucket')
- + api.checkout.try_test_data()
+ + api.checkout.try_test_data(
+ start_time=datetime.datetime.utcfromtimestamp(1600000000),
+ execution_timeout=120,
+ )
+ api.step_data('upload.get build id', retcode=1)
+ ls_export('step1', 'foo')
+ + api.time.seed(1600000000)
+ + api.time.step(20.0)
)
manifest = 'https://pigweed.googlesource.com/pigweed/manifest'
