bazel: Read resultstore link asynchronously
Read the resultstore link asynchronously so it's accessible while the
build is still running.
Bug: b/363338443
Change-Id: Id9908c4c6fd8059909ef9118e21e54869590d81c
Reviewed-on: https://pigweed-review.googlesource.com/c/infra/recipes/+/238815
Commit-Queue: Auto-Submit <auto-submit@pigweed-service-accounts.iam.gserviceaccount.com>
Pigweed-Auto-Submit: Rob Mohr <mohrr@google.com>
Lint: Lint 🤖 <android-build-ayeaye@system.gserviceaccount.com>
Reviewed-by: Ted Pudlik <tpudlik@google.com>
Presubmit-Verified: CQ Bot Account <pigweed-scoped@luci-project-accounts.iam.gserviceaccount.com>
diff --git a/recipe_modules/bazel/__init__.py b/recipe_modules/bazel/__init__.py
index 8250b08..e4c03ec 100644
--- a/recipe_modules/bazel/__init__.py
+++ b/recipe_modules/bazel/__init__.py
@@ -23,9 +23,11 @@
'recipe_engine/context',
'recipe_engine/defer',
'recipe_engine/file',
+ 'recipe_engine/futures',
'recipe_engine/json',
'recipe_engine/path',
'recipe_engine/platform',
'recipe_engine/properties',
'recipe_engine/step',
+ 'recipe_engine/time',
]
diff --git a/recipe_modules/bazel/api.py b/recipe_modules/bazel/api.py
index e97e895..26fd954 100644
--- a/recipe_modules/bazel/api.py
+++ b/recipe_modules/bazel/api.py
@@ -183,31 +183,60 @@
*args,
*base_args,
]
- defer(
- self.api.step,
- shlex.join(args),
- cmd,
- **kwargs,
- )
- self.api.path.mock_add_file(json_path)
- if self.api.path.isfile(json_path):
- with self.api.step.nest('resultstore link') as pres:
+ with self.api.step.nest(shlex.join(args)):
+ future = self.api.futures.spawn(
+ defer,
+ self.api.step,
+ 'bazel',
+ cmd,
+ **kwargs,
+ )
+
+ # Ensure the bazel step shows up before the
+ # resultstore link step.
+ self.api.time.sleep(1)
+
+ def read_json() -> bool:
+ if not self.api.path.isfile(json_path):
+ return False
+
data = self.api.file.read_json(
- 'read',
+ f'read {i}',
json_path,
- test_data={
- 'resultstore': 'https://result.store/',
- },
+ test_data=dict(
+ resultstore='https://result.store/',
+ ),
)
- if 'resultstore' in data:
- pres.links['resultstore'] = data[
- 'resultstore'
- ]
- else: # pragma: no cover
- pres.step_summary_text = (
- 'no resultstore link found'
- )
+
+ if 'resultstore' not in data:
+ return False # pragma: no cover
+
+ pres.links['resultstore'] = data['resultstore']
+ pres.step_summary_text = ''
+ return True
+
+ found_resultstore_link = False
+
+ with self.api.step.nest('resultstore link') as pres:
+ pres.step_summary_text = 'link not found'
+
+ for i in range(1, 5):
+ self.api.time.sleep(i)
+
+ if i > 1:
+ self.api.path.mock_add_file(json_path)
+
+ if read_json():
+ found_resultstore_link = True
+ break
+
+ if future.done:
+ break # pragma: no cover
+
+ _ = future.result()
+ if not found_resultstore_link:
+ read_json() # pragma: no cover
class BazelApi(recipe_api.RecipeApi):
diff --git a/recipe_modules/bazel/tests/full.py b/recipe_modules/bazel/tests/full.py
index 84fb1e5..da8c964 100644
--- a/recipe_modules/bazel/tests/full.py
+++ b/recipe_modules/bazel/tests/full.py
@@ -72,7 +72,7 @@
check = post_process.StepCommandContains
if invert:
check = post_process.StepCommandDoesNotContain
- return api.post_process(check, step, pattern)
+ return api.post_process(check, f'{step}.bazel', pattern)
def lacks_override(step: str, name: str, bzlmod=False):
return contains_override(
diff --git a/recipes/bazel.expected/simple.json b/recipes/bazel.expected/simple.json
index 5a78107..f837b92 100644
--- a/recipes/bazel.expected/simple.json
+++ b/recipes/bazel.expected/simple.json
@@ -1207,54 +1207,17 @@
"name": "default"
},
{
- "cmd": [
- "RECIPE_MODULE[pigweed::bazel]/resources/wrapper.py",
- "--json",
- "[CLEANUP]/tmp_tmp_3/metadata.json",
- "--",
- "[CLEANUP]/tmp_tmp_2/bazelisk",
- "build",
- "//...",
- "--experimental_ui_max_stdouterr_bytes=-1",
- "--config=remote_cache",
- "--bes_instance_name=pigweed-rbe-private",
- "--remote_instance_name=projects/pigweed-rbe-private/instances/default_instance",
- "--remote_upload_local_results=true"
- ],
- "cwd": "[START_DIR]/co",
- "env": {
- "BUILDBUCKET_ID": "0",
- "BUILDBUCKET_NAME": "project:bucket:builder",
- "BUILD_NUMBER": "0",
- "CCACHE_DIR": "[CACHE]/ccache",
- "CLICOLOR": "0",
- "CLICOLOR_FORCE": "0",
- "CTCACHE_DIR": "[CACHE]/clang_tidy",
- "GCC_COLORS": "",
- "GOCACHE": "[CACHE]/go",
- "NO_COLOR": "1",
- "PIP_CACHE_DIR": "[CACHE]/pip",
- "PW_ENVIRONMENT_NO_ERROR_ON_UNRECOGNIZED": "1",
- "PW_ENVSETUP_DISABLE_SPINNER": "1",
- "PW_PRESUBMIT_DISABLE_SUBPROCESS_CAPTURE": "1",
- "PW_TEST_VAR": "test_value",
- "PW_USE_COLOR": "",
- "TEST_TMPDIR": "[CACHE]/bazel",
- "TRIGGERING_CHANGES_JSON": "[CLEANUP]/tmp_tmp_1"
- },
+ "cmd": [],
"name": "default.build //...",
"~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@",
- "@@@STEP_LOG_LINE@json.output (read error)@JSON file was missing or unreadable:@@@",
- "@@@STEP_LOG_LINE@json.output (read error)@ [CLEANUP]/tmp_tmp_3/metadata.json@@@",
- "@@@STEP_LOG_END@json.output (read error)@@@"
+ "@@@STEP_NEST_LEVEL@1@@@"
]
},
{
"cmd": [],
- "name": "default.resultstore link",
+ "name": "default.build //....resultstore link",
"~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_NEST_LEVEL@2@@@",
"@@@STEP_LINK@resultstore@https://result.store/@@@"
]
},
@@ -1291,9 +1254,110 @@
"TRIGGERING_CHANGES_JSON": "[CLEANUP]/tmp_tmp_1"
},
"infra_step": true,
- "name": "default.resultstore link.read",
+ "name": "default.build //....resultstore link.read 2",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_LOG_LINE@metadata.json@{@@@",
+ "@@@STEP_LOG_LINE@metadata.json@ \"resultstore\": \"https://result.store/\"@@@",
+ "@@@STEP_LOG_LINE@metadata.json@}@@@",
+ "@@@STEP_LOG_END@metadata.json@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "RECIPE_MODULE[pigweed::bazel]/resources/wrapper.py",
+ "--json",
+ "[CLEANUP]/tmp_tmp_3/metadata.json",
+ "--",
+ "[CLEANUP]/tmp_tmp_2/bazelisk",
+ "build",
+ "//...",
+ "--experimental_ui_max_stdouterr_bytes=-1",
+ "--config=remote_cache",
+ "--bes_instance_name=pigweed-rbe-private",
+ "--remote_instance_name=projects/pigweed-rbe-private/instances/default_instance",
+ "--remote_upload_local_results=true"
+ ],
+ "cwd": "[START_DIR]/co",
+ "env": {
+ "BUILDBUCKET_ID": "0",
+ "BUILDBUCKET_NAME": "project:bucket:builder",
+ "BUILD_NUMBER": "0",
+ "CCACHE_DIR": "[CACHE]/ccache",
+ "CLICOLOR": "0",
+ "CLICOLOR_FORCE": "0",
+ "CTCACHE_DIR": "[CACHE]/clang_tidy",
+ "GCC_COLORS": "",
+ "GOCACHE": "[CACHE]/go",
+ "NO_COLOR": "1",
+ "PIP_CACHE_DIR": "[CACHE]/pip",
+ "PW_ENVIRONMENT_NO_ERROR_ON_UNRECOGNIZED": "1",
+ "PW_ENVSETUP_DISABLE_SPINNER": "1",
+ "PW_PRESUBMIT_DISABLE_SUBPROCESS_CAPTURE": "1",
+ "PW_TEST_VAR": "test_value",
+ "PW_USE_COLOR": "",
+ "TEST_TMPDIR": "[CACHE]/bazel",
+ "TRIGGERING_CHANGES_JSON": "[CLEANUP]/tmp_tmp_1"
+ },
+ "name": "default.build //....bazel",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@json.output (read error)@JSON file was missing or unreadable:@@@",
+ "@@@STEP_LOG_LINE@json.output (read error)@ [CLEANUP]/tmp_tmp_3/metadata.json@@@",
+ "@@@STEP_LOG_END@json.output (read error)@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "default.test //...",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "default.test //....resultstore link",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LINK@resultstore@https://result.store/@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[CLEANUP]/tmp_tmp_4/metadata.json",
+ "/path/to/tmp/"
+ ],
+ "cwd": "[START_DIR]/co",
+ "env": {
+ "BUILDBUCKET_ID": "0",
+ "BUILDBUCKET_NAME": "project:bucket:builder",
+ "BUILD_NUMBER": "0",
+ "CCACHE_DIR": "[CACHE]/ccache",
+ "CLICOLOR": "0",
+ "CLICOLOR_FORCE": "0",
+ "CTCACHE_DIR": "[CACHE]/clang_tidy",
+ "GCC_COLORS": "",
+ "GOCACHE": "[CACHE]/go",
+ "NO_COLOR": "1",
+ "PIP_CACHE_DIR": "[CACHE]/pip",
+ "PW_ENVIRONMENT_NO_ERROR_ON_UNRECOGNIZED": "1",
+ "PW_ENVSETUP_DISABLE_SPINNER": "1",
+ "PW_PRESUBMIT_DISABLE_SUBPROCESS_CAPTURE": "1",
+ "PW_TEST_VAR": "test_value",
+ "PW_USE_COLOR": "",
+ "TEST_TMPDIR": "[CACHE]/bazel",
+ "TRIGGERING_CHANGES_JSON": "[CLEANUP]/tmp_tmp_1"
+ },
+ "infra_step": true,
+ "name": "default.test //....resultstore link.read 2",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
"@@@STEP_LOG_LINE@metadata.json@{@@@",
"@@@STEP_LOG_LINE@metadata.json@ \"resultstore\": \"https://result.store/\"@@@",
"@@@STEP_LOG_LINE@metadata.json@}@@@",
@@ -1336,65 +1400,15 @@
"TEST_TMPDIR": "[CACHE]/bazel",
"TRIGGERING_CHANGES_JSON": "[CLEANUP]/tmp_tmp_1"
},
- "name": "default.test //...",
+ "name": "default.test //....bazel",
"~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_NEST_LEVEL@2@@@",
"@@@STEP_LOG_LINE@json.output (read error)@JSON file was missing or unreadable:@@@",
"@@@STEP_LOG_LINE@json.output (read error)@ [CLEANUP]/tmp_tmp_4/metadata.json@@@",
"@@@STEP_LOG_END@json.output (read error)@@@"
]
},
{
- "cmd": [],
- "name": "default.resultstore link (2)",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@",
- "@@@STEP_LINK@resultstore@https://result.store/@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "copy",
- "[CLEANUP]/tmp_tmp_4/metadata.json",
- "/path/to/tmp/"
- ],
- "cwd": "[START_DIR]/co",
- "env": {
- "BUILDBUCKET_ID": "0",
- "BUILDBUCKET_NAME": "project:bucket:builder",
- "BUILD_NUMBER": "0",
- "CCACHE_DIR": "[CACHE]/ccache",
- "CLICOLOR": "0",
- "CLICOLOR_FORCE": "0",
- "CTCACHE_DIR": "[CACHE]/clang_tidy",
- "GCC_COLORS": "",
- "GOCACHE": "[CACHE]/go",
- "NO_COLOR": "1",
- "PIP_CACHE_DIR": "[CACHE]/pip",
- "PW_ENVIRONMENT_NO_ERROR_ON_UNRECOGNIZED": "1",
- "PW_ENVSETUP_DISABLE_SPINNER": "1",
- "PW_PRESUBMIT_DISABLE_SUBPROCESS_CAPTURE": "1",
- "PW_TEST_VAR": "test_value",
- "PW_USE_COLOR": "",
- "TEST_TMPDIR": "[CACHE]/bazel",
- "TRIGGERING_CHANGES_JSON": "[CLEANUP]/tmp_tmp_1"
- },
- "infra_step": true,
- "name": "default.resultstore link (2).read",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@2@@@",
- "@@@STEP_LOG_LINE@metadata.json@{@@@",
- "@@@STEP_LOG_LINE@metadata.json@ \"resultstore\": \"https://result.store/\"@@@",
- "@@@STEP_LOG_LINE@metadata.json@}@@@",
- "@@@STEP_LOG_END@metadata.json@@@"
- ]
- },
- {
"name": "$result"
}
]
\ No newline at end of file
