bazel: Add wrapper that extracts resultstore links
Add a wrapper script that extracts resultstore links to a separate file
that the recipe can read, while still allowing stdout to be visible as
it is generated.
Bug: b/368050432, b/363338443
Change-Id: Ibc487340576694b619230178890702c6603c6b90
Reviewed-on: https://pigweed-review.googlesource.com/c/infra/recipes/+/238173
Reviewed-by: Ted Pudlik <tpudlik@google.com>
Commit-Queue: Rob Mohr <mohrr@google.com>
Lint: Lint 🤖 <android-build-ayeaye@system.gserviceaccount.com>
Presubmit-Verified: CQ Bot Account <pigweed-scoped@luci-project-accounts.iam.gserviceaccount.com>
diff --git a/recipe_modules/bazel/__init__.py b/recipe_modules/bazel/__init__.py
index e9d0090..4a70df7 100644
--- a/recipe_modules/bazel/__init__.py
+++ b/recipe_modules/bazel/__init__.py
@@ -23,6 +23,7 @@
'recipe_engine/context',
'recipe_engine/defer',
'recipe_engine/file',
+ 'recipe_engine/json',
'recipe_engine/path',
'recipe_engine/platform',
'recipe_engine/properties',
diff --git a/recipe_modules/bazel/api.py b/recipe_modules/bazel/api.py
index 381c077..84773ad 100644
--- a/recipe_modules/bazel/api.py
+++ b/recipe_modules/bazel/api.py
@@ -173,7 +173,17 @@
assert program in programs, f'{program} not in {programs}'
assert programs[program]
for args in programs[program]:
- cmd = [self.ensure(), *args, *base_args]
+ json_path = self.api.path.mkdtemp() / 'metadata.json'
+
+ cmd = [
+ self.api.bazel.resource('wrapper.py'),
+ '--json',
+ self.api.json.output(leak_to=json_path),
+ '--',
+ self.ensure(),
+ *args,
+ *base_args,
+ ]
defer(
self.api.step,
shlex.join(args),
@@ -181,6 +191,25 @@
**kwargs,
)
+ self.api.path.mock_add_file(json_path)
+ if self.api.path.isfile(json_path):
+ with self.api.step.nest('resultstore link') as pres:
+ data = self.api.file.read_json(
+ 'read',
+ json_path,
+ test_data={
+ 'resultstore': 'https://result.store/',
+ },
+ )
+ if 'resultstore' in data:
+ pres.links['resultstore'] = data[
+ 'resultstore'
+ ]
+ else: # pragma: no cover
+ pres.step_summary_text = (
+ 'no resultstore link found'
+ )
+
def nwise(iterable, n):
# nwise('ABCDEFG', 3) → ABC BCD CDE DEF EFG
diff --git a/recipe_modules/bazel/resources/wrapper.py b/recipe_modules/bazel/resources/wrapper.py
new file mode 100755
index 0000000..4c55c05
--- /dev/null
+++ b/recipe_modules/bazel/resources/wrapper.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""Invoke Bazel, and write a JSON file maybe pointing to ResultStore."""
+
+import argparse
+import json
+import re
+import subprocess
+import sys
+
+
+def run(*, cmd, json_path):
+ # Always write this file, even if we never see a resultstore link.
+ with open(json_path, 'w') as outs:
+ json.dump({}, outs)
+
+ proc = subprocess.Popen(
+ cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ )
+ for line in proc.stdout:
+ line = line.decode()
+ print(line, end='')
+ if match := re.search(
+ r'Streaming build results to:\s*(https?://.*?)\s*$',
+ line,
+ ):
+ with open(json_path, 'w') as outs:
+ json.dump(dict(resultstore=match.group(1)), outs)
+ break
+
+ for line in proc.stdout:
+ line = line.decode()
+ print(line, end='')
+
+ return proc.wait()
+
+
+def parse(argv=None):
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--json', dest='json_path', required=True)
+ parser.add_argument('cmd', nargs='+')
+ return vars(parser.parse_args(argv))
+
+
+def main(argv=None):
+ return run(**parse(argv))
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/recipes/bazel.expected/simple.json b/recipes/bazel.expected/simple.json
index 7297bc3..5a78107 100644
--- a/recipes/bazel.expected/simple.json
+++ b/recipes/bazel.expected/simple.json
@@ -1208,6 +1208,10 @@
},
{
"cmd": [
+ "RECIPE_MODULE[pigweed::bazel]/resources/wrapper.py",
+ "--json",
+ "[CLEANUP]/tmp_tmp_3/metadata.json",
+ "--",
"[CLEANUP]/tmp_tmp_2/bazelisk",
"build",
"//...",
@@ -1240,11 +1244,68 @@
},
"name": "default.build //...",
"~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output (read error)@JSON file was missing or unreadable:@@@",
+ "@@@STEP_LOG_LINE@json.output (read error)@ [CLEANUP]/tmp_tmp_3/metadata.json@@@",
+ "@@@STEP_LOG_END@json.output (read error)@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "default.resultstore link",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LINK@resultstore@https://result.store/@@@"
]
},
{
"cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[CLEANUP]/tmp_tmp_3/metadata.json",
+ "/path/to/tmp/"
+ ],
+ "cwd": "[START_DIR]/co",
+ "env": {
+ "BUILDBUCKET_ID": "0",
+ "BUILDBUCKET_NAME": "project:bucket:builder",
+ "BUILD_NUMBER": "0",
+ "CCACHE_DIR": "[CACHE]/ccache",
+ "CLICOLOR": "0",
+ "CLICOLOR_FORCE": "0",
+ "CTCACHE_DIR": "[CACHE]/clang_tidy",
+ "GCC_COLORS": "",
+ "GOCACHE": "[CACHE]/go",
+ "NO_COLOR": "1",
+ "PIP_CACHE_DIR": "[CACHE]/pip",
+ "PW_ENVIRONMENT_NO_ERROR_ON_UNRECOGNIZED": "1",
+ "PW_ENVSETUP_DISABLE_SPINNER": "1",
+ "PW_PRESUBMIT_DISABLE_SUBPROCESS_CAPTURE": "1",
+ "PW_TEST_VAR": "test_value",
+ "PW_USE_COLOR": "",
+ "TEST_TMPDIR": "[CACHE]/bazel",
+ "TRIGGERING_CHANGES_JSON": "[CLEANUP]/tmp_tmp_1"
+ },
+ "infra_step": true,
+ "name": "default.resultstore link.read",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@metadata.json@{@@@",
+ "@@@STEP_LOG_LINE@metadata.json@ \"resultstore\": \"https://result.store/\"@@@",
+ "@@@STEP_LOG_LINE@metadata.json@}@@@",
+ "@@@STEP_LOG_END@metadata.json@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "RECIPE_MODULE[pigweed::bazel]/resources/wrapper.py",
+ "--json",
+ "[CLEANUP]/tmp_tmp_4/metadata.json",
+ "--",
"[CLEANUP]/tmp_tmp_2/bazelisk",
"test",
"//...",
@@ -1277,7 +1338,60 @@
},
"name": "default.test //...",
"~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output (read error)@JSON file was missing or unreadable:@@@",
+ "@@@STEP_LOG_LINE@json.output (read error)@ [CLEANUP]/tmp_tmp_4/metadata.json@@@",
+ "@@@STEP_LOG_END@json.output (read error)@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "default.resultstore link (2)",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LINK@resultstore@https://result.store/@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[CLEANUP]/tmp_tmp_4/metadata.json",
+ "/path/to/tmp/"
+ ],
+ "cwd": "[START_DIR]/co",
+ "env": {
+ "BUILDBUCKET_ID": "0",
+ "BUILDBUCKET_NAME": "project:bucket:builder",
+ "BUILD_NUMBER": "0",
+ "CCACHE_DIR": "[CACHE]/ccache",
+ "CLICOLOR": "0",
+ "CLICOLOR_FORCE": "0",
+ "CTCACHE_DIR": "[CACHE]/clang_tidy",
+ "GCC_COLORS": "",
+ "GOCACHE": "[CACHE]/go",
+ "NO_COLOR": "1",
+ "PIP_CACHE_DIR": "[CACHE]/pip",
+ "PW_ENVIRONMENT_NO_ERROR_ON_UNRECOGNIZED": "1",
+ "PW_ENVSETUP_DISABLE_SPINNER": "1",
+ "PW_PRESUBMIT_DISABLE_SUBPROCESS_CAPTURE": "1",
+ "PW_TEST_VAR": "test_value",
+ "PW_USE_COLOR": "",
+ "TEST_TMPDIR": "[CACHE]/bazel",
+ "TRIGGERING_CHANGES_JSON": "[CLEANUP]/tmp_tmp_1"
+ },
+ "infra_step": true,
+ "name": "default.resultstore link (2).read",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@metadata.json@{@@@",
+ "@@@STEP_LOG_LINE@metadata.json@ \"resultstore\": \"https://result.store/\"@@@",
+ "@@@STEP_LOG_LINE@metadata.json@}@@@",
+ "@@@STEP_LOG_END@metadata.json@@@"
]
},
{
