build: Support installing packages
Reordering the options protobuf because the numbers don't matter in this
context and logically packages come before other options.
Adding a test to handle default (empty) values for packages, gn args,
and targets.
Bug: b/285350808
Change-Id: Ia82b8a7b1bdda8c336b2c755e452082292b3b52e
Reviewed-on: https://pigweed-review.googlesource.com/c/infra/recipes/+/149990
Reviewed-by: Anthony DiGirolamo <tonymd@google.com>
Pigweed-Auto-Submit: Rob Mohr <mohrr@google.com>
Commit-Queue: Auto-Submit <auto-submit@pigweed.google.com.iam.gserviceaccount.com>
diff --git a/recipe_modules/build/api.py b/recipe_modules/build/api.py
index 0dd116b..9af00b2 100644
--- a/recipe_modules/build/api.py
+++ b/recipe_modules/build/api.py
@@ -36,9 +36,19 @@
return Context(self.m, checkout_root, root, options)
def __call__(self, ctx):
+ self.install_packages(ctx)
self.gn_gen(ctx)
self.ninja(ctx)
+ def install_packages(self, ctx):
+ if not ctx.options.packages:
+ return
+
+ with self.m.step.nest('install packages'):
+ cmd = ['python', '-m', 'pw_cli', 'package', 'install']
+ for package in ctx.options.packages:
+ self.m.step(package, cmd + [package])
+
def gn_gen(self, ctx):
cmd = ['gn', 'gen']
diff --git a/recipe_modules/build/options.proto b/recipe_modules/build/options.proto
index b122ad8..c180cbe 100644
--- a/recipe_modules/build/options.proto
+++ b/recipe_modules/build/options.proto
@@ -16,9 +16,12 @@
package recipe_modules.pigweed.build;
message Options {
+ // List of names of pw_package packages to install.
+ repeated string packages = 1;
+
// List of gn args ("var=value", ...). Default: empty.
- repeated string gn_args = 1;
+ repeated string gn_args = 2;
// List of targets to build. Default: empty (build default target).
- repeated string ninja_targets = 2;
+ repeated string ninja_targets = 3;
}
diff --git a/recipe_modules/build/test_api.py b/recipe_modules/build/test_api.py
index d6f67b9..0e59e2f 100644
--- a/recipe_modules/build/test_api.py
+++ b/recipe_modules/build/test_api.py
@@ -20,11 +20,13 @@
class BuildTestApi(recipe_test_api.RecipeTestApi):
"""Test API for build."""
- def options(self, *, gn_args=(), ninja_targets=()):
+ def options(self, *, packages=(), gn_args=(), ninja_targets=()):
+ assert isinstance(packages, (list, tuple))
assert isinstance(gn_args, (list, tuple))
assert isinstance(ninja_targets, (list, tuple))
opts = options.Options()
+ opts.packages.extend(packages)
opts.gn_args.extend(gn_args)
opts.ninja_targets.extend(ninja_targets)
return opts
diff --git a/recipe_modules/build/tests/full.expected/default.json b/recipe_modules/build/tests/full.expected/default.json
new file mode 100644
index 0000000..4ac4eac
--- /dev/null
+++ b/recipe_modules/build/tests/full.expected/default.json
@@ -0,0 +1,486 @@
+[
+ {
+ "cmd": [
+ "gn",
+ "gen",
+ "--export-compile-commands",
+ "[START_DIR]/checkout/out"
+ ],
+ "cwd": "[START_DIR]/checkout",
+ "name": "gn gen"
+ },
+ {
+ "cmd": [],
+ "name": "timeout 11h 4m 38s"
+ },
+ {
+ "cmd": [
+ "ninja",
+ "-C",
+ "[START_DIR]/checkout/out"
+ ],
+ "luci_context": {
+ "deadline": {
+ "grace_period": 30.0,
+ "soft_deadline": -120.0
+ }
+ },
+ "name": "ninja"
+ },
+ {
+ "cmd": [],
+ "name": "save logs",
+ "~followup_annotations": [
+ "@@@STEP_LINK@description@https://url@@@",
+ "@@@STEP_FAILURE@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "save logs.logs",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "save logs.logs.glob",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "glob",
+ "[START_DIR]/checkout/out",
+ "*.gn",
+ "--hidden"
+ ],
+ "infra_step": true,
+ "name": "save logs.logs.glob.*.gn",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_LOG_END@glob@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "glob",
+ "[START_DIR]/checkout/out",
+ "*.log",
+ "--hidden"
+ ],
+ "infra_step": true,
+ "name": "save logs.logs.glob.*.log",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_LOG_LINE@glob@[START_DIR]/checkout/out/.ninja_log@@@",
+ "@@@STEP_LOG_LINE@glob@[START_DIR]/checkout/out/failure-summary.log@@@",
+ "@@@STEP_LOG_LINE@glob@[START_DIR]/checkout/out/links.json@@@",
+ "@@@STEP_LOG_END@glob@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "glob",
+ "[START_DIR]/checkout/out",
+ "*.json",
+ "--hidden"
+ ],
+ "infra_step": true,
+ "name": "save logs.logs.glob.*.json",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_LOG_END@glob@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "glob",
+ "[START_DIR]/checkout/out",
+ "*.compdb",
+ "--hidden"
+ ],
+ "infra_step": true,
+ "name": "save logs.logs.glob.*.compdb",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_LOG_END@glob@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "glob",
+ "[START_DIR]/checkout/out",
+ "*.graph",
+ "--hidden"
+ ],
+ "infra_step": true,
+ "name": "save logs.logs.glob.*.graph",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_LOG_END@glob@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "glob",
+ "[START_DIR]/checkout/out",
+ "*_log",
+ "--hidden"
+ ],
+ "infra_step": true,
+ "name": "save logs.logs.glob.*_log",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_LOG_END@glob@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[START_DIR]/checkout/out/.ninja_log",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "save logs.logs..ninja_log",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@.ninja_log@2000 5000 0 medium 0@@@",
+ "@@@STEP_LOG_LINE@.ninja_log@3000 8000 0 long 0@@@",
+ "@@@STEP_LOG_LINE@.ninja_log@malformed line@@@",
+ "@@@STEP_LOG_LINE@.ninja_log@4000 5000 0 short 0@@@",
+ "@@@STEP_LOG_LINE@.ninja_log@5000 x 0 malformed-end-time 0@@@",
+ "@@@STEP_LOG_END@.ninja_log@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[START_DIR]/checkout/out/failure-summary.log",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "save logs.logs.failure-summary.log",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@failure-summary.log@[5/10] foo.c@@@",
+ "@@@STEP_LOG_LINE@failure-summary.log@error: ???@@@",
+ "@@@STEP_LOG_END@failure-summary.log@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[START_DIR]/checkout/out/links.json",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "save logs.logs.links.json",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@links.json@[@@@",
+ "@@@STEP_LOG_LINE@links.json@ {@@@",
+ "@@@STEP_LOG_LINE@links.json@ \"description\": \"description\",@@@",
+ "@@@STEP_LOG_LINE@links.json@ \"url\": \"https://url\"@@@",
+ "@@@STEP_LOG_LINE@links.json@ }@@@",
+ "@@@STEP_LOG_LINE@links.json@]@@@",
+ "@@@STEP_LOG_END@links.json@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "save logs.failure summary",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_SUMMARY_TEXT@```\n[5/10] foo.c\nerror: ???\n```@@@",
+ "@@@STEP_LOG_LINE@full contents@[5/10] foo.c@@@",
+ "@@@STEP_LOG_LINE@full contents@error: ???@@@",
+ "@@@STEP_LOG_END@full contents@@@",
+ "@@@STEP_FAILURE@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "save logs.longest build steps",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "save logs.longest build steps.long",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_SUMMARY_TEXT@5.0s@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "save logs.longest build steps.medium",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_SUMMARY_TEXT@3.0s@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "save logs.longest build steps.short",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_SUMMARY_TEXT@1.0s@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[START_DIR]/checkout/out/export/build_logs"
+ ],
+ "infra_step": true,
+ "name": "save logs.mkdir build_logs",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "save logs.copy",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[START_DIR]/checkout/out/.ninja_log",
+ "[START_DIR]/checkout/out/export/build_logs/.ninja_log"
+ ],
+ "infra_step": true,
+ "name": "save logs.copy..ninja_log",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[START_DIR]/checkout/out/failure-summary.log",
+ "[START_DIR]/checkout/out/export/build_logs/failure-summary.log"
+ ],
+ "infra_step": true,
+ "name": "save logs.copy.failure-summary.log",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[START_DIR]/checkout/out/links.json",
+ "[START_DIR]/checkout/out/export/build_logs/links.json"
+ ],
+ "infra_step": true,
+ "name": "save logs.copy.links.json",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "gn",
+ "args",
+ "[START_DIR]/checkout/out",
+ "--list",
+ "--json"
+ ],
+ "cwd": "[START_DIR]/checkout",
+ "name": "all gn args",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@json.output@[]@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "archive to cas",
+ "~followup_annotations": [
+ "@@@SET_BUILD_PROPERTY@cas_build_digest@\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0\"@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "RECIPE_MODULE[recipe_engine::cas]/resources/infra.sha1",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "archive to cas.read infra revision",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@infra.sha1@git_revision:mock_infra_git_revision@@@",
+ "@@@STEP_LOG_END@infra.sha1@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "archive to cas.install infra/tools/luci/cas",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision"
+ ],
+ "infra_step": true,
+ "name": "archive to cas.install infra/tools/luci/cas.ensure package directory",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision",
+ "-ensure-file",
+ "infra/tools/luci/cas/${platform} git_revision:mock_infra_git_revision",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "archive to cas.install infra/tools/luci/cas.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-git_revision:moc\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/cas/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision/cas",
+ "archive",
+ "-cas-instance",
+ "projects/example-cas-server/instances/default_instance",
+ "-dump-digest",
+ "/path/to/tmp/",
+ "-paths-json",
+ "[[\"[START_DIR]/checkout/out\", \".\"]]"
+ ],
+ "infra_step": true,
+ "name": "archive to cas.archive",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LINK@CAS UI@https://cas-viewer.appspot.com/projects/example-cas-server/instances/default_instance/blobs/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0/tree@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision/cas",
+ "download",
+ "-cas-instance",
+ "projects/example-cas-server/instances/default_instance",
+ "-digest",
+ "digest",
+ "-dir",
+ "[START_DIR]/checkout/out"
+ ],
+ "infra_step": true,
+ "name": "download from cas"
+ },
+ {
+ "name": "$result"
+ }
+]
\ No newline at end of file
diff --git a/recipe_modules/build/tests/full.expected/full.json b/recipe_modules/build/tests/full.expected/full.json
index 23d731e..5116ffd 100644
--- a/recipe_modules/build/tests/full.expected/full.json
+++ b/recipe_modules/build/tests/full.expected/full.json
@@ -1,5 +1,23 @@
[
{
+ "cmd": [],
+ "name": "install packages"
+ },
+ {
+ "cmd": [
+ "python",
+ "-m",
+ "pw_cli",
+ "package",
+ "install",
+ "pkg"
+ ],
+ "name": "install packages.pkg",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
"cmd": [
"gn",
"gen",
diff --git a/recipe_modules/build/tests/full.py b/recipe_modules/build/tests/full.py
index f885415..b5cfbe7 100644
--- a/recipe_modules/build/tests/full.py
+++ b/recipe_modules/build/tests/full.py
@@ -44,7 +44,13 @@
api.test('full')
+ api.properties(
build_options=api.build.options(
- gn_args=['foo=true'], ninja_targets=['target'],
+ packages=['pkg'],
+ gn_args=['foo=true'],
+ ninja_targets=['target'],
)
)
)
+
+ yield (
+ api.test('default') + api.properties(build_options=api.build.options())
+ )
