environment: Sort logs
Change-Id: I25584051877beb2a176c457de9975713fe4c5fe8
Reviewed-on: https://pigweed-review.googlesource.com/c/infra/recipes/+/145472
Pigweed-Auto-Submit: Rob Mohr <mohrr@google.com>
Presubmit-Verified: CQ Bot Account <pigweed-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Oliver Newman <olivernewman@google.com>
Commit-Queue: Auto-Submit <auto-submit@pigweed.google.com.iam.gserviceaccount.com>
diff --git a/recipe_modules/environment/api.py b/recipe_modules/environment/api.py
index 02b5508..f47185e 100644
--- a/recipe_modules/environment/api.py
+++ b/recipe_modules/environment/api.py
@@ -106,6 +106,8 @@
def save_logs(self, env_root):
"""Read logfiles from the environment directory."""
with self.m.step.nest('logs') as pres, self.m.step.defer_results():
+ paths = []
+
for pattern in [
'*.bat',
'*.json',
@@ -119,13 +121,17 @@
'*/*.txt',
'**/pip_install_log.txt',
]:
- for path in self.m.file.glob_paths(
- f'glob environment/{pattern}', env_root, pattern
- ).get_result():
- log = self.m.file.read_text(
- f'read {self.m.path.basename(path)}', path
+ paths.extend(
+ self.m.file.glob_paths(
+ f'glob environment/{pattern}', env_root, pattern
).get_result()
- pres.logs[self.m.path.relpath(path, env_root)] = log
+ )
+
+ for path in sorted(paths):
+ log = self.m.file.read_text(
+ f'read {self.m.path.basename(path)}', path
+ ).get_result()
+ pres.logs[self.m.path.relpath(path, env_root)] = log
def _init_pigweed(
self,
diff --git a/recipe_modules/environment/tests/full.expected/doctor-fail.json b/recipe_modules/environment/tests/full.expected/doctor-fail.json
index c792d8a..3592f3a 100644
--- a/recipe_modules/environment/tests/full.expected/doctor-fail.json
+++ b/recipe_modules/environment/tests/full.expected/doctor-fail.json
@@ -228,60 +228,6 @@
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
- "copy",
- "[START_DIR]/environment/env.log",
- "/path/to/tmp/"
- ],
- "infra_step": true,
- "name": "environment.run pw_env_setup.logs.read env.log",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@3@@@",
- "@@@STEP_LOG_END@env.log@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "copy",
- "[START_DIR]/environment/foo.log",
- "/path/to/tmp/"
- ],
- "infra_step": true,
- "name": "environment.run pw_env_setup.logs.read foo.log",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@3@@@",
- "@@@STEP_LOG_END@foo.log@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "copy",
- "[START_DIR]/environment/log.log",
- "/path/to/tmp/"
- ],
- "infra_step": true,
- "name": "environment.run pw_env_setup.logs.read log.log",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@3@@@",
- "@@@STEP_LOG_END@log.log@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
"glob",
"[START_DIR]/environment",
"*.sh"
@@ -427,6 +373,60 @@
"--json-output",
"/path/to/tmp/json",
"copy",
+ "[START_DIR]/environment/env.log",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "environment.run pw_env_setup.logs.read env.log",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_LOG_END@env.log@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[START_DIR]/environment/foo.log",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "environment.run pw_env_setup.logs.read foo.log",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_LOG_END@foo.log@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[START_DIR]/environment/log.log",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "environment.run pw_env_setup.logs.read log.log",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_LOG_END@log.log@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
"[START_DIR]/environment/vars.json",
"/path/to/tmp/"
],
diff --git a/recipe_modules/environment/tests/full.expected/normal.json b/recipe_modules/environment/tests/full.expected/normal.json
index 1cd16c3..e7dcfb7 100644
--- a/recipe_modules/environment/tests/full.expected/normal.json
+++ b/recipe_modules/environment/tests/full.expected/normal.json
@@ -454,60 +454,6 @@
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
- "copy",
- "[START_DIR]/environment/env.log",
- "/path/to/tmp/"
- ],
- "infra_step": true,
- "name": "environment.run pw_env_setup.logs.read env.log",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@3@@@",
- "@@@STEP_LOG_END@env.log@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "copy",
- "[START_DIR]/environment/foo.log",
- "/path/to/tmp/"
- ],
- "infra_step": true,
- "name": "environment.run pw_env_setup.logs.read foo.log",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@3@@@",
- "@@@STEP_LOG_END@foo.log@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "copy",
- "[START_DIR]/environment/log.log",
- "/path/to/tmp/"
- ],
- "infra_step": true,
- "name": "environment.run pw_env_setup.logs.read log.log",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@3@@@",
- "@@@STEP_LOG_END@log.log@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
"glob",
"[START_DIR]/environment",
"*.sh"
@@ -653,6 +599,60 @@
"--json-output",
"/path/to/tmp/json",
"copy",
+ "[START_DIR]/environment/env.log",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "environment.run pw_env_setup.logs.read env.log",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_LOG_END@env.log@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[START_DIR]/environment/foo.log",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "environment.run pw_env_setup.logs.read foo.log",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_LOG_END@foo.log@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[START_DIR]/environment/log.log",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "environment.run pw_env_setup.logs.read log.log",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_LOG_END@log.log@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
"[START_DIR]/environment/vars.json",
"/path/to/tmp/"
],
diff --git a/recipe_modules/environment/tests/full.expected/override-cas.json b/recipe_modules/environment/tests/full.expected/override-cas.json
index 60cfe2c..14cc9ad 100644
--- a/recipe_modules/environment/tests/full.expected/override-cas.json
+++ b/recipe_modules/environment/tests/full.expected/override-cas.json
@@ -227,60 +227,6 @@
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
- "copy",
- "[START_DIR]/environment/env.log",
- "/path/to/tmp/"
- ],
- "infra_step": true,
- "name": "environment.run pw_env_setup.logs.read env.log",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@3@@@",
- "@@@STEP_LOG_END@env.log@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "copy",
- "[START_DIR]/environment/foo.log",
- "/path/to/tmp/"
- ],
- "infra_step": true,
- "name": "environment.run pw_env_setup.logs.read foo.log",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@3@@@",
- "@@@STEP_LOG_END@foo.log@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "copy",
- "[START_DIR]/environment/log.log",
- "/path/to/tmp/"
- ],
- "infra_step": true,
- "name": "environment.run pw_env_setup.logs.read log.log",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@3@@@",
- "@@@STEP_LOG_END@log.log@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
"glob",
"[START_DIR]/environment",
"*.sh"
@@ -426,6 +372,60 @@
"--json-output",
"/path/to/tmp/json",
"copy",
+ "[START_DIR]/environment/env.log",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "environment.run pw_env_setup.logs.read env.log",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_LOG_END@env.log@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[START_DIR]/environment/foo.log",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "environment.run pw_env_setup.logs.read foo.log",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_LOG_END@foo.log@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[START_DIR]/environment/log.log",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "environment.run pw_env_setup.logs.read log.log",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_LOG_END@log.log@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
"[START_DIR]/environment/vars.json",
"/path/to/tmp/"
],
diff --git a/recipe_modules/environment/tests/full.expected/override-cipd.json b/recipe_modules/environment/tests/full.expected/override-cipd.json
index fa7e082..aa0ee9e 100644
--- a/recipe_modules/environment/tests/full.expected/override-cipd.json
+++ b/recipe_modules/environment/tests/full.expected/override-cipd.json
@@ -227,60 +227,6 @@
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
- "copy",
- "[START_DIR]/environment/env.log",
- "/path/to/tmp/"
- ],
- "infra_step": true,
- "name": "environment.run pw_env_setup.logs.read env.log",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@3@@@",
- "@@@STEP_LOG_END@env.log@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "copy",
- "[START_DIR]/environment/foo.log",
- "/path/to/tmp/"
- ],
- "infra_step": true,
- "name": "environment.run pw_env_setup.logs.read foo.log",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@3@@@",
- "@@@STEP_LOG_END@foo.log@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "copy",
- "[START_DIR]/environment/log.log",
- "/path/to/tmp/"
- ],
- "infra_step": true,
- "name": "environment.run pw_env_setup.logs.read log.log",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@3@@@",
- "@@@STEP_LOG_END@log.log@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
"glob",
"[START_DIR]/environment",
"*.sh"
@@ -426,6 +372,60 @@
"--json-output",
"/path/to/tmp/json",
"copy",
+ "[START_DIR]/environment/env.log",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "environment.run pw_env_setup.logs.read env.log",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_LOG_END@env.log@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[START_DIR]/environment/foo.log",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "environment.run pw_env_setup.logs.read foo.log",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_LOG_END@foo.log@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[START_DIR]/environment/log.log",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "environment.run pw_env_setup.logs.read log.log",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_LOG_END@log.log@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
"[START_DIR]/environment/vars.json",
"/path/to/tmp/"
],
diff --git a/recipes/envtest.expected/fail.json b/recipes/envtest.expected/fail.json
index c4cd4e5..e685e35 100644
--- a/recipes/envtest.expected/fail.json
+++ b/recipes/envtest.expected/fail.json
@@ -1656,36 +1656,6 @@
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
- "copy",
- "[START_DIR]/environment/foo.json",
- "/path/to/tmp/"
- ],
- "infra_step": true,
- "luci_context": {
- "realm": {
- "name": "project:ci"
- },
- "resultdb": {
- "current_invocation": {
- "name": "invocations/build:8945511751514863184",
- "update_token": "token"
- },
- "hostname": "rdbhost"
- }
- },
- "name": "logs.read foo.json",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@",
- "@@@STEP_LOG_END@foo.json@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
"glob",
"[START_DIR]/environment",
"*.log"
@@ -1950,6 +1920,36 @@
]
},
{
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[START_DIR]/environment/foo.json",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "project:ci"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "logs.read foo.json",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_END@foo.json@@@"
+ ]
+ },
+ {
"failure": {
"failure": {},
"humanReason": "1 out of 1 aggregated steps failed: Step('run.sh') (retcode: 1)"
