Run single presubmit steps.
Add ability to run single presubmit steps instead of always
running everything.
Many small fixes:
* add CIPD lib directories to LD_LIBRARY_PATH
* set PW_ROOT (pretty sure not needed but matches user workflows)
* set VIRTUAL_ENV
* add "--editable=" to pip install (it's inexplicably required)
* search for Pigweed Python packages without Pigweed environment
(the virtualenv seems to screw with vpython)
* install from Pigweed requirements.txt
Bug: 3, 29
Change-Id: I549d7820015f79d315de43e7cfca4c692ace1c96
diff --git a/recipe_modules/environment/api.py b/recipe_modules/environment/api.py
index 7d4b6af..1c49a3e 100644
--- a/recipe_modules/environment/api.py
+++ b/recipe_modules/environment/api.py
@@ -35,8 +35,9 @@
def __init__(self, *args, **kwargs):
super(EnvironmentApi, self).__init__(*args, **kwargs)
self._cipd_dir = None
- self._venv_dir = None
self._path_prefixes = []
+ self._ldpath_prefixes = []
+ self._env = {}
self._initialized = False
def _process_ensure_path(self, ensure_path, ensure_file):
@@ -72,8 +73,22 @@
self._path_prefixes.append(self._cipd_dir)
self._path_prefixes.append(self._cipd_dir.join('bin'))
+ self._ldpath_prefixes.append(self._cipd_dir)
+ self._ldpath_prefixes.append(self._cipd_dir.join('lib'))
+ self._env['CIPD_INSTALL_DIR'] = self._cipd_dir
def _all_python_packages(self, checkout_root):
+ """Return all folders with setup.py entries.
+
+ Note: this function should not be called from within a virtualenv. It screws
+ with how vpython works.
+
+ Args:
+ checkout_root(Path): root of source tree
+
+ Returns:
+ A list of package paths.
+ """
files = self.m.file.listdir('ls **/setup.py', checkout_root, recursive=True)
matches = [
self.m.path.dirname(x)
@@ -87,23 +102,37 @@
def _init_python(self, checkout_root):
"""Initialize the Python environment. (Specifically, install 'pw'.)"""
- with self.m.step.nest('setup python'), self():
- self._venv_dir = self.m.path['start_dir'].join('venv')
- venv_bin = self._venv_dir.join('bin')
- python = venv_bin.join('python3')
- self.m.step('create venv', ['python3', '-m', 'venv', self._venv_dir])
- self.m.step('upgrade pip',
- [python, '-m', 'pip', 'install', '--upgrade', 'pip'])
+ with self.m.step.nest('setup python'):
+ packages = self._all_python_packages(checkout_root)
- pip_install_cmd = [python, '-m', 'pip', 'install']
- for package in self._all_python_packages(checkout_root):
- pip_install_cmd.append(package)
+ with self.m.step.nest('setup virtualenv'), self():
+ venv_dir = self.m.path['start_dir'].join('venv')
+ venv_bin = venv_dir.join('bin')
+ python = venv_bin.join('python3')
+ self.m.step('create venv', ['python3', '-m', 'venv', venv_dir])
+ self.m.step('upgrade pip',
+ [python, '-m', 'pip', 'install', '--upgrade', 'pip'])
- self.m.step('pip install', pip_install_cmd)
+ # Need to insert at beginning because venv python should trump cipd
+ # python.
+ self._path_prefixes.insert(0, venv_bin)
+ self._env['VIRTUAL_ENV'] = venv_dir
- # Need to insert at beginning because venv python should trump cipd
- # python.
- self._path_prefixes.insert(0, venv_bin)
+ # Need to exit and reenter 'with self()' to include new context from
+ # creating the virtualenv.
+
+ with self.m.step.nest('install packages'), self():
+ pip_install_prefix = (python, '-m', 'pip', 'install')
+ pw_cmd = list(pip_install_prefix)
+ for package in packages:
+ pw_cmd.append('--editable={}'.format(package))
+ self.m.step('pigweed tools', pw_cmd)
+
+ requirements = checkout_root.join(
+ 'env_setup/virtualenv/requirements.txt')
+ req_cmd = list(pip_install_prefix)
+ req_cmd.extend(('-r', requirements))
+ self.m.step('build requirements', req_cmd)
def init(self, checkout_root):
if not self._initialized:
@@ -112,9 +141,17 @@
# use the context of previous steps, and invoking self() is the
# easiest way to do so.
self._initialized = True
+ self._env['PW_ROOT'] = checkout_root
self._init_cipd(checkout_root)
self._init_python(checkout_root)
def __call__(self):
assert self._initialized
- return self.m.context(env_prefixes={'PATH': self._path_prefixes})
+
+ env_prefixes = {}
+ if self._path_prefixes:
+ env_prefixes['PATH'] = self._path_prefixes
+ if self._ldpath_prefixes:
+ env_prefixes['LD_LIBRARY_PATH'] = self._ldpath_prefixes
+
+ return self.m.context(env_prefixes=env_prefixes, env=self._env)
diff --git a/recipe_modules/environment/tests/full.expected/simple.json b/recipe_modules/environment/tests/full.expected/simple.json
index 0625ab4..7fcddbb 100644
--- a/recipe_modules/environment/tests/full.expected/simple.json
+++ b/recipe_modules/environment/tests/full.expected/simple.json
@@ -68,44 +68,6 @@
},
{
"cmd": [
- "python3",
- "-m",
- "venv",
- "[START_DIR]/venv"
- ],
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/cipd",
- "[START_DIR]/cipd/bin"
- ]
- },
- "name": "environment.setup python.create venv",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@2@@@"
- ]
- },
- {
- "cmd": [
- "[START_DIR]/venv/bin/python3",
- "-m",
- "pip",
- "install",
- "--upgrade",
- "pip"
- ],
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/cipd",
- "[START_DIR]/cipd/bin"
- ]
- },
- "name": "environment.setup python.upgrade pip",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@2@@@"
- ]
- },
- {
- "cmd": [
"vpython",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
@@ -115,12 +77,6 @@
"[START_DIR]",
"--recursive"
],
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/cipd",
- "[START_DIR]/cipd/bin"
- ]
- },
"infra_step": true,
"name": "environment.setup python.ls **/setup.py",
"~followup_annotations": [
@@ -141,26 +97,134 @@
]
},
{
+ "cmd": [],
+ "name": "environment.setup python.setup virtualenv",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
"cmd": [
- "[START_DIR]/venv/bin/python3",
+ "python3",
"-m",
- "pip",
- "install",
- "[START_DIR]/pw_cli/py",
- "[START_DIR]/pw_presubmit/py"
+ "venv",
+ "[START_DIR]/venv"
],
+ "env": {
+ "CIPD_INSTALL_DIR": "[START_DIR]/cipd",
+ "PW_ROOT": "[START_DIR]"
+ },
"env_prefixes": {
+ "LD_LIBRARY_PATH": [
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/lib"
+ ],
"PATH": [
"[START_DIR]/cipd",
"[START_DIR]/cipd/bin"
]
},
- "name": "environment.setup python.pip install",
+ "name": "environment.setup python.setup virtualenv.create venv",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/venv/bin/python3",
+ "-m",
+ "pip",
+ "install",
+ "--upgrade",
+ "pip"
+ ],
+ "env": {
+ "CIPD_INSTALL_DIR": "[START_DIR]/cipd",
+ "PW_ROOT": "[START_DIR]"
+ },
+ "env_prefixes": {
+ "LD_LIBRARY_PATH": [
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/lib"
+ ],
+ "PATH": [
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/bin"
+ ]
+ },
+ "name": "environment.setup python.setup virtualenv.upgrade pip",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "environment.setup python.install packages",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@2@@@"
]
},
{
+ "cmd": [
+ "[START_DIR]/venv/bin/python3",
+ "-m",
+ "pip",
+ "install",
+ "--editable=[START_DIR]/pw_cli/py",
+ "--editable=[START_DIR]/pw_presubmit/py"
+ ],
+ "env": {
+ "CIPD_INSTALL_DIR": "[START_DIR]/cipd",
+ "PW_ROOT": "[START_DIR]",
+ "VIRTUAL_ENV": "[START_DIR]/venv"
+ },
+ "env_prefixes": {
+ "LD_LIBRARY_PATH": [
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/lib"
+ ],
+ "PATH": [
+ "[START_DIR]/venv/bin",
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/bin"
+ ]
+ },
+ "name": "environment.setup python.install packages.pigweed tools",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/venv/bin/python3",
+ "-m",
+ "pip",
+ "install",
+ "-r",
+ "[START_DIR]/env_setup/virtualenv/requirements.txt"
+ ],
+ "env": {
+ "CIPD_INSTALL_DIR": "[START_DIR]/cipd",
+ "PW_ROOT": "[START_DIR]",
+ "VIRTUAL_ENV": "[START_DIR]/venv"
+ },
+ "env_prefixes": {
+ "LD_LIBRARY_PATH": [
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/lib"
+ ],
+ "PATH": [
+ "[START_DIR]/venv/bin",
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/bin"
+ ]
+ },
+ "name": "environment.setup python.install packages.build requirements",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@"
+ ]
+ },
+ {
"name": "$result"
}
]
\ No newline at end of file
diff --git a/recipes/pigweed.expected/pigweed.json b/recipes/pigweed.expected/pigweed.json
index 4848c39..9074cf1 100644
--- a/recipes/pigweed.expected/pigweed.json
+++ b/recipes/pigweed.expected/pigweed.json
@@ -452,44 +452,6 @@
},
{
"cmd": [
- "python3",
- "-m",
- "venv",
- "[START_DIR]/venv"
- ],
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/cipd",
- "[START_DIR]/cipd/bin"
- ]
- },
- "name": "environment.setup python.create venv",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@2@@@"
- ]
- },
- {
- "cmd": [
- "[START_DIR]/venv/bin/python3",
- "-m",
- "pip",
- "install",
- "--upgrade",
- "pip"
- ],
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/cipd",
- "[START_DIR]/cipd/bin"
- ]
- },
- "name": "environment.setup python.upgrade pip",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@2@@@"
- ]
- },
- {
- "cmd": [
"vpython",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
@@ -499,12 +461,6 @@
"[START_DIR]/checkout",
"--recursive"
],
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/cipd",
- "[START_DIR]/cipd/bin"
- ]
- },
"infra_step": true,
"name": "environment.setup python.ls **/setup.py",
"~followup_annotations": [
@@ -525,27 +481,135 @@
]
},
{
+ "cmd": [],
+ "name": "environment.setup python.setup virtualenv",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
"cmd": [
- "[START_DIR]/venv/bin/python3",
+ "python3",
"-m",
- "pip",
- "install",
- "[START_DIR]/checkout/pw_cli/py",
- "[START_DIR]/checkout/pw_presubmit/py"
+ "venv",
+ "[START_DIR]/venv"
],
+ "env": {
+ "CIPD_INSTALL_DIR": "[START_DIR]/cipd",
+ "PW_ROOT": "[START_DIR]/checkout"
+ },
"env_prefixes": {
+ "LD_LIBRARY_PATH": [
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/lib"
+ ],
"PATH": [
"[START_DIR]/cipd",
"[START_DIR]/cipd/bin"
]
},
- "name": "environment.setup python.pip install",
+ "name": "environment.setup python.setup virtualenv.create venv",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/venv/bin/python3",
+ "-m",
+ "pip",
+ "install",
+ "--upgrade",
+ "pip"
+ ],
+ "env": {
+ "CIPD_INSTALL_DIR": "[START_DIR]/cipd",
+ "PW_ROOT": "[START_DIR]/checkout"
+ },
+ "env_prefixes": {
+ "LD_LIBRARY_PATH": [
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/lib"
+ ],
+ "PATH": [
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/bin"
+ ]
+ },
+ "name": "environment.setup python.setup virtualenv.upgrade pip",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "environment.setup python.install packages",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@2@@@"
]
},
{
"cmd": [
+ "[START_DIR]/venv/bin/python3",
+ "-m",
+ "pip",
+ "install",
+ "--editable=[START_DIR]/checkout/pw_cli/py",
+ "--editable=[START_DIR]/checkout/pw_presubmit/py"
+ ],
+ "env": {
+ "CIPD_INSTALL_DIR": "[START_DIR]/cipd",
+ "PW_ROOT": "[START_DIR]/checkout",
+ "VIRTUAL_ENV": "[START_DIR]/venv"
+ },
+ "env_prefixes": {
+ "LD_LIBRARY_PATH": [
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/lib"
+ ],
+ "PATH": [
+ "[START_DIR]/venv/bin",
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/bin"
+ ]
+ },
+ "name": "environment.setup python.install packages.pigweed tools",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/venv/bin/python3",
+ "-m",
+ "pip",
+ "install",
+ "-r",
+ "[START_DIR]/checkout/env_setup/virtualenv/requirements.txt"
+ ],
+ "env": {
+ "CIPD_INSTALL_DIR": "[START_DIR]/cipd",
+ "PW_ROOT": "[START_DIR]/checkout",
+ "VIRTUAL_ENV": "[START_DIR]/venv"
+ },
+ "env_prefixes": {
+ "LD_LIBRARY_PATH": [
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/lib"
+ ],
+ "PATH": [
+ "[START_DIR]/venv/bin",
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/bin"
+ ]
+ },
+ "name": "environment.setup python.install packages.build requirements",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@"
+ ]
+ },
+ {
+ "cmd": [
"pw",
"presubmit",
"--repository",
@@ -553,7 +617,16 @@
"--output-directory",
"out"
],
+ "env": {
+ "CIPD_INSTALL_DIR": "[START_DIR]/cipd",
+ "PW_ROOT": "[START_DIR]/checkout",
+ "VIRTUAL_ENV": "[START_DIR]/venv"
+ },
"env_prefixes": {
+ "LD_LIBRARY_PATH": [
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/lib"
+ ],
"PATH": [
"[START_DIR]/venv/bin",
"[START_DIR]/cipd",
diff --git a/recipes/pigweed.expected/step.json b/recipes/pigweed.expected/step.json
new file mode 100644
index 0000000..5a6d435
--- /dev/null
+++ b/recipes/pigweed.expected/step.json
@@ -0,0 +1,672 @@
+[
+ {
+ "cmd": [],
+ "name": "checkout"
+ },
+ {
+ "cmd": [],
+ "name": "checkout.ensure gitiles",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[START_DIR]/cipd/gitiles",
+ "-ensure-file",
+ "infra/tools/luci/gitiles/${platform} latest",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "checkout.ensure gitiles.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/gitiles/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd/gitiles/gitiles",
+ "refs",
+ "-json-output",
+ "/path/to/tmp/json",
+ "https://pigweed.googlesource.com/pigweed/pigweed",
+ "refs/heads"
+ ],
+ "infra_step": true,
+ "name": "checkout.refs",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"refs/heads/master\": \"1234567890123456789012345678901234567890\"@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[START_DIR]/checkout"
+ ],
+ "infra_step": true,
+ "name": "checkout.makedirs",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "init"
+ ],
+ "cwd": "[START_DIR]/checkout",
+ "infra_step": true,
+ "name": "checkout.git init",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "remote",
+ "add",
+ "origin",
+ "https://pigweed.googlesource.com/pigweed/pigweed"
+ ],
+ "cwd": "[START_DIR]/checkout",
+ "infra_step": true,
+ "name": "checkout.git remote",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "checkout.cache",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[CACHE]/git/pigweed.googlesource.com-pigweed-pigweed"
+ ],
+ "cwd": "[START_DIR]/checkout",
+ "infra_step": true,
+ "name": "checkout.cache.makedirs",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "init",
+ "--bare"
+ ],
+ "cwd": "[CACHE]/git/pigweed.googlesource.com-pigweed-pigweed",
+ "infra_step": true,
+ "name": "checkout.cache.git init",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "config",
+ "remote.origin.url",
+ "https://pigweed.googlesource.com/pigweed/pigweed"
+ ],
+ "cwd": "[CACHE]/git/pigweed.googlesource.com-pigweed-pigweed",
+ "infra_step": true,
+ "name": "checkout.cache.remote set-url",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "config",
+ "--replace-all",
+ "remote.origin.fetch",
+ "+refs/heads/*:refs/heads/*",
+ "\\+refs/heads/\\*:.*"
+ ],
+ "cwd": "[CACHE]/git/pigweed.googlesource.com-pigweed-pigweed",
+ "infra_step": true,
+ "name": "checkout.cache.git config",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "fetch",
+ "--prune",
+ "--tags",
+ "origin"
+ ],
+ "cwd": "[CACHE]/git/pigweed.googlesource.com-pigweed-pigweed",
+ "infra_step": true,
+ "name": "checkout.cache.git fetch",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[START_DIR]/checkout/.git/objects/info"
+ ],
+ "cwd": "[START_DIR]/checkout",
+ "infra_step": true,
+ "name": "checkout.cache.makedirs object/info",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[CACHE]/git/pigweed.googlesource.com-pigweed-pigweed/objects\n",
+ "[START_DIR]/checkout/.git/objects/info/alternates"
+ ],
+ "cwd": "[START_DIR]/checkout",
+ "infra_step": true,
+ "name": "checkout.cache.alternates",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@alternates@[CACHE]/git/pigweed.googlesource.com-pigweed-pigweed/objects@@@",
+ "@@@STEP_LOG_END@alternates@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "fetch",
+ "--tags",
+ "origin"
+ ],
+ "cwd": "[START_DIR]/checkout",
+ "infra_step": true,
+ "name": "checkout.git fetch",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "checkout",
+ "-f",
+ "1234567890123456789012345678901234567890"
+ ],
+ "cwd": "[START_DIR]/checkout",
+ "infra_step": true,
+ "name": "checkout.git checkout",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[START_DIR]/checkout",
+ "infra_step": true,
+ "name": "checkout.git rev-parse",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "clean",
+ "-f",
+ "-d",
+ "-x"
+ ],
+ "cwd": "[START_DIR]/checkout",
+ "infra_step": true,
+ "name": "checkout.git clean",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "checkout.submodule",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "submodule",
+ "sync"
+ ],
+ "cwd": "[START_DIR]/checkout",
+ "infra_step": true,
+ "name": "checkout.submodule.git submodule sync",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "submodule",
+ "update",
+ "--init"
+ ],
+ "cwd": "[START_DIR]/checkout",
+ "infra_step": true,
+ "name": "checkout.submodule.git submodule update",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "checkout.ensure gerrit",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[START_DIR]/cipd/gerrit",
+ "-ensure-file",
+ "infra/tools/luci/gerrit/${platform} latest",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "checkout.ensure gerrit.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/gerrit/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd/gerrit/gerrit",
+ "change-detail",
+ "-host",
+ "https://pigweed-review.googlesource.com",
+ "-input",
+ "{\"change_id\": \"1234567890123456789012345678901234567890\"}",
+ "-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "checkout.get change details",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"branch\": \"master\"@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "rebase",
+ "origin/master"
+ ],
+ "cwd": "[START_DIR]/checkout",
+ "name": "checkout.git rebase",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "environment"
+ },
+ {
+ "cmd": [],
+ "name": "environment.setup cipd",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[START_DIR]/checkout/env_setup/cipd/pigweed.ensure",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "environment.setup cipd.read pigweed.ensure",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@pigweed.ensure@@@@",
+ "@@@STEP_LOG_LINE@pigweed.ensure@ # comment@@@",
+ "@@@STEP_LOG_LINE@pigweed.ensure@ cipd/path/${platform} version:42@@@",
+ "@@@STEP_LOG_LINE@pigweed.ensure@ @@@",
+ "@@@STEP_LOG_END@pigweed.ensure@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[START_DIR]/cipd",
+ "-ensure-file",
+ "cipd/path/${platform} version:42",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "name": "environment.setup cipd.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-version:42------\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"cipd/path/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "environment.setup python",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "listdir",
+ "[START_DIR]/checkout",
+ "--recursive"
+ ],
+ "infra_step": true,
+ "name": "environment.setup python.ls **/setup.py",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@listdir@[START_DIR]/checkout/pw_cli/py/setup.py@@@",
+ "@@@STEP_LOG_LINE@listdir@[START_DIR]/checkout/pw_presubmit/py/setup.py@@@",
+ "@@@STEP_LOG_END@listdir@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "environment.setup python.packages",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@matches@[START_DIR]/checkout/pw_cli/py@@@",
+ "@@@STEP_LOG_LINE@matches@[START_DIR]/checkout/pw_presubmit/py@@@",
+ "@@@STEP_LOG_END@matches@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "environment.setup python.setup virtualenv",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python3",
+ "-m",
+ "venv",
+ "[START_DIR]/venv"
+ ],
+ "env": {
+ "CIPD_INSTALL_DIR": "[START_DIR]/cipd",
+ "PW_ROOT": "[START_DIR]/checkout"
+ },
+ "env_prefixes": {
+ "LD_LIBRARY_PATH": [
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/lib"
+ ],
+ "PATH": [
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/bin"
+ ]
+ },
+ "name": "environment.setup python.setup virtualenv.create venv",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/venv/bin/python3",
+ "-m",
+ "pip",
+ "install",
+ "--upgrade",
+ "pip"
+ ],
+ "env": {
+ "CIPD_INSTALL_DIR": "[START_DIR]/cipd",
+ "PW_ROOT": "[START_DIR]/checkout"
+ },
+ "env_prefixes": {
+ "LD_LIBRARY_PATH": [
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/lib"
+ ],
+ "PATH": [
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/bin"
+ ]
+ },
+ "name": "environment.setup python.setup virtualenv.upgrade pip",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "environment.setup python.install packages",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/venv/bin/python3",
+ "-m",
+ "pip",
+ "install",
+ "--editable=[START_DIR]/checkout/pw_cli/py",
+ "--editable=[START_DIR]/checkout/pw_presubmit/py"
+ ],
+ "env": {
+ "CIPD_INSTALL_DIR": "[START_DIR]/cipd",
+ "PW_ROOT": "[START_DIR]/checkout",
+ "VIRTUAL_ENV": "[START_DIR]/venv"
+ },
+ "env_prefixes": {
+ "LD_LIBRARY_PATH": [
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/lib"
+ ],
+ "PATH": [
+ "[START_DIR]/venv/bin",
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/bin"
+ ]
+ },
+ "name": "environment.setup python.install packages.pigweed tools",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/venv/bin/python3",
+ "-m",
+ "pip",
+ "install",
+ "-r",
+ "[START_DIR]/checkout/env_setup/virtualenv/requirements.txt"
+ ],
+ "env": {
+ "CIPD_INSTALL_DIR": "[START_DIR]/cipd",
+ "PW_ROOT": "[START_DIR]/checkout",
+ "VIRTUAL_ENV": "[START_DIR]/venv"
+ },
+ "env_prefixes": {
+ "LD_LIBRARY_PATH": [
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/lib"
+ ],
+ "PATH": [
+ "[START_DIR]/venv/bin",
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/bin"
+ ]
+ },
+ "name": "environment.setup python.install packages.build requirements",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "pw",
+ "presubmit",
+ "--repository",
+ "[START_DIR]/checkout",
+ "--output-directory",
+ "out",
+ "--step",
+ "step1"
+ ],
+ "env": {
+ "CIPD_INSTALL_DIR": "[START_DIR]/cipd",
+ "PW_ROOT": "[START_DIR]/checkout",
+ "VIRTUAL_ENV": "[START_DIR]/venv"
+ },
+ "env_prefixes": {
+ "LD_LIBRARY_PATH": [
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/lib"
+ ],
+ "PATH": [
+ "[START_DIR]/venv/bin",
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/bin"
+ ]
+ },
+ "name": "step1"
+ },
+ {
+ "cmd": [
+ "pw",
+ "presubmit",
+ "--repository",
+ "[START_DIR]/checkout",
+ "--output-directory",
+ "out",
+ "--step",
+ "step2"
+ ],
+ "env": {
+ "CIPD_INSTALL_DIR": "[START_DIR]/cipd",
+ "PW_ROOT": "[START_DIR]/checkout",
+ "VIRTUAL_ENV": "[START_DIR]/venv"
+ },
+ "env_prefixes": {
+ "LD_LIBRARY_PATH": [
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/lib"
+ ],
+ "PATH": [
+ "[START_DIR]/venv/bin",
+ "[START_DIR]/cipd",
+ "[START_DIR]/cipd/bin"
+ ]
+ },
+ "name": "step2"
+ },
+ {
+ "name": "$result"
+ }
+]
\ No newline at end of file
diff --git a/recipes/pigweed.py b/recipes/pigweed.py
index 6fb07f4..4e40a71 100644
--- a/recipes/pigweed.py
+++ b/recipes/pigweed.py
@@ -14,6 +14,7 @@
"""Recipe for testing Pigweed using presubmit_checks.py script."""
+from recipe_engine.config import List
from recipe_engine.recipe_api import Property
DEPS = [
@@ -25,24 +26,41 @@
PROPERTIES = {
'remote': Property(
- kind=str, help='Remote repository',
- default='https://pigweed.googlesource.com/pigweed/pigweed'),
-}
+ kind=str,
+ help='Remote repository',
+ default='https://pigweed.googlesource.com/pigweed/pigweed',
+ ),
+
+ 'step': Property(
+ kind=List(str),
+ help="Step to run from 'pw presubmit'. See 'pw presubmit --help' "
+ 'for options. Default is to run all steps.',
+ default=[],
+ ),
+} # yapf: disable
-def RunSteps(api, remote): # pylint: disable=invalid-name
+def RunSteps(api, remote, step): # pylint: disable=invalid-name
+ """Run Pigweed presubmit checks."""
+
api.checkout(remote)
api.environment.init(api.checkout.root)
- # TODO(mohrr) use individual steps instead of presubmit.py
+ prefix = [
+ 'pw',
+ 'presubmit',
+ '--repository', api.checkout.root,
+ '--output-directory', 'out',
+ ]
+
with api.environment():
- cmd = [
- 'pw',
- 'presubmit',
- '--repository', api.checkout.root,
- '--output-directory', 'out',
- ]
- api.step('pw presubmit', cmd)
+ if step:
+ with api.step.defer_results():
+ for s in step:
+ api.step(s, prefix + ['--step', s])
+
+ else:
+ api.step('pw presubmit', prefix)
def GenTests(api): # pylint: disable=invalid-name
@@ -51,3 +69,10 @@
api.checkout.test_data() +
api.environment.test_data()
)
+
+ yield (
+ api.test('step') +
+ api.properties(step=['step1', 'step2']) +
+ api.checkout.test_data() +
+ api.environment.test_data()
+ )
