pw_env_setup: Switch to using config JSON files
Change-Id: I7e209f8fe862e745db00d56c3f14046fd25ceb32
Bug: 327
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/34140
Reviewed-by: Wyatt Hepler <hepler@google.com>
Commit-Queue: Rob Mohr <mohrr@google.com>
diff --git a/bootstrap.bat b/bootstrap.bat
index 4ef0023..5c46a29 100644
--- a/bootstrap.bat
+++ b/bootstrap.bat
@@ -111,9 +111,7 @@
--pw-root "%PW_ROOT%" ^
--shell-file "%shell_file%" ^
--install-dir "%_PW_ACTUAL_ENVIRONMENT_ROOT%" ^
- --use-pigweed-defaults ^
- --virtualenv-gn-target "%PW_ROOT%#pw_env_setup:python.install" ^
- --virtualenv-gn-target "%PW_ROOT%#pw_env_setup:target_support_packages.install" ^
+ --config-file "%PW_ROOT%/pw_env_setup/config.json" ^
--project-root "%PW_PROJECT_ROOT%"
goto activate_shell
diff --git a/bootstrap.sh b/bootstrap.sh
index 06c427b..71006b9 100644
--- a/bootstrap.sh
+++ b/bootstrap.sh
@@ -89,7 +89,7 @@
if [ "$(basename "$_BOOTSTRAP_PATH")" = "bootstrap.sh" ] || \
[ ! -f "$SETUP_SH" ] || \
[ ! -s "$SETUP_SH" ]; then
- pw_bootstrap --shell-file "$SETUP_SH" --install-dir "$_PW_ACTUAL_ENVIRONMENT_ROOT" --use-pigweed-defaults --json-file "$_PW_ACTUAL_ENVIRONMENT_ROOT/actions.json" --virtualenv-gn-out-dir "$PW_ROOT/out" --virtualenv-gn-target "$PW_ROOT#pw_env_setup:python.install" --virtualenv-gn-target "$PW_ROOT#pw_env_setup:target_support_packages.install"
+ pw_bootstrap --shell-file "$SETUP_SH" --install-dir "$_PW_ACTUAL_ENVIRONMENT_ROOT" --json-file "$_PW_ACTUAL_ENVIRONMENT_ROOT/actions.json" --config-file "$PW_ROOT/pw_env_setup/config.json"
pw_finalize bootstrap "$SETUP_SH"
else
pw_activate
diff --git a/pw_env_setup/compatibility.json b/pw_env_setup/compatibility.json
new file mode 100644
index 0000000..3c3cac7
--- /dev/null
+++ b/pw_env_setup/compatibility.json
@@ -0,0 +1,14 @@
+{
+ "cipd_package_files": [
+ "pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json",
+ "pw_env_setup/py/pw_env_setup/cipd_setup/luci.json"
+ "pw_env_setup/py/pw_env_setup/cipd_setup/compatibility.json"
+ ],
+ "virtualenv": {
+ "gn_root": ".",
+ "gn_targets": [
+ ":python.install",
+ ":target_support_packages.install"
+ ]
+ }
+}
diff --git a/pw_env_setup/config.json b/pw_env_setup/config.json
new file mode 100644
index 0000000..435be72
--- /dev/null
+++ b/pw_env_setup/config.json
@@ -0,0 +1,13 @@
+{
+ "cipd_package_files": [
+ "pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json",
+ "pw_env_setup/py/pw_env_setup/cipd_setup/luci.json"
+ ],
+ "virtualenv": {
+ "gn_root": ".",
+ "gn_targets": [
+ ":python.install",
+ ":target_support_packages.install"
+ ]
+ }
+}
diff --git a/pw_env_setup/docs.rst b/pw_env_setup/docs.rst
index 8c76c75..473f1b6 100644
--- a/pw_env_setup/docs.rst
+++ b/pw_env_setup/docs.rst
@@ -158,66 +158,45 @@
********************************************
Projects depending on Pigweed but using additional or different packages should
-copy the Pigweed `sample project`'s ``bootstrap.sh`` and update the call to
-``pw_bootstrap``. Search for "downstream" for other places that may require
-changes, like setting the ``PW_ROOT`` and ``PW_PROJECT_ROOT`` environment
-variables. Relevant arguments to ``pw_bootstrap`` are listed here.
+copy the Pigweed `sample project`'s ``bootstrap.sh`` and ``config.json`` and
+update the call to ``pw_bootstrap``. Search for "downstream" for other places
+that may require changes, like setting the ``PW_ROOT`` and ``PW_PROJECT_ROOT``
+environment variables. Explanations of parts of ``config.json`` are described
+here.
.. _sample project: https://pigweed.googlesource.com/pigweed/sample_project/+/master
-``--use-pigweed-defaults``
- Use Pigweed default values in addition to the other switches.
-
-``--cipd-package-file path/to/packages.json``
+``cipd_package_files``
CIPD package file. JSON file consisting of a list of dictionaries with "path"
and "tags" keys, where "tags" is a list of strings.
-``--virtualenv-requierements path/to/requirements.txt``
- Pip requirements file. Compiled with pip-compile.
+``virtualenv.gn_targets``
+ Target for installing Python packages. Downstream projects will need to
+ create targets to install their packages or only use Pigweed Python packages.
-``--virtualenv-gn-target path/to/directory#package-install-target``
- Target for installing Python packages, and the directory from which it must be
- run. Example for Pigweed: ``third_party/pigweed#:python.install`` (assuming
- Pigweed is included in the project at ``third_party/pigweed``). Downstream
- projects will need to create targets to install their packages and either
- choose a subset of Pigweed packages or use
- ``third_party/pigweed#:python.install`` to install all Pigweed packages.
+``virtualenv.gn_root``
+ The root directory of your GN build tree, relative to ``PW_PROJECT_ROOT``.
+ This is the directory your project's ``.gn`` file is located in. If you're
+ only installing Pigweed Python packages, use the location of the Pigweed
+ submodule.
-``--cargo-package-file path/to/packages.txt``
- Rust cargo packages to install. Lines with package name and version separated
- by a space. Has no effect without ``--enable-cargo``.
+An example of a config file is below.
-``--enable-cargo``
- Enable cargo package installation.
+.. code-block:: json
-An example of the changed env_setup.py line is below.
-
-.. code-block:: bash
-
- pw_bootstrap \
- --shell-file "$SETUP_SH" \
- --install-dir "$_PW_ACTUAL_ENVIRONMENT_ROOT" \
- --use-pigweed-defaults \
- --cipd-package-file "$PW_PROJECT_ROOT/path/to/cipd.json" \
- --virtualenv-gn-target "$PW_PROJECT_ROOT#:python.install"
-
-Projects wanting some of the Pigweed environment packages but not all of them
-should not use ``--use-pigweed-defaults`` and must manually add the references
-to Pigweed default packages through the other arguments. The arguments below
-are identical to using ``--use-pigweed-defaults``.
-
-.. code-block:: bash
-
- --cipd-package-file
- "$PW_ROOT/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json"
- --cipd-package-file
- "$PW_ROOT/pw_env_setup/py/pw_env_setup/cipd_setup/luci.json"
- --virtualenv-requirements
- "$PW_ROOT/pw_env_setup/py/pw_env_setup/virtualenv_setup/requirements.txt"
- --virtualenv-gn-target
- "$PW_ROOT#:python.install"
- --cargo-package-file
- "$PW_ROOT/pw_env_setup/py/pw_env_setup/cargo_setup/packages.txt"
+ {
+ "cipd_package_files": [
+ "pigweed/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json",
+ "pigweed/pw_env_setup/py/pw_env_setup/cipd_setup/luci.json"
+ "tools/packages.json"
+ ],
+ "virtualenv": {
+ "gn_root": ".",
+ "gn_targets": [
+ ":python.install",
+ ]
+ }
+ }
Environment Variables
*********************
diff --git a/pw_env_setup/py/pw_env_setup/env_setup.py b/pw_env_setup/py/pw_env_setup/env_setup.py
index a05663b..42470c8 100755
--- a/pw_env_setup/py/pw_env_setup/env_setup.py
+++ b/pw_env_setup/py/pw_env_setup/env_setup.py
@@ -166,6 +166,10 @@
return result
+class ConfigFileError(Exception):
+ pass
+
+
# TODO(mohrr) remove disable=useless-object-inheritance once in Python 3.
# pylint: disable=useless-object-inheritance
# pylint: disable=too-many-instance-attributes
@@ -176,7 +180,7 @@
use_pigweed_defaults, cipd_package_file, virtualenv_root,
virtualenv_requirements, virtualenv_gn_target,
virtualenv_gn_out_dir, cargo_package_file, enable_cargo,
- json_file, project_root):
+ json_file, project_root, config_file):
self._env = environment.Environment()
self._project_root = project_root
self._pw_root = pw_root
@@ -202,6 +206,9 @@
self._cargo_package_file = []
self._enable_cargo = enable_cargo
+ if config_file:
+ self._parse_config_file(config_file)
+
self._json_file = json_file
setup_root = os.path.join(pw_root, 'pw_env_setup', 'py',
@@ -238,6 +245,33 @@
self._env.add_replacement('_PW_ACTUAL_ENVIRONMENT_ROOT', install_dir)
self._env.add_replacement('PW_ROOT', pw_root)
+ def _parse_config_file(self, config_file):
+ config = json.load(config_file)
+
+ self._cipd_package_file.extend(
+ os.path.join(self._project_root, x)
+ for x in config.pop('cipd_package_files', ()))
+
+ virtualenv = config.pop('virtualenv', {})
+
+ if virtualenv.get('gn_root'):
+ root = os.path.join(self._project_root, virtualenv.pop('gn_root'))
+ else:
+ root = self._project_root
+
+ for target in virtualenv.pop('gn_targets', ()):
+ self._virtualenv_gn_targets.append(
+ virtualenv_setup.GnTarget('{}#{}'.format(root, target)))
+
+ if virtualenv:
+ raise ConfigFileError(
+ 'unrecognized option in {}: "virtualenv.{}"'.format(
+ config_file.name, next(iter(virtualenv))))
+
+ if config:
+ raise ConfigFileError('unrecognized option in {}: "{}"'.format(
+ config_file.name, next(iter(config))))
+
def _log(self, *args, **kwargs):
# Not using logging module because it's awkward to flush a log handler.
if self._quiet:
@@ -505,6 +539,12 @@
)
parser.add_argument(
+ '--config-file',
+ help='JSON file describing CIPD and virtualenv requirements.',
+ type=argparse.FileType('r'),
+ )
+
+ parser.add_argument(
'--use-pigweed-defaults',
help='Use Pigweed default values in addition to the given environment '
'variables.',
@@ -570,7 +610,7 @@
args = parser.parse_args(argv)
- one_required = (
+ others = (
'use_pigweed_defaults',
'cipd_package_file',
'virtualenv_requirements',
@@ -578,10 +618,17 @@
'cargo_package_file',
)
+ one_required = others + ('config_file', )
+
if not any(getattr(args, x) for x in one_required):
parser.error('At least one of ({}) is required'.format(', '.join(
'"--{}"'.format(x.replace('_', '-')) for x in one_required)))
+ if args.config_file and any(getattr(args, x) for x in others):
+ parser.error('Cannot combine --config-file with any of {}'.format(
+ ', '.join('"--{}"'.format(x.replace('_', '-'))
+ for x in one_required)))
+
return args
diff --git a/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py b/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py
index 8e798ea..9a227c8 100644
--- a/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py
+++ b/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py
@@ -218,6 +218,7 @@
build_dir,
'--args=dir_pigweed="{}"'.format(pw_root),
)
+ print(gn_cmd, file=outs)
subprocess.check_call(gn_cmd,
cwd=os.path.join(project_root,
gn_target.directory),
@@ -234,6 +235,7 @@
with open(ninja_log_path, 'w') as outs:
ninja_cmd = ['ninja', '-C', build_dir]
ninja_cmd.append(gn_target.target)
+ print(ninja_cmd, file=outs)
subprocess.check_call(ninja_cmd, stdout=outs, stderr=outs)
except subprocess.CalledProcessError as err:
with open(ninja_log_path, 'r') as ins:
