pw_env_setup: Use GN to install Python packages
For now leave the setup.py search in place to make migration easier for
downstream projects, but largely turn it off for Pigweed itself.
Bug: 287
Change-Id: I79e3a1ba64f98eec2b6617903fad634e8ef0b95a
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/23160
Commit-Queue: Rob Mohr <mohrr@google.com>
Reviewed-by: Wyatt Hepler <hepler@google.com>
diff --git a/bootstrap.bat b/bootstrap.bat
index c4e5265..41d3833 100644
--- a/bootstrap.bat
+++ b/bootstrap.bat
@@ -71,6 +71,10 @@
set "_pw_start_script=%PW_ROOT%\pw_env_setup\py\pw_env_setup\windows_env_start.py"
+if "%PW_PROJECT_ROOT%"=="" (
+ set "PW_PROJECT_ROOT=%PW_ROOT%"
+)
+
:: If PW_SKIP_BOOTSTRAP is set, only run the activation stage instead of the
:: complete env_setup.
if "%PW_SKIP_BOOTSTRAP%" == "" (
diff --git a/bootstrap.sh b/bootstrap.sh
index 246d007..a7fdd5f 100644
--- a/bootstrap.sh
+++ b/bootstrap.sh
@@ -69,6 +69,10 @@
PW_ROOT="$(dirname "$_BOOTSTRAP_PATH")"
export PW_ROOT
+# Please also set PW_PROJECT_ROOT to YOUR_PROJECT_ROOT.
+PW_PROJECT_ROOT="$PW_ROOT"
+export PW_PROJECT_ROOT
+
. "$PW_ROOT/pw_env_setup/util.sh"
pw_deactivate
diff --git a/pw_build/py/pw_build/exec.py b/pw_build/py/pw_build/exec.py
index 0b74458..b956c13 100644
--- a/pw_build/py/pw_build/exec.py
+++ b/pw_build/py/pw_build/exec.py
@@ -22,7 +22,11 @@
import sys
from typing import Dict, Optional
-import pw_cli.log
+# Need to be able to run without pw_cli installed in the virtualenv.
+try:
+ import pw_cli.log
+except ImportError:
+ pass
_LOG = logging.getLogger(__name__)
@@ -157,5 +161,7 @@
if __name__ == '__main__':
- pw_cli.log.install()
+ # If pw_cli is not yet installed in the virtualenv just skip it.
+ if 'pw_cli' in globals():
+ pw_cli.log.install()
sys.exit(main())
diff --git a/pw_build/python.gni b/pw_build/python.gni
index 6b523ca..9ca3a43 100644
--- a/pw_build/python.gni
+++ b/pw_build/python.gni
@@ -26,7 +26,7 @@
# - $name.lint.mypy - Runs mypy.
# - $name.lint.pylint - Runs pylint.
# - $name.tests - Runs all tests for this package.
-# - $name.install - Installs the package in a venv. (Not implemented.)
+# - $name.install - Installs the package in a venv.
# - $name.wheel - Builds a Python wheel for the package. (Not implemented.)
#
# TODO(pwbug/239): Implement installation and wheel building.
diff --git a/pw_cli/py/pw_cli/env.py b/pw_cli/py/pw_cli/env.py
index 4455fc6..021c98e 100644
--- a/pw_cli/py/pw_cli/env.py
+++ b/pw_cli/py/pw_cli/env.py
@@ -34,6 +34,7 @@
type=envparse.strict_bool,
default=False)
parser.add_var('PW_ENVIRONMENT_ROOT')
+ parser.add_var('PW_PROJECT_ROOT')
parser.add_var('PW_ROOT')
parser.add_var('PW_SKIP_BOOTSTRAP')
parser.add_var('PW_SUBPROCESS', type=envparse.strict_bool, default=False)
diff --git a/pw_env_setup/docs.rst b/pw_env_setup/docs.rst
index da1ec9c..640593f 100644
--- a/pw_env_setup/docs.rst
+++ b/pw_env_setup/docs.rst
@@ -28,14 +28,14 @@
changes to your system. This tooling is designed to be reused by any
project.
+.. _CIPD: https://github.com/luci/luci-go/tree/master/cipd
+
Users interact with ``pw_env_setup`` with two commands: ``. bootstrap.sh`` and
``. activate.sh``. The bootstrap command always pulls down the current versions
of CIPD packages and sets up the Python virtual environment. The activate
command reinitializes a previously configured environment, and if none is found,
runs bootstrap.
-.. _CIPD: https://github.com/luci/luci-go/tree/master/cipd
-
.. note::
On Windows the scripts used to set up the environment are ``bootstrap.bat``
and ``activate.bat``. For simplicity they will be referred to with the ``.sh``
@@ -77,17 +77,24 @@
# below for a more flexible way to handle this.
PROJ_SETUP_SCRIPT_PATH="$(pwd)/bootstrap.sh"
- export PROJ_ROOT="$(_python_abspath "$(dirname "$PROJ_SETUP_SCRIPT_PATH")")"
+ export PW_PROJECT_ROOT="$(_python_abspath "$(dirname "$PROJ_SETUP_SCRIPT_PATH")")"
# You may wish to check if the user is attempting to execute this script
# instead of sourcing it. See below for an example of how to handle that
# situation.
- # Source Pigweed's bootstrap script.
- # Using '.' instead of 'source' for dash compatibility. Since users don't use
- # dash directly, using 'source' in documentation so users don't get confused
- # and try to `./bootstrap.sh`.
- . "$PROJ_ROOT/third_party/pigweed/$(basename "$PROJ_SETUP_SCRIPT_PATH")"
+ # Source Pigweed's bootstrap utility script.
+ # Using '.' instead of 'source' for POSIX compatibility. Since users don't use
+ # dash directly, using 'source' in most documentation so users don't get
+ # confused and try to `./bootstrap.sh`.
+ . "$PW_PROJECT_ROOT/third_party/pigweed/pw_env_setup/util.sh"
+
+ pw_check_root "$PW_ROOT"
+ _PW_ACTUAL_ENVIRONMENT_ROOT="$(pw_get_env_root)"
+ export _PW_ACTUAL_ENVIRONMENT_ROOT
+ SETUP_SH="$_PW_ACTUAL_ENVIRONMENT_ROOT/activate.sh"
+ pw_bootstrap --args... # See below for details about args.
+ pw_finalize bootstrap "$SETUP_SH"
User-Friendliness
-----------------
@@ -150,27 +157,18 @@
case ${0##*/} in sh|dash) _pw_sourced=1;; esac
fi
- if [ "$_pw_sourced" -eq 0 ]; then
- _S_NAME=$(basename "$PROJ_SETUP_SCRIPT_PATH" .sh)
- echo "Error: Attempting to $_S_NAME in a subshell"
- echo " Since $_S_NAME.sh modifies your shell's environment variables, it"
- echo " must be sourced rather than executed. In particular, "
- echo " 'bash $_S_NAME.sh' will not work since the modified environment "
- echo " will get destroyed at the end of the script. Instead, source the "
- echo " script's contents in your shell:"
- echo ""
- echo " \$ source $_S_NAME.sh"
- exit 1
- fi
+ _pw_eval_sourced "$_pw_sourced"
Downstream Projects Using Different Packages
********************************************
Projects depending on Pigweed but using additional or different packages should
-copy Pigweed's ``bootstrap.sh`` and update the call to ``env_setup.py``. Search
-for "downstream" for other places that may require changes, like setting the
-``PW_ROOT`` environment variable. Relevant arguments to ``env_setup.py`` are
-listed here.
+copy the Pigweed `sample project`'s ``bootstrap.sh`` and update the call to
+``pw_bootstrap``. Search for "downstream" for other places that may require
+changes, like setting the ``PW_ROOT`` and ``PW_PROJECT_ROOT`` environment
+variables. Relevant arguments to ``pw_bootstrap`` are listed here.
+
+.. _sample project: https://pigweed.googlesource.com/pigweed/sample_project/+/master
``--use-pigweed-defaults``
Use Pigweed default values in addition to the other switches.
@@ -182,8 +180,13 @@
``--virtualenv-requierements path/to/requirements.txt``
Pip requirements file. Compiled with pip-compile.
-``--virtualenv-setup-py-root path/to/directory``
- Directory in which to recursively search for ``setup.py`` files.
+``--virtualenv-gn-target path/to/directory#package-install-target``
+ Target for installing Python packages, and the directory from which it must be
+ run. Example for Pigweed: ``third_party/pigweed#:python.install`` (assuming
+ Pigweed is included in the project at ``third_party/pigweed``). Downstream
+ projects will need to create targets to install their packages and either
+ choose a subset of Pigweed packages or use
+ ``third_party/pigweed#:python.install`` to install all Pigweed packages.
``--cargo-package-file path/to/packages.txt``
Rust cargo packages to install. Lines with package name and version separated
@@ -196,12 +199,12 @@
.. code-block:: bash
- "$ROOT/third_party/pigweed/pw_env_setup/py/pw_env_setup/env_setup.py" \
+ pw_bootstrap \
--shell-file "$SETUP_SH" \
--install-dir "$_PW_ACTUAL_ENVIRONMENT_ROOT" \
--use-pigweed-defaults \
- --cipd-package-file "$ROOT/path/to/cipd.json" \
- --virtualenv-setup-py-root "$ROOT"
+ --cipd-package-file "$PW_PROJECT_ROOT/path/to/cipd.json" \
+ --virtualenv-setup-py-root "$PW_PROJECT_ROOT"
Projects wanting some of the Pigweed environment packages but not all of them
should not use ``--use-pigweed-defaults`` and must manually add the references
@@ -216,8 +219,8 @@
"$PW_ROOT/pw_env_setup/py/pw_env_setup/cipd_setup/luci.json"
--virtualenv-requirements
"$PW_ROOT/pw_env_setup/py/pw_env_setup/virtualenv_setup/requirements.txt"
- --virtualenv-setup-py-root
- "$PW_ROOT"
+ --virtualenv-gn-target
+ "$PW_ROOT#:python.install"
--cargo-package-file
"$PW_ROOT/pw_env_setup/py/pw_env_setup/cargo_setup/packages.txt"
diff --git a/pw_env_setup/py/pw_env_setup/env_setup.py b/pw_env_setup/py/pw_env_setup/env_setup.py
index 94f4bd9..328e8c4 100755
--- a/pw_env_setup/py/pw_env_setup/env_setup.py
+++ b/pw_env_setup/py/pw_env_setup/env_setup.py
@@ -152,7 +152,7 @@
'warning: pattern "{}" matched 0 files'.format(pat))
files.extend(matches)
- if not files:
+ if globs and not files:
warnings.append('warning: matched 0 total files')
return files, warnings
@@ -174,8 +174,10 @@
def __init__(self, pw_root, cipd_cache_dir, shell_file, quiet, install_dir,
use_pigweed_defaults, cipd_package_file, virtualenv_root,
virtualenv_requirements, virtualenv_setup_py_root,
- cargo_package_file, enable_cargo, json_file):
+ virtualenv_gn_target, cargo_package_file, enable_cargo,
+ json_file, project_root):
self._env = environment.Environment()
+ self._project_root = project_root
self._pw_root = pw_root
self._setup_root = os.path.join(pw_root, 'pw_env_setup', 'py',
'pw_env_setup')
@@ -196,6 +198,7 @@
self._cipd_package_file = []
self._virtualenv_requirements = []
self._virtualenv_setup_py_root = []
+ self._virtualenv_gn_targets = []
self._cargo_package_file = []
self._enable_cargo = enable_cargo
@@ -218,15 +221,19 @@
self._virtualenv_requirements.append(
os.path.join(setup_root, 'virtualenv_setup',
'requirements.txt'))
- self._virtualenv_setup_py_root.append(pw_root)
+ self._virtualenv_gn_targets.append(
+ virtualenv_setup.GnTarget(
+ '{}#:python.install'.format(pw_root)))
self._cargo_package_file.append(
os.path.join(setup_root, 'cargo_setup', 'packages.txt'))
self._cipd_package_file.extend(cipd_package_file)
self._virtualenv_requirements.extend(virtualenv_requirements)
self._virtualenv_setup_py_root.extend(virtualenv_setup_py_root)
+ self._virtualenv_gn_targets.extend(virtualenv_gn_target)
self._cargo_package_file.extend(cargo_package_file)
+ self._env.set('PW_PROJECT_ROOT', project_root)
self._env.set('PW_ROOT', pw_root)
self._env.set('_PW_ACTUAL_ENVIRONMENT_ROOT', install_dir)
self._env.add_replacement('_PW_ACTUAL_ENVIRONMENT_ROOT', install_dir)
@@ -396,14 +403,19 @@
shutil.copyfile(new_python3, python3_copy)
new_python3 = python3_copy
- if not requirements and not setup_py_roots:
+ if (not requirements and not self._virtualenv_setup_py_root
+ and not self._virtualenv_gn_targets):
return result(_Result.Status.SKIPPED)
- if not virtualenv_setup.install(venv_path=self._virtualenv_root,
- requirements=requirements,
- setup_py_roots=setup_py_roots,
- python=new_python3,
- env=self._env):
+ if not virtualenv_setup.install(
+ project_root=self._project_root,
+ venv_path=self._virtualenv_root,
+ requirements=requirements,
+ gn_targets=self._virtualenv_gn_targets,
+ setup_py_roots=setup_py_roots,
+ python=new_python3,
+ env=self._env,
+ ):
return result(_Result.Status.FAILED)
return result(_Result.Status.DONE)
@@ -446,12 +458,21 @@
stderr=outs).strip()
except subprocess.CalledProcessError:
pw_root = None
+
parser.add_argument(
'--pw-root',
default=pw_root,
required=not pw_root,
)
+ project_root = os.environ.get('PW_PROJECT_ROOT', None) or pw_root
+
+ parser.add_argument(
+ '--project-root',
+ default=project_root,
+ required=not project_root,
+ )
+
parser.add_argument(
'--cipd-cache-dir',
default=os.environ.get('CIPD_CACHE_DIR',
@@ -507,6 +528,15 @@
)
parser.add_argument(
+ '--virtualenv-gn-target',
+ help=('GN targets that build and install Python packages. Format: '
+ "path/to/gn_root#target"),
+ default=[],
+ action='append',
+ type=virtualenv_setup.GnTarget,
+ )
+
+ parser.add_argument(
'--virtualenv-root',
help=('Basename of virtualenv directory. Default: '
'<install_dir>/pigweed-venv'),
@@ -540,6 +570,7 @@
'cipd_package_file',
'virtualenv_requirements',
'virtualenv_setup_py_root',
+ 'virtualenv_gn_target',
'cargo_package_file',
)
diff --git a/pw_env_setup/py/pw_env_setup/virtualenv_setup/__main__.py b/pw_env_setup/py/pw_env_setup/virtualenv_setup/__main__.py
index 06cca9e..cfdb3a6 100644
--- a/pw_env_setup/py/pw_env_setup/virtualenv_setup/__main__.py
+++ b/pw_env_setup/py/pw_env_setup/virtualenv_setup/__main__.py
@@ -26,6 +26,13 @@
def _main():
parser = argparse.ArgumentParser(description=__doc__)
+
+ project_root = os.environ.get('PW_PROJECT_ROOT', None)
+
+ parser.add_argument('--project-root',
+ default=project_root,
+ required=not project_root,
+ help='Path to overall project root.')
parser.add_argument('--venv_path',
required=True,
help='Path at which to create the venv')
@@ -39,6 +46,12 @@
default=[],
action='append',
help='places to search for setup.py files')
+ parser.add_argument('--gn-target',
+ dest='gn_targets',
+ default=[],
+ action='append',
+ type=virtualenv_setup.GnTarget,
+ help='GN targets that install packages')
parser.add_argument('--quick-setup',
dest='full_envsetup',
action='store_false',
diff --git a/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py b/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py
index 674f974..b725c86 100644
--- a/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py
+++ b/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py
@@ -17,11 +17,25 @@
import glob
import os
+import re
import subprocess
import sys
import tempfile
+class GnTarget(object): # pylint: disable=useless-object-inheritance
+ def __init__(self, val):
+ self.directory, self.target = val.split('#', 1)
+
+ @property
+ def name(self):
+ """A reasonably stable and unique name for each pair."""
+ result = '{}-{}'.format(
+ os.path.basename(os.path.normpath(self.directory)),
+ hash(self.directory + self.target))
+ return re.sub(r'[:/#_]*', '_', result)
+
+
def git_stdout(*args, **kwargs):
"""Run git, passing args as git params and kwargs to subprocess."""
return subprocess.check_output(['git'] + list(args), **kwargs).strip()
@@ -93,9 +107,11 @@
def install(
+ project_root,
venv_path,
full_envsetup=True,
requirements=(),
+ gn_targets=(),
python=sys.executable,
setup_py_roots=(),
env=None,
@@ -172,9 +188,33 @@
pip_install('--log', os.path.join(venv_path, 'pip-packages.log'),
*(find_args + package_args))
- if env:
- env.set('VIRTUAL_ENV', venv_path)
- env.prepend('PATH', venv_bin)
- env.clear('PYTHONHOME')
+ def install_packages(gn_target):
+ build = os.path.join(venv_path, gn_target.name)
+
+ gn_log = 'gn-gen-{}.log'.format(gn_target.name)
+ with open(os.path.join(venv_path, gn_log), 'w') as outs:
+ subprocess.check_call(('gn', 'gen', build),
+ cwd=os.path.join(project_root,
+ gn_target.directory),
+ stdout=outs,
+ stderr=outs)
+
+ ninja_log = 'ninja-{}.log'.format(gn_target.name)
+ with open(os.path.join(venv_path, ninja_log), 'w') as outs:
+ ninja_cmd = ['ninja', '-C', build]
+ ninja_cmd.append(gn_target.target)
+ subprocess.check_call(ninja_cmd, stdout=outs, stderr=outs)
+
+ if gn_targets:
+ if env:
+ env.set('VIRTUAL_ENV', venv_path)
+ env.prepend('PATH', venv_bin)
+ env.clear('PYTHONHOME')
+ with env():
+ for gn_target in gn_targets:
+ install_packages(gn_target)
+ else:
+ for gn_target in gn_targets:
+ install_packages(gn_target)
return True
diff --git a/pw_presubmit/py/pw_presubmit/environment.py b/pw_presubmit/py/pw_presubmit/environment.py
index 1b54a7a..23aa347 100644
--- a/pw_presubmit/py/pw_presubmit/environment.py
+++ b/pw_presubmit/py/pw_presubmit/environment.py
@@ -65,12 +65,24 @@
output_directory: Path,
setup_py_roots: Iterable[Union[Path, str]] = (),
requirements: Iterable[Union[Path, str]] = (),
+ gn_targets: Iterable[str] = (),
) -> None:
"""Sets up a virtualenv, assumes recent Python 3 is already installed."""
virtualenv_source = pigweed_root.joinpath('pw_env_setup', 'py',
'pw_env_setup',
'virtualenv_setup')
+ # Need to set VIRTUAL_ENV before invoking GN because the GN targets install
+ # directly to the current virtual env.
+ os.environ['VIRTUAL_ENV'] = str(output_directory)
+ os.environ['PATH'] = os.pathsep.join((
+ str(output_directory.joinpath('bin')),
+ os.environ['PATH'],
+ ))
+
+ if not gn_targets:
+ gn_targets = (f'{os.environ["PW_ROOT"]}#:python.install', )
+
# For speed, don't build the venv if it exists. Use --clean to rebuild.
if not output_directory.joinpath('pyvenv.cfg').is_file():
call(
@@ -79,10 +91,6 @@
f'--venv_path={output_directory}',
f'--requirements={virtualenv_source / "requirements.txt"}',
*(f'--requirements={x}' for x in requirements),
- *(f'--setup-py-root={p}' for p in [pigweed_root, *setup_py_roots]),
+ *(f'--setup-py-root={p}' for p in setup_py_roots),
+ *(f'--gn-target={t}' for t in gn_targets),
)
-
- os.environ['PATH'] = os.pathsep.join((
- str(output_directory.joinpath('bin')),
- os.environ['PATH'],
- ))
