Move env_setup to pw_env_setup
Bug: 67
Change-Id: I285e917968b2e24e0851753ea4802cef1fbe2d67
diff --git a/.gitignore b/.gitignore
index b9b2f70..9c7800a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -42,7 +42,7 @@
*_REMOTE_*.txt
# Env Setup
-env_setup/.setup.sh
-env_setup/.env_setup.bat
+pw_env_setup/.setup.sh
+pw_env_setup/.env_setup.bat
.cipd
.cargo
diff --git a/README.md b/README.md
index cd586ec..a2aa40c 100644
--- a/README.md
+++ b/README.md
@@ -20,14 +20,15 @@
```bash
$ git clone sso://pigweed.googlesource.com/pigweed/pigweed ~/pigweed
$ cd ~/pigweed
-$ env_setup/cipd/wrapper.py auth-login # Once per machine.
-$ . env_setup/bootstrap.sh
+# Only need to run auth-login once per machine.
+$ pw_env_setup/py/pw_env_setup/cipd_setup/wrapper.py auth-login
+$ . pw_env_setup/bootstrap.sh
```
-You can use `. env_setup/env_setup.sh` in place of `. env_setup/bootstrap.sh`.
+You can use `. pw_env_setup/setup.sh` in place of `. pw_env_setup/bootstrap.sh`.
Both should work every time, but `bootstrap.sh` tends to remove and reinstall
-things at the expense of time whereas `env_setup.sh` tends to do basic checks
-to see if time can be saved by skipping expensive operations.
+things at the expense of time whereas `setup.sh` assumes things have
+already been installed and only sets environment variables.
If you're using Homebrew and you get an error saying
`module 'http.client' has no attribute 'HTTPSConnection'` then your
diff --git a/env_setup/cipd/pigweed.json b/env_setup/cipd/pigweed.json
index 2c6129b..2ce9f0d 120000
--- a/env_setup/cipd/pigweed.json
+++ b/env_setup/cipd/pigweed.json
@@ -1 +1 @@
-../cipd_setup/pigweed.json
\ No newline at end of file
+../../pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json
\ No newline at end of file
diff --git a/env_setup/virtualenv/requirements.txt b/env_setup/virtualenv/requirements.txt
index 3c9f4d0..baf8449 120000
--- a/env_setup/virtualenv/requirements.txt
+++ b/env_setup/virtualenv/requirements.txt
@@ -1 +1 @@
-../virtualenv_setup/requirements.txt
\ No newline at end of file
+../../pw_env_setup/py/pw_env_setup/virtualenv_setup/requirements.txt
\ No newline at end of file
diff --git a/env_setup/bootstrap.sh b/pw_env_setup/bootstrap.sh
similarity index 100%
rename from env_setup/bootstrap.sh
rename to pw_env_setup/bootstrap.sh
diff --git a/env_setup/env_setup.bat b/pw_env_setup/env_setup.bat
similarity index 74%
rename from env_setup/env_setup.bat
rename to pw_env_setup/env_setup.bat
index d53a74b..7152cc1 100644
--- a/env_setup/env_setup.bat
+++ b/pw_env_setup/env_setup.bat
@@ -17,12 +17,12 @@
:: Calls a Powershell script that determines the correct PW_ROOT directory and
:: exports it as an environment variable.
-for /F "usebackq tokens=1" %%i in (`powershell %%~dp0..\..\env_setup\env_setup.ps1`) do set PW_ROOT=%%i
+for /F "usebackq tokens=1" %%i in (`powershell %%~dp0..\..\pw_env_setup\env_setup.ps1`) do set PW_ROOT=%%i
-set shell_file="%PW_ROOT%\env_setup\.env_setup.bat"
+set shell_file="%PW_ROOT%\pw_env_setup\.env_setup.bat"
if not exist %shell_file% (
- call python %PW_ROOT%\env_setup\env_setup.py --pw-root %PW_ROOT% --shell-file %shell_file%
+ call python %PW_ROOT%\pw_env_setup\py\pw_env_setup\env_setup.py --pw-root %PW_ROOT% --shell-file %shell_file%
)
call %shell_file%
diff --git a/env_setup/env_setup.ps1 b/pw_env_setup/env_setup.ps1
similarity index 100%
rename from env_setup/env_setup.ps1
rename to pw_env_setup/env_setup.ps1
diff --git a/env_setup/env_setup.sh b/pw_env_setup/env_setup.sh
similarity index 90%
rename from env_setup/env_setup.sh
rename to pw_env_setup/env_setup.sh
index 7e5c16f..a9d18e8 100644
--- a/env_setup/env_setup.sh
+++ b/pw_env_setup/env_setup.sh
@@ -1,4 +1,4 @@
-# Copyright 2019 The Pigweed Authors
+# Copyright 2020 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
@@ -36,7 +36,7 @@
PW_ROOT=$(dirname $(dirname $PW_SETUP_SCRIPT_PATH))
export PW_ROOT
-SETUP_SH="$PW_ROOT/env_setup/.setup.sh"
+SETUP_SH="$PW_ROOT/pw_env_setup/.setup.sh"
# Try to use Python 3 if possible by default, before Python 2.
if which python3 &> /dev/null; then
@@ -50,7 +50,7 @@
[ $(basename $PW_SETUP_SCRIPT_PATH) = "bootstrap.sh" ] || \
[ ! -f $SETUP_SH ] || \
[ ! -s $SETUP_SH ]; then
- $PYTHON $PW_ROOT/env_setup/env_setup.py --shell-file $SETUP_SH
+ $PYTHON $PW_ROOT/pw_env_setup/py/pw_env_setup/env_setup.py --shell-file $SETUP_SH
fi
. $SETUP_SH
diff --git a/env_setup/__init__.py b/pw_env_setup/py/pw_env_setup/__init__.py
similarity index 94%
rename from env_setup/__init__.py
rename to pw_env_setup/py/pw_env_setup/__init__.py
index 2c8334f..1b8f5a8 100644
--- a/env_setup/__init__.py
+++ b/pw_env_setup/py/pw_env_setup/__init__.py
@@ -11,3 +11,4 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
+"""Pigweed environment setup."""
diff --git a/env_setup/cargo_setup/__init__.py b/pw_env_setup/py/pw_env_setup/cargo_setup/__init__.py
similarity index 92%
rename from env_setup/cargo_setup/__init__.py
rename to pw_env_setup/py/pw_env_setup/cargo_setup/__init__.py
index 28a2a34..101d2f3 100644
--- a/env_setup/cargo_setup/__init__.py
+++ b/pw_env_setup/py/pw_env_setup/cargo_setup/__init__.py
@@ -27,8 +27,8 @@
# packages.txt contains packages one per line with two fields: package
# name and version.
- package_path = os.path.join(pw_root, 'env_setup', 'cargo_setup',
- 'packages.txt')
+ package_path = os.path.join(pw_root, 'pw_env_setup', 'py', 'pw_env_setup',
+ 'cargo_setup', 'packages.txt')
with env(), open(package_path, 'r') as ins:
for line in ins:
line = line.strip()
diff --git a/env_setup/cargo_setup/packages.txt b/pw_env_setup/py/pw_env_setup/cargo_setup/packages.txt
similarity index 100%
rename from env_setup/cargo_setup/packages.txt
rename to pw_env_setup/py/pw_env_setup/cargo_setup/packages.txt
diff --git a/env_setup/cipd_setup/.cipd_version b/pw_env_setup/py/pw_env_setup/cipd_setup/.cipd_version
similarity index 100%
rename from env_setup/cipd_setup/.cipd_version
rename to pw_env_setup/py/pw_env_setup/cipd_setup/.cipd_version
diff --git a/env_setup/cipd_setup/.cipd_version.digests b/pw_env_setup/py/pw_env_setup/cipd_setup/.cipd_version.digests
similarity index 100%
rename from env_setup/cipd_setup/.cipd_version.digests
rename to pw_env_setup/py/pw_env_setup/cipd_setup/.cipd_version.digests
diff --git a/env_setup/cipd_setup/__init__.py b/pw_env_setup/py/pw_env_setup/cipd_setup/__init__.py
similarity index 100%
rename from env_setup/cipd_setup/__init__.py
rename to pw_env_setup/py/pw_env_setup/cipd_setup/__init__.py
diff --git a/env_setup/cipd_setup/luci.json b/pw_env_setup/py/pw_env_setup/cipd_setup/luci.json
similarity index 100%
rename from env_setup/cipd_setup/luci.json
rename to pw_env_setup/py/pw_env_setup/cipd_setup/luci.json
diff --git a/env_setup/cipd_setup/pigweed.json b/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json
similarity index 100%
rename from env_setup/cipd_setup/pigweed.json
rename to pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json
diff --git a/env_setup/cipd_setup/update.py b/pw_env_setup/py/pw_env_setup/cipd_setup/update.py
similarity index 87%
rename from env_setup/cipd_setup/update.py
rename to pw_env_setup/py/pw_env_setup/cipd_setup/update.py
index eee7d34..5d3c4cf 100755
--- a/env_setup/cipd_setup/update.py
+++ b/pw_env_setup/py/pw_env_setup/cipd_setup/update.py
@@ -29,25 +29,28 @@
import subprocess
import sys
-SCRIPT_ROOT = os.path.abspath(os.path.dirname(__file__))
-GIT_ROOT = subprocess.check_output(['git', 'rev-parse', '--show-toplevel'],
- cwd=SCRIPT_ROOT).decode('utf-8').strip()
-
def parse(argv=None):
"""Parse arguments."""
+ script_root = os.path.join(os.environ['PW_ROOT'], 'pw_env_setup', 'py',
+ 'pw_env_setup', 'cipd_setup')
+ git_root = subprocess.check_output(
+ ('git', 'rev-parse', '--show-toplevel'),
+ cwd=script_root,
+ ).decode('utf-8').strip()
+
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument(
'--install-dir',
dest='root_install_dir',
- default=os.path.join(GIT_ROOT, '.cipd'),
+ default=os.path.join(git_root, '.cipd'),
)
parser.add_argument('--package-file',
dest='package_files',
action='append')
parser.add_argument('--cipd',
- default=os.path.join(SCRIPT_ROOT, 'wrapper.py'))
+ default=os.path.join(script_root, 'wrapper.py'))
parser.add_argument('--cache-dir',
default=os.environ.get(
'CIPD_CACHE_DIR',
@@ -111,8 +114,15 @@
env_vars.set('PW_CIPD_INSTALL_DIR', root_install_dir)
env_vars.set('CIPD_CACHE_DIR', cache_dir)
+ pw_root = None
+ if env_vars:
+ pw_root = env_vars.get('PW_ROOT', None)
+ if not pw_root:
+ pw_root = os.environ['PW_ROOT']
+
# Run cipd for each json file.
- default_packages = os.path.join(SCRIPT_ROOT, '*.json')
+ default_packages = os.path.join(pw_root, 'pw_env_setup', 'py',
+ 'pw_env_setup', 'cipd_setup', '*.json')
for package_file in package_files or glob.glob(default_packages):
ensure_file = os.path.join(
root_install_dir,
diff --git a/env_setup/cipd_setup/wrapper.py b/pw_env_setup/py/pw_env_setup/cipd_setup/wrapper.py
similarity index 100%
rename from env_setup/cipd_setup/wrapper.py
rename to pw_env_setup/py/pw_env_setup/cipd_setup/wrapper.py
diff --git a/env_setup/env_setup.py b/pw_env_setup/py/pw_env_setup/env_setup.py
similarity index 71%
rename from env_setup/env_setup.py
rename to pw_env_setup/py/pw_env_setup/env_setup.py
index f3a3622..755c2f3 100755
--- a/env_setup/env_setup.py
+++ b/pw_env_setup/py/pw_env_setup/env_setup.py
@@ -25,20 +25,52 @@
from __future__ import print_function
import argparse
+import copy
import glob
+import inspect
import os
import shutil
import subprocess
import sys
-# TODO(mohrr) remove import-error disabling, not sure why pylint has issues
-# with it.
-import cipd_setup.update # pylint: disable=import-error
-import cipd_setup.wrapper # pylint: disable=import-error
-import cargo_setup # pylint: disable=import-error
-import environment # pylint: disable=import-error
-import host_build_setup # pylint: disable=import-error
-import virtualenv_setup # pylint: disable=import-error
+# TODO(pwbug/67): Remove import hacks once the oxidized prebuilt binaries are
+# proven stable for first-time bootstrapping. For now, continue to support
+# running directly from source without assuming a functioning Python
+# environment when running for the first time.
+
+# If we're running oxidized, filesystem-centric import hacks won't work. In that
+# case, jump straight to the imports and assume oxidation brought in the deps.
+if not getattr(sys, 'oxidized', False):
+ try:
+ # Even if we're running from source, the user may have a functioning
+ # Python environment already set up. Prefer using it over hacks.
+ # pylint: disable=no-name-in-module
+ from pw_env_setup import cargo_setup
+ # pylint: enable=no-name-in-module
+ except ImportError:
+ old_sys_path = copy.deepcopy(sys.path)
+ filename = None
+ if hasattr(sys.modules[__name__], '__file__'):
+ filename = __file__
+ else:
+ # Try introspection in environments where __file__ is not populated.
+ filename = inspect.getfile(inspect.currentframe())
+ # If none of our strategies worked, the imports are going to fail.
+ if filename is None:
+ raise
+ sys.path.append(
+ os.path.abspath(
+ os.path.join(filename, os.path.pardir, os.path.pardir)))
+ import pw_env_setup # pylint: disable=unused-import
+ sys.path = old_sys_path
+
+# pylint: disable=wrong-import-position
+from pw_env_setup.cipd_setup import update as cipd_update
+from pw_env_setup.cipd_setup import wrapper as cipd_wrapper
+from pw_env_setup import cargo_setup
+from pw_env_setup import environment
+from pw_env_setup import host_build_setup
+from pw_env_setup import virtualenv_setup
# TODO(mohrr) remove disable=useless-object-inheritance once in Python 3.
@@ -49,6 +81,8 @@
super(EnvSetup, self).__init__(*args, **kwargs)
self._env = environment.Environment()
self._pw_root = pw_root
+ self._setup_root = os.path.join(pw_root, 'pw_env_setup', 'py',
+ 'pw_env_setup')
self._cipd_cache_dir = cipd_cache_dir
self._shell_file = shell_file
@@ -82,11 +116,11 @@
def cipd(self):
install_dir = os.path.join(self._pw_root, '.cipd')
- cipd_client = cipd_setup.wrapper.init(install_dir)
+ cipd_client = cipd_wrapper.init(install_dir)
package_files = glob.glob(
- os.path.join(self._pw_root, 'env_setup', 'cipd_setup', '*.json'))
- cipd_setup.update.update(
+ os.path.join(self._setup_root, 'cipd_setup', '*.json'))
+ cipd_update.update(
cipd=cipd_client,
root_install_dir=install_dir,
package_files=package_files,
@@ -101,8 +135,8 @@
venv_path = os.path.join(self._pw_root, '.python3-env')
- requirements = os.path.join(self._pw_root, 'env_setup',
- 'virtualenv_setup', 'requirements.txt')
+ requirements = os.path.join(self._setup_root, 'virtualenv_setup',
+ 'requirements.txt')
cipd_bin = os.path.join(
self._pw_root,
diff --git a/env_setup/environment.py b/pw_env_setup/py/pw_env_setup/environment.py
similarity index 100%
rename from env_setup/environment.py
rename to pw_env_setup/py/pw_env_setup/environment.py
diff --git a/env_setup/environment_test.py b/pw_env_setup/py/pw_env_setup/environment_test.py
similarity index 98%
rename from env_setup/environment_test.py
rename to pw_env_setup/py/pw_env_setup/environment_test.py
index c15e78a..8d083ac 100644
--- a/env_setup/environment_test.py
+++ b/pw_env_setup/py/pw_env_setup/environment_test.py
@@ -29,9 +29,7 @@
import tempfile
import unittest
-# TODO(mohrr) remove import-error disabling, not sure why pylint has issues
-# with it.
-import environment # pylint: disable=import-error
+from pw_env_setup import environment
class WrittenEnvFailure(Exception):
diff --git a/env_setup/host_build_setup.py b/pw_env_setup/py/pw_env_setup/host_build_setup.py
similarity index 75%
rename from env_setup/host_build_setup.py
rename to pw_env_setup/py/pw_env_setup/host_build_setup.py
index 1bed6b2..b289136 100644
--- a/env_setup/host_build_setup.py
+++ b/pw_env_setup/py/pw_env_setup/host_build_setup.py
@@ -20,7 +20,10 @@
def install(pw_root, env):
host_dir = os.path.join(pw_root, 'out', 'host')
with env():
- subprocess.check_call(['gn', 'gen', host_dir], cwd=pw_root)
- subprocess.check_call(['ninja', '-C', host_dir], cwd=pw_root)
+ try:
+ subprocess.check_call(['gn', 'gen', host_dir], cwd=pw_root)
+ subprocess.check_call(['ninja', '-C', host_dir], cwd=pw_root)
+ except subprocess.CalledProcessError:
+ env.echo('warning: host tools failed to build')
env.prepend('PATH', os.path.join(host_dir, 'host_tools'))
diff --git a/env_setup/virtualenv_setup/__init__.py b/pw_env_setup/py/pw_env_setup/virtualenv_setup/__init__.py
similarity index 100%
rename from env_setup/virtualenv_setup/__init__.py
rename to pw_env_setup/py/pw_env_setup/virtualenv_setup/__init__.py
diff --git a/env_setup/virtualenv_setup/__main__.py b/pw_env_setup/py/pw_env_setup/virtualenv_setup/__main__.py
similarity index 84%
rename from env_setup/virtualenv_setup/__main__.py
rename to pw_env_setup/py/pw_env_setup/virtualenv_setup/__main__.py
index e5324c5..385252d 100644
--- a/env_setup/virtualenv_setup/__main__.py
+++ b/pw_env_setup/py/pw_env_setup/virtualenv_setup/__main__.py
@@ -17,9 +17,9 @@
import os
import sys
-# TODO(mohrr) remove import-error disabling, not sure why pylint has issues
-# with it.
-import install # pylint: disable=import-error
+# TODO(pwbug/67) switch back to 'from pw_env_setup import virtualenv_setup'.
+# from pw_env_setup import virtualenv_setup
+import install as virtualenv_setup # pylint: disable=import-error
def _main():
@@ -42,11 +42,7 @@
default=sys.executable,
help='Python to use when creating virtualenv.')
- try:
- install.install(**vars(parser.parse_args()))
- except install.GitRepoNotFound:
- print('git repository not found', file=sys.stderr)
- return -1
+ virtualenv_setup.install(**vars(parser.parse_args()))
return 0
diff --git a/env_setup/virtualenv_setup/install.py b/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py
similarity index 97%
rename from env_setup/virtualenv_setup/install.py
rename to pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py
index 13733b7..c861369 100644
--- a/env_setup/virtualenv_setup/install.py
+++ b/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py
@@ -101,9 +101,6 @@
setup_py_files = git_list_files('setup.py', '*/setup.py', cwd=pw_root)
- # Don't install the env_setup "module".
- setup_py_files = [x for x in setup_py_files if b'env_setup' not in x]
-
# If not forcing full setup, check if all expected packages are installed,
# ignoring versions. If they are, skip reinstalling.
if not full_envsetup:
diff --git a/env_setup/virtualenv_setup/requirements.in b/pw_env_setup/py/pw_env_setup/virtualenv_setup/requirements.in
similarity index 100%
rename from env_setup/virtualenv_setup/requirements.in
rename to pw_env_setup/py/pw_env_setup/virtualenv_setup/requirements.in
diff --git a/env_setup/virtualenv_setup/requirements.txt b/pw_env_setup/py/pw_env_setup/virtualenv_setup/requirements.txt
similarity index 100%
rename from env_setup/virtualenv_setup/requirements.txt
rename to pw_env_setup/py/pw_env_setup/virtualenv_setup/requirements.txt
diff --git a/env_setup/setup.py b/pw_env_setup/py/setup.py
similarity index 92%
rename from env_setup/setup.py
rename to pw_env_setup/py/setup.py
index c203f5f..9cd86f4 100644
--- a/env_setup/setup.py
+++ b/pw_env_setup/py/setup.py
@@ -30,7 +30,9 @@
description='Environment setup for Pigweed',
packages=setuptools.find_packages(),
test_suite='setup.test_suite',
- entry_points={'console_scripts': ['_pw_env_setup = env_setup:main']},
+ entry_points={
+ 'console_scripts': ['_pw_env_setup = pw_env_setup.env_setup:main'],
+ },
package_data={
'pw_env_setup': [
'cargo_setup/packages.txt',
diff --git a/env_setup/setup.sh b/pw_env_setup/setup.sh
similarity index 100%
rename from env_setup/setup.sh
rename to pw_env_setup/setup.sh
diff --git a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
index dc3408a..1377b42 100755
--- a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
+++ b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
@@ -51,9 +51,11 @@
#
def init_cipd(ctx: PresubmitContext):
# TODO(mohrr) invoke by importing rather than by subprocess.
- call(sys.executable,
- ctx.repository_root.joinpath('env_setup/cipd_setup/update.py'),
- '--install-dir', ctx.output_directory)
+ call(
+ sys.executable,
+ ctx.repository_root.joinpath('pw_env_setup', 'py', 'pw_env_setup',
+ 'cipd_setup', 'update.py'),
+ '--install-dir', ctx.output_directory)
paths = [ctx.output_directory, ctx.output_directory.joinpath('bin')]
for base in ctx.output_directory.glob('*'):
@@ -68,7 +70,8 @@
def init_virtualenv(ctx: PresubmitContext):
"""Set up virtualenv, assumes recent Python 3 is already installed."""
- virtualenv_source = ctx.repository_root.joinpath('env_setup',
+ virtualenv_source = ctx.repository_root.joinpath('pw_env_setup', 'py',
+ 'pw_env_setup',
'virtualenv_setup')
# For speed, don't build the venv if it exists. Use --clean to recreate it.
diff --git a/targets/stm32f429i-disc1/target_docs.rst b/targets/stm32f429i-disc1/target_docs.rst
index 906ce55..462bc87 100644
--- a/targets/stm32f429i-disc1/target_docs.rst
+++ b/targets/stm32f429i-disc1/target_docs.rst
@@ -45,7 +45,7 @@
.. code:: sh
# Setup pigweed environment.
- $ . env_setup/setup.sh
+ $ . pw_env_setup/setup.sh
# Run test.
$ stm32f429i_disc1_unit_test_runner /path/to/binary
@@ -61,7 +61,7 @@
.. code:: sh
# Setup Pigweed environment.
- $ . env_setup/setup.sh
+ $ . pw_env_setup/setup.sh
# Run test.
$ pw test --root out/disco/ --runner stm32f429i_disc1_unit_test_runner
