| # Copyright 2020 The Pigweed Authors |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may not |
| # use this file except in compliance with the License. You may obtain a copy of |
| # the License at |
| # |
| # https://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations under |
| # the License. |
| """Environment utility functions. |
| |
| Usage: |
| api.environment.initialize(checkout_root=...) |
| with api.environment(): |
| ... |
| """ |
| |
| import contextlib |
| import pprint |
| |
| import attr |
| from recipe_engine import recipe_api |
| |
| |
| @attr.s |
| class Package(object): |
| name = attr.ib(type=bytes) |
| version = attr.ib(type=bytes) |
| |
| |
| class EnvironmentApi(recipe_api.RecipeApi): |
| """Environment utility functions.""" |
| |
| def __init__(self, props, *args, **kwargs): |
| super(EnvironmentApi, self).__init__(*args, **kwargs) |
| self._root_variable_name = str(props.root_variable_name) |
| self._relative_pigweed_root = str(props.relative_pigweed_root) |
| self._config_file = str(props.config_file) |
| self._cipd_dir = None |
| self._prefixes = {} |
| self._suffixes = {} |
| self._env = {} |
| self._initialized = False |
| self._cipd_installation_dirs = [] |
| |
| def _init_platform(self): |
| if self.m.platform.is_mac: |
| with self.m.step.nest('setup platform'): |
| with self.m.macos_sdk(): |
| pass |
| |
| def _init_misc_vars(self): |
| self._env['PW_ENVIRONMENT_NO_ERROR_ON_UNRECOGNIZED'] = '1' |
| self._env['PW_ENVSETUP_DISABLE_SPINNER'] = '1' |
| self._env['PW_PRESUBMIT_DISABLE_SUBPROCESS_CAPTURE'] = '1' |
| |
| if self.m.led.launched_by_led: |
| # Not using self.m.buildbucket_util.id because having relatively |
| # consistent length is important to some downstream projects when |
| # pulling this into their builds. |
| self._env['BUILDBUCKET_ID'] = 'test:{}'.format( |
| self.m.swarming.task_id |
| ) |
| self._env['BUILD_NUMBER'] = str(self.m.buildbucket.build.number) |
| |
| else: |
| self._env['BUILDBUCKET_ID'] = str(self.m.buildbucket.build.id) |
| self._env['BUILD_NUMBER'] = str(self.m.buildbucket.build.number) |
| |
| self._env['BUILDBUCKET_NAME'] = ':'.join( |
| ( |
| self.m.buildbucket.build.builder.project, |
| self.m.buildbucket.build.builder.bucket, |
| self.m.buildbucket.build.builder.builder, |
| ) |
| ) |
| |
| if self._env['BUILDBUCKET_NAME'] == '::': |
| self._env['BUILDBUCKET_NAME'] = 'project:bucket:builder' |
| |
| self._env['GOCACHE'] = self.m.path['cache'].join('go') |
| self._env['PIP_CACHE_DIR'] = self.m.path['cache'].join('pip') |
| # Bazel cache dir. |
| self._env['TEST_TMPDIR'] = self.m.path['cache'].join('bazel') |
| |
| def _init_pigweed(self, checkout_root, top_pres, use_constraint_file): |
| """Run pw_env_setup.""" |
| |
| if not self._config_file: |
| return |
| |
| def path(relative_path): |
| parts = [ |
| x for x in relative_path.split('/') if x not in ('.', u'.') |
| ] |
| if parts: |
| return checkout_root.join(*parts) |
| else: |
| return checkout_root # pragma: no cover |
| |
| pw_root = path(self._relative_pigweed_root) |
| |
| env_dir = self.m.path['start_dir'].join('environment') |
| json_file = env_dir.join('vars.json') |
| shell_file = env_dir.join('setup.sh') |
| venv_dir = env_dir.join('venv') |
| |
| self.m.file.ensure_directory( |
| 'mkdir {}'.format(self.m.path.basename(env_dir)), env_dir, |
| ) |
| |
| self.m.file.ensure_directory( |
| 'mkdir {}'.format(self.m.path.basename(venv_dir)), venv_dir, |
| ) |
| |
| cmd = [ |
| 'python3', |
| pw_root.join('pw_env_setup', 'py', 'pw_env_setup', 'env_setup.py'), |
| '--pw-root', |
| pw_root, |
| '--install-dir', |
| env_dir, |
| '--json-file', |
| json_file, |
| '--shell-file', |
| shell_file, |
| '--virtualenv-root', |
| venv_dir, |
| '--virtualenv-gn-out-dir', |
| self.m.build.dir, |
| '--use-existing-cipd', |
| '--strict', |
| ] |
| |
| if not use_constraint_file: |
| cmd.append('--unpin-pip-packages') |
| |
| cmd.extend(('--config-file', path(self._config_file))) |
| |
| top_pres.logs['vars.json'] = pprint.pformat( |
| self.m.file.read_json('read config', path(self._config_file)) |
| ) |
| |
| with self.m.step.defer_results(): |
| with self.m.step.nest('run pw_env_setup'): |
| with self(): |
| self.m.step('pw_env_setup', cmd) |
| |
| cipd_dir = env_dir.join('cipd') |
| |
| log_exts = ('.cfg', '.log', '.sh', '.bat', '.json', '.ensure') |
| |
| for directory in (env_dir, cipd_dir, venv_dir): |
| if not self.m.path.isdir(directory): |
| continue |
| |
| with self.m.step.nest(self.m.path.basename(directory)): |
| files = self.m.file.listdir( |
| 'ls', directory |
| ).get_result() |
| |
| for entry in files: |
| ext = self.m.path.splitext(entry)[1] |
| if ext not in log_exts: |
| continue |
| |
| self.m.file.read_text( |
| 'read {}'.format(self.m.path.basename(entry)), |
| entry, |
| ) |
| |
| json_data = self.m.file.read_json( |
| 'read json file', |
| json_file, |
| test_data={ |
| 'set': {'VIRTUAL_ENV': '/environment/virtualenv'}, |
| 'modify': { |
| 'PATH': {'append': ['/environment/bin']}, |
| 'LD_LIBRARY_PATH': {'prepend': ['/environment/lib']}, |
| }, |
| }, |
| ) |
| |
| top_pres.logs['vars.json'] = pprint.pformat(json_data) |
| |
| for var, value in json_data['set'].items(): |
| self._env[var] = value |
| |
| for var, actions in json_data['modify'].items(): |
| for value in actions.get('prepend', ()): |
| self._prefixes.setdefault(var, []) |
| self._prefixes[var].append(value) |
| for value in actions.get('append', ()): |
| self._suffixes.setdefault(var, []) |
| self._suffixes[var].append(value) |
| |
| def init(self, checkout_root, use_constraint_file=True): |
| pigweed_root = checkout_root |
| if self._relative_pigweed_root not in (None, '', '.'): |
| pigweed_root = checkout_root.join(self._relative_pigweed_root) |
| |
| if self._root_variable_name: |
| self._env[self._root_variable_name] = checkout_root |
| |
| if not self._initialized: |
| with self.m.step.nest('environment') as pres: |
| # Setting _initialized immediately because some setup steps need |
| # to use the context of previous steps, and invoking self() is |
| # the easiest way to do so. |
| self._initialized = True |
| self._env['PW_ROOT'] = pigweed_root |
| self._env['PW_PROJECT_ROOT'] = checkout_root |
| |
| self._init_platform() |
| self._init_misc_vars() |
| self._init_pigweed(checkout_root, pres, use_constraint_file) |
| |
| @contextlib.contextmanager |
| def __call__(self): |
| assert self._initialized |
| |
| # Using reversed() because things that are added later in environment |
| # setup need to override things that came earlier. |
| with self.m.context( |
| env_prefixes={k: reversed(v) for k, v in self._prefixes.items()}, |
| env_suffixes=self._suffixes, |
| env=self._env, |
| ): |
| with self.m.macos_sdk(): |
| yield self |
| |
| def __getattr__(self, name): |
| if name not in self._env: |
| raise AttributeError(name) |
| |
| return self._env.get(name) |
