Rob Mohr | 3b7da06 | 2020-04-30 14:30:57 -0700 | [diff] [blame] | 1 | # Copyright 2020 The Pigweed Authors |
Rob Mohr | b671bc7 | 2019-12-13 08:36:15 -0800 | [diff] [blame] | 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not |
| 4 | # use this file except in compliance with the License. You may obtain a copy of |
| 5 | # the License at |
| 6 | # |
| 7 | # https://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 12 | # License for the specific language governing permissions and limitations under |
| 13 | # the License. |
| 14 | """Environment utility functions. |
| 15 | |
| 16 | Usage: |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 17 | env = api.environment.initialize(checkout_root, env_options) |
| 18 | with env(): |
Rob Mohr | b671bc7 | 2019-12-13 08:36:15 -0800 | [diff] [blame] | 19 | ... |
| 20 | """ |
| 21 | |
Rob Mohr | 21f15eb | 2020-09-09 10:10:39 -0700 | [diff] [blame] | 22 | import contextlib |
Rob Mohr | 59a9b1a | 2020-11-10 14:49:43 -0800 | [diff] [blame] | 23 | import pprint |
Rob Mohr | 21f15eb | 2020-09-09 10:10:39 -0700 | [diff] [blame] | 24 | |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 25 | from PB.recipe_modules.pigweed.environment.options import Options |
| 26 | |
Rob Mohr | b671bc7 | 2019-12-13 08:36:15 -0800 | [diff] [blame] | 27 | import attr |
| 28 | from recipe_engine import recipe_api |
| 29 | |
| 30 | |
| 31 | @attr.s |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 32 | class Environment(object): |
| 33 | _api = attr.ib() |
| 34 | prefixes = attr.ib(default=attr.Factory(dict)) |
| 35 | suffixes = attr.ib(default=attr.Factory(dict)) |
| 36 | env = attr.ib(default=attr.Factory(dict)) |
| 37 | |
| 38 | @contextlib.contextmanager |
| 39 | def __call__(self): |
| 40 | # Using reversed() because things that are added later in environment |
| 41 | # setup need to override things that came earlier. |
| 42 | with self._api.context( |
| 43 | env_prefixes={k: reversed(v) for k, v in self.prefixes.items()}, |
| 44 | env_suffixes=self.suffixes, |
| 45 | env=self.env, |
| 46 | ): |
| 47 | with self._api.macos_sdk(): |
| 48 | yield self |
| 49 | |
| 50 | def __getattr__(self, name): |
| 51 | if name not in self.env: |
| 52 | raise AttributeError(name) |
| 53 | |
| 54 | return self.env.get(name) |
Rob Mohr | b671bc7 | 2019-12-13 08:36:15 -0800 | [diff] [blame] | 55 | |
| 56 | |
| 57 | class EnvironmentApi(recipe_api.RecipeApi): |
Rob Mohr | 5720460 | 2020-09-23 08:41:18 -0700 | [diff] [blame] | 58 | """Environment utility functions.""" |
Rob Mohr | b671bc7 | 2019-12-13 08:36:15 -0800 | [diff] [blame] | 59 | |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 60 | def _init_platform(self, env): |
Rob Mohr | 5720460 | 2020-09-23 08:41:18 -0700 | [diff] [blame] | 61 | if self.m.platform.is_mac: |
| 62 | with self.m.step.nest('setup platform'): |
| 63 | with self.m.macos_sdk(): |
| 64 | pass |
Rob Mohr | 21f15eb | 2020-09-09 10:10:39 -0700 | [diff] [blame] | 65 | |
Rob Mohr | 9a39718 | 2022-07-12 02:00:24 +0000 | [diff] [blame] | 66 | def _init_misc_vars(self, env, additional_variables=None): |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 67 | env.env['PW_ENVIRONMENT_NO_ERROR_ON_UNRECOGNIZED'] = '1' |
| 68 | env.env['PW_ENVSETUP_DISABLE_SPINNER'] = '1' |
| 69 | env.env['PW_PRESUBMIT_DISABLE_SUBPROCESS_CAPTURE'] = '1' |
Rob Mohr | 8360e16 | 2021-01-13 08:46:28 -0800 | [diff] [blame] | 70 | |
Rob Mohr | 9a39718 | 2022-07-12 02:00:24 +0000 | [diff] [blame] | 71 | env.env.update(additional_variables or {}) |
| 72 | |
Rob Mohr | a89b6ae | 2020-10-08 13:42:56 -0700 | [diff] [blame] | 73 | if self.m.led.launched_by_led: |
| 74 | # Not using self.m.buildbucket_util.id because having relatively |
| 75 | # consistent length is important to some downstream projects when |
| 76 | # pulling this into their builds. |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 77 | env.env['BUILDBUCKET_ID'] = 'test:{}'.format( |
Rob Mohr | 7ab2baf | 2020-10-16 11:48:29 -0700 | [diff] [blame] | 78 | self.m.swarming.task_id |
| 79 | ) |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 80 | env.env['BUILD_NUMBER'] = str(self.m.buildbucket.build.number) |
Rob Mohr | 6042b34 | 2020-10-22 16:33:40 -0700 | [diff] [blame] | 81 | |
Rob Mohr | a89b6ae | 2020-10-08 13:42:56 -0700 | [diff] [blame] | 82 | else: |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 83 | env.env['BUILDBUCKET_ID'] = str(self.m.buildbucket.build.id) |
| 84 | env.env['BUILD_NUMBER'] = str(self.m.buildbucket.build.number) |
Rob Mohr | a89b6ae | 2020-10-08 13:42:56 -0700 | [diff] [blame] | 85 | |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 86 | env.env['BUILDBUCKET_NAME'] = ':'.join( |
Rob Mohr | 7ab2baf | 2020-10-16 11:48:29 -0700 | [diff] [blame] | 87 | ( |
| 88 | self.m.buildbucket.build.builder.project, |
| 89 | self.m.buildbucket.build.builder.bucket, |
| 90 | self.m.buildbucket.build.builder.builder, |
| 91 | ) |
| 92 | ) |
Rob Mohr | a89b6ae | 2020-10-08 13:42:56 -0700 | [diff] [blame] | 93 | |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 94 | if env.env['BUILDBUCKET_NAME'] == '::': |
| 95 | env.env['BUILDBUCKET_NAME'] = 'project:bucket:builder' |
Rob Mohr | a89b6ae | 2020-10-08 13:42:56 -0700 | [diff] [blame] | 96 | |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 97 | env.env['GOCACHE'] = self.m.path['cache'].join('go') |
| 98 | env.env['PIP_CACHE_DIR'] = self.m.path['cache'].join('pip') |
Rob Mohr | d9b0533 | 2021-09-15 10:57:31 -0700 | [diff] [blame] | 99 | # Bazel cache dir. |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 100 | env.env['TEST_TMPDIR'] = self.m.path['cache'].join('bazel') |
Rob Mohr | 59a9b1a | 2020-11-10 14:49:43 -0800 | [diff] [blame] | 101 | |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 102 | def _init_pigweed( |
| 103 | self, |
| 104 | checkout_root, |
Rob Mohr | e47c658 | 2022-07-14 16:33:10 +0000 | [diff] [blame^] | 105 | env_dir, |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 106 | top_presentation, |
| 107 | use_constraint_file, |
| 108 | pigweed_root, |
| 109 | config_file, |
Rob Mohr | 5b95c71 | 2022-06-13 13:22:21 -0700 | [diff] [blame] | 110 | skip_submodule_check, |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 111 | env, |
| 112 | ): |
Rob Mohr | 59a9b1a | 2020-11-10 14:49:43 -0800 | [diff] [blame] | 113 | """Run pw_env_setup.""" |
| 114 | |
Rob Mohr | 59a9b1a | 2020-11-10 14:49:43 -0800 | [diff] [blame] | 115 | def path(relative_path): |
| 116 | parts = [ |
| 117 | x for x in relative_path.split('/') if x not in ('.', u'.') |
| 118 | ] |
| 119 | if parts: |
| 120 | return checkout_root.join(*parts) |
| 121 | else: |
| 122 | return checkout_root # pragma: no cover |
| 123 | |
Rob Mohr | 8afb1b4 | 2021-02-26 13:56:08 -0800 | [diff] [blame] | 124 | json_file = env_dir.join('vars.json') |
| 125 | shell_file = env_dir.join('setup.sh') |
| 126 | venv_dir = env_dir.join('venv') |
Rob Mohr | 59a9b1a | 2020-11-10 14:49:43 -0800 | [diff] [blame] | 127 | |
Rob Mohr | 8afb1b4 | 2021-02-26 13:56:08 -0800 | [diff] [blame] | 128 | self.m.file.ensure_directory( |
| 129 | 'mkdir {}'.format(self.m.path.basename(env_dir)), env_dir, |
| 130 | ) |
Rob Mohr | 59a9b1a | 2020-11-10 14:49:43 -0800 | [diff] [blame] | 131 | |
Rob Mohr | 8afb1b4 | 2021-02-26 13:56:08 -0800 | [diff] [blame] | 132 | self.m.file.ensure_directory( |
| 133 | 'mkdir {}'.format(self.m.path.basename(venv_dir)), venv_dir, |
| 134 | ) |
Rob Mohr | 59a9b1a | 2020-11-10 14:49:43 -0800 | [diff] [blame] | 135 | |
Rob Mohr | 8afb1b4 | 2021-02-26 13:56:08 -0800 | [diff] [blame] | 136 | cmd = [ |
Rob Mohr | c7e0639 | 2021-04-15 14:10:28 -0700 | [diff] [blame] | 137 | 'python3', |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 138 | pigweed_root.join( |
| 139 | 'pw_env_setup', 'py', 'pw_env_setup', 'env_setup.py' |
| 140 | ), |
Rob Mohr | 8afb1b4 | 2021-02-26 13:56:08 -0800 | [diff] [blame] | 141 | '--pw-root', |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 142 | pigweed_root, |
Rob Mohr | 8afb1b4 | 2021-02-26 13:56:08 -0800 | [diff] [blame] | 143 | '--install-dir', |
| 144 | env_dir, |
| 145 | '--json-file', |
| 146 | json_file, |
| 147 | '--shell-file', |
| 148 | shell_file, |
Rob Mohr | 21e3da8 | 2021-05-27 13:21:41 -0700 | [diff] [blame] | 149 | '--virtualenv-gn-out-dir', |
| 150 | self.m.build.dir, |
Rob Mohr | 47d3d5c | 2021-05-13 06:59:23 -0700 | [diff] [blame] | 151 | '--use-existing-cipd', |
Rob Mohr | 95ab034 | 2021-05-27 10:38:22 -0700 | [diff] [blame] | 152 | '--strict', |
Rob Mohr | 8afb1b4 | 2021-02-26 13:56:08 -0800 | [diff] [blame] | 153 | ] |
Rob Mohr | 59a9b1a | 2020-11-10 14:49:43 -0800 | [diff] [blame] | 154 | |
Rob Mohr | 5b95c71 | 2022-06-13 13:22:21 -0700 | [diff] [blame] | 155 | if skip_submodule_check: |
| 156 | cmd.append('--skip-submodule-check') |
| 157 | |
Rob Mohr | 8040819 | 2021-09-23 07:59:12 -0700 | [diff] [blame] | 158 | if not use_constraint_file: |
| 159 | cmd.append('--unpin-pip-packages') |
| 160 | |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 161 | cmd.extend(('--config-file', path(config_file))) |
Rob Mohr | 1889314 | 2021-02-26 07:26:49 -0800 | [diff] [blame] | 162 | |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 163 | top_presentation.logs['vars.json'] = pprint.pformat( |
| 164 | self.m.file.read_json('read config', path(config_file)) |
Rob Mohr | 001b8a7 | 2021-06-16 09:22:58 -0700 | [diff] [blame] | 165 | ) |
Rob Mohr | 8afb1b4 | 2021-02-26 13:56:08 -0800 | [diff] [blame] | 166 | |
| 167 | with self.m.step.defer_results(): |
| 168 | with self.m.step.nest('run pw_env_setup'): |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 169 | with env(): |
Rob Mohr | 8afb1b4 | 2021-02-26 13:56:08 -0800 | [diff] [blame] | 170 | self.m.step('pw_env_setup', cmd) |
| 171 | |
Rob Mohr | 4de0659 | 2021-06-17 11:21:49 -0700 | [diff] [blame] | 172 | cipd_dir = env_dir.join('cipd') |
Rob Mohr | 8afb1b4 | 2021-02-26 13:56:08 -0800 | [diff] [blame] | 173 | |
Rob Mohr | 4de0659 | 2021-06-17 11:21:49 -0700 | [diff] [blame] | 174 | log_exts = ('.cfg', '.log', '.sh', '.bat', '.json', '.ensure') |
Rob Mohr | 8afb1b4 | 2021-02-26 13:56:08 -0800 | [diff] [blame] | 175 | |
Rob Mohr | 3445395 | 2021-08-04 06:46:54 -0700 | [diff] [blame] | 176 | for directory in (env_dir, cipd_dir, venv_dir): |
| 177 | if not self.m.path.isdir(directory): |
Rob Mohr | 4cd5a20 | 2021-07-16 08:44:15 -0700 | [diff] [blame] | 178 | continue |
| 179 | |
Rob Mohr | 3445395 | 2021-08-04 06:46:54 -0700 | [diff] [blame] | 180 | with self.m.step.nest(self.m.path.basename(directory)): |
| 181 | files = self.m.file.listdir( |
| 182 | 'ls', directory |
| 183 | ).get_result() |
Rob Mohr | 8afb1b4 | 2021-02-26 13:56:08 -0800 | [diff] [blame] | 184 | |
Rob Mohr | 4de0659 | 2021-06-17 11:21:49 -0700 | [diff] [blame] | 185 | for entry in files: |
| 186 | ext = self.m.path.splitext(entry)[1] |
| 187 | if ext not in log_exts: |
| 188 | continue |
| 189 | |
| 190 | self.m.file.read_text( |
| 191 | 'read {}'.format(self.m.path.basename(entry)), |
| 192 | entry, |
| 193 | ) |
Rob Mohr | 59a9b1a | 2020-11-10 14:49:43 -0800 | [diff] [blame] | 194 | |
Rob Mohr | 8afb1b4 | 2021-02-26 13:56:08 -0800 | [diff] [blame] | 195 | json_data = self.m.file.read_json( |
| 196 | 'read json file', |
| 197 | json_file, |
| 198 | test_data={ |
| 199 | 'set': {'VIRTUAL_ENV': '/environment/virtualenv'}, |
| 200 | 'modify': { |
| 201 | 'PATH': {'append': ['/environment/bin']}, |
| 202 | 'LD_LIBRARY_PATH': {'prepend': ['/environment/lib']}, |
Rob Mohr | 59a9b1a | 2020-11-10 14:49:43 -0800 | [diff] [blame] | 203 | }, |
Rob Mohr | 8afb1b4 | 2021-02-26 13:56:08 -0800 | [diff] [blame] | 204 | }, |
| 205 | ) |
Rob Mohr | 59a9b1a | 2020-11-10 14:49:43 -0800 | [diff] [blame] | 206 | |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 207 | top_presentation.logs['vars.json'] = pprint.pformat(json_data) |
Rob Mohr | 59a9b1a | 2020-11-10 14:49:43 -0800 | [diff] [blame] | 208 | |
Rob Mohr | e3be66a | 2021-08-27 12:38:18 -0700 | [diff] [blame] | 209 | for var, value in json_data['set'].items(): |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 210 | env.env[var] = value |
Rob Mohr | 59a9b1a | 2020-11-10 14:49:43 -0800 | [diff] [blame] | 211 | |
Rob Mohr | e3be66a | 2021-08-27 12:38:18 -0700 | [diff] [blame] | 212 | for var, actions in json_data['modify'].items(): |
Rob Mohr | 8afb1b4 | 2021-02-26 13:56:08 -0800 | [diff] [blame] | 213 | for value in actions.get('prepend', ()): |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 214 | env.prefixes.setdefault(var, []) |
| 215 | env.prefixes[var].append(value) |
Rob Mohr | 8afb1b4 | 2021-02-26 13:56:08 -0800 | [diff] [blame] | 216 | for value in actions.get('append', ()): |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 217 | env.suffixes.setdefault(var, []) |
| 218 | env.suffixes[var].append(value) |
Rob Mohr | 59a9b1a | 2020-11-10 14:49:43 -0800 | [diff] [blame] | 219 | |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 220 | def init( |
| 221 | self, checkout_root, options=None, use_constraint_file=True, |
| 222 | ): |
Rob Mohr | 59a9b1a | 2020-11-10 14:49:43 -0800 | [diff] [blame] | 223 | pigweed_root = checkout_root |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 224 | env = Environment(api=self.m) |
Rob Mohr | 59a9b1a | 2020-11-10 14:49:43 -0800 | [diff] [blame] | 225 | |
Rob Mohr | e47c658 | 2022-07-14 16:33:10 +0000 | [diff] [blame^] | 226 | env_dir = checkout_root.join('environment') |
| 227 | |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 228 | if not options.config_file: |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 229 | return env |
Rob Mohr | 3b7da06 | 2020-04-30 14:30:57 -0700 | [diff] [blame] | 230 | |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 231 | if options.relative_pigweed_root not in (None, '', '.'): |
| 232 | pigweed_root = checkout_root.join(options.relative_pigweed_root) |
| 233 | |
| 234 | if options.root_variable_name: |
| 235 | env.env[options.root_variable_name] = checkout_root |
| 236 | |
| 237 | with self.m.step.nest('environment') as pres: |
| 238 | # Setting _initialized immediately because some setup steps need |
| 239 | # to use the context of previous steps, and invoking self() is |
| 240 | # the easiest way to do so. |
| 241 | env.env['PW_ROOT'] = pigweed_root |
| 242 | env.env['PW_PROJECT_ROOT'] = checkout_root |
| 243 | |
| 244 | self._init_platform(env) |
Rob Mohr | 9a39718 | 2022-07-12 02:00:24 +0000 | [diff] [blame] | 245 | self._init_misc_vars(env, options.additional_variables) |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 246 | self._init_pigweed( |
| 247 | checkout_root=checkout_root, |
Rob Mohr | e47c658 | 2022-07-14 16:33:10 +0000 | [diff] [blame^] | 248 | env_dir=env_dir, |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 249 | top_presentation=pres, |
| 250 | use_constraint_file=use_constraint_file, |
| 251 | pigweed_root=pigweed_root, |
| 252 | config_file=options.config_file, |
Rob Mohr | 5b95c71 | 2022-06-13 13:22:21 -0700 | [diff] [blame] | 253 | skip_submodule_check=options.skip_submodule_check, |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 254 | env=env, |
| 255 | ) |
| 256 | |
Rob Mohr | faa377f | 2022-01-19 14:38:36 -0800 | [diff] [blame] | 257 | return env |