blob: 8ded3ab17bfed9a315f0c6682205970d0b7353c8 [file] [log] [blame]
Rob Mohr3b7da062020-04-30 14:30:57 -07001# Copyright 2020 The Pigweed Authors
Rob Mohrb671bc72019-12-13 08:36:15 -08002#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4# use this file except in compliance with the License. You may obtain a copy of
5# the License at
6#
7# https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations under
13# the License.
14"""Environment utility functions.
15
16Usage:
Rob Mohr500364a2023-02-01 15:05:30 +000017env = api.environment.initialize(checkout, env_options)
Rob Mohrfaa377f2022-01-19 14:38:36 -080018with env():
Rob Mohrb671bc72019-12-13 08:36:15 -080019 ...
20"""
21
Rob Mohr21f15eb2020-09-09 10:10:39 -070022import contextlib
Rob Mohr465feb32023-07-27 01:15:24 +000023import dataclasses
Rob Mohr59a9b1a2020-11-10 14:49:43 -080024import pprint
Rob Mohr465feb32023-07-27 01:15:24 +000025from typing import Any, Dict
Rob Mohr21f15eb2020-09-09 10:10:39 -070026
Rob Mohrfaa377f2022-01-19 14:38:36 -080027from PB.recipe_modules.pigweed.environment.options import Options
28
Rob Mohrb671bc72019-12-13 08:36:15 -080029import attr
Rob Mohr465feb32023-07-27 01:15:24 +000030from recipe_engine import config_types, recipe_api
Rob Mohrb671bc72019-12-13 08:36:15 -080031
32
Rob Mohr465feb32023-07-27 01:15:24 +000033@dataclasses.dataclass
Rob Mohr82c0e352022-07-27 20:57:40 +000034class Environment:
Rob Mohr465feb32023-07-27 01:15:24 +000035 api: Any
36 dir: config_types.Path
37 checkout: Any
38 prefixes: Dict = dataclasses.field(default_factory=dict)
39 suffixes: Dict = dataclasses.field(default_factory=dict)
40 env: Dict = dataclasses.field(default_factory=dict)
41 override_gn_args: Dict = dataclasses.field(default_factory=dict)
Rob Mohrfaa377f2022-01-19 14:38:36 -080042
43 @contextlib.contextmanager
44 def __call__(self):
45 # Using reversed() because things that are added later in environment
46 # setup need to override things that came earlier.
Rob Mohr465feb32023-07-27 01:15:24 +000047 with self.api.context(
Rob Mohrfaa377f2022-01-19 14:38:36 -080048 env_prefixes={k: reversed(v) for k, v in self.prefixes.items()},
49 env_suffixes=self.suffixes,
50 env=self.env,
51 ):
Rob Mohr465feb32023-07-27 01:15:24 +000052 with self.api.macos_sdk():
Rob Mohrfaa377f2022-01-19 14:38:36 -080053 yield self
54
55 def __getattr__(self, name):
56 if name not in self.env:
57 raise AttributeError(name)
58
59 return self.env.get(name)
Rob Mohrb671bc72019-12-13 08:36:15 -080060
61
62class EnvironmentApi(recipe_api.RecipeApi):
Rob Mohr57204602020-09-23 08:41:18 -070063 """Environment utility functions."""
Rob Mohrb671bc72019-12-13 08:36:15 -080064
Rob Mohrfaa377f2022-01-19 14:38:36 -080065 def _init_platform(self, env):
Rob Mohr57204602020-09-23 08:41:18 -070066 if self.m.platform.is_mac:
67 with self.m.step.nest('setup platform'):
68 with self.m.macos_sdk():
69 pass
Rob Mohr21f15eb2020-09-09 10:10:39 -070070
Rob Mohr9a397182022-07-12 02:00:24 +000071 def _init_misc_vars(self, env, additional_variables=None):
Rob Mohrfaa377f2022-01-19 14:38:36 -080072 env.env['PW_ENVIRONMENT_NO_ERROR_ON_UNRECOGNIZED'] = '1'
73 env.env['PW_ENVSETUP_DISABLE_SPINNER'] = '1'
74 env.env['PW_PRESUBMIT_DISABLE_SUBPROCESS_CAPTURE'] = '1'
Rob Mohr33bf5df2023-03-27 22:24:15 +000075 env.env['PW_USE_COLOR'] = ''
Rob Mohre440a9d2023-05-17 23:05:48 +000076 env.env['CLICOLOR'] = '0' # See https://bixense.com/clicolors/.
Rob Mohr8360e162021-01-13 08:46:28 -080077
Rob Mohr9a397182022-07-12 02:00:24 +000078 env.env.update(additional_variables or {})
79
Rob Mohra89b6ae2020-10-08 13:42:56 -070080 if self.m.led.launched_by_led:
Rob Mohre181c022022-09-16 18:11:17 +000081 # Not using self.m.buildbucket_util.id because some downstream
82 # projects need this id to never be longer than a typical
83 # buildbucket id. Shorter is fine.
84 env.env['BUILDBUCKET_ID'] = '0'
Rob Mohrfaa377f2022-01-19 14:38:36 -080085 env.env['BUILD_NUMBER'] = str(self.m.buildbucket.build.number)
Rob Mohr6042b342020-10-22 16:33:40 -070086
Rob Mohra89b6ae2020-10-08 13:42:56 -070087 else:
Rob Mohrfaa377f2022-01-19 14:38:36 -080088 env.env['BUILDBUCKET_ID'] = str(self.m.buildbucket.build.id)
89 env.env['BUILD_NUMBER'] = str(self.m.buildbucket.build.number)
Rob Mohra89b6ae2020-10-08 13:42:56 -070090
Rob Mohrfaa377f2022-01-19 14:38:36 -080091 env.env['BUILDBUCKET_NAME'] = ':'.join(
Rob Mohr7ab2baf2020-10-16 11:48:29 -070092 (
93 self.m.buildbucket.build.builder.project,
94 self.m.buildbucket.build.builder.bucket,
95 self.m.buildbucket.build.builder.builder,
96 )
97 )
Rob Mohra89b6ae2020-10-08 13:42:56 -070098
Rob Mohrfaa377f2022-01-19 14:38:36 -080099 if env.env['BUILDBUCKET_NAME'] == '::':
100 env.env['BUILDBUCKET_NAME'] = 'project:bucket:builder'
Rob Mohra89b6ae2020-10-08 13:42:56 -0700101
Rob Mohrfabf49c2023-04-11 20:08:52 +0000102 env.env['CTCACHE_DIR'] = self.m.path['cache'] / 'clang_tidy'
103 env.env['GOCACHE'] = self.m.path['cache'] / 'go'
104 env.env['PIP_CACHE_DIR'] = self.m.path['cache'] / 'pip'
105 env.env['TEST_TMPDIR'] = self.m.path['cache'] / 'bazel'
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800106
Rob Mohrb2d926c2023-02-01 15:07:42 +0000107 env.env['TRIGGERING_CHANGES_JSON'] = env.checkout.changes_json
108
Rob Mohrfaa377f2022-01-19 14:38:36 -0800109 def _init_pigweed(
110 self,
Rob Mohr500364a2023-02-01 15:05:30 +0000111 checkout,
Rob Mohrfaa377f2022-01-19 14:38:36 -0800112 top_presentation,
113 use_constraint_file,
114 pigweed_root,
Rob Mohr9794dc82023-07-22 00:19:43 +0000115 options,
Rob Mohrfaa377f2022-01-19 14:38:36 -0800116 env,
117 ):
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800118 """Run pw_env_setup."""
119
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800120 def path(relative_path):
121 parts = [
122 x for x in relative_path.split('/') if x not in ('.', u'.')
123 ]
124 if parts:
Rob Mohr500364a2023-02-01 15:05:30 +0000125 return checkout.root.join(*parts)
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800126 else:
Rob Mohr500364a2023-02-01 15:05:30 +0000127 return checkout.root # pragma: no cover
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800128
Rob Mohrfabf49c2023-04-11 20:08:52 +0000129 json_file = env.dir / 'vars.json'
130 shell_file = env.dir / 'setup.sh'
131 venv_dir = env.dir / 'venv'
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800132
Rob Mohr8afb1b42021-02-26 13:56:08 -0800133 self.m.file.ensure_directory(
Rob Mohr4806b6b2023-02-03 18:03:32 +0000134 f'mkdir {self.m.path.basename(env.dir)}', env.dir,
Rob Mohr8afb1b42021-02-26 13:56:08 -0800135 )
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800136
Rob Mohr8afb1b42021-02-26 13:56:08 -0800137 self.m.file.ensure_directory(
Rob Mohr4806b6b2023-02-03 18:03:32 +0000138 f'mkdir {self.m.path.basename(venv_dir)}', venv_dir,
Rob Mohr8afb1b42021-02-26 13:56:08 -0800139 )
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800140
Rob Mohr8afb1b42021-02-26 13:56:08 -0800141 cmd = [
Rob Mohrc7e06392021-04-15 14:10:28 -0700142 'python3',
Rob Mohrfabf49c2023-04-11 20:08:52 +0000143 (
144 pigweed_root
145 / 'pw_env_setup'
146 / 'py'
147 / 'pw_env_setup'
148 / 'env_setup.py'
Rob Mohrfaa377f2022-01-19 14:38:36 -0800149 ),
Rob Mohr8afb1b42021-02-26 13:56:08 -0800150 '--pw-root',
Rob Mohrfaa377f2022-01-19 14:38:36 -0800151 pigweed_root,
Rob Mohr8afb1b42021-02-26 13:56:08 -0800152 '--install-dir',
Rob Mohrdcc61c82022-08-16 18:01:54 +0000153 env.dir,
Rob Mohr8afb1b42021-02-26 13:56:08 -0800154 '--json-file',
155 json_file,
156 '--shell-file',
157 shell_file,
Rob Mohr21e3da82021-05-27 13:21:41 -0700158 '--virtualenv-gn-out-dir',
Rob Mohrfabf49c2023-04-11 20:08:52 +0000159 env.dir / 'out',
Rob Mohr47d3d5c2021-05-13 06:59:23 -0700160 '--use-existing-cipd',
Rob Mohr95ab0342021-05-27 10:38:22 -0700161 '--strict',
Rob Mohr8afb1b42021-02-26 13:56:08 -0800162 ]
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800163
Rob Mohr9794dc82023-07-22 00:19:43 +0000164 if options.skip_submodule_check:
Rob Mohr5b95c712022-06-13 13:22:21 -0700165 cmd.append('--skip-submodule-check')
166
Rob Mohr80408192021-09-23 07:59:12 -0700167 if not use_constraint_file:
168 cmd.append('--unpin-pip-packages')
169
Rob Mohr3534bfe2023-07-24 19:02:55 +0000170 for f in options.additional_cipd_files:
Rob Mohre4031732023-09-18 22:53:36 +0000171 f = f.replace('$PW_ROOT', str(pigweed_root))
Rob Mohr3534bfe2023-07-24 19:02:55 +0000172 cmd.extend(('--additional-cipd-file', f))
173
Rob Mohr36fd4632023-09-18 16:39:24 +0000174 cmd.extend(
175 ('--config-file', path(options.config_file or 'pigweed.json'))
176 )
Rob Mohr18893142021-02-26 07:26:49 -0800177
Rob Mohr4e7bea62022-07-28 14:19:31 +0000178 top_presentation.logs['config.json'] = pprint.pformat(
Rob Mohr9794dc82023-07-22 00:19:43 +0000179 self.m.file.read_json('read config', path(options.config_file))
Rob Mohr001b8a72021-06-16 09:22:58 -0700180 )
Rob Mohr8afb1b42021-02-26 13:56:08 -0800181
182 with self.m.step.defer_results():
Rob Mohr78f51822023-06-06 18:54:08 +0000183 with self.m.step.nest('run pw_env_setup') as pres:
Rob Mohr4a778e32022-09-28 15:05:54 +0000184 with env(), self.m.default_timeout():
Rob Mohr8afb1b42021-02-26 13:56:08 -0800185 self.m.step('pw_env_setup', cmd)
186
Rob Mohr9a0d6af2023-04-28 20:30:45 +0000187 self.m.file.listdir('ls', env.dir, recursive=True).get_result()
188
Rob Mohr78f51822023-06-06 18:54:08 +0000189 self.m.save_logs((env.dir,), pres=pres)
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800190
Rob Mohr8afb1b42021-02-26 13:56:08 -0800191 json_data = self.m.file.read_json(
192 'read json file',
193 json_file,
194 test_data={
195 'set': {'VIRTUAL_ENV': '/environment/virtualenv'},
196 'modify': {
197 'PATH': {'append': ['/environment/bin']},
198 'LD_LIBRARY_PATH': {'prepend': ['/environment/lib']},
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800199 },
Rob Mohr8afb1b42021-02-26 13:56:08 -0800200 },
201 )
Rob Mohrb1eff262022-09-07 16:58:47 +0000202 top_presentation.logs['vars.json'] = pprint.pformat(json_data)
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800203
Rob Mohrfabf49c2023-04-11 20:08:52 +0000204 env_gni_path = (
205 checkout.root / 'build_overrides' / 'pigweed_environment.gni'
Rob Mohr07314452022-08-16 18:22:52 +0000206 )
Rob Mohr07314452022-08-16 18:22:52 +0000207
Rob Mohrb1eff262022-09-07 16:58:47 +0000208 self.m.path.mock_add_file(env_gni_path)
209 if self.m.path.isfile(env_gni_path):
210 environment_gni = self.m.file.read_text(
211 'read gni file', env_gni_path
212 )
213 top_presentation.logs['pigweed_environment.gni'] = environment_gni
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800214
Rob Mohre3be66a2021-08-27 12:38:18 -0700215 for var, value in json_data['set'].items():
Rob Mohrfaa377f2022-01-19 14:38:36 -0800216 env.env[var] = value
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800217
Rob Mohre3be66a2021-08-27 12:38:18 -0700218 for var, actions in json_data['modify'].items():
Rob Mohr8afb1b42021-02-26 13:56:08 -0800219 for value in actions.get('prepend', ()):
Rob Mohrfaa377f2022-01-19 14:38:36 -0800220 env.prefixes.setdefault(var, [])
221 env.prefixes[var].append(value)
Rob Mohr8afb1b42021-02-26 13:56:08 -0800222 for value in actions.get('append', ()):
Rob Mohrfaa377f2022-01-19 14:38:36 -0800223 env.suffixes.setdefault(var, [])
224 env.suffixes[var].append(value)
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800225
Rob Mohrdc93dd92022-08-13 00:15:04 +0000226 def _toolchain_override(self, env):
227 """Checks for a toolchain override and applies it."""
228
229 # Using '$fuchsia/build' properties to simplify interface with the
230 # Fuchsia Toolchain team.
231 fuchsia_build_props = self.m.properties.thaw().get('$fuchsia/build', {})
232 toolchain_props = fuchsia_build_props.get('clang_toolchain', {})
233 if not toolchain_props:
234 return
235
236 with self.m.step.nest('toolchain override'):
237 with self.m.context(infra_steps=True):
Rob Mohrfabf49c2023-04-11 20:08:52 +0000238 toolchain_dir = env.dir / 'override' / 'clang_toolchain'
Rob Mohrdc93dd92022-08-13 00:15:04 +0000239
240 if toolchain_props['source'] == 'cipd':
241 pkgs = self.m.cipd.EnsureFile()
242 pkgs.add_package(
243 'fuchsia/third_party/clang/${platform}',
244 toolchain_props['version'],
245 )
Rob Mohrbf732212022-09-12 18:11:22 +0000246 self.m.cipd.ensure(toolchain_dir, pkgs)
Rob Mohrdc93dd92022-08-13 00:15:04 +0000247
248 elif toolchain_props['source'] in 'isolated':
Rob Mohr6da5ed82022-09-12 18:20:29 +0000249 with self.m.cas.with_instance(
250 'projects/chromium-swarm/instances/default_instance'
251 ):
252 self.m.cas.download(
253 'download',
254 digest=toolchain_props['version'],
255 output_dir=toolchain_dir,
256 )
Rob Mohrdc93dd92022-08-13 00:15:04 +0000257
258 else: # pragma: no cover
259 raise KeyError(
260 f'clang toolchain source {toolchain_props["source"]} '
261 'not recognized'
262 )
263
264 env.prefixes.setdefault('PATH', [])
Rob Mohrbf732212022-09-12 18:11:22 +0000265 env.prefixes['PATH'].append(toolchain_dir)
Rob Mohrfabf49c2023-04-11 20:08:52 +0000266 env.prefixes['PATH'].append(toolchain_dir / 'bin')
Rob Mohrbf732212022-09-12 18:11:22 +0000267
Rob Mohrfabf49c2023-04-11 20:08:52 +0000268 clang_prefix = toolchain_dir / 'bin'
Rob Mohrbf732212022-09-12 18:11:22 +0000269 env.override_gn_args[
270 'pw_toolchain_CLANG_PREFIX'
Rob Mohra65a1972022-09-13 16:25:39 +0000271 ] = f'{clang_prefix}/'
Rob Mohrdc93dd92022-08-13 00:15:04 +0000272
Rob Mohrfaa377f2022-01-19 14:38:36 -0800273 def init(
Rob Mohr500364a2023-02-01 15:05:30 +0000274 self, checkout, options=None, use_constraint_file=True,
Rob Mohrfaa377f2022-01-19 14:38:36 -0800275 ):
Rob Mohr500364a2023-02-01 15:05:30 +0000276 pigweed_root = checkout.root
Rob Mohrbf732212022-09-12 18:11:22 +0000277 env = Environment(
Rob Mohrfabf49c2023-04-11 20:08:52 +0000278 api=self.m, dir=checkout.root / 'environment', checkout=checkout,
Rob Mohrbf732212022-09-12 18:11:22 +0000279 )
Rob Mohre47c6582022-07-14 16:33:10 +0000280
Rob Mohre4475152022-11-16 20:04:36 +0000281 # If in recipe tests always add at least one variable to make it easier
282 # to test use of variables in recipes.
283 if self._test_data.enabled:
284 env.env['PW_TEST_VAR'] = 'test_value'
285
Rob Mohrfaa377f2022-01-19 14:38:36 -0800286 if not options.config_file:
Rob Mohrfaa377f2022-01-19 14:38:36 -0800287 return env
Rob Mohr3b7da062020-04-30 14:30:57 -0700288
Rob Mohrfaa377f2022-01-19 14:38:36 -0800289 if options.relative_pigweed_root not in (None, '', '.'):
Rob Mohrfabf49c2023-04-11 20:08:52 +0000290 pigweed_root = checkout.root / options.relative_pigweed_root
Rob Mohrfaa377f2022-01-19 14:38:36 -0800291
Rob Mohrfaa377f2022-01-19 14:38:36 -0800292 with self.m.step.nest('environment') as pres:
Rob Mohrfaa377f2022-01-19 14:38:36 -0800293 env.env['PW_ROOT'] = pigweed_root
Rob Mohr500364a2023-02-01 15:05:30 +0000294 env.env['PW_PROJECT_ROOT'] = checkout.root
Rob Mohrfaa377f2022-01-19 14:38:36 -0800295
296 self._init_platform(env)
Rob Mohr9a397182022-07-12 02:00:24 +0000297 self._init_misc_vars(env, options.additional_variables)
Rob Mohrfaa377f2022-01-19 14:38:36 -0800298 self._init_pigweed(
Rob Mohr500364a2023-02-01 15:05:30 +0000299 checkout=checkout,
Rob Mohrfaa377f2022-01-19 14:38:36 -0800300 top_presentation=pres,
301 use_constraint_file=use_constraint_file,
302 pigweed_root=pigweed_root,
Rob Mohr9794dc82023-07-22 00:19:43 +0000303 options=options,
Rob Mohrfaa377f2022-01-19 14:38:36 -0800304 env=env,
305 )
Rob Mohrdc93dd92022-08-13 00:15:04 +0000306 self._toolchain_override(env)
Rob Mohrfaa377f2022-01-19 14:38:36 -0800307
Rob Mohr322b7aa2022-09-07 19:45:05 +0000308 with env():
309 # If 'pw doctor' fails we can continue, but show the doctor
310 # failure in red in the UI.
311 try:
312 self.m.step('doctor', ['python', '-m', 'pw_cli', 'doctor'])
313 except self.m.step.StepFailure:
314 pass
315
Rob Mohrfaa377f2022-01-19 14:38:36 -0800316 return env