blob: 3d6d74be553f8163b2c2fe4ae692ee18b41651aa [file] [log] [blame]
Rob Mohr3b7da062020-04-30 14:30:57 -07001# Copyright 2020 The Pigweed Authors
Rob Mohrb671bc72019-12-13 08:36:15 -08002#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4# use this file except in compliance with the License. You may obtain a copy of
5# the License at
6#
7# https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations under
13# the License.
14"""Environment utility functions.
15
16Usage:
Rob Mohr500364a2023-02-01 15:05:30 +000017env = api.environment.initialize(checkout, env_options)
Rob Mohrfaa377f2022-01-19 14:38:36 -080018with env():
Rob Mohrb671bc72019-12-13 08:36:15 -080019 ...
20"""
21
Rob Mohr21f15eb2020-09-09 10:10:39 -070022import contextlib
Rob Mohr465feb32023-07-27 01:15:24 +000023import dataclasses
Rob Mohr59a9b1a2020-11-10 14:49:43 -080024import pprint
Rob Mohr465feb32023-07-27 01:15:24 +000025from typing import Any, Dict
Rob Mohr21f15eb2020-09-09 10:10:39 -070026
Rob Mohrfaa377f2022-01-19 14:38:36 -080027from PB.recipe_modules.pigweed.environment.options import Options
28
Rob Mohr465feb32023-07-27 01:15:24 +000029from recipe_engine import config_types, recipe_api
Rob Mohrb671bc72019-12-13 08:36:15 -080030
31
Rob Mohr465feb32023-07-27 01:15:24 +000032@dataclasses.dataclass
Rob Mohr82c0e352022-07-27 20:57:40 +000033class Environment:
Rob Mohr465feb32023-07-27 01:15:24 +000034 api: Any
35 dir: config_types.Path
36 checkout: Any
37 prefixes: Dict = dataclasses.field(default_factory=dict)
38 suffixes: Dict = dataclasses.field(default_factory=dict)
39 env: Dict = dataclasses.field(default_factory=dict)
40 override_gn_args: Dict = dataclasses.field(default_factory=dict)
Rob Mohrfaa377f2022-01-19 14:38:36 -080041
42 @contextlib.contextmanager
43 def __call__(self):
44 # Using reversed() because things that are added later in environment
45 # setup need to override things that came earlier.
Rob Mohr465feb32023-07-27 01:15:24 +000046 with self.api.context(
Rob Mohrfaa377f2022-01-19 14:38:36 -080047 env_prefixes={k: reversed(v) for k, v in self.prefixes.items()},
48 env_suffixes=self.suffixes,
49 env=self.env,
50 ):
Rob Mohr465feb32023-07-27 01:15:24 +000051 with self.api.macos_sdk():
Rob Mohrfaa377f2022-01-19 14:38:36 -080052 yield self
53
54 def __getattr__(self, name):
55 if name not in self.env:
56 raise AttributeError(name)
57
58 return self.env.get(name)
Rob Mohrb671bc72019-12-13 08:36:15 -080059
60
Rob Mohr0567a302023-09-19 16:19:42 +000061def path(relative_path, checkout):
62 parts = [x for x in relative_path.split('/') if x not in ('.', u'.')]
63 if parts:
64 return checkout.root.join(*parts)
65 else:
66 return checkout.root # pragma: no cover
67
68
Rob Mohrb671bc72019-12-13 08:36:15 -080069class EnvironmentApi(recipe_api.RecipeApi):
Rob Mohr57204602020-09-23 08:41:18 -070070 """Environment utility functions."""
Rob Mohrb671bc72019-12-13 08:36:15 -080071
Rob Mohrd260a4e2023-11-13 19:00:16 +000072 Environment = Environment
73
Rob Mohrfaa377f2022-01-19 14:38:36 -080074 def _init_platform(self, env):
Rob Mohr57204602020-09-23 08:41:18 -070075 if self.m.platform.is_mac:
76 with self.m.step.nest('setup platform'):
77 with self.m.macos_sdk():
78 pass
Rob Mohr21f15eb2020-09-09 10:10:39 -070079
Rob Mohr9a397182022-07-12 02:00:24 +000080 def _init_misc_vars(self, env, additional_variables=None):
Rob Mohrfaa377f2022-01-19 14:38:36 -080081 env.env['PW_ENVIRONMENT_NO_ERROR_ON_UNRECOGNIZED'] = '1'
82 env.env['PW_ENVSETUP_DISABLE_SPINNER'] = '1'
83 env.env['PW_PRESUBMIT_DISABLE_SUBPROCESS_CAPTURE'] = '1'
Rob Mohr33bf5df2023-03-27 22:24:15 +000084 env.env['PW_USE_COLOR'] = ''
Rob Mohrc00a2622023-10-18 23:49:02 +000085 env.env['CLICOLOR'] = '0' # Formerly on https://bixense.com/clicolors/.
86 env.env['NO_COLOR'] = '1' # See https://no-color.org.
Rob Mohrd62d6a12023-09-26 18:42:22 +000087 # This should tell ninja to disable colors based on implementation at
88 # https://github.com/ninja-build/ninja/blob/master/src/line_printer.cc#L60.
89 env.env['CLICOLOR_FORCE'] = '0'
Rob Mohr8360e162021-01-13 08:46:28 -080090
Rob Mohr9a397182022-07-12 02:00:24 +000091 env.env.update(additional_variables or {})
92
Rob Mohra89b6ae2020-10-08 13:42:56 -070093 if self.m.led.launched_by_led:
Rob Mohre181c022022-09-16 18:11:17 +000094 # Not using self.m.buildbucket_util.id because some downstream
95 # projects need this id to never be longer than a typical
96 # buildbucket id. Shorter is fine.
97 env.env['BUILDBUCKET_ID'] = '0'
Rob Mohrfaa377f2022-01-19 14:38:36 -080098 env.env['BUILD_NUMBER'] = str(self.m.buildbucket.build.number)
Rob Mohr6042b342020-10-22 16:33:40 -070099
Rob Mohra89b6ae2020-10-08 13:42:56 -0700100 else:
Rob Mohrfaa377f2022-01-19 14:38:36 -0800101 env.env['BUILDBUCKET_ID'] = str(self.m.buildbucket.build.id)
102 env.env['BUILD_NUMBER'] = str(self.m.buildbucket.build.number)
Rob Mohra89b6ae2020-10-08 13:42:56 -0700103
Rob Mohrfaa377f2022-01-19 14:38:36 -0800104 env.env['BUILDBUCKET_NAME'] = ':'.join(
Rob Mohr7ab2baf2020-10-16 11:48:29 -0700105 (
106 self.m.buildbucket.build.builder.project,
107 self.m.buildbucket.build.builder.bucket,
108 self.m.buildbucket.build.builder.builder,
109 )
110 )
Rob Mohra89b6ae2020-10-08 13:42:56 -0700111
Rob Mohrfaa377f2022-01-19 14:38:36 -0800112 if env.env['BUILDBUCKET_NAME'] == '::':
113 env.env['BUILDBUCKET_NAME'] = 'project:bucket:builder'
Rob Mohra89b6ae2020-10-08 13:42:56 -0700114
Rob Mohrfabf49c2023-04-11 20:08:52 +0000115 env.env['CTCACHE_DIR'] = self.m.path['cache'] / 'clang_tidy'
116 env.env['GOCACHE'] = self.m.path['cache'] / 'go'
117 env.env['PIP_CACHE_DIR'] = self.m.path['cache'] / 'pip'
118 env.env['TEST_TMPDIR'] = self.m.path['cache'] / 'bazel'
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800119
Rob Mohrb2d926c2023-02-01 15:07:42 +0000120 env.env['TRIGGERING_CHANGES_JSON'] = env.checkout.changes_json
121
Rob Mohrfaa377f2022-01-19 14:38:36 -0800122 def _init_pigweed(
123 self,
Rob Mohr500364a2023-02-01 15:05:30 +0000124 checkout,
Rob Mohrfaa377f2022-01-19 14:38:36 -0800125 top_presentation,
126 use_constraint_file,
127 pigweed_root,
Rob Mohr9794dc82023-07-22 00:19:43 +0000128 options,
Rob Mohrfaa377f2022-01-19 14:38:36 -0800129 env,
130 ):
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800131 """Run pw_env_setup."""
132
Rob Mohrfabf49c2023-04-11 20:08:52 +0000133 json_file = env.dir / 'vars.json'
134 shell_file = env.dir / 'setup.sh'
135 venv_dir = env.dir / 'venv'
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800136
Rob Mohr8afb1b42021-02-26 13:56:08 -0800137 self.m.file.ensure_directory(
Rob Mohr4806b6b2023-02-03 18:03:32 +0000138 f'mkdir {self.m.path.basename(env.dir)}', env.dir,
Rob Mohr8afb1b42021-02-26 13:56:08 -0800139 )
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800140
Rob Mohr8afb1b42021-02-26 13:56:08 -0800141 self.m.file.ensure_directory(
Rob Mohr4806b6b2023-02-03 18:03:32 +0000142 f'mkdir {self.m.path.basename(venv_dir)}', venv_dir,
Rob Mohr8afb1b42021-02-26 13:56:08 -0800143 )
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800144
Rob Mohr8afb1b42021-02-26 13:56:08 -0800145 cmd = [
Rob Mohrc7e06392021-04-15 14:10:28 -0700146 'python3',
Rob Mohrfabf49c2023-04-11 20:08:52 +0000147 (
148 pigweed_root
149 / 'pw_env_setup'
150 / 'py'
151 / 'pw_env_setup'
152 / 'env_setup.py'
Rob Mohrfaa377f2022-01-19 14:38:36 -0800153 ),
Rob Mohr8afb1b42021-02-26 13:56:08 -0800154 '--pw-root',
Rob Mohrfaa377f2022-01-19 14:38:36 -0800155 pigweed_root,
Rob Mohr8afb1b42021-02-26 13:56:08 -0800156 '--install-dir',
Rob Mohrdcc61c82022-08-16 18:01:54 +0000157 env.dir,
Rob Mohr8afb1b42021-02-26 13:56:08 -0800158 '--json-file',
159 json_file,
160 '--shell-file',
161 shell_file,
Rob Mohr21e3da82021-05-27 13:21:41 -0700162 '--virtualenv-gn-out-dir',
Rob Mohrfabf49c2023-04-11 20:08:52 +0000163 env.dir / 'out',
Rob Mohr47d3d5c2021-05-13 06:59:23 -0700164 '--use-existing-cipd',
Rob Mohr95ab0342021-05-27 10:38:22 -0700165 '--strict',
Rob Mohr8afb1b42021-02-26 13:56:08 -0800166 ]
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800167
Rob Mohr9794dc82023-07-22 00:19:43 +0000168 if options.skip_submodule_check:
Rob Mohr5b95c712022-06-13 13:22:21 -0700169 cmd.append('--skip-submodule-check')
170
Rob Mohr80408192021-09-23 07:59:12 -0700171 if not use_constraint_file:
172 cmd.append('--unpin-pip-packages')
173
Rob Mohr3534bfe2023-07-24 19:02:55 +0000174 for f in options.additional_cipd_files:
Rob Mohre4031732023-09-18 22:53:36 +0000175 f = f.replace('$PW_ROOT', str(pigweed_root))
Rob Mohr3534bfe2023-07-24 19:02:55 +0000176 cmd.extend(('--additional-cipd-file', f))
177
Rob Mohr0567a302023-09-19 16:19:42 +0000178 cmd.append('--config-file')
179 cmd.append(path(options.config_file or 'pigweed.json', checkout))
Rob Mohr8afb1b42021-02-26 13:56:08 -0800180
Rob Mohr2e6a9d02023-10-27 14:29:33 +0000181 with self.m.step.nest('run pw_env_setup') as pres:
Rob Mohr84f58bb2023-11-08 22:06:57 +0000182 with self.m.defer.context() as defer:
183 with env(), self.m.default_timeout():
184 defer(self.m.step, 'pw_env_setup', cmd)
Rob Mohr8afb1b42021-02-26 13:56:08 -0800185
Rob Mohr84f58bb2023-11-08 22:06:57 +0000186 defer(self.m.file.listdir, 'ls', env.dir, recursive=True)
Rob Mohr9a0d6af2023-04-28 20:30:45 +0000187
Rob Mohr84f58bb2023-11-08 22:06:57 +0000188 defer(self.m.save_logs, (env.dir,), pres=pres)
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800189
Rob Mohr8afb1b42021-02-26 13:56:08 -0800190 json_data = self.m.file.read_json(
191 'read json file',
192 json_file,
193 test_data={
194 'set': {'VIRTUAL_ENV': '/environment/virtualenv'},
195 'modify': {
196 'PATH': {'append': ['/environment/bin']},
197 'LD_LIBRARY_PATH': {'prepend': ['/environment/lib']},
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800198 },
Rob Mohr8afb1b42021-02-26 13:56:08 -0800199 },
200 )
Rob Mohrb1eff262022-09-07 16:58:47 +0000201 top_presentation.logs['vars.json'] = pprint.pformat(json_data)
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800202
Rob Mohrfabf49c2023-04-11 20:08:52 +0000203 env_gni_path = (
204 checkout.root / 'build_overrides' / 'pigweed_environment.gni'
Rob Mohr07314452022-08-16 18:22:52 +0000205 )
Rob Mohr07314452022-08-16 18:22:52 +0000206
Rob Mohrb1eff262022-09-07 16:58:47 +0000207 self.m.path.mock_add_file(env_gni_path)
208 if self.m.path.isfile(env_gni_path):
209 environment_gni = self.m.file.read_text(
210 'read gni file', env_gni_path
211 )
212 top_presentation.logs['pigweed_environment.gni'] = environment_gni
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800213
Rob Mohre3be66a2021-08-27 12:38:18 -0700214 for var, value in json_data['set'].items():
Rob Mohrfaa377f2022-01-19 14:38:36 -0800215 env.env[var] = value
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800216
Rob Mohre3be66a2021-08-27 12:38:18 -0700217 for var, actions in json_data['modify'].items():
Rob Mohr8afb1b42021-02-26 13:56:08 -0800218 for value in actions.get('prepend', ()):
Rob Mohrfaa377f2022-01-19 14:38:36 -0800219 env.prefixes.setdefault(var, [])
220 env.prefixes[var].append(value)
Rob Mohr8afb1b42021-02-26 13:56:08 -0800221 for value in actions.get('append', ()):
Rob Mohrfaa377f2022-01-19 14:38:36 -0800222 env.suffixes.setdefault(var, [])
223 env.suffixes[var].append(value)
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800224
Rob Mohrdc93dd92022-08-13 00:15:04 +0000225 def _toolchain_override(self, env):
226 """Checks for a toolchain override and applies it."""
227
Oliver Newman630409a2023-09-27 17:33:25 +0000228 # Using '$fuchsia/checkout' properties to simplify interface with the
Rob Mohrdc93dd92022-08-13 00:15:04 +0000229 # Fuchsia Toolchain team.
Oliver Newman630409a2023-09-27 17:33:25 +0000230 fuchsia_build_props = self.m.properties.thaw().get('$fuchsia/checkout', {})
Rob Mohrdc93dd92022-08-13 00:15:04 +0000231 toolchain_props = fuchsia_build_props.get('clang_toolchain', {})
232 if not toolchain_props:
233 return
234
235 with self.m.step.nest('toolchain override'):
236 with self.m.context(infra_steps=True):
Rob Mohrfabf49c2023-04-11 20:08:52 +0000237 toolchain_dir = env.dir / 'override' / 'clang_toolchain'
Rob Mohrdc93dd92022-08-13 00:15:04 +0000238
Oliver Newman630409a2023-09-27 17:33:25 +0000239 if cipd_version := toolchain_props.get('cipd_version'):
Rob Mohrdc93dd92022-08-13 00:15:04 +0000240 pkgs = self.m.cipd.EnsureFile()
241 pkgs.add_package(
242 'fuchsia/third_party/clang/${platform}',
Oliver Newman630409a2023-09-27 17:33:25 +0000243 cipd_version,
Rob Mohrdc93dd92022-08-13 00:15:04 +0000244 )
Rob Mohrbf732212022-09-12 18:11:22 +0000245 self.m.cipd.ensure(toolchain_dir, pkgs)
Rob Mohrdc93dd92022-08-13 00:15:04 +0000246
Oliver Newman630409a2023-09-27 17:33:25 +0000247 elif cas_digest := toolchain_props.get('cas_digest'):
Rob Mohr6da5ed82022-09-12 18:20:29 +0000248 with self.m.cas.with_instance(
249 'projects/chromium-swarm/instances/default_instance'
250 ):
251 self.m.cas.download(
252 'download',
Oliver Newman630409a2023-09-27 17:33:25 +0000253 digest=cas_digest,
Rob Mohr6da5ed82022-09-12 18:20:29 +0000254 output_dir=toolchain_dir,
255 )
Rob Mohrdc93dd92022-08-13 00:15:04 +0000256
257 else: # pragma: no cover
258 raise KeyError(
Oliver Newman630409a2023-09-27 17:33:25 +0000259 f'invalid clang toolchain properties: {toolchain_props!r}'
Rob Mohrdc93dd92022-08-13 00:15:04 +0000260 )
261
262 env.prefixes.setdefault('PATH', [])
Rob Mohrbf732212022-09-12 18:11:22 +0000263 env.prefixes['PATH'].append(toolchain_dir)
Rob Mohrfabf49c2023-04-11 20:08:52 +0000264 env.prefixes['PATH'].append(toolchain_dir / 'bin')
Rob Mohrbf732212022-09-12 18:11:22 +0000265
Rob Mohrfabf49c2023-04-11 20:08:52 +0000266 clang_prefix = toolchain_dir / 'bin'
Rob Mohrbf732212022-09-12 18:11:22 +0000267 env.override_gn_args[
268 'pw_toolchain_CLANG_PREFIX'
Rob Mohra65a1972022-09-13 16:25:39 +0000269 ] = f'{clang_prefix}/'
Rob Mohrdc93dd92022-08-13 00:15:04 +0000270
Rob Mohrfaa377f2022-01-19 14:38:36 -0800271 def init(
Rob Mohr500364a2023-02-01 15:05:30 +0000272 self, checkout, options=None, use_constraint_file=True,
Rob Mohrfaa377f2022-01-19 14:38:36 -0800273 ):
Rob Mohr500364a2023-02-01 15:05:30 +0000274 pigweed_root = checkout.root
Rob Mohrbf732212022-09-12 18:11:22 +0000275 env = Environment(
Rob Mohrfabf49c2023-04-11 20:08:52 +0000276 api=self.m, dir=checkout.root / 'environment', checkout=checkout,
Rob Mohrbf732212022-09-12 18:11:22 +0000277 )
Rob Mohre47c6582022-07-14 16:33:10 +0000278
Rob Mohre4475152022-11-16 20:04:36 +0000279 # If in recipe tests always add at least one variable to make it easier
280 # to test use of variables in recipes.
281 if self._test_data.enabled:
282 env.env['PW_TEST_VAR'] = 'test_value'
283
Rob Mohr2cb42052023-09-26 18:40:07 +0000284 if not options:
285 options = Options()
286
Rob Mohrfaa377f2022-01-19 14:38:36 -0800287 if not options.config_file:
Rob Mohr2cb42052023-09-26 18:40:07 +0000288 # Always set certain variables even without an environment config.
289 self._init_misc_vars(env, options.additional_variables)
Rob Mohrfaa377f2022-01-19 14:38:36 -0800290 return env
Rob Mohr3b7da062020-04-30 14:30:57 -0700291
Rob Mohrfaa377f2022-01-19 14:38:36 -0800292 if options.relative_pigweed_root not in (None, '', '.'):
Rob Mohrfabf49c2023-04-11 20:08:52 +0000293 pigweed_root = checkout.root / options.relative_pigweed_root
Rob Mohrfaa377f2022-01-19 14:38:36 -0800294
Rob Mohrfaa377f2022-01-19 14:38:36 -0800295 with self.m.step.nest('environment') as pres:
Rob Mohrd5764982023-11-16 18:59:44 +0000296 with self.m.step.nest('options') as options_pres:
297 options_pres.step_summary_text = repr(options)
298
Rob Mohr0567a302023-09-19 16:19:42 +0000299 cfg_json = self.m.file.read_json(
300 'read config',
301 path(options.config_file, checkout),
302 test_data={
303 'pw': {
304 'pw_env_setup': {'relative_pigweed_root': 'pigweed',},
305 },
306 },
307 )
308 pres.logs['config.json'] = pprint.pformat(cfg_json)
309
310 cfg_json = cfg_json.get('pw', cfg_json)
311 cfg_json = cfg_json.get('pw_env_setup', cfg_json)
312 if 'relative_pigweed_root' in cfg_json:
313 pigweed_root = checkout.root / cfg_json['relative_pigweed_root']
314
Rob Mohr500364a2023-02-01 15:05:30 +0000315 env.env['PW_PROJECT_ROOT'] = checkout.root
Rob Mohr0567a302023-09-19 16:19:42 +0000316 env.env['PW_ROOT'] = pigweed_root
Rob Mohrfaa377f2022-01-19 14:38:36 -0800317
318 self._init_platform(env)
Rob Mohr9a397182022-07-12 02:00:24 +0000319 self._init_misc_vars(env, options.additional_variables)
Rob Mohrfaa377f2022-01-19 14:38:36 -0800320 self._init_pigweed(
Rob Mohr500364a2023-02-01 15:05:30 +0000321 checkout=checkout,
Rob Mohrfaa377f2022-01-19 14:38:36 -0800322 top_presentation=pres,
323 use_constraint_file=use_constraint_file,
324 pigweed_root=pigweed_root,
Rob Mohr9794dc82023-07-22 00:19:43 +0000325 options=options,
Rob Mohrfaa377f2022-01-19 14:38:36 -0800326 env=env,
327 )
Rob Mohrdc93dd92022-08-13 00:15:04 +0000328 self._toolchain_override(env)
Rob Mohrfaa377f2022-01-19 14:38:36 -0800329
Rob Mohr322b7aa2022-09-07 19:45:05 +0000330 with env():
331 # If 'pw doctor' fails we can continue, but show the doctor
332 # failure in red in the UI.
333 try:
334 self.m.step('doctor', ['python', '-m', 'pw_cli', 'doctor'])
335 except self.m.step.StepFailure:
336 pass
337
Rob Mohrfaa377f2022-01-19 14:38:36 -0800338 return env