blob: 7d922e8b9569c4240465b1628cfb068ec9dcccb3 [file] [log] [blame]
Rob Mohr3b7da062020-04-30 14:30:57 -07001# Copyright 2020 The Pigweed Authors
Rob Mohrb671bc72019-12-13 08:36:15 -08002#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4# use this file except in compliance with the License. You may obtain a copy of
5# the License at
6#
7# https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations under
13# the License.
14"""Environment utility functions.
15
16Usage:
Rob Mohrfaa377f2022-01-19 14:38:36 -080017env = api.environment.initialize(checkout_root, env_options)
18with env():
Rob Mohrb671bc72019-12-13 08:36:15 -080019 ...
20"""
21
Rob Mohr21f15eb2020-09-09 10:10:39 -070022import contextlib
Rob Mohr59a9b1a2020-11-10 14:49:43 -080023import pprint
Rob Mohr21f15eb2020-09-09 10:10:39 -070024
Rob Mohrfaa377f2022-01-19 14:38:36 -080025from PB.recipe_modules.pigweed.environment.options import Options
26
Rob Mohrb671bc72019-12-13 08:36:15 -080027import attr
28from recipe_engine import recipe_api
29
30
31@attr.s
Rob Mohr82c0e352022-07-27 20:57:40 +000032class Environment:
Rob Mohrfaa377f2022-01-19 14:38:36 -080033 _api = attr.ib()
Rob Mohrdcc61c82022-08-16 18:01:54 +000034 dir = attr.ib()
Rob Mohrfaa377f2022-01-19 14:38:36 -080035 prefixes = attr.ib(default=attr.Factory(dict))
36 suffixes = attr.ib(default=attr.Factory(dict))
37 env = attr.ib(default=attr.Factory(dict))
38
39 @contextlib.contextmanager
40 def __call__(self):
41 # Using reversed() because things that are added later in environment
42 # setup need to override things that came earlier.
43 with self._api.context(
44 env_prefixes={k: reversed(v) for k, v in self.prefixes.items()},
45 env_suffixes=self.suffixes,
46 env=self.env,
47 ):
48 with self._api.macos_sdk():
49 yield self
50
51 def __getattr__(self, name):
52 if name not in self.env:
53 raise AttributeError(name)
54
55 return self.env.get(name)
Rob Mohrb671bc72019-12-13 08:36:15 -080056
57
58class EnvironmentApi(recipe_api.RecipeApi):
Rob Mohr57204602020-09-23 08:41:18 -070059 """Environment utility functions."""
Rob Mohrb671bc72019-12-13 08:36:15 -080060
Rob Mohrfaa377f2022-01-19 14:38:36 -080061 def _init_platform(self, env):
Rob Mohr57204602020-09-23 08:41:18 -070062 if self.m.platform.is_mac:
63 with self.m.step.nest('setup platform'):
64 with self.m.macos_sdk():
65 pass
Rob Mohr21f15eb2020-09-09 10:10:39 -070066
Rob Mohr9a397182022-07-12 02:00:24 +000067 def _init_misc_vars(self, env, additional_variables=None):
Rob Mohrfaa377f2022-01-19 14:38:36 -080068 env.env['PW_ENVIRONMENT_NO_ERROR_ON_UNRECOGNIZED'] = '1'
69 env.env['PW_ENVSETUP_DISABLE_SPINNER'] = '1'
70 env.env['PW_PRESUBMIT_DISABLE_SUBPROCESS_CAPTURE'] = '1'
Rob Mohr8360e162021-01-13 08:46:28 -080071
Rob Mohr9a397182022-07-12 02:00:24 +000072 env.env.update(additional_variables or {})
73
Rob Mohra89b6ae2020-10-08 13:42:56 -070074 if self.m.led.launched_by_led:
75 # Not using self.m.buildbucket_util.id because having relatively
76 # consistent length is important to some downstream projects when
77 # pulling this into their builds.
Rob Mohrfaa377f2022-01-19 14:38:36 -080078 env.env['BUILDBUCKET_ID'] = 'test:{}'.format(
Rob Mohr7ab2baf2020-10-16 11:48:29 -070079 self.m.swarming.task_id
80 )
Rob Mohrfaa377f2022-01-19 14:38:36 -080081 env.env['BUILD_NUMBER'] = str(self.m.buildbucket.build.number)
Rob Mohr6042b342020-10-22 16:33:40 -070082
Rob Mohra89b6ae2020-10-08 13:42:56 -070083 else:
Rob Mohrfaa377f2022-01-19 14:38:36 -080084 env.env['BUILDBUCKET_ID'] = str(self.m.buildbucket.build.id)
85 env.env['BUILD_NUMBER'] = str(self.m.buildbucket.build.number)
Rob Mohra89b6ae2020-10-08 13:42:56 -070086
Rob Mohrfaa377f2022-01-19 14:38:36 -080087 env.env['BUILDBUCKET_NAME'] = ':'.join(
Rob Mohr7ab2baf2020-10-16 11:48:29 -070088 (
89 self.m.buildbucket.build.builder.project,
90 self.m.buildbucket.build.builder.bucket,
91 self.m.buildbucket.build.builder.builder,
92 )
93 )
Rob Mohra89b6ae2020-10-08 13:42:56 -070094
Rob Mohrfaa377f2022-01-19 14:38:36 -080095 if env.env['BUILDBUCKET_NAME'] == '::':
96 env.env['BUILDBUCKET_NAME'] = 'project:bucket:builder'
Rob Mohra89b6ae2020-10-08 13:42:56 -070097
Rob Mohr9863ed82022-08-15 20:14:54 +000098 env.env['CTCACHE_DIR'] = self.m.path['cache'].join('clang_tidy')
Rob Mohrfaa377f2022-01-19 14:38:36 -080099 env.env['GOCACHE'] = self.m.path['cache'].join('go')
100 env.env['PIP_CACHE_DIR'] = self.m.path['cache'].join('pip')
Rob Mohrfaa377f2022-01-19 14:38:36 -0800101 env.env['TEST_TMPDIR'] = self.m.path['cache'].join('bazel')
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800102
Rob Mohrfaa377f2022-01-19 14:38:36 -0800103 def _init_pigweed(
104 self,
105 checkout_root,
106 top_presentation,
107 use_constraint_file,
108 pigweed_root,
109 config_file,
Rob Mohr5b95c712022-06-13 13:22:21 -0700110 skip_submodule_check,
Rob Mohrfaa377f2022-01-19 14:38:36 -0800111 env,
112 ):
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800113 """Run pw_env_setup."""
114
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800115 def path(relative_path):
116 parts = [
117 x for x in relative_path.split('/') if x not in ('.', u'.')
118 ]
119 if parts:
120 return checkout_root.join(*parts)
121 else:
122 return checkout_root # pragma: no cover
123
Rob Mohrdcc61c82022-08-16 18:01:54 +0000124 json_file = env.dir.join('vars.json')
125 shell_file = env.dir.join('setup.sh')
126 venv_dir = env.dir.join('venv')
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800127
Rob Mohr8afb1b42021-02-26 13:56:08 -0800128 self.m.file.ensure_directory(
Rob Mohrdcc61c82022-08-16 18:01:54 +0000129 'mkdir {}'.format(self.m.path.basename(env.dir)), env.dir,
Rob Mohr8afb1b42021-02-26 13:56:08 -0800130 )
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800131
Rob Mohr8afb1b42021-02-26 13:56:08 -0800132 self.m.file.ensure_directory(
133 'mkdir {}'.format(self.m.path.basename(venv_dir)), venv_dir,
134 )
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800135
Rob Mohr8afb1b42021-02-26 13:56:08 -0800136 cmd = [
Rob Mohrc7e06392021-04-15 14:10:28 -0700137 'python3',
Rob Mohrfaa377f2022-01-19 14:38:36 -0800138 pigweed_root.join(
139 'pw_env_setup', 'py', 'pw_env_setup', 'env_setup.py'
140 ),
Rob Mohr8afb1b42021-02-26 13:56:08 -0800141 '--pw-root',
Rob Mohrfaa377f2022-01-19 14:38:36 -0800142 pigweed_root,
Rob Mohr8afb1b42021-02-26 13:56:08 -0800143 '--install-dir',
Rob Mohrdcc61c82022-08-16 18:01:54 +0000144 env.dir,
Rob Mohr8afb1b42021-02-26 13:56:08 -0800145 '--json-file',
146 json_file,
147 '--shell-file',
148 shell_file,
Rob Mohr21e3da82021-05-27 13:21:41 -0700149 '--virtualenv-gn-out-dir',
150 self.m.build.dir,
Rob Mohr47d3d5c2021-05-13 06:59:23 -0700151 '--use-existing-cipd',
Rob Mohr95ab0342021-05-27 10:38:22 -0700152 '--strict',
Rob Mohr8afb1b42021-02-26 13:56:08 -0800153 ]
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800154
Rob Mohr5b95c712022-06-13 13:22:21 -0700155 if skip_submodule_check:
156 cmd.append('--skip-submodule-check')
157
Rob Mohr80408192021-09-23 07:59:12 -0700158 if not use_constraint_file:
159 cmd.append('--unpin-pip-packages')
160
Rob Mohrfaa377f2022-01-19 14:38:36 -0800161 cmd.extend(('--config-file', path(config_file)))
Rob Mohr18893142021-02-26 07:26:49 -0800162
Rob Mohr4e7bea62022-07-28 14:19:31 +0000163 top_presentation.logs['config.json'] = pprint.pformat(
Rob Mohrfaa377f2022-01-19 14:38:36 -0800164 self.m.file.read_json('read config', path(config_file))
Rob Mohr001b8a72021-06-16 09:22:58 -0700165 )
Rob Mohr8afb1b42021-02-26 13:56:08 -0800166
167 with self.m.step.defer_results():
168 with self.m.step.nest('run pw_env_setup'):
Rob Mohrfaa377f2022-01-19 14:38:36 -0800169 with env():
Rob Mohr8afb1b42021-02-26 13:56:08 -0800170 self.m.step('pw_env_setup', cmd)
171
Rob Mohrdcc61c82022-08-16 18:01:54 +0000172 cipd_dir = env.dir.join('cipd')
Rob Mohr8afb1b42021-02-26 13:56:08 -0800173
Rob Mohr4de06592021-06-17 11:21:49 -0700174 log_exts = ('.cfg', '.log', '.sh', '.bat', '.json', '.ensure')
Rob Mohr8afb1b42021-02-26 13:56:08 -0800175
Rob Mohrdcc61c82022-08-16 18:01:54 +0000176 for directory in (env.dir, cipd_dir, venv_dir):
Rob Mohr34453952021-08-04 06:46:54 -0700177 if not self.m.path.isdir(directory):
Rob Mohr4cd5a202021-07-16 08:44:15 -0700178 continue
179
Rob Mohr34453952021-08-04 06:46:54 -0700180 with self.m.step.nest(self.m.path.basename(directory)):
181 files = self.m.file.listdir(
182 'ls', directory
183 ).get_result()
Rob Mohr8afb1b42021-02-26 13:56:08 -0800184
Rob Mohr4de06592021-06-17 11:21:49 -0700185 for entry in files:
186 ext = self.m.path.splitext(entry)[1]
187 if ext not in log_exts:
188 continue
189
190 self.m.file.read_text(
191 'read {}'.format(self.m.path.basename(entry)),
192 entry,
193 )
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800194
Rob Mohr8afb1b42021-02-26 13:56:08 -0800195 json_data = self.m.file.read_json(
196 'read json file',
197 json_file,
198 test_data={
199 'set': {'VIRTUAL_ENV': '/environment/virtualenv'},
200 'modify': {
201 'PATH': {'append': ['/environment/bin']},
202 'LD_LIBRARY_PATH': {'prepend': ['/environment/lib']},
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800203 },
Rob Mohr8afb1b42021-02-26 13:56:08 -0800204 },
205 )
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800206
Rob Mohrfaa377f2022-01-19 14:38:36 -0800207 top_presentation.logs['vars.json'] = pprint.pformat(json_data)
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800208
Rob Mohre3be66a2021-08-27 12:38:18 -0700209 for var, value in json_data['set'].items():
Rob Mohrfaa377f2022-01-19 14:38:36 -0800210 env.env[var] = value
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800211
Rob Mohre3be66a2021-08-27 12:38:18 -0700212 for var, actions in json_data['modify'].items():
Rob Mohr8afb1b42021-02-26 13:56:08 -0800213 for value in actions.get('prepend', ()):
Rob Mohrfaa377f2022-01-19 14:38:36 -0800214 env.prefixes.setdefault(var, [])
215 env.prefixes[var].append(value)
Rob Mohr8afb1b42021-02-26 13:56:08 -0800216 for value in actions.get('append', ()):
Rob Mohrfaa377f2022-01-19 14:38:36 -0800217 env.suffixes.setdefault(var, [])
218 env.suffixes[var].append(value)
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800219
Rob Mohrdc93dd92022-08-13 00:15:04 +0000220 def _toolchain_override(self, env):
221 """Checks for a toolchain override and applies it."""
222
223 # Using '$fuchsia/build' properties to simplify interface with the
224 # Fuchsia Toolchain team.
225 fuchsia_build_props = self.m.properties.thaw().get('$fuchsia/build', {})
226 toolchain_props = fuchsia_build_props.get('clang_toolchain', {})
227 if not toolchain_props:
228 return
229
230 with self.m.step.nest('toolchain override'):
231 with self.m.context(infra_steps=True):
232 root_dir = self.m.path.mkdtemp('clang')
233
234 if toolchain_props['source'] == 'cipd':
235 pkgs = self.m.cipd.EnsureFile()
236 pkgs.add_package(
237 'fuchsia/third_party/clang/${platform}',
238 toolchain_props['version'],
239 )
240 self.m.cipd.ensure(root_dir, pkgs)
241
242 elif toolchain_props['source'] in 'isolated':
243 self.m.cas.download(
244 'download',
245 digest=toolchain_props['version'],
246 output_dir=root_dir,
247 )
248
249 else: # pragma: no cover
250 raise KeyError(
251 f'clang toolchain source {toolchain_props["source"]} '
252 'not recognized'
253 )
254
255 env.prefixes.setdefault('PATH', [])
256 env.prefixes['PATH'].append(root_dir)
257 env.prefixes['PATH'].append(root_dir.join('bin'))
258
Rob Mohrfaa377f2022-01-19 14:38:36 -0800259 def init(
260 self, checkout_root, options=None, use_constraint_file=True,
261 ):
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800262 pigweed_root = checkout_root
Rob Mohrdcc61c82022-08-16 18:01:54 +0000263 env = Environment(api=self.m, dir=checkout_root.join('environment'))
Rob Mohre47c6582022-07-14 16:33:10 +0000264
Rob Mohrfaa377f2022-01-19 14:38:36 -0800265 if not options.config_file:
Rob Mohrfaa377f2022-01-19 14:38:36 -0800266 return env
Rob Mohr3b7da062020-04-30 14:30:57 -0700267
Rob Mohrfaa377f2022-01-19 14:38:36 -0800268 if options.relative_pigweed_root not in (None, '', '.'):
269 pigweed_root = checkout_root.join(options.relative_pigweed_root)
270
271 if options.root_variable_name:
272 env.env[options.root_variable_name] = checkout_root
273
274 with self.m.step.nest('environment') as pres:
275 # Setting _initialized immediately because some setup steps need
276 # to use the context of previous steps, and invoking self() is
277 # the easiest way to do so.
278 env.env['PW_ROOT'] = pigweed_root
279 env.env['PW_PROJECT_ROOT'] = checkout_root
280
281 self._init_platform(env)
Rob Mohr9a397182022-07-12 02:00:24 +0000282 self._init_misc_vars(env, options.additional_variables)
Rob Mohrfaa377f2022-01-19 14:38:36 -0800283 self._init_pigweed(
284 checkout_root=checkout_root,
285 top_presentation=pres,
286 use_constraint_file=use_constraint_file,
287 pigweed_root=pigweed_root,
288 config_file=options.config_file,
Rob Mohr5b95c712022-06-13 13:22:21 -0700289 skip_submodule_check=options.skip_submodule_check,
Rob Mohrfaa377f2022-01-19 14:38:36 -0800290 env=env,
291 )
Rob Mohrdc93dd92022-08-13 00:15:04 +0000292 self._toolchain_override(env)
Rob Mohrfaa377f2022-01-19 14:38:36 -0800293
Rob Mohrfaa377f2022-01-19 14:38:36 -0800294 return env