blob: 69cd70e6d1df66805a0966b6c74ab63e80b1087c [file] [log] [blame]
Rob Mohr3b7da062020-04-30 14:30:57 -07001# Copyright 2020 The Pigweed Authors
Rob Mohrb671bc72019-12-13 08:36:15 -08002#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4# use this file except in compliance with the License. You may obtain a copy of
5# the License at
6#
7# https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations under
13# the License.
14"""Environment utility functions.
15
16Usage:
Rob Mohrfaa377f2022-01-19 14:38:36 -080017env = api.environment.initialize(checkout_root, env_options)
18with env():
Rob Mohrb671bc72019-12-13 08:36:15 -080019 ...
20"""
21
Rob Mohr21f15eb2020-09-09 10:10:39 -070022import contextlib
Rob Mohr59a9b1a2020-11-10 14:49:43 -080023import pprint
Rob Mohr21f15eb2020-09-09 10:10:39 -070024
Rob Mohrfaa377f2022-01-19 14:38:36 -080025from PB.recipe_modules.pigweed.environment.options import Options
26
Rob Mohrb671bc72019-12-13 08:36:15 -080027import attr
28from recipe_engine import recipe_api
29
30
31@attr.s
Rob Mohrfaa377f2022-01-19 14:38:36 -080032class Environment(object):
33 _api = attr.ib()
34 prefixes = attr.ib(default=attr.Factory(dict))
35 suffixes = attr.ib(default=attr.Factory(dict))
36 env = attr.ib(default=attr.Factory(dict))
37
38 @contextlib.contextmanager
39 def __call__(self):
40 # Using reversed() because things that are added later in environment
41 # setup need to override things that came earlier.
42 with self._api.context(
43 env_prefixes={k: reversed(v) for k, v in self.prefixes.items()},
44 env_suffixes=self.suffixes,
45 env=self.env,
46 ):
47 with self._api.macos_sdk():
48 yield self
49
50 def __getattr__(self, name):
51 if name not in self.env:
52 raise AttributeError(name)
53
54 return self.env.get(name)
Rob Mohrb671bc72019-12-13 08:36:15 -080055
56
57class EnvironmentApi(recipe_api.RecipeApi):
Rob Mohr57204602020-09-23 08:41:18 -070058 """Environment utility functions."""
Rob Mohrb671bc72019-12-13 08:36:15 -080059
Rob Mohrfaa377f2022-01-19 14:38:36 -080060 def _init_platform(self, env):
Rob Mohr57204602020-09-23 08:41:18 -070061 if self.m.platform.is_mac:
62 with self.m.step.nest('setup platform'):
63 with self.m.macos_sdk():
64 pass
Rob Mohr21f15eb2020-09-09 10:10:39 -070065
Rob Mohr9a397182022-07-12 02:00:24 +000066 def _init_misc_vars(self, env, additional_variables=None):
Rob Mohrfaa377f2022-01-19 14:38:36 -080067 env.env['PW_ENVIRONMENT_NO_ERROR_ON_UNRECOGNIZED'] = '1'
68 env.env['PW_ENVSETUP_DISABLE_SPINNER'] = '1'
69 env.env['PW_PRESUBMIT_DISABLE_SUBPROCESS_CAPTURE'] = '1'
Rob Mohr8360e162021-01-13 08:46:28 -080070
Rob Mohr9a397182022-07-12 02:00:24 +000071 env.env.update(additional_variables or {})
72
Rob Mohra89b6ae2020-10-08 13:42:56 -070073 if self.m.led.launched_by_led:
74 # Not using self.m.buildbucket_util.id because having relatively
75 # consistent length is important to some downstream projects when
76 # pulling this into their builds.
Rob Mohrfaa377f2022-01-19 14:38:36 -080077 env.env['BUILDBUCKET_ID'] = 'test:{}'.format(
Rob Mohr7ab2baf2020-10-16 11:48:29 -070078 self.m.swarming.task_id
79 )
Rob Mohrfaa377f2022-01-19 14:38:36 -080080 env.env['BUILD_NUMBER'] = str(self.m.buildbucket.build.number)
Rob Mohr6042b342020-10-22 16:33:40 -070081
Rob Mohra89b6ae2020-10-08 13:42:56 -070082 else:
Rob Mohrfaa377f2022-01-19 14:38:36 -080083 env.env['BUILDBUCKET_ID'] = str(self.m.buildbucket.build.id)
84 env.env['BUILD_NUMBER'] = str(self.m.buildbucket.build.number)
Rob Mohra89b6ae2020-10-08 13:42:56 -070085
Rob Mohrfaa377f2022-01-19 14:38:36 -080086 env.env['BUILDBUCKET_NAME'] = ':'.join(
Rob Mohr7ab2baf2020-10-16 11:48:29 -070087 (
88 self.m.buildbucket.build.builder.project,
89 self.m.buildbucket.build.builder.bucket,
90 self.m.buildbucket.build.builder.builder,
91 )
92 )
Rob Mohra89b6ae2020-10-08 13:42:56 -070093
Rob Mohrfaa377f2022-01-19 14:38:36 -080094 if env.env['BUILDBUCKET_NAME'] == '::':
95 env.env['BUILDBUCKET_NAME'] = 'project:bucket:builder'
Rob Mohra89b6ae2020-10-08 13:42:56 -070096
Rob Mohrfaa377f2022-01-19 14:38:36 -080097 env.env['GOCACHE'] = self.m.path['cache'].join('go')
98 env.env['PIP_CACHE_DIR'] = self.m.path['cache'].join('pip')
Rob Mohrd9b05332021-09-15 10:57:31 -070099 # Bazel cache dir.
Rob Mohrfaa377f2022-01-19 14:38:36 -0800100 env.env['TEST_TMPDIR'] = self.m.path['cache'].join('bazel')
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800101
Rob Mohrfaa377f2022-01-19 14:38:36 -0800102 def _init_pigweed(
103 self,
104 checkout_root,
Rob Mohre47c6582022-07-14 16:33:10 +0000105 env_dir,
Rob Mohrfaa377f2022-01-19 14:38:36 -0800106 top_presentation,
107 use_constraint_file,
108 pigweed_root,
109 config_file,
Rob Mohr5b95c712022-06-13 13:22:21 -0700110 skip_submodule_check,
Rob Mohrfaa377f2022-01-19 14:38:36 -0800111 env,
112 ):
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800113 """Run pw_env_setup."""
114
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800115 def path(relative_path):
116 parts = [
117 x for x in relative_path.split('/') if x not in ('.', u'.')
118 ]
119 if parts:
120 return checkout_root.join(*parts)
121 else:
122 return checkout_root # pragma: no cover
123
Rob Mohr8afb1b42021-02-26 13:56:08 -0800124 json_file = env_dir.join('vars.json')
125 shell_file = env_dir.join('setup.sh')
126 venv_dir = env_dir.join('venv')
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800127
Rob Mohr8afb1b42021-02-26 13:56:08 -0800128 self.m.file.ensure_directory(
129 'mkdir {}'.format(self.m.path.basename(env_dir)), env_dir,
130 )
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800131
Rob Mohr8afb1b42021-02-26 13:56:08 -0800132 self.m.file.ensure_directory(
133 'mkdir {}'.format(self.m.path.basename(venv_dir)), venv_dir,
134 )
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800135
Rob Mohr8afb1b42021-02-26 13:56:08 -0800136 cmd = [
Rob Mohrc7e06392021-04-15 14:10:28 -0700137 'python3',
Rob Mohrfaa377f2022-01-19 14:38:36 -0800138 pigweed_root.join(
139 'pw_env_setup', 'py', 'pw_env_setup', 'env_setup.py'
140 ),
Rob Mohr8afb1b42021-02-26 13:56:08 -0800141 '--pw-root',
Rob Mohrfaa377f2022-01-19 14:38:36 -0800142 pigweed_root,
Rob Mohr8afb1b42021-02-26 13:56:08 -0800143 '--install-dir',
144 env_dir,
145 '--json-file',
146 json_file,
147 '--shell-file',
148 shell_file,
Rob Mohr21e3da82021-05-27 13:21:41 -0700149 '--virtualenv-gn-out-dir',
150 self.m.build.dir,
Rob Mohr47d3d5c2021-05-13 06:59:23 -0700151 '--use-existing-cipd',
Rob Mohr95ab0342021-05-27 10:38:22 -0700152 '--strict',
Rob Mohr8afb1b42021-02-26 13:56:08 -0800153 ]
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800154
Rob Mohr5b95c712022-06-13 13:22:21 -0700155 if skip_submodule_check:
156 cmd.append('--skip-submodule-check')
157
Rob Mohr80408192021-09-23 07:59:12 -0700158 if not use_constraint_file:
159 cmd.append('--unpin-pip-packages')
160
Rob Mohrfaa377f2022-01-19 14:38:36 -0800161 cmd.extend(('--config-file', path(config_file)))
Rob Mohr18893142021-02-26 07:26:49 -0800162
Rob Mohrfaa377f2022-01-19 14:38:36 -0800163 top_presentation.logs['vars.json'] = pprint.pformat(
164 self.m.file.read_json('read config', path(config_file))
Rob Mohr001b8a72021-06-16 09:22:58 -0700165 )
Rob Mohr8afb1b42021-02-26 13:56:08 -0800166
167 with self.m.step.defer_results():
168 with self.m.step.nest('run pw_env_setup'):
Rob Mohrfaa377f2022-01-19 14:38:36 -0800169 with env():
Rob Mohr8afb1b42021-02-26 13:56:08 -0800170 self.m.step('pw_env_setup', cmd)
171
Rob Mohr4de06592021-06-17 11:21:49 -0700172 cipd_dir = env_dir.join('cipd')
Rob Mohr8afb1b42021-02-26 13:56:08 -0800173
Rob Mohr4de06592021-06-17 11:21:49 -0700174 log_exts = ('.cfg', '.log', '.sh', '.bat', '.json', '.ensure')
Rob Mohr8afb1b42021-02-26 13:56:08 -0800175
Rob Mohr34453952021-08-04 06:46:54 -0700176 for directory in (env_dir, cipd_dir, venv_dir):
177 if not self.m.path.isdir(directory):
Rob Mohr4cd5a202021-07-16 08:44:15 -0700178 continue
179
Rob Mohr34453952021-08-04 06:46:54 -0700180 with self.m.step.nest(self.m.path.basename(directory)):
181 files = self.m.file.listdir(
182 'ls', directory
183 ).get_result()
Rob Mohr8afb1b42021-02-26 13:56:08 -0800184
Rob Mohr4de06592021-06-17 11:21:49 -0700185 for entry in files:
186 ext = self.m.path.splitext(entry)[1]
187 if ext not in log_exts:
188 continue
189
190 self.m.file.read_text(
191 'read {}'.format(self.m.path.basename(entry)),
192 entry,
193 )
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800194
Rob Mohr8afb1b42021-02-26 13:56:08 -0800195 json_data = self.m.file.read_json(
196 'read json file',
197 json_file,
198 test_data={
199 'set': {'VIRTUAL_ENV': '/environment/virtualenv'},
200 'modify': {
201 'PATH': {'append': ['/environment/bin']},
202 'LD_LIBRARY_PATH': {'prepend': ['/environment/lib']},
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800203 },
Rob Mohr8afb1b42021-02-26 13:56:08 -0800204 },
205 )
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800206
Rob Mohrfaa377f2022-01-19 14:38:36 -0800207 top_presentation.logs['vars.json'] = pprint.pformat(json_data)
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800208
Rob Mohre3be66a2021-08-27 12:38:18 -0700209 for var, value in json_data['set'].items():
Rob Mohrfaa377f2022-01-19 14:38:36 -0800210 env.env[var] = value
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800211
Rob Mohre3be66a2021-08-27 12:38:18 -0700212 for var, actions in json_data['modify'].items():
Rob Mohr8afb1b42021-02-26 13:56:08 -0800213 for value in actions.get('prepend', ()):
Rob Mohrfaa377f2022-01-19 14:38:36 -0800214 env.prefixes.setdefault(var, [])
215 env.prefixes[var].append(value)
Rob Mohr8afb1b42021-02-26 13:56:08 -0800216 for value in actions.get('append', ()):
Rob Mohrfaa377f2022-01-19 14:38:36 -0800217 env.suffixes.setdefault(var, [])
218 env.suffixes[var].append(value)
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800219
Rob Mohrfaa377f2022-01-19 14:38:36 -0800220 def init(
221 self, checkout_root, options=None, use_constraint_file=True,
222 ):
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800223 pigweed_root = checkout_root
Rob Mohrfaa377f2022-01-19 14:38:36 -0800224 env = Environment(api=self.m)
Rob Mohr59a9b1a2020-11-10 14:49:43 -0800225
Rob Mohre47c6582022-07-14 16:33:10 +0000226 env_dir = checkout_root.join('environment')
227
Rob Mohrfaa377f2022-01-19 14:38:36 -0800228 if not options.config_file:
Rob Mohrfaa377f2022-01-19 14:38:36 -0800229 return env
Rob Mohr3b7da062020-04-30 14:30:57 -0700230
Rob Mohrfaa377f2022-01-19 14:38:36 -0800231 if options.relative_pigweed_root not in (None, '', '.'):
232 pigweed_root = checkout_root.join(options.relative_pigweed_root)
233
234 if options.root_variable_name:
235 env.env[options.root_variable_name] = checkout_root
236
237 with self.m.step.nest('environment') as pres:
238 # Setting _initialized immediately because some setup steps need
239 # to use the context of previous steps, and invoking self() is
240 # the easiest way to do so.
241 env.env['PW_ROOT'] = pigweed_root
242 env.env['PW_PROJECT_ROOT'] = checkout_root
243
244 self._init_platform(env)
Rob Mohr9a397182022-07-12 02:00:24 +0000245 self._init_misc_vars(env, options.additional_variables)
Rob Mohrfaa377f2022-01-19 14:38:36 -0800246 self._init_pigweed(
247 checkout_root=checkout_root,
Rob Mohre47c6582022-07-14 16:33:10 +0000248 env_dir=env_dir,
Rob Mohrfaa377f2022-01-19 14:38:36 -0800249 top_presentation=pres,
250 use_constraint_file=use_constraint_file,
251 pigweed_root=pigweed_root,
252 config_file=options.config_file,
Rob Mohr5b95c712022-06-13 13:22:21 -0700253 skip_submodule_check=options.skip_submodule_check,
Rob Mohrfaa377f2022-01-19 14:38:36 -0800254 env=env,
255 )
256
Rob Mohrfaa377f2022-01-19 14:38:36 -0800257 return env