recipe_modules/environment/api.py - infra/recipes - Git at Google

 # Copyright 2020 The Pigweed Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not
 # use this file except in compliance with the License. You may obtain a copy of
 # the License at
 #
 #     https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations under
 # the License.
 """Environment utility functions.

 Usage:
 env = api.environment.initialize(checkout, env_options)
 with env():
   ...
 """

 from __future__ import annotations

 import contextlib
 import dataclasses
 import pprint
 from typing import TYPE_CHECKING

 from PB.recipe_modules.pigweed.environment.options import Options
 from recipe_engine import recipe_api

 if TYPE_CHECKING:  # pragma: no cover
     from recipe_engine import config_types, engine_types
     from RECIPE_MODULES.pigweed.checkout import api as checkout_api


 @dataclasses.dataclass
 class Environment:
     api: recipe_api.RecipeApi
     dir: config_types.Path
     checkout: checkout_api.CheckoutContext
     prefixes: dict[str, list[str]] = dataclasses.field(default_factory=dict)
     suffixes: dict[str, list[str]] = dataclasses.field(default_factory=dict)
     env: dict[str, str] = dataclasses.field(default_factory=dict)
     override_gn_args: dict = dataclasses.field(default_factory=dict)

     @contextlib.contextmanager
     def __call__(self):
         with self.api.macos_sdk():
             # Using reversed() because things that are added later in
             # environment setup need to override things that came earlier.
             with self.api.context(
                 env_prefixes={k: reversed(v) for k, v in self.prefixes.items()},
                 env_suffixes=self.suffixes,
                 env=self.env,
             ):
                 yield self

     def __getattr__(self, name) -> str:
         if name not in self.env:
             raise AttributeError(name)

         return self.env.get(name)


 def path(
     relative_path: config_types.Path,
     checkout: checkout_api.CheckoutContext,
 ) -> config_types.Path:
     parts = [x for x in relative_path.split('/') if x not in ('.', u'.')]
     if parts:
         return checkout.root.joinpath(*parts)
     else:
         return checkout.root  # pragma: no cover


 class EnvironmentApi(recipe_api.RecipeApi):
     """Environment utility functions."""

     Environment = Environment

     def _init_platform(self, env: Environment) -> None:
         if self.m.platform.is_mac:
             with self.m.step.nest('setup platform'):
                 with self.m.macos_sdk():
                     pass

     def _init_misc_vars(
         self,
         env: Environment,
         additional_variables: dict[str, str] | None = None,
     ) -> None:
         env.env['PW_ENVIRONMENT_NO_ERROR_ON_UNRECOGNIZED'] = '1'
         env.env['PW_ENVSETUP_DISABLE_SPINNER'] = '1'
         env.env['PW_PRESUBMIT_DISABLE_SUBPROCESS_CAPTURE'] = '1'
         env.env['PW_USE_COLOR'] = ''
         env.env['CLICOLOR'] = '0'  # Formerly on https://bixense.com/clicolors/.
         env.env['NO_COLOR'] = '1'  # See https://no-color.org.
         # This should tell ninja to disable colors based on implementation at
         # https://github.com/ninja-build/ninja/blob/master/src/line_printer.cc#L60.
         env.env['CLICOLOR_FORCE'] = '0'
         env.env['GCC_COLORS'] = ''

         env.env.update(additional_variables or {})

         if self.m.led.launched_by_led and not self.m.led.led_build:
             # Not using self.m.buildbucket_util.id because some downstream
             # projects need this id to never be longer than a typical
             # buildbucket id. Shorter is fine.
             env.env['BUILDBUCKET_ID'] = '0'
             env.env['BUILD_NUMBER'] = str(self.m.buildbucket.build.number)

         else:
             env.env['BUILDBUCKET_ID'] = str(self.m.buildbucket.build.id)
             env.env['BUILD_NUMBER'] = str(self.m.buildbucket.build.number)

         env.env['BUILDBUCKET_NAME'] = ':'.join(
             (
                 self.m.buildbucket.build.builder.project,
                 self.m.buildbucket.build.builder.bucket,
                 self.m.buildbucket.build.builder.builder,
             )
         )

         if env.env['BUILDBUCKET_NAME'] == '::':
             env.env['BUILDBUCKET_NAME'] = 'project:bucket:builder'

         env.env['CCACHE_DIR'] = self.m.path.cache_dir / 'ccache'
         env.env['CTCACHE_DIR'] = self.m.path.cache_dir / 'clang_tidy'
         env.env['GOCACHE'] = self.m.path.cache_dir / 'go'
         env.env['PIP_CACHE_DIR'] = self.m.path.cache_dir / 'pip'
         env.env['TEST_TMPDIR'] = self.m.path.cache_dir / 'bazel'

         env.env['TRIGGERING_CHANGES_JSON'] = env.checkout.changes_json

     def _init_pigweed(
         self,
         checkout: checkout_api.CheckoutContext,
         top_presentation: engine_types.StepPresentation,
         use_constraint_file: bool,
         pigweed_root: config_types.Path,
         options: Options,
         env: Environment,
     ) -> None:
         """Run pw_env_setup."""

         json_file = env.dir / 'vars.json'
         shell_file = env.dir / 'setup.sh'
         venv_dir = env.dir / 'venv'

         self.m.file.ensure_directory(
             f'mkdir {self.m.path.basename(env.dir)}',
             env.dir,
         )

         self.m.file.ensure_directory(
             f'mkdir {self.m.path.basename(venv_dir)}',
             venv_dir,
         )

         cmd: list[str | config_types.Path] = [
             'python3',
             (
                 pigweed_root
                 / 'pw_env_setup'
                 / 'py'
                 / 'pw_env_setup'
                 / 'env_setup.py'
             ),
             '--pw-root',
             pigweed_root,
             '--install-dir',
             env.dir,
             '--json-file',
             json_file,
             '--shell-file',
             shell_file,
             '--virtualenv-gn-out-dir',
             env.dir / 'out',
             '--use-existing-cipd',
             '--strict',
             '--disable-rosetta',
         ]

         if options.skip_submodule_check:
             cmd.append('--skip-submodule-check')

         if not use_constraint_file:
             cmd.append('--unpin-pip-packages')

         for f in options.additional_cipd_files:
             f = f.replace('$PW_ROOT', str(pigweed_root))
             cmd.extend(('--additional-cipd-file', f))

         cmd.append('--config-file')
         cmd.append(path(options.config_file or 'pigweed.json', checkout))

         with self.m.step.nest('run pw_env_setup') as pres:
             with self.m.defer.context() as defer:
                 with env(), self.m.default_timeout():
                     result = defer(self.m.step, 'pw_env_setup', cmd)

                 defer(self.m.file.listdir, 'ls', env.dir, recursive=True)

                 defer(
                     self.m.save_logs,
                     dirs=(env.dir,),
                     pres=pres,
                     step_passed=result.is_ok(),
                     step_name='environment-setup',
                 )

         json_data = self.m.file.read_json(
             'read json file',
             json_file,
             test_data={
                 'set': {'VIRTUAL_ENV': '/environment/virtualenv'},
                 'modify': {
                     'PATH': {'append': ['/environment/bin']},
                     'LD_LIBRARY_PATH': {'prepend': ['/environment/lib']},
                 },
             },
         )
         top_presentation.logs['vars.json'] = pprint.pformat(json_data)

         env_gni_path = (
             checkout.root / 'build_overrides' / 'pigweed_environment.gni'
         )

         self.m.path.mock_add_file(env_gni_path)
         if self.m.path.isfile(env_gni_path):
             environment_gni = self.m.file.read_text(
                 'read gni file', env_gni_path
             )
             top_presentation.logs['pigweed_environment.gni'] = environment_gni

         for var, value in json_data['set'].items():
             env.env[var] = value

         for var, actions in json_data['modify'].items():
             for value in actions.get('prepend', ()):
                 env.prefixes.setdefault(var, [])
                 env.prefixes[var].append(value)
             for value in actions.get('append', ()):
                 env.suffixes.setdefault(var, [])
                 env.suffixes[var].append(value)

     def _toolchain_override(self, env: Environment) -> None:
         """Checks for a toolchain override and applies it."""

         # Using '$fuchsia/checkout' properties to simplify interface with the
         # Fuchsia Toolchain team.
         fuchsia_build_props = self.m.properties.thaw().get(
             '$fuchsia/checkout', {}
         )
         toolchain_props = fuchsia_build_props.get('clang_toolchain', {})
         if not toolchain_props:
             return

         with self.m.step.nest('toolchain override'):
             with self.m.context(infra_steps=True):
                 toolchain_dir = env.dir / 'override' / 'clang_toolchain'

                 if cipd_version := toolchain_props.get('cipd_version'):
                     pkgs = self.m.cipd.EnsureFile()
                     pkgs.add_package(
                         'fuchsia/third_party/clang/${platform}',
                         cipd_version,
                     )
                     self.m.cipd.ensure(toolchain_dir, pkgs)

                 elif cas_digest := toolchain_props.get('cas_digest'):
                     with self.m.cas.with_instance(
                         'projects/chromium-swarm/instances/default_instance'
                     ):
                         self.m.cas.download(
                             'download',
                             digest=cas_digest,
                             output_dir=toolchain_dir,
                         )

                 else:  # pragma: no cover
                     raise KeyError(
                         f'invalid clang toolchain properties: {toolchain_props!r}'
                     )

                 env.prefixes.setdefault('PATH', [])
                 env.prefixes['PATH'].append(toolchain_dir)
                 env.prefixes['PATH'].append(toolchain_dir / 'bin')

                 clang_prefix = toolchain_dir / 'bin'
                 env.override_gn_args['pw_toolchain_CLANG_PREFIX'] = (
                     f'{clang_prefix}/'
                 )

     @recipe_api.ignore_warnings('recipe_engine/PYTHON2_DEPRECATED')
     def init(
         self,
         checkout: checkout_api.CheckoutContext,
         options: Options | None = None,
         use_constraint_file: bool = True,
     ) -> Environment:
         pigweed_root = checkout.root
         env = Environment(
             api=self.m,
             dir=checkout.root / 'environment',
             checkout=checkout,
         )

         # If in recipe tests always add at least one variable to make it easier
         # to test use of variables in recipes.
         if self._test_data.enabled:
             env.env['PW_TEST_VAR'] = 'test_value'

         if not options:
             options = Options()

         if not options.config_file:
             # Always set certain variables even without an environment config.
             self._init_misc_vars(env, options.additional_variables)
             return env

         if options.relative_pigweed_root not in (None, '', '.'):
             pigweed_root = checkout.root / options.relative_pigweed_root

         with self.m.step.nest('environment') as pres:
             with self.m.step.nest('options') as options_pres:
                 options_pres.step_summary_text = repr(options)

             cfg_json = self.m.file.read_json(
                 'read config',
                 path(options.config_file, checkout),
                 test_data={
                     'pw': {
                         'pw_env_setup': {
                             'relative_pigweed_root': 'pigweed',
                         },
                     },
                 },
             )
             pres.logs['config.json'] = pprint.pformat(cfg_json)

             cfg_json = cfg_json.get('pw', cfg_json)
             cfg_json = cfg_json.get('pw_env_setup', cfg_json)
             if 'relative_pigweed_root' in cfg_json:
                 pigweed_root = checkout.root / cfg_json['relative_pigweed_root']

             env.env['PW_PROJECT_ROOT'] = checkout.root
             env.env['PW_ROOT'] = pigweed_root

             self._init_platform(env)
             self._init_misc_vars(env, options.additional_variables)
             self._init_pigweed(
                 checkout=checkout,
                 top_presentation=pres,
                 use_constraint_file=use_constraint_file,
                 pigweed_root=pigweed_root,
                 options=options,
                 env=env,
             )
             self._toolchain_override(env)

             with env():
                 try:
                     self.m.step('doctor', ['python', '-m', 'pw_cli', 'doctor'])
                 except self.m.step.StepFailure:
                     if not options.allow_doctor_to_fail:
                         raise

         return env
	# Copyright 2020 The Pigweed Authors
	#
	# Licensed under the Apache License, Version 2.0 (the "License"); you may not
	# use this file except in compliance with the License. You may obtain a copy of
	# the License at
	#
	# https://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
	# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
	# License for the specific language governing permissions and limitations under
	# the License.
	"""Environment utility functions.

	Usage:
	env = api.environment.initialize(checkout, env_options)
	with env():
	...
	"""

	from __future__ import annotations

	import contextlib
	import dataclasses
	import pprint
	from typing import TYPE_CHECKING

	from PB.recipe_modules.pigweed.environment.options import Options
	from recipe_engine import recipe_api

	if TYPE_CHECKING: # pragma: no cover
	from recipe_engine import config_types, engine_types
	from RECIPE_MODULES.pigweed.checkout import api as checkout_api


	@dataclasses.dataclass
	class Environment:
	api: recipe_api.RecipeApi
	dir: config_types.Path
	checkout: checkout_api.CheckoutContext
	prefixes: dict[str, list[str]] = dataclasses.field(default_factory=dict)
	suffixes: dict[str, list[str]] = dataclasses.field(default_factory=dict)
	env: dict[str, str] = dataclasses.field(default_factory=dict)
	override_gn_args: dict = dataclasses.field(default_factory=dict)

	@contextlib.contextmanager
	def __call__(self):
	with self.api.macos_sdk():
	# Using reversed() because things that are added later in
	# environment setup need to override things that came earlier.
	with self.api.context(
	env_prefixes={k: reversed(v) for k, v in self.prefixes.items()},
	env_suffixes=self.suffixes,
	env=self.env,
	):
	yield self

	def __getattr__(self, name) -> str:
	if name not in self.env:
	raise AttributeError(name)

	return self.env.get(name)


	def path(
	relative_path: config_types.Path,
	checkout: checkout_api.CheckoutContext,
	) -> config_types.Path:
	parts = [x for x in relative_path.split('/') if x not in ('.', u'.')]
	if parts:
	return checkout.root.joinpath(*parts)
	else:
	return checkout.root # pragma: no cover


	class EnvironmentApi(recipe_api.RecipeApi):
	"""Environment utility functions."""

	Environment = Environment

	def _init_platform(self, env: Environment) -> None:
	if self.m.platform.is_mac:
	with self.m.step.nest('setup platform'):
	with self.m.macos_sdk():
	pass

	def _init_misc_vars(
	self,
	env: Environment,
	additional_variables: dict[str, str] \| None = None,
	) -> None:
	env.env['PW_ENVIRONMENT_NO_ERROR_ON_UNRECOGNIZED'] = '1'
	env.env['PW_ENVSETUP_DISABLE_SPINNER'] = '1'
	env.env['PW_PRESUBMIT_DISABLE_SUBPROCESS_CAPTURE'] = '1'
	env.env['PW_USE_COLOR'] = ''
	env.env['CLICOLOR'] = '0' # Formerly on https://bixense.com/clicolors/.
	env.env['NO_COLOR'] = '1' # See https://no-color.org.
	# This should tell ninja to disable colors based on implementation at
	# https://github.com/ninja-build/ninja/blob/master/src/line_printer.cc#L60.
	env.env['CLICOLOR_FORCE'] = '0'
	env.env['GCC_COLORS'] = ''

	env.env.update(additional_variables or {})

	if self.m.led.launched_by_led and not self.m.led.led_build:
	# Not using self.m.buildbucket_util.id because some downstream
	# projects need this id to never be longer than a typical
	# buildbucket id. Shorter is fine.
	env.env['BUILDBUCKET_ID'] = '0'
	env.env['BUILD_NUMBER'] = str(self.m.buildbucket.build.number)

	else:
	env.env['BUILDBUCKET_ID'] = str(self.m.buildbucket.build.id)
	env.env['BUILD_NUMBER'] = str(self.m.buildbucket.build.number)

	env.env['BUILDBUCKET_NAME'] = ':'.join(
	(
	self.m.buildbucket.build.builder.project,
	self.m.buildbucket.build.builder.bucket,
	self.m.buildbucket.build.builder.builder,
	)
	)

	if env.env['BUILDBUCKET_NAME'] == '::':
	env.env['BUILDBUCKET_NAME'] = 'project:bucket:builder'

	env.env['CCACHE_DIR'] = self.m.path.cache_dir / 'ccache'
	env.env['CTCACHE_DIR'] = self.m.path.cache_dir / 'clang_tidy'
	env.env['GOCACHE'] = self.m.path.cache_dir / 'go'
	env.env['PIP_CACHE_DIR'] = self.m.path.cache_dir / 'pip'
	env.env['TEST_TMPDIR'] = self.m.path.cache_dir / 'bazel'

	env.env['TRIGGERING_CHANGES_JSON'] = env.checkout.changes_json

	def _init_pigweed(
	self,
	checkout: checkout_api.CheckoutContext,
	top_presentation: engine_types.StepPresentation,
	use_constraint_file: bool,
	pigweed_root: config_types.Path,
	options: Options,
	env: Environment,
	) -> None:
	"""Run pw_env_setup."""

	json_file = env.dir / 'vars.json'
	shell_file = env.dir / 'setup.sh'
	venv_dir = env.dir / 'venv'

	self.m.file.ensure_directory(
	f'mkdir {self.m.path.basename(env.dir)}',
	env.dir,
	)

	self.m.file.ensure_directory(
	f'mkdir {self.m.path.basename(venv_dir)}',
	venv_dir,
	)

	cmd: list[str \| config_types.Path] = [
	'python3',
	(
	pigweed_root
	/ 'pw_env_setup'
	/ 'py'
	/ 'pw_env_setup'
	/ 'env_setup.py'
	),
	'--pw-root',
	pigweed_root,
	'--install-dir',
	env.dir,
	'--json-file',
	json_file,
	'--shell-file',
	shell_file,
	'--virtualenv-gn-out-dir',
	env.dir / 'out',
	'--use-existing-cipd',
	'--strict',
	'--disable-rosetta',
	]

	if options.skip_submodule_check:
	cmd.append('--skip-submodule-check')

	if not use_constraint_file:
	cmd.append('--unpin-pip-packages')

	for f in options.additional_cipd_files:
	f = f.replace('$PW_ROOT', str(pigweed_root))
	cmd.extend(('--additional-cipd-file', f))

	cmd.append('--config-file')
	cmd.append(path(options.config_file or 'pigweed.json', checkout))

	with self.m.step.nest('run pw_env_setup') as pres:
	with self.m.defer.context() as defer:
	with env(), self.m.default_timeout():
	result = defer(self.m.step, 'pw_env_setup', cmd)

	defer(self.m.file.listdir, 'ls', env.dir, recursive=True)

	defer(
	self.m.save_logs,
	dirs=(env.dir,),
	pres=pres,
	step_passed=result.is_ok(),
	step_name='environment-setup',
	)

	json_data = self.m.file.read_json(
	'read json file',
	json_file,
	test_data={
	'set': {'VIRTUAL_ENV': '/environment/virtualenv'},
	'modify': {
	'PATH': {'append': ['/environment/bin']},
	'LD_LIBRARY_PATH': {'prepend': ['/environment/lib']},
	},
	},
	)
	top_presentation.logs['vars.json'] = pprint.pformat(json_data)

	env_gni_path = (
	checkout.root / 'build_overrides' / 'pigweed_environment.gni'
	)

	self.m.path.mock_add_file(env_gni_path)
	if self.m.path.isfile(env_gni_path):
	environment_gni = self.m.file.read_text(
	'read gni file', env_gni_path
	)
	top_presentation.logs['pigweed_environment.gni'] = environment_gni

	for var, value in json_data['set'].items():
	env.env[var] = value

	for var, actions in json_data['modify'].items():
	for value in actions.get('prepend', ()):
	env.prefixes.setdefault(var, [])
	env.prefixes[var].append(value)
	for value in actions.get('append', ()):
	env.suffixes.setdefault(var, [])
	env.suffixes[var].append(value)

	def _toolchain_override(self, env: Environment) -> None:
	"""Checks for a toolchain override and applies it."""

	# Using '$fuchsia/checkout' properties to simplify interface with the
	# Fuchsia Toolchain team.
	fuchsia_build_props = self.m.properties.thaw().get(
	'$fuchsia/checkout', {}
	)
	toolchain_props = fuchsia_build_props.get('clang_toolchain', {})
	if not toolchain_props:
	return

	with self.m.step.nest('toolchain override'):
	with self.m.context(infra_steps=True):
	toolchain_dir = env.dir / 'override' / 'clang_toolchain'

	if cipd_version := toolchain_props.get('cipd_version'):
	pkgs = self.m.cipd.EnsureFile()
	pkgs.add_package(
	'fuchsia/third_party/clang/${platform}',
	cipd_version,
	)
	self.m.cipd.ensure(toolchain_dir, pkgs)

	elif cas_digest := toolchain_props.get('cas_digest'):
	with self.m.cas.with_instance(
	'projects/chromium-swarm/instances/default_instance'
	):
	self.m.cas.download(
	'download',
	digest=cas_digest,
	output_dir=toolchain_dir,
	)

	else: # pragma: no cover
	raise KeyError(
	f'invalid clang toolchain properties: {toolchain_props!r}'
	)

	env.prefixes.setdefault('PATH', [])
	env.prefixes['PATH'].append(toolchain_dir)
	env.prefixes['PATH'].append(toolchain_dir / 'bin')

	clang_prefix = toolchain_dir / 'bin'
	env.override_gn_args['pw_toolchain_CLANG_PREFIX'] = (
	f'{clang_prefix}/'
	)

	@recipe_api.ignore_warnings('recipe_engine/PYTHON2_DEPRECATED')
	def init(
	self,
	checkout: checkout_api.CheckoutContext,
	options: Options \| None = None,
	use_constraint_file: bool = True,
	) -> Environment:
	pigweed_root = checkout.root
	env = Environment(
	api=self.m,
	dir=checkout.root / 'environment',
	checkout=checkout,
	)

	# If in recipe tests always add at least one variable to make it easier
	# to test use of variables in recipes.
	if self._test_data.enabled:
	env.env['PW_TEST_VAR'] = 'test_value'

	if not options:
	options = Options()

	if not options.config_file:
	# Always set certain variables even without an environment config.
	self._init_misc_vars(env, options.additional_variables)
	return env

	if options.relative_pigweed_root not in (None, '', '.'):
	pigweed_root = checkout.root / options.relative_pigweed_root

	with self.m.step.nest('environment') as pres:
	with self.m.step.nest('options') as options_pres:
	options_pres.step_summary_text = repr(options)

	cfg_json = self.m.file.read_json(
	'read config',
	path(options.config_file, checkout),
	test_data={
	'pw': {
	'pw_env_setup': {
	'relative_pigweed_root': 'pigweed',
	},
	},
	},
	)
	pres.logs['config.json'] = pprint.pformat(cfg_json)

	cfg_json = cfg_json.get('pw', cfg_json)
	cfg_json = cfg_json.get('pw_env_setup', cfg_json)
	if 'relative_pigweed_root' in cfg_json:
	pigweed_root = checkout.root / cfg_json['relative_pigweed_root']

	env.env['PW_PROJECT_ROOT'] = checkout.root
	env.env['PW_ROOT'] = pigweed_root

	self._init_platform(env)
	self._init_misc_vars(env, options.additional_variables)
	self._init_pigweed(
	checkout=checkout,
	top_presentation=pres,
	use_constraint_file=use_constraint_file,
	pigweed_root=pigweed_root,
	options=options,
	env=env,
	)
	self._toolchain_override(env)

	with env():
	try:
	self.m.step('doctor', ['python', '-m', 'pw_cli', 'doctor'])
	except self.m.step.StepFailure:
	if not options.allow_doctor_to_fail:
	raise

	return env