recipes/pw_presubmit.py - infra/recipes - Git at Google

 # Copyright 2020 The Pigweed Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not
 # use this file except in compliance with the License. You may obtain a copy of
 # the License at
 #
 #     https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations under
 # the License.
 """Recipe for testing Pigweed using presubmit_checks.py script."""

 import datetime
 import re
 from typing import Generator

 from PB.go.chromium.org.luci.buildbucket.proto import common
 from PB.recipes.pigweed.pw_presubmit import InputProperties, StepName
 from PB.recipe_engine import result

 from recipe_engine import (
     config_types,
     post_process,
     recipe_api,
     recipe_test_api,
 )

 DEPS = [
     'fuchsia/gsutil',
     'pigweed/checkout',
     'pigweed/ci_status',
     'pigweed/environment',
     'pigweed/pw_presubmit',
     'pigweed/util',
     'recipe_engine/cq',
     'recipe_engine/cv',
     'recipe_engine/defer',
     'recipe_engine/file',
     'recipe_engine/futures',
     'recipe_engine/json',
     'recipe_engine/path',
     'recipe_engine/properties',
     'recipe_engine/raw_io',
     'recipe_engine/step',
     'recipe_engine/time',
 ]

 PROPERTIES = InputProperties

 # The path to a public key used to sign release builds. Only set on release
 # builders.
 RELEASE_PUBKEY_PATH = '/etc/release_keys/release_key_pub.pem'

 # The name of the public key file uploaded in release builds.
 RELEASE_PUBKEY_FILENAME = 'publickey.pem'


 def _try_sign_archive(
     api: recipe_api.RecipeScriptApi,
     archive_path: config_types.Path,
     name: str,
 ) -> str:
     cmd: list[str | config_types.Path] = [
         'vpython3',
         '-vpython-spec',
         api.resource('sign.py.vpython'),
         '-u',
         api.resource('sign.py'),
         '--archive-file',
         archive_path,
     ]

     return api.step(
         f'sign {name}',
         cmd,
         stdout=api.raw_io.output_text(),
     ).stdout


 def RunSteps(
     api: recipe_api.RecipeScriptApi,
     props: InputProperties,
 ) -> result.RawResult | None:
     """Run Pigweed presubmit checks."""
     gcs_bucket = props.gcs_bucket

     if res := api.ci_status.exit_early_in_recipe_testing_if_failing():
         return res  # pragma: no cover

     checkout = api.checkout(props.checkout_options)
     env = api.environment.init(checkout, props.environment_options)

     with env():
         presubmit = api.pw_presubmit.init(checkout, props.pw_presubmit_options)

         for change in checkout.changes:
             if 'build-errors: continue' in change.commit_message.lower():
                 presubmit.options.continue_after_build_error = True

         with api.defer.context() as defer:
             for step in presubmit.steps:
                 defer(api.pw_presubmit.run, ctx=presubmit, step=step, env=env)

     metadata = {}
     steps_with_metadata = set()
     for step in presubmit.steps:
         for metadata_type, data in step.metadata.items():
             metadata.setdefault(metadata_type, {})
             for key, value in data.items():
                 steps_with_metadata.add(step.name)
                 metadata[metadata_type][f'{step.name}.{key}'] = value

     if metadata:
         # Change metadata output like the following:
         #
         #   "binary_sizes":  {
         #     "step1.foo": 123,
         #     "step1.bar": 456,
         #   }
         #
         # For (STEP_NAME_DEFAULT and one step) or WITH_WITHOUT_STEP_NAME:
         #
         #   "binary_sizes":  {
         #     "foo": 123,
         #     "bar": 456,
         #     "step1.foo": 123,
         #     "step1.bar": 456,
         #   }
         #
         # For ONLY_WITHOUT_STEP_NAME:
         #
         #   "binary_sizes":  {
         #     "foo": 123,
         #     "bar": 456,
         #   }
         #
         # For (STEP_NAME_DEFAULT and multiple steps) or ONLY_WITH_STEP_NAME
         # (unchanged):
         #
         #   "binary_sizes":  {
         #     "step1.foo": 123,
         #     "step1.bar": 456,
         #   }
         #
         # These options exist because we might be comparing size outputs from
         # steps with different names, or we might want soft transitions from
         # one step name to another.

         with api.step.nest('metadata') as pres:
             step_usage = StepName.Name(props.metadata_step_name_usage)
             if step_usage == 'STEP_NAME_DEFAULT':
                 if len(steps_with_metadata) == 1:
                     step_usage = 'WITH_WITHOUT_STEP_NAME'
                 else:
                     step_usage = 'ONLY_WITH_STEP_NAME'

             for data in metadata.values():
                 for key in set(data.keys()):
                     if step_usage == 'WITH_WITHOUT_STEP_NAME':
                         # Need both 'foo' and 'step1.foo'.
                         data[key.split('.', 1)[1]] = data[key]
                     elif step_usage == 'ONLY_WITH_STEP_NAME':
                         # Only need 'step1.foo', good as is.
                         pass
                     elif step_usage == 'ONLY_WITHOUT_STEP_NAME':
                         # Only need 'foo', need to delete 'step1.foo'.
                         data[key.split('.', 1)[1]] = data[key]
                         del data[key]
                     else:
                         raise ValueError(str(step_usage))  # pragma: no cover

             for name, data in metadata.items():
                 pres.properties[name] = data

     if gcs_bucket:
         uploaded_public_key = False

         with api.step.nest('upload') as pres:
             with env():
                 namespace = api.pw_presubmit.build_id(presubmit)

             checkout_dir = api.path.start_dir / 'checkout_upload'
             checkout.snapshot_to_dir(checkout_dir)
             futures = [
                 api.futures.spawn(
                     api.gsutil.upload_namespaced_directory,
                     source=checkout_dir,
                     bucket=gcs_bucket,
                     subpath='checkout',
                     namespace=namespace,
                 )
             ]

             futures.append(
                 api.futures.spawn(
                     api.gsutil.upload_namespaced_file,
                     source=api.json.input(api.util.build_metadata()),
                     bucket=gcs_bucket,
                     subpath='build_metadata.json',
                     namespace=namespace,
                 )
             )

             for step in presubmit.steps:
                 if not step.export_dir:
                     continue  # pragma: no cover

                 # In testing this will never be true because of the
                 # mock_add_file() call for binary_sizes.json.
                 if not api.path.exists(step.export_dir):
                     continue  # pragma: no cover

                 for entry in api.file.listdir(
                     f'ls {step.name}/{presubmit.options.export_dir_name}',
                     step.export_dir,
                     recursive=True,
                 ):
                     metadata = None

                     ext = api.path.splitext(entry)[1]
                     if ext in props.extensions_to_sign:
                         signature = _try_sign_archive(
                             api,
                             entry,
                             name=api.path.relpath(entry, presubmit.root),
                         )
                         if signature:
                             metadata = {
                                 "x-goog-meta-signature": signature,
                             }
                             if not uploaded_public_key:
                                 futures.append(
                                     api.futures.spawn(
                                         api.gsutil.upload_namespaced_file,
                                         source=RELEASE_PUBKEY_PATH,
                                         bucket=gcs_bucket,
                                         subpath=RELEASE_PUBKEY_FILENAME,
                                         namespace=namespace,
                                     )
                                 )
                                 uploaded_public_key = True

                     futures.append(
                         api.futures.spawn(
                             api.gsutil.upload_namespaced_file,
                             source=entry,
                             bucket=gcs_bucket,
                             subpath='{}/{}'.format(
                                 step.name,
                                 api.path.relpath(entry, step.export_dir),
                             ),
                             namespace=namespace,
                             metadata=metadata,
                         )
                     )

             # Need to wait for results but don't care about their values.
             _ = [f.result() for f in futures]

             # This file tells other users of the bucket that the upload is
             # complete.
             api.gsutil.upload_namespaced_file(
                 source=api.raw_io.input(''),
                 bucket=gcs_bucket,
                 subpath='upload_complete',
                 namespace=namespace,
             )

             browse_link = api.gsutil.namespaced_directory_url(gcs_bucket)
             pres.links['browse'] = browse_link

             return result.RawResult(
                 summary_markdown=f'[artifacts]({browse_link})',
                 status=common.SUCCESS,
             )


 def GenTests(api) -> Generator[recipe_test_api.TestData, None, None]:
     """Create tests."""

     def ls_export(step_name, *files):
         return api.path.exists(
             api.path.start_dir / 'presubmit' / step_name / 'export'
         ) + api.step_data(
             f'upload.ls {step_name}/export',
             api.file.listdir(files),
         )

     def signature(step_name, filename):
         return api.step_data(
             f'upload.sign {step_name}/export/{filename}',
             stdout=api.raw_io.output_text('John Hancock'),
         )

     def properties(
         *,
         num_ci_failures_to_trigger_exiting_early=0,
         extensions_to_sign=('.out',),
         gcs_bucket=None,
         metadata_step_name_usage=None,
         **kwargs,
     ):
         props = InputProperties()
         props.checkout_options.CopyFrom(api.checkout.git_options())
         props.pw_presubmit_options.CopyFrom(api.pw_presubmit.options(**kwargs))
         if metadata_step_name_usage:
             props.metadata_step_name_usage = StepName.Value(
                 metadata_step_name_usage
             )
         props.extensions_to_sign.extend(extensions_to_sign)
         props.num_ci_failures_to_trigger_exiting_early = (
             num_ci_failures_to_trigger_exiting_early
         )
         if gcs_bucket:
             props.gcs_bucket = gcs_bucket
         return api.properties(props)

     def ran(x):
         return api.post_process(post_process.MustRun, x)

     def drop_expectations_must_be_last():
         return api.post_process(post_process.DropExpectation)

     yield api.test(
         'one_step_no_exit_passing_in_ci',
         properties(step=['step1']),
         api.checkout.try_test_data(),
         api.cv(run_mode=api.cq.DRY_RUN),
         ran('step1'),
         drop_expectations_must_be_last(),
     )

     yield api.test(
         'one_step_no_exit_not_tryjob',
         properties(step=['step1']),
         api.checkout.ci_test_data(),
         api.cv(run_mode=api.cv.DRY_RUN),
         ran('step1'),
         drop_expectations_must_be_last(),
     )

     yield api.test(
         'one_step_no_exit_not_in_cv',
         properties(step=['step1']),
         api.checkout.try_test_data(),
         ran('step1'),
         drop_expectations_must_be_last(),
     )

     yield api.test(
         'two_steps',
         properties(step=['step1', 'step2'], gcs_bucket='bucket'),
         api.checkout.try_test_data(
             start_time=datetime.datetime.utcfromtimestamp(1600000000),
             execution_timeout=120,
         ),
         api.checkout.cl_branch_parents(message='Build-Errors: continue'),
         api.step_data('upload.get build id', retcode=1),
         ls_export('step1', 'foo'),
         api.time.seed(1600000000),
         api.time.step(20.0),
         ran('step1'),
         ran('step2'),
         ran('upload'),
         drop_expectations_must_be_last(),
     )

     yield api.test(
         'sign',
         properties(
             step=['release'],
             gcs_bucket='bucket',
             extensions_to_sign=['.foo'],
             metadata_step_name_usage='ONLY_WITHOUT_STEP_NAME',
         ),
         api.checkout.ci_test_data(),
         ls_export('release', '1.foo', '2.bar'),
         signature('release', '1.foo'),
         ran('release'),
         ran('upload.sign release/export/1.foo'),
         drop_expectations_must_be_last(),
     )
	# Copyright 2020 The Pigweed Authors
	#
	# Licensed under the Apache License, Version 2.0 (the "License"); you may not
	# use this file except in compliance with the License. You may obtain a copy of
	# the License at
	#
	# https://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
	# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
	# License for the specific language governing permissions and limitations under
	# the License.
	"""Recipe for testing Pigweed using presubmit_checks.py script."""

	import datetime
	import re
	from typing import Generator

	from PB.go.chromium.org.luci.buildbucket.proto import common
	from PB.recipes.pigweed.pw_presubmit import InputProperties, StepName
	from PB.recipe_engine import result

	from recipe_engine import (
	config_types,
	post_process,
	recipe_api,
	recipe_test_api,
	)

	DEPS = [
	'fuchsia/gsutil',
	'pigweed/checkout',
	'pigweed/ci_status',
	'pigweed/environment',
	'pigweed/pw_presubmit',
	'pigweed/util',
	'recipe_engine/cq',
	'recipe_engine/cv',
	'recipe_engine/defer',
	'recipe_engine/file',
	'recipe_engine/futures',
	'recipe_engine/json',
	'recipe_engine/path',
	'recipe_engine/properties',
	'recipe_engine/raw_io',
	'recipe_engine/step',
	'recipe_engine/time',
	]

	PROPERTIES = InputProperties

	# The path to a public key used to sign release builds. Only set on release
	# builders.
	RELEASE_PUBKEY_PATH = '/etc/release_keys/release_key_pub.pem'

	# The name of the public key file uploaded in release builds.
	RELEASE_PUBKEY_FILENAME = 'publickey.pem'


	def _try_sign_archive(
	api: recipe_api.RecipeScriptApi,
	archive_path: config_types.Path,
	name: str,
	) -> str:
	cmd: list[str \| config_types.Path] = [
	'vpython3',
	'-vpython-spec',
	api.resource('sign.py.vpython'),
	'-u',
	api.resource('sign.py'),
	'--archive-file',
	archive_path,
	]

	return api.step(
	f'sign {name}',
	cmd,
	stdout=api.raw_io.output_text(),
	).stdout


	def RunSteps(
	api: recipe_api.RecipeScriptApi,
	props: InputProperties,
	) -> result.RawResult \| None:
	"""Run Pigweed presubmit checks."""
	gcs_bucket = props.gcs_bucket

	if res := api.ci_status.exit_early_in_recipe_testing_if_failing():
	return res # pragma: no cover

	checkout = api.checkout(props.checkout_options)
	env = api.environment.init(checkout, props.environment_options)

	with env():
	presubmit = api.pw_presubmit.init(checkout, props.pw_presubmit_options)

	for change in checkout.changes:
	if 'build-errors: continue' in change.commit_message.lower():
	presubmit.options.continue_after_build_error = True

	with api.defer.context() as defer:
	for step in presubmit.steps:
	defer(api.pw_presubmit.run, ctx=presubmit, step=step, env=env)

	metadata = {}
	steps_with_metadata = set()
	for step in presubmit.steps:
	for metadata_type, data in step.metadata.items():
	metadata.setdefault(metadata_type, {})
	for key, value in data.items():
	steps_with_metadata.add(step.name)
	metadata[metadata_type][f'{step.name}.{key}'] = value

	if metadata:
	# Change metadata output like the following:
	#
	# "binary_sizes": {
	# "step1.foo": 123,
	# "step1.bar": 456,
	# }
	#
	# For (STEP_NAME_DEFAULT and one step) or WITH_WITHOUT_STEP_NAME:
	#
	# "binary_sizes": {
	# "foo": 123,
	# "bar": 456,
	# "step1.foo": 123,
	# "step1.bar": 456,
	# }
	#
	# For ONLY_WITHOUT_STEP_NAME:
	#
	# "binary_sizes": {
	# "foo": 123,
	# "bar": 456,
	# }
	#
	# For (STEP_NAME_DEFAULT and multiple steps) or ONLY_WITH_STEP_NAME
	# (unchanged):
	#
	# "binary_sizes": {
	# "step1.foo": 123,
	# "step1.bar": 456,
	# }
	#
	# These options exist because we might be comparing size outputs from
	# steps with different names, or we might want soft transitions from
	# one step name to another.

	with api.step.nest('metadata') as pres:
	step_usage = StepName.Name(props.metadata_step_name_usage)
	if step_usage == 'STEP_NAME_DEFAULT':
	if len(steps_with_metadata) == 1:
	step_usage = 'WITH_WITHOUT_STEP_NAME'
	else:
	step_usage = 'ONLY_WITH_STEP_NAME'

	for data in metadata.values():
	for key in set(data.keys()):
	if step_usage == 'WITH_WITHOUT_STEP_NAME':
	# Need both 'foo' and 'step1.foo'.
	data[key.split('.', 1)[1]] = data[key]
	elif step_usage == 'ONLY_WITH_STEP_NAME':
	# Only need 'step1.foo', good as is.
	pass
	elif step_usage == 'ONLY_WITHOUT_STEP_NAME':
	# Only need 'foo', need to delete 'step1.foo'.
	data[key.split('.', 1)[1]] = data[key]
	del data[key]
	else:
	raise ValueError(str(step_usage)) # pragma: no cover

	for name, data in metadata.items():
	pres.properties[name] = data

	if gcs_bucket:
	uploaded_public_key = False

	with api.step.nest('upload') as pres:
	with env():
	namespace = api.pw_presubmit.build_id(presubmit)

	checkout_dir = api.path.start_dir / 'checkout_upload'
	checkout.snapshot_to_dir(checkout_dir)
	futures = [
	api.futures.spawn(
	api.gsutil.upload_namespaced_directory,
	source=checkout_dir,
	bucket=gcs_bucket,
	subpath='checkout',
	namespace=namespace,
	)
	]

	futures.append(
	api.futures.spawn(
	api.gsutil.upload_namespaced_file,
	source=api.json.input(api.util.build_metadata()),
	bucket=gcs_bucket,
	subpath='build_metadata.json',
	namespace=namespace,
	)
	)

	for step in presubmit.steps:
	if not step.export_dir:
	continue # pragma: no cover

	# In testing this will never be true because of the
	# mock_add_file() call for binary_sizes.json.
	if not api.path.exists(step.export_dir):
	continue # pragma: no cover

	for entry in api.file.listdir(
	f'ls {step.name}/{presubmit.options.export_dir_name}',
	step.export_dir,
	recursive=True,
	):
	metadata = None

	ext = api.path.splitext(entry)[1]
	if ext in props.extensions_to_sign:
	signature = _try_sign_archive(
	api,
	entry,
	name=api.path.relpath(entry, presubmit.root),
	)
	if signature:
	metadata = {
	"x-goog-meta-signature": signature,
	}
	if not uploaded_public_key:
	futures.append(
	api.futures.spawn(
	api.gsutil.upload_namespaced_file,
	source=RELEASE_PUBKEY_PATH,
	bucket=gcs_bucket,
	subpath=RELEASE_PUBKEY_FILENAME,
	namespace=namespace,
	)
	)
	uploaded_public_key = True

	futures.append(
	api.futures.spawn(
	api.gsutil.upload_namespaced_file,
	source=entry,
	bucket=gcs_bucket,
	subpath='{}/{}'.format(
	step.name,
	api.path.relpath(entry, step.export_dir),
	),
	namespace=namespace,
	metadata=metadata,
	)
	)

	# Need to wait for results but don't care about their values.
	_ = [f.result() for f in futures]

	# This file tells other users of the bucket that the upload is
	# complete.
	api.gsutil.upload_namespaced_file(
	source=api.raw_io.input(''),
	bucket=gcs_bucket,
	subpath='upload_complete',
	namespace=namespace,
	)

	browse_link = api.gsutil.namespaced_directory_url(gcs_bucket)
	pres.links['browse'] = browse_link

	return result.RawResult(
	summary_markdown=f'[artifacts]({browse_link})',
	status=common.SUCCESS,
	)


	def GenTests(api) -> Generator[recipe_test_api.TestData, None, None]:
	"""Create tests."""

	def ls_export(step_name, *files):
	return api.path.exists(
	api.path.start_dir / 'presubmit' / step_name / 'export'
	) + api.step_data(
	f'upload.ls {step_name}/export',
	api.file.listdir(files),
	)

	def signature(step_name, filename):
	return api.step_data(
	f'upload.sign {step_name}/export/{filename}',
	stdout=api.raw_io.output_text('John Hancock'),
	)

	def properties(
	*,
	num_ci_failures_to_trigger_exiting_early=0,
	extensions_to_sign=('.out',),
	gcs_bucket=None,
	metadata_step_name_usage=None,
	**kwargs,
	):
	props = InputProperties()
	props.checkout_options.CopyFrom(api.checkout.git_options())
	props.pw_presubmit_options.CopyFrom(api.pw_presubmit.options(**kwargs))
	if metadata_step_name_usage:
	props.metadata_step_name_usage = StepName.Value(
	metadata_step_name_usage
	)
	props.extensions_to_sign.extend(extensions_to_sign)
	props.num_ci_failures_to_trigger_exiting_early = (
	num_ci_failures_to_trigger_exiting_early
	)
	if gcs_bucket:
	props.gcs_bucket = gcs_bucket
	return api.properties(props)

	def ran(x):
	return api.post_process(post_process.MustRun, x)

	def drop_expectations_must_be_last():
	return api.post_process(post_process.DropExpectation)

	yield api.test(
	'one_step_no_exit_passing_in_ci',
	properties(step=['step1']),
	api.checkout.try_test_data(),
	api.cv(run_mode=api.cq.DRY_RUN),
	ran('step1'),
	drop_expectations_must_be_last(),
	)

	yield api.test(
	'one_step_no_exit_not_tryjob',
	properties(step=['step1']),
	api.checkout.ci_test_data(),
	api.cv(run_mode=api.cv.DRY_RUN),
	ran('step1'),
	drop_expectations_must_be_last(),
	)

	yield api.test(
	'one_step_no_exit_not_in_cv',
	properties(step=['step1']),
	api.checkout.try_test_data(),
	ran('step1'),
	drop_expectations_must_be_last(),
	)

	yield api.test(
	'two_steps',
	properties(step=['step1', 'step2'], gcs_bucket='bucket'),
	api.checkout.try_test_data(
	start_time=datetime.datetime.utcfromtimestamp(1600000000),
	execution_timeout=120,
	),
	api.checkout.cl_branch_parents(message='Build-Errors: continue'),
	api.step_data('upload.get build id', retcode=1),
	ls_export('step1', 'foo'),
	api.time.seed(1600000000),
	api.time.step(20.0),
	ran('step1'),
	ran('step2'),
	ran('upload'),
	drop_expectations_must_be_last(),
	)

	yield api.test(
	'sign',
	properties(
	step=['release'],
	gcs_bucket='bucket',
	extensions_to_sign=['.foo'],
	metadata_step_name_usage='ONLY_WITHOUT_STEP_NAME',
	),
	api.checkout.ci_test_data(),
	ls_export('release', '1.foo', '2.bar'),
	signature('release', '1.foo'),
	ran('release'),
	ran('upload.sign release/export/1.foo'),
	drop_expectations_must_be_last(),
	)