recipes/roller.py - infra/recipes - Git at Google

 # Copyright 2024 The Pigweed Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not
 # use this file except in compliance with the License. You may obtain a copy of
 # the License at
 #
 #     https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations under
 # the License.
 """Combined roller for submodules, repo projects, etc."""

 from __future__ import annotations

 from typing import TYPE_CHECKING

 import attrs
 from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb
 from PB.recipe_engine import result as result_pb
 from PB.recipe_modules.fuchsia.auto_roller.options import (
     Options as AutoRollerOptions,
 )
 from PB.recipes.pigweed.roller import InputProperties
 from recipe_engine import post_process

 if TYPE_CHECKING:  # pragma: no cover
     from typing import Generator
     from recipe_engine import recipe_test_api

 DEPS = [
     'fuchsia/auto_roller',
     'fuchsia/git_roll_util',
     'fuchsia/gitiles',
     'fuchsia/roll_commit_message',
     'pigweed/abandon_old_changes',
     'pigweed/bazel_roll',
     'pigweed/checkout',
     'pigweed/cipd_roll',
     'pigweed/copy_roll',
     'pigweed/repo_roll',
     'pigweed/submodule_roll',
     'pigweed/txt_roll',
     'recipe_engine/file',
     'recipe_engine/properties',
     'recipe_engine/step',
     'recipe_engine/time',
 ]

 PROPERTIES = InputProperties


 def merge_auto_roller_overrides(
     auto_roller_options: AutoRollerOptions,
     override_auto_roller_options: AutoRollerOptions,
 ):
     result = AutoRollerOptions()
     result.CopyFrom(auto_roller_options)
     result.MergeFrom(override_auto_roller_options)
     return result


 def RunSteps(  # pylint: disable=invalid-name
     api: recipe_api.RecipeScriptApi,
     props: InputProperties,
 ):
     api.abandon_old_changes(host=props.checkout_options.remote)

     # The checkout module will try to use trigger data to pull in a specific
     # patch. Since the triggering commit is in a different repository that
     # needs to be disabled.
     props.checkout_options.use_trigger = False
     checkout = api.checkout(props.checkout_options)

     rolls = []
     attempts = 0

     for submodule in props.submodules:
         attempts += 1
         rolls.extend(api.submodule_roll.update(checkout, submodule))

     for cipd_package in props.bazel_cipd_packages:
         attempts += 1
         rolls.extend(api.bazel_roll.update_cipd_package(checkout, cipd_package))

     for git_repo in props.bazel_git_repositories:
         attempts += 1
         rolls.extend(api.bazel_roll.update_git_repository(checkout, git_repo))

     for project in props.repo_tool_projects:
         attempts += 1
         rolls.extend(api.repo_roll.update_project(checkout, project))

     for cipd_package in props.cipd_packages:
         attempts += 1
         rolls.extend(api.cipd_roll.update_package(checkout, cipd_package))

     for txt_entry in props.txt_entries:
         attempts += 1
         rolls.extend(api.txt_roll.update(checkout, txt_entry))

     for copy_entry in props.copy_entries:
         attempts += 1
         rolls.extend(api.copy_roll.update(checkout, copy_entry))

     if not attempts:
         summary = 'roller is not configured to roll anything'
         api.step.empty(summary)
         return result_pb.RawResult(
             summary_markdown=summary,
             status=common_pb.INFRA_FAILURE,
         )

     if not rolls:
         summary = 'nothing to roll'
         api.step.empty(summary)
         return result_pb.RawResult(
             summary_markdown=summary,
             status=common_pb.SUCCESS,
         )

     author_override: api.git_roll_util.Author | None = None
     if props.forge_author:
         author_override = api.git_roll_util.get_author_override(*rolls)

     commit_message = api.roll_commit_message.format(
         *rolls,
         roll_prefix='roll:',
         send_comment=True,
         commit_divider=props.commit_divider,
         header_override=props.header_override,
         footers=list(props.footer or [f'Roll-Count: {len(rolls)}']),
     )

     pres = api.step.empty('commit message').presentation
     pres.logs['commit message'] = commit_message
     for roll in rolls:
         pres.properties[roll.short_name()] = roll.output_property()
     pres.properties['_roll_count'] = len(rolls)

     auto_roller_options = merge_auto_roller_overrides(
         props.auto_roller_options, props.override_auto_roller_options
     )

     change = api.auto_roller.attempt_roll(
         auto_roller_options,
         repo_dir=checkout.root,
         commit_message=commit_message,
         author_override=author_override,
     )

     result = api.auto_roller.raw_result(change)
     api.time.sleep(props.post_roll_sleep_secs)
     return result


 def GenTests(api) -> Generator[recipe_test_api.TestData, None, None]:
     """Create tests."""

     def ensure_list(x):
         if isinstance(x, tuple):
             return list(x)
         if isinstance(x, list):
             return x  # pragma: no cover
         return [x]

     def properties(
         *,
         bazel_cipd_packages=(),
         bazel_git_repositories=(),
         cipd_packages=(),
         copy_entries=(),
         repo_tool_projects=(),
         submodules=(),
         txt_entries=(),
         **kwargs,
     ):
         props = InputProperties(**kwargs)
         props.checkout_options.CopyFrom(api.checkout.git_options())

         props.bazel_cipd_packages.extend(ensure_list(bazel_cipd_packages))
         props.bazel_git_repositories.extend(ensure_list(bazel_git_repositories))
         props.cipd_packages.extend(ensure_list(cipd_packages))
         props.copy_entries.extend(ensure_list(copy_entries))
         props.repo_tool_projects.extend(ensure_list(repo_tool_projects))
         props.submodules.extend(ensure_list(submodules))
         props.txt_entries.extend(ensure_list(txt_entries))

         props.auto_roller_options.dry_run = True
         props.auto_roller_options.remote = api.checkout.pigweed_repo

         return api.properties(props)

     yield api.test(
         'empty',
         properties(),
         api.post_process(
             post_process.MustRun,
             'roller is not configured to roll anything',
         ),
         api.post_process(post_process.DropExpectation),
         status='INFRA_FAILURE',
     )

     yield api.test(
         'noop',
         properties(
             bazel_cipd_packages=api.bazel_roll.cipd_package(
                 spec='bazel-cipd/${platform}',
             ),
             bazel_git_repositories=api.bazel_roll.git_repo(
                 name='bazel-git',
                 remote='https://pigweed.googlesource.com/bazel-git',
             ),
             cipd_packages=api.cipd_roll.package_props('cipd/${platform}'),
             copy_entries=api.copy_roll.entry('copy.txt'),
             repo_tool_projects=api.repo_roll.project('project'),
             submodules=api.submodule_roll.entry('submodule'),
             txt_entries=api.txt_roll.entry('text.txt'),
         ),
         api.bazel_roll.workspace_file(
             'bazel-cipd',
             api.bazel_roll.cipd_entry(
                 spec='bazel-cipd/${platform}',
                 tag='git_revision:397a2597cdc237f3026e6143b683be4b9ab60540',
             ),
             api.bazel_roll.git_entry(
                 remote='https://pigweed.googlesource.com/bazel-git',
             ),
         ),
         api.gitiles.log('bazel-git.log bazel-git', 'A', n=0),
         api.cipd_roll.read_file_step_data(
             'cipd',
             'pigweed.json',
             api.cipd_roll.package(
                 'cipd/${platform}',
                 'git_revision:397a2597cdc237f3026e6143b683be4b9ab60540',
                 platforms=['linux-amd64', 'windows-amd64'],
             ),
         ),
         api.step_data(
             'copy.txt.read destination copy.txt',
             api.file.read_text('abc'),
         ),
         api.gitiles.fetch('copy.txt.read source copy.txt', 'abc'),
         api.repo_roll.read_step_data('project'),
         api.gitiles.log('project.log project', 'A', n=0),
         api.submodule_roll.gitmodules(
             prefix='submodule',
             submodule='submodule',
         ),
         api.gitiles.log('submodule.log submodule', 'A', n=0),
         api.gitiles.log('text.txt.log text.txt', 'A', n=0),
         api.post_process(post_process.MustRun, 'nothing to roll'),
         api.post_process(post_process.DropExpectation),
     )

     def expected_roll(name, **expected):
         def compare(actual):
             for k, v in expected.items():
                 if k not in actual or actual[k] != v:
                     return False  # pragma: no cover
             return True

         return api.post_process(
             post_process.PropertyMatchesCallable,
             name,
             compare,
         )

     yield api.test(
         'mega',
         properties(
             bazel_cipd_packages=api.bazel_roll.cipd_package(
                 name='bazel-cipd/${plat}',
             ),
             bazel_git_repositories=api.bazel_roll.git_repo(name='bazel-git'),
             cipd_packages=api.cipd_roll.package_props('cipd/${platform}'),
             copy_entries=api.copy_roll.entry('copy.txt'),
             repo_tool_projects=api.repo_roll.project('project'),
             submodules=api.submodule_roll.entry('submodule'),
             txt_entries=api.txt_roll.entry('text.txt'),
             forge_author=True,
             post_roll_sleep_secs=61,
         ),
         api.gitiles.log('bazel-git.log bazel-git', 'A'),
         expected_roll('bazel-git', old='1' * 40, new='h3ll0'),
         api.cipd_roll.read_file_step_data(
             'cipd',
             'pigweed.json',
             api.cipd_roll.package(
                 'cipd/${platform}',
                 'git_revision:123',
                 platforms=['linux-amd64', 'windows-amd64'],
             ),
         ),
         expected_roll(
             'cipd',
             old='git_revision:123',
             new='git_revision:397a2597cdc237f3026e6143b683be4b9ab60540',
         ),
         api.step_data(
             'copy.txt.read destination copy.txt',
             api.file.read_text('old'),
         ),
         api.gitiles.fetch('copy.txt.read source copy.txt', 'new'),
         expected_roll('copy.txt', old='old', new='new'),
         api.repo_roll.read_step_data('project'),
         api.gitiles.log('project.log project', 'A'),
         expected_roll('project', old='main', new='h3ll0'),
         api.submodule_roll.gitmodules(
             prefix='submodule',
             submodule='submodule',
         ),
         api.gitiles.log('submodule.log submodule', 'A'),
         expected_roll('submodule', old='1' * 40, new='h3ll0'),
         api.gitiles.log('text.txt.log text.txt', 'A'),
         expected_roll('text.txt', old='1' * 40, new='h3ll0'),
         api.auto_roller.dry_run_success(),
         api.post_process(post_process.MustRun, 'sleep 61'),
         api.post_process(post_process.PropertyEquals, '_roll_count', 7),
         api.post_process(post_process.DropExpectation),
     )
	# Copyright 2024 The Pigweed Authors
	#
	# Licensed under the Apache License, Version 2.0 (the "License"); you may not
	# use this file except in compliance with the License. You may obtain a copy of
	# the License at
	#
	# https://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
	# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
	# License for the specific language governing permissions and limitations under
	# the License.
	"""Combined roller for submodules, repo projects, etc."""

	from __future__ import annotations

	from typing import TYPE_CHECKING

	import attrs
	from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb
	from PB.recipe_engine import result as result_pb
	from PB.recipe_modules.fuchsia.auto_roller.options import (
	Options as AutoRollerOptions,
	)
	from PB.recipes.pigweed.roller import InputProperties
	from recipe_engine import post_process

	if TYPE_CHECKING: # pragma: no cover
	from typing import Generator
	from recipe_engine import recipe_test_api

	DEPS = [
	'fuchsia/auto_roller',
	'fuchsia/git_roll_util',
	'fuchsia/gitiles',
	'fuchsia/roll_commit_message',
	'pigweed/abandon_old_changes',
	'pigweed/bazel_roll',
	'pigweed/checkout',
	'pigweed/cipd_roll',
	'pigweed/copy_roll',
	'pigweed/repo_roll',
	'pigweed/submodule_roll',
	'pigweed/txt_roll',
	'recipe_engine/file',
	'recipe_engine/properties',
	'recipe_engine/step',
	'recipe_engine/time',
	]

	PROPERTIES = InputProperties


	def merge_auto_roller_overrides(
	auto_roller_options: AutoRollerOptions,
	override_auto_roller_options: AutoRollerOptions,
	):
	result = AutoRollerOptions()
	result.CopyFrom(auto_roller_options)
	result.MergeFrom(override_auto_roller_options)
	return result


	def RunSteps( # pylint: disable=invalid-name
	api: recipe_api.RecipeScriptApi,
	props: InputProperties,
	):
	api.abandon_old_changes(host=props.checkout_options.remote)

	# The checkout module will try to use trigger data to pull in a specific
	# patch. Since the triggering commit is in a different repository that
	# needs to be disabled.
	props.checkout_options.use_trigger = False
	checkout = api.checkout(props.checkout_options)

	rolls = []
	attempts = 0

	for submodule in props.submodules:
	attempts += 1
	rolls.extend(api.submodule_roll.update(checkout, submodule))

	for cipd_package in props.bazel_cipd_packages:
	attempts += 1
	rolls.extend(api.bazel_roll.update_cipd_package(checkout, cipd_package))

	for git_repo in props.bazel_git_repositories:
	attempts += 1
	rolls.extend(api.bazel_roll.update_git_repository(checkout, git_repo))

	for project in props.repo_tool_projects:
	attempts += 1
	rolls.extend(api.repo_roll.update_project(checkout, project))

	for cipd_package in props.cipd_packages:
	attempts += 1
	rolls.extend(api.cipd_roll.update_package(checkout, cipd_package))

	for txt_entry in props.txt_entries:
	attempts += 1
	rolls.extend(api.txt_roll.update(checkout, txt_entry))

	for copy_entry in props.copy_entries:
	attempts += 1
	rolls.extend(api.copy_roll.update(checkout, copy_entry))

	if not attempts:
	summary = 'roller is not configured to roll anything'
	api.step.empty(summary)
	return result_pb.RawResult(
	summary_markdown=summary,
	status=common_pb.INFRA_FAILURE,
	)

	if not rolls:
	summary = 'nothing to roll'
	api.step.empty(summary)
	return result_pb.RawResult(
	summary_markdown=summary,
	status=common_pb.SUCCESS,
	)

	author_override: api.git_roll_util.Author \| None = None
	if props.forge_author:
	author_override = api.git_roll_util.get_author_override(*rolls)

	commit_message = api.roll_commit_message.format(
	*rolls,
	roll_prefix='roll:',
	send_comment=True,
	commit_divider=props.commit_divider,
	header_override=props.header_override,
	footers=list(props.footer or [f'Roll-Count: {len(rolls)}']),
	)

	pres = api.step.empty('commit message').presentation
	pres.logs['commit message'] = commit_message
	for roll in rolls:
	pres.properties[roll.short_name()] = roll.output_property()
	pres.properties['_roll_count'] = len(rolls)

	auto_roller_options = merge_auto_roller_overrides(
	props.auto_roller_options, props.override_auto_roller_options
	)

	change = api.auto_roller.attempt_roll(
	auto_roller_options,
	repo_dir=checkout.root,
	commit_message=commit_message,
	author_override=author_override,
	)

	result = api.auto_roller.raw_result(change)
	api.time.sleep(props.post_roll_sleep_secs)
	return result


	def GenTests(api) -> Generator[recipe_test_api.TestData, None, None]:
	"""Create tests."""

	def ensure_list(x):
	if isinstance(x, tuple):
	return list(x)
	if isinstance(x, list):
	return x # pragma: no cover
	return [x]

	def properties(
	*,
	bazel_cipd_packages=(),
	bazel_git_repositories=(),
	cipd_packages=(),
	copy_entries=(),
	repo_tool_projects=(),
	submodules=(),
	txt_entries=(),
	**kwargs,
	):
	props = InputProperties(**kwargs)
	props.checkout_options.CopyFrom(api.checkout.git_options())

	props.bazel_cipd_packages.extend(ensure_list(bazel_cipd_packages))
	props.bazel_git_repositories.extend(ensure_list(bazel_git_repositories))
	props.cipd_packages.extend(ensure_list(cipd_packages))
	props.copy_entries.extend(ensure_list(copy_entries))
	props.repo_tool_projects.extend(ensure_list(repo_tool_projects))
	props.submodules.extend(ensure_list(submodules))
	props.txt_entries.extend(ensure_list(txt_entries))

	props.auto_roller_options.dry_run = True
	props.auto_roller_options.remote = api.checkout.pigweed_repo

	return api.properties(props)

	yield api.test(
	'empty',
	properties(),
	api.post_process(
	post_process.MustRun,
	'roller is not configured to roll anything',
	),
	api.post_process(post_process.DropExpectation),
	status='INFRA_FAILURE',
	)

	yield api.test(
	'noop',
	properties(
	bazel_cipd_packages=api.bazel_roll.cipd_package(
	spec='bazel-cipd/${platform}',
	),
	bazel_git_repositories=api.bazel_roll.git_repo(
	name='bazel-git',
	remote='https://pigweed.googlesource.com/bazel-git',
	),
	cipd_packages=api.cipd_roll.package_props('cipd/${platform}'),
	copy_entries=api.copy_roll.entry('copy.txt'),
	repo_tool_projects=api.repo_roll.project('project'),
	submodules=api.submodule_roll.entry('submodule'),
	txt_entries=api.txt_roll.entry('text.txt'),
	),
	api.bazel_roll.workspace_file(
	'bazel-cipd',
	api.bazel_roll.cipd_entry(
	spec='bazel-cipd/${platform}',
	tag='git_revision:397a2597cdc237f3026e6143b683be4b9ab60540',
	),
	api.bazel_roll.git_entry(
	remote='https://pigweed.googlesource.com/bazel-git',
	),
	),
	api.gitiles.log('bazel-git.log bazel-git', 'A', n=0),
	api.cipd_roll.read_file_step_data(
	'cipd',
	'pigweed.json',
	api.cipd_roll.package(
	'cipd/${platform}',
	'git_revision:397a2597cdc237f3026e6143b683be4b9ab60540',
	platforms=['linux-amd64', 'windows-amd64'],
	),
	),
	api.step_data(
	'copy.txt.read destination copy.txt',
	api.file.read_text('abc'),
	),
	api.gitiles.fetch('copy.txt.read source copy.txt', 'abc'),
	api.repo_roll.read_step_data('project'),
	api.gitiles.log('project.log project', 'A', n=0),
	api.submodule_roll.gitmodules(
	prefix='submodule',
	submodule='submodule',
	),
	api.gitiles.log('submodule.log submodule', 'A', n=0),
	api.gitiles.log('text.txt.log text.txt', 'A', n=0),
	api.post_process(post_process.MustRun, 'nothing to roll'),
	api.post_process(post_process.DropExpectation),
	)

	def expected_roll(name, **expected):
	def compare(actual):
	for k, v in expected.items():
	if k not in actual or actual[k] != v:
	return False # pragma: no cover
	return True

	return api.post_process(
	post_process.PropertyMatchesCallable,
	name,
	compare,
	)

	yield api.test(
	'mega',
	properties(
	bazel_cipd_packages=api.bazel_roll.cipd_package(
	name='bazel-cipd/${plat}',
	),
	bazel_git_repositories=api.bazel_roll.git_repo(name='bazel-git'),
	cipd_packages=api.cipd_roll.package_props('cipd/${platform}'),
	copy_entries=api.copy_roll.entry('copy.txt'),
	repo_tool_projects=api.repo_roll.project('project'),
	submodules=api.submodule_roll.entry('submodule'),
	txt_entries=api.txt_roll.entry('text.txt'),
	forge_author=True,
	post_roll_sleep_secs=61,
	),
	api.gitiles.log('bazel-git.log bazel-git', 'A'),
	expected_roll('bazel-git', old='1' * 40, new='h3ll0'),
	api.cipd_roll.read_file_step_data(
	'cipd',
	'pigweed.json',
	api.cipd_roll.package(
	'cipd/${platform}',
	'git_revision:123',
	platforms=['linux-amd64', 'windows-amd64'],
	),
	),
	expected_roll(
	'cipd',
	old='git_revision:123',
	new='git_revision:397a2597cdc237f3026e6143b683be4b9ab60540',
	),
	api.step_data(
	'copy.txt.read destination copy.txt',
	api.file.read_text('old'),
	),
	api.gitiles.fetch('copy.txt.read source copy.txt', 'new'),
	expected_roll('copy.txt', old='old', new='new'),
	api.repo_roll.read_step_data('project'),
	api.gitiles.log('project.log project', 'A'),
	expected_roll('project', old='main', new='h3ll0'),
	api.submodule_roll.gitmodules(
	prefix='submodule',
	submodule='submodule',
	),
	api.gitiles.log('submodule.log submodule', 'A'),
	expected_roll('submodule', old='1' * 40, new='h3ll0'),
	api.gitiles.log('text.txt.log text.txt', 'A'),
	expected_roll('text.txt', old='1' * 40, new='h3ll0'),
	api.auto_roller.dry_run_success(),
	api.post_process(post_process.MustRun, 'sleep 61'),
	api.post_process(post_process.PropertyEquals, '_roll_count', 7),
	api.post_process(post_process.DropExpectation),
	)