blob: e6eda7144899261d10cb9a8603dfe3b4df6d290c [file]
# Copyright 2024 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""Combined roller for various types of dependencies.
This recipe is designed to automate the process of updating (or "rolling")
different kinds of dependencies within a project. It supports rolling:
- Git submodules
- Bazel dependencies (both CIPD packages and Git repositories)
- Android Repo tool projects
- CIPD packages
- Text file entries
- Variable entries in files
- Direct file copies
The roller checks each configured dependency for new versions and, if updates
are found, creates a commit and potentially a code review (e.g., Gerrit CL)
with the changes. It uses the fuchsia/auto_roller module for managing the
roll process and interacting with code review systems.
"""
from __future__ import annotations
import contextlib
import itertools
import re
from collections.abc import Iterator, Sequence
from typing import Any
from google.protobuf import json_format
from recipe_engine import post_process, recipe_api, recipe_test_api
from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb
from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb
from PB.recipe_engine import result as result_pb
from PB.recipe_modules.fuchsia.auto_roller.options import (
Options as AutoRollerOptions,
)
from PB.recipe_modules.pigweed.bazel_roll.cipd_package import BazelCipdPackage
from PB.recipe_modules.pigweed.bazel_roll.git_repository import GitRepository
from PB.recipe_modules.pigweed.cipd_roll.package import Package
from PB.recipe_modules.pigweed.copy_roll.copy_entry import CopyEntry
from PB.recipe_modules.pigweed.merge_roll.merge import MergeEntry
from PB.recipe_modules.pigweed.module_bazel_lock_roll.module_bazel_lock import (
ModuleBazelLock,
)
from PB.recipe_modules.pigweed.repo_roll.project import Project
from PB.recipe_modules.pigweed.run_script_roll.run_script_entry import (
RunScriptEntry,
)
from PB.recipe_modules.pigweed.submodule_roll.submodule import SubmoduleEntry
from PB.recipe_modules.pigweed.txt_roll.txt_entry import TxtEntry
from PB.recipe_modules.pigweed.variable_roll.variable_entry import (
CipdVariableEntry,
VariableEntry,
)
from PB.recipes.pigweed.roller import InputProperties
from RECIPE_MODULES.pigweed.raw_result import api as raw_result_api
DEPS = [
'fuchsia/auto_roller',
'fuchsia/git_roll_util',
'fuchsia/gitiles',
'fuchsia/roll_commit_message',
'pigweed/abandon_old_changes',
'pigweed/bazel_roll',
'pigweed/checkout',
'pigweed/cipd_roll',
'pigweed/copy_roll',
'pigweed/environment',
'pigweed/merge_roll',
'pigweed/module_bazel_lock_roll',
'pigweed/raw_result',
'pigweed/repo_roll',
'pigweed/run_script_roll',
'pigweed/submodule_roll',
'pigweed/txt_roll',
'pigweed/util',
'pigweed/variable_roll',
'recipe_engine/buildbucket',
'recipe_engine/file',
'recipe_engine/json',
'recipe_engine/properties',
'recipe_engine/step',
'recipe_engine/time',
]
PROPERTIES = InputProperties
READY_POLL_INTERVAL_SECONDS = 20
def merge_auto_roller_overrides(
auto_roller_options: AutoRollerOptions,
override_auto_roller_options: AutoRollerOptions,
) -> AutoRollerOptions:
result = AutoRollerOptions()
result.CopyFrom(auto_roller_options)
result.MergeFrom(override_auto_roller_options)
return result
@raw_result_api.wrap_run_steps
def RunSteps( # pylint: disable=invalid-name
api: recipe_api.RecipeScriptApi,
props: InputProperties,
) -> result_pb.RawResult | None:
api.abandon_old_changes(host=props.checkout_options.remote)
not_ready = api.step.empty('not ready').presentation
not_ready.properties['ready_to_upload'] = False
if props.match_triggering_branch:
triggering_changes = api.checkout.triggering_changes(
props.checkout_options
)
if len(triggering_changes) != 1:
api.step.empty(
f'len(triggering_changes) {len(triggering_changes)} != 1',
status='FAILURE',
)
change = triggering_changes[0]
if not change.submitted:
api.step.empty(
f'triggering change {change} is not submitted',
status='FAILURE',
)
props.checkout_options.branch = change.branch
props.auto_roller_options.upstream_ref = change.branch
props.override_auto_roller_options.upstream_ref = change.branch
checkout = api.checkout(props.checkout_options)
env: api.checkout.Environment | None = None
if props.environment_options.config_file:
env = api.environment.init(checkout, props.environment_options)
rolls = []
attempts = 0
with contextlib.ExitStack() as stack:
if env:
stack.enter_context(env())
for submodule in props.submodules:
attempts += 1
rolls.extend(api.submodule_roll.update(checkout, submodule))
for cipd_package in props.bazel_cipd_packages:
attempts += 1
rolls.extend(
api.bazel_roll.update_cipd_package(checkout, cipd_package),
)
for git_repo in props.bazel_git_repositories:
attempts += 1
rolls.extend(
api.bazel_roll.update_git_repository(checkout, git_repo),
)
for project in props.repo_tool_projects:
attempts += 1
rolls.extend(api.repo_roll.update_project(checkout, project))
for cipd_package in props.cipd_packages:
attempts += 1
rolls.extend(api.cipd_roll.update_package(checkout, cipd_package))
for cipd_variable in props.cipd_variables:
attempts += 1
rolls.extend(api.variable_roll.update_cipd(checkout, cipd_variable))
for txt_entry in props.txt_entries:
attempts += 1
rolls.extend(api.txt_roll.update(checkout, txt_entry))
for variable_entry in props.variable_entries:
attempts += 1
rolls.extend(api.variable_roll.update(checkout, variable_entry))
for copy_entry in props.copy_entries:
attempts += 1
rolls.extend(api.copy_roll.update(checkout, copy_entry))
for lock_entry in props.module_bazel_lock_entries:
attempts += 1
rolls.extend(
api.module_bazel_lock_roll.update(checkout, lock_entry),
)
for merge_entry in props.merge_entries:
attempts += 1
rolls.extend(
api.merge_roll.merge(checkout, merge_entry),
)
for run_script_entry in props.run_script_entries:
attempts += 1
rolls.extend(
api.run_script_roll.run(checkout, run_script_entry),
)
if not attempts:
summary = 'roller is not configured to roll anything'
pres = api.step.empty(summary).presentation
pres.properties['ready_to_upload'] = True
return result_pb.RawResult(
summary_markdown=summary,
status=common_pb.INFRA_FAILURE,
)
if not rolls:
summary = 'nothing to roll'
pres = api.step.empty(summary).presentation
pres.properties['_roll_count'] = 0
pres.properties['_roll_names'] = []
pres.properties['ready_to_upload'] = True
return result_pb.RawResult(
summary_markdown=summary,
status=common_pb.SUCCESS,
)
author_override: api.git_roll_util.Author | None = None
if props.forge_author:
author_override = api.git_roll_util.get_author_override(*rolls)
commit_message = api.roll_commit_message.format(
*rolls,
roll_prefix='roll:',
send_comment=True,
commit_divider=props.commit_divider,
header_override=props.header_override,
footers=list(props.footer or [f'Roll-Count: {len(rolls)}']),
)
roll_names = set()
with api.step.nest('commit message') as pres:
pres.logs['commit message'] = commit_message
for roll in rolls:
roll_names.add(roll.short_name())
roll_pres = api.step.empty(roll.short_name()).presentation
output_props = roll.output_property()
roll_pres.properties[roll.short_name()] = output_props
roll_pres.step_summary_text = repr(output_props)
pres.properties['_roll_count'] = len(rolls)
pres.properties['_roll_names'] = [x.short_name() for x in rolls]
auto_roller_options = merge_auto_roller_overrides(
props.auto_roller_options, props.override_auto_roller_options
)
hashtags = set(auto_roller_options.hashtags)
for roll_name in roll_names:
hashtags.add(f'roll-{roll_name}')
del auto_roller_options.hashtags[:]
auto_roller_options.hashtags.extend(sorted(hashtags))
if not props.dont_dry_run_in_bisection:
for tag in api.buildbucket.build.tags:
if tag.key == 'user_agent' and tag.value == 'bisector':
auto_roller_options.dry_run = True
pres = api.step.empty('ready_to_upload').presentation
pres.properties['ready_to_upload'] = True
if props.parent_build_id:
with api.step.nest('wait'):
for i in itertools.count():
test_data = build_pb.Build(id=int(props.parent_build_id))
if i == 0:
test_data.output.properties.update({})
elif i == 1:
test_data.output.properties.update(
{'continue_to_upload': False}
)
else:
test_data.output.properties.update(
{'continue_to_upload': True}
)
parent = api.buildbucket.get(
build_id=int(props.parent_build_id),
test_data=test_data,
)
if not parent:
break # pragma: no cover
properties = json_format.MessageToDict(parent.output.properties)
if properties.get('continue_to_upload'):
break
if api.util.on_last_iteration(READY_POLL_INTERVAL_SECONDS):
return result_pb.RawResult( # pragma: no cover
summary_markdown=(
'Failed to get "continue_to_upload" from parent.'
),
status=common_pb.FAILURE,
)
api.time.sleep(READY_POLL_INTERVAL_SECONDS)
auto_roller_dir = checkout.root
if props.subdirectory:
auto_roller_dir /= props.subdirectory
change = api.auto_roller.attempt_roll(
auto_roller_options,
repo_dir=auto_roller_dir,
commit_message=commit_message,
author_override=author_override,
)
result = api.auto_roller.raw_result(change)
api.time.sleep(props.post_roll_sleep_secs)
return result
def GenTests(
api: recipe_test_api.RecipeTestApi,
) -> Iterator[recipe_test_api.TestData]:
"""Create tests."""
def ensure_list(x: Any) -> list:
if isinstance(x, tuple):
return list(x)
if isinstance(x, list):
return x # pragma: no cover
return [x]
def SequenceOrScalar(T):
return Sequence[T] | T # pragma: no cover
def properties(
*,
bazel_cipd_packages: SequenceOrScalar[BazelCipdPackage] = (),
bazel_git_repositories: SequenceOrScalar[GitRepository] = (),
cipd_packages: SequenceOrScalar[Package] = (),
cipd_variables: SequenceOrScalar[CipdVariableEntry] = (),
copy_entries: SequenceOrScalar[CopyEntry] = (),
merge_entries: SequenceOrScalar[MergeEntry] = (),
module_bazel_lock_entries: SequenceOrScalar[ModuleBazelLock] = (),
repo_tool_projects: SequenceOrScalar[Project] = (),
run_script_entries: SequenceOrScalar[RunScriptEntry] = (),
submodules: SequenceOrScalar[SubmoduleEntry] = (),
txt_entries: SequenceOrScalar[TxtEntry] = (),
variable_entries: SequenceOrScalar[VariableEntry] = (),
dry_run: bool = True,
subdirectory: str | None = None,
parent_build_id: str | int | None = None,
**kwargs: Any,
) -> recipe_test_api.TestData:
props = InputProperties(**kwargs)
props.checkout_options.CopyFrom(
api.checkout.git_options(use_trigger=False),
)
props.bazel_cipd_packages.extend(ensure_list(bazel_cipd_packages))
props.bazel_git_repositories.extend(ensure_list(bazel_git_repositories))
props.cipd_packages.extend(ensure_list(cipd_packages))
props.cipd_variables.extend(ensure_list(cipd_variables))
props.copy_entries.extend(ensure_list(copy_entries))
props.merge_entries.extend(ensure_list(merge_entries))
props.module_bazel_lock_entries.extend(
ensure_list(module_bazel_lock_entries),
)
props.repo_tool_projects.extend(ensure_list(repo_tool_projects))
props.run_script_entries.extend(ensure_list(run_script_entries))
props.submodules.extend(ensure_list(submodules))
props.txt_entries.extend(ensure_list(txt_entries))
props.variable_entries.extend(ensure_list(variable_entries))
props.auto_roller_options.dry_run = dry_run
props.auto_roller_options.remote = api.checkout.pigweed_repo
if parent_build_id:
props.parent_build_id = str(parent_build_id)
if subdirectory:
props.subdirectory = subdirectory
return api.properties(props)
yield api.test(
'empty',
properties(),
api.environment.properties(),
api.post_check(
post_process.MustRun,
'roller is not configured to roll anything',
),
api.post_process(post_process.DropExpectation),
status='INFRA_FAILURE',
)
yield api.test(
'noop',
properties(
bazel_cipd_packages=api.bazel_roll.cipd_package(
spec='bazel-cipd/${platform}',
),
bazel_git_repositories=api.bazel_roll.git_repo(
name='bazel-git',
remote='https://pigweed.googlesource.com/bazel-git',
),
cipd_packages=api.cipd_roll.package_props('cipd/${platform}'),
cipd_variables=api.variable_roll.cipd_entry(
path='MODULE.bazel',
spec='variable-cipd/${plat}',
variable='FOO',
),
copy_entries=api.copy_roll.entry('copy.txt'),
merge_entries=api.merge_roll.entry('staging'),
module_bazel_lock_entries=api.module_bazel_lock_roll.entry(),
repo_tool_projects=api.repo_roll.project('project'),
submodules=api.submodule_roll.entry('submodule'),
txt_entries=api.txt_roll.entry('text.txt'),
variable_entries=api.variable_roll.entry('var.txt', variable='FOO'),
),
api.bazel_roll.workspace_file(
'bazel-cipd',
api.bazel_roll.cipd_entry(
spec='bazel-cipd/${platform}',
tag='git_revision:397a2597cdc237f3026e6143b683be4b9ab60540',
),
api.bazel_roll.git_entry(
remote='https://pigweed.googlesource.com/bazel-git',
),
),
api.gitiles.log('bazel-git.log bazel-git', 'A', n=0),
api.cipd_roll.read_file_step_data(
'cipd',
'pigweed.json',
api.cipd_roll.package(
'cipd/${platform}',
'git_revision:397a2597cdc237f3026e6143b683be4b9ab60540',
platforms=['linux-amd64', 'windows-amd64'],
),
),
api.step_data(
'copy.txt.read destination copy.txt',
api.file.read_text('abc'),
),
api.step_data(
'copy.txt.read source copy.txt', api.file.read_text('abc')
),
api.gitiles.log('staging.log staging', 'A', n=0),
api.repo_roll.read_step_data('project'),
api.gitiles.log('project.log project', 'A', n=0),
api.submodule_roll.gitmodules(
prefix='submodule',
submodule='submodule',
),
api.gitiles.log('submodule.log submodule', 'A', n=0),
api.gitiles.log('text.txt.log text.txt', 'A', n=0),
api.gitiles.log('var.txt[FOO].log var.txt[FOO]', 'A', n=0),
api.variable_roll.cipd_read(
'MODULE.bazel[FOO]',
'FOO = "git_revision:397a2597cdc237f3026e6143b683be4b9ab60540"',
needs_update=False,
),
api.post_check(post_process.MustRun, 'nothing to roll'),
# api.post_process(post_process.DropExpectation),
)
def expected_roll(name, **expected):
def compare(check, actual):
for k, v in expected.items():
if not check(k in actual):
return False # pragma: no cover
if not check(actual[k] == v):
return False # pragma: no cover
return True
return api.post_check(
post_process.PropertyMatchesCallable,
name,
compare,
)
yield api.test(
'one',
api.buildbucket.ci_build(
tags=[common_pb.StringPair(key='user_agent', value='bisector')],
git_repo='https://host.googlesource.com/prefix/project',
revision='h3ll0',
),
properties(
cipd_packages=api.cipd_roll.package_props('cipd/${platform}'),
dry_run=False,
subdirectory='subdir',
parent_build_id='12345',
),
api.cipd_roll.read_file_step_data(
'cipd',
'pigweed.json',
api.cipd_roll.package(
'cipd/${platform}',
'git_revision:123',
platforms=['linux-amd64', 'windows-amd64'],
),
),
expected_roll(
'cipd',
old='git_revision:123',
new='git_revision:397a2597cdc237f3026e6143b683be4b9ab60540',
),
api.auto_roller.dry_run_success(),
api.post_check(post_process.PropertyEquals, '_roll_count', 1),
# api.post_process(post_process.DropExpectation),
)
yield api.test(
'match_triggering_branch_success',
api.buildbucket.ci_build(
git_repo=api.checkout.pigweed_repo,
git_ref='refs/heads/release-1.0',
),
api.override_step_data(
'change data.process gitiles commit.number',
api.json.output(
[
{
'_number': '1234',
'branch': 'release-1.0',
'project': 'pigweed/pigweed',
}
]
),
),
properties(
cipd_packages=api.cipd_roll.package_props('cipd/${platform}'),
match_triggering_branch=True,
),
api.cipd_roll.read_file_step_data(
'cipd',
'pigweed.json',
api.cipd_roll.package(
'cipd/${platform}',
'git_revision:123',
platforms=['linux-amd64', 'windows-amd64'],
),
),
expected_roll(
'cipd',
old='git_revision:123',
new='git_revision:397a2597cdc237f3026e6143b683be4b9ab60540',
),
api.auto_roller.dry_run_success(),
api.post_check(post_process.PropertyEquals, '_roll_count', 1),
api.post_check(
post_process.StepCommandContains,
'git push',
[re.compile(r'refs/for/release-1.0')],
),
api.post_process(post_process.DropExpectation),
)
yield api.test(
'match_triggering_branch_not_submitted',
api.checkout.try_test_data(),
properties(
cipd_packages=api.cipd_roll.package_props('cipd/${platform}'),
match_triggering_branch=True,
),
api.post_check(
post_process.MustRunRE,
r'.*triggering change .* is not submitted',
),
api.post_process(post_process.DropExpectation),
status='FAILURE',
)
yield api.test(
'match_triggering_branch_multiple_changes',
api.checkout.try_test_data(
gerrit_changes=[
api.checkout.cl('host1', 'proj1', 123),
api.checkout.cl('host2', 'proj2', 456),
]
),
properties(
cipd_packages=api.cipd_roll.package_props('cipd/${platform}'),
match_triggering_branch=True,
),
api.post_check(
post_process.MustRun,
'len(triggering_changes) 2 != 1',
),
api.post_process(post_process.DropExpectation),
status='FAILURE',
)
yield api.test(
'match_triggering_branch_no_changes',
properties(
cipd_packages=api.cipd_roll.package_props('cipd/${platform}'),
match_triggering_branch=True,
),
api.post_check(
post_process.MustRun,
'len(triggering_changes) 0 != 1',
),
api.post_process(post_process.DropExpectation),
status='FAILURE',
)
yield api.test(
'mega',
api.buildbucket.ci_build(
tags=[common_pb.StringPair(key='user_agent', value='bisector')],
git_repo='https://host.googlesource.com/prefix/project',
revision='h3ll0',
),
properties(
bazel_cipd_packages=api.bazel_roll.cipd_package(
name='bazel-cipd/${plat}',
),
bazel_git_repositories=api.bazel_roll.git_repo(name='bazel-git'),
cipd_packages=api.cipd_roll.package_props('cipd/${platform}'),
cipd_variables=api.variable_roll.cipd_entry(
path='MODULE.bazel',
spec='variable-cipd/${plat}',
variable='FOO',
),
copy_entries=api.copy_roll.entry('copy.txt'),
merge_entries=api.merge_roll.entry('staging'),
module_bazel_lock_entries=api.module_bazel_lock_roll.entry(),
repo_tool_projects=api.repo_roll.project('project'),
run_script_entries=api.run_script_roll.entry('script.sh'),
submodules=api.submodule_roll.entry('submodule'),
txt_entries=api.txt_roll.entry('text.txt'),
variable_entries=api.variable_roll.entry('var.txt', variable='FOO'),
forge_author=True,
post_roll_sleep_secs=61,
dry_run=False,
),
api.gitiles.log('bazel-git.log bazel-git', 'A'),
expected_roll(
'bazel-git',
old='255c6325c4c654e17e6b35142e3912c86f1718f2',
new='3e30158f2a7caccb7a9f6632a60011e7a44e1e5c',
),
api.cipd_roll.read_file_step_data(
'cipd',
'pigweed.json',
api.cipd_roll.package(
'cipd/${platform}',
'git_revision:123',
platforms=['linux-amd64', 'windows-amd64'],
),
),
expected_roll(
'cipd',
old='git_revision:123',
new='git_revision:397a2597cdc237f3026e6143b683be4b9ab60540',
),
expected_roll(
'variable-cipd',
old='git_revision:0000000000000000000000000000000000000000',
new='git_revision:397a2597cdc237f3026e6143b683be4b9ab60540',
),
api.step_data(
'copy.txt.read destination copy.txt',
api.file.read_text('old'),
),
api.step_data(
'copy.txt.read source copy.txt', api.file.read_text('new')
),
expected_roll('copy.txt', old='old', new='new'),
api.gitiles.log('staging.log staging', 'A'),
expected_roll(
'staging',
old='255c6325c4c654e17e6b35142e3912c86f1718f2',
new='3e30158f2a7caccb7a9f6632a60011e7a44e1e5c',
),
api.repo_roll.read_step_data('project'),
api.gitiles.log('project.log project', 'A'),
expected_roll(
'project',
old='255c6325c4c654e17e6b35142e3912c86f1718f2',
new='3e30158f2a7caccb7a9f6632a60011e7a44e1e5c',
),
api.submodule_roll.gitmodules(
prefix='submodule',
submodule='submodule',
),
api.gitiles.log('submodule.log submodule', 'A'),
expected_roll(
'submodule',
old='255c6325c4c654e17e6b35142e3912c86f1718f2',
new='3e30158f2a7caccb7a9f6632a60011e7a44e1e5c',
),
api.gitiles.log('text.txt.log text.txt', 'A'),
expected_roll(
'text.txt',
old='255c6325c4c654e17e6b35142e3912c86f1718f2',
new='3e30158f2a7caccb7a9f6632a60011e7a44e1e5c',
),
api.gitiles.log('var.txt[FOO].log var.txt[FOO]', 'A'),
expected_roll(
'var.txt[FOO]',
old='255c6325c4c654e17e6b35142e3912c86f1718f2',
new='3e30158f2a7caccb7a9f6632a60011e7a44e1e5c',
),
api.auto_roller.success(),
api.post_check(post_process.MustRun, 'sleep 61'),
api.post_check(post_process.PropertyEquals, '_roll_count', 11),
# api.post_process(post_process.DropExpectation),
)