| # Copyright 2024 The Pigweed Authors |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may not |
| # use this file except in compliance with the License. You may obtain a copy of |
| # the License at |
| # |
| # https://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations under |
| # the License. |
| """Combined roller for various types of dependencies. |
| |
| This recipe is designed to automate the process of updating (or "rolling") |
| different kinds of dependencies within a project. It supports rolling: |
| - Git submodules |
| - Bazel dependencies (both CIPD packages and Git repositories) |
| - Android Repo tool projects |
| - CIPD packages |
| - Text file entries |
| - Variable entries in files |
| - Direct file copies |
| |
| The roller checks each configured dependency for new versions and, if updates |
| are found, creates a commit and potentially a code review (e.g., Gerrit CL) |
| with the changes. It uses the fuchsia/auto_roller module for managing the |
| roll process and interacting with code review systems. |
| """ |
| |
| from __future__ import annotations |
| |
| from collections.abc import Iterator |
| |
| from recipe_engine import post_process, recipe_api, recipe_test_api |
| |
| from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb |
| from PB.recipe_engine import result as result_pb |
| from PB.recipe_modules.fuchsia.auto_roller.options import ( |
| Options as AutoRollerOptions, |
| ) |
| from PB.recipes.pigweed.roller import InputProperties |
| |
| DEPS = [ |
| 'fuchsia/auto_roller', |
| 'fuchsia/git_roll_util', |
| 'fuchsia/gitiles', |
| 'fuchsia/roll_commit_message', |
| 'pigweed/abandon_old_changes', |
| 'pigweed/bazel_roll', |
| 'pigweed/checkout', |
| 'pigweed/cipd_roll', |
| 'pigweed/copy_roll', |
| 'pigweed/module_bazel_lock_roll', |
| 'pigweed/repo_roll', |
| 'pigweed/submodule_roll', |
| 'pigweed/txt_roll', |
| 'pigweed/variable_roll', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/file', |
| 'recipe_engine/properties', |
| 'recipe_engine/step', |
| 'recipe_engine/time', |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| |
| def merge_auto_roller_overrides( |
| auto_roller_options: AutoRollerOptions, |
| override_auto_roller_options: AutoRollerOptions, |
| ) -> AutoRollerOptions: |
| result = AutoRollerOptions() |
| result.CopyFrom(auto_roller_options) |
| result.MergeFrom(override_auto_roller_options) |
| return result |
| |
| |
| def RunSteps( # pylint: disable=invalid-name |
| api: recipe_api.RecipeScriptApi, |
| props: InputProperties, |
| ) -> result_pb.RawResult | None: |
| api.abandon_old_changes(host=props.checkout_options.remote) |
| |
| # The checkout module will try to use trigger data to pull in a specific |
| # patch. Since the triggering commit is in a different repository that |
| # needs to be disabled. |
| props.checkout_options.use_trigger = False |
| checkout = api.checkout(props.checkout_options) |
| |
| rolls = [] |
| attempts = 0 |
| |
| for submodule in props.submodules: |
| attempts += 1 |
| rolls.extend(api.submodule_roll.update(checkout, submodule)) |
| |
| for cipd_package in props.bazel_cipd_packages: |
| attempts += 1 |
| rolls.extend(api.bazel_roll.update_cipd_package(checkout, cipd_package)) |
| |
| for git_repo in props.bazel_git_repositories: |
| attempts += 1 |
| rolls.extend(api.bazel_roll.update_git_repository(checkout, git_repo)) |
| |
| for project in props.repo_tool_projects: |
| attempts += 1 |
| rolls.extend(api.repo_roll.update_project(checkout, project)) |
| |
| for cipd_package in props.cipd_packages: |
| attempts += 1 |
| rolls.extend(api.cipd_roll.update_package(checkout, cipd_package)) |
| |
| for cipd_variable in props.cipd_variables: |
| attempts += 1 |
| rolls.extend(api.variable_roll.update_cipd(checkout, cipd_variable)) |
| |
| for txt_entry in props.txt_entries: |
| attempts += 1 |
| rolls.extend(api.txt_roll.update(checkout, txt_entry)) |
| |
| for variable_entry in props.variable_entries: |
| attempts += 1 |
| rolls.extend(api.variable_roll.update(checkout, variable_entry)) |
| |
| for copy_entry in props.copy_entries: |
| attempts += 1 |
| rolls.extend(api.copy_roll.update(checkout, copy_entry)) |
| |
| for lock_entry in props.module_bazel_lock_entries: |
| attempts += 1 |
| rolls.extend(api.module_bazel_lock_roll.update(checkout, lock_entry)) |
| |
| if not attempts: |
| summary = 'roller is not configured to roll anything' |
| api.step.empty(summary) |
| return result_pb.RawResult( |
| summary_markdown=summary, |
| status=common_pb.INFRA_FAILURE, |
| ) |
| |
| if not rolls: |
| summary = 'nothing to roll' |
| pres = api.step.empty(summary).presentation |
| pres.properties['_roll_count'] = len(rolls) |
| |
| return result_pb.RawResult( |
| summary_markdown=summary, |
| status=common_pb.SUCCESS, |
| ) |
| |
| author_override: api.git_roll_util.Author | None = None |
| if props.forge_author: |
| author_override = api.git_roll_util.get_author_override(*rolls) |
| |
| commit_message = api.roll_commit_message.format( |
| *rolls, |
| roll_prefix='roll:', |
| send_comment=True, |
| commit_divider=props.commit_divider, |
| header_override=props.header_override, |
| footers=list(props.footer or [f'Roll-Count: {len(rolls)}']), |
| ) |
| |
| roll_names = set() |
| with api.step.nest('commit message') as pres: |
| pres.logs['commit message'] = commit_message |
| for roll in rolls: |
| roll_names.add(roll.short_name()) |
| roll_pres = api.step.empty(roll.short_name()).presentation |
| output_props = roll.output_property() |
| roll_pres.properties[roll.short_name()] = output_props |
| roll_pres.step_summary_text = repr(output_props) |
| pres.properties['_roll_count'] = len(rolls) |
| |
| auto_roller_options = merge_auto_roller_overrides( |
| props.auto_roller_options, props.override_auto_roller_options |
| ) |
| |
| hashtags = set(auto_roller_options.hashtags) |
| for roll_name in roll_names: |
| hashtags.add(f'roll-{roll_name}') |
| del auto_roller_options.hashtags[:] |
| auto_roller_options.hashtags.extend(sorted(hashtags)) |
| |
| for tag in api.buildbucket.build.tags: |
| if tag.key == 'user_agent' and tag.value == 'bisector': |
| auto_roller_options.dry_run = True |
| |
| change = api.auto_roller.attempt_roll( |
| auto_roller_options, |
| repo_dir=checkout.root, |
| commit_message=commit_message, |
| author_override=author_override, |
| ) |
| |
| result = api.auto_roller.raw_result(change) |
| api.time.sleep(props.post_roll_sleep_secs) |
| return result |
| |
| |
| def GenTests( |
| api: recipe_test_api.RecipeTestApi, |
| ) -> Iterator[recipe_test_api.TestData]: |
| """Create tests.""" |
| |
| def ensure_list(x) -> list: |
| if isinstance(x, tuple): |
| return list(x) |
| if isinstance(x, list): |
| return x # pragma: no cover |
| return [x] |
| |
| def properties( |
| *, |
| bazel_cipd_packages=(), |
| bazel_git_repositories=(), |
| cipd_packages=(), |
| cipd_variables=(), |
| copy_entries=(), |
| module_bazel_lock_entries=(), |
| repo_tool_projects=(), |
| submodules=(), |
| txt_entries=(), |
| variable_entries=(), |
| dry_run=True, |
| **kwargs, |
| ): |
| props = InputProperties(**kwargs) |
| props.checkout_options.CopyFrom(api.checkout.git_options()) |
| |
| props.bazel_cipd_packages.extend(ensure_list(bazel_cipd_packages)) |
| props.bazel_git_repositories.extend(ensure_list(bazel_git_repositories)) |
| props.cipd_packages.extend(ensure_list(cipd_packages)) |
| props.cipd_variables.extend(ensure_list(cipd_variables)) |
| props.copy_entries.extend(ensure_list(copy_entries)) |
| props.module_bazel_lock_entries.extend( |
| ensure_list(module_bazel_lock_entries), |
| ) |
| props.repo_tool_projects.extend(ensure_list(repo_tool_projects)) |
| props.submodules.extend(ensure_list(submodules)) |
| props.txt_entries.extend(ensure_list(txt_entries)) |
| props.variable_entries.extend(ensure_list(variable_entries)) |
| |
| props.auto_roller_options.dry_run = dry_run |
| props.auto_roller_options.remote = api.checkout.pigweed_repo |
| |
| return api.properties(props) |
| |
| yield api.test( |
| 'empty', |
| properties(), |
| api.post_process( |
| post_process.MustRun, |
| 'roller is not configured to roll anything', |
| ), |
| api.post_process(post_process.DropExpectation), |
| status='INFRA_FAILURE', |
| ) |
| |
| yield api.test( |
| 'noop', |
| properties( |
| bazel_cipd_packages=api.bazel_roll.cipd_package( |
| spec='bazel-cipd/${platform}', |
| ), |
| bazel_git_repositories=api.bazel_roll.git_repo( |
| name='bazel-git', |
| remote='https://pigweed.googlesource.com/bazel-git', |
| ), |
| cipd_packages=api.cipd_roll.package_props('cipd/${platform}'), |
| cipd_variables=api.variable_roll.cipd_entry( |
| path='MODULE.bazel', |
| spec='variable-cipd/${plat}', |
| variable='FOO', |
| ), |
| copy_entries=api.copy_roll.entry('copy.txt'), |
| module_bazel_lock_entries=api.module_bazel_lock_roll.entry(), |
| repo_tool_projects=api.repo_roll.project('project'), |
| submodules=api.submodule_roll.entry('submodule'), |
| txt_entries=api.txt_roll.entry('text.txt'), |
| variable_entries=api.variable_roll.entry('var.txt', variable='FOO'), |
| ), |
| api.bazel_roll.workspace_file( |
| 'bazel-cipd', |
| api.bazel_roll.cipd_entry( |
| spec='bazel-cipd/${platform}', |
| tag='git_revision:397a2597cdc237f3026e6143b683be4b9ab60540', |
| ), |
| api.bazel_roll.git_entry( |
| remote='https://pigweed.googlesource.com/bazel-git', |
| ), |
| ), |
| api.gitiles.log('bazel-git.log bazel-git', 'A', n=0), |
| api.cipd_roll.read_file_step_data( |
| 'cipd', |
| 'pigweed.json', |
| api.cipd_roll.package( |
| 'cipd/${platform}', |
| 'git_revision:397a2597cdc237f3026e6143b683be4b9ab60540', |
| platforms=['linux-amd64', 'windows-amd64'], |
| ), |
| ), |
| api.step_data( |
| 'copy.txt.read destination copy.txt', |
| api.file.read_text('abc'), |
| ), |
| api.gitiles.fetch('copy.txt.read source copy.txt', 'abc'), |
| api.repo_roll.read_step_data('project'), |
| api.gitiles.log('project.log project', 'A', n=0), |
| api.submodule_roll.gitmodules( |
| prefix='submodule', |
| submodule='submodule', |
| ), |
| api.gitiles.log('submodule.log submodule', 'A', n=0), |
| api.gitiles.log('text.txt.log text.txt', 'A', n=0), |
| api.gitiles.log('var.txt[FOO].log var.txt[FOO]', 'A', n=0), |
| api.variable_roll.cipd_read( |
| 'MODULE.bazel[FOO]', |
| 'FOO = "git_revision:397a2597cdc237f3026e6143b683be4b9ab60540"', |
| needs_update=False, |
| ), |
| api.post_process(post_process.MustRun, 'nothing to roll'), |
| # api.post_process(post_process.DropExpectation), |
| ) |
| |
| def expected_roll(name, **expected): |
| def compare(actual): |
| for k, v in expected.items(): |
| if k not in actual or actual[k] != v: |
| return False # pragma: no cover |
| return True |
| |
| return api.post_process( |
| post_process.PropertyMatchesCallable, |
| name, |
| compare, |
| ) |
| |
| yield api.test( |
| 'mega', |
| api.buildbucket.ci_build( |
| tags=[common_pb.StringPair(key='user_agent', value='bisector')], |
| git_repo='https://host.googlesource.com/prefix/project', |
| revision='h3ll0', |
| ), |
| properties( |
| bazel_cipd_packages=api.bazel_roll.cipd_package( |
| name='bazel-cipd/${plat}', |
| ), |
| bazel_git_repositories=api.bazel_roll.git_repo(name='bazel-git'), |
| cipd_packages=api.cipd_roll.package_props('cipd/${platform}'), |
| cipd_variables=api.variable_roll.cipd_entry( |
| path='MODULE.bazel', |
| spec='variable-cipd/${plat}', |
| variable='FOO', |
| ), |
| copy_entries=api.copy_roll.entry('copy.txt'), |
| module_bazel_lock_entries=api.module_bazel_lock_roll.entry(), |
| repo_tool_projects=api.repo_roll.project('project'), |
| submodules=api.submodule_roll.entry('submodule'), |
| txt_entries=api.txt_roll.entry('text.txt'), |
| variable_entries=api.variable_roll.entry('var.txt', variable='FOO'), |
| forge_author=True, |
| post_roll_sleep_secs=61, |
| dry_run=False, |
| ), |
| api.gitiles.log('bazel-git.log bazel-git', 'A'), |
| expected_roll( |
| 'bazel-git', |
| old='255c6325c4c654e17e6b35142e3912c86f1718f2', |
| new='3e30158f2a7caccb7a9f6632a60011e7a44e1e5c', |
| ), |
| api.cipd_roll.read_file_step_data( |
| 'cipd', |
| 'pigweed.json', |
| api.cipd_roll.package( |
| 'cipd/${platform}', |
| 'git_revision:123', |
| platforms=['linux-amd64', 'windows-amd64'], |
| ), |
| ), |
| expected_roll( |
| 'cipd', |
| old='git_revision:123', |
| new='git_revision:397a2597cdc237f3026e6143b683be4b9ab60540', |
| ), |
| expected_roll( |
| 'variable-cipd', |
| old='git_revision:0000000000000000000000000000000000000000', |
| new='git_revision:397a2597cdc237f3026e6143b683be4b9ab60540', |
| ), |
| api.step_data( |
| 'copy.txt.read destination copy.txt', |
| api.file.read_text('old'), |
| ), |
| api.gitiles.fetch('copy.txt.read source copy.txt', 'new'), |
| expected_roll('copy.txt', old='old', new='new'), |
| api.repo_roll.read_step_data('project'), |
| api.gitiles.log('project.log project', 'A'), |
| expected_roll( |
| 'project', |
| old='255c6325c4c654e17e6b35142e3912c86f1718f2', |
| new='3e30158f2a7caccb7a9f6632a60011e7a44e1e5c', |
| ), |
| api.submodule_roll.gitmodules( |
| prefix='submodule', |
| submodule='submodule', |
| ), |
| api.gitiles.log('submodule.log submodule', 'A'), |
| expected_roll( |
| 'submodule', |
| old='255c6325c4c654e17e6b35142e3912c86f1718f2', |
| new='3e30158f2a7caccb7a9f6632a60011e7a44e1e5c', |
| ), |
| api.gitiles.log('text.txt.log text.txt', 'A'), |
| expected_roll( |
| 'text.txt', |
| old='255c6325c4c654e17e6b35142e3912c86f1718f2', |
| new='3e30158f2a7caccb7a9f6632a60011e7a44e1e5c', |
| ), |
| api.gitiles.log('var.txt[FOO].log var.txt[FOO]', 'A'), |
| expected_roll( |
| 'var.txt[FOO]', |
| old='255c6325c4c654e17e6b35142e3912c86f1718f2', |
| new='3e30158f2a7caccb7a9f6632a60011e7a44e1e5c', |
| ), |
| api.auto_roller.dry_run_success(), |
| api.post_process(post_process.MustRun, 'sleep 61'), |
| api.post_process(post_process.PropertyEquals, '_roll_count', 9), |
| # api.post_process(post_process.DropExpectation), |
| ) |
