| # Copyright 2021 The Pigweed Authors |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may not |
| # use this file except in compliance with the License. You may obtain a copy of |
| # the License at |
| # |
| # https://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations under |
| # the License. |
| """Roll multiple submodules of a git repository.""" |
| |
| # TODO(pwbug/433) Merge with submodule_roller.py. |
| |
| import re |
| |
| import attr |
| from PB.recipes.pigweed.multiple_submodule_roller import InputProperties |
| from recipe_engine import post_process |
| from six.moves import configparser |
| from six import StringIO |
| |
| DEPS = [ |
| 'fuchsia/auto_roller', |
| 'fuchsia/git', |
| 'fuchsia/status_check', |
| 'pigweed/checkout', |
| 'pigweed/roll_util', |
| 'recipe_engine/context', |
| 'recipe_engine/file', |
| 'recipe_engine/properties', |
| 'recipe_engine/step', |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| PYTHON_VERSION_COMPATIBILITY = "PY3" |
| |
| |
| @attr.s |
| class _Submodule(object): |
| _api = attr.ib() |
| path = attr.ib(type=str) |
| name = attr.ib(type=str) |
| branch = attr.ib(type=str) |
| remote = attr.ib(type=str, default=None) |
| |
| @property |
| def dir(self): |
| return self._api.checkout.root.join(self.path) |
| |
| |
| @attr.s |
| class _RevisionChange(object): |
| old = attr.ib(type=str) |
| new = attr.ib(type=str) |
| |
| |
| def _update_submodule(api, path, new_revision): |
| old_revision = api.checkout.get_revision( |
| path, 'get old revision', test_data='1' * 40 |
| ) |
| |
| with api.context(cwd=path): |
| api.git('git fetch', 'fetch', 'origin', new_revision) |
| api.git('git checkout', 'checkout', 'FETCH_HEAD') |
| |
| # In case new_revision is a branch name we need to retrieve the hash it |
| # resolved to. |
| if not re.search(r'^[0-9a-f]{40}$', new_revision): |
| new_revision = api.checkout.get_revision( |
| path, 'get new revision', test_data='2' * 40 |
| ) |
| |
| return _RevisionChange(old=old_revision, new=new_revision) |
| |
| |
| def RunSteps(api, props): # pylint: disable=invalid-name |
| submodules = [ |
| _Submodule(api=api, path=x.path, name=x.name, branch=x.branch) |
| for x in props.submodules |
| ] |
| dry_run = props.dry_run |
| cc_authors_on_rolls = props.cc_authors_on_rolls |
| cc_reviewers_on_rolls = props.cc_reviewers_on_rolls |
| cc_domains = props.cc_domains |
| always_cc = props.always_cc |
| |
| # The checkout module will try to use trigger data to pull in a specific |
| # patch. Since the triggering commit is in a different repository that |
| # needs to be disabled. |
| api.checkout(use_trigger=False) |
| |
| # Confirm the given path is actually a submodule. |
| gitmodules = api.file.read_text( |
| 'read .gitmodules', api.checkout.root.join('.gitmodules') |
| ) |
| # Example .gitmodules file: |
| # [submodule "third_party/pigweed"] |
| # path = third_party/pigweed |
| # url = https://pigweed.googlesource.com/pigweed/pigweed |
| |
| # configparser doesn't like leading whitespace on lines, despite what its |
| # documentation says. |
| gitmodules = re.sub(r'\n\s+', '\n', gitmodules) |
| parser = configparser.RawConfigParser() |
| parser.readfp(StringIO(gitmodules)) |
| |
| rolls = {} |
| |
| for submodule in submodules: |
| if not submodule.name: |
| submodule.name = submodule.path |
| |
| with api.step.nest(submodule.name) as pres: |
| section = 'submodule "{}"'.format(submodule.name) |
| if not parser.has_section(section): |
| sections = parser.sections() |
| submodules = sorted( |
| re.sub(r'^.*"(.*)"$', r'\1', x) for x in sections |
| ) |
| raise api.step.StepFailure( |
| 'no submodule "{}" (submodules: {})'.format( |
| submodule.name, |
| ', '.join('"{}"'.format(x) for x in submodules), |
| ) |
| ) |
| |
| if not submodule.branch: |
| try: |
| submodule.branch = parser.get(section, 'branch') |
| except configparser.NoOptionError: |
| submodule.branch = 'main' |
| |
| submodule.remote = api.roll_util.normalize_remote( |
| parser.get(section, 'url'), api.checkout.remote, |
| ) |
| |
| change = _update_submodule(api, submodule.dir, submodule.branch) |
| |
| direction = api.roll_util.get_roll_direction( |
| submodule.dir, change.old, change.new |
| ) |
| |
| # If the primary roll is not necessary or is backwards we can exit |
| # immediately and don't need to check deps. |
| if api.roll_util.can_roll(direction): |
| rolls[submodule.path] = api.roll_util.Roll( |
| project_name=str(submodule.path), |
| old_revision=change.old, |
| new_revision=change.new, |
| proj_dir=submodule.dir, |
| direction=direction, |
| nest_steps=False, |
| ) |
| |
| else: |
| pres.step_summary_text = 'no roll required' |
| api.roll_util.skip_roll_step( |
| submodule.remote, change.old, change.new |
| ) |
| |
| if not rolls: |
| with api.step.nest('nothing to roll, exiting'): |
| return |
| |
| cc = set() |
| if cc_authors_on_rolls: |
| cc.update(api.roll_util.authors(*rolls.values())) |
| if cc_reviewers_on_rolls: |
| cc.update(api.roll_util.reviewers(*rolls.values())) |
| |
| def include_cc(email): |
| return api.roll_util.include_cc( |
| email, cc_domains, api.checkout.gerrit_host() |
| ) |
| |
| # include_cc() writes steps, so we want things sorted before calling it. |
| cc = sorted(set(cc)) |
| cc = [x for x in cc if include_cc(x)] |
| |
| roll_kwargs = {} |
| if always_cc: |
| roll_kwargs['cc'] = cc |
| else: |
| roll_kwargs['cc_on_failure'] = cc |
| |
| change = api.auto_roller.attempt_roll( |
| gerrit_host=api.checkout.gerrit_host(), |
| gerrit_project=api.checkout.gerrit_project(), |
| upstream_ref=api.checkout.branch, |
| repo_dir=api.checkout.root, |
| commit_message=api.roll_util.message(*rolls.values()), |
| dry_run=dry_run, |
| labels_to_set=api.roll_util.labels_to_set, |
| labels_to_wait_on=api.roll_util.labels_to_wait_on, |
| bot_commit=props.bot_commit, |
| **roll_kwargs |
| ) |
| |
| return api.auto_roller.raw_result(change) |
| |
| |
| def GenTests(api): # pylint: disable=invalid-name |
| """Create tests.""" |
| |
| def _url(x): |
| if x.startswith(('https://', 'sso://', '.')): |
| return x |
| return 'https://foo.googlesource.com/' + x |
| |
| def submodules(*subs): |
| res = [] |
| for sub in subs: |
| if isinstance(sub, str): |
| res.append(dict(path=sub)) |
| elif isinstance(sub, dict): |
| res.append(sub) |
| else: |
| raise ValueError(repr(sub)) # pragma: no cover |
| return res |
| |
| def gitmodules(**submodules): |
| branches = {} |
| for k, v in submodules.items(): |
| if k.endswith('_branch'): |
| branches[k.replace('_branch', '')] = v |
| |
| for x in branches: |
| del submodules['{}_branch'.format(x)] |
| |
| text = [] |
| for k, v in submodules.items(): |
| text.append( |
| '[submodule "{0}"]\n\tpath = {0}\n\turl = {1}\n'.format( |
| k, _url(v) |
| ) |
| ) |
| if k in branches: |
| text.append('\tbranch = {}\n'.format(branches[k])) |
| |
| return api.step_data( |
| 'read .gitmodules', api.file.read_text(''.join(text)) |
| ) |
| |
| def properties(submodules, **kwargs): |
| new_kwargs = api.checkout.git_properties() |
| new_kwargs['submodules'] = submodules |
| new_kwargs.update(kwargs) |
| new_kwargs.setdefault('dry_run', True) |
| return api.properties(**new_kwargs) |
| |
| def commit_data(name, **kwargs): |
| return api.roll_util.commit_data( |
| name, |
| api.roll_util.commit('a' * 40, 'foo\nbar\n\nChange-Id: I1111'), |
| **kwargs |
| ) |
| |
| yield ( |
| api.status_check.test('success') |
| + properties( |
| submodules('a1', 'b2'), cc_authors_on_rolls=True, always_cc=True, |
| ) |
| + commit_data('a1', prefix='') |
| + commit_data('b2', prefix='') |
| + gitmodules(a1='sso://foo/a1', b2='sso://foo/b2') |
| + api.roll_util.forward_roll('a1') |
| + api.roll_util.forward_roll('b2') |
| + api.auto_roller.dry_run_success() |
| ) |
| |
| yield ( |
| api.status_check.test('partial_noop') |
| + properties(submodules('a1', 'b2'), cc_reviewers_on_rolls=True) |
| + commit_data('a1', prefix='') |
| + gitmodules(a1='sso://foo/a1', b2='sso://foo/b2') |
| + api.roll_util.forward_roll('a1') |
| + api.roll_util.noop_roll('b2') |
| + api.auto_roller.dry_run_success() |
| ) |
| |
| yield ( |
| api.status_check.test('noop') |
| + properties(submodules('a1', {'path': 'b2'})) |
| + gitmodules(a1='a1', b2='b2', b2_branch='branch') |
| + api.roll_util.noop_roll('a1') |
| + api.roll_util.noop_roll('b2') |
| ) |
| |
| yield ( |
| api.status_check.test('missing', status='failure') |
| + properties(submodules('a1', 'b2'), cc_authors_on_rolls=True) |
| + gitmodules(a1='sso://foo/a1') |
| ) |