| # Copyright 2020 The Pigweed Authors |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may not |
| # use this file except in compliance with the License. You may obtain a copy of |
| # the License at |
| # |
| # https://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations under |
| # the License. |
| """Find dependencies on pending CLs.""" |
| |
| import re |
| |
| import attr |
| from recipe_engine import recipe_api |
| |
| |
| @attr.s |
| class Change(object): |
| """Details of a matching change. |
| |
| This is meant to match the GerritChange message defined at |
| https://source.chromium.org/chromium/infra/infra/+/HEAD:go/src/go.chromium.org/luci/buildbucket/proto/common.proto?q=GerritChange |
| A few fields have been added for use by this module. |
| """ |
| |
| gerrit_name = attr.ib(type=str) |
| change = attr.ib(converter=int) |
| project = attr.ib(type=str, default=None) |
| patchset = attr.ib(type=int, default=None) |
| status = attr.ib(type=str, default=None) |
| commit = attr.ib(type=str, default=None) |
| parents = attr.ib(type=list, default=attr.Factory(list)) |
| |
| @property |
| def host(self): |
| return '{}-review.googlesource.com'.format(self.gerrit_name) |
| |
| @property |
| def link(self): |
| return 'https://{}/{}'.format(self.host, self.change) |
| |
| @property |
| def name(self): |
| return '{}:{}'.format(self.gerrit_name, self.change) |
| |
| @property |
| def remote(self): |
| return 'https://{}.googlesource.com/{}'.format( |
| self.gerrit_name, self.project |
| ) |
| |
| @property |
| def gerrit_url(self): |
| return 'https://{}-review.googlesource.com/c/{}'.format( |
| self.gerrit_name, self.change |
| ) |
| |
| def __str__(self): |
| return self.name |
| |
| |
| class CqDepsApi(recipe_api.RecipeApi): |
| """Find dependencies on pending CLs.""" |
| |
| def __init__(self, props, *args, **kwargs): |
| super(CqDepsApi, self).__init__(*args, **kwargs) |
| self._details = {} |
| self._enabled = not props.disabled |
| self._ignore_errors = props.ignore_errors |
| |
| def _lookup_cl(self, gerrit_name, commit): |
| query_results = self.m.gerrit.change_query( |
| 'number {}'.format(commit), |
| 'commit:{}'.format(commit), |
| host='{}-review.googlesource.com'.format(gerrit_name), |
| max_attempts=2, |
| timeout=30, |
| # The strange value for _number is to ensure different hashes result |
| # in different CL numbers by default. This way we don't accidentally |
| # create a cycle by specifying a fixed default value. |
| test_data=self.m.json.test_api.output( |
| [{'_number': sum(ord(x) for x in commit),}] |
| ), |
| ).json.output |
| if query_results: |
| return query_results[0] |
| return None |
| |
| def resolve(self, gerrit_name, number, statuses=('NEW',)): |
| """Recursively resolve dependencies of a CL. |
| |
| Resolve dependencies of a CL by parsing its commit message. Looks for |
| lines like below, and only in the last "paragraph". |
| |
| Requires: gerritname:1, commas-supported:2 |
| Requires: name:4,spaces-not-required:3 |
| Requires: name:5,,,,extra-commas-ignored:6,,,, |
| |
| Args: |
| gerrit_name (str): Name of Gerrit host of initial change. |
| number (int|str): Gerrit number or git commit hash of initial |
| change. |
| statuses (seq(str)): CL statuses to accept. |
| |
| Returns a tuple of resolved Change objects (excluding the given change) |
| and unresolved Change objects (which are not completely populated). |
| This includes dependencies of parent changes with 'NEW' status but not |
| the parent changes themselves. |
| """ |
| |
| with self.m.step.nest('resolve CL deps', status='last') as pres: |
| if not self._enabled: |
| with self.m.step.nest('dependency processing disabled'): |
| return ((), ()) |
| |
| with self.m.context(infra_steps=True): |
| if isinstance(number, str): |
| cl = self._lookup_cl(gerrit_name, number) |
| if not cl: |
| return ((), ()) |
| number = cl['_number'] |
| |
| seen = set() |
| deps = {} |
| parents = [] |
| unresolved = [] |
| initial_change = Change(gerrit_name, number) |
| to_process = [initial_change] + initial_change.parents |
| # Want all initial details calls to come from this step level so |
| # their step names are predictable. Further calls can be made |
| # anywhere and will hit the cache. |
| _ = self._change_details(initial_change) |
| |
| while to_process: |
| current = to_process.pop() |
| deps[current.name] = current |
| seen.add(current.name) |
| |
| parents.extend(current.parents) |
| to_process.extend( |
| x for x in current.parents if x.name not in seen |
| ) |
| |
| with self.m.step.nest( |
| 'parents {}'.format(current.name) |
| ) as pres: |
| if current.parents: |
| pres.step_summary_text = ', '.join( |
| x.name for x in current.parents |
| ) |
| else: |
| pres.step_summary_text = ( |
| 'all parents already submitted' |
| ) |
| |
| for dep in self._resolve(current): |
| if dep.name not in deps: |
| details = self._change_details(dep) |
| if not details: |
| unresolved.append(dep) |
| continue |
| if details['status'] not in statuses: |
| continue |
| |
| if dep.name not in seen: |
| to_process.append(dep) |
| deps[dep.name] = dep |
| |
| # If any dependency is a parent of another dependency remove the |
| # parent since it's already being accounted for by patching in |
| # its child. |
| for parent in parents: |
| if parent.name in deps: |
| del deps[parent.name] |
| |
| del deps[initial_change.name] |
| |
| for dep in deps.values(): |
| pres.links[dep.name] = dep.link |
| |
| # Make sure the last step passes because _change_details might |
| # fail for an individual CL in ways we don't care about. |
| with self.m.step.nest('pass'): |
| pass |
| |
| return deps.values(), unresolved |
| |
| def _parse_commit_message(self, change): |
| """Convert a commit message into a dependency list. |
| |
| For example, |
| |
| Requires: gerritname:1, commas-supported:2 |
| Requires: name:4,spaces-not-required:3 |
| Requires: 5,,,,extra-commas-ignored:6,,,, |
| |
| becomes |
| |
| [ |
| Change("gerritname", 1), |
| Change("commas-supported", 2), |
| Change("spaces-not-required", 3), |
| Change("name", 4), |
| Change(change.gerrit_name, 5), |
| Change("extra-commas-ignored", 6), |
| ] |
| """ |
| |
| details = self._change_details(change) |
| current_revision = details['revisions'][details['current_revision']] |
| commit_message = current_revision['commit']['message'] |
| last_paragraph = commit_message.split('\n\n')[-1] |
| |
| for line in last_paragraph.split('\n'): |
| if ':' not in line: |
| continue |
| name, value = line.split(':', 1) |
| |
| if name != 'Requires': |
| continue |
| |
| values = value.split(',') |
| for val in values: |
| val = val.strip() |
| if not val: |
| continue |
| |
| gerrit_name = change.gerrit_name |
| if ':' in val: |
| gerrit_name, number = val.split(':') |
| else: |
| number = val |
| |
| if not number.isdigit(): |
| if self._ignore_errors: |
| continue |
| |
| else: |
| with self.m.step.nest('invalid requirement') as pres: |
| pres.step_summary_text = ( |
| 'Format is "<gerrit-host>:<cl-number>" or ' |
| '"<cl-number>", {!r} does not match'.format(val) |
| ) |
| pres.status = 'FAILURE' |
| raise self.m.step.StepFailure('invalid requirement') |
| |
| yield Change(gerrit_name, number) |
| |
| def _resolve(self, change): |
| """Resolve dependencies of the given change. |
| |
| Retrieve and parse the commit message of the given change, extracting |
| from "Requires:" lines (see resolve function above for details). Does |
| not attempt to verify if these dependent changes are accessible or |
| even valid. |
| |
| Args: |
| change (Change): Change to resolve dependencies for. |
| |
| Returns a list of changes on which the given change depends. This |
| includes dependencies of parent changes with 'NEW' status but not the |
| parent changes themselves. |
| """ |
| |
| with self.m.step.nest('resolve deps for {}'.format(change)) as pres: |
| deps = [] |
| for dep in self._parse_commit_message(change): |
| deps.append(dep) |
| pres.links[dep.name] = dep.link |
| if not deps: |
| pres.step_summary_text = 'no dependencies' |
| return deps |
| |
| def _change_details(self, change, _kw_only=(), test_status='NEW'): |
| assert not _kw_only |
| if change.name in self._details: |
| return self._details[change.name] |
| |
| try: |
| step = self.m.gerrit.change_details( |
| 'details {}'.format(change), |
| change_id=str(change.change), |
| host=change.host, |
| query_params=['CURRENT_COMMIT', 'CURRENT_REVISION',], |
| max_attempts=2, |
| timeout=30, |
| test_data=self.m.json.test_api.output( |
| { |
| 'current_revision': 'HASH', |
| 'revisions': { |
| 'HASH': { |
| '_number': 1, |
| 'commit': { |
| 'message': '', |
| 'parents': [{'commit': 'PARENT',}], |
| }, |
| } |
| }, |
| 'project': 'project', |
| 'status': test_status, |
| } |
| ), |
| ) |
| details = step.json.output |
| |
| except self.m.step.StepFailure: |
| details = {} |
| |
| self._details[change.name] = details |
| if details: |
| change.project = details['project'] |
| current_revision = details['revisions'][details['current_revision']] |
| change.patchset = current_revision['_number'] |
| change.status = details['status'] |
| change.commit = details['current_revision'] |
| |
| # If this CL is unsubmitted grab its parent and put it in the |
| # cache. We could have several submitted parents in self._details, |
| # but that's ok. |
| if details['status'] == 'NEW': |
| for parent in current_revision['commit']['parents']: |
| parent_result = self._lookup_cl( |
| change.gerrit_name, parent['commit'], |
| ) |
| # There could be no parent CL because this is the first |
| # commit or because the parent was submitted without going |
| # through Gerrit. |
| if not parent_result: |
| continue |
| |
| parent_change = Change( |
| gerrit_name=change.gerrit_name, |
| change=parent_result['_number'], |
| ) |
| self._change_details(parent_change, test_status='SUBMITTED') |
| if parent_change.status == 'NEW': |
| if parent_change.commit != parent['commit']: |
| raise self.m.step.StepFailure( |
| '{} has parent {} which is not the current ' |
| 'revision of its parent, {} ({})'.format( |
| change.name, |
| parent['commit'], |
| parent_change.name, |
| parent_change.commit, |
| ) |
| ) |
| |
| change.parents.append(parent_change) |
| |
| return self._details[change.name] |
