blob: c78a85168d329cac19a00adb157567e298384241 [file] [log] [blame]
Rob Mohr092a4462020-06-16 10:23:54 -07001# Copyright 2020 The Pigweed Authors
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4# use this file except in compliance with the License. You may obtain a copy of
5# the License at
6#
7# https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations under
13# the License.
Rob Mohrd39ea9e2020-09-22 15:01:07 -070014"""Utility functions for rollers."""
Rob Mohr092a4462020-06-16 10:23:54 -070015
16import pprint
17import re
Rob Mohrcbee3fc2020-11-30 14:07:42 -080018import urlparse
Rob Mohr092a4462020-06-16 10:23:54 -070019
20import attr
Rob Mohrf7e30152020-11-18 09:56:30 -080021import enum
Rob Mohr092a4462020-06-16 10:23:54 -070022from recipe_engine import recipe_api
23
24# If we're embedding the original commit message, prepend 'Original-' to lines
25# which begin with these tags.
26ESCAPE_TAGS = [
27 'Bug:',
28 'Fixed:',
Rob Mohr2ec49a02020-12-15 08:37:21 -080029 'Fixes:',
Rob Mohrf6ee7cb2020-10-26 14:05:06 -070030 'Requires:',
Rob Mohr092a4462020-06-16 10:23:54 -070031 'Reviewed-on:',
32]
33
34# If we're embedding the original commit message, remove lines which contain
35# these tags.
36FILTER_TAGS = [
37 'API-Review:',
38 'Acked-by:',
39 'CC:',
40 'CQ-Do-Not-Cancel-Tryjobs:',
41 'Change-Id:',
42 'Commit-Queue:',
Rob Mohr4c6a5382021-02-25 08:50:44 -080043 'Cq-Cl-Tag:',
Rob Mohr092a4462020-06-16 10:23:54 -070044 'Reviewed-by:',
Rob Mohr4c6a5382021-02-25 08:50:44 -080045 'Roller-URL:',
Rob Mohr092a4462020-06-16 10:23:54 -070046 'Signed-off-by:',
47 'Testability-Review:',
48 'Tested-by:',
Rob Mohr7cb53c42020-11-18 10:45:13 -080049 'Auto-Submit',
50 re.compile(r'^\w+-Auto-Submit:'),
Rob Mohr092a4462020-06-16 10:23:54 -070051]
52
53
Rob Mohr7cb53c42020-11-18 10:45:13 -080054def _match_tag(line, tag):
55 if hasattr(tag, 'match'):
56 return tag.match(line)
57 return line.startswith(tag)
58
59
Rob Mohr092a4462020-06-16 10:23:54 -070060def _sanitize_message(message):
61 """Sanitize lines of a commit message.
62
63 Prepend 'Original-' to lines which begin with ESCAPE_TAGS. Filter
64 out lines which begin with FILTER_TAGS.
65 """
66 return '\n'.join(
67 "Original-" + line
68 if any((line.startswith(tag) for tag in ESCAPE_TAGS))
69 else line
70 for line in message.splitlines()
Rob Mohr7cb53c42020-11-18 10:45:13 -080071 if not any((_match_tag(line, tag) for tag in FILTER_TAGS))
Rob Mohr092a4462020-06-16 10:23:54 -070072 )
73
74
Rob Mohrf7e30152020-11-18 09:56:30 -080075class _Direction(enum.Enum):
76 CURRENT = 'CURRENT'
77 FORWARD = 'FORWARD'
78 BACKWARD = 'BACKWARD'
79 REBASE = 'REBASE'
80
81
Rob Mohr092a4462020-06-16 10:23:54 -070082@attr.s
83class Commit(object):
Rob Mohr36984592021-03-31 15:46:38 -070084 hash = attr.ib(type=str)
85 message = attr.ib(type=str)
86 author = attr.ib(type=str)
87 owner = attr.ib(type=str)
88 reviewers = attr.ib(type=tuple)
Rob Mohr092a4462020-06-16 10:23:54 -070089
90
Rob Mohrf6ee7cb2020-10-26 14:05:06 -070091@attr.s
92class Roll(object):
93 _api = attr.ib()
94 project_name = attr.ib(type=str)
95 old_revision = attr.ib(type=str)
96 new_revision = attr.ib(type=str)
97 proj_dir = attr.ib(type=str)
Rob Mohrf7e30152020-11-18 09:56:30 -080098 direction = attr.ib(type=str)
Rob Mohr36984592021-03-31 15:46:38 -070099 commits = attr.ib(type=tuple, default=None)
100 remote = attr.ib(type=str, default=None)
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700101
Rob Mohrf7e30152020-11-18 09:56:30 -0800102 @direction.validator
103 def check(self, _, value): # pragma: no cover
104 if value not in _Direction:
105 raise ValueError('invalid direction: {}'.format(value))
106 if value == _Direction.CURRENT:
107 raise ValueError('attempt to do a no-op roll')
108
Rob Mohr36984592021-03-31 15:46:38 -0700109 def __attrs_post_init__(self):
110 self._set_remote()
111 self._set_commits()
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700112
Rob Mohr36984592021-03-31 15:46:38 -0700113 def _set_commits(self):
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700114 with self._api.context(cwd=self.proj_dir):
115 with self._api.step.nest(self.project_name):
116 commits = []
Rob Mohrf7e30152020-11-18 09:56:30 -0800117 if (
Rob Mohr476a7042020-11-30 13:33:52 -0800118 _is_hash(self.old_revision)
119 and self.direction == _Direction.FORWARD
Rob Mohrf7e30152020-11-18 09:56:30 -0800120 ):
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700121 base = self.old_revision
122 else:
123 base = '{}~5'.format(self.new_revision)
124
125 for commit in (
126 self._api.git(
127 'git log',
128 'log',
129 '{}..{}'.format(base, self.new_revision),
Rob Mohr36984592021-03-31 15:46:38 -0700130 '--pretty=format:%H\n%ae\n%B',
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700131 # Separate entries with null bytes since most entries
132 # will contain newlines ("%B" is the full commit
133 # message, not just the first line.)
134 '-z',
135 stdout=self._api.raw_io.output(),
136 )
137 .stdout.strip('\0')
138 .split('\0')
139 ):
Rob Mohr36984592021-03-31 15:46:38 -0700140 hash, author, message = commit.split('\n', 2)
141 match = re.search(r'Change-Id: (I\w+)', message)
142 owner = None
143 reviewers = []
144 if match:
145 change_id = match.group(1)
Rob Mohr7023db22021-04-01 12:59:05 -0700146 step = self._api.gerrit.change_details(
Rob Mohr36984592021-03-31 15:46:38 -0700147 'get {}'.format(change_id),
148 change_id,
149 host='{}-review.googlesource.com'.format(
150 self.gerrit_name
151 ),
152 test_data=self._api.json.test_api.output(
153 {
154 'owner': {'email': 'owner@example.com'},
155 'reviewers': {
156 'REVIEWER': [
157 {'email': 'reviewer@example.com'},
158 {'email': 'nobody@google.com'},
159 {
160 'email': 'robot@gserviceaccount.com'
161 },
162 ],
163 },
164 }
165 ),
Rob Mohr7023db22021-04-01 12:59:05 -0700166 ok_ret='any',
167 )
168
169 if step.exc_result.retcode == 0:
170 details = step.json.output
171 owner = details['owner']['email']
172 for reviewer in details['reviewers']['REVIEWER']:
173 reviewers.append(reviewer['email'])
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700174
Rob Mohr36984592021-03-31 15:46:38 -0700175 commits.append(
176 Commit(
177 hash=hash,
178 author=author,
179 owner=owner,
180 reviewers=tuple(reviewers),
181 message=message,
182 )
183 )
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700184
Rob Mohr36984592021-03-31 15:46:38 -0700185 self.commits = tuple(commits)
Rob Mohrcbee3fc2020-11-30 14:07:42 -0800186
Rob Mohr36984592021-03-31 15:46:38 -0700187 def _set_remote(self):
Rob Mohrcbee3fc2020-11-30 14:07:42 -0800188 api = self._api
189
190 with api.step.nest('remote'), api.context(cwd=self.proj_dir):
191 name = api.git(
192 'name',
193 'remote',
194 stdout=api.raw_io.output(),
195 step_test_data=lambda: api.raw_io.test_api.stream_output(
196 'origin'
197 ),
198 ).stdout.strip()
199
200 remote = api.git(
201 'url',
202 'remote',
203 'get-url',
204 name,
205 stdout=api.raw_io.output(),
206 step_test_data=lambda: api.raw_io.test_api.stream_output(
207 'sso://pigweed/pigweed/pigweed'
208 ),
209 ).stdout.strip()
210
Rob Mohr36984592021-03-31 15:46:38 -0700211 self.remote = api.sso.sso_to_https(remote)
Rob Mohrcbee3fc2020-11-30 14:07:42 -0800212
213 @property
214 def gerrit_name(self):
215 return urlparse.urlparse(self.remote).netloc.split('.')[0]
216
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700217
218@attr.s
219class Message(object):
220 name = attr.ib(type=str)
221 template = attr.ib(type=str)
222 kwargs = attr.ib(type=dict)
223 num_commits = attr.ib(type=int)
224 footer = attr.ib(type=tuple, default=())
225
226 def render(self, with_footer=True):
227 result = [self.template.format(**self.kwargs)]
228 if with_footer:
229 result.extend(x for x in self.footer)
230 return '\n'.join(result)
231
232
Rob Mohr092a4462020-06-16 10:23:54 -0700233def _is_hash(value):
Rob Mohr57204602020-09-23 08:41:18 -0700234 return re.match(r'^[0-9a-fA-F]{40}', value)
Rob Mohr092a4462020-06-16 10:23:54 -0700235
236
Rob Mohrd39ea9e2020-09-22 15:01:07 -0700237class RollUtilApi(recipe_api.RecipeApi):
Rob Mohr1eb8f912020-10-07 13:17:17 -0700238 def __init__(self, props, *args, **kwargs):
239 super(RollUtilApi, self).__init__(*args, **kwargs)
240 self.labels_to_set = {x.label: x.value for x in props.labels_to_set}
241 self.labels_to_wait_on = props.labels_to_wait_on
Rob Mohr1c1ef302021-03-08 12:23:14 -0800242 self.footer = []
Rob Mohr1eb8f912020-10-07 13:17:17 -0700243
Rob Mohr36984592021-03-31 15:46:38 -0700244 def authors(self, *roll):
245 authors = set()
246 for r in roll:
247 for commit in r.commits:
248 if commit.author:
249 authors.add(commit.author)
250 if commit.owner:
251 authors.add(commit.owner)
252 return authors
253
254 def reviewers(self, *roll):
255 reviewers = set()
256 for r in roll:
257 for commit in r.commits:
258 reviewers.update(commit.reviewers)
259 return reviewers
260
Rob Mohrb04227c2021-04-01 12:38:33 -0700261 def can_cc_on_roll(self, email, host):
Rob Mohr36984592021-03-31 15:46:38 -0700262 # Assume all queried accounts exist on Gerrit in testing except for
263 # nobody@google.com.
264 test_data = self.m.json.test_api.output([{'_account_id': 123}])
265 if email == 'nobody@google.com':
266 test_data = self.m.json.test_api.output([])
267
268 return self.m.gerrit.account_query(
269 email, 'email:{}'.format(email), host=host, test_data=test_data,
270 ).json.output
271
Rob Mohrb04227c2021-04-01 12:38:33 -0700272 def include_cc(self, email, cc_domains, host):
273 with self.m.step.nest('cc {}'.format(email)) as pres:
274 domain = email.split('@', 1)[1]
275 if domain.endswith('gserviceaccount.com'):
276 pres.step_summary_text = 'not CCing, robot account'
277 return False
278 if cc_domains and domain not in cc_domains:
279 pres.step_summary_text = 'not CCing, domain excluded'
280 return False
281 if not self.can_cc_on_roll(email, host=host):
282 pres.step_summary_text = 'not CCing, no account in Gerrit'
283 return False
284
285 pres.step_summary_text = 'CCing'
286 return True
287
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700288 def _single_commit_roll_message(self, roll):
Rob Mohr57204602020-09-23 08:41:18 -0700289 template = """
Rob Mohr092a4462020-06-16 10:23:54 -0700290[roll {project_name}] {sanitized_message}
291
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700292{project_name} Rolled-Commits: {old_revision:.15}..{new_revision:.15}
293 """.strip()
294
295 commit = roll.commits[0]
Rob Mohr092a4462020-06-16 10:23:54 -0700296
Rob Mohr57204602020-09-23 08:41:18 -0700297 kwargs = {
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700298 'project_name': roll.project_name,
Rob Mohr57204602020-09-23 08:41:18 -0700299 'original_message': commit.message,
300 'sanitized_message': _sanitize_message(commit.message),
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700301 'old_revision': roll.old_revision,
302 'new_revision': roll.new_revision,
Rob Mohr57204602020-09-23 08:41:18 -0700303 }
Rob Mohr092a4462020-06-16 10:23:54 -0700304
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700305 message = Message(
306 name=roll.project_name,
307 template=template,
308 kwargs=kwargs,
309 num_commits=1,
310 footer=tuple(self.footer),
311 )
Rob Mohr092a4462020-06-16 10:23:54 -0700312
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700313 with self.m.step.nest(
314 'message for {}'.format(roll.project_name)
315 ) as pres:
Rob Mohr57204602020-09-23 08:41:18 -0700316 pres.logs['template'] = template
317 pres.logs['kwargs'] = pprint.pformat(kwargs)
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700318 pres.logs['message'] = message.render()
Rob Mohr092a4462020-06-16 10:23:54 -0700319
Rob Mohr57204602020-09-23 08:41:18 -0700320 return message
Rob Mohr092a4462020-06-16 10:23:54 -0700321
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700322 def _multiple_commits_roll_message(self, roll):
Rob Mohr57204602020-09-23 08:41:18 -0700323 template = """
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700324[{project_name}] Roll {num_commits} commits
Rob Mohr092a4462020-06-16 10:23:54 -0700325
326{one_liners}
327
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700328{project_name} Rolled-Commits: {old_revision:.15}..{new_revision:.15}
Rob Mohr092a4462020-06-16 10:23:54 -0700329 """.strip()
330
Rob Mohr57204602020-09-23 08:41:18 -0700331 one_liners = [
332 '{:.15} {}'.format(commit.hash, commit.message.splitlines()[0][:50])
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700333 for commit in roll.commits
Rob Mohr57204602020-09-23 08:41:18 -0700334 ]
Rob Mohr092a4462020-06-16 10:23:54 -0700335
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700336 num_commits = len(roll.commits)
337 if not _is_hash(roll.old_revision):
338 num_commits = 'multiple'
Rob Mohr57204602020-09-23 08:41:18 -0700339 one_liners.append('...')
Rob Mohr092a4462020-06-16 10:23:54 -0700340
Rob Mohr57204602020-09-23 08:41:18 -0700341 kwargs = {
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700342 'project_name': roll.project_name,
343 'num_commits': num_commits,
Rob Mohr57204602020-09-23 08:41:18 -0700344 'one_liners': '\n'.join(one_liners),
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700345 'old_revision': roll.old_revision,
346 'new_revision': roll.new_revision,
Rob Mohr57204602020-09-23 08:41:18 -0700347 }
Rob Mohr092a4462020-06-16 10:23:54 -0700348
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700349 message = Message(
350 name=roll.project_name,
351 template=template,
352 kwargs=kwargs,
353 num_commits=num_commits,
354 footer=tuple(self.footer),
355 )
Rob Mohr092a4462020-06-16 10:23:54 -0700356
Rob Mohr57204602020-09-23 08:41:18 -0700357 with self.m.step.nest('message') as pres:
358 pres.logs['template'] = template
359 pres.logs['kwargs'] = pprint.pformat(kwargs)
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700360 pres.logs['message'] = message.render()
Rob Mohr092a4462020-06-16 10:23:54 -0700361
Rob Mohr57204602020-09-23 08:41:18 -0700362 return message
Rob Mohr092a4462020-06-16 10:23:54 -0700363
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700364 def _single_roll_message(self, roll):
365 if len(roll.commits) > 1:
366 return self._multiple_commits_roll_message(roll)
367 return self._single_commit_roll_message(roll)
Rob Mohr092a4462020-06-16 10:23:54 -0700368
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700369 def _multiple_rolls_message(self, *rolls):
370 rolls = sorted(rolls, key=lambda x: x.project_name)
371
372 messages = []
373 for roll in rolls:
374 messages.append(self._single_roll_message(roll))
375
376 texts = [
377 '[roll {}] Roll {} commits'.format(
378 ', '.join(x.name for x in messages),
379 sum(x.num_commits for x in messages),
380 )
381 ]
382 texts.extend(x.render(with_footer=False) for x in messages)
383 texts.append('\n'.join('{}'.format(x) for x in self.footer))
384
385 return '\n\n'.join(texts)
386
387 def Roll(self, **kwargs):
388 """Create a Roll. See Roll class above for details."""
389 return Roll(api=self.m, **kwargs)
390
391 def message(self, *rolls):
Rob Mohr57204602020-09-23 08:41:18 -0700392 with self.m.step.nest('roll message'):
Rob Mohrf6ee7cb2020-10-26 14:05:06 -0700393 if len(rolls) > 1:
394 return self._multiple_rolls_message(*rolls)
395 return self._single_roll_message(*rolls).render()
Rob Mohrd39ea9e2020-09-22 15:01:07 -0700396
Rob Mohrf7e30152020-11-18 09:56:30 -0800397 Direction = _Direction
398
Rob Mohr476a7042020-11-30 13:33:52 -0800399 def get_roll_direction(self, git_dir, old, new, name='get roll direction'):
Rob Mohrf7e30152020-11-18 09:56:30 -0800400 """Return Direction of roll."""
Rob Mohr57204602020-09-23 08:41:18 -0700401 if old == new:
402 with self.m.step.nest(name) as pres:
403 pres.step_summary_text = 'up-to-date'
Rob Mohrf7e30152020-11-18 09:56:30 -0800404 return self.Direction.CURRENT
Rob Mohrd39ea9e2020-09-22 15:01:07 -0700405
Rob Mohr57204602020-09-23 08:41:18 -0700406 with self.m.context(git_dir):
Rob Mohrf7e30152020-11-18 09:56:30 -0800407 with self.m.step.nest(name) as pres:
408 forward = self.m.git(
409 'is forward',
410 'merge-base',
411 '--is-ancestor',
412 old,
413 new,
414 ok_ret=(0, 1),
415 )
Rob Mohrd39ea9e2020-09-22 15:01:07 -0700416
Rob Mohrf7e30152020-11-18 09:56:30 -0800417 backward = self.m.git(
418 'is backward',
419 'merge-base',
420 '--is-ancestor',
421 new,
422 old,
423 ok_ret=(0, 1),
424 )
Rob Mohrd39ea9e2020-09-22 15:01:07 -0700425
Rob Mohrf7e30152020-11-18 09:56:30 -0800426 if (
427 forward.exc_result.retcode == 0
428 and backward.exc_result.retcode != 0
429 ):
430 pres.step_summary_text = 'forward'
431 return self.Direction.FORWARD
Rob Mohrd39ea9e2020-09-22 15:01:07 -0700432
Rob Mohrf7e30152020-11-18 09:56:30 -0800433 if (
434 forward.exc_result.retcode != 0
435 and backward.exc_result.retcode == 0
436 ):
437 pres.step_summary_text = 'backward'
438 return self.Direction.BACKWARD
439
Rob Mohrcbee3fc2020-11-30 14:07:42 -0800440 # If new and old are ancestors of each other then this is the
441 # same commit. We should only hit this during testing because
442 # the comparison at the top of the function should have caught
443 # this situation.
444 if (
445 forward.exc_result.retcode == 0
446 and backward.exc_result.retcode == 0
447 ):
448 with self.m.step.nest(name) as pres:
449 pres.step_summary_text = 'up-to-date'
450 return self.Direction.CURRENT
451
Rob Mohrf7e30152020-11-18 09:56:30 -0800452 # If old is not an ancestor of new and new is not an ancestor
453 # of old then history was rewritten in some manner but we still
454 # need to update the pin.
455 pres.step_summary_text = 'rebase'
456 return self.Direction.REBASE
457
458 def can_roll(self, direction):
459 return direction in (self.Direction.FORWARD, self.Direction.REBASE)
460
461 def skip_roll_step(self, remote, old_revision, new_revision):
Rob Mohr57204602020-09-23 08:41:18 -0700462 with self.m.step.nest('cancelling roll') as pres:
463 fmt = (
464 'not updating from {old} to {new} because {old} is newer '
465 'than {new}'
466 )
467 if old_revision == new_revision:
Rob Mohrad16bc12020-09-28 06:39:55 -0700468 fmt = (
469 'not updating from {old} to {new} because they are '
470 'identical'
471 )
Rob Mohr57204602020-09-23 08:41:18 -0700472 pres.step_summary_text = fmt.format(
473 old=old_revision[0:7], new=new_revision[0:7]
474 )
475 pres.links[old_revision] = '{}/+/{}'.format(remote, old_revision)
476 pres.links[new_revision] = '{}/+/{}'.format(remote, new_revision)