| # Copyright 2020 The Pigweed Authors |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may not |
| # use this file except in compliance with the License. You may obtain a copy of |
| # the License at |
| # |
| # https://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations under |
| # the License. |
| """Update a CIPD package to the latest version.""" |
| |
| import collections |
| import re |
| |
| from PB.recipes.pigweed.cipd_roller import InputProperties |
| |
| DEPS = [ |
| 'fuchsia/auto_roller', |
| 'fuchsia/buildbucket_util', |
| 'fuchsia/status_check', |
| 'pigweed/checkout', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/cipd', |
| 'recipe_engine/file', |
| 'recipe_engine/json', |
| 'recipe_engine/path', |
| 'recipe_engine/properties', |
| 'recipe_engine/step', |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| COMMIT_MESSAGE = """ |
| roll: {package_name} |
| |
| From: {old_version} |
| To: {new_version} |
| |
| CQ-Do-Not-Cancel-Tryjobs: true |
| Build-Errors: continue |
| """.strip() |
| |
| |
| def _is_platform(part): |
| """Return true for platform-style strings. |
| |
| Example matches: "linux-amd64", "${platform}", "${os}-amd64", etc. |
| Example non-matches: "clang", "amd64", "linux". |
| """ |
| if '{' in part: |
| return True |
| try: |
| os, arch = part.split('-') |
| return os in ('linux', 'mac', 'windows') |
| except ValueError: |
| return False |
| |
| |
| def find_shared_tags(api, package_tags, tag): |
| """Attempts to find a tag shared by all packages. |
| |
| This function can be used if the intersection of the sets of tags |
| associated with different-platform packages with the same 'ref' is empty. |
| It finds a tag shared by all packages, with as many of them as possible |
| matching 'ref'. |
| """ |
| # Find the most common tags. We use the sorted dict keys for determinism. |
| package_paths = sorted(package_tags.keys()) |
| counter = collections.Counter() |
| for path in package_paths: |
| counter.update(package_tags[path]) |
| most_common_tags = counter.most_common() |
| |
| with api.step.nest("find shared tag"): |
| for tag_candidate, _ in most_common_tags: |
| # There is at least one package for which the version with the |
| # specified 'ref' does not have this tag. See if there exists a |
| # version of this package that *does* have this tag. If so, use |
| # that version. |
| updated_tags = dict() |
| for package_path in package_paths: |
| if tag_candidate in package_tags[package_path]: |
| # For this package we already have a version with this tag, |
| # nothing to do. |
| continue |
| try: |
| package_data = api.cipd.describe( |
| package_path, tag_candidate |
| ) |
| except api.step.StepFailure: |
| # No luck: there exists no version with this tag. |
| break |
| updated_tags[package_path] = set( |
| x.tag |
| for x in package_data.tags |
| if x.tag.startswith(tag + ':') |
| ) |
| else: |
| # We found a version of each package with the tag_candidate. |
| merged_tags = dict() |
| merged_tags.update(package_tags) |
| merged_tags.update(updated_tags) |
| tags = set.intersection(*merged_tags.values()) |
| # Should always succeed. |
| assert len(tags) > 0 |
| # Update package_tags to be consistent with the returned tags. |
| package_tags.update(updated_tags) |
| return tags |
| |
| # We failed to find any tag that meets our criteria. |
| return set() |
| |
| |
| def RunSteps(api, props): |
| cipd_json_path = props.cipd_json_path |
| package_name = props.package_name |
| package_spec = props.package_spec |
| ref = props.ref |
| tag = props.tag |
| allow_mismatched_refs = props.allow_mismatched_refs |
| |
| props.checkout_options.use_trigger = False |
| checkout = api.checkout(props.checkout_options) |
| |
| cipd_json_path = checkout.root.join(*re.split(r'[\\/]+', cipd_json_path)) |
| |
| if not package_name: |
| # Turn foo/bar/baz/${platform} and foo/bar/baz/${os=mac}-${arch} into |
| # 'baz'. |
| package_name = [ |
| part for part in package_spec.split('/') if not _is_platform(part) |
| ][-1] |
| |
| basename = api.path.basename(cipd_json_path) |
| cipd_json = api.file.read_json('read {}'.format(basename), cipd_json_path) |
| packages = cipd_json |
| if isinstance(cipd_json, dict): |
| packages = cipd_json['packages'] |
| old_version = None |
| package = None |
| for package in packages: |
| if package['path'] == package_spec: |
| old_version = package['tags'][0] |
| break |
| else: |
| raise api.step.StepFailure( |
| "couldn't find package {} in {}".format( |
| package_spec, cipd_json_path |
| ) |
| ) |
| |
| assert package.get('platforms'), 'platforms empty in json' |
| platforms = package.get('platforms') |
| base, name = package_spec.rstrip('/').rsplit('/', 1) |
| if _is_platform(name): |
| package_paths = ['{}/{}'.format(base, x) for x in platforms] |
| else: |
| package_paths = [package_spec] |
| |
| package_tags = {} |
| tags = None |
| for package_path in package_paths: |
| try: |
| package_data = api.cipd.describe(package_path, ref) |
| |
| except api.step.StepFailure: |
| # If we got here this package doesn't have the correct ref. This is |
| # likely because it's a new platform for an existing package. In |
| # that case ignore this platform when checking that refs agree on |
| # package versions. We still need at least one platform to have the |
| # ref or the checks below will fail. |
| pass |
| |
| else: |
| package_tags[package_path] = set( |
| x.tag for x in package_data.tags if x.tag.startswith(tag + ':') |
| ) |
| if tags is None: |
| tags = set(package_tags[package_path]) |
| else: |
| tags.intersection_update(package_tags[package_path]) |
| |
| if not tags and allow_mismatched_refs: |
| # The package with the requested ref has non-overlapping tag values for |
| # different platforms. Try relaxing the requirement that all packages |
| # come from the same ref, and see if this allows us to find a set with |
| # shared tag values. |
| tags = find_shared_tags(api, package_tags, tag) |
| |
| with api.step.nest('common tags') as presentation: |
| presentation.step_summary_text = '\n'.join(sorted(tags)) |
| |
| if not tags: |
| err_lines = ['no common tags across "{}" refs of packages'.format(ref)] |
| for package_path, package_tags in sorted(package_tags.items()): |
| err_lines.append('') |
| err_lines.append(package_path) |
| for tag in package_tags: |
| err_lines.append(tag) |
| |
| raise api.step.StepFailure('<br>'.join(err_lines)) |
| # Deterministically pick one of the common tags. |
| new_version = sorted(tags)[0] |
| package['tags'] = [new_version] |
| |
| # Verify there's only one instance of each platform package with this tag. |
| with api.step.nest('check number of instances'): |
| for package_path in package_paths: |
| api.cipd.describe(package_path, new_version) |
| |
| with api.step.nest('new_version') as presentation: |
| presentation.step_summary_text = new_version |
| |
| if old_version in tags: |
| with api.step.nest('already up-to-date') as presentation: |
| presentation.step_summary_text = ( |
| 'current version {} in common tags' |
| ).format(old_version) |
| return |
| |
| api.file.write_text( |
| 'write {}'.format(basename), |
| cipd_json_path, |
| api.json.dumps(cipd_json, indent=2, separators=(',', ': ')) + '\n', |
| ) |
| |
| change = api.auto_roller.attempt_roll( |
| props.auto_roller_options, |
| repo_dir=api.path.dirname(cipd_json_path), |
| commit_message=COMMIT_MESSAGE.format( |
| package_name=package_name, |
| package_spec=package_spec, |
| old_version=old_version, |
| new_version=new_version, |
| builder=api.buildbucket.builder_name, |
| build_id=api.buildbucket_util.id, |
| ), |
| ) |
| |
| return api.auto_roller.raw_result(change) |
| |
| |
| def GenTests(api): # pylint: disable=invalid-name |
| """Create tests.""" |
| |
| def package(cipd_path, old_version, platforms=None): |
| result = { |
| 'path': cipd_path, |
| 'tags': [old_version], |
| '_comment': 'comments should be preserved', |
| } |
| |
| if platforms is not None: |
| result['platforms'] = list(platforms) |
| |
| return result |
| |
| def describe(package_path, *tags, **kwargs): |
| return api.step_data( |
| 'cipd describe {}'.format(package_path), |
| api.cipd.example_describe( |
| package_path, |
| test_data_tags=[ |
| '{}:{}'.format(name, value) for name, value in tags |
| ], |
| ), |
| **kwargs, |
| ) |
| |
| def no_ref(package_paths): |
| res = None |
| for package_path in package_paths: |
| step = describe(package_path, retcode=1) |
| res = res + step if res else step |
| return res |
| |
| def no_common_tags(package_paths, tagname='git_revision'): |
| res = None |
| for i, package_path in enumerate(package_paths): |
| step = describe(package_path, (tagname, i)) |
| res = res + step if res else step |
| return res |
| |
| def multiple_common_tags(package_paths, tagname='git_revision'): |
| res = None |
| for package_path in package_paths: |
| step = describe( |
| package_path, (tagname, 1), (tagname, 2), (tagname, 3) |
| ) |
| res = res + step if res else step |
| return res |
| |
| def relaxing_works(package_paths, tagname='git_revision'): |
| return api.step_data( |
| 'find shared tag.cipd describe {}'.format(package_paths[1]), |
| api.cipd.example_describe( |
| package_paths[1], test_data_tags=['{}:{}'.format(tagname, 0)], |
| ), |
| ) |
| |
| def relaxing_does_not_work(package_paths, tagname='git_revision'): |
| # No version of package_paths[1] with hoped-for tag. |
| return api.step_data( |
| 'find shared tag.cipd describe {}'.format(package_paths[0]), |
| api.cipd.example_describe(package_paths[0]), |
| retcode=1, |
| ) + api.step_data( |
| 'find shared tag.cipd describe {}'.format(package_paths[1]), |
| api.cipd.example_describe(package_paths[1]), |
| retcode=1, |
| ) |
| |
| def read_file_step_data(package_json_name, *packages): |
| return api.step_data( |
| 'read {}'.format(package_json_name), |
| api.file.read_json({'packages': packages}), |
| ) |
| |
| spec = 'pigweed/host_tools/${platform}' |
| paths = ( |
| 'pigweed/host_tools/linux-amd64', |
| 'pigweed/host_tools/windows-amd64', |
| ) |
| |
| def properties(package_spec=spec, dry_run=True, **kwargs): |
| new_kwargs = { |
| 'package_spec': package_spec, |
| 'ref': 'latest', |
| 'tag': 'git_revision', |
| 'cipd_json_path': 'pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json', |
| } |
| new_kwargs.update(api.checkout.git_properties()) |
| new_kwargs.setdefault( |
| 'auto_roller_options', |
| {'dry_run': dry_run, 'remote': api.checkout.pigweed_repo,}, |
| ) |
| new_kwargs.update(kwargs) |
| return api.properties(**new_kwargs) |
| |
| yield ( |
| api.status_check.test('success') |
| + properties() |
| + api.checkout.ci_test_data() |
| + read_file_step_data( |
| 'pigweed.json', |
| package( |
| spec, |
| 'git_revision:123', |
| platforms=['linux-amd64', 'windows-amd64'], |
| ), |
| ) |
| + api.auto_roller.dry_run_success() |
| ) |
| |
| yield ( |
| api.status_check.test('bad_package_spec', status='failure') |
| + properties(package_spec='bad-{}'.format(spec)) |
| + api.checkout.ci_test_data() |
| + read_file_step_data('pigweed.json', package(spec, 'git_revision:123')) |
| ) |
| |
| yield ( |
| api.status_check.test('no_common_tags', status='failure') |
| + properties() |
| + api.checkout.ci_test_data() |
| + read_file_step_data( |
| 'pigweed.json', |
| package( |
| spec, |
| 'git_revision:123', |
| platforms=('linux-amd64', 'windows-amd64'), |
| ), |
| ) |
| + no_common_tags(paths) |
| ) |
| |
| yield ( |
| api.status_check.test('no_common_tags_but_relaxing_ref_mismatch_helps') |
| + properties(allow_mismatched_refs=True) |
| + api.checkout.ci_test_data() |
| + read_file_step_data( |
| 'pigweed.json', |
| package( |
| spec, |
| 'git_revision:123', |
| platforms=('linux-amd64', 'windows-amd64'), |
| ), |
| ) |
| # Package 0 exists at git_revision:0, package 1 at git_revision:1 |
| + no_common_tags(paths) |
| # However, package 1 also exists at git_revision:0, so that revision is |
| # good to use. |
| + relaxing_works(paths) |
| + api.auto_roller.dry_run_success() |
| ) |
| |
| yield ( |
| api.status_check.test( |
| 'no_common_tags_and_relaxing_ref_mismatch_does_not_help', |
| status='failure', |
| ) |
| + properties(allow_mismatched_refs=True) |
| + api.checkout.ci_test_data() |
| + read_file_step_data( |
| 'pigweed.json', |
| package( |
| spec, |
| 'git_revision:123', |
| platforms=('linux-amd64', 'windows-amd64'), |
| ), |
| ) |
| # Package 0 exists at git_revision:0, package 1 at git_revision:1 |
| + no_common_tags(paths) |
| # However, package 1 does not exist at git_revision:0. |
| + relaxing_does_not_work(paths) |
| ) |
| |
| yield ( |
| api.status_check.test('multiple_common_tags') |
| + properties() |
| + api.checkout.ci_test_data() |
| + multiple_common_tags(paths) |
| + read_file_step_data( |
| 'pigweed.json', |
| package( |
| spec, |
| 'git_revision:2', |
| platforms=('linux-amd64', 'windows-amd64'), |
| ), |
| ) |
| ) |
| |
| yield ( |
| api.status_check.test('missing_tag') |
| + properties() |
| + api.checkout.ci_test_data() |
| + multiple_common_tags(paths) |
| + no_ref(['pigweed/host_tools/fake-amd64']) |
| + read_file_step_data( |
| 'pigweed.json', |
| package( |
| spec, |
| 'git_revision:2', |
| platforms=('linux-amd64', 'windows-amd64', 'fake-amd64'), |
| ), |
| ) |
| ) |
| |
| no_curly_spec = 'pigweed/host_tools/linux-amd64' |
| yield ( |
| api.status_check.test('no_curlies_in_spec') |
| + properties(package_spec=no_curly_spec) |
| + api.checkout.ci_test_data() |
| + read_file_step_data( |
| 'pigweed.json', |
| package( |
| no_curly_spec, |
| 'git_revision:123', |
| platforms=('linux-amd64', 'windows-amd64'), |
| ), |
| ) |
| + api.auto_roller.dry_run_success() |
| ) |
| |
| yield ( |
| api.status_check.test('platform-independent') |
| + properties(package_spec='foo/bar/baz') |
| + api.checkout.ci_test_data() |
| + read_file_step_data( |
| 'pigweed.json', |
| package( |
| 'foo/bar/baz', |
| 'git_revision:123', |
| platforms=['linux-amd64', 'mac-amd64'], |
| ), |
| ) |
| + api.auto_roller.dry_run_success() |
| ) |
