# Copyright 2020 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""Recipe for testing Pigweed using presubmit_checks.py script."""

import collections
import re

from PB.recipes.pigweed.cipd_roller import InputProperties

DEPS = [
    'fuchsia/auto_roller',
    'fuchsia/buildbucket_util',
    'fuchsia/status_check',
    'pigweed/checkout',
    'pigweed/roll_util',
    'recipe_engine/buildbucket',
    'recipe_engine/cipd',
    'recipe_engine/file',
    'recipe_engine/json',
    'recipe_engine/path',
    'recipe_engine/properties',
    'recipe_engine/step',
]

PROPERTIES = InputProperties

COMMIT_MESSAGE = """
roll: {package_name}

From: {old_version}
To: {new_version}

CQ-Do-Not-Cancel-Tryjobs: true
""".strip()


def _is_platform(part):
    if '{' in part:
        return True
    os = part.split('-')[0]
    return os in ('linux', 'mac', 'windows')


def find_shared_tags(api, package_tags, tag):
    """Attempts to find a tag shared by all packages.

    This function can be used if the intersection of the sets of tags
    associated with different-platform packages with the same 'ref' is empty.
    It finds a tag shared by all packages, with as many of them as possible
    matching 'ref'.
    """
    # Find the most common tags.  We use the sorted dict keys for determinism.
    package_paths = sorted(package_tags.keys())
    counter = collections.Counter()
    for path in package_paths:
        counter.update(package_tags[path])
    most_common_tags = counter.most_common()

    with api.step.nest("find shared tag"):
        for tag_candidate, _ in most_common_tags:
            # There is at least one package for which the version with the
            # specified 'ref' does not have this tag. See if there exists a
            # version of this package that *does* have this tag.  If so, use
            # that version.
            updated_tags = dict()
            for package_path in package_paths:
                if tag_candidate in package_tags[package_path]:
                    # For this package we already have a version with this tag,
                    # nothing to do.
                    continue
                try:
                    package_data = api.cipd.describe(
                        package_path, tag_candidate
                    )
                except api.step.StepFailure:
                    # No luck: there exists no version with this tag.
                    break
                updated_tags[package_path] = set(
                    x.tag
                    for x in package_data.tags
                    if x.tag.startswith(tag + ':')
                )
            else:
                # We found a version of each package with the tag_candidate.
                merged_tags = dict()
                merged_tags.update(package_tags)
                merged_tags.update(updated_tags)
                tags = set.intersection(*merged_tags.values())
                # Should always succeed.
                assert len(tags) > 0
                # Update package_tags to be consistent with the returned tags.
                package_tags.update(updated_tags)
                return tags

    # We failed to find any tag that meets our criteria.
    return set()


def RunSteps(api, props):
    cipd_json_path = props.cipd_json_path
    package_name = props.package_name
    package_spec = props.package_spec
    ref = props.ref
    tag = props.tag
    dry_run = props.dry_run
    allow_mismatched_refs = props.allow_mismatched_refs

    props.checkout_options.use_trigger = False
    checkout = api.checkout(props.checkout_options)

    cipd_json_path = checkout.root.join(*re.split(r'[\\/]+', cipd_json_path))

    if not package_name:
        # Turn foo/bar/baz/${platform} and foo/bar/baz/${os=mac}-${arch} into
        # 'baz'.
        package_name = [
            part for part in package_spec.split('/') if not _is_platform(part)
        ][-1]

    basename = api.path.basename(cipd_json_path)
    cipd_json = api.file.read_json('read {}'.format(basename), cipd_json_path)
    packages = cipd_json
    if isinstance(cipd_json, dict):
        packages = cipd_json['packages']
    old_version = None
    package = None
    for package in packages:
        if package['path'] == package_spec:
            old_version = package['tags'][0]
            break
    else:
        raise api.step.StepFailure(
            "couldn't find package {} in {}".format(
                package_spec, cipd_json_path
            )
        )

    assert package.get('platforms'), 'platforms empty in json'
    platforms = package.get('platforms')
    base, name = package_spec.rstrip('/').rsplit('/', 1)
    if _is_platform(name):
        package_paths = ['{}/{}'.format(base, x) for x in platforms]
    else:
        package_paths = [package_spec]

    package_tags = {}
    tags = None
    for package_path in package_paths:
        try:
            package_data = api.cipd.describe(package_path, ref)

        except api.step.StepFailure:
            # If we got here this package doesn't have the correct ref. This is
            # likely because it's a new platform for an existing package. In
            # that case ignore this platform when checking that refs agree on
            # package versions. We still need at least one platform to have the
            # ref or the checks below will fail.
            pass

        else:
            package_tags[package_path] = set(
                x.tag for x in package_data.tags if x.tag.startswith(tag + ':')
            )
            if tags is None:
                tags = set(package_tags[package_path])
            else:
                tags.intersection_update(package_tags[package_path])

    if not tags and allow_mismatched_refs:
        # The package with the requested ref has non-overlapping tag values for
        # different platforms.  Try relaxing the requirement that all packages
        # come from the same ref, and see if this allows us to find a set with
        # shared tag values.
        tags = find_shared_tags(api, package_tags, tag)

    with api.step.nest('common tags') as presentation:
        presentation.step_summary_text = '\n'.join(sorted(tags))

    if not tags:
        err_lines = ['no common tags across "{}" refs of packages'.format(ref)]
        for package_path, package_tags in sorted(package_tags.items()):
            err_lines.append('')
            err_lines.append(package_path)
            for tag in package_tags:
                err_lines.append(tag)

        raise api.step.StepFailure('<br>'.join(err_lines))
    # Deterministically pick one of the common tags.
    new_version = sorted(tags)[0]
    package['tags'] = [new_version]

    # Verify there's only one instance of each platform package with this tag.
    with api.step.nest('check number of instances'):
        for package_path in package_paths:
            api.cipd.describe(package_path, new_version)

    with api.step.nest('new_version') as presentation:
        presentation.step_summary_text = new_version

    if old_version in tags:
        with api.step.nest('already up-to-date') as presentation:
            presentation.step_summary_text = (
                'current version {} in common tags'
            ).format(old_version)
        return

    api.file.write_text(
        'write {}'.format(basename),
        cipd_json_path,
        api.json.dumps(cipd_json, indent=2, separators=(',', ': ')) + '\n',
    )

    change = api.auto_roller.attempt_roll(
        api.auto_roller.Options(
            remote=checkout.options.remote,
            upstream_ref=checkout.options.branch,
            dry_run=dry_run,
            labels_to_set=api.roll_util.labels_to_set,
            labels_to_wait_on=api.roll_util.labels_to_wait_on,
            bot_commit=props.bot_commit,
        ),
        repo_dir=api.path.dirname(cipd_json_path),
        commit_message=COMMIT_MESSAGE.format(
            package_name=package_name,
            package_spec=package_spec,
            old_version=old_version,
            new_version=new_version,
            builder=api.buildbucket.builder_name,
            build_id=api.buildbucket_util.id,
        ),
    )

    return api.auto_roller.raw_result(change)


def GenTests(api):  # pylint: disable=invalid-name
    """Create tests."""

    def package(cipd_path, old_version, platforms=None):
        result = {
            'path': cipd_path,
            'tags': [old_version],
            '_comment': 'comments should be preserved',
        }

        if platforms is not None:
            result['platforms'] = list(platforms)

        return result

    def describe(package_path, *tags, **kwargs):
        return api.step_data(
            'cipd describe {}'.format(package_path),
            api.cipd.example_describe(
                package_path,
                test_data_tags=[
                    '{}:{}'.format(name, value) for name, value in tags
                ],
            ),
            **kwargs,
        )

    def no_ref(package_paths):
        res = None
        for package_path in package_paths:
            step = describe(package_path, retcode=1)
            res = res + step if res else step
        return res

    def no_common_tags(package_paths, tagname='git_revision'):
        res = None
        for i, package_path in enumerate(package_paths):
            step = describe(package_path, (tagname, i))
            res = res + step if res else step
        return res

    def multiple_common_tags(package_paths, tagname='git_revision'):
        res = None
        for package_path in package_paths:
            step = describe(
                package_path, (tagname, 1), (tagname, 2), (tagname, 3)
            )
            res = res + step if res else step
        return res

    def relaxing_works(package_paths, tagname='git_revision'):
        return api.step_data(
            'find shared tag.cipd describe {}'.format(package_paths[1]),
            api.cipd.example_describe(
                package_paths[1], test_data_tags=['{}:{}'.format(tagname, 0)],
            ),
        )

    def relaxing_does_not_work(package_paths, tagname='git_revision'):
        # No version of package_paths[1] with hoped-for tag.
        return api.step_data(
            'find shared tag.cipd describe {}'.format(package_paths[0]),
            api.cipd.example_describe(package_paths[0]),
            retcode=1,
        ) + api.step_data(
            'find shared tag.cipd describe {}'.format(package_paths[1]),
            api.cipd.example_describe(package_paths[1]),
            retcode=1,
        )

    def read_file_step_data(package_json_name, *packages):
        return api.step_data(
            'read {}'.format(package_json_name),
            api.file.read_json({'packages': packages}),
        )

    spec = 'pigweed/host_tools/${platform}'
    paths = (
        'pigweed/host_tools/linux-amd64',
        'pigweed/host_tools/windows-amd64',
    )

    def properties(package_spec=spec, dry_run=True, **kwargs):
        new_kwargs = {
            'package_spec': package_spec,
            'dry_run': dry_run,
            'ref': 'latest',
            'tag': 'git_revision',
            'cipd_json_path': 'pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json',
        }
        new_kwargs.update(api.checkout.git_properties())
        new_kwargs.update(kwargs)
        return api.properties(**new_kwargs)

    yield (
        api.status_check.test('success')
        + properties()
        + api.checkout.ci_test_data()
        + read_file_step_data(
            'pigweed.json',
            package(
                spec,
                'git_revision:123',
                platforms=['linux-amd64', 'windows-amd64'],
            ),
        )
        + api.auto_roller.dry_run_success()
    )

    yield (
        api.status_check.test('bad_package_spec', status='failure')
        + properties(package_spec='bad-{}'.format(spec))
        + api.checkout.ci_test_data()
        + read_file_step_data('pigweed.json', package(spec, 'git_revision:123'))
    )

    yield (
        api.status_check.test('no_common_tags', status='failure')
        + properties()
        + api.checkout.ci_test_data()
        + read_file_step_data(
            'pigweed.json',
            package(
                spec,
                'git_revision:123',
                platforms=('linux-amd64', 'windows-amd64'),
            ),
        )
        + no_common_tags(paths)
    )

    yield (
        api.status_check.test('no_common_tags_but_relaxing_ref_mismatch_helps')
        + properties(allow_mismatched_refs=True)
        + api.checkout.ci_test_data()
        + read_file_step_data(
            'pigweed.json',
            package(
                spec,
                'git_revision:123',
                platforms=('linux-amd64', 'windows-amd64'),
            ),
        )
        # Package 0 exists at git_revision:0, package 1 at git_revision:1
        + no_common_tags(paths)
        # However, package 1 also exists at git_revision:0, so that revision is
        # good to use.
        + relaxing_works(paths)
        + api.auto_roller.dry_run_success()
    )

    yield (
        api.status_check.test(
            'no_common_tags_and_relaxing_ref_mismatch_does_not_help',
            status='failure',
        )
        + properties(allow_mismatched_refs=True)
        + api.checkout.ci_test_data()
        + read_file_step_data(
            'pigweed.json',
            package(
                spec,
                'git_revision:123',
                platforms=('linux-amd64', 'windows-amd64'),
            ),
        )
        # Package 0 exists at git_revision:0, package 1 at git_revision:1
        + no_common_tags(paths)
        # However, package 1 does not exist at git_revision:0.
        + relaxing_does_not_work(paths)
    )

    yield (
        api.status_check.test('multiple_common_tags')
        + properties()
        + api.checkout.ci_test_data()
        + multiple_common_tags(paths)
        + read_file_step_data(
            'pigweed.json',
            package(
                spec,
                'git_revision:2',
                platforms=('linux-amd64', 'windows-amd64'),
            ),
        )
    )

    yield (
        api.status_check.test('missing_tag')
        + properties()
        + api.checkout.ci_test_data()
        + multiple_common_tags(paths)
        + no_ref(['pigweed/host_tools/fake-amd64'])
        + read_file_step_data(
            'pigweed.json',
            package(
                spec,
                'git_revision:2',
                platforms=('linux-amd64', 'windows-amd64', 'fake-amd64'),
            ),
        )
    )

    no_curly_spec = 'pigweed/host_tools/linux-amd64'
    yield (
        api.status_check.test('no_curlies_in_spec')
        + properties(package_spec=no_curly_spec)
        + api.checkout.ci_test_data()
        + read_file_step_data(
            'pigweed.json',
            package(
                no_curly_spec,
                'git_revision:123',
                platforms=('linux-amd64', 'windows-amd64'),
            ),
        )
        + api.auto_roller.dry_run_success()
    )

    yield (
        api.status_check.test('platform-independent')
        + properties(package_spec='foo/bar/baz')
        + api.checkout.ci_test_data()
        + read_file_step_data(
            'pigweed.json',
            package(
                'foo/bar/baz',
                'git_revision:123',
                platforms=['linux-amd64', 'mac-amd64'],
            ),
        )
        + api.auto_roller.dry_run_success()
    )