# Copyright 2020 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""This module provides advanced source code checkout capabilities.

The checkout recipe module is a critical component for CI/CQ, responsible for
retrieving source code from version control systems. It ensures that the
correct version of the code, including dependencies, is available for builds,
tests, and deployments.

The module's primary function is to check out code from Git repositories. It
can perform simple checkouts of a given remote and branch, or it can use the
Android Repo Tool to work with manifest-based projects. When using the repo
tool, it can initialize the workspace, sync all projects, and correctly handle
manifest branches and groups.

A key feature of this module is its deep integration with Gerrit. It can
process Gerrit changes provided by LUCI builds, and it can apply stacks of
dependent changes. It uses the fuchsia/gerrit and pigweed/cq_deps modules to
resolve and apply dependent CLs that are specified in `patches.json` files.
When applying changes, it can handle both direct commits and rebasing of
changes onto a target branch.

Submodule management is also a core capability. The module can initialize and
update all submodules, or it can be configured to work with a specific subset
of submodules. It can also intelligently apply Gerrit changes to the correct
submodule repository by matching remote URLs and branches.

Additionally, the module supports applying changes to projects managed as Bazel
external repositories. It can parse Bazel workspace files to find repositories
matching the triggering changes, check them out, and provide overrides for Bazel
builds.

To improve performance, the module supports caching of repository data. This
can significantly speed up checkout times in CI environments. The checkout
process is highly configurable, with options to control timeouts, retry
attempts, remote URL rewriting, and more.

The module also provides a `CheckoutContext` object that encapsulates the
state of the checkout. This includes information about the root directory, the
applied changes, and the final revision of the code. This context object can
be used by other recipe modules to interact with the checked-out code.
"""

from __future__ import annotations

import contextlib
import re
import urllib
import xml.etree.ElementTree
from collections.abc import Generator, Iterator, Sequence
from typing import Any, TypedDict

from google.protobuf import json_format

from recipe_engine import config_types, recipe_api

from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb
from PB.go.chromium.org.luci.scheduler.api.scheduler.v1 import (
    triggers as triggers_pb,
)
from PB.recipe_modules.pigweed.checkout.options import Options

from . import data

PIGWEED_REMOTE = 'https://pigweed.googlesource.com/pigweed/pigweed'

Change = data.Change
CheckoutContext = data.CheckoutContext
Manifest = data.Manifest
Project = data.Project
Remote = data.Remote
StatusOfChanges = data.StatusOfChanges
Submodule = data.Submodule
Url = data.Url
to_dict = data.to_dict


class CheckoutApi(recipe_api.RecipeApi):
    """Calls to checkout code."""

    Change = Change
    CheckoutContext = CheckoutContext

    def fake_context(self):  # pragma: no cover
        ctx = CheckoutContext(self.m)
        ctx.top = ctx.root = self.m.path.start_dir / 'checkout'
        ctx.options = Options(remote=self.test_api.pigweed_repo)
        ctx.changes = [
            Change(
                number=1234,
                remote='https://pigweed.googlesource.com/pigweed/pigweed',
                ref='5' * 40,
                branch='main',
            ),
        ]
        return ctx

    def _read_manifest(
        self, manifest_remote: str, manifest_file: str
    ) -> Manifest:
        """Reads manifest file to get git repo locations."""

        with self.m.step.nest('read manifest') as read_step:
            manifest_text: str = self.m.file.read_text(
                'read file', manifest_file
            )
            read_step.logs['raw'] = manifest_text

            xml_tree = xml.etree.ElementTree.fromstring(manifest_text)

            manifest = Manifest()

            for remote in xml_tree.iter('remote'):
                with self.m.step.nest('log') as pres:
                    pres.step_summary_text = repr(remote.attrib)
                remote = Remote(**remote.attrib)
                if remote.fetch.startswith('..'):
                    rest = re.sub(r'^(..)(/..)*', '', remote.fetch)
                    parsed = urllib.parse.urlparse(manifest_remote)
                    remote.fetch = f'{parsed.scheme}://{parsed.netloc}' + rest
                remote.fetch = Url(remote.fetch)
                remote.fetch.https = self.m.sso.sso_to_https(remote.fetch.url)
                manifest.remotes[remote.name] = remote

            defaults = {}
            for default in xml_tree.iter('default'):
                defaults.update(default.attrib)

            for project in xml_tree.iter('project'):
                name = project.attrib['name']
                path = project.attrib.get('path', name)

                if 'remote' in project.attrib:
                    remote = project.attrib['remote']
                elif 'remote' in defaults:
                    remote = defaults['remote']
                else:  # pragma: no cover
                    assert False, f'remote not specified for {name}'

                assert remote in manifest.remotes, (
                    f'Remote {remote} does not exist'
                )

                if 'revision' in project.attrib:
                    revision = project.attrib['revision']
                elif manifest.remotes[remote].revision:
                    revision = manifest.remotes[remote].revision
                elif 'revision' in defaults:
                    revision = defaults['revision']
                else:  # pragma: no cover
                    assert False, f'revision not specified for {name}'

                if 'upstream' in project.attrib:
                    upstream = project.attrib['upstream']
                elif 'upstream' in defaults:  # pragma: no cover
                    # This is unlikely to be used and hard to test--it requires
                    # a completely separate manifest definition, otherwise the
                    # 'else' condition won't be covered. It's also simple.
                    upstream = defaults['upstream']
                else:
                    upstream = revision

                # urllib.urljoin does something different than what's desired
                # here.
                url = '/'.join(
                    (
                        manifest.remotes[remote].fetch.https.rstrip('/'),
                        name.lstrip('/'),
                    )
                )
                manifest.projects.append(
                    Project(
                        name=name,
                        path=path,
                        remote=remote,
                        revision=revision,
                        upstream=upstream,
                        url=url,
                    )
                )

            self.m.file.write_json(
                'manifest json',
                self.m.path.start_dir / 'manifest.json',
                manifest.dict(),
            )

        return manifest

    def _process_gerrit_change(
        self,
        ctx: CheckoutContext,
        host: str,
        change_id: str | int,
        project: str = 'pigweed/pigweed',
        patchset: int | None = None,
        primary: bool = False,
    ) -> Change:
        """Process a LUCI GerritChange and return a Change object."""

        host = self.m.gerrit.normalize_host(host)
        gitiles_host = host.replace(
            '-review.googlesource.com', '.googlesource.com'
        )
        gerrit_name = gitiles_host.split('.')[0]
        details = self.m.gerrit.change_details(
            'details',
            change_id=str(change_id),
            host=host,
            max_attempts=5,
            query_params=[
                'CURRENT_COMMIT',
                'CURRENT_REVISION',
                'SUBMIT_REQUIREMENTS',
            ],
            timeout=30,
            test_data=self.m.json.test_api.output(
                {
                    'branch': 'main',
                    'current_revision': 'f' * 40,
                    'revisions': {
                        'f' * 40: {
                            '_number': 3,
                            'commit': {
                                'parents': [{}],
                                'message': '',
                            },
                            'uploader': {
                                'email': 'noreply@google.com',
                            },
                        }
                    },
                    'project': project,
                    'submit_requirements': [
                        {
                            'name': 'Code-Review',
                            'status': 'OVERRIDDEN',
                        },
                    ],
                }
            ),
        ).json.output
        branch = details['branch']

        remote = f'https://{gitiles_host}/{details["project"]}'.strip('/')
        rebase = not ctx.options.force_no_rebase

        current_revision = details['revisions'][details['current_revision']]
        is_merge = len(current_revision['commit']['parents']) > 1
        if is_merge:
            rebase = False

        if not patchset:
            patchset = current_revision['_number']

        ref = f'refs/changes/{change_id % 100:02}/{change_id}/{patchset}'

        submit_requirements: dict[str, str] = {}
        for req in details.get('submit_requirements', ()):
            submit_requirements[req['name']] = req['status']

        uploader_info: dict[str, Any] = current_revision.get('real_uploader')
        if not uploader_info:
            uploader_info = current_revision.get('uploader')
            if not uploader_info and self.m.recipe_testing.enabled:
                uploader_info = {  # pragma: no cover
                    'email': 'noreply@google.com',
                }

        uploader: str | None = None
        if uploader_info:
            uploader = uploader_info['email']

        return Change(
            number=int(change_id),
            patchset=patchset,
            remote=remote,
            ref=ref,
            rebase=rebase,
            is_merge=is_merge,
            branch=branch,
            gerrit_name=gerrit_name,
            submitted=False,
            commit_message=current_revision['commit']['message'],
            project=details['project'],
            topic=details.get('topic', None) or None,
            current_revision=details['current_revision'],
            primary=primary,
            submit_requirements=submit_requirements,
            uploader=uploader,
        )

    def _process_gerrit_changes(
        self,
        ctx: CheckoutContext,
        bb_input: build_pb.Build.Input,
    ) -> Generator[Change, None, None]:
        seen = set()
        for i, change in enumerate(bb_input.gerrit_changes):
            with self.m.step.nest(str(i)):
                result = self._process_gerrit_change(
                    ctx=ctx,
                    host=change.host,
                    project=change.project,
                    change_id=change.change,
                    patchset=change.patchset,
                    primary=True,
                )
                yield result
                seen.add(result.name)

        cq_deps_result = self.m.cq_deps.resolve(
            result.gerrit_name,
            result.number,
            result.topic,
        )
        for dep in cq_deps_result.resolved:
            # dep.name should only appear in seen if there are multiple
            # gerrit_changes from buildbucket and a later one depends on an
            # earlier one. If buildbucket has multiple gerrit_changes the
            # cq_deps module is not needed here, so this is just double-checking
            # something that shouldn't happen.
            if dep.name in seen:  # pragma: no cover
                continue
            seen.add(dep.name)
            yield self._process_gerrit_change(
                ctx=ctx,
                host=dep.host,
                project=dep.project,
                change_id=dep.change,
                primary=False,
            )

        for cl in cq_deps_result.unresolved:
            yield Change(
                number=cl.change,
                remote=None,
                ref=None,
                rebase=None,
                project=None,
                branch=None,
                gerrit_name=cl.gerrit_name,
                submitted=False,
            )

    def number_for_hash(
        self,
        host: str,
        commit_hash: str,
        branch: str = 'main',
    ) -> dict[str, Any]:
        if 'github.com' in host or 'github-review' in host:
            return None  # pragma: no cover

        try:
            results = self.m.gerrit.change_query(
                'number',
                f'commit:{commit_hash}',
                host=host,
                max_attempts=5,
                timeout=30,
                test_data=self.m.json.test_api.output(
                    [
                        {
                            '_number': '1234',
                            'branch': branch,
                            'project': 'pigweed',
                        }
                    ]
                ),
            ).json.output
            # Skip this change if it didn't go through Gerrit.
            if results and len(results) == 1:
                return results[0]
        except self.m.step.StepFailure:  # pragma: no cover
            pass

        return None

    def _change_data(
        self,
        ctx: CheckoutContext,
        remote: str = None,
        branch: str = None,
        fallback_to_head: bool = True,
    ) -> tuple[Change, ...]:
        bb_input: build_pb.Build.Input = self.m.buildbucket.build.input
        results: list[Change] = []

        triggers: dict[str, triggers_pb.Trigger] = {}
        for trigger in self.m.scheduler.triggers:
            if trigger.gitiles:
                triggers[trigger.gitiles.revision] = trigger

        with self.m.step.nest('change data'):
            if bb_input.gerrit_changes:
                with self.m.step.nest('process gerrit changes'):
                    results.extend(self._process_gerrit_changes(ctx, bb_input))

            elif bb_input.gitiles_commit.id:
                with self.m.step.nest('process gitiles commit'):
                    commit = bb_input.gitiles_commit
                    assert commit.host
                    if commit.project:
                        remote = f'https://{commit.host}/{commit.project}'

                    host = commit.host.replace(
                        '.googlesource.com', '-review.googlesource.com'
                    )
                    gerrit_name = commit.host.split('.')[0]

                    result = self.number_for_hash(host, commit.id)

                    number: str | None = None
                    if result:
                        number = result['_number']

                    results.append(
                        Change(
                            number=number,
                            remote=remote,
                            ref=commit.id,
                            rebase=False,
                            branch=commit.ref.removeprefix('refs/heads/'),
                            gerrit_name=gerrit_name,
                            submitted=True,
                            project=commit.project,
                            current_revision=commit.id,
                            primary=True,
                        )
                    )

            if not results and fallback_to_head:
                # If not triggered by a gitiles_poller gitiles_commit may be
                # empty. In that case treat the most recent commit on the
                # remote as the triggering commit. This is a good assumption
                # except for Android Repo Tool projects, unless all projects
                # are pinned to commits instead of tracking branches. However,
                # even if this is wrong it's close enough to have utility.
                head: str = self.m.git.get_remote_branch_head(remote, branch)
                gerrit_name: str = urllib.parse.urlparse(remote).netloc.split(
                    '.'
                )[0]
                host: str = f'{gerrit_name}-review.googlesource.com'
                result: dict[str, Any] = self.number_for_hash(host, head)

                results.append(
                    Change(
                        number=result['_number'] if result else 0,
                        remote=remote,
                        ref=head,
                        rebase=False,
                        branch=result['branch'] if result else branch,
                        gerrit_name=gerrit_name,
                        project=None,
                        submitted=True,
                    )
                )

            with self.m.step.nest('changes'):
                for result in results:
                    with self.m.step.nest(result.name) as change_data_pres:
                        change_data_pres.step_summary_text = repr(result)

            return tuple(results)

    def _matching_branches(
        self,
        repo: str,
        branches: Sequence[str],
        name: str = 'has branch',
        **kwargs,
    ):
        """Returns the subset of the given branches that exist on gitiles."""
        matches: set[str] = set()
        with self.m.step.nest(name), self.m.context(infra_steps=True):
            for branch in branches:
                head: str = self.m.git.get_remote_branch_head(
                    repo,
                    branch,
                    step_name=f'git ls-remote {branch}',
                    step_test_data=lambda: self.m.raw_io.test_api.stream_output_text(
                        ''
                    ),
                    **kwargs,
                )
                with self.m.step.nest('head') as pres:
                    pres.step_summary_text = repr(head)

                if head:
                    matches.add(branch)
        return sorted(matches)

    def _apply_change(
        self,
        ctx: CheckoutContext,
        change: Change,
        cwd: config_types.Path = None,
    ):
        """Applies the given change to the given directory.

        Args:
            ctx: Checkout context object.
            change: Change to apply.
            cwd: Working directory, defaults to current directory.
        """
        with self._apply_change_context(ctx=ctx, change=change, cwd=cwd):
            pass

    @contextlib.contextmanager
    def _apply_change_context(
        self,
        ctx: CheckoutContext,
        change: Change,
        cwd: config_types.Path = None,
    ) -> Iterator[None]:
        """Applies the given change to the given directory.

        Args:
            ctx: Checkout context object.
            change: Change to apply.
            cwd: Working directory, defaults to current directory.
        """
        kwargs: dict[str, Any] = {'cwd': cwd} if cwd else {}
        change.applied = True
        change.path = self.m.path.relpath(cwd or ctx.top, ctx.top)

        try:
            apply_step: str = f'apply {change.name}'
            with self.m.context(**kwargs), self.m.step.nest(apply_step) as pres:
                pres.links['gerrit'] = change.gerrit_url
                pres.links['gitiles'] = change.gitiles_url
                if cwd:
                    pres.step_summary_text = str(
                        self.m.path.relpath(cwd, ctx.top)
                    )

                with self.m.context(infra_steps=True):
                    # 'git fetch' fails if a submodule pin in the patch isn't
                    # present in the remote (for example, if the pin is only
                    # present in the uploader's workspace). Use
                    # '--no-recurse-submodules' here so 'git fetch' doesn't fail
                    # but instead 'git rebase' or 'git submodule update' fails
                    # later (important because those are not infra steps). Also
                    # don't use '--recurse-submodules' in 'git checkout' for
                    # similar reasons.
                    with self.m.default_timeout():
                        self.m.utils.retry(
                            lambda: self.m.git.fetch(
                                change.remote,
                                change.ref,
                                recurse_submodules=False,
                                step_name='git fetch patch',
                            ),
                            max_attempts=ctx.options.max_attempts,
                        )
                    self.m.git(
                        'git checkout patch',
                        'checkout',
                        '--force',
                        '-b',
                        'working',
                        'FETCH_HEAD',
                    )

                # These remain unused if change.submitted is False.
                remote: str | None = None
                remote_branch: str | None = None

                with self.m.context(infra_steps=True):
                    # Change "https://foo.googlesource.com/bar"
                    #     to "https___foo_googlesource_com_bar".
                    # In Android Repo Tool projects, the remote for the manifest
                    # is often configured in a way that seems incorrect. Instead
                    # of relying on it, create a whole new remote every time
                    # that is always correct.
                    remote = re.sub(r'[^\w]', '_', change.remote)
                    remote_branch = '/'.join((remote, change.branch))
                    self.m.git(
                        'git remote add',
                        'remote',
                        'add',
                        remote,
                        change.remote,
                    )

                    with self.m.default_timeout():
                        self.m.utils.retry(
                            lambda: self.m.git.fetch(
                                remote,
                                f'refs/heads/{change.branch}',
                                prune=False,
                                step_name='git fetch branch',
                            ),
                            max_attempts=ctx.options.max_attempts,
                        )

                    self.m.git(
                        'git set upstream',
                        'branch',
                        f'--set-upstream-to={remote_branch}',
                    )

                if not change.submitted:
                    with self.m.context(infra_steps=True):
                        self.m.git(
                            'pre-rebase log', 'log', '--oneline', '-n', '10'
                        )

                if change.submitted:
                    change.base = self.m.git.rev_parse(
                        'HEAD',
                        step_test_data=lambda: self.m.raw_io.test_api.stream_output_text(
                            'HEAD_' * 8,
                        ),
                    )
                    change.base_type = 'submitted_commit_hash'

                elif change.rebase:
                    self.m.git('git rebase', 'rebase', remote_branch)

                    change.base = self.m.git.rev_parse(
                        remote_branch,
                        step_test_data=lambda: self.m.raw_io.test_api.stream_output_text(
                            'REMOTE_BRANCH_' * 3,
                        ),
                    )
                    change.base_type = 'remote_branch_tip'

                else:
                    change.base = self.m.git(
                        'merge-base',
                        'merge-base',
                        'HEAD',
                        remote_branch,
                        stdout=self.m.raw_io.output_text(),
                        step_test_data=lambda: self.m.raw_io.test_api.stream_output_text(
                            'MERGEBASE_' * 4,
                        ),
                    ).stdout
                    change.base_type = 'merge-base'

                with self.m.context(infra_steps=True):
                    self.m.git(
                        'post-rebase log', 'log', '--oneline', '-n', '10'
                    )

                # In most cases this is redundant or unnecessary, but it
                # shouldn't cause problems. It's necessary when a superproject
                # CL is updating a submodule pin and we need to sync the
                # submodule to the new revision.
                with self.m.default_timeout():
                    # See b/243673776 for why we detach before updating
                    # submodules.
                    self.m.git('detach', 'checkout', '--detach')
                    self.m.git.submodule_sync(recursive=True)
                    self.m.git.submodule_update(
                        recursive=True,
                        timeout=ctx.options.submodule_timeout_sec,
                    )
                    self.m.git('reattach', 'checkout', '-')

                yield

        finally:
            pass

    def _check_unapplied_changes(self, changes: Sequence[Change]):
        applied: list[Change] = []
        failed_to_apply: list[Change] = []
        if not changes:  # pragma: no cover
            return None

        def handle_unapplied_change(change):
            with self.m.step.nest(f'failed to apply {change.name}') as pres:
                pres.status = 'WARNING'
                pres.links['gerrit'] = change.gerrit_url
                pres.links['gitiles'] = change.gitiles_url
            failed_to_apply.append(change)

        with self.m.context(infra_steps=True):
            if all(not x.applied for x in changes):
                with self.m.step.nest('no changes were applied') as pres:
                    pres.status = 'FAILURE'
                    for change in changes:
                        handle_unapplied_change(change)
                    pres.properties['changes'] = [x.name for x in changes]

                raise self.m.step.InfraFailure(
                    'could not find triggering changes in checkout'
                )

            elif any(not x.applied for x in changes):
                with self.m.step.nest('some changes were not applied') as pres:
                    pres.status = 'WARNING'
                    for change in changes:
                        if change.applied:
                            applied.append(change)
                        else:
                            handle_unapplied_change(change)

            else:
                applied.extend(changes)

        with self.m.step.nest('status') as pres:
            pres.step_summary_text = (
                f'applied {applied}\nnot applied {failed_to_apply}'
            )

        return StatusOfChanges(
            applied=tuple(applied),
            not_applied=tuple(failed_to_apply),
        )

    def _cached_checkout(
        self,
        remote: str,
        path: config_types.Path,
        ref: str,
        submodules: bool,
        included_submodules: Sequence[str] | None = None,
        excluded_submodules: Sequence[str] | None = None,
        submodule_timeout_sec: int = 10 * 60,
        cache: bool = True,
        use_packfiles: bool = False,
        max_attempts: int = 1,
        **kwargs,
    ):
        submodule_paths = included_submodules = included_submodules or []

        if cache:
            with self.m.step.nest('cache') as pres, self.m.cache.guard('git'):
                parsed_remote = urllib.parse.urlparse(remote)
                cache_name = (
                    parsed_remote.hostname
                    + parsed_remote.path.replace('-', '--').replace('/', '-')
                )
                cache_path = self.m.path.cache_dir / 'git' / cache_name
                self.m.file.ensure_directory('makedirs', cache_path)

                with self.m.context(cwd=cache_path):
                    dotgit = cache_path / '.git'
                    if self.m.path.exists(dotgit):  # pragma: no cover
                        hit_pres = self.m.step.empty('hit').presentation
                        hit_pres.properties[f'checkout {remote}'] = 'cache hit'
                        self.m.git.config_remove_section(
                            'remote.origin', **kwargs
                        )
                        pres.step_summary_text = 'hit'
                    else:
                        miss_pres = self.m.step.empty('miss').presentation
                        miss_pres.properties[f'checkout {remote}'] = (
                            'cache miss'
                        )
                        self.m.git.init(bare=False, **kwargs)
                        pres.step_summary_text = 'miss'

                    self.m.git.config(
                        'remote.origin.url',
                        remote,
                        step_name='remote set-url',
                        **kwargs,
                    )

                    if use_packfiles:
                        self.m.git.config(
                            'fetch.uriprotocols',
                            'https',
                            step_name='enable packfiles',
                            **kwargs,
                        )
                    else:
                        self.m.git.config(
                            'fetch.uriprotocols',
                            '',
                            step_name='disable packfiles',
                            **kwargs,
                        )

                    with self.m.default_timeout():
                        try:
                            self.m.utils.retry(
                                lambda: self.m.git.fetch(
                                    repository='origin',
                                    prune=True,
                                    tags=True,
                                    recurse_submodules=submodules,
                                    **kwargs,
                                ),
                                max_attempts=max_attempts,
                            )

                        # If the checkout failed save the git config. It might
                        # not be helpful, but it shouldn't hurt.
                        except self.m.step.StepFailure:
                            with self.m.step.nest('git config'):
                                self.m.git.config(
                                    '--list', '--local', step_name='local'
                                )
                                self.m.git.config(
                                    '--list', '--global', step_name='global'
                                )
                            raise

                    self.m.git.merge(ref='FETCH_HEAD', **kwargs)

                    if included_submodules and excluded_submodules:
                        raise self.m.step.InfraFailure(
                            'cannot specify both included_submodules and '
                            'excluded_submodules'
                        )

                    submodule_paths = included_submodules
                    if excluded_submodules:
                        submodule_status = self.m.git(
                            'submodule status',
                            'submodule',
                            'status',
                            stdout=self.m.raw_io.output_text(),
                            step_test_data=lambda: self.test_api.m.raw_io.stream_output_text(
                                '-0000000000000000000000000000000000000000 pigweed (abc123)\n'
                                '-1111111111111111111111111111111111111111 nanopb (heads/branch)\n'
                            ),
                        ).stdout.splitlines()

                        submodule_paths = [
                            x.split()[1] for x in submodule_status
                        ]
                        for sub in excluded_submodules:
                            if sub not in submodule_paths:
                                raise self.m.step.InfraFailure(
                                    f'excluded submodule {sub} is not a submodule'
                                )
                            with self.m.step.nest(f'excluding submodule {sub}'):
                                pass
                            submodule_paths.remove(sub)

                    for sub in submodule_paths:
                        with self.m.step.nest(f'including submodule {sub}'):
                            pass

                    self.m.git.submodule_sync(recursive=True, **kwargs)
                    if submodules or submodule_paths:
                        with self.m.default_timeout():
                            self.m.git.submodule_update(
                                recursive=True,
                                force=True,
                                paths=submodule_paths,
                                timeout=submodule_timeout_sec,
                                **kwargs,
                            )

                    if not submodules:
                        # Even though submodules weren't requested, if the cache
                        # had any active submodules we need to update them.
                        # Otherwise we'll get weird situations in rolls where an
                        # uninvolved submodule will be rolled back.
                        with self.m.default_timeout():
                            self.m.git.submodule_update(
                                recursive=True,
                                force=True,
                                init=False,
                                timeout=submodule_timeout_sec,
                                **kwargs,
                            )

            self.m.file.copytree(
                'copy from cache', cache_path, path, symlinks=True
            )

        # Deliberately not combining contexts into one line so it's obvious to
        # both devs and Python which one is "outer" and which is "inner".
        with self.m.step.nest('git checkout'):
            with self.m.default_timeout():
                self.m.git_checkout(
                    repo=remote,
                    path=path,
                    cache=False,
                    revision=ref,
                    recursive=submodules,
                    submodules=submodules,
                    submodule_force=submodules,
                    submodule_paths=submodule_paths,
                    submodule_timeout=submodule_timeout_sec,
                    step_name='',
                    use_packfiles=use_packfiles,
                    max_attempts=max_attempts,
                )

    def _git(self, ctx: CheckoutContext):
        """Checkout code from git."""

        super_branch = self._matching_branch(ctx) or ctx.options.branch

        with self.m.context(infra_steps=True):
            cache = not ctx.options.do_not_cache
            build = self.m.buildbucket.build
            if 'pigweed.disable_git_cache' in build.input.experiments:
                cache = False

            self._cached_checkout(
                ctx.options.remote,
                path=ctx.top,
                ref=super_branch,
                cache=cache,
                submodules=ctx.options.initialize_submodules,
                submodule_timeout_sec=ctx.options.submodule_timeout_sec,
                included_submodules=ctx.options.included_submodules,
                excluded_submodules=ctx.options.excluded_submodules,
                use_packfiles=not ctx.options.do_not_use_packfiles,
                max_attempts=ctx.options.max_attempts,
            )

        with self.m.context(cwd=ctx.top):
            got_revision = None
            got_revision_type = 'no_trigger'

            submodules = []

            if ctx.options.use_trigger:
                got_revision = self.m.git.rev_parse(
                    'HEAD',
                    step_test_data=lambda: self.m.raw_io.test_api.stream_output_text(
                        'HEAD' * 10,
                    ),
                )

                # Check for CLs for the top-level repository.
                for change in ctx.changes:
                    if ctx.remotes_equivalent(
                        ctx.options.remote, change.remote
                    ):
                        self._apply_change(ctx, change)
                        got_revision = change.base
                        got_revision_type = change.base_type

                submodules = ctx.submodules(recursive=True)

                # Check for CLs for submodules.
                # There are three rough cases:
                # 1. Zero submodules have matching remotes. In this case we do
                #    nothing.
                # 2. Exactly one submodule has a matching remote. In this case,
                #    the change is applied to that submodule, even if the branch
                #    does not match.
                # 3. Multiple submodules have matching remotes.
                #    1. Exactly one of them matches the triggering change's
                #       branch. In this case, the change is applied to this
                #       submodule.
                #    2. Zero or multiple submodules match both the remote and
                #       the branch of the triggering change. In these cases we
                #       error out.
                for change in ctx.changes:
                    with self.m.step.nest(f'matching {change.name}') as pres:
                        pres.links['gerrit'] = change.gerrit_url
                        pres.links['gitiles'] = change.gitiles_url

                        matching_submodules = []
                        for submodule in submodules:
                            if submodule.initialized:
                                if ctx.remotes_equivalent(
                                    submodule.remote, change.remote
                                ):
                                    self.m.step.empty(
                                        f'match: {submodule.path} '
                                        f'({submodule.remote})'
                                    )
                                    matching_submodules.append(submodule)
                                else:
                                    self.m.step.empty(
                                        f'no match: {submodule.path} '
                                        f'({submodule.remote})'
                                    )

                        if not matching_submodules:
                            pres.step_summary_text = 'no matching submodules'
                            continue

                        if len(matching_submodules) > 1:
                            submodule_info = ', '.join(
                                f'{self.m.path.relpath(sub.path, ctx.top)} '
                                f'(branch {sub.branch})'
                                for sub in matching_submodules
                            )

                            matching_with_branch = []
                            for submodule in matching_submodules:
                                if submodule.branch == change.branch:
                                    matching_with_branch.append(submodule)

                            if len(matching_with_branch) == 1:
                                pres.step_summary_text = (
                                    'one matching submodule'
                                )
                                matching_submodules = matching_with_branch

                            elif len(matching_with_branch) > 1:
                                pres.step_summary_text = (
                                    'too many submodules match the branch'
                                )
                                raise self.m.step.StepFailure(
                                    f'change {change.name} (branch '
                                    f'{change.branch}) matches multiple '
                                    f'submodules ({submodule_info}), but too '
                                    'many branches match'
                                )

                            else:
                                pres.step_summary_text = (
                                    'zero submodules match the branch'
                                )
                                raise self.m.step.StepFailure(
                                    f'change {change.name} '
                                    f'(branch {change.branch}) matches '
                                    f'multiple submodules ({submodule_info}) '
                                    'but no branches match'
                                )

                    if len(matching_submodules) == 1:
                        submodule = matching_submodules[0]
                        if not ctx.options.initialize_submodules:
                            with self.m.default_timeout():
                                self.m.git.submodule_sync(recursive=True)
                                self.m.git.submodule_update(
                                    paths=(submodule.path,)
                                )
                        self._apply_change(ctx, change, cwd=submodule.path)

                def _vars_primitive_only(x):
                    return {
                        k: v
                        for k, v in to_dict(x).items()
                        if isinstance(v, (int, str, bool, type(None)))
                    }

                applied_changes = [
                    _vars_primitive_only(x) for x in ctx.changes if x.applied
                ]
                ctx.changes_json = self.m.path.mkstemp()
                self.m.file.write_json(
                    'write changes.json',
                    ctx.changes_json,
                    applied_changes,
                )

            # Run git log for both the top-level checkout and every submodule.
            with self.m.step.nest('git log'):
                self.m.git(str(ctx.top), 'log', '--oneline', '-n', '10')
                for submodule in sorted(submodules):
                    with self.m.context(cwd=submodule.path):
                        self.m.git(
                            str(submodule.path),
                            'log',
                            '--oneline',
                            '-n',
                            '10',
                        )

            if got_revision:
                with self.m.step.nest('base') as pres:
                    pres.properties['got_revision'] = got_revision
                    # got_revision_type isn't needed by anything but helps
                    # explain why got_revision is the value it is.
                    pres.properties['got_revision_type'] = got_revision_type

    def _matching_branch(self, ctx: CheckoutContext):
        """Return if there are manifest branches that match the triggering CLs.

        If the triggering change is on a branch name that is also present in the
        manifest or superproject remote, use that branch when checking out the
        project.

        Args:
            ctx (CheckoutContext): Context object.

        Raises:
            StepFailure if there are multiple matching branches.

        Returns:
            One matching branch name, or None.
        """
        if not ctx.options.match_branch or not ctx.options.use_trigger:
            with self.m.step.nest('not matching branch names'):
                return

        kind = 'manifest' if ctx.options.use_repo else 'superproject'

        manifest_branch = None
        branch_names = sorted(
            set(
                x.branch
                for x in ctx.changes
                if x.branch not in ('master', 'main', None)
            )
        )

        if not branch_names:
            with self.m.step.nest('no non-standard branch names'):
                return

        with self.m.step.nest('branch names') as pres:
            pres.step_summary_text = str(branch_names)

        matching_branches = self._matching_branches(
            ctx.options.remote, branch_names, name=f'{kind} has branch'
        )
        if not matching_branches:
            with self.m.step.nest('no branch names match'):
                return

        if len(matching_branches) > 1:
            with self.m.step.nest(
                f'too many matching branches ({", ".join(matching_branches)})'
            ) as pres:
                pres.step_summary_text = (
                    "Can't figure out which {} branch to use. Remove some "
                    '"Requires:" lines to simplify the checkout.'.format(kind)
                )
                raise self.m.step.StepFailure('multiple matching branches')

        manifest_branch = matching_branches.pop()
        self.m.step(
            f'changing {kind} branch to {manifest_branch}',
            None,
        )
        return manifest_branch

    def _repo(self, ctx: CheckoutContext):
        """Checkout code from an Android Repo Tool manifest."""

        # Git makes the top-level folder, Repo requires caller to make it.
        self.m.file.ensure_directory('mkdir checkout', ctx.top)

        with self.m.context(cwd=ctx.top):
            manifest_branch = self._matching_branch(ctx) or ctx.options.branch

            with self.m.context(infra_steps=True):
                kwargs = {}
                if ctx.options.repo_init_timeout_sec:
                    kwargs['timeout'] = ctx.options.repo_init_timeout_sec
                    kwargs['attempts'] = ctx.options.number_of_attempts
                if ctx.options.manifest_groups:
                    kwargs['groups'] = ctx.options.manifest_groups

                self.m.repo.init(
                    manifest_url=ctx.options.remote,
                    manifest_branch=manifest_branch,
                    manifest_name=ctx.options.manifest_file,
                    clone_bundle=not ctx.options.repo_no_clone_bundle,
                    **kwargs,
                )

            manifests_dir = ctx.top / '.repo' / 'manifests'
            # If the triggering CL is a manifest change, apply it before running
            # sync.
            if ctx.options.use_trigger:
                for change in ctx.changes:
                    if change.remote and ctx.remotes_equivalent(
                        ctx.options.remote, change.remote
                    ):
                        with self._apply_change_context(
                            ctx, change, cwd=manifests_dir
                        ):
                            # Right now the upstream of 'working' is the local
                            # 'default' branch. 'repo sync' complains if the
                            # upstream isn't remote, so it's changed to the
                            # remote branch that's identical to 'default'.
                            self.m.git(
                                'git branch',
                                'branch',
                                f'--set-upstream-to=origin/{manifest_branch}',
                            )

            ctx.manifest = self._read_manifest(
                ctx.options.remote,
                manifests_dir / ctx.options.manifest_file,
            )

            for _, remote_host in sorted(ctx.manifest.remotes.items()):
                if remote_host.fetch.url.startswith('sso://'):
                    self.m.sso.configure_insteadof(remote_host.fetch.url)

            with self.m.context(infra_steps=True):
                kwargs = {}
                if ctx.options.repo_sync_timeout_sec:
                    kwargs['timeout'] = ctx.options.repo_sync_timeout_sec
                    kwargs['attempts'] = ctx.options.number_of_attempts
                self.m.repo.sync(
                    force_sync=True,
                    current_branch=True,
                    jobs=self.m.platform.cpu_count,
                    **kwargs,
                )
                self.m.repo.start('base')

            if ctx.options.use_trigger:
                for change in ctx.changes:
                    for entry in ctx.manifest.projects:
                        if ctx.remotes_equivalent(entry.url, change.remote):
                            with self._apply_change_context(
                                ctx,
                                change,
                                cwd=entry.path_object(ctx.top),
                            ):
                                with self.m.step.nest(
                                    'compare branch name'
                                ) as pres:
                                    pres.step_summary_text = (
                                        'CL branch: {}\nupstream branch: {}'
                                    ).format(change.branch, entry.upstream)

    def _workspace(self, ctx: CheckoutContext):
        if not ctx.options.eligible_workspace_paths or all(
            x.applied for x in ctx.changes
        ):
            return

        repos_dir = self.m.path.start_dir / 'bazel_repos'
        self.m.file.ensure_directory(f'mkdir {repos_dir}', repos_dir)

        with self.m.step.nest('workspace'):
            for change in ctx.changes:
                if change.applied:
                    continue  # pragma: no cover

                with self.m.step.nest(change.name) as pres:
                    pres.links['gerrit'] = change.gerrit_url
                    pres.links['gitiles'] = change.gitiles_url

                    for workspace in ctx.options.eligible_workspace_paths:
                        workspace_path = ctx.root / workspace
                        self.m.path.mock_add_file(workspace_path)
                        if not self.m.path.isfile(workspace_path):
                            continue  # pragma: no cover

                        with self.m.step.nest(workspace) as ws_pres:
                            ws_pres.step_summary_text = str(workspace_path)

                            repos = self.m.bazel_roll.retrieve_attributes(
                                project_value=change.remote,
                                path=workspace_path,
                                matches=ctx.remotes_equivalent,
                                search_key='remote',
                            )

                            if not repos:
                                continue

                            name = change.remote
                            name = name.replace('http://', '')
                            name = name.replace('https://', '')
                            name = name.replace('sso://', '')
                            name = name.replace('.git.corp.google.com', '')
                            name = name.replace('.googlesource.com', '')
                            name = name.replace('/', '_')
                            path = repos_dir / name

                            self.m.git_checkout(
                                change.remote,
                                path=path,
                                step_name=f'checkout {name}',
                                ignore_build_input=True,
                                cache=False,
                                max_attempts=ctx.options.max_attempts,
                            )

                            self._apply_change(ctx, change, path)

                            for repo in repos:
                                if 'strip_prefix' in repo:
                                    path = path / repo['strip_prefix']
                                ctx.bazel_overrides[repo['name']] = path

    def _check_gerrit_changes(self, ctx: CheckoutContext):
        """Check that we don't have multiple changes for the same repo."""
        seen = {}

        for change in ctx.changes:
            if change.submitted:
                continue

            for remote, prev_change in seen.items():
                if ctx.remotes_equivalent(change.remote, remote):
                    failure = (
                        f'Change {prev_change.name} and change {change.name} '
                        f'are from equivalent remotes ({prev_change.remote!r} '
                        f'and {change.remote!r}). This is not supported. '
                        'Instead, stack these changes, or only refer to the '
                        "top from a 'patches.json' file in another repository."
                    )
                    pres = self.m.step.empty(
                        'multiple changes for same repo',
                    ).presentation
                    pres.step_summary_text = failure
                    pres.status = 'FAILURE'
                    pres.links[prev_change.name] = prev_change.gerrit_url
                    pres.links[change.name] = change.gerrit_url
                    raise self.m.step.StepFailure(failure)

            seen[change.remote] = change

    def _configure_insteadof(self, ctx: CheckoutContext):
        """Configure git to use some urls in place of others."""
        if not ctx.options.rewrites:
            return

        with self.m.step.nest('insteadof'):
            for rewrite in ctx.options.rewrites:
                self.m.git(
                    f'{rewrite.original} to {rewrite.final}',
                    'config',
                    '--global',
                    '--add',
                    f'url.{rewrite.final}.insteadof',
                    rewrite.original,
                )

            self.m.git('rewrites', 'config', '--get-regexp', '^url.*')

    def _name(self, options: Options):
        """Turn "https://foo/bar/baz.git" into "baz"."""
        name = options.remote.rstrip('/').removesuffix('.git')
        parts = name.split('/')
        if options.use_repo and parts[-1] == 'manifest':
            parts.pop(-1)
        return f'checkout {parts[-1]}'

    class EquivalentRemotes(TypedDict):
        remotes: list[str]

    def load_equivalent_remotes(
        self,
        equivalent_remotes_input: list[EquivalentRemotes],
    ):
        equivalent_remotes = {}
        for remotes in equivalent_remotes_input:
            new_remotes = [
                self.m.sso.sso_to_https(x) for x in remotes['remotes']
            ]
            for remote in new_remotes:
                assert remote not in equivalent_remotes
                equivalent_remotes[remote] = new_remotes
        return equivalent_remotes

    def remotes_equivalent(
        self,
        equivalent_remotes: dict[str, list[str]],
        remote1: str | None,
        remote2: str | None,
    ) -> bool:
        # Sometimes remote1 or remote2 is None. In that case we shouldn't
        # convert sso to https.
        if remote1:
            remote1 = self.m.sso.sso_to_https(remote1).removesuffix('.git')
        if remote2:
            remote2 = self.m.sso.sso_to_https(remote2).removesuffix('.git')
        if remote1 == remote2:
            return True

        # TODO: b/400532999 - Remove hard-coded equivalency support.
        hardcoded_equivalent_remotes = (
            (
                'https://fuchsia.googlesource.com/third_party/pigweed.googlesource.com/pigweed/pigweed',
                'https://pigweed.googlesource.com/pigweed/pigweed',
            ),
        )

        def extend_unique(dest, to_add):
            for item in to_add:
                if item not in dest:
                    dest.append(item)

        # TODO: b/400532999 - Remove hard-coded equivalency support.
        for hardcoded_equivalent_group in hardcoded_equivalent_remotes:
            final_group = list(hardcoded_equivalent_group)
            for remote in hardcoded_equivalent_group:
                if remote in equivalent_remotes:
                    extend_unique(final_group, equivalent_remotes[remote])
            for remote in final_group:
                equivalent_remotes.setdefault(remote, [])
                extend_unique(equivalent_remotes[remote], final_group)

        return remote1 in equivalent_remotes.get(remote2, ())

    def triggering_changes(self, options: Options) -> tuple[Change]:
        ctx = CheckoutContext(self.m)
        ctx.options = options
        ctx.changes = []

        return self._change_data(
            ctx,
            remote=options.remote,
            branch=options.branch,
            fallback_to_head=False,
        )

    def __call__(
        self,
        options: Options,
        root: config_types.Path | None = None,
        name: str = None,
    ) -> CheckoutContext:
        """Checkout code."""

        checkout_name = name or self._name(options)

        assert options.remote

        initial_options = repr(options)
        options.manifest_file = options.manifest_file or 'default.xml'
        options.max_attempts = options.max_attempts or 2
        options.repo_init_timeout_sec = options.repo_init_timeout_sec or 2 * 60
        options.repo_sync_timeout_sec = options.repo_sync_timeout_sec or 15 * 60
        options.number_of_attempts = options.number_of_attempts or 3
        options.submodule_timeout_sec = options.submodule_timeout_sec or 10 * 60
        final_options = repr(options)

        ctx = CheckoutContext(self.m)
        ctx.options = options
        ctx.changes = []

        root = root or self.m.path.start_dir / 'co'
        ctx.top = root
        if ctx.options.root_subdirectory:
            ctx.root = ctx.top / ctx.options.root_subdirectory
        else:
            ctx.root = ctx.top

        ctx.equivalent_remotes = self.load_equivalent_remotes(
            json_format.MessageToDict(options).get('equivalentRemotes', []),
        )

        with self.m.step.nest(checkout_name) as pres:
            with self.m.step.nest('options') as options_pres:
                options_pres.step_summary_text = initial_options

            with self.m.step.nest('options with defaults') as options_pres:
                options_pres.step_summary_text = final_options

            if options.remote.endswith('.git'):
                options.remote = options.remote[:-4]

            if options.use_trigger:
                ctx.changes = self._change_data(
                    ctx, options.remote, options.branch
                )

            self._check_gerrit_changes(ctx)

            self._configure_insteadof(ctx)

            if options.use_repo:
                self._repo(ctx)

            else:
                self._git(ctx)

            self._workspace(ctx)

            if not options.use_repo:
                with self.m.context(cwd=ctx.top):
                    self.m.git.clean(force=2, recursive=True)
                    try:
                        self.m.git.status()
                    except self.m.step.StepFailure:  # pragma: no cover
                        pass

            ctx.status = self._check_unapplied_changes(ctx.changes)

            if ctx.status:
                for change in ctx.status.applied:
                    pres.links[f'applied {change.name_with_path}'] = (
                        change.gerrit_url
                    )

                for change in ctx.status.not_applied:
                    pres.links[f'failed to apply {change.name}'] = (
                        change.gerrit_url
                    )

            snapshot_dir = self.m.path.start_dir / 'snapshot'
            ctx.snapshot_to_dir(snapshot_dir)

        return ctx

    def get_revision(
        self,
        root: config_types.Path,
        name: str = 'git log',
        test_data: str = 'HASH',
    ):
        """Like self.revision, but works for secondary checkouts."""
        with self.m.context(cwd=root):
            step = self.m.git(
                name,
                'log',
                '--max-count=1',
                '--pretty=format:%H',
                stdout=self.m.raw_io.output_text(),
                step_test_data=lambda: self.test_api.m.raw_io.stream_output_text(
                    test_data,
                ),
            )

            # Sometimes test data has additional information, following a
            # newline. Only keep the part before the first newline.
            result = step.stdout.strip().split('\n')[0]
            step.presentation.step_summary_text = result
            return result