recipes/docs_builder.py

# Copyright 2020 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""Generate docs using GN/Ninja and upload to GCS."""

from __future__ import annotations

from typing import TYPE_CHECKING

from PB.go.chromium.org.luci.buildbucket.proto import common
from PB.recipes.pigweed.docs_builder import InputProperties
from PB.recipe_engine import result
from recipe_engine import post_process

if TYPE_CHECKING:  # pragma: no cover
    from typing import Any, Generator, Sequence
    from recipe_engine import config_types, recipe_api, recipe_test_api

DEPS = [
    'fuchsia/buildbucket_util',
    'fuchsia/gerrit',
    'fuchsia/gsutil',
    'pigweed/bazel',
    'pigweed/build',
    'pigweed/checkout',
    'pigweed/environment',
    'pigweed/pw_presubmit',
    'recipe_engine/buildbucket',
    'recipe_engine/file',
    'recipe_engine/json',
    'recipe_engine/path',
    'recipe_engine/properties',
    'recipe_engine/step',
    'recipe_engine/swarming',
]

PROPERTIES = InputProperties


def RunSteps(
    api: recipe_api.RecipeScriptApi,
    props: InputProperties,
) -> result.RawResult:
    checkout: api.checkout.CheckoutContext = api.checkout(
        props.checkout_options
    )

    out_dir: config_types.Path
    if props.HasField('bazel_options'):
        default_html_path = 'k8-fastbuild/bin/docs/docs/_build/html'
        out_dir = checkout.root / 'bazel-out'

        runner: api.bazel.BazelRunner = api.bazel.new_runner(
            checkout,
            props.bazel_options,
            download_all_artifacts=True,
        )
        runner.run()

    else:
        default_html_path = 'docs/gen/docs/html'

        env: api.environment.Environment = api.environment.init(
            checkout, props.environment_options
        )

        if props.HasField('build_options'):
            build: api.build.BuildContext = api.build.create(
                checkout.root, props.build_options
            )
            with env():
                api.build(build)
            out_dir = build.root

        elif props.HasField('pw_presubmit_options'):
            with env():
                presubmit: api.pw_presubmit.PresubmitContext = (
                    api.pw_presubmit.init(checkout, props.pw_presubmit_options)
                )
                assert len(presubmit.steps) == 1
                for step in presubmit.steps:
                    api.pw_presubmit.run(ctx=presubmit, step=step, env=env)
                    out_dir = step.dir

        else:
            assert False  # pragma: no cover

    build_html = out_dir / (props.html_path or default_html_path)
    html = api.path.start_dir / 'html'
    api.file.listdir('ls build-html', build_html)
    api.file.copytree('copy', build_html, html)
    api.file.listdir('ls html', html)
    api.file.chmod('chmod', html, '777', recursive=True)

    api.file.write_text(
        'write buildbucket_id',
        html / 'buildbucket.id',
        f'{api.buildbucket.build.id}\n',
    )

    applied_changes: Sequence[api.checkout.Change] = checkout.applied_changes()

    paths: Sequence[str]
    if props.dry_run:
        paths = [
            f'testing/swarming-{api.swarming.task_id}/{checkout.revision()}',
        ]
    elif api.buildbucket_util.is_tryjob:
        paths = [str(api.buildbucket.build.id)]
        if len(applied_changes) == 1:
            paths.append(str(applied_changes[0].number))
    else:
        paths = [checkout.revision()]

    bucket: str = props.bucket or 'pigweed-docs'
    for path in paths:
        api.gsutil.rsync(
            bucket=bucket,
            src=html,
            dst=path,
            recursive=True,
            multithreaded=True,
            name=f'upload {path}',
        )

    # If in a tryjob there's no need to even fetch metadata.
    if not api.buildbucket_util.is_tryjob:
        # Getting gerrit details even if this is a dry run so testing will show
        # if it would have worked.
        change: dict[str, Any] = api.gerrit.change_details(
            'get change details',
            change_id=str(checkout.revision()),
            host=checkout.gerrit_host(),
            max_attempts=5,
            timeout=30,
            test_data=api.json.test_api.output({'branch': 'main'}),
        ).json.output

        if change['branch'] in ('master', 'main') and not props.dry_run:
            head: str = checkout.root / 'HEAD'
            api.file.write_text('write HEAD', head, checkout.revision())
            api.gsutil.upload(bucket=bucket, src=head, dst='HEAD')

            api.gsutil.rsync(
                bucket=bucket,
                src=html,
                dst='latest',
                recursive=True,
                multithreaded=True,
            )

    link_template: str = props.link_template or 'https://pigweed.dev/?rev={}'
    links: list[str] = []
    for path in paths:
        links.append(link_template.format(path))
    step = api.step.empty('set output properties')
    step.presentation.properties['links'] = links

    joined_links: str = ", ".join(links)

    if props.comment and api.buildbucket_util.is_tryjob:
        for change in applied_changes:
            api.gerrit.set_review(
                'post CL comment',
                change.number,
                host=change.gerrit_host,
                message=f'Docs available at: {joined_links}',
                notify='NONE',
                ignore_automatic_attention_set_rules=True,
            )

    return result.RawResult(
        summary_markdown=f'Docs available at {joined_links}.',
        status=common.SUCCESS,
    )


def GenTests(api) -> Generator[recipe_test_api.TestData, None, None]:
    def properties(**kwargs):
        props = InputProperties(**kwargs)
        props.checkout_options.CopyFrom(api.checkout.git_options())
        return api.properties(props)

    def ran(x):
        return api.post_process(post_process.MustRun, x)

    def did_not_run(x):
        return api.post_process(post_process.DoesNotRun, x)

    def url_is(url):
        return api.post_process(
            post_process.PropertyMatchesCallable,
            'links',
            lambda value: url in value,
        )

    def drop():
        return api.post_process(post_process.DropExpectation)

    yield api.test(
        'docs-dry_run-build',
        properties(
            dry_run=True,
            build_options=api.build.options(ninja_targets=['target']),
        ),
        ran('gn gen'),
        ran('ninja'),
        did_not_run('step'),
        did_not_run('docs.build //docs'),
        ran('upload testing/swarming-fake-task-id/HASH'),
        url_is('https://pigweed.dev/?rev=testing/swarming-fake-task-id/HASH'),
        drop(),
    )

    yield api.test(
        'docs-postsubmit-bazel',
        properties(
            dry_run=False,
            bazel_options=api.bazel.options(program='docs'),
        ),
        api.bazel.config(),
        ran('docs.build //docs'),
        did_not_run('step'),
        did_not_run('gn gen'),
        did_not_run('ninja'),
        ran('upload HASH'),
        ran('write HEAD'),
        ran('gsutil cp'),
        ran('gsutil rsync'),
        url_is('https://pigweed.dev/?rev=HASH'),
        drop(),
    )

    build_id = 881234567890
    change_id = 123456

    yield api.test(
        'docs-presubmit',
        api.checkout.try_test_data(build_id=build_id),
        properties(
            comment=True,
            dry_run=False,
            pw_presubmit_options=api.pw_presubmit.options(step=['step']),
            link_template='https://pending.docs/{}',
        ),
        ran('step'),
        did_not_run('gn gen'),
        did_not_run('ninja'),
        did_not_run('docs.build //docs'),
        ran(f'upload {build_id}'),
        ran(f'upload {change_id}'),
        ran('post CL comment'),
        url_is(f'https://pending.docs/{build_id}'),
        url_is(f'https://pending.docs/{change_id}'),
        drop(),
    )