| # Copyright 2020 The Pigweed Authors |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may not |
| # use this file except in compliance with the License. You may obtain a copy of |
| # the License at |
| # |
| # https://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations under |
| # the License. |
| """Generate docs using GN/Ninja and upload to GCS.""" |
| |
| from __future__ import annotations |
| |
| from typing import TYPE_CHECKING |
| |
| from PB.go.chromium.org.luci.buildbucket.proto import common |
| from PB.recipes.pigweed.docs_builder import InputProperties |
| from PB.recipe_engine import result |
| from recipe_engine import post_process |
| |
| if TYPE_CHECKING: # pragma: no cover |
| from typing import Any, Generator, Sequence |
| from recipe_engine import config_types, recipe_api, recipe_test_api |
| |
| DEPS = [ |
| 'fuchsia/buildbucket_util', |
| 'fuchsia/gerrit', |
| 'fuchsia/gsutil', |
| 'pigweed/build', |
| 'pigweed/checkout', |
| 'pigweed/environment', |
| 'pigweed/pw_presubmit', |
| 'recipe_engine/file', |
| 'recipe_engine/json', |
| 'recipe_engine/properties', |
| 'recipe_engine/step', |
| 'recipe_engine/swarming', |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| |
| def RunSteps( |
| api: recipe_api.RecipeScriptApi, |
| props: InputProperties, |
| ) -> result.RawResult: |
| checkout: api.checkout.CheckoutContext = api.checkout( |
| props.checkout_options |
| ) |
| env: api.environment.Environment = api.environment.init( |
| checkout, props.environment_options |
| ) |
| |
| out_dir: config_types.Path |
| if props.HasField('build_options'): |
| build: api.build.BuildContext = api.build.create( |
| checkout.root, props.build_options |
| ) |
| with env(): |
| api.build(build) |
| out_dir = build.root |
| elif props.HasField('pw_presubmit_options'): |
| with env(): |
| presubmit: api.pw_presubmit.PresubmitContext = ( |
| api.pw_presubmit.init(checkout, props.pw_presubmit_options) |
| ) |
| assert len(presubmit.steps) == 1 |
| for step in presubmit.steps: |
| api.pw_presubmit.run(ctx=presubmit, step=step, env=env) |
| out_dir = step.dir |
| else: |
| assert False # pragma: no cover |
| |
| html_path: str = props.html_path or 'docs/gen/docs/html' |
| html: config_types.Path = out_dir.joinpath(*html_path.split('/')) |
| api.file.write_text( |
| 'write buildbucket_id', |
| html / 'buildbucket.id', |
| f'{api.buildbucket_util.id}\n', |
| ) |
| |
| applied_changes: Sequence[api.checkout.Change] = checkout.applied_changes() |
| |
| paths: Sequence[str] |
| if props.dry_run: |
| paths = [ |
| f'testing/swarming-{api.swarming.task_id}/{checkout.revision()}', |
| ] |
| elif api.buildbucket_util.is_tryjob: |
| paths = [api.buildbucket_util.id] |
| if len(applied_changes) == 1: |
| paths.append(str(applied_changes[0].number)) |
| else: |
| paths = [checkout.revision()] |
| |
| bucket: str = props.bucket or 'pigweed-docs' |
| for path in paths: |
| api.gsutil.rsync( |
| bucket=bucket, |
| src=html, |
| dst=path, |
| recursive=True, |
| multithreaded=True, |
| name=f'upload {path}', |
| ) |
| |
| # If in a tryjob there's no need to even fetch metadata. |
| if not api.buildbucket_util.is_tryjob: |
| # Getting gerrit details even if this is a dry run so testing will show |
| # if it would have worked. |
| change: dict[str, Any] = api.gerrit.change_details( |
| 'get change details', |
| change_id=str(checkout.revision()), |
| host=checkout.gerrit_host(), |
| max_attempts=5, |
| timeout=30, |
| test_data=api.json.test_api.output({'branch': 'main'}), |
| ).json.output |
| |
| if change['branch'] in ('master', 'main') and not props.dry_run: |
| head: str = checkout.root / 'HEAD' |
| api.file.write_text('write HEAD', head, checkout.revision()) |
| api.gsutil.upload(bucket=bucket, src=head, dst='HEAD') |
| |
| api.gsutil.rsync( |
| bucket=bucket, |
| src=html, |
| dst='latest', |
| recursive=True, |
| multithreaded=True, |
| ) |
| |
| link_template: str = props.link_template or 'https://pigweed.dev/?rev={}' |
| links: list[str] = [] |
| for path in paths: |
| links.append(link_template.format(path)) |
| step = api.step.empty('set output properties') |
| step.presentation.properties['links'] = links |
| |
| joined_links: str = ", ".join(links) |
| |
| if props.comment and api.buildbucket_util.is_tryjob: |
| for change in applied_changes: |
| api.gerrit.set_review( |
| 'post CL comment', |
| change.number, |
| host=change.gerrit_host, |
| message=f'Docs available at: {joined_links}', |
| notify='NONE', |
| ignore_automatic_attention_set_rules=True, |
| ) |
| |
| return result.RawResult( |
| summary_markdown=f'Docs available at {joined_links}.', |
| status=common.SUCCESS, |
| ) |
| |
| |
| def GenTests(api) -> Generator[recipe_test_api.TestData, None, None]: |
| def properties(**kwargs): |
| props = InputProperties(**kwargs) |
| props.checkout_options.CopyFrom(api.checkout.git_options()) |
| return api.properties(props) |
| |
| def ran(x): |
| return api.post_process(post_process.MustRun, x) |
| |
| def did_not_run(x): |
| return api.post_process(post_process.DoesNotRun, x) |
| |
| def url_is(url): |
| return api.post_process( |
| post_process.PropertyMatchesCallable, |
| 'links', |
| lambda value: url in value, |
| ) |
| |
| def drop(): |
| return api.post_process(post_process.DropExpectation) |
| |
| yield api.test( |
| 'docs-dry_run-build', |
| properties( |
| dry_run=True, |
| build_options=api.build.options(ninja_targets=['target']), |
| ), |
| ran('gn gen'), |
| ran('ninja'), |
| did_not_run('step'), |
| ran('upload testing/swarming-fake-task-id/HASH'), |
| url_is('https://pigweed.dev/?rev=testing/swarming-fake-task-id/HASH'), |
| drop(), |
| ) |
| |
| yield api.test( |
| 'docs-postsubmit', |
| properties( |
| dry_run=False, |
| pw_presubmit_options=api.pw_presubmit.options(step=['step']), |
| ), |
| ran('step'), |
| did_not_run('gn gen'), |
| did_not_run('ninja'), |
| ran('upload HASH'), |
| ran('write HEAD'), |
| ran('gsutil cp'), |
| ran('gsutil rsync'), |
| url_is('https://pigweed.dev/?rev=HASH'), |
| drop(), |
| ) |
| |
| build_id = 881234567890 |
| change_id = 123456 |
| |
| yield api.test( |
| 'docs-presubmit', |
| api.checkout.try_test_data(build_id=build_id), |
| properties( |
| comment=True, |
| dry_run=False, |
| pw_presubmit_options=api.pw_presubmit.options(step=['step']), |
| link_template='https://pending.docs/{}', |
| ), |
| ran('step'), |
| did_not_run('gn gen'), |
| did_not_run('ninja'), |
| ran(f'upload {build_id}'), |
| ran(f'upload {change_id}'), |
| ran('post CL comment'), |
| url_is(f'https://pending.docs/{build_id}'), |
| url_is(f'https://pending.docs/{change_id}'), |
| drop(), |
| ) |