| # Copyright 2020 The Pigweed Authors |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may not |
| # use this file except in compliance with the License. You may obtain a copy of |
| # the License at |
| # |
| # https://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations under |
| # the License. |
| """Generate docs using GN/Ninja and upload to GCS.""" |
| |
| from typing import Any, Generator, Sequence |
| |
| from PB.go.chromium.org.luci.buildbucket.proto import common |
| from PB.recipes.pigweed.docs_builder import InputProperties |
| from PB.recipe_engine import result |
| from recipe_engine import ( |
| config_types, |
| post_process, |
| recipe_api, |
| recipe_test_api, |
| ) |
| |
| DEPS = [ |
| 'fuchsia/buildbucket_util', |
| 'fuchsia/gerrit', |
| 'fuchsia/gsutil', |
| 'pigweed/build', |
| 'pigweed/checkout', |
| 'pigweed/environment', |
| 'pigweed/pw_presubmit', |
| 'recipe_engine/file', |
| 'recipe_engine/json', |
| 'recipe_engine/properties', |
| 'recipe_engine/swarming', |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| |
| def RunSteps( |
| api: recipe_api.RecipeScriptApi, |
| props: InputProperties, |
| ) -> result.RawResult: |
| checkout: api.checkout.CheckoutContext = api.checkout( |
| props.checkout_options |
| ) |
| env: api.environment.Environment = api.environment.init( |
| checkout, props.environment_options |
| ) |
| |
| out_dir: config_types.Path |
| if props.HasField('build_options'): |
| build: api.build.BuildContext = api.build.create( |
| checkout.root, props.build_options |
| ) |
| with env(): |
| api.build(build) |
| out_dir = build.root |
| elif props.HasField('pw_presubmit_options'): |
| with env(): |
| presubmit: api.pw_presubmit.PresubmitContext = ( |
| api.pw_presubmit.init(checkout, props.pw_presubmit_options) |
| ) |
| assert len(presubmit.steps) == 1 |
| for step in presubmit.steps: |
| api.pw_presubmit.run(ctx=presubmit, step=step, env=env) |
| out_dir = step.dir |
| else: |
| assert False # pragma: no cover |
| |
| html_path: str = props.html_path or 'docs/gen/docs/html' |
| html: config_types.Path = out_dir.joinpath(*html_path.split('/')) |
| |
| applied_changes: Sequence[api.checkout.Change] = checkout.applied_changes() |
| |
| paths: Sequence[str] |
| if props.dry_run: |
| paths = [ |
| f'testing/swarming-{api.swarming.task_id}/{checkout.revision()}', |
| ] |
| elif api.buildbucket_util.is_tryjob: |
| paths = [api.buildbucket_util.id] |
| if len(applied_changes) == 1: |
| paths.append(str(applied_changes[0].number)) |
| else: |
| paths = [checkout.revision()] |
| |
| bucket: str = props.bucket or 'pigweed-docs' |
| for path in paths: |
| api.gsutil.upload( |
| bucket=bucket, |
| src=html, |
| dst=path, |
| recursive=True, |
| multithreaded=True, |
| name=f'upload {path}', |
| ) |
| |
| # If in a tryjob there's no need to even fetch metadata. |
| if not api.buildbucket_util.is_tryjob: |
| # Getting gerrit details even if this is a dry run so testing will show |
| # if it would have worked. |
| change: dict[str, Any] = api.gerrit.change_details( |
| 'get change details', |
| change_id=str(checkout.revision()), |
| host=checkout.gerrit_host(), |
| max_attempts=5, |
| timeout=30, |
| test_data=api.json.test_api.output({'branch': 'main'}), |
| ).json.output |
| |
| if change['branch'] in ('master', 'main') and not props.dry_run: |
| head: str = checkout.root / 'HEAD' |
| api.file.write_text('write HEAD', head, checkout.revision()) |
| api.gsutil.upload(bucket=bucket, src=head, dst='HEAD') |
| |
| api.gsutil.rsync( |
| bucket=bucket, |
| src=html, |
| dst='latest', |
| recursive=True, |
| multithreaded=True, |
| ) |
| |
| link_template: str = props.link_template or 'https://pigweed.dev/?rev={}' |
| links: list[str] = [] |
| for path in paths: |
| link: str = link_template.format(path) |
| |
| if api.buildbucket_util.is_tryjob and '?' not in link: |
| if len(applied_changes) == 1: |
| change = applied_changes[0] |
| link += ( |
| f'?gerrit_name={change.gerrit_name}&change={change.number}&' |
| f'patchset={change.patchset}' |
| ) |
| |
| links.append(link) |
| |
| joined_links: str = ", ".join(links) |
| |
| if props.comment and api.buildbucket_util.is_tryjob: |
| for change in applied_changes: |
| api.gerrit.set_review( |
| "post CL comment", |
| change.number, |
| host=change.gerrit_host, |
| message=f'Docs available at: {joined_links}', |
| notify="NONE", |
| ignore_automatic_attention_set_rules=True, |
| ) |
| |
| return result.RawResult( |
| summary_markdown=f'Docs available at {joined_links}.', |
| status=common.SUCCESS, |
| ) |
| |
| |
| def GenTests(api) -> Generator[recipe_test_api.TestData, None, None]: |
| def properties(**kwargs): |
| props = InputProperties(**kwargs) |
| props.checkout_options.CopyFrom(api.checkout.git_options()) |
| return api.properties(props) |
| |
| def ran(x): |
| return api.post_process(post_process.MustRun, x) |
| |
| def did_not_run(x): |
| return api.post_process(post_process.DoesNotRun, x) |
| |
| def drop(): |
| return api.post_process(post_process.DropExpectation) |
| |
| yield ( |
| api.test('docs-dry_run-build') |
| + properties( |
| dry_run=True, |
| build_options=api.build.options(ninja_targets=['target']), |
| ) |
| + ran('gn gen') |
| + ran('ninja') |
| + did_not_run('step') |
| + ran('upload testing/swarming-fake-task-id/HASH') |
| + drop() |
| ) |
| |
| yield ( |
| api.test('docs-postsubmit') |
| + properties( |
| dry_run=False, |
| pw_presubmit_options=api.pw_presubmit.options(step=['step']), |
| ) |
| + ran('step') |
| + did_not_run('gn gen') |
| + did_not_run('ninja') |
| + ran('upload HASH') |
| + ran('write HEAD') |
| + ran('gsutil cp') |
| + ran('gsutil rsync') |
| + drop() |
| ) |
| |
| build_id = 881234567890 |
| |
| yield ( |
| api.test('docs-presubmit') |
| + api.checkout.try_test_data(build_id=build_id) |
| + properties( |
| comment=True, |
| dry_run=False, |
| pw_presubmit_options=api.pw_presubmit.options(step=['step']), |
| link_template='https://pending.docs/{}', |
| ) |
| + ran('step') |
| + did_not_run('gn gen') |
| + did_not_run('ninja') |
| + ran(f'upload {build_id}') |
| + ran('upload 123456') |
| + ran('post CL comment') |
| + drop() |
| ) |