recipe_modules/build/api.py

# Copyright 2021 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""Calls to build code."""

from recipe_engine import recipe_api


class BuildApi(recipe_api.RecipeApi):
    """Calls to build code."""

    CAS_DIGEST_PROPERTY_NAME = 'cas_build_digest'

    def __init__(self, props, *args, **kwargs):
        super(BuildApi, self).__init__(*args, **kwargs)
        # Make these attributes public so recipes can override them.
        self.gn_args = list(props.gn_args)
        self.ninja_targets = list(props.ninja_targets)
        self.dir = None

    def initialize(self):
        self.dir = self.m.path['start_dir'].join('build')

    def gn_gen(self, checkout_dir, options=None):
        # TODO(pwbug/600) Make options required.
        cmd = ['gn', 'gen']

        # TODO(pwbug/600) Always use options.
        gn_args = options.gn_args if options else self.gn_args

        for gn_arg in gn_args:
            cmd.append('--args={}'.format(gn_arg))

        # Infrequently needed but harmless to always add this.
        cmd.append('--export-compile-commands')

        cmd.append(self.dir)

        with self.m.context(cwd=checkout_dir):
            self.m.step('gn gen', cmd)

    def get_gn_args(self, checkout_root=None, test_data=None):
        context_kwargs = {'cwd': checkout_root} if checkout_root else {}
        with self.m.context(**context_kwargs):
            cmd = ['gn', 'args', self.dir, '--list', '--json']
            args = self.m.step(
                'all gn args',
                cmd,
                stdout=self.m.json.output(),
                step_test_data=lambda: self.m.json.test_api.output_stream(
                    test_data or []
                ),
            ).stdout
            return {x['name']: x for x in args or ()}

    def ninja(self, options=None):
        # TODO(pwbug/600) Make options required.
        cmd = ['ninja', '-C', self.dir]
        # TODO(pwbug/600) Always use options.
        cmd.extend(options.ninja_targets if options else self.ninja_targets)
        self.m.step('ninja', cmd)

    def __call__(self, checkout_dir, options=None):
        # TODO(pwbug/600) Make options required.
        self.gn_gen(checkout_dir, options)
        self.ninja(options)

    def archive_to_cas(self):
        # TODO(pwbug/389) Only archive necessary files.
        with self.m.step.nest('archive to cas') as pres:
            digest = self.m.cas.archive('archive', self.dir, self.dir)
            pres.properties[self.CAS_DIGEST_PROPERTY_NAME] = digest

    def download_from_cas(self, digest):
        return self.m.cas.download('download from cas', digest, self.dir)

    def log_longest_build_steps(self, ninja_log):
        """Parse the build log and log the longest-running build steps."""
        steps = []
        for line in ninja_log.splitlines():
            try:
                start_ms, end_ms, _, name, _ = line.split()
                duration = (int(end_ms) - int(start_ms)) / 1000.0
                steps.append((duration, name))
            except (ValueError, TypeError):
                # This processing is best-effort and should never be the cause
                # of a build failure. In case there's something wrong with this
                # logfile silently ignore the error--in that case it's very
                # likely something else also went wrong and that should be the
                # error presented to the user.
                pass

        steps.sort(reverse=True)

        if steps:
            with self.m.step.nest('longest build steps'):
                for dur, name in steps[0:10]:
                    with self.m.step.nest(name) as pres:
                        pres.step_summary_text = '{:.1f}s'.format(dur)

    def save_logs(self, build_dir=None, export_dir=None):
        """Save common build logs from the build directory.

        Read common build logs so they appear in logdog and if export_dir is
        set copy these logs there. If there's a ninja log call
        log_longest_build_steps() on it.
        """

        if build_dir is None:
            build_dir = self.dir

        files = [
            'args.gn',
            'build.log',
            'compile_commands.json',
            'ninja.compdb',
            'ninja.graph',
            'ninja.log',
            'step.log',
            '.ninja_log',
        ]
        self.m.path.mock_add_file(build_dir.join('ninja.log'))

        found_files = []

        # Read these files and discard them so contents will be in logdog.
        ninja_log = None
        for name in files:
            path = build_dir.join(name)
            if not self.m.path.isfile(path):
                continue
            found_files.append(path)

            test_data = ''
            if name in ('.ninja_log', 'ninja.log'):
                test_data = (
                    '2000 5000 0 medium 0\n'
                    '3000 8000 0 long 0\n'
                    'malformed line\n'
                    '4000 5000 0 short 0\n'
                    '5000 x 0 malformed-end-time 0\n'
                )

            # No need to defer results here, but since some callers will be
            # deferring results and others won't this makes it so we always
            # need to call .get_result().
            with self.m.step.defer_results():
                contents = self.m.file.read_text(
                    name, path, test_data=test_data
                )

                if name in ('.ninja_log', 'ninja.log'):
                    ninja_log = contents.get_result()

        if ninja_log:
            self.log_longest_build_steps(ninja_log)

        if export_dir and found_files:
            with self.m.step.nest('copy'):
                for path in found_files:
                    name = self.m.path.basename(path)
                    self.m.file.copy(name, path, export_dir.join(name))