#!/usr/bin/env python3

# Copyright 2020 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""Runs the local presubmit checks for the Pigweed repository."""

import argparse
import json
import logging
import os
from pathlib import Path
import re
import shutil
import subprocess
import sys
from typing import Callable, Iterable, List, Sequence, TextIO

try:
    import pw_presubmit
except ImportError:
    # Append the pw_presubmit package path to the module search path to allow
    # running this module without installing the pw_presubmit package.
    sys.path.append(os.path.dirname(os.path.dirname(
        os.path.abspath(__file__))))
    import pw_presubmit

import pw_package.pigweed_packages

from pw_presubmit import (
    build,
    cli,
    cpp_checks,
    format_code,
    git_repo,
    call,
    filter_paths,
    inclusive_language,
    npm_presubmit,
    plural,
    presubmit,
    PresubmitContext,
    PresubmitFailure,
    Programs,
    python_checks,
    shell_checks,
    todo_check,
)
from pw_presubmit.install_hook import install_git_hook

_LOG = logging.getLogger(__name__)

pw_package.pigweed_packages.initialize()

# Trigger builds if files with these extensions change.
_BUILD_FILE_FILTER = presubmit.FileFilter(
    suffix=(*format_code.C_FORMAT.extensions, '.py', '.rst', '.gn', '.gni'))


def _at_all_optimization_levels(target):
    for level in ('debug', 'size_optimized', 'speed_optimized'):
        yield f'{target}_{level}'


#
# Build presubmit checks
#
def gn_clang_build(ctx: PresubmitContext):
    """Checks all compile targets that rely on LLVM tooling."""
    build_targets = [
        *_at_all_optimization_levels('host_clang'),
        'cpp14_compatibility',
        'cpp20_compatibility',
        'asan',
        'tsan',
        'ubsan',
        'runtime_sanitizers',

        # TODO(b/234876100): msan will not work until the C++ standard library
        # included in the sysroot has a variant built with msan.
    ]

    # clang-tidy doesn't run on Windows.
    if sys.platform != 'win32':
        build_targets.append('static_analysis')

    # QEMU doesn't run on Windows.
    if sys.platform != 'win32':
        # TODO(b/244604080): For the pw::InlineString tests, qemu_clang_debug
        #     and qemu_clang_speed_optimized produce a binary too large for the
        #     QEMU target's 256KB flash. Restore debug and speed optimized
        #     builds when this is fixed.
        build_targets.append('qemu_clang_size_optimized')

    # TODO(b/240982565): SocketStream currently requires Linux.
    if sys.platform.startswith('linux'):
        build_targets.append('integration_tests')

    build.gn_gen(ctx)
    build.ninja(ctx, *build_targets)


@_BUILD_FILE_FILTER.apply_to_check()
def gn_gcc_build(ctx: PresubmitContext):
    build.gn_gen(ctx)
    build.ninja(ctx, *_at_all_optimization_levels('host_gcc'))


_HOST_COMPILER = 'gcc' if sys.platform == 'win32' else 'clang'


def gn_host_build(ctx: PresubmitContext):
    build.gn_gen(ctx)
    build.ninja(ctx, *_at_all_optimization_levels(f'host_{_HOST_COMPILER}'))


@_BUILD_FILE_FILTER.apply_to_check()
def gn_quick_build_check(ctx: PresubmitContext):
    """Checks the state of the GN build by running gn gen and gn check."""
    build.gn_gen(ctx)


@_BUILD_FILE_FILTER.apply_to_check()
def gn_full_build_check(ctx: PresubmitContext) -> None:
    build_targets = [
        *_at_all_optimization_levels('stm32f429i'),
        *_at_all_optimization_levels(f'host_{_HOST_COMPILER}'),
        'python.tests',
        'python.lint',
        'docs',
        'fuzzers',
        'pw_env_setup:pypi_pigweed_python_source_tree',
    ]

    # TODO(b/234645359): Re-enable on Windows when compatibility tests build.
    if sys.platform != 'win32':
        build_targets.append('cpp14_compatibility')
        build_targets.append('cpp20_compatibility')

    # TODO(b/240982565): SocketStream currently requires Linux.
    if sys.platform.startswith('linux'):
        build_targets.append('integration_tests')

    build.gn_gen(ctx)
    build.ninja(ctx, *build_targets)


@_BUILD_FILE_FILTER.apply_to_check()
def gn_full_qemu_check(ctx: PresubmitContext):
    build.gn_gen(ctx)
    build.ninja(
        ctx,
        *_at_all_optimization_levels('qemu_gcc'),
        *_at_all_optimization_levels('qemu_clang'),
    )


@_BUILD_FILE_FILTER.apply_to_check()
def gn_combined_build_check(ctx: PresubmitContext) -> None:
    """Run most host and device (QEMU) tests."""
    build_targets = [
        *_at_all_optimization_levels('stm32f429i'),
        *_at_all_optimization_levels(f'host_{_HOST_COMPILER}'),
        'python.tests',
        'python.lint',
        'docs',
        'fuzzers',
        'pw_env_setup:pypi_pigweed_python_source_tree',
    ]

    # TODO(b/234645359): Re-enable on Windows when compatibility tests build.
    if sys.platform != 'win32':
        build_targets.append('cpp14_compatibility')
        build_targets.append('cpp20_compatibility')

    # clang-tidy doesn't run on Windows.
    if sys.platform != 'win32':
        build_targets.append('static_analysis')

    # QEMU doesn't run on Windows.
    if sys.platform != 'win32':
        build_targets.extend(_at_all_optimization_levels('qemu_gcc'))

        # TODO(b/244604080): For the pw::InlineString tests, qemu_clang_debug
        #     and qemu_clang_speed_optimized produce a binary too large for the
        #     QEMU target's 256KB flash. Restore debug and speed optimized
        #     builds when this is fixed.
        build_targets.append('qemu_clang_size_optimized')

    # TODO(b/240982565): SocketStream currently requires Linux.
    if sys.platform.startswith('linux'):
        build_targets.append('integration_tests')

    build.gn_gen(ctx)
    build.ninja(ctx, *build_targets)


@_BUILD_FILE_FILTER.apply_to_check()
def gn_arm_build(ctx: PresubmitContext):
    build.gn_gen(ctx)
    build.ninja(ctx, *_at_all_optimization_levels('stm32f429i'))


@_BUILD_FILE_FILTER.apply_to_check()
def stm32f429i(ctx: PresubmitContext):
    build.gn_gen(ctx, pw_use_test_server=True)
    with build.test_server('stm32f429i_disc1_test_server', ctx.output_dir):
        build.ninja(ctx, *_at_all_optimization_levels('stm32f429i'))


@_BUILD_FILE_FILTER.apply_to_check()
def gn_boringssl_build(ctx: PresubmitContext):
    build.install_package(ctx.package_root, 'boringssl')
    build.gn_gen(ctx,
                 dir_pw_third_party_boringssl='"{}"'.format(ctx.package_root /
                                                            'boringssl'))
    build.ninja(
        ctx,
        *_at_all_optimization_levels('stm32f429i'),
        *_at_all_optimization_levels('host_clang'),
    )


@_BUILD_FILE_FILTER.apply_to_check()
def gn_nanopb_build(ctx: PresubmitContext):
    build.install_package(ctx.package_root, 'nanopb')
    build.gn_gen(ctx,
                 dir_pw_third_party_nanopb='"{}"'.format(ctx.package_root /
                                                         'nanopb'))
    build.ninja(
        ctx,
        *_at_all_optimization_levels('stm32f429i'),
        *_at_all_optimization_levels('host_clang'),
    )


@_BUILD_FILE_FILTER.apply_to_check()
def gn_crypto_mbedtls_build(ctx: PresubmitContext):
    build.install_package(ctx.package_root, 'mbedtls')
    build.gn_gen(
        ctx,
        dir_pw_third_party_mbedtls='"{}"'.format(ctx.package_root / 'mbedtls'),
        pw_crypto_SHA256_BACKEND='"{}"'.format(ctx.root /
                                               'pw_crypto:sha256_mbedtls'),
        pw_crypto_ECDSA_BACKEND='"{}"'.format(ctx.root /
                                              'pw_crypto:ecdsa_mbedtls'))
    build_targets = [*_at_all_optimization_levels(f'host_{_HOST_COMPILER}')]

    # TODO(b/240982565): SocketStream currently requires Linux.
    if sys.platform.startswith('linux'):
        build_targets.append('integration_tests')

    build.ninja(ctx, *build_targets)


@_BUILD_FILE_FILTER.apply_to_check()
def gn_crypto_boringssl_build(ctx: PresubmitContext):
    build.install_package(ctx.package_root, 'boringssl')
    build.gn_gen(
        ctx,
        dir_pw_third_party_boringssl='"{}"'.format(ctx.package_root /
                                                   'boringssl'),
        pw_crypto_SHA256_BACKEND='"{}"'.format(ctx.root /
                                               'pw_crypto:sha256_boringssl'),
        pw_crypto_ECDSA_BACKEND='"{}"'.format(ctx.root /
                                              'pw_crypto:ecdsa_boringssl'),
    )
    build_targets = [*_at_all_optimization_levels(f'host_{_HOST_COMPILER}')]

    # TODO(b/240982565): SocketStream currently requires Linux.
    if sys.platform.startswith('linux'):
        build_targets.append('integration_tests')

    build.ninja(ctx, *build_targets)


@_BUILD_FILE_FILTER.apply_to_check()
def gn_crypto_micro_ecc_build(ctx: PresubmitContext):
    build.install_package(ctx.package_root, 'micro-ecc')
    build.gn_gen(
        ctx,
        dir_pw_third_party_micro_ecc='"{}"'.format(ctx.package_root /
                                                   'micro-ecc'),
        pw_crypto_ECDSA_BACKEND='"{}"'.format(ctx.root /
                                              'pw_crypto:ecdsa_uecc'),
    )
    build_targets = [*_at_all_optimization_levels(f'host_{_HOST_COMPILER}')]

    # TODO(b/240982565): SocketStream currently requires Linux.
    if sys.platform.startswith('linux'):
        build_targets.append('integration_tests')

    build.ninja(ctx, *build_targets)


@_BUILD_FILE_FILTER.apply_to_check()
def gn_teensy_build(ctx: PresubmitContext):
    build.install_package(ctx.package_root, 'teensy')
    build.gn_gen(ctx,
                 pw_arduino_build_CORE_PATH='"{}"'.format(str(
                     ctx.package_root)),
                 pw_arduino_build_CORE_NAME='teensy',
                 pw_arduino_build_PACKAGE_NAME='teensy/avr',
                 pw_arduino_build_BOARD='teensy40')
    build.ninja(ctx, *_at_all_optimization_levels('arduino'))


@_BUILD_FILE_FILTER.apply_to_check()
def gn_software_update_build(ctx: PresubmitContext):
    build.install_package(ctx.package_root, 'nanopb')
    build.install_package(ctx.package_root, 'protobuf')
    build.install_package(ctx.package_root, 'mbedtls')
    build.install_package(ctx.package_root, 'micro-ecc')
    build.gn_gen(
        ctx,
        dir_pw_third_party_protobuf='"{}"'.format(ctx.package_root /
                                                  'protobuf'),
        dir_pw_third_party_nanopb='"{}"'.format(ctx.package_root / 'nanopb'),
        dir_pw_third_party_micro_ecc='"{}"'.format(ctx.package_root /
                                                   'micro-ecc'),
        pw_crypto_ECDSA_BACKEND='"{}"'.format(ctx.root /
                                              'pw_crypto:ecdsa_uecc'),
        dir_pw_third_party_mbedtls='"{}"'.format(ctx.package_root / 'mbedtls'),
        pw_crypto_SHA256_BACKEND='"{}"'.format(ctx.root /
                                               'pw_crypto:sha256_mbedtls'))
    build.ninja(
        ctx,
        *_at_all_optimization_levels('host_clang'),
    )


@_BUILD_FILE_FILTER.apply_to_check()
def gn_pw_system_demo_build(ctx: PresubmitContext):
    build.install_package(ctx.package_root, 'freertos')
    build.install_package(ctx.package_root, 'nanopb')
    build.install_package(ctx.package_root, 'stm32cube_f4')
    build.gn_gen(
        ctx,
        dir_pw_third_party_freertos='"{}"'.format(ctx.package_root /
                                                  'freertos'),
        dir_pw_third_party_nanopb='"{}"'.format(ctx.package_root / 'nanopb'),
        dir_pw_third_party_stm32cube_f4='"{}"'.format(ctx.package_root /
                                                      'stm32cube_f4'),
    )
    build.ninja(ctx, 'pw_system_demo')


@_BUILD_FILE_FILTER.apply_to_check()
def gn_qemu_build(ctx: PresubmitContext):
    build.gn_gen(ctx)
    build.ninja(ctx, *_at_all_optimization_levels('qemu_gcc'))


@_BUILD_FILE_FILTER.apply_to_check()
def gn_qemu_clang_build(ctx: PresubmitContext):
    build.gn_gen(ctx)
    build.ninja(ctx, *_at_all_optimization_levels('qemu_clang'))


def gn_docs_build(ctx: PresubmitContext):
    build.gn_gen(ctx)
    build.ninja(ctx, 'docs')


def gn_host_tools(ctx: PresubmitContext):
    build.gn_gen(ctx)
    build.ninja(ctx, 'host_tools')


@filter_paths(endswith=format_code.C_FORMAT.extensions)
def oss_fuzz_build(ctx: PresubmitContext):
    build.gn_gen(ctx, pw_toolchain_OSS_FUZZ_ENABLED=True)
    build.ninja(ctx, "fuzzers")


def _run_cmake(ctx: PresubmitContext, toolchain='host_clang') -> None:
    build.install_package(ctx.package_root, 'nanopb')

    env = None
    if 'clang' in toolchain:
        env = build.env_with_clang_vars()

    toolchain_path = ctx.root / 'pw_toolchain' / toolchain / 'toolchain.cmake'
    build.cmake(ctx,
                f'-DCMAKE_TOOLCHAIN_FILE={toolchain_path}',
                '-DCMAKE_EXPORT_COMPILE_COMMANDS=1',
                f'-Ddir_pw_third_party_nanopb={ctx.package_root / "nanopb"}',
                '-Dpw_third_party_nanopb_ADD_SUBDIRECTORY=ON',
                env=env)


@filter_paths(endswith=(*format_code.C_FORMAT.extensions, '.cmake',
                        'CMakeLists.txt'))
def cmake_clang(ctx: PresubmitContext):
    _run_cmake(ctx, toolchain='host_clang')
    build.ninja(ctx, 'pw_apps', 'pw_run_tests.modules')


@filter_paths(endswith=(*format_code.C_FORMAT.extensions, '.cmake',
                        'CMakeLists.txt'))
def cmake_gcc(ctx: PresubmitContext):
    _run_cmake(ctx, toolchain='host_gcc')
    build.ninja(ctx, 'pw_apps', 'pw_run_tests.modules')


# TODO(b/235882003): Slowly remove targets from here that work with bazel until
# none remain.
_TARGETS_THAT_DO_NOT_BUILD_WITH_BAZEL = (
    '-//pw_arduino_build',
    '-//pw_blob_store/...:all',
    '-//pw_boot/...:all',
    '-//pw_cpu_exception_cortex_m/...:all',
    '-//pw_chrono:chrono_proto_pb2',
    '-//pw_chrono/py/...:all',
    '-//pw_crypto/...:all',  # TODO(b/236321905) Remove when passing.
    '-//pw_file/...:all',
    '-//pw_hdlc/rpc_example',  # TODO(b/241575924) Remove when passing.
    '-//pw_i2c_mcuxpresso/...:all',
    '-//pw_kvs/...:all',
    '-//pw_log:log_proto_py_pb2',  # TODO(b/241456982) Remove when passing.
    '-//pw_log:log_proto_py_pb2_genproto',
    '-//pw_log_null/...:all',
    '-//pw_log_string/...:all',
    '-//pw_metric/...:all',
    '-//pw_minimal_cpp_stdlib/...:all',
    '-//pw_persistent_ram/...:all',
    '-//pw_snapshot/py/...:all',
    '-//pw_snapshot:metadata_proto_py_pb2',
    '-//pw_snapshot:metadata_proto_py_pb2_genproto',
    '-//pw_snapshot:snapshot_proto_py_pb2',
    '-//pw_snapshot:snapshot_proto_py_pb2_genproto',
    # TODO(b/232427554): Get pw_software_update to build.
    '-//pw_software_update:bundled_update_py_pb2',
    '-//pw_software_update:bundled_update_py_pb2_genproto',
    '-//pw_software_update:bundled_update_service',
    '-//pw_software_update:bundled_update_service_test',
    '-//pw_software_update:bundled_update_service_pwpb',
    '-//pw_software_update:bundled_update_service_pwpb_test',
    '-//pw_software_update:update_bundle',
    '-//pw_software_update:update_bundle_test',
    '-//pw_spi/...:all',
    '-//pw_sys_io_arduino/...:all',
    '-//pw_sys_io_mcuxpresso/...:all',
    '-//pw_sys_io_stm32cube/...:all',
    '-//pw_system/...:all',
    '-//pw_thread/py/...:all',
    '-//pw_thread:thread_proto_py_pb2',
    '-//pw_thread:thread_proto_py_pb2_genproto',
    '-//pw_thread:thread_snapshot_service_py_pb2',
    '-//pw_thread:thread_snapshot_service_py_pb2_genproto',
    '-//pw_thread_embos/...:all',
    '-//pw_thread_freertos/...:all',
    '-//pw_thread_threadx/...:all',
    '-//pw_tls_client/...:all',
    '-//pw_tls_client_boringssl/...:all',
    '-//pw_tls_client_mbedtls/...:all',
    # TODO(b/241456982) Remove when passing.
    '-//pw_tokenizer:tokenizer_proto_py_pb2',
    '-//pw_tokenizer:tokenizer_proto_py_pb2_genproto',
    '-//pw_trace/...:all',
    '-//pw_trace_tokenized/...:all',
    '-//pw_work_queue/...:all',
    '-//targets/arduino/...:all',
    '-//targets/emcraft_sf2_som/...:all',
    '-//targets/lm3s6965evb_qemu/...:all',
    '-//targets/mimxrt595_evk/...:all',
    '-//targets/stm32f429i_disc1/...:all',
    '-//targets/stm32f429i_disc1_stm32cube/...:all',
    '-//targets/rp2040/...:all',
    '-//third_party/boringssl/...:all',
    '-//third_party/micro_ecc/...:all',
)

# TODO(b/235882003): Slowly remove targets from here that work with bazel until
# none remain.
_TARGETS_THAT_DO_NOT_TEST_WITH_BAZEL = _TARGETS_THAT_DO_NOT_BUILD_WITH_BAZEL + (
    '-//pw_malloc_freelist/...:all', )


@filter_paths(endswith=(*format_code.C_FORMAT.extensions, '.bazel', '.bzl',
                        'BUILD'))
def bazel_test(ctx: PresubmitContext) -> None:
    """Runs bazel test on each bazel compatible module"""
    build.bazel(ctx, 'test', '--test_output=errors', '--', '//...',
                *_TARGETS_THAT_DO_NOT_TEST_WITH_BAZEL)


@filter_paths(endswith=(*format_code.C_FORMAT.extensions, '.bazel', '.bzl',
                        'BUILD'))
def bazel_build(ctx: PresubmitContext) -> None:
    """Runs Bazel build on each Bazel compatible module."""
    build.bazel(ctx, 'build', '--', '//...',
                *_TARGETS_THAT_DO_NOT_BUILD_WITH_BAZEL)


def pw_transfer_integration_test(ctx: PresubmitContext) -> None:
    """Runs the pw_transfer cross-language integration test only.

    This test is not part of the regular bazel build because it's slow and
    intended to run in CI only.
    """
    build.bazel(
        ctx, 'test',
        '//pw_transfer/integration_test:cross_language_integration_test',
        '--test_output=errors')


#
# General presubmit checks
#


def _clang_system_include_paths(lang: str) -> List[str]:
    """Generate default system header paths.

    Returns the list of system include paths used by the host
    clang installation.
    """
    # Dump system include paths with preprocessor verbose.
    command = [
        'clang++', '-Xpreprocessor', '-v', '-x', f'{lang}', f'{os.devnull}',
        '-fsyntax-only'
    ]
    process = subprocess.run(command,
                             check=True,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT)

    # Parse the command output to retrieve system include paths.
    # The paths are listed one per line.
    output = process.stdout.decode(errors='backslashreplace')
    include_paths: List[str] = []
    for line in output.splitlines():
        path = line.strip()
        if os.path.exists(path):
            include_paths.append(f'-isystem{path}')

    return include_paths


def edit_compile_commands(in_path: Path, out_path: Path,
                          func: Callable[[str, str, str], str]) -> None:
    """Edit the selected compile command file.

    Calls the input callback on all triplets (file, directory, command) in
    the input compile commands database. The return value replaces the old
    compile command in the output database.
    """
    with open(in_path) as in_file:
        compile_commands = json.load(in_file)
        for item in compile_commands:
            item['command'] = func(item['file'], item['directory'],
                                   item['command'])
    with open(out_path, 'w') as out_file:
        json.dump(compile_commands, out_file, indent=2)


_EXCLUDE_FROM_COPYRIGHT_NOTICE: Sequence[str] = (
    # Configuration
    r'^(?:.+/)?\..+$',
    r'\bPW_PLUGINS$',
    r'\bconstraint.list$',
    # Metadata
    r'^docker/tag$',
    r'\bAUTHORS$',
    r'\bLICENSE$',
    r'\bOWNERS$',
    r'\bPIGWEED_MODULES$',
    r'\brequirements.txt$',
    r'\bgo.(mod|sum)$',
    r'\bpackage.json$',
    r'\byarn.lock$',
    r'\bpackage-lock.json$',
    # Data files
    r'\.bin$',
    r'\.csv$',
    r'\.elf$',
    r'\.gif$',
    r'\.ico$',
    r'\.jpg$',
    r'\.json$',
    r'\.png$',
    r'\.svg$',
    r'\.xml$',
    # Documentation
    r'\.md$',
    r'\.rst$',
    # Generated protobuf files
    r'\.pb\.h$',
    r'\.pb\.c$',
    r'\_pb2.pyi?$',
    # Diff/Patch files
    r'\.diff$',
    r'\.patch$',
)

# Regular expression for the copyright comment. "\1" refers to the comment
# characters and "\2" refers to space after the comment characters, if any.
# All period characters are escaped using a replace call.
_COPYRIGHT = re.compile(
r"""(#|//|::| \*|)( ?)Copyright 2\d{3} The Pigweed Authors
\1
\1\2Licensed under the Apache License, Version 2.0 \(the "License"\); you may not
\1\2use this file except in compliance with the License. You may obtain a copy of
\1\2the License at
\1
\1(?:\2    |\t)https://www.apache.org/licenses/LICENSE-2.0
\1
\1\2Unless required by applicable law or agreed to in writing, software
\1\2distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
\1\2WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
\1\2License for the specific language governing permissions and limitations under
\1\2the License.
""".replace('.', r'\.'), re.MULTILINE)  # pylint: disable=line-too-long # yapf: disable

_SKIP_LINE_PREFIXES = (
    '#!',
    '@echo off',
    ':<<',
    '/*',
    ' * @jest-environment jsdom',
    ' */',
    '{#',  # Jinja comment block
    '# -*- coding: utf-8 -*-',
    '<!--',
)


def _read_notice_lines(file: TextIO) -> Iterable[str]:
    lines = iter(file)
    try:
        # Read until the first line of the copyright notice.
        line = next(lines)
        while line.isspace() or line.startswith(_SKIP_LINE_PREFIXES):
            line = next(lines)

        yield line

        for _ in range(12):  # The notice is 13 lines; read the remaining 12.
            yield next(lines)
    except StopIteration:
        return


@filter_paths(exclude=_EXCLUDE_FROM_COPYRIGHT_NOTICE)
def copyright_notice(ctx: PresubmitContext):
    """Checks that the Pigweed copyright notice is present."""
    errors = []

    for path in ctx.paths:
        if path.stat().st_size == 0:
            continue  # Skip empty files

        with path.open() as file:
            if not _COPYRIGHT.match(''.join(_read_notice_lines(file))):
                errors.append(path)

    if errors:
        _LOG.warning('%s with a missing or incorrect copyright notice:\n%s',
                     plural(errors, 'file'), '\n'.join(str(e) for e in errors))
        raise PresubmitFailure


_BAZEL_SOURCES_IN_BUILD = tuple(format_code.C_FORMAT.extensions)
_GN_SOURCES_IN_BUILD = ('setup.cfg', '.toml', '.rst', '.py',
                        *_BAZEL_SOURCES_IN_BUILD)

SOURCE_FILES_FILTER = presubmit.FileFilter(endswith=_GN_SOURCES_IN_BUILD,
                                           suffix=('.bazel', '.bzl', '.gn',
                                                   '.gni'),
                                           exclude=(r'zephyr.*/',
                                                    r'android.*/'))


@SOURCE_FILES_FILTER.apply_to_check()
def source_is_in_build_files(ctx: PresubmitContext):
    """Checks that source files are in the GN and Bazel builds."""
    missing = build.check_builds_for_files(
        _BAZEL_SOURCES_IN_BUILD,
        _GN_SOURCES_IN_BUILD,
        ctx.paths,
        bazel_dirs=[ctx.root],
        gn_build_files=git_repo.list_files(pathspecs=['BUILD.gn', '*BUILD.gn'],
                                           repo_path=ctx.root))

    if missing:
        _LOG.warning(
            'All source files must appear in BUILD and BUILD.gn files')
        raise PresubmitFailure

    _run_cmake(ctx)
    cmake_missing = build.check_compile_commands_for_files(
        ctx.output_dir / 'compile_commands.json',
        (f for f in ctx.paths if f.suffix in ('.c', '.cc')))
    if cmake_missing:
        _LOG.warning('The CMake build is missing %d files', len(cmake_missing))
        _LOG.warning('Files missing from CMake:\n%s',
                     '\n'.join(str(f) for f in cmake_missing))
        # TODO(hepler): Many files are missing from the CMake build. Make this
        #     check an error when the missing files are fixed.
        # raise PresubmitFailure


def build_env_setup(ctx: PresubmitContext):
    if 'PW_CARGO_SETUP' not in os.environ:
        _LOG.warning(
            'Skipping build_env_setup since PW_CARGO_SETUP is not set')
        return

    tmpl = ctx.root.joinpath('pw_env_setup', 'py', 'pyoxidizer.bzl.tmpl')
    out = ctx.output_dir.joinpath('pyoxidizer.bzl')

    with open(tmpl, 'r') as ins:
        cfg = ins.read().replace('${PW_ROOT}', str(ctx.root))
        with open(out, 'w') as outs:
            outs.write(cfg)

    call('pyoxidizer', 'build', cwd=ctx.output_dir)


def _valid_capitalization(word: str) -> bool:
    """Checks that the word has a capital letter or is not a regular word."""
    return bool(
        any(c.isupper() for c in word)  # Any capitalizatian (iTelephone)
        or not word.isalpha()  # Non-alphabetical (cool_stuff.exe)
        or shutil.which(word))  # Matches an executable (clangd)


def commit_message_format(_: PresubmitContext):
    """Checks that the top commit's message is correctly formatted."""
    lines = git_repo.commit_message().splitlines()

    # Show limits and current commit message in log.
    _LOG.debug('%-25s%+25s%+22s', 'Line limits', '72|', '72|')
    for line in lines:
        _LOG.debug(line)

    if not lines:
        _LOG.error('The commit message is too short!')
        raise PresubmitFailure

    # Ignore Gerrit-generated reverts.
    if ('Revert' in lines[0]
            and 'This reverts commit ' in git_repo.commit_message()
            and 'Reason for revert: ' in git_repo.commit_message()):
        _LOG.warning('Ignoring apparent Gerrit-generated revert')
        return

    # Ignore Gerrit-generated relands
    if ('Reland' in lines[0]
            and 'This is a reland of ' in git_repo.commit_message() and
            "Original change's description: " in git_repo.commit_message()):
        _LOG.warning('Ignoring apparent Gerrit-generated reland')
        return

    errors = 0

    if len(lines[0]) > 72:
        _LOG.warning("The commit message's first line must be no longer than "
                     '72 characters.')
        _LOG.warning('The first line is %d characters:\n  %s', len(lines[0]),
                     lines[0])
        errors += 1

    if lines[0].endswith('.'):
        _LOG.warning(
            "The commit message's first line must not end with a period:\n %s",
            lines[0])
        errors += 1

    # Check that the first line matches the expected pattern.
    match = re.match(r'^[\w/]+(?:{[\w ,]+})?: (?P<desc>.+)$', lines[0])
    if not match:
        _LOG.warning('The first line does not match the expected format')
        _LOG.warning(
            'Expected:\n\n  module_or_target: The description\n\n'
            'Found:\n\n  %s\n', lines[0])
        errors += 1
    elif not _valid_capitalization(match.group('desc').split()[0]):
        _LOG.warning(
            'The first word after the ":" in the first line ("%s") must be '
            'capitalized:\n  %s',
            match.group('desc').split()[0], lines[0])
        errors += 1

    if len(lines) > 1 and lines[1]:
        _LOG.warning("The commit message's second line must be blank.")
        _LOG.warning('The second line has %d characters:\n  %s', len(lines[1]),
                     lines[1])
        errors += 1

    # Ignore the line length check for Copybara imports so they can include the
    # commit hash and description for imported commits.
    if not errors and ('Copybara import' in lines[0]
                       and 'GitOrigin-RevId:' in git_repo.commit_message()):
        _LOG.warning('Ignoring Copybara import')
        return

    # Check that the lines are 72 characters or less, but skip any lines that
    # might possibly have a URL, path, or metadata in them. Also skip any lines
    # with non-ASCII characters.
    for i, line in enumerate(lines[2:], 3):
        if any(c in line for c in ':/>') or not line.isascii():
            continue

        if len(line) > 72:
            _LOG.warning(
                'Commit message lines must be no longer than 72 characters.')
            _LOG.warning('Line %d has %d characters:\n  %s', i, len(line),
                         line)
            errors += 1

    if errors:
        _LOG.error('Found %s in the commit message', plural(errors, 'error'))
        raise PresubmitFailure


@filter_paths(endswith=(*format_code.C_FORMAT.extensions, '.py'))
def static_analysis(ctx: PresubmitContext):
    """Runs all available static analysis tools."""
    build.gn_gen(ctx)
    build.ninja(ctx, 'python.lint', 'static_analysis')


def renode_check(ctx: PresubmitContext):
    """Placeholder for future check."""
    _LOG.info('%s %s', ctx.root, ctx.output_dir)


#
# Presubmit check programs
#

OTHER_CHECKS = (
    # Build that attempts to duplicate the build OSS-Fuzz does. Currently
    # failing.
    oss_fuzz_build,
    # TODO(b/235277910): Enable all Bazel tests when they're fixed.
    bazel_test,
    cmake_clang,
    cmake_gcc,
    gn_boringssl_build,
    build.gn_gen_check,
    gn_full_build_check,
    gn_full_qemu_check,
    gn_combined_build_check,
    gn_clang_build,
    gn_gcc_build,
    pw_transfer_integration_test,
    renode_check,
    static_analysis,
    stm32f429i,
    npm_presubmit.npm_test,
    todo_check.create(todo_check.BUGS_OR_USERNAMES),
)

# The misc program differs from other_checks in that checks in the misc
# program block CQ on Linux.
MISC = (
    gn_nanopb_build,
    gn_pw_system_demo_build,
    gn_teensy_build,
)

SANITIZERS = (cpp_checks.all_sanitizers(), )

# TODO(b/243380637) Merge into SECURITY.
CRYPTO = (
    gn_crypto_mbedtls_build,
    gn_crypto_boringssl_build,
    gn_crypto_micro_ecc_build,
)

SECURITY = (
    CRYPTO,
    gn_software_update_build,
)

# Avoid running all checks on specific paths.
PATH_EXCLUSIONS = (re.compile(r'\bthird_party/fuchsia/repo/'), )

_LINTFORMAT = (
    commit_message_format,
    copyright_notice,
    format_code.presubmit_checks(),
    inclusive_language.inclusive_language.with_filter(exclude=(
        r'\byarn.lock$',
        r'\bpackage-lock.json$',
    )),
    cpp_checks.pragma_once,
    build.bazel_lint,
    source_is_in_build_files,
    shell_checks.shellcheck if shutil.which('shellcheck') else (),
)

LINTFORMAT = (
    _LINTFORMAT,
    pw_presubmit.python_checks.check_python_versions,
    pw_presubmit.python_checks.gn_python_lint,
)

QUICK = (
    _LINTFORMAT,
    gn_quick_build_check,
    # TODO(b/34884583): Re-enable CMake and Bazel for Mac after we have fixed
    # the clang issues. The problem is that all clang++ invocations need the
    # two extra flags: "-nostdc++" and "${clang_prefix}/../lib/libc++.a".
    cmake_clang if sys.platform != 'darwin' else (),
)

FULL = (
    _LINTFORMAT,
    gn_host_build,
    gn_arm_build,
    gn_docs_build,
    gn_host_tools,
    bazel_test if sys.platform == 'linux' else (),
    bazel_build if sys.platform == 'linux' else (),
    # On Mac OS, system 'gcc' is a symlink to 'clang' by default, so skip GCC
    # host builds on Mac for now. Skip it on Windows too, since gn_host_build
    # already uses 'gcc' on Windows.
    gn_gcc_build if sys.platform not in ('darwin', 'win32') else (),
    # Windows doesn't support QEMU yet.
    gn_qemu_build if sys.platform != 'win32' else (),
    gn_qemu_clang_build if sys.platform != 'win32' else (),
    source_is_in_build_files,
    python_checks.gn_python_check,
    python_checks.gn_python_test_coverage,
    build_env_setup,
    # Skip gn_teensy_build if running on Windows. The Teensycore installer is
    # an exe that requires an admin role.
    gn_teensy_build if sys.platform in ['linux', 'darwin'] else (),
)

PROGRAMS = Programs(
    full=FULL,
    lintformat=LINTFORMAT,
    misc=MISC,
    other_checks=OTHER_CHECKS,
    quick=QUICK,
    crypto=CRYPTO,
    sanitizers=SANITIZERS,
    security=SECURITY,
)


def parse_args() -> argparse.Namespace:
    """Creates an argument parser and parses arguments."""

    parser = argparse.ArgumentParser(description=__doc__)
    cli.add_arguments(parser, PROGRAMS, 'quick')
    parser.add_argument(
        '--install',
        action='store_true',
        help='Install the presubmit as a Git pre-push hook and exit.')

    return parser.parse_args()


def run(install: bool, exclude: list, **presubmit_args) -> int:
    """Entry point for presubmit."""

    if install:
        install_git_hook('pre-push', [
            'python', '-m', 'pw_presubmit.pigweed_presubmit', '--base',
            'origin/main..HEAD', '--program', 'quick'
        ])
        return 0

    exclude.extend(PATH_EXCLUSIONS)
    return cli.run(exclude=exclude, **presubmit_args)


def main() -> int:
    """Run the presubmit for the Pigweed repository."""
    return run(**vars(parse_args()))


if __name__ == '__main__':
    try:
        # If pw_cli is available, use it to initialize logs.
        from pw_cli import log

        log.install(logging.INFO)
    except ImportError:
        # If pw_cli isn't available, display log messages like a simple print.
        logging.basicConfig(format='%(message)s', level=logging.INFO)

    sys.exit(main())