#!/usr/bin/env python3

# Copyright 2020 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""Runs the local presubmit checks for the Pigweed repository."""

import argparse
import itertools
import logging
import os
from pathlib import Path
import re
import shutil
import subprocess
import sys
from typing import Callable, Dict, Sequence, Tuple

try:
    import pw_presubmit
except ImportError:
    # Append the pw_presubmit package path to the module search path to allow
    # running this module without installing the pw_presubmit package.
    sys.path.append(os.path.dirname(os.path.dirname(
        os.path.abspath(__file__))))
    import pw_presubmit

from pw_presubmit import python_checks
from pw_presubmit import format_code, PresubmitContext
from pw_presubmit.install_hook import install_hook
from pw_presubmit import call, filter_paths, log_run, plural, PresubmitFailure

_LOG = logging.getLogger(__name__)


#
# Initialization
#
def init_cipd(ctx: PresubmitContext):
    # TODO(mohrr) invoke by importing rather than by subprocess.

    # TODO(pwbug/138) find way to support dependent project package files.

    cmd = [
        sys.executable,
        ctx.repository_root.joinpath('pw_env_setup', 'py', 'pw_env_setup',
                                     'cipd_setup', 'update.py'),
        '--install-dir', ctx.output_directory,
    ]  # yapf: disable

    package_files = ctx.repository_root.joinpath('pw_env_setup', 'py',
                                                 'pw_env_setup',
                                                 'cipd_setup').glob('*.json')

    for package_file in package_files:
        cmd.extend(('--package-file', package_file))

    call(*cmd)

    paths = [ctx.output_directory, ctx.output_directory.joinpath('bin')]
    for base in ctx.output_directory.glob('*'):
        paths.append(base)
        paths.append(base.joinpath('bin'))

    paths.append(Path(os.environ['PATH']))

    os.environ['PATH'] = os.pathsep.join(str(x) for x in paths)
    _LOG.debug('PATH %s', os.environ['PATH'])


def init_virtualenv(ctx: PresubmitContext):
    """Set up virtualenv, assumes recent Python 3 is already installed."""
    virtualenv_source = ctx.repository_root.joinpath('pw_env_setup', 'py',
                                                     'pw_env_setup',
                                                     'virtualenv_setup')

    # TODO(pwbug/138) find way to support dependent project requirements.

    # For speed, don't build the venv if it exists. Use --clean to recreate it.
    if not ctx.output_directory.joinpath('pyvenv.cfg').is_file():
        call(
            'python3',
            virtualenv_source,
            f'--venv_path={ctx.output_directory}',
            '--requirements={}'.format(
                virtualenv_source.joinpath('requirements.txt')),
            '--setup-py-roots={}'.format(ctx.repository_root),
        )

    os.environ['PATH'] = os.pathsep.join((
        str(ctx.output_directory.joinpath('bin')),
        os.environ['PATH'],
    ))


INIT: Tuple[Callable, ...] = (
    init_cipd,
    init_virtualenv,
)


#
# GN presubmit checks
#
def gn_args(**kwargs) -> str:
    return '--args=' + ' '.join(f'{arg}={val}' for arg, val in kwargs.items())


def gn_gen(*args, ctx: PresubmitContext, path=None, **kwargs) -> None:
    call('gn',
         'gen',
         path or ctx.output_directory,
         '--color=always',
         '--check',
         *args,
         cwd=ctx.repository_root,
         **kwargs)


def ninja(*args, ctx: PresubmitContext, **kwargs):
    call('ninja',
         '-C',
         ctx.output_directory,
         *args,
         cwd=ctx.repository_root,
         **kwargs)


_CLANG_GEN_ARGS = gn_args(
    pw_target_config='"//targets/host/target_config.gni"',
    pw_target_toolchain='"//pw_toolchain:host_clang_os"')

_DOCS_GEN_ARGS = gn_args(pw_target_config='"//targets/docs/target_config.gni"')


def gn_clang_build(ctx: PresubmitContext):
    gn_gen(_CLANG_GEN_ARGS, ctx=ctx)
    ninja(ctx=ctx)


@filter_paths(endswith=format_code.C_FORMAT.extensions)
def gn_gcc_build(ctx: PresubmitContext):
    gn_gen(gn_args(pw_target_config='"//targets/host/target_config.gni"',
                   pw_target_toolchain='"//pw_toolchain:host_gcc_os"'),
           ctx=ctx)
    ninja(ctx=ctx)


_ARM_GEN_ARGS = gn_args(
    pw_target_config='"//targets/stm32f429i-disc1/target_config.gni"')


@filter_paths(endswith=format_code.C_FORMAT.extensions)
def gn_arm_build(ctx: PresubmitContext):
    gn_gen(_ARM_GEN_ARGS, ctx=ctx)
    ninja(ctx=ctx)


def gn_docs_build(ctx: PresubmitContext):
    gn_gen(_DOCS_GEN_ARGS, ctx=ctx)
    ninja('docs:docs', ctx=ctx)


_QEMU_GEN_ARGS = gn_args(
    pw_target_config='"//targets/lm3s6965evb-qemu/target_config.gni"')

GN: Tuple[Callable, ...] = (
    gn_clang_build,
    gn_arm_build,
    gn_docs_build,
)

# On Mac OS, 'gcc' is a symlink to 'clang', so skip GCC host builds on Mac.
if sys.platform != 'darwin':
    GN = GN + (gn_gcc_build, )


def gn_host_tools(ctx: PresubmitContext):
    gn_gen(gn_args(pw_target_config='"//targets/host/target_config.gni"',
                   pw_target_toolchain='"//pw_toolchain:host_clang_os"',
                   pw_build_host_tools='true'),
           ctx=ctx)
    ninja('host_tools', ctx=ctx)


#
# C++ presubmit checks
#
# TODO(pwbug/45) Probably want additional checks.
_CLANG_TIDY_CHECKS = ('modernize-use-override', )


@filter_paths(endswith=format_code.C_FORMAT.extensions)
def clang_tidy(ctx: PresubmitContext):
    gn_gen('--export-compile-commands', _CLANG_GEN_ARGS, ctx=ctx)
    ninja(ctx=ctx)
    ninja('-t', 'compdb', 'objcxx', 'cxx', ctx=ctx)

    run_clang_tidy = None
    for var in ('PW_PIGWEED_CIPD_INSTALL_DIR', 'PW_CIPD_INSTALL_DIR'):
        if var in os.environ:
            possibility = os.path.join(os.environ[var],
                                       'share/clang/run-clang-tidy.py')
            if os.path.isfile(possibility):
                run_clang_tidy = possibility
                break

    checks = ','.join(_CLANG_TIDY_CHECKS)
    call(
        run_clang_tidy,
        f'-p={ctx.output_directory}',
        f'-checks={checks}',
        # TODO(pwbug/45) not sure if this is needed.
        # f'-extra-arg-before=-warnings-as-errors={checks}',
        *ctx.paths)


CC: Tuple[Callable, ...] = (
    pw_presubmit.pragma_once,
    # TODO(pwbug/45): Enable clang-tidy when it passes.
    # clang_tidy,
)


#
# CMake presubmit checks
#
def _env_with_clang_cc_vars():
    env = os.environ.copy()
    env['CC'] = env['LD'] = env['AS'] = 'clang'
    env['CXX'] = 'clang++'
    return env


@filter_paths(endswith=(*format_code.C_FORMAT.extensions, '.cmake',
                        'CMakeLists.txt'))
def cmake_tests(ctx: PresubmitContext):
    env = _env_with_clang_cc_vars()
    call('cmake',
         '-B', ctx.output_directory,
         '-S', ctx.repository_root,
         '-G', 'Ninja',
         env=env)  # yapf: disable
    ninja('pw_run_tests.modules', ctx=ctx)


CMAKE: Tuple[Callable, ...] = ()

# TODO(pwbug/141): Re-enable this after we have fixed the CMake toolchain issue
# on Mac. The problem is that all clang++ invocations need the two extra
# flags: "-nostdc++" and "${clang_prefix}../lib/libc++.a".
if sys.platform != 'darwin':
    CMAKE = (cmake_tests, )


#
# Bazel presubmit checks
#
@filter_paths(endswith=(*format_code.C_FORMAT.extensions, '.bzl', 'BUILD'))
def bazel_test(ctx: PresubmitContext):
    try:
        call('bazel',
             'test',
             '//...',
             '--verbose_failures',
             '--verbose_explanations',
             '--worker_verbose',
             '--symlink_prefix',
             ctx.output_directory.joinpath('bazel-'),
             cwd=ctx.repository_root)
    except:
        _LOG.info('If the Bazel build inexplicably fails while the '
                  'other builds are passing, try deleting the Bazel cache:\n'
                  '    rm -rf ~/.cache/bazel')
        raise


BAZEL: Tuple[Callable, ...] = ()

# TODO(pwbug/141): Re-enable this after we have fixed the Bazel toolchain issue
# on Mac. The problem is that all clang++ invocations need the two extra flags:
# "-nostdc++" and "${clang_prefix}../lib/libc++.a".
if sys.platform != 'darwin':
    BAZEL = (bazel_test, )

#
# Code format presubmit checks
#
COPYRIGHT_FIRST_LINE = re.compile(
    r'^(#|//| \*|REM|::) Copyright 20\d\d The Pigweed Authors$')

COPYRIGHT_LINES = tuple("""\

 Licensed under the Apache License, Version 2.0 (the "License"); you may not
 use this file except in compliance with the License. You may obtain a copy of
 the License at

     https://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 License for the specific language governing permissions and limitations under
 the License.
""".splitlines(True))

_EXCLUDE_FROM_COPYRIGHT_NOTICE: Sequence[str] = (
    r'(?:.+/)?\..+',
    r'AUTHORS',
    r'LICENSE',
    r'OWNERS',
    r'.*\.elf',
    r'.*\.gif',
    r'.*\.jpg',
    r'.*\.json',
    r'.*\.md',
    r'.*\.png',
    r'.*\.rst',
    r'(?:.+/)?requirements.txt',
    r'(.+/)?go.(mod|sum)',
)


@filter_paths(exclude=_EXCLUDE_FROM_COPYRIGHT_NOTICE)
def copyright_notice(ctx: PresubmitContext):
    """Checks that the copyright notice is present."""

    errors = []

    for path in ctx.paths:
        _LOG.debug('Checking %s', path)
        with open(path) as file:
            # Skip shebang and blank lines
            line = file.readline()
            while line and (line.startswith(
                ('#!', '/*', '@echo off', '# -*-')) or not line.strip()):
                line = file.readline()

            first_line = COPYRIGHT_FIRST_LINE.match(line)
            if not first_line:
                _LOG.debug('%s: invalid first line %r', path, line)
                errors.append(path)
                continue

            comment = first_line.group(1)

            for expected, actual in zip(COPYRIGHT_LINES, file):
                if comment + expected != actual:
                    _LOG.debug('  bad line: %r', actual)
                    _LOG.debug('  expected: %r', comment + expected)
                    errors.append(path)
                    break

    if errors:
        _LOG.warning('%s with a missing or incorrect copyright notice:\n%s',
                     pw_presubmit.plural(errors, 'file'),
                     '\n'.join(str(e) for e in errors))
        raise PresubmitFailure


CODE_FORMAT: Tuple[Callable, ...] = (
    copyright_notice, *format_code.PRESUBMIT_CHECKS)  # yapf: disable

#
# General presubmit checks
#


def _get_paths_from_command(*args, ctx: PresubmitContext, **kwargs):
    """Runs a command and reads Bazel or GN //-style paths from it."""
    process = log_run(*args,
                      stdout=subprocess.PIPE,
                      stderr=subprocess.DEVNULL,
                      cwd=ctx.repository_root,
                      **kwargs)
    files = set()

    for line in process.stdout.splitlines():
        path = line.strip().lstrip(b'/').replace(b':', b'/').decode()
        path = ctx.repository_root.joinpath(path)
        if path.is_file():
            _LOG.debug('Found file %s', path)
            files.add(path)

    return files


_SOURCES_IN_BUILD = '.rst', *format_code.C_FORMAT.extensions


@filter_paths(endswith=(*_SOURCES_IN_BUILD, 'BUILD', '.bzl', '.gn', '.gni'))
def source_is_in_build_files(ctx: PresubmitContext):
    """Checks that source files are in the GN and Bazel builds."""

    # Collect all paths in the Bazel build.
    build_bazel = _get_paths_from_command('bazel',
                                          'query',
                                          'kind("source file", //...:*)',
                                          ctx=ctx)

    # Collect all paths in the ARM and Clang GN builds.
    arm_dir = ctx.output_directory.joinpath('arm')
    gn_gen(_ARM_GEN_ARGS, ctx=ctx, path=arm_dir)
    build_gn = _get_paths_from_command('gn', 'desc', arm_dir, '*', ctx=ctx)

    clang_dir = ctx.output_directory.joinpath('clang')
    gn_gen(_CLANG_GEN_ARGS, ctx=ctx, path=clang_dir)
    build_gn.update(
        _get_paths_from_command('gn', 'desc', clang_dir, '*', ctx=ctx))

    docs_dir = ctx.output_directory.joinpath('docs')
    gn_gen(_DOCS_GEN_ARGS, ctx=ctx, path=docs_dir)
    build_gn.update(
        _get_paths_from_command('gn', 'desc', docs_dir, '*', ctx=ctx))

    qemu_dir = ctx.output_directory.joinpath('qemu')
    gn_gen(_QEMU_GEN_ARGS, ctx=ctx, path=qemu_dir)
    build_gn.update(
        _get_paths_from_command('gn', 'desc', qemu_dir, '*', ctx=ctx))

    missing_bazel = []
    missing_gn = []

    for path in (p for p in ctx.paths if p.suffix in _SOURCES_IN_BUILD):
        if path.suffix != '.rst' and path not in build_bazel:
            missing_bazel.append(path)
        if path not in build_gn:
            missing_gn.append(path)

    # TODO(pwbug/176) Replace this workaround for fuzzers.
    missing_bazel = [p for p in missing_bazel if 'fuzzer' not in p.name]

    if missing_bazel or missing_gn:
        for build, files in [('Bazel', missing_bazel), ('GN', missing_gn)]:
            if files:
                _LOG.warning('%s are missing from the %s build:\n%s',
                             plural(files, 'file'), build,
                             '\n'.join(str(x) for x in files))

        _LOG.warning(
            'All source files must appear in BUILD and BUILD.gn files')
        raise PresubmitFailure


GENERAL: Tuple[Callable, ...] = (source_is_in_build_files, )


def build_env_setup(ctx: PresubmitContext):
    if 'PW_CARGO_SETUP' not in os.environ:
        _LOG.warning(
            'Skipping build_env_setup since PW_CARGO_SETUP is not set')
        return

    tmpl = ctx.repository_root.joinpath('pw_env_setup', 'py', 'pyoxidizer.bzl')
    out = ctx.output_directory.joinpath('pyoxidizer.bzl')

    with open(tmpl, 'r') as ins:
        cfg = ins.read().replace('${PW_ROOT}', str(ctx.repository_root))
        with open(out, 'w') as outs:
            outs.write(cfg)

    call('pyoxidizer', 'build', cwd=ctx.output_directory)


BUILD_ENV_SETUP = (build_env_setup, )


BROKEN: Tuple[Callable, ...] = (
    # TODO(pwbug/45): Remove clang-tidy from BROKEN when it passes.
    clang_tidy,
    # Host tools are not broken but take long on slow internet connections.
    # They're still run in CQ, but not in 'pw presubmit'.
    gn_host_tools,
)  # yapf: disable

#
# Presubmit check programs
#
QUICK_PRESUBMIT: Tuple[Callable, ...] = (
    *INIT,
    *CODE_FORMAT,
    *GENERAL,
    *CC,
    gn_clang_build,
    gn_arm_build,
    *python_checks.ALL,
)

FULL_PRESUBMIT: Tuple[Callable, ...] = (
    INIT + CODE_FORMAT + GENERAL + CC + GN +
    python_checks.ALL + CMAKE + BAZEL + BUILD_ENV_SETUP)  # yapf: disable

PROGRAMS: Dict[str, Tuple] = {
    'broken': BROKEN,
    'full': FULL_PRESUBMIT,
    'quick': QUICK_PRESUBMIT,
}

ALL_STEPS = {c.__name__: c for c in itertools.chain(*PROGRAMS.values())}


def argument_parser(parser=None) -> argparse.ArgumentParser:
    """Create argument parser."""

    if parser is None:
        parser = argparse.ArgumentParser(description=__doc__)

    parser.add_argument(
        '-o',
        '--output-directory',
        type=Path,
        help='Output directory (default: <repo root>/.presubmit)',
    )

    parser.add_argument(
        '--clean',
        action='store_true',
        help='Delete the entire output directory before starting.',
    )
    parser.add_argument(
        '--clean-py',
        action='store_true',
        help=('Delete the Python virtualenv in the output directory before '
              'starting.'),
    )

    exclusive = parser.add_mutually_exclusive_group()
    exclusive.add_argument(
        '--install',
        action='store_true',
        help='Install the presubmit as a Git pre-push hook and exits.')
    exclusive.add_argument('-p',
                           '--program',
                           dest='program_name',
                           choices=[x for x in PROGRAMS if x != 'broken'],
                           default='full',
                           help='Which presubmit program to run')

    exclusive.add_argument(
        '--step',
        dest='steps',
        choices=sorted(ALL_STEPS),
        action='append',
        help='Provide explicit steps instead of running a predefined program.',
    )

    pw_presubmit.add_arguments(parser)

    return parser


def main(
        program_name: str,
        clean: bool,
        clean_py: bool,
        install: bool,
        repository: Path,
        output_directory: Path,
        steps: Sequence[str],
        **presubmit_args,
) -> int:
    """Entry point for presubmit."""

    os.environ.setdefault('PW_ROOT',
                          str(pw_presubmit.git_repo_path(repo=repository)))

    if not output_directory:
        output_directory = pw_presubmit.git_repo_path('.presubmit',
                                                      repo=repository)
    environment = output_directory
    _LOG.debug('Using environment at %s', environment)

    if clean and environment.exists():
        shutil.rmtree(environment)
    elif clean_py and environment.joinpath('init_virtualenv').exists():
        shutil.rmtree(environment.joinpath('init_virtualenv'))

    if install:
        install_hook(__file__, 'pre-push',
                     ['--base', 'origin/master..HEAD', '--program', 'quick'],
                     repository)
        return 0

    program: Sequence = PROGRAMS[program_name]
    if steps:
        program = [ALL_STEPS[name] for name in steps]

    if pw_presubmit.run_presubmit(program,
                                  repository=repository,
                                  output_directory=output_directory,
                                  **presubmit_args):
        return 0

    return 1


if __name__ == '__main__':
    # By default, display log messages like a simple print statement.
    logging.basicConfig(format='%(message)s', level=logging.INFO)
    sys.exit(main(**vars(argument_parser().parse_args())))