pw_presubmit: Check for files in the CMake build
- Add functions for checking if source files are present in
compile_commands.json.
- Check for sources in CMake files in the source_is_in_build_files
presubmit check. Report missing files, but don't fail, since there are
many files missing.
Change-Id: I3def8a0c19760256f70c5c263023a1c2ff04d1bb
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/28740
Reviewed-by: Rob Mohr <mohrr@google.com>
Commit-Queue: Wyatt Hepler <hepler@google.com>
diff --git a/pw_presubmit/py/pw_presubmit/build.py b/pw_presubmit/py/pw_presubmit/build.py
index 563f8ef..835a95e 100644
--- a/pw_presubmit/py/pw_presubmit/build.py
+++ b/pw_presubmit/py/pw_presubmit/build.py
@@ -14,11 +14,14 @@
"""Functions for building code during presubmit checks."""
import collections
+import itertools
+import json
import logging
import os
from pathlib import Path
import re
-from typing import Container, Dict, Iterable, List, Mapping, Set, Tuple
+from typing import (Collection, Container, Dict, Iterable, List, Mapping, Set,
+ Tuple, Union)
from pw_package import package_manager
from pw_presubmit import call, log_run, plural, PresubmitFailure, tools
@@ -152,6 +155,38 @@
yield path
+def _read_compile_commands(compile_commands: Path) -> dict:
+ with compile_commands.open('rb') as fd:
+ return json.load(fd)
+
+
+def compiled_files(compile_commands: Path) -> Iterable[Path]:
+ for command in _read_compile_commands(compile_commands):
+ file = Path(command['file'])
+ if file.is_absolute():
+ yield file
+ else:
+ yield file.joinpath(command['directory']).resolve()
+
+
+def check_compile_commands_for_files(
+ compile_commands: Union[Path, Iterable[Path]],
+ files: Iterable[Path],
+ extensions: Collection[str] = ('.c', '.cc', '.cpp'),
+) -> List[Path]:
+ """Checks for paths in one or more compile_commands.json files.
+
+ Only checks C and C++ source files by default.
+ """
+ if isinstance(compile_commands, Path):
+ compile_commands = [compile_commands]
+
+ compiled = frozenset(
+ itertools.chain.from_iterable(
+ compiled_files(cmds) for cmds in compile_commands))
+ return [f for f in files if f not in compiled and f.suffix in extensions]
+
+
def check_builds_for_files(
bazel_extensions_to_check: Container[str],
gn_extensions_to_check: Container[str],
diff --git a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
index d75f498..f069720 100755
--- a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
+++ b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
@@ -158,17 +158,22 @@
)
-@filter_paths(endswith=(*format_code.C_FORMAT.extensions, '.cmake',
- 'CMakeLists.txt'))
-def cmake_tests(ctx: PresubmitContext):
+def _run_cmake(ctx: PresubmitContext) -> None:
build.install_package(ctx.package_root, 'nanopb')
toolchain = ctx.root / 'pw_toolchain' / 'host_clang' / 'toolchain.cmake'
build.cmake(ctx.root,
ctx.output_dir,
f'-DCMAKE_TOOLCHAIN_FILE={toolchain}',
+ '-DCMAKE_EXPORT_COMPILE_COMMANDS=1',
f'-Ddir_pw_third_party_nanopb={ctx.package_root / "nanopb"}',
env=build.env_with_clang_vars())
+
+
+@filter_paths(endswith=(*format_code.C_FORMAT.extensions, '.cmake',
+ 'CMakeLists.txt'))
+def cmake_tests(ctx: PresubmitContext):
+ _run_cmake(ctx)
build.ninja(ctx.output_dir, 'pw_apps', 'pw_run_tests.modules')
@@ -396,6 +401,18 @@
'All source files must appear in BUILD and BUILD.gn files')
raise PresubmitFailure
+ _run_cmake(ctx)
+ cmake_missing = build.check_compile_commands_for_files(
+ ctx.output_dir / 'compile_commands.json',
+ (f for f in ctx.paths if f.suffix in ('.c', '.cc')))
+ if cmake_missing:
+ _LOG.warning('The CMake build is missing %d files', len(cmake_missing))
+ _LOG.warning('Files missing from CMake:\n%s',
+ '\n'.join(str(f) for f in cmake_missing))
+ # TODO(hepler): Many files are missing from the CMake build. Make this
+ # check an error when the missing files are fixed.
+ # raise PresubmitFailure
+
def build_env_setup(ctx: PresubmitContext):
if 'PW_CARGO_SETUP' not in os.environ:
