pw_presubmit: Add nanopb presubmit checks Change-Id: Ib72e4b31402bd754fb09d95740ebbcf87d99a554 Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/22226 Reviewed-by: Wyatt Hepler <hepler@google.com> Commit-Queue: Rob Mohr <mohrr@google.com>
diff --git a/pw_package/py/pw_package/package_manager.py b/pw_package/py/pw_package/package_manager.py index dd20b44..dd89dba 100644 --- a/pw_package/py/pw_package/package_manager.py +++ b/pw_package/py/pw_package/package_manager.py
@@ -14,11 +14,12 @@ """Install and remove optional packages.""" import argparse +import dataclasses import logging import os import pathlib import shutil -from typing import List +from typing import Dict, List, Tuple _LOG: logging.Logger = logging.getLogger(__name__) @@ -59,7 +60,7 @@ """ -_PACKAGES = {} +_PACKAGES: Dict[str, Package] = {} def register(package_class: type) -> None: @@ -67,58 +68,93 @@ _PACKAGES[obj.name] = obj +@dataclasses.dataclass +class Packages: + all: Tuple[str, ...] + installed: Tuple[str, ...] + available: Tuple[str, ...] + + class PackageManager: """Install and remove optional packages.""" - def __init__(self): - self._pkg_root: pathlib.Path = None + def __init__(self, root: pathlib.Path): + self._pkg_root = root + os.makedirs(root, exist_ok=True) - def install(self, package: str, force=False): + def install(self, package: str, force: bool = False) -> None: pkg = _PACKAGES[package] if force: self.remove(package) - _LOG.info('Installing %s...', pkg.name) pkg.install(self._pkg_root / pkg.name) - _LOG.info('Installing %s...done.', pkg.name) - return 0 - def remove(self, package: str): # pylint: disable=no-self-use + def remove(self, package: str) -> None: pkg = _PACKAGES[package] - _LOG.info('Removing %s...', pkg.name) pkg.remove(self._pkg_root / pkg.name) - _LOG.info('Removing %s...done.', pkg.name) - return 0 - def status(self, package: str): # pylint: disable=no-self-use + def status(self, package: str) -> bool: pkg = _PACKAGES[package] path = self._pkg_root / pkg.name - if os.path.isdir(path) and pkg.status(path): - _LOG.info('%s is installed.', pkg.name) - return 0 + return os.path.isdir(path) and pkg.status(path) - _LOG.info('%s is not installed.', pkg.name) - return -1 - - def list(self): # pylint: disable=no-self-use - _LOG.info('Installed packages:') + def list(self) -> Packages: + installed = [] available = [] for package in sorted(_PACKAGES.keys()): pkg = _PACKAGES[package] if pkg.status(self._pkg_root / pkg.name): - _LOG.info(' %s', pkg.name) + installed.append(pkg.name) else: available.append(pkg.name) + + return Packages( + all=tuple(_PACKAGES.keys()), + installed=tuple(installed), + available=tuple(available), + ) + + +class PackageManagerCLI: + """Command-line interface to PackageManager.""" + def __init__(self): + self._mgr: PackageManager = None + + def install(self, package: str, force: bool = False) -> int: + _LOG.info('Installing %s...', package) + self._mgr.install(package, force) + _LOG.info('Installing %s...done.', package) + return 0 + + def remove(self, package: str) -> int: + _LOG.info('Removing %s...', package) + self._mgr.remove(package) + _LOG.info('Removing %s...done.', package) + return 0 + + def status(self, package: str) -> int: + if self._mgr.status(package): + _LOG.info('%s is installed.', package) + return 0 + + _LOG.info('%s is not installed.', package) + return -1 + + def list(self) -> int: + packages = self._mgr.list() + + _LOG.info('Installed packages:') + for package in packages.installed: + _LOG.info(' %s', package) _LOG.info('') _LOG.info('Available packages:') - for pkg_name in available: - _LOG.info(' %s', pkg_name) + for package in packages.available: + _LOG.info(' %s', package) _LOG.info('') return 0 - def run(self, command: str, pkg_root: pathlib.Path, **kwargs): - os.makedirs(pkg_root, exist_ok=True) - self._pkg_root = pkg_root + def run(self, command: str, pkg_root: pathlib.Path, **kwargs) -> int: + self._mgr = PackageManager(pkg_root) return getattr(self, command)(**kwargs) @@ -144,4 +180,4 @@ def run(**kwargs): - return PackageManager().run(**kwargs) + return PackageManagerCLI().run(**kwargs)
diff --git a/pw_package/py/pw_package/py.typed b/pw_package/py/pw_package/py.typed new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pw_package/py/pw_package/py.typed
diff --git a/pw_presubmit/py/BUILD.gn b/pw_presubmit/py/BUILD.gn index 4109256..a9b96e7 100644 --- a/pw_presubmit/py/BUILD.gn +++ b/pw_presubmit/py/BUILD.gn
@@ -35,5 +35,8 @@ "presubmit_test.py", "tools_test.py", ] - python_deps = [ "$dir_pw_cli/py" ] + python_deps = [ + "$dir_pw_cli/py", + "$dir_pw_package/py", + ] }
diff --git a/pw_presubmit/py/pw_presubmit/build.py b/pw_presubmit/py/pw_presubmit/build.py index c08ef3d..72d2aab 100644 --- a/pw_presubmit/py/pw_presubmit/build.py +++ b/pw_presubmit/py/pw_presubmit/build.py
@@ -20,11 +20,25 @@ import re from typing import Container, Dict, Iterable, List, Mapping, Set, Tuple +from pw_package import package_manager from pw_presubmit import call, log_run, plural, PresubmitFailure, tools _LOG = logging.getLogger(__name__) +def install_package(root: Path, name: str) -> None: + """Install package with given name in given path.""" + mgr = package_manager.PackageManager(root) + + if not mgr.list(): + raise PresubmitFailure( + 'no packages configured, please import your pw_package ' + 'configuration module') + + if not mgr.status(name): + mgr.install(name) + + def gn_args(**kwargs) -> str: """Builds a string to use for the --args argument to gn gen.""" return '--args=' + ' '.join(f'{arg}={val}' for arg, val in kwargs.items())
diff --git a/pw_presubmit/py/pw_presubmit/cli.py b/pw_presubmit/py/pw_presubmit/cli.py index de4f956..b43fc69 100644 --- a/pw_presubmit/py/pw_presubmit/cli.py +++ b/pw_presubmit/py/pw_presubmit/cli.py
@@ -108,6 +108,11 @@ type=Path, help='Output directory (default: <repo root>/.presubmit)', ) + parser.add_argument( + '--package-root', + type=Path, + help='Package root directory (default: <output directory>/packages)', + ) exclusive = parser.add_mutually_exclusive_group() exclusive.add_argument( @@ -127,6 +132,7 @@ def run( program: Sequence[Callable], output_directory: Path, + package_root: Path, clear: bool, root: Path = None, repositories: Collection[Path] = (), @@ -141,6 +147,7 @@ defaults to the root of the current directory's repository program: from the --program option output_directory: from --output-directory option + package_root: from --package-root option clear: from the --clear option **other_args: remaining arguments defined by by add_arguments @@ -156,6 +163,9 @@ if not output_directory: output_directory = root / '.presubmit' + if not package_root: + package_root = output_directory / 'packages' + _LOG.debug('Using environment at %s', output_directory) if clear: @@ -171,6 +181,7 @@ root, repositories, output_directory=output_directory, + package_root=package_root, **other_args): return 0
diff --git a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py index 28a2896..6396dc0 100755 --- a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py +++ b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
@@ -97,6 +97,21 @@ @filter_paths(endswith=_BUILD_EXTENSIONS) +def gn_nanopb_build(ctx: PresubmitContext): + build.install_package(ctx.package_root, 'nanopb') + build.gn_gen(ctx.root, + ctx.output_dir, + dir_pw_third_party_nanopb='"{}"'.format(ctx.package_root / + 'nanopb'), + pw_protobuf_GENERATORS='["nanopb", "nanopb_rpc", "pwpb"]') + build.ninja( + ctx.output_dir, + *_at_all_optimization_levels('stm32f429i'), + *_at_all_optimization_levels('host_clang'), + ) + + +@filter_paths(endswith=_BUILD_EXTENSIONS) def gn_qemu_build(ctx: PresubmitContext): build.gn_gen(ctx.root, ctx.output_dir) build.ninja(ctx.output_dir, *_at_all_optimization_levels('qemu')) @@ -445,6 +460,7 @@ # failing. oss_fuzz_build, bazel_test, + gn_nanopb_build, ) QUICK = (
diff --git a/pw_presubmit/py/pw_presubmit/presubmit.py b/pw_presubmit/py/pw_presubmit/presubmit.py index 51ef1f3..a572a9a 100644 --- a/pw_presubmit/py/pw_presubmit/presubmit.py +++ b/pw_presubmit/py/pw_presubmit/presubmit.py
@@ -177,6 +177,7 @@ repos: Tuple[Path, ...] output_dir: Path paths: Tuple[Path, ...] + package_root: Path def relative_paths(self, start: Optional[Path] = None) -> Tuple[Path, ...]: return tuple( @@ -203,13 +204,15 @@ class Presubmit: """Runs a series of presubmit checks on a list of files.""" def __init__(self, root: Path, repos: Sequence[Path], - output_directory: Path, paths: Sequence[Path]): + output_directory: Path, paths: Sequence[Path], + package_root: Path): self._root = root.resolve() self._repos = tuple(repos) self._output_directory = output_directory.resolve() self._paths = tuple(paths) self._relative_paths = tuple( tools.relative_paths(self._paths, self._root)) + self._package_root = package_root.resolve() def run(self, program: Program, keep_going: bool = False) -> bool: """Executes a series of presubmit checks on the paths.""" @@ -317,6 +320,7 @@ repos=self._repos, output_dir=output_directory, paths=paths, + package_root=self._package_root, ) finally: @@ -381,6 +385,7 @@ paths: Sequence[str] = (), exclude: Sequence[Pattern] = (), output_directory: Optional[Path] = None, + package_root: Path = None, keep_going: bool = False) -> bool: """Lists files in the current Git repo and runs a Presubmit with them. @@ -403,6 +408,7 @@ paths: optional list of Git pathspecs to run the checks against exclude: regular expressions for Posix-style paths to exclude output_directory: where to place output files + package_root: where to place package files keep_going: whether to continue running checks if an error occurs Returns: @@ -430,7 +436,16 @@ if output_directory is None: output_directory = root / '.presubmit' - presubmit = Presubmit(root, repos, output_directory, files) + if package_root is None: + package_root = output_directory / 'packages' + + presubmit = Presubmit( + root=root, + repos=repos, + output_directory=output_directory, + paths=files, + package_root=package_root, + ) if not isinstance(program, Program): program = Program('', program)
diff --git a/pw_presubmit/py/setup.py b/pw_presubmit/py/setup.py index 3f97b96..8539057 100644 --- a/pw_presubmit/py/setup.py +++ b/pw_presubmit/py/setup.py
@@ -25,6 +25,7 @@ 'mypy==0.782', 'pylint==2.5.3', 'yapf==0.30.0', + 'pw_package', ], packages=setuptools.find_packages(), package_data={'pw_presubmit': ['py.typed']},