pw_presubmit: Add nanopb presubmit checks
Change-Id: Ib72e4b31402bd754fb09d95740ebbcf87d99a554
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/22226
Reviewed-by: Wyatt Hepler <hepler@google.com>
Commit-Queue: Rob Mohr <mohrr@google.com>
diff --git a/pw_package/py/pw_package/package_manager.py b/pw_package/py/pw_package/package_manager.py
index dd20b44..dd89dba 100644
--- a/pw_package/py/pw_package/package_manager.py
+++ b/pw_package/py/pw_package/package_manager.py
@@ -14,11 +14,12 @@
"""Install and remove optional packages."""
import argparse
+import dataclasses
import logging
import os
import pathlib
import shutil
-from typing import List
+from typing import Dict, List, Tuple
_LOG: logging.Logger = logging.getLogger(__name__)
@@ -59,7 +60,7 @@
"""
-_PACKAGES = {}
+_PACKAGES: Dict[str, Package] = {}
def register(package_class: type) -> None:
@@ -67,58 +68,93 @@
_PACKAGES[obj.name] = obj
+@dataclasses.dataclass
+class Packages:
+ all: Tuple[str, ...]
+ installed: Tuple[str, ...]
+ available: Tuple[str, ...]
+
+
class PackageManager:
"""Install and remove optional packages."""
- def __init__(self):
- self._pkg_root: pathlib.Path = None
+ def __init__(self, root: pathlib.Path):
+ self._pkg_root = root
+ os.makedirs(root, exist_ok=True)
- def install(self, package: str, force=False):
+ def install(self, package: str, force: bool = False) -> None:
pkg = _PACKAGES[package]
if force:
self.remove(package)
- _LOG.info('Installing %s...', pkg.name)
pkg.install(self._pkg_root / pkg.name)
- _LOG.info('Installing %s...done.', pkg.name)
- return 0
- def remove(self, package: str): # pylint: disable=no-self-use
+ def remove(self, package: str) -> None:
pkg = _PACKAGES[package]
- _LOG.info('Removing %s...', pkg.name)
pkg.remove(self._pkg_root / pkg.name)
- _LOG.info('Removing %s...done.', pkg.name)
- return 0
- def status(self, package: str): # pylint: disable=no-self-use
+ def status(self, package: str) -> bool:
pkg = _PACKAGES[package]
path = self._pkg_root / pkg.name
- if os.path.isdir(path) and pkg.status(path):
- _LOG.info('%s is installed.', pkg.name)
- return 0
+ return os.path.isdir(path) and pkg.status(path)
- _LOG.info('%s is not installed.', pkg.name)
- return -1
-
- def list(self): # pylint: disable=no-self-use
- _LOG.info('Installed packages:')
+ def list(self) -> Packages:
+ installed = []
available = []
for package in sorted(_PACKAGES.keys()):
pkg = _PACKAGES[package]
if pkg.status(self._pkg_root / pkg.name):
- _LOG.info(' %s', pkg.name)
+ installed.append(pkg.name)
else:
available.append(pkg.name)
+
+ return Packages(
+ all=tuple(_PACKAGES.keys()),
+ installed=tuple(installed),
+ available=tuple(available),
+ )
+
+
+class PackageManagerCLI:
+ """Command-line interface to PackageManager."""
+ def __init__(self):
+ self._mgr: PackageManager = None
+
+ def install(self, package: str, force: bool = False) -> int:
+ _LOG.info('Installing %s...', package)
+ self._mgr.install(package, force)
+ _LOG.info('Installing %s...done.', package)
+ return 0
+
+ def remove(self, package: str) -> int:
+ _LOG.info('Removing %s...', package)
+ self._mgr.remove(package)
+ _LOG.info('Removing %s...done.', package)
+ return 0
+
+ def status(self, package: str) -> int:
+ if self._mgr.status(package):
+ _LOG.info('%s is installed.', package)
+ return 0
+
+ _LOG.info('%s is not installed.', package)
+ return -1
+
+ def list(self) -> int:
+ packages = self._mgr.list()
+
+ _LOG.info('Installed packages:')
+ for package in packages.installed:
+ _LOG.info(' %s', package)
_LOG.info('')
_LOG.info('Available packages:')
- for pkg_name in available:
- _LOG.info(' %s', pkg_name)
+ for package in packages.available:
+ _LOG.info(' %s', package)
_LOG.info('')
return 0
- def run(self, command: str, pkg_root: pathlib.Path, **kwargs):
- os.makedirs(pkg_root, exist_ok=True)
- self._pkg_root = pkg_root
+ def run(self, command: str, pkg_root: pathlib.Path, **kwargs) -> int:
+ self._mgr = PackageManager(pkg_root)
return getattr(self, command)(**kwargs)
@@ -144,4 +180,4 @@
def run(**kwargs):
- return PackageManager().run(**kwargs)
+ return PackageManagerCLI().run(**kwargs)
diff --git a/pw_package/py/pw_package/py.typed b/pw_package/py/pw_package/py.typed
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/pw_package/py/pw_package/py.typed
diff --git a/pw_presubmit/py/BUILD.gn b/pw_presubmit/py/BUILD.gn
index 4109256..a9b96e7 100644
--- a/pw_presubmit/py/BUILD.gn
+++ b/pw_presubmit/py/BUILD.gn
@@ -35,5 +35,8 @@
"presubmit_test.py",
"tools_test.py",
]
- python_deps = [ "$dir_pw_cli/py" ]
+ python_deps = [
+ "$dir_pw_cli/py",
+ "$dir_pw_package/py",
+ ]
}
diff --git a/pw_presubmit/py/pw_presubmit/build.py b/pw_presubmit/py/pw_presubmit/build.py
index c08ef3d..72d2aab 100644
--- a/pw_presubmit/py/pw_presubmit/build.py
+++ b/pw_presubmit/py/pw_presubmit/build.py
@@ -20,11 +20,25 @@
import re
from typing import Container, Dict, Iterable, List, Mapping, Set, Tuple
+from pw_package import package_manager
from pw_presubmit import call, log_run, plural, PresubmitFailure, tools
_LOG = logging.getLogger(__name__)
+def install_package(root: Path, name: str) -> None:
+ """Install package with given name in given path."""
+ mgr = package_manager.PackageManager(root)
+
+ if not mgr.list():
+ raise PresubmitFailure(
+ 'no packages configured, please import your pw_package '
+ 'configuration module')
+
+ if not mgr.status(name):
+ mgr.install(name)
+
+
def gn_args(**kwargs) -> str:
"""Builds a string to use for the --args argument to gn gen."""
return '--args=' + ' '.join(f'{arg}={val}' for arg, val in kwargs.items())
diff --git a/pw_presubmit/py/pw_presubmit/cli.py b/pw_presubmit/py/pw_presubmit/cli.py
index de4f956..b43fc69 100644
--- a/pw_presubmit/py/pw_presubmit/cli.py
+++ b/pw_presubmit/py/pw_presubmit/cli.py
@@ -108,6 +108,11 @@
type=Path,
help='Output directory (default: <repo root>/.presubmit)',
)
+ parser.add_argument(
+ '--package-root',
+ type=Path,
+ help='Package root directory (default: <output directory>/packages)',
+ )
exclusive = parser.add_mutually_exclusive_group()
exclusive.add_argument(
@@ -127,6 +132,7 @@
def run(
program: Sequence[Callable],
output_directory: Path,
+ package_root: Path,
clear: bool,
root: Path = None,
repositories: Collection[Path] = (),
@@ -141,6 +147,7 @@
defaults to the root of the current directory's repository
program: from the --program option
output_directory: from --output-directory option
+ package_root: from --package-root option
clear: from the --clear option
**other_args: remaining arguments defined by by add_arguments
@@ -156,6 +163,9 @@
if not output_directory:
output_directory = root / '.presubmit'
+ if not package_root:
+ package_root = output_directory / 'packages'
+
_LOG.debug('Using environment at %s', output_directory)
if clear:
@@ -171,6 +181,7 @@
root,
repositories,
output_directory=output_directory,
+ package_root=package_root,
**other_args):
return 0
diff --git a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
index 28a2896..6396dc0 100755
--- a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
+++ b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
@@ -97,6 +97,21 @@
@filter_paths(endswith=_BUILD_EXTENSIONS)
+def gn_nanopb_build(ctx: PresubmitContext):
+ build.install_package(ctx.package_root, 'nanopb')
+ build.gn_gen(ctx.root,
+ ctx.output_dir,
+ dir_pw_third_party_nanopb='"{}"'.format(ctx.package_root /
+ 'nanopb'),
+ pw_protobuf_GENERATORS='["nanopb", "nanopb_rpc", "pwpb"]')
+ build.ninja(
+ ctx.output_dir,
+ *_at_all_optimization_levels('stm32f429i'),
+ *_at_all_optimization_levels('host_clang'),
+ )
+
+
+@filter_paths(endswith=_BUILD_EXTENSIONS)
def gn_qemu_build(ctx: PresubmitContext):
build.gn_gen(ctx.root, ctx.output_dir)
build.ninja(ctx.output_dir, *_at_all_optimization_levels('qemu'))
@@ -445,6 +460,7 @@
# failing.
oss_fuzz_build,
bazel_test,
+ gn_nanopb_build,
)
QUICK = (
diff --git a/pw_presubmit/py/pw_presubmit/presubmit.py b/pw_presubmit/py/pw_presubmit/presubmit.py
index 51ef1f3..a572a9a 100644
--- a/pw_presubmit/py/pw_presubmit/presubmit.py
+++ b/pw_presubmit/py/pw_presubmit/presubmit.py
@@ -177,6 +177,7 @@
repos: Tuple[Path, ...]
output_dir: Path
paths: Tuple[Path, ...]
+ package_root: Path
def relative_paths(self, start: Optional[Path] = None) -> Tuple[Path, ...]:
return tuple(
@@ -203,13 +204,15 @@
class Presubmit:
"""Runs a series of presubmit checks on a list of files."""
def __init__(self, root: Path, repos: Sequence[Path],
- output_directory: Path, paths: Sequence[Path]):
+ output_directory: Path, paths: Sequence[Path],
+ package_root: Path):
self._root = root.resolve()
self._repos = tuple(repos)
self._output_directory = output_directory.resolve()
self._paths = tuple(paths)
self._relative_paths = tuple(
tools.relative_paths(self._paths, self._root))
+ self._package_root = package_root.resolve()
def run(self, program: Program, keep_going: bool = False) -> bool:
"""Executes a series of presubmit checks on the paths."""
@@ -317,6 +320,7 @@
repos=self._repos,
output_dir=output_directory,
paths=paths,
+ package_root=self._package_root,
)
finally:
@@ -381,6 +385,7 @@
paths: Sequence[str] = (),
exclude: Sequence[Pattern] = (),
output_directory: Optional[Path] = None,
+ package_root: Path = None,
keep_going: bool = False) -> bool:
"""Lists files in the current Git repo and runs a Presubmit with them.
@@ -403,6 +408,7 @@
paths: optional list of Git pathspecs to run the checks against
exclude: regular expressions for Posix-style paths to exclude
output_directory: where to place output files
+ package_root: where to place package files
keep_going: whether to continue running checks if an error occurs
Returns:
@@ -430,7 +436,16 @@
if output_directory is None:
output_directory = root / '.presubmit'
- presubmit = Presubmit(root, repos, output_directory, files)
+ if package_root is None:
+ package_root = output_directory / 'packages'
+
+ presubmit = Presubmit(
+ root=root,
+ repos=repos,
+ output_directory=output_directory,
+ paths=files,
+ package_root=package_root,
+ )
if not isinstance(program, Program):
program = Program('', program)
diff --git a/pw_presubmit/py/setup.py b/pw_presubmit/py/setup.py
index 3f97b96..8539057 100644
--- a/pw_presubmit/py/setup.py
+++ b/pw_presubmit/py/setup.py
@@ -25,6 +25,7 @@
'mypy==0.782',
'pylint==2.5.3',
'yapf==0.30.0',
+ 'pw_package',
],
packages=setuptools.find_packages(),
package_data={'pw_presubmit': ['py.typed']},
