pw_build: Python API
- Improvements to the BuildCommand and BuildRecipe APIs
- Allow --build-system-commands for pw watch and pw build commands
to be chained.
- Added logfile related options:
--logfile FILE to log build output
--separate-logfiles option to break out build output to a separate
log file per out directory.
- run_builds() function to execute a collection of BuildRecipes
- run_watch() function to pw-watch a collection of BuildRecipes
- Improvements to the print_build_summary output. Pending builds are
now shown with slug '...'.
- Hookup -j, --keep-going, and --colors options for bazel
Change-Id: I9b856d4324b3851b8e0071626992de8c2f71e2bd
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/124990
Pigweed-Auto-Submit: Anthony DiGirolamo <tonymd@google.com>
Commit-Queue: Auto-Submit <auto-submit@pigweed.google.com.iam.gserviceaccount.com>
Reviewed-by: Chad Norvell <chadnorvell@google.com>
diff --git a/pw_build/py/build_recipe_test.py b/pw_build/py/build_recipe_test.py
index 4b5e7a5..6361da5 100644
--- a/pw_build/py/build_recipe_test.py
+++ b/pw_build/py/build_recipe_test.py
@@ -14,6 +14,7 @@
"""Tests for pw_watch.build_recipe"""
from pathlib import Path
+import shlex
import unittest
from parameterized import parameterized # type: ignore
@@ -63,7 +64,7 @@
'cmake shell command',
BuildCommand(
build_dir=Path('outcmake'),
- command_string='cmake -G Ninja -S ./ -B outcmake',
+ command=shlex.split('cmake -G Ninja -S ./ -B outcmake'),
),
# result
['cmake', '-G', 'Ninja', '-S', './', '-B', 'outcmake'],
@@ -72,7 +73,7 @@
'gn shell command',
BuildCommand(
build_dir=Path('out'),
- command_string='gn gen out --export-compile-commands',
+ command=shlex.split('gn gen out --export-compile-commands'),
),
# result
['gn', 'gen', 'out', '--export-compile-commands'],
@@ -81,11 +82,22 @@
'python shell command',
BuildCommand(
build_dir=Path('outpytest'),
- command_string='python pw_build/py/build_recipe_test.py',
+ command=shlex.split(
+ 'python pw_build/py/build_recipe_test.py'
+ ),
),
# result
['python', 'pw_build/py/build_recipe_test.py'],
),
+ (
+ 'gn shell command with a list',
+ BuildCommand(
+ build_dir=Path('out'),
+ command=['gn', 'gen', 'out', '--export-compile-commands'],
+ ),
+ # result
+ ['gn', 'gen', 'out', '--export-compile-commands'],
+ ),
]
)
def test_build_command_get_args(
diff --git a/pw_build/py/project_builder_prefs_test.py b/pw_build/py/project_builder_prefs_test.py
index 1944ad4..9ede88f 100644
--- a/pw_build/py/project_builder_prefs_test.py
+++ b/pw_build/py/project_builder_prefs_test.py
@@ -85,7 +85,10 @@
changed_args = {
'jobs': 8,
'colors': False,
- 'build_system_commands': [['default', 'bazel']],
+ 'build_system_commands': [
+ ['out', 'bazel build'],
+ ['out', 'bazel test'],
+ ],
}
args_dict.update(changed_args)
@@ -94,7 +97,13 @@
# apply_command_line_args modifies build_system_commands to match the
# prefs dict format.
changed_args['build_system_commands'] = {
- 'default': {'command': 'bazel'}
+ 'default': {'commands': [{'command': 'ninja', 'extra_args': []}]},
+ 'out': {
+ 'commands': [
+ {'command': 'bazel', 'extra_args': ['build']},
+ {'command': 'bazel', 'extra_args': ['test']},
+ ],
+ },
}
# Check that only args changed from their defaults are applied.
diff --git a/pw_build/py/pw_build/build_recipe.py b/pw_build/py/pw_build/build_recipe.py
index 6ed1af7..3387afb 100644
--- a/pw_build/py/pw_build/build_recipe.py
+++ b/pw_build/py/pw_build/build_recipe.py
@@ -15,9 +15,10 @@
"""Watch build config dataclasses."""
from dataclasses import dataclass, field
+import logging
from pathlib import Path
import shlex
-from typing import List, Optional, TYPE_CHECKING
+from typing import Callable, List, Optional, TYPE_CHECKING
if TYPE_CHECKING:
from pw_build.project_builder_prefs import ProjectBuilderPrefs
@@ -27,28 +28,91 @@
"""Exception for requesting unsupported build systems."""
+class UnknownBuildDir(Exception):
+ """Exception for an unknown build dir before command running."""
+
+
@dataclass
class BuildCommand:
- """Store details of a single build step."""
+ """Store details of a single build step.
- build_dir: Path
+ Example usage:
+
+ .. code-block:: python
+
+ from pw_build.build_recipe import BuildCommand, BuildRecipe
+
+ def should_gen_gn(out: Path):
+ return not (out / 'build.ninja').is_file()
+
+ cmd1 = BuildCommand(build_dir='out',
+ command=['gn', 'gen', '{build_dir}'],
+ run_if=should_gen_gn)
+
+ cmd2 = BuildCommand(build_dir='out',
+ build_system_command='ninja',
+ build_system_extra_args=['-k', '0'],
+ targets=['default']),
+
+ Args:
+ build_dir: Output directory for this build command. This can be omitted
+ if the BuildCommand is included in the steps of a BuildRecipe.
+ build_system_command: This command should end with ``ninja``, ``make``,
+ or ``bazel``.
+ build_system_extra_args: A list of extra arguments passed to the
+ build_system_command. If running ``bazel test`` include ``test`` as
+ an extra arg here.
+ targets: Optional list of targets to build in the build_dir.
+ command: List of strings to run as a command. These are passed to
+ subprocess.run(). Any instances of the ``'{build_dir}'`` string
+ literal will be replaced at run time with the out directory.
+ run_if: A callable function to run before executing this
+ BuildCommand. The callable takes one Path arg for the build_dir. If
+ the callable returns true this command is executed. All
+ BuildCommands are run by default.
+ """
+
+ build_dir: Optional[Path] = None
build_system_command: Optional[str] = None
build_system_extra_args: List[str] = field(default_factory=list)
targets: List[str] = field(default_factory=list)
- command_string: str = ''
+ command: List[str] = field(default_factory=list)
+ run_if: Callable[[Path], bool] = lambda _build_dir: True
def __post_init__(self) -> None:
- self._expanded_args: List[str] = shlex.split(self.command_string)
+ # Copy self._expanded_args from the command list.
+ self._expanded_args: List[str] = []
+ if self.command:
+ self._expanded_args = self.command
+
+ def should_run(self) -> bool:
+ """Return True if this build command should be run."""
+ if self.build_dir:
+ return self.run_if(self.build_dir)
+ return True
+
+ def _get_starting_build_system_args(self) -> List[str]:
+ """Return flags that appear immediately after the build command."""
+ assert self.build_system_command
+ assert self.build_dir
+ if self.build_system_command.endswith('bazel'):
+ return ['--output_base', str(self.build_dir)]
+ return []
def _get_build_system_args(self) -> List[str]:
assert self.build_system_command
+ assert self.build_dir
+
+ # Both make and ninja use -C for a build directory.
if self.build_system_command.endswith(
'make'
) or self.build_system_command.endswith('ninja'):
return ['-C', str(self.build_dir), *self.targets]
+ # Bazel relies on --output_base which is handled by the
+ # _get_starting_build_system_args() function.
if self.build_system_command.endswith('bazel'):
- return ['--output_base', str(self.build_dir), *self.targets]
+ return [*self.targets]
raise UnknownBuildSystem(
f'\n\nUnknown build system command "{self.build_system_command}" '
@@ -56,34 +120,174 @@
'Supported commands: ninja, bazel, make'
)
+ def _resolve_expanded_args(self) -> List[str]:
+ """Replace instances of '{build_dir}' with the self.build_dir."""
+ resolved_args = []
+ for arg in self._expanded_args:
+ if arg == "{build_dir}":
+ if not self.build_dir:
+ raise UnknownBuildDir(
+ '\n\nUnknown "{build_dir}" value for command:\n'
+ f' {self._expanded_args}\n'
+ f'In BuildCommand: {repr(self)}\n\n'
+ 'Check build_dir is set for the above BuildCommand'
+ 'or included as a step to a BuildRecipe.'
+ )
+ resolved_args.append(str(self.build_dir.resolve()))
+ else:
+ resolved_args.append(arg)
+ return resolved_args
+
+ def ninja_command(self) -> bool:
+ if self.build_system_command and self.build_system_command.endswith(
+ 'ninja'
+ ):
+ return True
+ return False
+
+ def bazel_command(self) -> bool:
+ if self.build_system_command and self.build_system_command.endswith(
+ 'bazel'
+ ):
+ return True
+ return False
+
+ def bazel_build_command(self) -> bool:
+ if self.bazel_command() and 'build' in self.build_system_extra_args:
+ return True
+ return False
+
def get_args(
self,
- additional_build_args: Optional[List[str]] = None,
+ additional_ninja_args: Optional[List[str]] = None,
+ additional_bazel_args: Optional[List[str]] = None,
+ additional_bazel_build_args: Optional[List[str]] = None,
) -> List[str]:
- if self.build_system_command:
- extra_args = []
- extra_args.extend(self.build_system_extra_args)
- if additional_build_args:
- extra_args.extend(additional_build_args)
- command = [
- self.build_system_command,
- *extra_args,
- *self._get_build_system_args(),
- ]
- return command
- return self._expanded_args
+ # If this is a plain command step, return self._expanded_args as-is.
+ if not self.build_system_command:
+ return self._resolve_expanded_args()
+
+ # Assmemble user-defined extra args.
+ extra_args = []
+ extra_args.extend(self.build_system_extra_args)
+ if additional_ninja_args and self.ninja_command():
+ extra_args.extend(additional_ninja_args)
+
+ if additional_bazel_build_args and self.bazel_build_command():
+ extra_args.extend(additional_bazel_build_args)
+
+ if additional_bazel_args and self.bazel_command():
+ extra_args.extend(additional_bazel_args)
+
+ # Construct the build system command args.
+ command = [
+ self.build_system_command,
+ *self._get_starting_build_system_args(),
+ *extra_args,
+ *self._get_build_system_args(),
+ ]
+ return command
def __str__(self) -> str:
return ' '.join(shlex.quote(arg) for arg in self.get_args())
@dataclass
+class BuildRecipeStatus:
+ current_step: str = ''
+ percent: float = 0.0
+ error_count: int = 0
+ return_code: Optional[int] = None
+
+ def pending(self) -> bool:
+ return self.return_code is None
+
+ def failed(self) -> bool:
+ if self.return_code is not None:
+ return self.return_code != 0
+ return False
+
+ def passed(self) -> bool:
+ if self.return_code is not None:
+ return self.return_code == 0
+ return False
+
+
+@dataclass
class BuildRecipe:
+ """Dataclass to store a list of BuildCommands.
+
+ Example usage:
+
+ .. code-block:: python
+
+ from pw_build.build_recipe import BuildCommand, BuildRecipe
+
+ def should_gen_gn(out: Path) -> bool:
+ return not (out / 'build.ninja').is_file()
+
+ recipe = BuildRecipe(
+ build_dir='out',
+ title='Vanilla Ninja Build',
+ steps=[
+ BuildCommand(command=['gn', 'gen', '{build_dir}'],
+ run_if=should_gen_gn),
+ BuildCommand(build_system_command='ninja',
+ build_system_extra_args=['-k', '0'],
+ targets=['default']),
+ ],
+ )
+
+ Args:
+ build_dir: Output directory for this BuildRecipe. On init this out dir
+ is set for all included steps.
+ steps: List of BuildCommands to run.
+ title: Custom title. The build_dir is used if this is ommited.
+ """
+
build_dir: Path
- steps: List[BuildCommand]
- build_system_command: Optional[str] = None
+ steps: List[BuildCommand] = field(default_factory=list)
title: Optional[str] = None
+ def __post_init__(self) -> None:
+ # Update all included steps to use this recipe's build_dir.
+ for step in self.steps:
+ if self.build_dir:
+ step.build_dir = self.build_dir
+
+ # Set logging variables
+ self._logger: Optional[logging.Logger] = None
+ self._logfile: Optional[Path] = None
+ self._status: BuildRecipeStatus = BuildRecipeStatus()
+
+ def set_targets(self, new_targets: List[str]) -> None:
+ """Reset all build step targets."""
+ for step in self.steps:
+ step.targets = new_targets
+
+ def set_logger(self, logger: logging.Logger) -> None:
+ self._logger = logger
+
+ def set_logfile(self, log_file: Path) -> None:
+ self._logfile = log_file
+
+ def reset_status(self) -> None:
+ self._status = BuildRecipeStatus()
+
+ @property
+ def status(self) -> BuildRecipeStatus:
+ return self._status
+
+ @property
+ def log(self) -> logging.Logger:
+ if self._logger:
+ return self._logger
+ return logging.getLogger()
+
+ @property
+ def logfile(self) -> Optional[Path]:
+ return self._logfile
+
@property
def display_name(self) -> str:
if self.title:
@@ -96,10 +300,11 @@
)
def __str__(self) -> str:
- message = f"{self.display_name}"
+ message = self.display_name
targets = self.targets()
if targets:
- message = '{} ({})'.format(message, ' '.join(self.targets()))
+ target_list = ' '.join(self.targets())
+ message = f'{message} -- {target_list}'
return message
@@ -112,9 +317,7 @@
build_recipes.append(
BuildRecipe(
build_dir=Path.cwd(),
- steps=[
- BuildCommand(Path.cwd(), command_string=command_str)
- ],
+ steps=[BuildCommand(command=shlex.split(command_str))],
title=command_str,
)
)
@@ -122,27 +325,25 @@
for build_dir, targets in prefs.build_directories.items():
steps: List[BuildCommand] = []
build_path = Path(build_dir)
- (
- build_system_command,
- build_system_extra_args,
- ) = prefs.build_system_commands(build_dir)
-
if not targets:
targets = []
- steps.append(
- BuildCommand(
- build_dir=build_path,
- build_system_command=build_system_command,
- build_system_extra_args=build_system_extra_args,
- targets=targets,
+
+ for (
+ build_system_command,
+ build_system_extra_args,
+ ) in prefs.build_system_commands(build_dir):
+ steps.append(
+ BuildCommand(
+ build_system_command=build_system_command,
+ build_system_extra_args=build_system_extra_args,
+ targets=targets,
+ )
)
- )
build_recipes.append(
BuildRecipe(
build_dir=build_path,
steps=steps,
- build_system_command=build_system_command,
)
)
diff --git a/pw_build/py/pw_build/project_builder.py b/pw_build/py/pw_build/project_builder.py
index 0068321..6b91756 100644
--- a/pw_build/py/pw_build/project_builder.py
+++ b/pw_build/py/pw_build/project_builder.py
@@ -13,21 +13,36 @@
# the License.
"""Build a Pigweed Project.
+Run arbitrary commands or invoke build systems (Ninja, Bazel and make) on one or
+more build directories.
+
Usage examples:
+ # Build the default target in out/ using ninja.
+ python -m pw_build.project_builder -C out
+
# Build pw_run_tests.modules in the out/cmake directory
- pw build -C out/cmake pw_run_tests.modules
+ python -m pw_build.project_builder -C out/cmake pw_run_tests.modules
# Build the default target in out/ and pw_apps in out/cmake
- pw build -C out -C out/cmake pw_apps
+ python -m pw_build.project_builder -C out -C out/cmake pw_apps
- # Find a directory and build python.tests, and build pw_apps in out/cmake
- pw build python.tests -C out/cmake pw_apps
+ # Build python.tests in out/ and pw_apps in out/cmake/
+ python -m pw_build.project_builder python.tests -C out/cmake pw_apps
+
+ # Run 'bazel build' and 'bazel test' on the target '//...' in outbazel/
+ python -m pw_build.project_builder --run-command 'mkdir -p outbazel'
+ -C outbazel '//...'
+ --build-system-command outbazel 'bazel build'
+ --build-system-command outbazel 'bazel test'
"""
import argparse
+import concurrent.futures
import os
import logging
+from pathlib import Path
+import re
import shlex
import sys
import subprocess
@@ -42,11 +57,11 @@
NamedTuple,
)
+
import pw_cli.log
import pw_cli.env
from pw_build.build_recipe import BuildRecipe, create_build_recipes
from pw_build.project_builder_prefs import ProjectBuilderPrefs
-
from pw_build.project_builder_argparse import add_project_builder_arguments
_COLOR = pw_cli.color.colors()
@@ -79,10 +94,15 @@
class ProjectBuilderCharset(NamedTuple):
slug_ok: str
slug_fail: str
+ slug_building: str
-ASCII_CHARSET = ProjectBuilderCharset(_COLOR.green('OK '), _COLOR.red('FAIL'))
-EMOJI_CHARSET = ProjectBuilderCharset('✔️ ', '💥')
+ASCII_CHARSET = ProjectBuilderCharset(
+ _COLOR.green('OK '),
+ _COLOR.red('FAIL'),
+ _COLOR.yellow('... '),
+)
+EMOJI_CHARSET = ProjectBuilderCharset('✔️ ', '❌', '⏱️ ')
def _exit(*args) -> NoReturn:
@@ -98,37 +118,262 @@
sys.exit(1)
-def _execute_command(command: list, env: dict) -> bool:
+_NINJA_BUILD_STEP = re.compile(
+ r'^\[(?P<step>[0-9]+)/(?P<total_steps>[0-9]+)\] (?P<action>.*)$'
+)
+
+_NINJA_FAILURE_TEXT = '\033[31mFAILED: '
+
+
+def execute_command_no_logging(
+ command: List,
+ env: Dict,
+ recipe: BuildRecipe,
+ # pylint: disable=unused-argument
+ logger: logging.Logger = _LOG,
+ line_processed_callback: Optional[Callable[[BuildRecipe], None]] = None,
+ # pylint: enable=unused-argument
+) -> bool:
print()
current_build = subprocess.run(command, env=env, errors='replace')
print()
+ recipe.status.return_code = current_build.returncode
return current_build.returncode == 0
-class ProjectBuilder:
+def execute_command_with_logging(
+ command: List,
+ env: Dict,
+ recipe: BuildRecipe,
+ logger: logging.Logger = _LOG,
+ line_processed_callback: Optional[Callable[[BuildRecipe], None]] = None,
+) -> bool:
+ """Run a command in a subprocess and log all output."""
+ current_stdout = ''
+ returncode = None
+
+ with subprocess.Popen(
+ command,
+ env=env,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ errors='replace',
+ ) as proc:
+ # Empty line at the start.
+ logger.info('')
+
+ while returncode is None:
+ output = ''
+ error_output = ''
+
+ if proc.stdout:
+ output += proc.stdout.readline()
+ current_stdout += output
+ if proc.stderr:
+ error_output += proc.stderr.readline()
+ current_stdout += error_output
+
+ if not output and not error_output:
+ returncode = proc.poll()
+ continue
+
+ line_match_result = _NINJA_BUILD_STEP.match(output)
+ if line_match_result:
+ matches = line_match_result.groupdict()
+ recipe.status.current_step = line_match_result.group(0)
+ recipe.status.percent = float(
+ int(matches.get('step', 0))
+ / int(matches.get('total_steps', 1))
+ )
+
+ if output.startswith(_NINJA_FAILURE_TEXT):
+ logger.error(output.strip())
+ recipe.status.error_count += 1
+
+ else:
+ # Mypy output mixes character encoding in its colored output
+ # due to it's use of the curses module retrieving the 'sgr0'
+ # (or exit_attribute_mode) capability from the host
+ # machine's terminfo database.
+ #
+ # This can result in this sequence ending up in STDOUT as
+ # b'\x1b(B\x1b[m'. (B tells terminals to interpret text as
+ # USASCII encoding but will appear in prompt_toolkit as a B
+ # character.
+ #
+ # The following replace calls will strip out those
+ # instances.
+ logger.info(
+ output.replace('\x1b(B\x1b[m', '')
+ .replace('\x1b[1m', '')
+ .strip()
+ )
+
+ if line_processed_callback:
+ line_processed_callback(recipe)
+
+ recipe.status.return_code = returncode
+ # Empty line at the end.
+ logger.info('')
+
+ return returncode == 0
+
+
+def log_build_recipe_start(
+ index_message: str,
+ project_builder: 'ProjectBuilder',
+ cfg: BuildRecipe,
+ logger: logging.Logger = _LOG,
+) -> None:
+ if project_builder.separate_build_file_logging:
+ cfg.log.propagate = False
+
+ build_start_msg = ' '.join(
+ [index_message, _COLOR.cyan('Starting ==>'), str(cfg)]
+ )
+
+ logger.info(build_start_msg)
+ if cfg.logfile:
+ logger.info(
+ '%s %s: %s',
+ index_message,
+ _COLOR.yellow('Logging to'),
+ cfg.logfile.resolve(),
+ )
+ cfg.log.info(build_start_msg)
+
+
+def log_build_recipe_finish(
+ index_message: str,
+ project_builder: 'ProjectBuilder', # pylint: disable=unused-argument
+ cfg: BuildRecipe,
+ logger: logging.Logger = _LOG,
+) -> None:
+
+ if cfg.status.failed():
+ level = logging.ERROR
+ tag = _COLOR.red('(FAIL)')
+ else:
+ level = logging.INFO
+ tag = _COLOR.green('(OK)')
+
+ build_finish_msg = [
+ level,
+ '%s %s %s %s',
+ index_message,
+ _COLOR.cyan('Finished ==>'),
+ cfg,
+ tag,
+ ]
+ logger.log(*build_finish_msg)
+ if cfg.logfile:
+ cfg.log.log(*build_finish_msg)
+
+
+class MissingGlobalLogfile(Exception):
+ """Exception raised if a global logfile is not specifed."""
+
+
+class ProjectBuilder: # pylint: disable=too-many-instance-attributes
"""Pigweed Project Builder"""
def __init__(
+ # pylint: disable=too-many-arguments
self,
build_recipes: Sequence[BuildRecipe],
jobs: Optional[int] = None,
banners: bool = True,
keep_going: bool = False,
abort_callback: Callable = _exit,
- execute_command: Callable[[List, Dict], bool] = _execute_command,
+ execute_command: Callable[
+ [List, Dict, BuildRecipe, logging.Logger, Optional[Callable]], bool
+ ] = execute_command_no_logging,
charset: ProjectBuilderCharset = ASCII_CHARSET,
colors: bool = True,
+ separate_build_file_logging: bool = False,
+ root_logger: logging.Logger = _LOG,
+ root_logfile: Optional[Path] = None,
+ log_level: int = logging.INFO,
):
- self.colors = colors
self.charset: ProjectBuilderCharset = charset
self.abort_callback = abort_callback
self.execute_command = execute_command
self.banners = banners
self.build_recipes = build_recipes
- self.extra_ninja_args = [] if jobs is None else ['-j', f'{jobs}']
+ self.max_name_width = max(
+ [len(str(step.display_name)) for step in self.build_recipes]
+ )
+
+ self.extra_ninja_args: List[str] = []
+ self.extra_bazel_args: List[str] = []
+ self.extra_bazel_build_args: List[str] = []
+
+ if jobs:
+ job_args = ['-j', f'{jobs}']
+ self.extra_ninja_args.extend(job_args)
+ self.extra_bazel_build_args.extend(job_args)
if keep_going:
self.extra_ninja_args.extend(['-k', '0'])
+ self.extra_bazel_build_args.extend(['-k'])
+
+ self.colors = colors
+ if colors:
+ self.extra_bazel_args.append('--color=yes')
+ else:
+ self.extra_bazel_args.append('--color=no')
+
+ if separate_build_file_logging and not root_logfile:
+ raise MissingGlobalLogfile(
+ '\n\nA logfile must be specified if using separate logs per '
+ 'build directory.'
+ )
+
+ self.separate_build_file_logging = separate_build_file_logging
+ self.default_log_level = log_level
+ self.default_logfile = root_logfile
+
+ if root_logfile:
+ timestamp_fmt = _COLOR.black_on_white('%(asctime)s') + ' '
+ formatter = logging.Formatter(
+ timestamp_fmt + '%(levelname)s %(message)s', '%Y%m%d %H:%M:%S'
+ )
+
+ self.execute_command = execute_command_with_logging
+
+ build_log_filehandler = logging.FileHandler(
+ root_logfile, encoding='utf-8'
+ )
+ build_log_filehandler.setLevel(log_level)
+ build_log_filehandler.setFormatter(formatter)
+ root_logger.addHandler(build_log_filehandler)
+
+ if not separate_build_file_logging:
+ for recipe in self.build_recipes:
+ recipe.set_logger(root_logger)
+ else:
+ for recipe in self.build_recipes:
+ new_logger = logging.getLogger(
+ f'{root_logger.name}.{recipe.display_name}'
+ )
+ new_logger.setLevel(log_level)
+ new_logger.propagate = False
+ new_logfile = root_logfile.parent / (
+ root_logfile.stem
+ + '_'
+ + recipe.display_name.replace(' ', '_')
+ + root_logfile.suffix
+ )
+
+ new_log_filehandler = logging.FileHandler(
+ new_logfile, encoding='utf-8'
+ )
+ new_log_filehandler.setLevel(log_level)
+ new_log_filehandler.setFormatter(formatter)
+ new_logger.addHandler(new_log_filehandler)
+
+ recipe.set_logger(new_logger)
+ recipe.set_logfile(new_logfile)
def __len__(self) -> int:
return len(self.build_recipes)
@@ -140,7 +385,10 @@
return (build_recipe for build_recipe in self.build_recipes)
def run_build(
- self, cfg: BuildRecipe, env: Dict, index_message: Optional[str] = ''
+ self,
+ cfg: BuildRecipe,
+ env: Dict,
+ index_message: Optional[str] = '',
) -> bool:
"""Run a single build config."""
if self.colors:
@@ -150,26 +398,45 @@
env['CLICOLOR_FORCE'] = '1'
build_succeded = False
+ cfg.reset_status()
for command_step in cfg.steps:
command_args = command_step.get_args(
- additional_build_args=self.extra_ninja_args
- )
-
- _LOG.info(
- '%s Running ==> %s',
- index_message,
- ' '.join(shlex.quote(arg) for arg in command_args),
+ additional_ninja_args=self.extra_ninja_args,
+ additional_bazel_args=self.extra_bazel_args,
+ additional_bazel_build_args=self.extra_bazel_build_args,
)
# Verify that the build output directories exist.
- if command_step.build_system_command is not None and (
- not cfg.build_dir.is_dir()
+ if (
+ command_step.build_system_command is not None
+ and cfg.build_dir
+ and (not cfg.build_dir.is_dir())
):
self.abort_callback(
'Build directory does not exist: %s', cfg.build_dir
)
- build_succeded = self.execute_command(command_args, env)
+ quoted_command_args = ' '.join(
+ shlex.quote(arg) for arg in command_args
+ )
+ build_succeded = True
+ if command_step.should_run():
+ cfg.log.info(
+ '%s %s %s',
+ index_message,
+ _COLOR.blue('Run ==>'),
+ quoted_command_args,
+ )
+ build_succeded = self.execute_command(
+ command_args, env, cfg, cfg.log, None
+ )
+ else:
+ cfg.log.info(
+ '%s %s %s',
+ index_message,
+ _COLOR.yellow('Skipped ==>'),
+ quoted_command_args,
+ )
# Don't run further steps if a command fails.
if not build_succeded:
break
@@ -177,41 +444,106 @@
return build_succeded
def print_build_summary(
- self, builds_succeeded: List[bool], cancelled: bool = False
+ self,
+ cancelled: bool = False,
+ logger: logging.Logger = _LOG,
) -> None:
- """Print build results summary table."""
+ """Print build status summary table."""
+
+ build_descriptions = []
+ build_status = []
+
+ for cfg in self:
+ description = [str(cfg.display_name).ljust(self.max_name_width)]
+ description.append(' '.join(cfg.targets()))
+ build_descriptions.append(' '.join(description))
+
+ if cfg.status.passed():
+ build_status.append(self.charset.slug_ok)
+ elif cfg.status.failed():
+ build_status.append(self.charset.slug_fail)
+ else:
+ build_status.append(self.charset.slug_building)
+
if not cancelled:
- _LOG.info('')
- _LOG.info(' .------------------------------------')
- _LOG.info(' |')
- for (succeeded, cmd) in zip(
- builds_succeeded, [str(cfg) for cfg in self]
- ):
- slug = (
- self.charset.slug_ok
- if succeeded
- else self.charset.slug_fail
- )
- _LOG.info(' | %s %s', slug, cmd)
- _LOG.info(' |')
- _LOG.info(" '------------------------------------")
+ logger.info(' ╔════════════════════════════════════')
+ logger.info(' ║')
+
+ for (slug, cmd) in zip(build_status, build_descriptions):
+ logger.info(' ║ %s %s', slug, cmd)
+
+ logger.info(' ║')
+ logger.info(" ╚════════════════════════════════════")
else:
# Build was interrupted.
- _LOG.info('')
- _LOG.info(' .------------------------------------')
- _LOG.info(' |')
- _LOG.info(' | %s- interrupted', self.charset.slug_fail)
- _LOG.info(' |')
- _LOG.info(" '------------------------------------")
+ logger.info('')
+ logger.info(' ╔════════════════════════════════════')
+ logger.info(' ║')
+ logger.info(' ║ %s- interrupted', self.charset.slug_fail)
+ logger.info(' ║')
+ logger.info(" ╚════════════════════════════════════")
# Show a large color banner for the overall result.
- if self.banners:
- if all(builds_succeeded) and not cancelled:
+ if self.banners and not any(recipe.status.pending() for recipe in self):
+ if all(recipe.status.passed() for recipe in self) and not cancelled:
for line in PASS_MESSAGE.splitlines():
- _LOG.info(_COLOR.green(line))
+ logger.info(_COLOR.green(line))
else:
for line in FAIL_MESSAGE.splitlines():
- _LOG.info(_COLOR.red(line))
+ logger.info(_COLOR.red(line))
+
+
+def run_recipe(
+ index: int, project_builder: ProjectBuilder, cfg: BuildRecipe, env
+) -> None:
+ num_builds = len(project_builder)
+ index_message = f'[{index}/{num_builds}]'
+
+ log_build_recipe_start(index_message, project_builder, cfg)
+
+ try:
+ project_builder.run_build(cfg, env, index_message=index_message)
+ # Ctrl-C on Unix generates KeyboardInterrupt
+ # Ctrl-Z on Windows generates EOFError
+ except (KeyboardInterrupt, EOFError):
+ _exit_due_to_interrupt()
+
+ log_build_recipe_finish(index_message, project_builder, cfg)
+
+
+def run_builds(project_builder: ProjectBuilder, workers: int = 1) -> None:
+ """Execute build steps in the ProjectBuilder and print a summary."""
+ num_builds = len(project_builder)
+ _LOG.info('Starting build with %d directories', num_builds)
+ if project_builder.default_logfile:
+ _LOG.info(
+ '%s: %s',
+ _COLOR.yellow('Logging to'),
+ project_builder.default_logfile.resolve(),
+ )
+
+ env = os.environ.copy()
+
+ # Print status before starting
+ project_builder.print_build_summary()
+
+ with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
+ future_to_index = {}
+ for i, cfg in enumerate(project_builder, 1):
+ future_to_index[
+ executor.submit(run_recipe, i, project_builder, cfg, env)
+ ] = i
+
+ try:
+ for future in concurrent.futures.as_completed(future_to_index):
+ future.result()
+ # Ctrl-C on Unix generates KeyboardInterrupt
+ # Ctrl-Z on Windows generates EOFError
+ except (KeyboardInterrupt, EOFError):
+ _exit_due_to_interrupt()
+
+ # Print status when finished
+ project_builder.print_build_summary()
def main() -> None:
@@ -235,50 +567,28 @@
prefs.apply_command_line_args(args)
build_recipes = create_build_recipes(prefs)
+ log_level = logging.DEBUG if args.debug_logging else logging.INFO
+
+ pw_cli.log.install(
+ level=log_level,
+ use_color=args.colors,
+ hide_timestamp=False,
+ )
+
project_builder = ProjectBuilder(
build_recipes=build_recipes,
jobs=args.jobs,
banners=args.banners,
keep_going=args.keep_going,
+ colors=args.colors,
charset=charset,
+ separate_build_file_logging=args.separate_logfiles,
+ root_logfile=args.logfile,
+ root_logger=_LOG,
+ log_level=log_level,
)
- pw_cli.log.install(
- level=logging.DEBUG if args.debug_logging else logging.INFO,
- use_color=args.colors,
- hide_timestamp=False,
- )
-
- builds_succeeded = []
- num_builds = len(project_builder)
- _LOG.info('Starting build with %d directories', num_builds)
-
- env = os.environ.copy()
-
- for i, cfg in enumerate(project_builder, 1):
- index = f'[{i}/{num_builds}]'
- build_start_msg = '{} Starting {}'.format(index, cfg)
- _LOG.info(build_start_msg)
-
- try:
- builds_succeeded.append(
- project_builder.run_build(cfg, env, index_message=index)
- )
- # Ctrl-C on Unix generates KeyboardInterrupt
- # Ctrl-Z on Windows generates EOFError
- except (KeyboardInterrupt, EOFError):
- _exit_due_to_interrupt()
-
- if builds_succeeded[-1]:
- level = logging.INFO
- tag = '(OK)'
- else:
- level = logging.ERROR
- tag = '(FAIL)'
-
- _LOG.log(level, '%s Finished %s %s', index, cfg, tag)
-
- project_builder.print_build_summary(builds_succeeded)
+ run_builds(project_builder)
if __name__ == '__main__':
diff --git a/pw_build/py/pw_build/project_builder_argparse.py b/pw_build/py/pw_build/project_builder_argparse.py
index 3c00dc9..603d114 100644
--- a/pw_build/py/pw_build/project_builder_argparse.py
+++ b/pw_build/py/pw_build/project_builder_argparse.py
@@ -14,6 +14,7 @@
"""Pigweed Project Builder Common argparse."""
import argparse
+from pathlib import Path
def add_project_builder_arguments(
@@ -21,16 +22,6 @@
) -> argparse.ArgumentParser:
"""Add ProjectBuilder.main specific arguments."""
parser.add_argument(
- '-k',
- '--keep-going',
- action='store_true',
- help=(
- 'Keep building past the first failure. This is '
- 'equivalent to passing "-k 0" to ninja.'
- ),
- )
-
- parser.add_argument(
'default_build_targets',
nargs='*',
metavar='target',
@@ -59,26 +50,58 @@
),
)
- parser.add_argument(
+ build_options_group = parser.add_argument_group(
+ title='Build system control options'
+ )
+ build_options_group.add_argument(
'-j',
'--jobs',
type=int,
- help="Number of cores to use; defaults to Ninja's default",
+ help="Specify the number of cores to use for each build system.",
+ )
+ build_options_group.add_argument(
+ '-k',
+ '--keep-going',
+ action='store_true',
+ help=(
+ 'Keep building past the first failure. This is equivalent to '
+ 'running "ninja -k 0" or "bazel build -k".'
+ ),
)
- parser.add_argument(
- '--debug-logging', action='store_true', help='Enable debug logging.'
+ logfile_group = parser.add_argument_group(title='Log file options')
+ logfile_group.add_argument(
+ '--logfile',
+ type=Path,
+ help='Global build output log file.',
)
+ logfile_group.add_argument(
+ '--separate-logfiles',
+ action='store_true',
+ help=(
+ 'Create separate log files per build directory. Requires setting '
+ 'the --logfile option.'
+ ),
+ )
+
+ logfile_group.add_argument(
+ '--debug-logging',
+ action='store_true',
+ help='Enable Python build execution tool debug logging.',
+ )
+
+ output_group = parser.add_argument_group(title='Display output options')
+
# TODO(b/248257406) Use argparse.BooleanOptionalAction when Python 3.8 is
# no longer supported.
- parser.add_argument(
+ output_group.add_argument(
'--banners',
action='store_true',
default=True,
help='Show pass/fail banners.',
)
- parser.add_argument(
+ output_group.add_argument(
'--no-banners',
action='store_false',
dest='banners',
@@ -87,13 +110,13 @@
# TODO(b/248257406) Use argparse.BooleanOptionalAction when Python 3.8 is
# no longer supported.
- parser.add_argument(
+ output_group.add_argument(
'--colors',
action='store_true',
default=True,
help='Force color output from ninja.',
)
- parser.add_argument(
+ output_group.add_argument(
'--no-colors',
action='store_false',
dest='colors',
@@ -107,7 +130,7 @@
default=[],
dest='build_system_commands',
metavar=('directory', 'command'),
- help='Build system command.',
+ help='Build system command for . Default: ninja',
)
parser.add_argument(
@@ -115,7 +138,7 @@
action='append',
default=[],
help=(
- 'Additional build commands to run. These are run before any -C '
+ 'Additional commands to run. These are run before any -C '
'arguments and may be repeated. For example: '
"--run-command 'bazel build //pw_cli/...'"
"--run-command 'bazel test //pw_cli/...'"
diff --git a/pw_build/py/pw_build/project_builder_prefs.py b/pw_build/py/pw_build/project_builder_prefs.py
index 53db42a..a6c3515 100644
--- a/pw_build/py/pw_build/project_builder_prefs.py
+++ b/pw_build/py/pw_build/project_builder_prefs.py
@@ -15,6 +15,7 @@
import argparse
import copy
+import shlex
from typing import Any, Callable, Dict, List, Tuple, Union
from pw_cli.toml_config_loader_mixin import TomlConfigLoaderMixin
@@ -23,8 +24,12 @@
# Config settings not available as a command line options go here.
'build_system_commands': {
'default': {
- 'command': 'ninja',
- 'extra_args': [],
+ 'commands': [
+ {
+ 'command': 'ninja',
+ 'extra_args': [],
+ },
+ ],
},
},
}
@@ -64,7 +69,7 @@
project_user_file=False,
user_file=False,
default_config=_DEFAULT_CONFIG,
- environment_var='PW_BUILD_RECIPE_FILE',
+ environment_var='PW_BUILD_CONFIG_FILE',
)
def reset_config(self) -> None:
@@ -78,9 +83,21 @@
) -> Dict[str, Any]:
result = copy.copy(_DEFAULT_CONFIG['build_system_commands'])
for out_dir, command in argparse_input:
- new_dir = result.get('out_dir', {})
- new_dir['command'] = command
- result[out_dir] = new_dir
+ new_dir_spec = result.get(out_dir, {})
+ # Get existing commands list
+ new_commands = new_dir_spec.get('commands', [])
+
+ # Convert 'ninja -k 1' to 'ninja' and ['-k', '1']
+ extra_args = []
+ command_tokens = shlex.split(command)
+ if len(command_tokens) > 1:
+ extra_args = command_tokens[1:]
+ command = command_tokens[0]
+
+ # Append the command step
+ new_commands.append({'command': command, 'extra_args': extra_args})
+ new_dir_spec['commands'] = new_commands
+ result[out_dir] = new_dir_spec
return result
def apply_command_line_args(self, new_args: argparse.Namespace) -> None:
@@ -150,12 +167,17 @@
return build_system_commands
- def build_system_commands(self, build_dir: str) -> Tuple[str, List[str]]:
+ def build_system_commands(
+ self, build_dir: str
+ ) -> List[Tuple[str, List[str]]]:
build_system_commands = self._get_build_system_commands_for(build_dir)
- command: str = build_system_commands.get('command', 'ninja')
- extra_args: List[str] = build_system_commands.get('extra_args', [])
- if not extra_args:
- extra_args = []
- assert isinstance(command, str)
- assert isinstance(extra_args, list)
- return command, extra_args
+
+ command_steps: List[Tuple[str, List[str]]] = []
+ commands: List[Dict[str, Any]] = build_system_commands.get(
+ 'commands', []
+ )
+ for command_step in commands:
+ command_steps.append(
+ (command_step['command'], command_step['extra_args'])
+ )
+ return command_steps
diff --git a/pw_watch/py/pw_watch/argparser.py b/pw_watch/py/pw_watch/argparser.py
index 4a2671f..30ec790 100644
--- a/pw_watch/py/pw_watch/argparser.py
+++ b/pw_watch/py/pw_watch/argparser.py
@@ -53,7 +53,11 @@
parser: argparse.ArgumentParser,
) -> argparse.ArgumentParser:
"""Sets up an argument parser for pw watch."""
- parser.add_argument(
+ parser = add_project_builder_arguments(parser)
+
+ watch_group = parser.add_argument_group(title='Watch options')
+
+ watch_group.add_argument(
'--patterns',
help=(
WATCH_PATTERN_DELIMITER + '-delimited list of globs to '
@@ -62,7 +66,7 @@
default=WATCH_PATTERN_DELIMITER.join(WATCH_PATTERNS),
)
- parser.add_argument(
+ watch_group.add_argument(
'--ignore_patterns',
dest='ignore_patterns_string',
help=(
@@ -71,7 +75,7 @@
),
)
- parser.add_argument(
+ watch_group.add_argument(
'--exclude_list',
nargs='+',
type=Path,
@@ -79,15 +83,13 @@
default=[],
)
- parser.add_argument(
+ watch_group.add_argument(
'--no-restart',
dest='restart',
action='store_false',
help='do not restart ongoing builds if files change',
)
- parser = add_project_builder_arguments(parser)
-
parser.add_argument(
'--serve-docs',
dest='serve_docs',
@@ -110,7 +112,7 @@
'--serve-docs-path',
dest='serve_docs_path',
type=Path,
- default="docs/gen/docs",
+ default='docs/gen/docs',
help='Set the path for the docs to serve. Default to docs/gen/docs'
' in the build directory.',
)
diff --git a/pw_watch/py/pw_watch/debounce.py b/pw_watch/py/pw_watch/debounce.py
index 644be7e..dca6758 100644
--- a/pw_watch/py/pw_watch/debounce.py
+++ b/pw_watch/py/pw_watch/debounce.py
@@ -18,7 +18,7 @@
import threading
from abc import ABC, abstractmethod
-_LOG = logging.getLogger('pw_watch')
+_LOG = logging.getLogger('pw_build.watch')
class DebouncedFunction(ABC):
diff --git a/pw_watch/py/pw_watch/watch.py b/pw_watch/py/pw_watch/watch.py
index 2a1e506..610bb5f 100755
--- a/pw_watch/py/pw_watch/watch.py
+++ b/pw_watch/py/pw_watch/watch.py
@@ -14,15 +14,15 @@
# the License.
"""Watch files for changes and rebuild.
-pw watch runs Ninja in a build directory when source files change. It works with
-any Ninja project (GN or CMake).
+Run arbitrary commands or invoke build systems (Ninja, Bazel and make) on one or
+more build directories whenever source files change.
Usage examples:
- # Find a build directory and build the default target
- pw watch
+ # Build the default target in out/ using ninja.
+ pw watch -C out
- # Find a build directory and build the stm32f429i target
+ # Build python.lint and stm32f429i targets in out/ using ninja.
pw watch python.lint stm32f429i
# Build pw_run_tests.modules in the out/cmake directory
@@ -31,14 +31,19 @@
# Build the default target in out/ and pw_apps in out/cmake
pw watch -C out -C out/cmake pw_apps
- # Find a directory and build python.tests, and build pw_apps in out/cmake
+ # Build python.tests in out/ and pw_apps in out/cmake/
pw watch python.tests -C out/cmake pw_apps
+
+ # Run 'bazel build' and 'bazel test' on the target '//...' in outbazel/
+ pw watch --run-command 'mkdir -p outbazel'
+ -C outbazel '//...'
+ --build-system-command outbazel 'bazel build'
+ --build-system-command outbazel 'bazel test'
"""
import argparse
import errno
import http.server
-from itertools import zip_longest
import logging
import os
from pathlib import Path
@@ -73,6 +78,9 @@
from pw_build.build_recipe import BuildRecipe, create_build_recipes
from pw_build.project_builder import (
ProjectBuilder,
+ execute_command_with_logging,
+ log_build_recipe_start,
+ log_build_recipe_finish,
ASCII_CHARSET,
EMOJI_CHARSET,
)
@@ -83,13 +91,16 @@
import pw_cli.plugins
import pw_console.python_logging
-from pw_watch.argparser import WATCH_PATTERN_DELIMITER, add_parser_arguments
+from pw_watch.argparser import (
+ WATCH_PATTERN_DELIMITER,
+ WATCH_PATTERNS,
+ add_parser_arguments,
+)
from pw_watch.debounce import DebouncedFunction, Debouncer
from pw_watch.watch_app import WatchAppPrefs, WatchApp
_COLOR = pw_cli.color.colors()
-_LOG = logging.getLogger('pw_watch')
-_NINJA_LOG = logging.getLogger('pw_watch_ninja_output')
+_LOG = logging.getLogger('pw_build.watch')
_ERRNO_INOTIFY_LIMIT_REACHED = 28
# Suppress events under 'fsevents', generated by watchdog on every file
@@ -146,13 +157,14 @@
restart: bool = True,
fullscreen: bool = False,
banners: bool = True,
+ use_logfile: bool = False,
+ separate_logfiles: bool = False,
):
super().__init__()
self.banners = banners
self.status_message: Optional[OneStyleAndTextTuple] = None
self.result_message: Optional[StyleAndTextTuples] = None
- self.current_stdout = ''
self.current_build_step = ''
self.current_build_percent = 0.0
self.current_build_errors = 0
@@ -164,6 +176,9 @@
self.fullscreen_enabled = fullscreen
self.watch_app: Optional[WatchApp] = None
+ self.use_logfile = use_logfile
+ self.separate_logfiles = separate_logfiles
+
# Initialize self._current_build to an empty subprocess.
self._current_build = subprocess.Popen('', shell=True, errors='replace')
@@ -172,7 +187,6 @@
# Track state of a build. These need to be members instead of locals
# due to the split between dispatch(), run(), and on_complete().
self.matching_path: Optional[Path] = None
- self.builds_succeeded: List[bool] = []
if not self.fullscreen_enabled:
self.wait_for_keypress_thread = threading.Thread(
@@ -271,58 +285,88 @@
' Watching for changes. Ctrl-C to exit; enter to rebuild'
)
)
- _LOG.info('')
- _LOG.info('Change detected: %s', self.matching_path)
+ if self.matching_path:
+ _LOG.info('')
+ _LOG.info('Change detected: %s', self.matching_path)
self._clear_screen()
- self.builds_succeeded = []
num_builds = len(self.project_builder)
_LOG.info('Starting build with %d directories', num_builds)
+ if self.project_builder.default_logfile:
+ _LOG.info(
+ '%s: %s',
+ _COLOR.yellow('Logging to'),
+ self.project_builder.default_logfile.resolve(),
+ )
+
env = os.environ.copy()
# Force colors in Pigweed subcommands run through the watcher.
env['PW_USE_COLOR'] = '1'
# Force Ninja to output ANSI colors
env['CLICOLOR_FORCE'] = '1'
+ # Reset status messages
+ for cfg in self.project_builder:
+ cfg.reset_status()
+ self.create_result_message()
+
for i, cfg in enumerate(self.project_builder, 1):
- index = f'[{i}/{num_builds}]'
- build_start_msg = '{} Starting {}'.format(index, cfg)
- _LOG.info(build_start_msg)
+ self.run_recipe(i, cfg, env)
- self.builds_succeeded.append(
- self.project_builder.run_build(cfg, env, index_message=index)
- )
- if self.builds_succeeded[-1]:
- level = logging.INFO
- tag = '(OK)'
- else:
- level = logging.ERROR
- tag = '(FAIL)'
+ def run_recipe(self, index: int, cfg: BuildRecipe, env) -> None:
+ num_builds = len(self.project_builder)
+ index_message = f'[{index}/{num_builds}]'
- _LOG.log(level, '%s Finished %s %s', index, cfg, tag)
- self.create_result_message()
+ log_build_recipe_start(
+ index_message, self.project_builder, cfg, logger=_LOG
+ )
+
+ self.project_builder.run_build(
+ cfg,
+ env,
+ index_message=index_message,
+ )
+
+ log_build_recipe_finish(
+ index_message,
+ self.project_builder,
+ cfg,
+ logger=_LOG,
+ )
+
+ self.create_result_message()
def create_result_message(self):
- """TODO(tonymd) Add docstring."""
+ """Update the prompt_toolkit formatted build result message."""
if not self.fullscreen_enabled:
return
self.result_message = []
first_building_target_found = False
- for (succeeded, command) in zip_longest(
- self.builds_succeeded,
- [cfg.build_dir for cfg in self.project_builder],
- ):
- if succeeded:
+ for cfg in self.project_builder:
+ # Add the status
+ if cfg.status.passed():
self.result_message.append(
(
'class:theme-fg-green',
'OK'.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH),
)
)
- elif succeeded is None and not first_building_target_found:
+ elif cfg.status.failed():
+ self.result_message.append(
+ (
+ 'class:theme-fg-red',
+ 'Failed'.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH),
+ )
+ )
+ # NOTE: This condition assumes each build dir is run in serial
+ elif first_building_target_found:
+ self.result_message.append(
+ ('', ''.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH))
+ )
+ else:
first_building_target_found = True
self.result_message.append(
(
@@ -330,98 +374,78 @@
'Building'.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH),
)
)
- elif first_building_target_found:
- self.result_message.append(
- ('', ''.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH))
- )
- else:
- self.result_message.append(
- (
- 'class:theme-fg-red',
- 'Failed'.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH),
- )
- )
- self.result_message.append(('', f' {command}\n'))
- def execute_command(self, command: list, env: dict) -> bool:
+ # Add the build directory
+ self.result_message.append(('', f' {cfg.display_name}\n'))
+
+ def execute_command(
+ self,
+ command: list,
+ env: dict,
+ recipe: BuildRecipe,
+ # pylint: disable=unused-argument
+ *args,
+ **kwargs,
+ # pylint: enable=unused-argument
+ ) -> bool:
"""Runs a command with a blank before/after for visual separation."""
- self.current_build_errors = 0
self.status_message = (
'class:theme-fg-yellow',
'Building'.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH),
)
+
if self.fullscreen_enabled:
- return self._execute_command_watch_app(command, env)
+ return self._execute_command_watch_app(command, env, recipe)
+
+ if self.separate_logfiles:
+ return execute_command_with_logging(
+ command, env, recipe, logger=recipe.log
+ )
+ if self.use_logfile:
+ return execute_command_with_logging(
+ command, env, recipe, logger=_LOG
+ )
+
print()
self._current_build = subprocess.Popen(
command, env=env, errors='replace'
)
returncode = self._current_build.wait()
print()
+ recipe.status.return_code = returncode
return returncode == 0
- def _execute_command_watch_app(self, command: list, env: dict) -> bool:
+ def _execute_command_watch_app(
+ self,
+ command: list,
+ env: dict,
+ recipe: BuildRecipe,
+ ) -> bool:
"""Runs a command with and outputs the logs."""
if not self.watch_app:
return False
- self.current_stdout = ''
- returncode = None
- with subprocess.Popen(
- command,
- env=env,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- errors='replace',
- ) as proc:
- self._current_build = proc
- # Empty line at the start.
- _NINJA_LOG.info('')
- while returncode is None:
- if not proc.stdout:
- continue
+ def new_line_callback(recipe: BuildRecipe) -> None:
+ self.current_build_step = recipe.status.current_step
+ self.current_build_percent = recipe.status.percent
+ self.current_build_errors = recipe.status.error_count
- output = proc.stdout.readline()
- self.current_stdout += output
-
- line_match_result = self.NINJA_BUILD_STEP.match(output)
- if line_match_result:
- matches = line_match_result.groupdict()
- self.current_build_step = line_match_result.group(0)
- self.current_build_percent = float(
- int(matches.get('step', 0))
- / int(matches.get('total_steps', 1))
- )
-
- elif output.startswith(WatchApp.NINJA_FAILURE_TEXT):
- _NINJA_LOG.critical(output.strip())
- self.current_build_errors += 1
-
- else:
- # Mypy output mixes character encoding in its colored output
- # due to it's use of the curses module retrieving the 'sgr0'
- # (or exit_attribute_mode) capability from the host
- # machine's terminfo database.
- #
- # This can result in this sequence ending up in STDOUT as
- # b'\x1b(B\x1b[m'. (B tells terminals to interpret text as
- # USASCII encoding but will appear in prompt_toolkit as a B
- # character.
- #
- # The following replace calls will strip out those
- # instances.
- _NINJA_LOG.info(
- output.replace('\x1b(B\x1b[m', '')
- .replace('\x1b[1m', '')
- .strip()
- )
+ if self.watch_app:
self.watch_app.redraw_ui()
- returncode = proc.poll()
- # Empty line at the end.
- _NINJA_LOG.info('')
+ desired_logger = _LOG
+ if self.separate_logfiles:
+ desired_logger = recipe.log
- return returncode == 0
+ result = execute_command_with_logging(
+ command,
+ env,
+ recipe,
+ logger=desired_logger,
+ line_processed_callback=new_line_callback,
+ )
+
+ return result
# Implementation of DebouncedFunction.cancel()
def cancel(self) -> bool:
@@ -442,7 +466,7 @@
)
_LOG.error('Finished; build was interrupted')
- elif all(self.builds_succeeded):
+ elif all(recipe.status.passed() for recipe in self.project_builder):
self.status_message = (
'class:theme-fg-green',
'Succeeded'.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH),
@@ -462,7 +486,7 @@
else:
# Show a more distinct colored banner.
self.project_builder.print_build_summary(
- self.builds_succeeded, cancelled=cancelled
+ cancelled=cancelled, logger=_LOG
)
if self.watch_app:
@@ -704,28 +728,62 @@
threading.Thread(None, server_thread, 'pw_docs_server').start()
+def watch_logging_init(log_level: int, fullscreen: bool) -> None:
+ # Logging setup
+ if not fullscreen:
+ pw_cli.log.install(
+ level=log_level,
+ use_color=True,
+ hide_timestamp=False,
+ )
+ return
+
+ watch_logfile = pw_console.python_logging.create_temp_log_file(
+ prefix=__package__
+ )
+ pw_cli.log.install(
+ level=logging.DEBUG,
+ use_color=True,
+ hide_timestamp=False,
+ log_file=watch_logfile,
+ )
+ pw_console.python_logging.setup_python_logging(
+ last_resort_filename=watch_logfile
+ )
+
+
def watch_setup( # pylint: disable=too-many-locals
- build_recipes: List[BuildRecipe],
- default_build_targets: List[str], # pylint: disable=unused-argument
- build_directories: List[str], # pylint: disable=unused-argument
- build_system_commands: List[str], # pylint: disable=unused-argument
- run_command: List[str], # pylint: disable=unused-argument
- patterns: str,
- ignore_patterns_string: str,
- exclude_list: List[Path],
- restart: bool,
- jobs: Optional[int],
- serve_docs: bool,
- serve_docs_port: int,
- serve_docs_path: Path,
- fullscreen: bool,
- banners: bool,
- keep_going: bool,
- debug_logging: bool, # pylint: disable=unused-argument
- colors: bool,
+ project_builder: ProjectBuilder,
+ # NOTE: The following args should have defaults matching argparse. This
+ # allows use of watch_setup by other project build scripts.
+ patterns: str = WATCH_PATTERN_DELIMITER.join(WATCH_PATTERNS),
+ ignore_patterns_string: str = '',
+ exclude_list: Optional[List[Path]] = None,
+ restart: bool = True,
+ serve_docs: bool = False,
+ serve_docs_port: int = 8000,
+ serve_docs_path: Path = Path('docs/gen/docs'),
+ fullscreen: bool = False,
+ banners: bool = True,
+ logfile: Optional[Path] = None,
+ separate_logfiles: bool = False,
+ # pylint: disable=unused-argument
+ default_build_targets: Optional[List[str]] = None,
+ build_directories: Optional[List[str]] = None,
+ build_system_commands: Optional[List[str]] = None,
+ run_command: Optional[List[str]] = None,
+ jobs: Optional[int] = None,
+ keep_going: bool = False,
+ colors: bool = True,
+ debug_logging: bool = False,
+ # pylint: enable=unused-argument
# pylint: disable=too-many-arguments
-) -> Tuple[str, PigweedBuildWatcher, List[Path]]:
+) -> Tuple[PigweedBuildWatcher, List[Path]]:
"""Watches files and runs Ninja commands when they change."""
+ watch_logging_init(
+ log_level=project_builder.default_log_level, fullscreen=fullscreen
+ )
+
_LOG.info('Starting Pigweed build watcher')
# Get pigweed directory information from environment variable PW_ROOT.
@@ -735,7 +793,11 @@
if Path.cwd().resolve() not in [pw_root, *pw_root.parents]:
_exit_due_to_pigweed_not_installed()
+ build_recipes = project_builder.build_recipes
+
# Preset exclude list for pigweed directory.
+ if not exclude_list:
+ exclude_list = []
exclude_list += get_common_excludes()
# Add build directories to the exclude list.
exclude_list.extend(
@@ -754,11 +816,6 @@
build_recipes[0].build_dir, serve_docs_path, port=serve_docs_port
)
- # Try to make a short display path for the watched directory that has
- # "$HOME" instead of the full home directory. This is nice for users
- # who have deeply nested home directories.
- path_to_log = str(Path().resolve()).replace(str(Path.home()), '$HOME')
-
# Ignore the user-specified patterns.
ignore_patterns = (
ignore_patterns_string.split(WATCH_PATTERN_DELIMITER)
@@ -766,21 +823,6 @@
else []
)
- env = pw_cli.env.pigweed_environment()
- if env.PW_EMOJI:
- charset = EMOJI_CHARSET
- else:
- charset = ASCII_CHARSET
-
- project_builder = ProjectBuilder(
- build_recipes=build_recipes,
- jobs=jobs,
- banners=banners,
- keep_going=keep_going,
- charset=charset,
- colors=colors,
- )
-
event_handler = PigweedBuildWatcher(
project_builder=project_builder,
patterns=patterns.split(WATCH_PATTERN_DELIMITER),
@@ -788,19 +830,25 @@
restart=restart,
fullscreen=fullscreen,
banners=banners,
+ use_logfile=bool(logfile),
+ separate_logfiles=bool(separate_logfiles),
)
project_builder.execute_command = event_handler.execute_command
- return path_to_log, event_handler, exclude_list
+ return event_handler, exclude_list
def watch(
- path_to_log: Path,
event_handler: PigweedBuildWatcher,
exclude_list: List[Path],
):
"""Watches files and runs Ninja commands when they change."""
+ # Try to make a short display path for the watched directory that has
+ # "$HOME" instead of the full home directory. This is nice for users
+ # who have deeply nested home directories.
+ path_to_log = str(Path().resolve()).replace(str(Path.home()), '$HOME')
+
try:
# It can take awhile to configure the filesystem watcher, so have the
# message reflect that with the "...". Run inside the try: to
@@ -839,6 +887,35 @@
raise err
+def run_watch(
+ event_handler: PigweedBuildWatcher,
+ exclude_list: List[Path],
+ prefs: Optional[WatchAppPrefs] = None,
+ fullscreen: bool = False,
+) -> None:
+ """Start pw_watch."""
+ if not prefs:
+ prefs = WatchAppPrefs(load_argparse_arguments=add_parser_arguments)
+
+ if fullscreen:
+ watch_thread = Thread(
+ target=watch,
+ args=(event_handler, exclude_list),
+ daemon=True,
+ )
+ watch_thread.start()
+ watch_app = WatchApp(
+ event_handler=event_handler,
+ prefs=prefs,
+ )
+
+ event_handler.watch_app = watch_app
+ watch_app.run()
+
+ else:
+ watch(event_handler, exclude_list)
+
+
def main() -> None:
"""Watch files for changes and rebuild."""
parser = argparse.ArgumentParser(
@@ -852,46 +929,33 @@
prefs.apply_command_line_args(args)
build_recipes = create_build_recipes(prefs)
- path_to_log, event_handler, exclude_list = watch_setup(
- build_recipes=build_recipes, **vars(args)
+ env = pw_cli.env.pigweed_environment()
+ if env.PW_EMOJI:
+ charset = EMOJI_CHARSET
+ else:
+ charset = ASCII_CHARSET
+
+ project_builder = ProjectBuilder(
+ build_recipes=build_recipes,
+ jobs=args.jobs,
+ banners=args.banners,
+ keep_going=args.keep_going,
+ colors=args.colors,
+ charset=charset,
+ separate_build_file_logging=args.separate_logfiles,
+ root_logfile=args.logfile,
+ root_logger=_LOG,
+ log_level=logging.DEBUG if args.debug_logging else logging.INFO,
)
- if args.fullscreen:
- watch_logfile = pw_console.python_logging.create_temp_log_file(
- prefix=__package__
- )
- pw_cli.log.install(
- level=logging.DEBUG,
- use_color=True,
- hide_timestamp=False,
- log_file=watch_logfile,
- )
- pw_console.python_logging.setup_python_logging(
- last_resort_filename=watch_logfile
- )
+ event_handler, exclude_list = watch_setup(project_builder, **vars(args))
- watch_thread = Thread(
- target=watch,
- args=(path_to_log, event_handler, exclude_list),
- daemon=True,
- )
- watch_thread.start()
- watch_app = WatchApp(
- event_handler=event_handler,
- prefs=prefs,
- debug_logging=args.debug_logging,
- log_file_name=watch_logfile,
- )
-
- event_handler.watch_app = watch_app
- watch_app.run()
- else:
- pw_cli.log.install(
- level=logging.DEBUG if args.debug_logging else logging.INFO,
- use_color=True,
- hide_timestamp=False,
- )
- watch(Path(path_to_log), event_handler, exclude_list)
+ run_watch(
+ event_handler,
+ exclude_list,
+ prefs=prefs,
+ fullscreen=args.fullscreen,
+ )
if __name__ == '__main__':
diff --git a/pw_watch/py/pw_watch/watch_app.py b/pw_watch/py/pw_watch/watch_app.py
index 58669ed..3886f80 100644
--- a/pw_watch/py/pw_watch/watch_app.py
+++ b/pw_watch/py/pw_watch/watch_app.py
@@ -20,7 +20,7 @@
import re
import sys
import time
-from typing import Callable, Dict, List, NoReturn, Optional
+from typing import Callable, Dict, Iterable, List, NoReturn, Optional
from prompt_toolkit.application import Application
from prompt_toolkit.clipboard.pyperclip import PyperclipClipboard
@@ -160,8 +160,7 @@
return self._config.get('show_python_logger', False)
-_NINJA_LOG = logging.getLogger('pw_watch_ninja_output')
-_LOG = logging.getLogger('pw_watch')
+_LOG = logging.getLogger('pw_build.watch')
class WatchWindowManager(WindowManager):
@@ -184,7 +183,6 @@
self,
event_handler,
prefs: WatchAppPrefs,
- debug_logging: bool = False,
log_file_name: Optional[str] = None,
):
@@ -215,49 +213,25 @@
self.log_ui_update_frequency = 0.1 # 10 FPS
self._last_ui_update_time = time.time()
- self.ninja_log_pane = LogPane(
- application=self, pane_title='Pigweed Watch'
+ debug_logging = (
+ event_handler.project_builder.default_log_level == logging.DEBUG
)
- self.ninja_log_pane.add_log_handler(_NINJA_LOG, level_name='INFO')
- self.ninja_log_pane.add_log_handler(
- _LOG, level_name=('DEBUG' if debug_logging else 'INFO')
- )
- # Set python log format to just the message itself.
- self.ninja_log_pane.log_view.log_store.formatter = logging.Formatter(
- '%(message)s'
- )
- self.ninja_log_pane.table_view = False
- # Disable line wrapping for improved error visibility.
- if self.ninja_log_pane.wrap_lines:
- self.ninja_log_pane.toggle_wrap_lines()
- # Blank right side toolbar text
- self.ninja_log_pane._pane_subtitle = ' '
- self.ninja_log_view = self.ninja_log_pane.log_view
+ level_name = 'DEBUG' if debug_logging else 'INFO'
+ if event_handler.separate_logfiles:
+ for recipe in reversed(event_handler.project_builder.build_recipes):
+ self.add_build_log_pane(
+ recipe.display_name,
+ recipe.log,
+ level_name=level_name,
+ )
- # Make tab and shift-tab search for next and previous error
- next_error_bindings = KeyBindings()
-
- @next_error_bindings.add('s-tab')
- def _previous_error(_event):
- self.jump_to_error(backwards=True)
-
- @next_error_bindings.add('tab')
- def _next_error(_event):
- self.jump_to_error()
-
- existing_log_bindings: Optional[
- KeyBindingsBase
- ] = self.ninja_log_pane.log_content_control.key_bindings
-
- key_binding_list: List[KeyBindingsBase] = []
- if existing_log_bindings:
- key_binding_list.append(existing_log_bindings)
- key_binding_list.append(next_error_bindings)
- self.ninja_log_pane.log_content_control.key_bindings = (
- merge_key_bindings(key_binding_list)
+ self.add_build_log_pane(
+ 'Pigweed Watch',
+ _LOG,
+ level_name=level_name,
)
- self.window_manager.add_pane(self.ninja_log_pane)
+ self.ninja_log_pane = list(self.all_log_panes())[0]
self.time_waster = Twenty48Pane(include_resize_handle=True)
self.time_waster.application = self
@@ -358,7 +332,8 @@
)
self.layout = Layout(
- self.root_container, focused_element=self.ninja_log_pane
+ self.root_container,
+ focused_element=self.ninja_log_pane,
)
self.application: Application = Application(
@@ -384,6 +359,52 @@
plugin_logger_name='pw_watch_stdout_checker',
)
+ def add_build_log_pane(
+ self, title: str, logger: logging.Logger, level_name: str
+ ) -> None:
+ """Setup a new build log pane."""
+ new_log_pane = LogPane(application=self, pane_title=title)
+ new_log_pane.add_log_handler(logger, level_name=level_name)
+
+ # Set python log format to just the message itself.
+ new_log_pane.log_view.log_store.formatter = logging.Formatter(
+ '%(message)s'
+ )
+
+ new_log_pane.table_view = False
+
+ # Disable line wrapping for improved error visibility.
+ if new_log_pane.wrap_lines:
+ new_log_pane.toggle_wrap_lines()
+
+ # Blank right side toolbar text
+ new_log_pane._pane_subtitle = ' ' # pylint: disable=protected-access
+
+ # Make tab and shift-tab search for next and previous error
+ next_error_bindings = KeyBindings()
+
+ @next_error_bindings.add('s-tab')
+ def _previous_error(_event):
+ self.jump_to_error(backwards=True)
+
+ @next_error_bindings.add('tab')
+ def _next_error(_event):
+ self.jump_to_error()
+
+ existing_log_bindings: Optional[
+ KeyBindingsBase
+ ] = new_log_pane.log_content_control.key_bindings
+
+ key_binding_list: List[KeyBindingsBase] = []
+ if existing_log_bindings:
+ key_binding_list.append(existing_log_bindings)
+ key_binding_list.append(next_error_bindings)
+ new_log_pane.log_content_control.key_bindings = merge_key_bindings(
+ key_binding_list
+ )
+
+ self.window_manager.add_pane(new_log_pane)
+
def logs_redraw(self):
emit_time = time.time()
# Has enough time passed since last UI redraw due to new logs?
@@ -445,13 +466,19 @@
# pylint: disable=no-self-use
return False
+ def all_log_panes(self) -> Iterable[LogPane]:
+ for pane in self.window_manager.active_panes():
+ if isinstance(pane, LogPane):
+ yield pane
+
def clear_ninja_log(self) -> None:
- self.ninja_log_view.log_store.clear_logs()
- self.ninja_log_view._restart_filtering() # pylint: disable=protected-access
- self.ninja_log_view.view_mode_changed()
- # Re-enable follow if needed
- if not self.ninja_log_view.follow:
- self.ninja_log_view.toggle_follow()
+ for pane in self.all_log_panes():
+ pane.log_view.log_store.clear_logs()
+ pane.log_view._restart_filtering() # pylint: disable=protected-access
+ pane.log_view.view_mode_changed()
+ # Re-enable follow if needed
+ if not pane.log_view.follow:
+ pane.log_view.toggle_follow()
def run_build(self):
"""Manually trigger a rebuild."""
@@ -459,8 +486,9 @@
self.event_handler.rebuild()
def rebuild_on_filechange(self):
- self.ninja_log_view.log_store.clear_logs()
- self.ninja_log_view.view_mode_changed()
+ for pane in self.all_log_panes():
+ pane.log_view.log_store.clear_logs()
+ pane.log_view.view_mode_changed()
def get_statusbar_text(self):
status = self.event_handler.status_message
