pw_system: Console log files and busy wait fix
- Add host and device logfile options
- Merge host and device into single logfile option
- Remove ANSI colors from logfiles
- Add IPython option in case the pw_console TUI is not needed
- Set pyserial timeout to small timespan to avoid 100% CPU usage
Change-Id: I308c78e7a157d914359b2a77c511768f23932d68
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/108555
Commit-Queue: Anthony DiGirolamo <tonymd@google.com>
Reviewed-by: Armando Montanez <amontanez@google.com>
diff --git a/pw_system/py/pw_system/console.py b/pw_system/py/pw_system/console.py
index ba491a1..90450ed 100644
--- a/pw_system/py/pw_system/console.py
+++ b/pw_system/py/pw_system/console.py
@@ -52,22 +52,24 @@
import socket
import serial # type: ignore
+import IPython # type: ignore
import pw_cli.log
-import pw_console.python_logging
-from pw_console import PwConsoleEmbed
-from pw_console.pyserial_wrapper import SerialWithLogging
-from pw_console.plugins.bandwidth_toolbar import BandwidthToolbar
+from pw_console.embed import PwConsoleEmbed
from pw_console.log_store import LogStore
+from pw_console.plugins.bandwidth_toolbar import BandwidthToolbar
+from pw_console.pyserial_wrapper import SerialWithLogging
+import pw_console.python_logging
+from pw_rpc.console_tools.console import flattened_rpc_completions
+from pw_system.device import Device
+from pw_tokenizer.detokenize import AutoUpdatingDetokenizer
+
+# Default proto imports:
from pw_log.proto import log_pb2
from pw_metric_proto import metric_service_pb2
from pw_thread_protos import thread_snapshot_service_pb2
-from pw_rpc.console_tools.console import flattened_rpc_completions
-from pw_tokenizer.detokenize import AutoUpdatingDetokenizer
from pw_unit_test_proto import unit_test_pb2
-from pw_system.device import Device
-
_LOG = logging.getLogger('tools')
_DEVICE_LOG = logging.getLogger('rpc_device')
_SERIAL_DEBUG = logging.getLogger('pw_console.serial_debug_logger')
@@ -102,7 +104,34 @@
default=sys.stdout.buffer,
help=('The file to which to write device output (HDLC channel 1); '
'provide - or omit for stdout.'))
- parser.add_argument('--logfile', help='Console debug log file.')
+
+ # Log file options
+ parser.add_argument(
+ '--logfile',
+ default='pw_console-logs.txt',
+ help=('Default log file. This will contain host side '
+ 'log messages only unles the '
+ '--merge-device-and-host-logs argument is used.'),
+ )
+
+ parser.add_argument(
+ '--merge-device-and-host-logs',
+ action='store_true',
+ help=('Include device logs in the default --logfile.'
+ 'These are normally shown in a separate device '
+ 'only log file.'),
+ )
+
+ parser.add_argument(
+ '--host-logfile',
+ help=('Additional host only log file. Normally all logs in the '
+ 'default logfile are host only.'),
+ )
+
+ parser.add_argument('--device-logfile',
+ default='pw_console-device-logs.txt',
+ help='Device only log file.')
+
group.add_argument('-s',
'--socket-addr',
type=str,
@@ -119,11 +148,16 @@
parser.add_argument('--proto-globs',
nargs='+',
default=[],
- help='glob pattern for .proto files')
+ help='glob pattern for .proto files.')
parser.add_argument('-v',
'--verbose',
action='store_true',
- help='Enables debug logging when set')
+ help='Enables debug logging when set.')
+ parser.add_argument('--ipython',
+ action='store_true',
+ dest='use_ipython',
+ help='Use IPython instead of pw_console.')
+
return parser.parse_args()
@@ -133,16 +167,19 @@
yield Path(file)
-def _start_ipython_terminal(
+def _start_python_terminal(
device: Device,
device_log_store: LogStore,
root_log_store: LogStore,
serial_debug_log_store: LogStore,
log_file: str,
+ host_logfile: str,
+ device_logfile: str,
serial_debug: bool = False,
config_file_path: Optional[Path] = None,
+ use_ipython: bool = False,
) -> None:
- """Starts an interactive IPython terminal with preset variables."""
+ """Starts an interactive Python terminal with preset variables."""
local_variables = dict(
client=device.client,
device=device,
@@ -166,6 +203,19 @@
DEVICE_LOG.warning('Message appears in Device Logs window.')
""")
+ welcome_message += '\n\nLogs are being saved to:\n ' + log_file
+ if host_logfile:
+ welcome_message += '\nHost logs are being saved to:\n ' + host_logfile
+ if device_logfile:
+ welcome_message += ('\nDevice logs are being saved to:\n ' +
+ device_logfile)
+
+ if use_ipython:
+ print(welcome_message)
+ IPython.terminal.embed.InteractiveShellEmbed().mainloop(
+ local_ns=local_variables, module=argparse.Namespace())
+ return
+
client_info = device.info()
completions = flattened_rpc_completions([client_info])
@@ -219,18 +269,22 @@
return self.socket.recv(num_bytes)
-#pylint: disable=too-many-arguments
+#pylint: disable=too-many-arguments,too-many-locals
def console(device: str,
baudrate: int,
proto_globs: Collection[str],
token_databases: Collection[Path],
socket_addr: str,
logfile: str,
+ host_logfile: str,
+ device_logfile: str,
output: Any,
serial_debug: bool = False,
config_file: Optional[Path] = None,
verbose: bool = False,
- compiled_protos: Optional[List[ModuleType]] = None) -> int:
+ compiled_protos: Optional[List[ModuleType]] = None,
+ merge_device_and_host_logs: bool = False,
+ use_ipython: bool = False) -> int:
"""Starts an interactive RPC console for HDLC."""
# argparse.FileType doesn't correctly handle '-' for binary files.
if output is sys.stdout:
@@ -256,11 +310,37 @@
logfile = pw_console.python_logging.create_temp_log_file()
log_level = logging.DEBUG if verbose else logging.INFO
- pw_cli.log.install(log_level, True, False, logfile)
+
+ pw_cli.log.install(level=log_level,
+ use_color=False,
+ hide_timestamp=False,
+ log_file=logfile)
+
+ if device_logfile:
+ pw_cli.log.install(level=log_level,
+ use_color=False,
+ hide_timestamp=False,
+ log_file=device_logfile,
+ logger=_DEVICE_LOG)
+ if host_logfile:
+ pw_cli.log.install(level=log_level,
+ use_color=False,
+ hide_timestamp=False,
+ log_file=host_logfile,
+ logger=_ROOT_LOG)
+
+ if merge_device_and_host_logs:
+ # Add device logs to the default logfile.
+ pw_cli.log.install(level=log_level,
+ use_color=False,
+ hide_timestamp=False,
+ log_file=logfile,
+ logger=_DEVICE_LOG)
+
_LOG.setLevel(log_level)
_DEVICE_LOG.setLevel(log_level)
_ROOT_LOG.setLevel(log_level)
- _SERIAL_DEBUG.setLevel(log_level)
+ _SERIAL_DEBUG.setLevel(logging.DEBUG)
detokenizer = None
if token_databases:
@@ -299,7 +379,11 @@
serial_device = serial_impl(
device,
baudrate,
- timeout=0, # Non-blocking mode
+ # Timeout in seconds. This should be a very small value. Setting to
+ # zero makes pyserial read() non-blocking which will cause the host
+ # machine to busy loop and 100% CPU usage.
+ # https://pythonhosted.org/pyserial/pyserial_api.html#serial.Serial
+ timeout=0.1,
)
read = lambda: serial_device.read(8192)
write = serial_device.write
@@ -327,9 +411,18 @@
timestamp_decoder=timestamp_decoder,
rpc_timeout_s=5)
- _start_ipython_terminal(device_client, device_log_store, root_log_store,
- serial_debug_log_store, logfile, serial_debug,
- config_file)
+ _start_python_terminal(
+ device=device_client,
+ device_log_store=device_log_store,
+ root_log_store=root_log_store,
+ serial_debug_log_store=serial_debug_log_store,
+ log_file=logfile,
+ host_logfile=host_logfile,
+ device_logfile=device_logfile,
+ serial_debug=serial_debug,
+ config_file_path=config_file,
+ use_ipython=use_ipython,
+ )
return 0
