pw_console: Open any Python logger interactively
- File > Open Logger menu that displays all known python loggers.
Clicking one opens a new log window with that logger.
- File > Log Table View > Show Python File and Logger options.
These preferences toggle displaying each log line's
Python logger and file:linenumber columns in table view.
- Set logging.lastResort to a file handler. This prevent loggers
with no handlers from outputting text to stderr and corrupting
the prompt_toolkit UI.
Change-Id: I6f36384764621b0f62ad6c51687b5b125c91da44
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/69280
Reviewed-by: Joe Ethier <jethier@google.com>
Commit-Queue: Anthony DiGirolamo <tonymd@google.com>
diff --git a/pw_console/py/log_store_test.py b/pw_console/py/log_store_test.py
index a8097d4..9c0763d 100644
--- a/pw_console/py/log_store_test.py
+++ b/pw_console/py/log_store_test.py
@@ -24,6 +24,8 @@
def _create_log_store():
log_store = LogStore(
prefs=ConsolePrefs(project_file=False, user_file=False))
+
+ assert not log_store.table.prefs.show_python_file
viewer = MagicMock()
viewer.new_logs_arrived = MagicMock()
log_store.register_viewer(viewer)
diff --git a/pw_console/py/log_view_test.py b/pw_console/py/log_view_test.py
index 1c68dc2..f740851 100644
--- a/pw_console/py/log_view_test.py
+++ b/pw_console/py/log_view_test.py
@@ -226,12 +226,15 @@
log_view.scroll_to_top()
log_view.render_content()
- self.assertEqual(log_view.get_cursor_position(), Point(x=0, y=0))
+ self.assertEqual(log_view.get_cursor_position(), Point(x=0, y=2))
+
log_view.scroll_to_bottom()
log_view.render_content()
- self.assertEqual(log_view.get_cursor_position(), Point(x=0, y=3))
+ self.assertEqual(log_view.get_cursor_position(), Point(x=0, y=5))
expected_line_cache = [
+ [('', '\n')],
+ [('', '\n')],
[
('class:log-time', '20210713 00:00:00'),
('', ' '),
@@ -263,14 +266,12 @@
('class:selected-log-line class:log-level-10', 'DEBUG'),
('class:selected-log-line ', ' '),
('class:selected-log-line ', 'Test log 3'),
- ('class:selected-log-line ', ' \n')
+ ('class:selected-log-line ', ' \n')
]),
] # yapf: disable
- self.assertEqual(
- list(log_view._line_fragment_cache # pylint: disable=protected-access
- ),
- expected_line_cache)
+ self.assertEqual(expected_line_cache,
+ list(log_view._line_fragment_cache)) # pylint: disable=protected-access
def test_clear_scrollback(self) -> None:
"""Test various functions with clearing log scrollback history."""
diff --git a/pw_console/py/pw_console/__main__.py b/pw_console/py/pw_console/__main__.py
index 3ab540e..a84fb0c 100644
--- a/pw_console/py/pw_console/__main__.py
+++ b/pw_console/py/pw_console/__main__.py
@@ -18,7 +18,6 @@
import logging
from pathlib import Path
import sys
-from typing import List
import pw_cli.log
import pw_cli.argument_types
@@ -80,19 +79,21 @@
use_color=True,
hide_timestamp=False,
log_file=args.console_debug_log_file,
- logger=logging.getLogger(__package__))
- logging.getLogger(__package__).propagate = False
+ logger=logging.getLogger('pw_console'))
global_vars = None
- default_loggers: List = []
+ default_loggers = {}
if args.test_mode:
- default_loggers = [
+ fake_logger = logging.getLogger(
+ pw_console.console_app.FAKE_DEVICE_LOGGER_NAME)
+ default_loggers = {
# Don't include pw_console package logs (_LOG) in the log pane UI.
# Add the fake logger for test_mode.
- logging.getLogger(pw_console.console_app.FAKE_DEVICE_LOGGER_NAME)
- ]
+ 'Fake Device Logs': [fake_logger],
+ 'PwConsole Debug': [logging.getLogger('pw_console')],
+ }
# Give access to adding log messages from the repl via: `LOG.warning()`
- global_vars = dict(LOG=default_loggers[0])
+ global_vars = dict(LOG=fake_logger)
help_text = None
app_title = None
diff --git a/pw_console/py/pw_console/console_app.py b/pw_console/py/pw_console/console_app.py
index 38f55d4..7d1941e 100644
--- a/pw_console/py/pw_console/console_app.py
+++ b/pw_console/py/pw_console/console_app.py
@@ -58,6 +58,7 @@
from pw_console.help_window import HelpWindow
import pw_console.key_bindings
from pw_console.log_pane import LogPane
+from pw_console.python_logging import all_loggers
from pw_console.pw_ptpython_repl import PwPtPythonRepl
from pw_console.repl_pane import ReplPane
import pw_console.style
@@ -68,8 +69,10 @@
_LOG = logging.getLogger(__package__)
# Fake logger for --test-mode
-FAKE_DEVICE_LOGGER_NAME = 'fake_device.1'
+FAKE_DEVICE_LOGGER_NAME = 'pw_console_fake_device'
_FAKE_DEVICE_LOG = logging.getLogger(FAKE_DEVICE_LOGGER_NAME)
+# Don't send fake_device logs to the root Python logger.
+_FAKE_DEVICE_LOG.propagate = False
class FloatingMessageBar(ConditionalContainer):
@@ -85,11 +88,12 @@
def _add_log_handler_to_pane(logger: Union[str, logging.Logger],
- pane: 'LogPane') -> None:
+ pane: 'LogPane',
+ level_name: Optional[str] = None) -> None:
"""A log pane handler for a given logger instance."""
if not pane:
return
- pane.add_log_handler(logger)
+ pane.add_log_handler(logger, level_name=level_name)
class ConsoleApp:
@@ -331,6 +335,17 @@
self.update_menu_items()
self.focus_main_menu()
+ def open_new_log_pane_for_logger(
+ self,
+ logger_name: str,
+ level_name='NOTSET',
+ window_title: Optional[str] = None) -> None:
+ pane_title = window_title if window_title else logger_name
+ self.run_pane_menu_option(
+ functools.partial(self.add_log_handler,
+ pane_title, [logger_name],
+ log_level_name=level_name))
+
def set_ui_theme(self, theme_name: str) -> Callable:
call_function = functools.partial(
self.run_pane_menu_option,
@@ -345,7 +360,27 @@
return call_function
def update_menu_items(self):
- self.root_container.menu_items = self._create_menu_items()
+ self.menu_items = self._create_menu_items()
+ self.root_container.menu_items = self.menu_items
+
+ def _create_logger_submenu(self):
+ submenu = [
+ MenuItem(
+ 'root',
+ handler=functools.partial(self.open_new_log_pane_for_logger,
+ '',
+ window_title='root'),
+ )
+ ]
+ all_logger_names = sorted([logger.name for logger in all_loggers()])
+ for logger_name in all_logger_names:
+ submenu.append(
+ MenuItem(
+ logger_name,
+ handler=functools.partial(
+ self.open_new_log_pane_for_logger, logger_name),
+ ))
+ return submenu
def _create_menu_items(self):
themes_submenu = [
@@ -394,6 +429,34 @@
MenuItem(
'[File]',
children=[
+ MenuItem('Open Logger',
+ children=self._create_logger_submenu()),
+ MenuItem(
+ 'Log Table View',
+ children=[
+ MenuItem(
+ '{check} Show Python File'.format(
+ check=pw_console.widgets.checkbox.
+ to_checkbox_text(
+ self.prefs.show_python_file, end='')),
+ handler=functools.partial(
+ self.run_pane_menu_option,
+ functools.partial(self.toggle_pref_option,
+ 'show_python_file')),
+ ),
+ MenuItem(
+ '{check} Show Python Logger'.format(
+ check=pw_console.widgets.checkbox.
+ to_checkbox_text(
+ self.prefs.show_python_logger,
+ end='')),
+ handler=functools.partial(
+ self.run_pane_menu_option,
+ functools.partial(self.toggle_pref_option,
+ 'show_python_logger')),
+ ),
+ ]),
+ MenuItem('-'),
MenuItem(
'Themes',
children=themes_submenu,
@@ -520,9 +583,14 @@
if self.application:
self.focus_main_menu()
+ def toggle_pref_option(self, setting_name):
+ self.prefs.toggle_bool_option(setting_name)
+
def load_theme(self, theme_name=None):
"""Regenerate styles for the current theme_name."""
self._current_theme = pw_console.style.generate_styles(theme_name)
+ if theme_name:
+ self.prefs.set_ui_theme(theme_name)
def _create_log_pane(self, title=None) -> 'LogPane':
# Create one log pane.
@@ -537,10 +605,12 @@
def apply_window_config(self) -> None:
self.window_manager.apply_config(self.prefs)
- def add_log_handler(self,
- window_title: str,
- logger_instances: Iterable[logging.Logger],
- separate_log_panes=False) -> Optional[LogPane]:
+ def add_log_handler(
+ self,
+ window_title: str,
+ logger_instances: Iterable[logging.Logger],
+ separate_log_panes: bool = False,
+ log_level_name: Optional[str] = None) -> Optional[LogPane]:
"""Add the Log pane as a handler for this logger instance."""
existing_log_pane = None
@@ -554,7 +624,7 @@
existing_log_pane = self._create_log_pane(title=window_title)
for logger in logger_instances:
- _add_log_handler_to_pane(logger, existing_log_pane)
+ _add_log_handler_to_pane(logger, existing_log_pane, log_level_name)
self.window_manager.update_root_container_body()
self.update_menu_items()
@@ -641,12 +711,22 @@
if test_mode:
background_log_task = asyncio.create_task(self.log_forever())
+ background_menu_updater_task = asyncio.create_task(
+ self.background_menu_updater())
try:
unused_result = await self.application.run_async(
set_exception_handler=True)
finally:
if test_mode:
background_log_task.cancel()
+ background_menu_updater_task.cancel()
+
+ async def background_menu_updater(self):
+ """Periodically update main menu items to capture new logger names."""
+ while True:
+ await asyncio.sleep(30)
+ _LOG.debug('Update main menu items')
+ self.update_menu_items()
async def log_forever(self):
"""Test mode async log generator coroutine that runs forever."""
diff --git a/pw_console/py/pw_console/console_prefs.py b/pw_console/py/pw_console/console_prefs.py
index 61b2d0b..1a79ad7 100644
--- a/pw_console/py/pw_console/console_prefs.py
+++ b/pw_console/py/pw_console/console_prefs.py
@@ -20,6 +20,8 @@
import yaml
+from pw_console.style import get_theme_colors
+
_DEFAULT_REPL_HISTORY: Path = Path.home() / '.pw_console_history'
_DEFAULT_SEARCH_HISTORY: Path = Path.home() / '.pw_console_search'
@@ -36,6 +38,8 @@
'column_order_omit_unspecified_columns': False,
'column_order': [],
'column_colors': {},
+ 'show_python_file': False,
+ 'show_python_logger': False,
'hide_date_from_log_time': False,
# Window arrangement
'windows': {},
@@ -121,6 +125,13 @@
def ui_theme(self) -> str:
return self._config.get('ui_theme', '')
+ def set_ui_theme(self, theme_name: str):
+ self._config['ui_theme'] = theme_name
+
+ @property
+ def theme_colors(self):
+ return get_theme_colors(self.ui_theme)
+
@property
def code_theme(self) -> str:
return self._config.get('code_theme', '')
@@ -156,6 +167,19 @@
return self._config.get('hide_date_from_log_time', False)
@property
+ def show_python_file(self) -> bool:
+ return self._config.get('show_python_file', False)
+
+ @property
+ def show_python_logger(self) -> bool:
+ return self._config.get('show_python_logger', False)
+
+ def toggle_bool_option(self, name: str):
+ existing_setting = self._config[name]
+ assert isinstance(existing_setting, bool)
+ self._config[name] = not existing_setting
+
+ @property
def column_order(self) -> list:
return self._config.get('column_order', [])
@@ -194,7 +218,8 @@
titles = []
for column in self.windows.values():
for window_key_title, window_dict in column.items():
+ window_options = window_dict if window_dict else {}
# Use 'duplicate_of: Title' if it exists, otherwise use the key.
- titles.append(window_dict.get('duplicate_of',
- window_key_title))
+ titles.append(
+ window_options.get('duplicate_of', window_key_title))
return set(titles)
diff --git a/pw_console/py/pw_console/embed.py b/pw_console/py/pw_console/embed.py
index cf650ca..9ba65d7 100644
--- a/pw_console/py/pw_console/embed.py
+++ b/pw_console/py/pw_console/embed.py
@@ -14,7 +14,6 @@
"""pw_console embed class."""
import asyncio
-import copy
import logging
from pathlib import Path
from typing import Dict, List, Iterable, Optional, Union
@@ -74,6 +73,10 @@
'some_variable', 'Variable',
}
)
+
+ # Setup Python loggers to output to a file instead of STDOUT.
+ console.setup_python_logging()
+
# Then run the console with:
console.embed()
@@ -151,32 +154,17 @@
if window_pane.pane_title() in self.hidden_by_default_windows:
window_pane.show_pane = False
- def setup_python_logging(self):
- """Disable log handlers for full screen prompt_toolkit applications."""
- self.setup_python_logging_called = True
- for logger in pw_console.python_logging.all_loggers():
- # Make sure all known loggers propagate to the root logger.
- logger.propagate = True
- # Remove all stdout and stdout & stderr handlers to prevent
- # corrupting the prompt_toolkit user interface.
- for handler in copy.copy(logger.handlers):
- # Must use type() check here since this returns True:
- # isinstance(logging.FileHandler, logging.StreamHandler)
- if type(handler) == logging.StreamHandler: # pylint: disable=unidiomatic-typecheck
- logger.removeHandler(handler)
+ def setup_python_logging(self, last_resort_filename: Optional[str] = None):
+ """Disable log handlers for full screen prompt_toolkit applications.
- # Prevent these loggers from propagating to the root logger.
- logging.getLogger('pw_console').propagate = False
- # prompt_toolkit triggered debug log messages
- logging.getLogger('prompt_toolkit').propagate = False
- logging.getLogger('prompt_toolkit.buffer').propagate = False
- logging.getLogger('parso.python.diff').propagate = False
- logging.getLogger('parso.cache').propagate = False
- # Set asyncio log level to WARNING
- logging.getLogger('asyncio').setLevel(logging.WARNING)
- # Always set DEBUG level for serial debug.
- logging.getLogger('pw_console.serial_debug_logger').setLevel(
- logging.DEBUG)
+ Args:
+ last_resort_filename: If specified use this file as a fallback for
+ unhandled python logging messages. Normally Python will output
+ any log messages with no handlers to STDERR as a fallback. If
+ none, a temp file will be created instead.
+ """
+ self.setup_python_logging_called = True
+ pw_console.python_logging.setup_python_logging(last_resort_filename)
def hide_windows(self, *window_titles):
for window_title in window_titles:
diff --git a/pw_console/py/pw_console/log_line.py b/pw_console/py/pw_console/log_line.py
index cc30675..9d13d29 100644
--- a/pw_console/py/pw_console/log_line.py
+++ b/pw_console/py/pw_console/log_line.py
@@ -73,6 +73,11 @@
for key, value in extra_fields.items():
self.metadata.fields[key] = value
+ lineno = self.record.lineno
+ file_name = str(self.record.filename)
+ self.metadata.fields['py_file'] = f'{file_name}:{lineno}'
+ self.metadata.fields['py_logger'] = str(self.record.name)
+
return self.metadata
def get_fragments(self) -> StyleAndTextTuples:
diff --git a/pw_console/py/pw_console/log_view.py b/pw_console/py/pw_console/log_view.py
index 2695e1e..d00eb4a 100644
--- a/pw_console/py/pw_console/log_view.py
+++ b/pw_console/py/pw_console/log_view.py
@@ -435,6 +435,7 @@
return Point(0, 0)
# For each line rendered in the last pass:
for row, line in enumerate(self._line_fragment_cache):
+ # TODO(tonymd): This assumes every row contains exactly one '\n'
column = 0
# For each style string and raw text tuple in this line:
for style_str, text, *_ in line:
@@ -661,8 +662,8 @@
# will push the table header to the top.
if total_used_lines < self._window_height:
empty_line_count = self._window_height - total_used_lines
- self._line_fragment_cache.appendleft([('', '\n' * empty_line_count)
- ])
+ for i in range(empty_line_count):
+ self._line_fragment_cache.appendleft([('', '\n')])
self._line_fragment_cache_flattened = (
pw_console.text_formatting.flatten_formatted_text_tuples(
diff --git a/pw_console/py/pw_console/python_logging.py b/pw_console/py/pw_console/python_logging.py
index 3453201..1d494a8 100644
--- a/pw_console/py/pw_console/python_logging.py
+++ b/pw_console/py/pw_console/python_logging.py
@@ -13,10 +13,11 @@
# the License.
"""Python logging helper fuctions."""
+import copy
import logging
import tempfile
from datetime import datetime
-from typing import Iterator
+from typing import Iterator, Optional
def all_loggers() -> Iterator[logging.Logger]:
@@ -44,3 +45,53 @@
log_file_name = log_file.name
return log_file_name
+
+
+def set_logging_last_resort_file_handler(
+ file_name: Optional[str] = None) -> None:
+ log_file = file_name if file_name else create_temp_log_file()
+ logging.lastResort = logging.FileHandler(log_file)
+
+
+def disable_stdout_handlers(logger: logging.Logger) -> None:
+ """Remove all stdout and stdout & stderr logger handlers."""
+ for handler in copy.copy(logger.handlers):
+ # Must use type() check here since this returns True:
+ # isinstance(logging.FileHandler, logging.StreamHandler)
+ if type(handler) == logging.StreamHandler: # pylint: disable=unidiomatic-typecheck
+ logger.removeHandler(handler)
+
+
+def setup_python_logging(last_resort_filename: Optional[str] = None) -> None:
+ """Disable log handlers for full screen prompt_toolkit applications."""
+ disable_stdout_handlers(logging.getLogger())
+
+ if logging.lastResort is not None:
+ set_logging_last_resort_file_handler(last_resort_filename)
+
+ for logger in all_loggers():
+ # Make sure all known loggers propagate to the root logger.
+ logger.propagate = True
+ # Prevent stdout handlers from corrupting the prompt_toolkit UI.
+ disable_stdout_handlers(logger)
+
+ # Prevent these loggers from propagating to the root logger.
+ hidden_host_loggers = [
+ 'pw_console',
+ 'pw_console.plugins',
+
+ # prompt_toolkit triggered debug log messages
+ 'prompt_toolkit',
+ 'prompt_toolkit.buffer',
+ 'parso.python.diff',
+ 'parso.cache',
+ 'pw_console.serial_debug_logger',
+ ]
+ for logger_name in hidden_host_loggers:
+ logging.getLogger(logger_name).propagate = False
+
+ # Set asyncio log level to WARNING
+ logging.getLogger('asyncio').setLevel(logging.WARNING)
+
+ # Always set DEBUG level for serial debug.
+ logging.getLogger('pw_console.serial_debug_logger').setLevel(logging.DEBUG)
diff --git a/pw_console/py/pw_console/repl_pane.py b/pw_console/py/pw_console/repl_pane.py
index 83c1585..4869ba8 100644
--- a/pw_console/py/pw_console/repl_pane.py
+++ b/pw_console/py/pw_console/repl_pane.py
@@ -333,7 +333,9 @@
def _log_executed_code(self, code, prefix=''):
"""Log repl command input text along with a prefix string."""
text = self.get_output_buffer_text([code], show_index=False)
- _LOG.debug('[PYTHON] %s\n%s', prefix, text)
+ text = text.strip()
+ for line in text.splitlines():
+ _LOG.debug('[PYTHON %s] %s', prefix, line.strip())
async def periodically_check_stdout(self, user_code: UserCodeExecution,
stdout_proxy, stderr_proxy):
diff --git a/pw_console/py/pw_console/style.py b/pw_console/py/pw_console/style.py
index d4dbf3a..9b56c92 100644
--- a/pw_console/py/pw_console/style.py
+++ b/pw_console/py/pw_console/style.py
@@ -167,7 +167,6 @@
magenta_accent = '#e27e8d'
-
_THEME_NAME_MAPPING = {
'moonlight': MoonlightColors(),
'nord': NordColors(),
@@ -176,6 +175,12 @@
'high-contrast-dark': HighContrastDarkColors(),
} # yapf: disable
+
+def get_theme_colors(theme_name=''):
+ theme = _THEME_NAME_MAPPING.get(theme_name, DarkColors())
+ return theme
+
+
def generate_styles(theme_name='dark'):
"""Return prompt_toolkit styles for the given theme name."""
# Use DarkColors() if name not found.
diff --git a/pw_console/py/pw_console/widgets/table.py b/pw_console/py/pw_console/widgets/table.py
index 51fb65e..6b3f550 100644
--- a/pw_console/py/pw_console/widgets/table.py
+++ b/pw_console/py/pw_console/widgets/table.py
@@ -15,7 +15,6 @@
import collections
import copy
-import logging
from prompt_toolkit.formatted_text import StyleAndTextTuples
@@ -23,8 +22,6 @@
from pw_console.log_line import LogLine
import pw_console.text_formatting
-_LOG = logging.getLogger(__package__)
-
class TableView:
"""Store column information and render logs into formatted tables."""
@@ -72,23 +69,28 @@
def _ordered_column_widths(self):
"""Return each column and width in the preferred order."""
- # Reverse sort if no custom ordering.
- if not self.prefs.column_order:
- return self.column_widths.items()
+ if self.prefs.column_order:
+ # Get ordered_columns
+ columns = copy.copy(self.column_widths)
+ ordered_columns = {}
- # Get ordered_columns
- columns = copy.copy(self.column_widths)
- ordered_columns = {}
+ for column_name in self.prefs.column_order:
+ # If valid column name
+ if column_name in columns:
+ ordered_columns[column_name] = columns.pop(column_name)
- for column_name in self.prefs.column_order:
- # If valid column name
- if column_name in columns:
- ordered_columns[column_name] = columns.pop(column_name)
- # NOTE: Any remaining columns not specified by the user are not shown.
- # Perhaps a user setting could add them at the end. To add them in:
- if not self.prefs.omit_unspecified_columns:
- for column_name in columns:
- ordered_columns[column_name] = columns[column_name]
+ # Add remaining columns unless user preference to hide them.
+ if not self.prefs.omit_unspecified_columns:
+ for column_name in columns:
+ ordered_columns[column_name] = columns[column_name]
+ else:
+ ordered_columns = copy.copy(self.column_widths)
+
+ if not self.prefs.show_python_file and 'py_file' in ordered_columns:
+ del ordered_columns['py_file']
+ if not self.prefs.show_python_logger and 'py_logger' in ordered_columns:
+ del ordered_columns['py_logger']
+
return ordered_columns.items()
def update_metadata_column_widths(self, log: LogLine):
diff --git a/pw_console/py/pw_console/window_manager.py b/pw_console/py/pw_console/window_manager.py
index 3fe48a3..5cd5721 100644
--- a/pw_console/py/pw_console/window_manager.py
+++ b/pw_console/py/pw_console/window_manager.py
@@ -413,7 +413,8 @@
self.window_lists[column_index].display_mode = DisplayMode.STACK
# Add windows to the this column (window_list)
- for window_title, window_options in windows.items():
+ for window_title, window_dict in windows.items():
+ window_options = window_dict if window_dict else {}
new_pane = None
desired_window_title = window_title
# Check for duplicate_of: Title value
diff --git a/pw_console/py/table_test.py b/pw_console/py/table_test.py
index a6325d5..06343d8 100644
--- a/pw_console/py/table_test.py
+++ b/pw_console/py/table_test.py
@@ -108,14 +108,18 @@
table = TableView(self.prefs)
for log in logs:
table.update_metadata_column_widths(log)
+ metadata_fields = {
+ k: v
+ for k, v in log.metadata.fields.items()
+ if k not in ['py_file', 'py_logger']
+ }
# update_metadata_column_widths shoulp populate self.metadata.fields
- self.assertEqual(log.metadata.fields,
- log.record.extra_metadata_fields)
+ self.assertEqual(metadata_fields, log.record.extra_metadata_fields)
# Check expected column widths
results = {
k: v
for k, v in dict(table.column_widths).items()
- if k not in ['time', 'level']
+ if k not in ['time', 'level', 'py_file', 'py_logger']
}
self.assertCountEqual(expected_widths, results)
@@ -160,6 +164,7 @@
def test_formatted_header(self, _name, logs, expected_headers) -> None:
"""Test colum widths calculation."""
table = TableView(self.prefs)
+
for log, header in zip(logs, expected_headers):
table.update_metadata_column_widths(log)
self.assertEqual(table.formatted_header(), header)
