pw_tokenizer: Tool for detokenizing from serial
pw_tokenizer.serial_detokenizer detokenizes prefixed Base64-encoded
strings in the output from a serial port.
Change-Id: I0675192d40fff2cf2db5db8a219b3326b2780b36
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/15365
Commit-Queue: Wyatt Hepler <hepler@google.com>
Reviewed-by: Armando Montanez <amontanez@google.com>
diff --git a/pw_tokenizer/docs.rst b/pw_tokenizer/docs.rst
index 9a5b369..48d66a7 100644
--- a/pw_tokenizer/docs.rst
+++ b/pw_tokenizer/docs.rst
@@ -692,6 +692,29 @@
TransmitLogMessage(base64_buffer, base64_size);
}
+Command line utilities
+^^^^^^^^^^^^^^^^^^^^^^
+``pw_tokenizer`` provides two standalone command line utilities for detokenizing
+Base64-encoded tokenized strings.
+
+* ``detokenize.py`` -- Detokenizes Base64-encoded strings in files or from
+ stdin.
+* ``detokenize_serial.py`` -- Detokenizes Base64-encoded strings from a
+ connected serial device.
+
+If the ``pw_tokenizer`` Python package is installed, these tools may be executed
+as runnable modules. For example:
+
+.. code-block::
+
+ # Detokenize Base64-encoded strings in a file
+ python -m pw_tokenizer.detokenize -i input_file.txt
+
+ # Detokenize Base64-encoded strings in output from a serial device
+ python -m pw_tokenizer.detokenize_serial --device /dev/ttyACM0
+
+See the ``--help`` options for these tools for full usage information.
+
Deployment war story
====================
The tokenizer module was developed to bring tokenized logging to an
diff --git a/pw_tokenizer/py/pw_tokenizer/__main__.py b/pw_tokenizer/py/pw_tokenizer/__main__.py
index 3ed2835..4a7bdf2 100644
--- a/pw_tokenizer/py/pw_tokenizer/__main__.py
+++ b/pw_tokenizer/py/pw_tokenizer/__main__.py
@@ -15,4 +15,4 @@
from pw_tokenizer import detokenize
-detokenize._main(detokenize._parse_args()) # pylint: disable=protected-access
+detokenize.main()
diff --git a/pw_tokenizer/py/pw_tokenizer/detokenize.py b/pw_tokenizer/py/pw_tokenizer/detokenize.py
index 2ec0369..040978e 100755
--- a/pw_tokenizer/py/pw_tokenizer/detokenize.py
+++ b/pw_tokenizer/py/pw_tokenizer/detokenize.py
@@ -343,13 +343,14 @@
return decode_and_detokenize
+BASE64_PREFIX = b'$'
DEFAULT_RECURSION = 9
def detokenize_base64_live(detokenizer,
input_file,
output,
- prefix=b'$',
+ prefix=BASE64_PREFIX,
recursion=DEFAULT_RECURSION):
"""Reads chars one-at-a-time and decodes messages; SLOW for big files."""
transform = _detokenize_prefixed_base64(detokenizer, prefix, recursion)
@@ -367,7 +368,7 @@
def detokenize_base64_to_file(detokenizer,
data,
output,
- prefix=b'$',
+ prefix=BASE64_PREFIX,
recursion=DEFAULT_RECURSION):
"""Decodes prefixed Base64 messages in data; decodes to an output file."""
transform = _detokenize_prefixed_base64(detokenizer, prefix, recursion)
@@ -389,7 +390,7 @@
def detokenize_base64(detokenizer,
data,
- prefix=b'$',
+ prefix=BASE64_PREFIX,
recursion=DEFAULT_RECURSION):
"""Decodes and replaces prefixed Base64 messages in the provided data.
@@ -428,7 +429,7 @@
def _parse_args():
- """Parse and return command line arguments."""
+ """Parses and return command line arguments."""
parser = argparse.ArgumentParser(
description=__doc__,
@@ -460,7 +461,7 @@
subparser.add_argument(
'-p',
'--prefix',
- default='$',
+ default=BASE64_PREFIX,
help=('The one-character prefix that signals the start of a '
'Base64-encoded message. (default: $)'))
subparser.add_argument(
@@ -473,7 +474,9 @@
return parser.parse_args()
-def _main(args):
+def main():
+ args = _parse_args()
+
handler = args.handler
del args.handler
@@ -483,4 +486,4 @@
if __name__ == '__main__':
if sys.version_info[0] < 3:
sys.exit('ERROR: The detokenizer command line tools require Python 3.')
- _main(_parse_args())
+ sys.exit(main())
diff --git a/pw_tokenizer/py/pw_tokenizer/serial_detokenizer.py b/pw_tokenizer/py/pw_tokenizer/serial_detokenizer.py
new file mode 100644
index 0000000..d017a8c
--- /dev/null
+++ b/pw_tokenizer/py/pw_tokenizer/serial_detokenizer.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Decodes and detokenizes Base64-encoded strings in serial output.
+
+The output is printed or saved to a file. Input is not supported.
+"""
+
+import argparse
+import sys
+from typing import BinaryIO, Iterable
+
+import serial
+from pw_tokenizer import database, detokenize, tokens
+
+
+def _parse_args():
+ """Parses and return command line arguments."""
+
+ parser = argparse.ArgumentParser(
+ description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ parents=[database.token_databases_parser()])
+ parser.add_argument('-d',
+ '--device',
+ required=True,
+ help='The serial device from which to read')
+ parser.add_argument('-b',
+ '--baudrate',
+ type=int,
+ default=115200,
+ help='The baud rate for the serial device')
+ parser.add_argument('-o',
+ '--output',
+ type=argparse.FileType('wb'),
+ default=sys.stdout.buffer,
+ help=('The file to which to write the output; '
+ 'provide - or omit for stdout.'))
+ parser.add_argument(
+ '-p',
+ '--prefix',
+ default=detokenize.BASE64_PREFIX,
+ help=('The one-character prefix that signals the start of a '
+ 'Base64-encoded message. (default: $)'))
+ parser.add_argument(
+ '-s',
+ '--show_errors',
+ action='store_true',
+ help=('Show error messages instead of conversion specifiers when '
+ 'arguments cannot be decoded.'))
+
+ return parser.parse_args()
+
+
+def _detokenize_serial(databases: Iterable, device: serial.Serial,
+ baudrate: int, show_errors: bool, output: BinaryIO,
+ prefix: str) -> None:
+ if output is sys.stdout:
+ output = sys.stdout.buffer
+
+ detokenizer = detokenize.Detokenizer(tokens.Database.merged(*databases),
+ show_errors=show_errors)
+ serial_device = serial.Serial(port=device, baudrate=baudrate)
+
+ try:
+ detokenize.detokenize_base64_live(detokenizer, serial_device, output,
+ prefix)
+ except KeyboardInterrupt:
+ output.flush()
+
+
+def main():
+ _detokenize_serial(**vars(_parse_args()))
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/pw_tokenizer/py/setup.py b/pw_tokenizer/py/setup.py
index 3b3dec2..9d9a96e 100644
--- a/pw_tokenizer/py/setup.py
+++ b/pw_tokenizer/py/setup.py
@@ -23,4 +23,5 @@
description='Tools for decoding tokenized strings',
packages=setuptools.find_packages(),
test_suite='setup.test_suite',
+ extra_requires=['serial'],
)
