pw_trace: Add python chrome trace generator
Add a python tool to generate traces which can be viewed using
chrome://tracing.
Change-Id: Ibe2d248e287a9329d00abec6002defb907a1b64e
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/14804
Commit-Queue: Rob Oliver <rgoliver@google.com>
Reviewed-by: (☞ﾟ∀ﾟ)☞ Tennessee Carmel-Veilleux <tennessee@google.com>
diff --git a/pw_trace/docs.rst b/pw_trace/docs.rst
index 39beda7..37e7d89 100644
--- a/pw_trace/docs.rst
+++ b/pw_trace/docs.rst
@@ -179,6 +179,19 @@
- *data* - A pointer to a buffer of arbitrary caller-provided data (void*).
- *size* - The size of the data (size_t).
+Currently the included python tool supports a few different options for
+*data_format_string*:
+
+- *@pw_arg_label* - Uses the string in the data as the trace event label.
+- *@pw_arg_group* - Uses the string in the data as the trace event group.
+- *@pw_arg_counter* - Uses the data as a little endian integer value, and
+ visualizes it as a counter value in the trace (on a graph).
+- *@pw_py_struct_fmt:* - Interprets the string after the ":" as a python struct
+ format string, and uses that format string to unpack the data elements. This
+ can be used to either provide a single value type, or provide multiple
+ different values with a variety of types. Options for format string types can
+ be found here: https://docs.python.org/3/library/struct.html#format-characters
+
.. tip::
It is ok for only one event of a start/end pair to contain data, as long the
@@ -244,8 +257,8 @@
provided.
- *PW_TRACE_FLAGS_DEFAULT*: Default value if no flags are provided.
-- *PW_TRACE_TRACE_ID_DEFAULT*: Default value if not trace_id provided.
-- *PW_TRACE_GROUP_LABEL_DEFAULT*: Default value if not group_label provided.
+- *PW_TRACE_TRACE_ID_DEFAULT*: Default value if no trace_id provided.
+- *PW_TRACE_GROUP_LABEL_DEFAULT*: Default value if no group_label provided.
----------
Sample App
@@ -267,3 +280,18 @@
Jobs are intentionally made to have multiple stages of processing (simulated by
being re-added to the work-queue). This results in multiple jobs being handled
at the same time, the trace_id is used to separate these traces.
+
+----------------------
+Python Trace Generator
+----------------------
+The Python tool is still in early developments, but currently it supports
+generating a list of json lines from a list of trace events.
+
+To view the trace, these lines can be saved to a file and loaded into
+chrome://tracing.
+
+Future work will look to add:
+
+- Config options to customize output.
+- A method of providing custom data formatters.
+- Perfetto support.
diff --git a/pw_trace/py/pw_trace/__init__.py b/pw_trace/py/pw_trace/__init__.py
new file mode 100644
index 0000000..af9ef88
--- /dev/null
+++ b/pw_trace/py/pw_trace/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Package that supports traces."""
diff --git a/pw_trace/py/pw_trace/trace.py b/pw_trace/py/pw_trace/trace.py
new file mode 100755
index 0000000..f30ba2f
--- /dev/null
+++ b/pw_trace/py/pw_trace/trace.py
@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+r"""
+
+Trace module which creates trace files from a list of trace events.
+
+This is a work in progress, future work will look to add:
+ - Config options to customize output.
+ - A method of providing custom data formatters.
+ - Perfetto support.
+"""
+from enum import Enum
+import json
+import logging
+import struct
+from typing import Iterable, NamedTuple
+
+_LOG = logging.getLogger('pw_trace')
+
+
+class TraceType(Enum):
+ Invalid = 0
+ Instantaneous = 1
+ InstantaneousGroup = 2
+ AsyncStart = 3
+ AsyncStep = 4
+ AsyncEnd = 5
+ DurationStart = 6
+ DurationEnd = 7
+ DurationGroupStart = 8
+ DurationGroupEnd = 9
+
+
+class TraceEvent(NamedTuple):
+ event_type: TraceType
+ module: str
+ label: str
+ timestamp: int
+ group: str = ""
+ trace_id: int = 0
+ flags: int = 0
+ has_data: bool = False
+ data_fmt: str = ""
+ data: bytes = b''
+
+
+def event_has_trace_id(event_type):
+ return event_type in {
+ "kPwTraceEvent_AsyncStart", "kPwTraceEvent_AsyncStep",
+ "kPwTraceEvent_AsyncEnd"
+ }
+
+
+def generate_trace_json(events: Iterable[TraceEvent]):
+ """Generates a list of JSON lines from provided trace events."""
+ json_lines = []
+ for event in events:
+ if event.module is None or event.timestamp is None or \
+ event.event_type is None or event.label is None:
+ _LOG.error("Invalid sample")
+ continue
+
+ line = {
+ "pid": event.module,
+ "name": (event.label),
+ "ts": event.timestamp
+ }
+ if event.event_type == TraceType.DurationStart:
+ line["ph"] = "B"
+ line["tid"] = event.label
+ elif event.event_type == TraceType.DurationEnd:
+ line["ph"] = "E"
+ line["tid"] = event.label
+ elif event.event_type == TraceType.DurationGroupStart:
+ line["ph"] = "B"
+ line["tid"] = event.group
+ elif event.event_type == TraceType.DurationGroupEnd:
+ line["ph"] = "E"
+ line["tid"] = event.group
+ elif event.event_type == TraceType.Instantaneous:
+ line["ph"] = "I"
+ line["s"] = "p"
+ elif event.event_type == TraceType.InstantaneousGroup:
+ line["ph"] = "I"
+ line["s"] = "t"
+ line["tid"] = event.group
+ elif event.event_type == TraceType.AsyncStart:
+ line["ph"] = "b"
+ line["scope"] = event.group
+ line["tid"] = event.group
+ line["cat"] = event.module
+ line["id"] = event.trace_id
+ line["args"] = {"id": line["id"]}
+ elif event.event_type == TraceType.AsyncStep:
+ line["ph"] = "n"
+ line["scope"] = event.group
+ line["tid"] = event.group
+ line["cat"] = event.module
+ line["id"] = event.trace_id
+ line["args"] = {"id": line["id"]}
+ elif event.event_type == TraceType.AsyncEnd:
+ line["ph"] = "e"
+ line["scope"] = event.group
+ line["tid"] = event.group
+ line["cat"] = event.module
+ line["id"] = event.trace_id
+ line["args"] = {"id": line["id"]}
+ else:
+ _LOG.error("Unknown event type, skipping")
+ continue
+
+ # Handle Data
+ if event.has_data:
+ if event.data_fmt == "@pw_arg_label":
+ line["name"] = event.data.decode("utf-8")
+ elif event.data_fmt == "@pw_arg_group":
+ line["tid"] = event.data.decode("utf-8")
+ elif event.data_fmt == "@pw_arg_counter":
+ line["ph"] = "C"
+ line["args"] = {
+ line["name"]: int.from_bytes(event.data, "little")
+ }
+ elif event.data_fmt.startswith("@pw_py_struct_fmt:"):
+ items = struct.unpack_from(
+ event.data_fmt[len("@pw_py_struct_fmt:"):], event.data)
+ args = {}
+ for i, item in enumerate(items):
+ args["data_" + str(i)] = item
+ line["args"] = args
+
+ # Encode as JSON
+ json_lines.append(json.dumps(line))
+
+ return json_lines
diff --git a/pw_trace/py/setup.py b/pw_trace/py/setup.py
new file mode 100644
index 0000000..65e6126
--- /dev/null
+++ b/pw_trace/py/setup.py
@@ -0,0 +1,26 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""The pw_trace package."""
+
+import setuptools
+
+setuptools.setup(
+ name='pw_trace',
+ version='0.0.1',
+ author='Pigweed Authors',
+ author_email='pigweed-developers@googlegroups.com',
+ description='Tools for dealing with trace data',
+ packages=setuptools.find_packages(),
+ test_suite='setup.test_suite',
+)
diff --git a/pw_trace/py/trace_test.py b/pw_trace/py/trace_test.py
new file mode 100755
index 0000000..f9bab22
--- /dev/null
+++ b/pw_trace/py/trace_test.py
@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Tests the trace module."""
+
+import json
+import struct
+import unittest
+
+from pw_trace import trace
+
+test_events = [
+ trace.TraceEvent(trace.TraceType.Instantaneous, "m1", "L1", 1),
+ trace.TraceEvent(trace.TraceType.InstantaneousGroup, "m2", "L2", 2, "G2"),
+ trace.TraceEvent(trace.TraceType.AsyncStep, "m3", "L3", 3, "G3", 103),
+ trace.TraceEvent(trace.TraceType.DurationStart, "m4", "L4", 4),
+ trace.TraceEvent(trace.TraceType.DurationGroupStart, "m5", "L5", 5, "G5"),
+ trace.TraceEvent(trace.TraceType.AsyncStart, "m6", "L6", 6, "G6", 106),
+ trace.TraceEvent(trace.TraceType.DurationEnd, "m7", "L7", 7),
+ trace.TraceEvent(trace.TraceType.DurationGroupEnd, "m8", "L8", 8, "G8"),
+ trace.TraceEvent(trace.TraceType.AsyncEnd, "m9", "L9", 9, "G9", 109)
+]
+
+test_json = [
+ {"ph": "I", "pid": "m1", "name": "L1", "ts": 1, "s": "p"},
+ {"ph": "I", "pid": "m2", "tid": "G2", "name": "L2", "ts": 2, "s": "t"},
+ {"ph": "n", "pid": "m3", "tid": "G3", "name": "L3", "ts": 3, \
+ "scope": "G3", "cat": "m3", "id": 103, "args": {"id": 103}},
+ {"ph": "B", "pid": "m4", "tid": "L4", "name": "L4", "ts": 4},
+ {"ph": "B", "pid": "m5", "tid": "G5", "name": "L5", "ts": 5},
+ {"ph": "b", "pid": "m6", "tid": "G6", "name": "L6", "ts": 6, \
+ "scope": "G6", "cat": "m6", "id": 106, "args": {"id": 106}},
+ {"ph": "E", "pid": "m7", "tid": "L7", "name": "L7", "ts": 7},
+ {"ph": "E", "pid": "m8", "tid": "G8", "name": "L8", "ts": 8},
+ {"ph": "e", "pid": "m9", "tid": "G9", "name": "L9", "ts": 9, \
+ "scope": "G9", "cat": "m9", "id": 109, "args": {"id": 109}},
+]
+
+
+class TestTraceGenerateJson(unittest.TestCase):
+ """Tests generate json with various events."""
+ def test_generate_single_json_event(self):
+ event = trace.TraceEvent(event_type=trace.TraceType.Instantaneous,
+ module="module",
+ label="label",
+ timestamp=10)
+ json_lines = trace.generate_trace_json([event])
+ self.assertEqual(1, len(json_lines))
+ self.assertEqual(json.loads(json_lines[0]), {
+ "ph": "I",
+ "pid": "module",
+ "name": "label",
+ "ts": 10,
+ "s": "p"
+ })
+
+ def test_generate_multiple_json_events(self):
+ json_lines = trace.generate_trace_json(test_events)
+ self.assertEqual(len(test_json), len(json_lines))
+ for actual, expected in zip(json_lines, test_json):
+ self.assertEqual(expected, json.loads(actual))
+
+ def test_generate_json_data_arg_label(self):
+ event = trace.TraceEvent(
+ event_type=trace.TraceType.Instantaneous,
+ module="module",
+ label="", # Is replaced by data string
+ timestamp=10,
+ has_data=True,
+ data_fmt="@pw_arg_label",
+ data=bytes("arg", "utf-8"))
+ json_lines = trace.generate_trace_json([event])
+ self.assertEqual(1, len(json_lines))
+ self.assertEqual(json.loads(json_lines[0]), {
+ "ph": "I",
+ "pid": "module",
+ "name": "arg",
+ "ts": 10,
+ "s": "p"
+ })
+
+ def test_generate_json_data_arg_group(self):
+ event = trace.TraceEvent(event_type=trace.TraceType.InstantaneousGroup,
+ module="module",
+ label="label",
+ timestamp=10,
+ has_data=True,
+ data_fmt="@pw_arg_group",
+ data=bytes("arg", "utf-8"))
+ json_lines = trace.generate_trace_json([event])
+ self.assertEqual(1, len(json_lines))
+ self.assertEqual(
+ json.loads(json_lines[0]), {
+ "ph": "I",
+ "pid": "module",
+ "name": "label",
+ "tid": "arg",
+ "ts": 10,
+ "s": "t"
+ })
+
+ def test_generate_json_data_counter(self):
+ event = trace.TraceEvent(event_type=trace.TraceType.Instantaneous,
+ module="module",
+ label="counter",
+ timestamp=10,
+ has_data=True,
+ data_fmt="@pw_arg_counter",
+ data=(5).to_bytes(4, byteorder="little"))
+ json_lines = trace.generate_trace_json([event])
+ self.assertEqual(1, len(json_lines))
+ self.assertEqual(
+ json.loads(json_lines[0]), {
+ "ph": "C",
+ "pid": "module",
+ "name": "counter",
+ "ts": 10,
+ "s": "p",
+ "args": {
+ "counter": 5
+ }
+ })
+
+ def test_generate_json_data_struct_fmt_single(self):
+ event = trace.TraceEvent(event_type=trace.TraceType.Instantaneous,
+ module="module",
+ label="counter",
+ timestamp=10,
+ has_data=True,
+ data_fmt="@pw_py_struct_fmt:H",
+ data=(5).to_bytes(2, byteorder="little"))
+ json_lines = trace.generate_trace_json([event])
+ self.assertEqual(1, len(json_lines))
+ self.assertEqual(
+ json.loads(json_lines[0]), {
+ "ph": "I",
+ "pid": "module",
+ "name": "counter",
+ "ts": 10,
+ "s": "p",
+ "args": {
+ "data_0": 5
+ }
+ })
+
+ def test_generate_json_data_struct_fmt_multi(self):
+ event = trace.TraceEvent(event_type=trace.TraceType.Instantaneous,
+ module="module",
+ label="counter",
+ timestamp=10,
+ has_data=True,
+ data_fmt="@pw_py_struct_fmt:Hl",
+ data=struct.pack("Hl", 5, 2))
+ json_lines = trace.generate_trace_json([event])
+ self.assertEqual(1, len(json_lines))
+ self.assertEqual(
+ json.loads(json_lines[0]), {
+ "ph": "I",
+ "pid": "module",
+ "name": "counter",
+ "ts": 10,
+ "s": "p",
+ "args": {
+ "data_0": 5,
+ "data_1": 2
+ }
+ })
+
+
+if __name__ == '__main__':
+ unittest.main()
