pw_trace: Add basic RPCs to get trace
Add a basic RPC service to trace which currently provides 3 RPCs:
- Enable(bool), Turns tracing on or off
- IsEnabled, Returns if tracing is currently on or off
- GetTraceData, Streams the encoded trace data.
This CL also adds a python script which uses these RPCs to retrieve
and decode trace data from a connected device.
Change-Id: Iaceb3fa87017939a17512738101af70bf335504a
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/31880
Reviewed-by: Paul Mathieu <paulmathieu@google.com>
Commit-Queue: Rob Oliver <rgoliver@google.com>
diff --git a/BUILD.gn b/BUILD.gn
index 0c1ebdb..ae068a3 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -201,6 +201,7 @@
"$dir_pw_trace:trace_example_basic",
"$dir_pw_trace_tokenized:trace_tokenized_example_basic",
"$dir_pw_trace_tokenized:trace_tokenized_example_filter",
+ "$dir_pw_trace_tokenized:trace_tokenized_example_rpc",
"$dir_pw_trace_tokenized:trace_tokenized_example_trigger",
]
}
diff --git a/pw_trace_tokenized/BUILD b/pw_trace_tokenized/BUILD
index 4c3175f..1944d71 100644
--- a/pw_trace_tokenized/BUILD
+++ b/pw_trace_tokenized/BUILD
@@ -61,6 +61,24 @@
)
pw_cc_library(
+ name = "trace_rpc_service",
+ hdrs = [
+ "public/pw_trace_tokenized/trace_rpc_service_nanopb.h",
+ ],
+ includes = [
+ "public",
+ ],
+ srcs = [
+ "trace_rpc_service_nanopb.cc",
+ ],
+ deps = [
+ "//pw_log",
+ "//pw_trace",
+ "//pw_trace_tokenized_buffer",
+ ],
+)
+
+pw_cc_library(
name = "trace_buffer_headers",
hdrs = [
"public/pw_trace_tokenized/trace_buffer.h",
@@ -205,3 +223,17 @@
],
srcs = [ "example/filter.cc" ]
)
+
+pw_cc_library(
+ name = "trace_tokenized_example_rpc",
+ deps = [
+ ":pw_trace_rpc_service",
+ "//dir_pw_rpc:server",
+ "//dir_pw_rpc:system_server",
+ "//pw_log",
+ "//pw_hdlc",
+ "//dir_pw_trace",
+ "//dir_pw_trace:pw_trace_sample_app",
+ ],
+ srcs = [ "example/rpc.cc" ]
+)
\ No newline at end of file
diff --git a/pw_trace_tokenized/BUILD.gn b/pw_trace_tokenized/BUILD.gn
index c031e89..8468e8a 100644
--- a/pw_trace_tokenized/BUILD.gn
+++ b/pw_trace_tokenized/BUILD.gn
@@ -15,7 +15,10 @@
import("//build_overrides/pigweed.gni")
import("$dir_pw_build/module_config.gni")
+import("$dir_pw_build/target_types.gni")
import("$dir_pw_docgen/docs.gni")
+import("$dir_pw_protobuf_compiler/proto.gni")
+import("$dir_pw_third_party/nanopb/nanopb.gni")
import("$dir_pw_unit_test/test.gni")
declare_args() {
@@ -85,6 +88,25 @@
defines = [ "PW_TRACE_BUFFER_SIZE_BYTES=${pw_trace_tokenized_BUFFER_SIZE}" ]
}
+pw_proto_library("trace_rpc_service_proto") {
+ sources = [ "pw_trace_protos/trace_rpc.proto" ]
+ inputs = [ "pw_trace_protos/trace_rpc.options" ]
+}
+
+pw_source_set("trace_rpc_service") {
+ public_configs = [ ":public_include_path" ]
+ public_deps = [ ":trace_rpc_service_proto.nanopb_rpc" ]
+ deps = [
+ ":tokenized_trace_buffer",
+ "$dir_pw_log",
+ "$dir_pw_trace",
+ ]
+ sources = [
+ "public/pw_trace_tokenized/trace_rpc_service_nanopb.h",
+ "trace_rpc_service_nanopb.cc",
+ ]
+}
+
pw_source_set("tokenized_trace_buffer") {
deps = [ ":pw_trace_tokenized_core" ]
public_deps = [
@@ -209,3 +231,21 @@
]
sources = [ "example/filter.cc" ]
}
+
+if (dir_pw_third_party_nanopb == "") {
+ group("trace_tokenized_example_rpc") {
+ }
+} else {
+ pw_executable("trace_tokenized_example_rpc") {
+ sources = [ "example/rpc.cc" ]
+ deps = [
+ ":trace_rpc_service",
+ "$dir_pw_hdlc",
+ "$dir_pw_log",
+ "$dir_pw_rpc:server",
+ "$dir_pw_rpc/system_server",
+ "$dir_pw_trace",
+ "$dir_pw_trace:trace_sample_app",
+ ]
+ }
+}
diff --git a/pw_trace_tokenized/example/rpc.cc b/pw_trace_tokenized/example/rpc.cc
new file mode 100644
index 0000000..8126d6a
--- /dev/null
+++ b/pw_trace_tokenized/example/rpc.cc
@@ -0,0 +1,66 @@
+// Copyright 2021 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+//==============================================================================
+/*
+BUILD
+ninja -C out
+host_clang_debug/obj/pw_trace_tokenized/bin/trace_tokenized_example_rpc
+
+RUN
+.out/host_clang_debug/obj/pw_trace_tokenized/bin/trace_tokenized_example_rpc
+
+DECODE
+python pw_trace_tokenized/py/pw_trace_tokenized/get_trace.py
+ -s localhost:33000
+ -o trace.json
+ -t
+ out/host_clang_debug/obj/pw_trace_tokenized/bin/trace_tokenized_example_rpc
+ pw_trace_tokenized/pw_trace_protos/trace_rpc.proto
+
+VIEW
+In chrome navigate to chrome://tracing, and load the trace.json file.
+*/
+#include <thread>
+
+#include "pw_log/log.h"
+#include "pw_rpc/server.h"
+#include "pw_rpc_system_server/rpc_server.h"
+#include "pw_trace/example/sample_app.h"
+#include "pw_trace/trace.h"
+#include "pw_trace_tokenized/trace_rpc_service_nanopb.h"
+
+namespace {
+
+pw::trace::TraceService trace_service;
+
+void RpcThread() {
+ pw::rpc::system_server::Init();
+
+ // Set up the server and start processing data.
+ pw::rpc::system_server::Server().RegisterService(trace_service);
+ pw::rpc::system_server::Start();
+}
+
+} // namespace
+
+int main() {
+ std::thread rpc_thread(RpcThread);
+
+ // Enable tracing.
+ PW_TRACE_SET_ENABLED(true);
+
+ PW_LOG_INFO("Running basic trace example...\n");
+ RunTraceSampleApp();
+ return 0;
+}
\ No newline at end of file
diff --git a/pw_trace_tokenized/public/pw_trace_tokenized/trace_rpc_service_nanopb.h b/pw_trace_tokenized/public/pw_trace_tokenized/trace_rpc_service_nanopb.h
new file mode 100644
index 0000000..cc326c1
--- /dev/null
+++ b/pw_trace_tokenized/public/pw_trace_tokenized/trace_rpc_service_nanopb.h
@@ -0,0 +1,35 @@
+// Copyright 2021 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+#pragma once
+
+#include "pw_trace_protos/trace_rpc.rpc.pb.h"
+
+namespace pw::trace {
+
+class TraceService final : public generated::TraceService<TraceService> {
+ public:
+ pw::Status Enable(ServerContext&,
+ const pw_trace_TraceEnableMessage& request,
+ pw_trace_TraceEnableMessage& response);
+
+ pw::Status IsEnabled(ServerContext&,
+ const pw_trace_Empty& request,
+ pw_trace_TraceEnableMessage& response);
+
+ void GetTraceData(ServerContext&,
+ const pw_trace_Empty& request,
+ ServerWriter<pw_trace_TraceDataMessage>& writer);
+};
+
+} // namespace pw::trace
diff --git a/pw_trace_tokenized/pw_trace_protos/trace_rpc.options b/pw_trace_tokenized/pw_trace_protos/trace_rpc.options
new file mode 100644
index 0000000..37508ac
--- /dev/null
+++ b/pw_trace_tokenized/pw_trace_protos/trace_rpc.options
@@ -0,0 +1,15 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+pw.trace.TraceDataMessage.data max_size:64
diff --git a/pw_trace_tokenized/pw_trace_protos/trace_rpc.proto b/pw_trace_tokenized/pw_trace_protos/trace_rpc.proto
new file mode 100644
index 0000000..39971e7
--- /dev/null
+++ b/pw_trace_tokenized/pw_trace_protos/trace_rpc.proto
@@ -0,0 +1,32 @@
+// Copyright 2021 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+syntax = "proto3";
+
+package pw.trace;
+
+service TraceService {
+ rpc Enable(TraceEnableMessage) returns (TraceEnableMessage) {}
+ rpc IsEnabled(Empty) returns (TraceEnableMessage) {}
+ rpc GetTraceData(Empty) returns (stream TraceDataMessage) {}
+}
+
+message Empty {}
+
+message TraceEnableMessage {
+ bool enable = 1;
+}
+
+message TraceDataMessage {
+ bytes data = 1;
+}
diff --git a/pw_trace_tokenized/py/BUILD.gn b/pw_trace_tokenized/py/BUILD.gn
index 00b85f8..ef75ba6 100644
--- a/pw_trace_tokenized/py/BUILD.gn
+++ b/pw_trace_tokenized/py/BUILD.gn
@@ -20,6 +20,7 @@
setup = [ "setup.py" ]
sources = [
"pw_trace_tokenized/__init__.py",
+ "pw_trace_tokenized/get_trace.py",
"pw_trace_tokenized/trace_tokenized.py",
]
python_deps = [ "$dir_pw_tokenizer/py" ]
diff --git a/pw_trace_tokenized/py/pw_trace_tokenized/get_trace.py b/pw_trace_tokenized/py/pw_trace_tokenized/get_trace.py
new file mode 100755
index 0000000..988f62b
--- /dev/null
+++ b/pw_trace_tokenized/py/pw_trace_tokenized/get_trace.py
@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+# Copyright 2021 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+r"""
+Generates json trace files viewable using chrome://tracing using RPCs from a
+connected HdlcRpcClient.
+
+Example usage:
+python pw_trace_tokenized/py/pw_trace_tokenized/get_trace.py -s localhost:33000
+ -o trace.json
+ -t out/host_clang_debug/obj/pw_trace_tokenized/bin/trace_tokenized_example_rpc
+ pw_trace_tokenized/pw_trace_protos/trace_rpc.proto
+"""
+import argparse
+import logging
+import glob
+from pathlib import Path
+import sys
+from typing import Collection, Iterable, Iterator
+import serial # type: ignore
+from pw_tokenizer import database
+from pw_trace import trace
+from pw_hdlc.rpc import HdlcRpcClient, default_channels
+from pw_hdlc.rpc_console import SocketClientImpl
+from pw_trace_tokenized import trace_tokenized
+
+_LOG = logging.getLogger('pw_trace_tokenizer')
+
+PW_RPC_MAX_PACKET_SIZE = 256
+SOCKET_SERVER = 'localhost'
+SOCKET_PORT = 33000
+MKFIFO_MODE = 0o666
+
+
+def _expand_globs(globs: Iterable[str]) -> Iterator[Path]:
+ for pattern in globs:
+ for file in glob.glob(pattern, recursive=True):
+ yield Path(file)
+
+
+def get_hdlc_rpc_client(device: str, baudrate: int,
+ proto_globs: Collection[str], socket_addr: str,
+ **kwargs):
+ """Get the HdlcRpcClient based on arguments."""
+ del kwargs # ignore
+ if not proto_globs:
+ proto_globs = ['**/*.proto']
+
+ protos = list(_expand_globs(proto_globs))
+
+ if not protos:
+ _LOG.critical('No .proto files were found with %s',
+ ', '.join(proto_globs))
+ _LOG.critical('At least one .proto file is required')
+ return 1
+
+ _LOG.debug('Found %d .proto files found with %s', len(protos),
+ ', '.join(proto_globs))
+
+ # TODO(rgoliver): When pw has a generalized transport for RPC this should
+ # use it so it isn't specific to HDLC
+ if socket_addr is None:
+ serial_device = serial.Serial(device, baudrate, timeout=1)
+ read = lambda: serial_device.read(8192)
+ write = serial_device.write
+ else:
+ try:
+ socket_device = SocketClientImpl(socket_addr)
+ read = socket_device.read
+ write = socket_device.write
+ except ValueError:
+ _LOG.exception('Failed to initialize socket at %s', socket_addr)
+ return 1
+
+ return HdlcRpcClient(read, protos, default_channels(write))
+
+
+def get_trace_data_from_device(client):
+ """ Get the trace data using RPC from a Client"""
+ data = b''
+ result = \
+ client.client.channel(1).rpcs.pw.trace.TraceService.GetTraceData().get()
+ for streamed_data in result:
+ data = data + bytes([len(streamed_data.data)])
+ data = data + streamed_data.data
+ _LOG.debug(''.join(format(x, '02x') for x in streamed_data.data))
+ return data
+
+
+def _parse_args():
+ """Parse and return command line arguments."""
+
+ parser = argparse.ArgumentParser(
+ description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+ group = parser.add_mutually_exclusive_group(required=True)
+ group.add_argument('-d', '--device', help='the serial port to use')
+ parser.add_argument('-b',
+ '--baudrate',
+ type=int,
+ default=115200,
+ help='the baud rate to use')
+ group.add_argument('-s',
+ '--socket-addr',
+ type=str,
+ help='use socket to connect to server, type default for\
+ localhost:33000, or manually input the server address:port')
+ parser.add_argument('-o',
+ '--trace_output',
+ dest='trace_output_file',
+ help=('The json file to which to write the output.'))
+ parser.add_argument(
+ '-t',
+ '--trace_token_database',
+ help='Databases (ELF, binary, or CSV) to use to lookup trace tokens.')
+ parser.add_argument('proto_globs',
+ nargs='+',
+ help='glob pattern for .proto files')
+
+ return parser.parse_args()
+
+
+def _main(args):
+ token_database = \
+ database.load_token_database(args.trace_token_database, domain="trace")
+ _LOG.info(database.database_summary(token_database))
+ client = get_hdlc_rpc_client(**vars(args))
+ data = get_trace_data_from_device(client)
+ events = trace_tokenized.get_trace_events([token_database], data)
+ json_lines = trace.generate_trace_json(events)
+ trace_tokenized.save_trace_file(json_lines, args.trace_output_file)
+
+
+if __name__ == '__main__':
+ if sys.version_info[0] < 3:
+ sys.exit('ERROR: The detokenizer command line tools require Python 3.')
+ _main(_parse_args())
diff --git a/pw_trace_tokenized/py/pw_trace_tokenized/trace_tokenized.py b/pw_trace_tokenized/py/pw_trace_tokenized/trace_tokenized.py
index 3e88a8a..fd16b3c 100755
--- a/pw_trace_tokenized/py/pw_trace_tokenized/trace_tokenized.py
+++ b/pw_trace_tokenized/py/pw_trace_tokenized/trace_tokenized.py
@@ -107,6 +107,7 @@
def parse_trace_event(buffer, db, last_time, ticks_per_second=1000):
+ """Parse a single trace event from bytes"""
us_per_tick = 1000000 / ticks_per_second
idx = 0
# Read token
@@ -116,6 +117,8 @@
# Decode token
if len(db.token_to_entries[token]) == 0:
_LOG.error("token not found: %08x", token)
+ return None
+
token_string = str(db.token_to_entries[token][0])
# Read time
@@ -138,31 +141,52 @@
return create_trace_event(token_string, timestamp_us, trace_id, data)
-def get_trace_events_from_file(databases, input_file_name):
+def get_trace_events(databases, raw_trace_data):
"""Handles the decoding traces."""
db = tokens.Database.merged(*databases)
last_timestamp = 0
events = []
- with open(input_file_name, "rb") as input_file:
- bytes_read = input_file.read()
- idx = 0
+ idx = 0
- while idx + 1 < len(bytes_read):
- # Read size
- size = int(bytes_read[idx])
- if idx + size > len(bytes_read):
- _LOG.error("incomplete file")
- break
+ while idx + 1 < len(raw_trace_data):
+ # Read size
+ size = int(raw_trace_data[idx])
+ if idx + size > len(raw_trace_data):
+ _LOG.error("incomplete file")
+ break
- event = parse_trace_event(bytes_read[idx + 1:idx + 1 + size], db,
- last_timestamp)
+ event = parse_trace_event(raw_trace_data[idx + 1:idx + 1 + size], db,
+ last_timestamp)
+ if event:
last_timestamp = event.timestamp_us
events.append(event)
- idx = idx + size + 1
+ idx = idx + size + 1
return events
+def get_trace_data_from_file(input_file_name):
+ """Handles the decoding traces."""
+ with open(input_file_name, "rb") as input_file:
+ return input_file.read()
+ return None
+
+
+def save_trace_file(trace_lines, file_name):
+ """Handles generating the trace file."""
+ with open(file_name, 'w') as output_file:
+ output_file.write("[")
+ for line in trace_lines:
+ output_file.write("%s,\n" % line)
+ output_file.write("{}]")
+
+
+def get_trace_events_from_file(databases, input_file_name):
+ """Get trace events from a file."""
+ raw_trace_data = get_trace_data_from_file(input_file_name)
+ return get_trace_events(databases, raw_trace_data)
+
+
def _parse_args():
"""Parse and return command line arguments."""
@@ -190,10 +214,7 @@
def _main(args):
events = get_trace_events_from_file(args.databases, args.input_file)
json_lines = trace.generate_trace_json(events)
-
- with open(args.output_file, 'w') as output_file:
- for line in json_lines:
- output_file.write("%s,\n" % line)
+ save_trace_file(json_lines, args.output_file)
if __name__ == '__main__':
diff --git a/pw_trace_tokenized/trace_rpc_service_nanopb.cc b/pw_trace_tokenized/trace_rpc_service_nanopb.cc
new file mode 100644
index 0000000..a6893da
--- /dev/null
+++ b/pw_trace_tokenized/trace_rpc_service_nanopb.cc
@@ -0,0 +1,63 @@
+// Copyright 2021 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+//==============================================================================
+
+#include "pw_trace_tokenized/trace_rpc_service_nanopb.h"
+
+#include "pw_log/log.h"
+#include "pw_preprocessor/util.h"
+#include "pw_trace_tokenized/trace_buffer.h"
+#include "pw_trace_tokenized/trace_tokenized.h"
+
+namespace pw::trace {
+
+pw::Status TraceService::Enable(ServerContext&,
+ const pw_trace_TraceEnableMessage& request,
+ pw_trace_TraceEnableMessage& response) {
+ TokenizedTrace::Instance().Enable(request.enable);
+ response.enable = TokenizedTrace::Instance().IsEnabled();
+ return PW_STATUS_OK;
+}
+
+pw::Status TraceService::IsEnabled(ServerContext&,
+ const pw_trace_Empty&,
+ pw_trace_TraceEnableMessage& response) {
+ response.enable = TokenizedTrace::Instance().IsEnabled();
+ return PW_STATUS_OK;
+}
+
+void TraceService::GetTraceData(
+ ServerContext&,
+ const pw_trace_Empty&,
+ ServerWriter<pw_trace_TraceDataMessage>& writer) {
+ pw_trace_TraceDataMessage buffer = pw_trace_TraceDataMessage_init_default;
+ size_t size = 0;
+ pw::ring_buffer::PrefixedEntryRingBuffer* trace_buffer =
+ pw::trace::GetBuffer();
+
+ while (trace_buffer->PeekFront(
+ std::as_writable_bytes(std::span(buffer.data.bytes)), &size) !=
+ pw::Status::OutOfRange()) {
+ trace_buffer->PopFront();
+ buffer.data.size = size;
+ pw::Status status = writer.Write(buffer);
+ if (!status.ok()) {
+ PW_LOG_ERROR("Error sending trace; abandoning trace dump. Error: %s",
+ status.str());
+ break;
+ }
+ }
+ writer.Finish();
+}
+} // namespace pw::trace
\ No newline at end of file
