pw_assert_tokenized: Highly compressed asserts
This change introduces a highly compressed tokenized assert backend that
drops some of the rich PW_CHECK() support in favor of maximum
compactness.
Change-Id: Ifc34a352d071223f712c34d450e15c7c6dc818a3
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/67261
Pigweed-Auto-Submit: Armando Montanez <amontanez@google.com>
Reviewed-by: Keir Mierle <keir@google.com>
Commit-Queue: Auto-Submit <auto-submit@pigweed.google.com.iam.gserviceaccount.com>
diff --git a/docs/BUILD.gn b/docs/BUILD.gn
index a1391b1..8c71a71 100644
--- a/docs/BUILD.gn
+++ b/docs/BUILD.gn
@@ -69,6 +69,7 @@
"$dir_pw_assert:docs",
"$dir_pw_assert_basic:docs",
"$dir_pw_assert_log:docs",
+ "$dir_pw_assert_tokenized:docs",
"$dir_pw_assert_zephyr:docs",
"$dir_pw_base64:docs",
"$dir_pw_bloat:docs",
diff --git a/modules.gni b/modules.gni
index b78953e..8b9dc58 100644
--- a/modules.gni
+++ b/modules.gni
@@ -24,6 +24,7 @@
dir_pw_assert = get_path_info("pw_assert", "abspath")
dir_pw_assert_basic = get_path_info("pw_assert_basic", "abspath")
dir_pw_assert_log = get_path_info("pw_assert_log", "abspath")
+ dir_pw_assert_tokenized = get_path_info("pw_assert_tokenized", "abspath")
dir_pw_assert_zephyr = get_path_info("pw_assert_zephyr", "abspath")
dir_pw_base64 = get_path_info("pw_base64", "abspath")
dir_pw_bloat = get_path_info("pw_bloat", "abspath")
diff --git a/pw_assert_tokenized/BUILD.bazel b/pw_assert_tokenized/BUILD.bazel
new file mode 100644
index 0000000..bd2fe86
--- /dev/null
+++ b/pw_assert_tokenized/BUILD.bazel
@@ -0,0 +1,47 @@
+# Copyright 2022 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+load(
+ "//pw_build:pigweed.bzl",
+ "pw_cc_library",
+)
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+pw_cc_library(
+ name = "pw_assert_tokenized",
+ srcs = [
+ "log_handler.cc",
+ ],
+ hdrs = [
+ "assert_public_overrides/pw_assert_backend/assert_lite_backend.h",
+ "check_public_overrides/pw_assert_backend/assert_backend.h",
+ "public/pw_assert_tokenized/assert_tokenized.h",
+ "public/pw_assert_tokenized/check_tokenized.h",
+ "public/pw_assert_tokenized/handler.h",
+ ],
+ includes = [
+ "assert_public_overrides",
+ "check_public_overrides",
+ "public",
+ ],
+ deps = [
+ "//pw_assert",
+ "//pw_log_tokenized,",
+ "//pw_preprocessor",
+ "//pw_tokenizer",
+ ],
+)
diff --git a/pw_assert_tokenized/BUILD.gn b/pw_assert_tokenized/BUILD.gn
new file mode 100644
index 0000000..42561a4
--- /dev/null
+++ b/pw_assert_tokenized/BUILD.gn
@@ -0,0 +1,110 @@
+# Copyright 2022 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import("//build_overrides/pigweed.gni")
+
+import("$dir_pw_build/target_types.gni")
+import("$dir_pw_docgen/docs.gni")
+
+declare_args() {
+ pw_assert_tokenized_HANDLER_BACKEND = "$dir_pw_assert_tokenized:log_handler"
+}
+
+config("public_include_path") {
+ include_dirs = [ "public" ]
+ visibility = [ ":*" ]
+}
+
+config("assert_backend_config") {
+ include_dirs = [ "assert_public_overrides" ]
+ visibility = [ ":*" ]
+}
+
+config("check_backend_config") {
+ include_dirs = [ "check_public_overrides" ]
+ visibility = [ ":*" ]
+}
+
+pw_source_set("handler") {
+ public_configs = [ ":public_include_path" ]
+ public_deps = [ "$dir_pw_preprocessor" ]
+ public = [ "public/pw_assert_tokenized/handler.h" ]
+}
+
+pw_source_set("assert_backend") {
+ public_configs = [
+ ":public_include_path",
+ ":assert_backend_config",
+ ]
+ public_deps = [
+ ":handler",
+ "$dir_pw_tokenizer",
+ ]
+ public = [
+ "assert_public_overrides/pw_assert_backend/assert_lite_backend.h",
+ "public/pw_assert_tokenized/assert_tokenized.h",
+ ]
+}
+
+pw_source_set("assert_backend.impl") {
+ public_deps = [ pw_assert_tokenized_HANDLER_BACKEND ]
+}
+
+pw_source_set("check_backend") {
+ public_configs = [
+ ":public_include_path",
+ ":check_backend_config",
+ ]
+ public_deps = [
+ ":handler",
+ "$dir_pw_tokenizer",
+ ]
+ public = [
+ "check_public_overrides/pw_assert_backend/assert_backend.h",
+ "public/pw_assert_tokenized/check_tokenized.h",
+ ]
+}
+
+pw_source_set("check_backend.impl") {
+ public_deps = [ pw_assert_tokenized_HANDLER_BACKEND ]
+}
+
+pw_source_set("pw_assert_tokenized") {
+ public_deps = [
+ ":assert_backend",
+ ":check_backend",
+ ]
+}
+
+pw_source_set("pw_assert_tokenized.impl") {
+ deps = [
+ ":assert_backend.impl",
+ ":check_backend.impl",
+ ]
+}
+
+pw_source_set("log_handler") {
+ deps = [
+ ":handler",
+ "$dir_pw_assert:config",
+ "$dir_pw_base64",
+ "$dir_pw_log",
+ "$dir_pw_log_tokenized",
+ ]
+ sources = [ "log_handler.cc" ]
+}
+
+pw_doc_group("docs") {
+ sources = [ "docs.rst" ]
+}
diff --git a/pw_assert_tokenized/assert_public_overrides/pw_assert_backend/assert_lite_backend.h b/pw_assert_tokenized/assert_public_overrides/pw_assert_backend/assert_lite_backend.h
new file mode 100644
index 0000000..561f521
--- /dev/null
+++ b/pw_assert_tokenized/assert_public_overrides/pw_assert_backend/assert_lite_backend.h
@@ -0,0 +1,16 @@
+// Copyright 2022 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+#pragma once
+
+#include "pw_assert_tokenized/assert_tokenized.h"
diff --git a/pw_assert_tokenized/check_public_overrides/pw_assert_backend/assert_backend.h b/pw_assert_tokenized/check_public_overrides/pw_assert_backend/assert_backend.h
new file mode 100644
index 0000000..4a1e3ae
--- /dev/null
+++ b/pw_assert_tokenized/check_public_overrides/pw_assert_backend/assert_backend.h
@@ -0,0 +1,16 @@
+// Copyright 2022 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+#pragma once
+
+#include "pw_assert_tokenized/check_tokenized.h"
diff --git a/pw_assert_tokenized/docs.rst b/pw_assert_tokenized/docs.rst
new file mode 100644
index 0000000..bc645a7
--- /dev/null
+++ b/pw_assert_tokenized/docs.rst
@@ -0,0 +1,95 @@
+.. _module-pw_assert_tokenized:
+
+===================
+pw_assert_tokenized
+===================
+
+--------
+Overview
+--------
+The ``pw_assert_tokenized`` module provides ``PW_ASSERT()`` and ``PW_CHECK_*()``
+backends for the ``pw_assert`` module. These backends are much more space
+efficient than using ``pw_assert_log`` with ``pw_log_tokenized`` The tradeoff,
+however, is that ``PW_CHECK_*()`` macros are much more limited as all argument
+values are discarded. This means only constant string information is captured in
+the reported tokens.
+
+* **PW_ASSERT()**: The ``PW_ASSERT()`` macro will capture the file name and line
+ number of the assert statement. By default, it is passed to the logging system
+ to produce a string like this:
+
+ PW_ASSERT() or PW_DASSERT() failure at
+ pw_result/public/pw_result/result.h:63
+
+* **PW_CHECK_\*()**: The ``PW_CHECK_*()`` macros work in contexts where
+ tokenization is fully supported, so they are able to capture the CHECK
+ statement expression and any provided string literal in addition to the file
+ name:
+
+ Check failure in pw_metric/size_report/base.cc: \*unoptimizable >= 0,
+ Ensure this CHECK logic stays.
+
+ Evaluated values of ``PW_CHECK_*()`` statements are not captured, and any
+ string formatting arguments are also not captured. This minimizes call-site
+ cost as only two arguments are ever passed to the handler (the calculated
+ token, and the line number of the statement).
+
+ Note that the line number is passed to the tokenized logging system as
+ metadata, but is not part of the tokenized string. This is to ensure the
+ CHECK callsite maximizes efficiency by only passing two arguments to the
+ handler.
+
+In both cases, the assert handler is only called with two arguments: a 32-bit
+token to represent a string, and the integer line number of the callsite.
+
+-----
+Setup
+-----
+
+#. Set ``pw_assert_BACKEND = "$dir_pw_assert_tokenized:check_backend"`` and
+ ``pw_assert_LITE_BACKEND = "$dir_pw_assert_tokenized:assert_backend"`` in
+ your target configuration.
+#. Ensure your target provides ``pw_tokenizer_GLOBAL_HANDLER_BACKEND``. By
+ default, pw_assert_tokenized will forward assert failures to the tokenizer
+ handler as logs. The tokenizer handler should check for ``LOG_LEVEL_FATAL``
+ and properly divert to a crash handler.
+#. Add file name tokens to your token database. pw_assert_tokenized can't create
+ file name tokens that can be parsed out of the final compiled binary. The
+ ``pw_relative_source_file_names``
+ `GN template <module-pw_build-relative-source-file-names>`_ can be used to
+ collect the names of all source files used in your final executable into a
+ JSON file, which can then be included in the creation of a tokenizer
+ database.
+
+Example file name token database setup
+--------------------------------------
+
+.. code-block::
+
+ pw_executable("main") {
+ deps = [
+ # ...
+ ]
+ sources = [ "main.cc" ]
+ }
+
+ pw_tokenizer_database("log_tokens") {
+ database = "tools/tokenized_logs.csv"
+ deps = [
+ ":source_file_names",
+ ":main",
+ ]
+ optional_paths = [ "$root_build_dir/**/*.elf" ]
+ input_databases = [ "$target_gen_dir/source_file_names.json" ]
+ }
+
+ # Extracts all source/header file names from "main" and its transitive
+ # dependencies for tokenization.
+ pw_relative_source_file_names("source_file_names") {
+ deps = [ ":main" ]
+ outputs = [ "$target_gen_dir/source_file_names.json" ]
+ }
+
+
+.. warning::
+ This module is experimental and does not provide a stable API.
diff --git a/pw_assert_tokenized/log_handler.cc b/pw_assert_tokenized/log_handler.cc
new file mode 100644
index 0000000..ea559ea
--- /dev/null
+++ b/pw_assert_tokenized/log_handler.cc
@@ -0,0 +1,66 @@
+// Copyright 2022 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+#include <cstdint>
+#include <cstring>
+#include <memory>
+#include <span>
+
+#include "pw_assert/config.h"
+#include "pw_assert_tokenized/handler.h"
+#include "pw_base64/base64.h"
+#include "pw_log/log.h"
+#include "pw_log_tokenized/log_tokenized.h"
+
+extern "C" void pw_assert_tokenized_HandleAssertFailure(
+ uint32_t tokenized_file_name, int line_number) {
+ // Buffer size for binary->base64 conversion with a null terminator.
+ constexpr size_t kBufferSize =
+ pw::base64::EncodedSize(sizeof(tokenized_file_name)) + 1;
+ std::byte* hash_buffer = reinterpret_cast<std::byte*>(&tokenized_file_name);
+ char base64_buffer[kBufferSize];
+
+ size_t len =
+ pw::base64::Encode(std::span(hash_buffer, sizeof(tokenized_file_name)),
+ std::span(base64_buffer));
+ base64_buffer[len] = '\0';
+#if PW_ASSERT_ENABLE_DEBUG
+ PW_LOG(PW_LOG_LEVEL_FATAL,
+ PW_LOG_FLAGS,
+ "PW_ASSERT() or PW_DASSERT() failure at $%s:%d",
+ base64_buffer,
+ line_number);
+#else
+ PW_LOG(PW_LOG_LEVEL_FATAL,
+ PW_LOG_FLAGS,
+ "PW_ASSERT() failure. Note: PW_DASSERT disabled $%s:%d",
+ base64_buffer,
+ line_number);
+#endif // PW_ASSERT_ENABLE_DEBUG
+ PW_UNREACHABLE;
+}
+
+extern "C" void pw_assert_tokenized_HandleCheckFailure(
+ uint32_t tokenized_message, int line_number) {
+ // TODO(amontanez): There should be a less-hacky way to assemble this.
+ const uint32_t payload = _PW_LOG_TOKENIZED_LEVEL(PW_LOG_LEVEL_FATAL) |
+ _PW_LOG_TOKENIZED_FLAGS(PW_LOG_FLAGS) |
+ _PW_LOG_TOKENIZED_LINE(line_number);
+ uint8_t token_buffer[sizeof(tokenized_message)];
+ memcpy(token_buffer, &tokenized_message, sizeof(tokenized_message));
+
+ pw_tokenizer_HandleEncodedMessageWithPayload(
+ payload, token_buffer, sizeof(token_buffer));
+ PW_UNREACHABLE;
+}
diff --git a/pw_assert_tokenized/public/pw_assert_tokenized/assert_tokenized.h b/pw_assert_tokenized/public/pw_assert_tokenized/assert_tokenized.h
new file mode 100644
index 0000000..7d3811d
--- /dev/null
+++ b/pw_assert_tokenized/public/pw_assert_tokenized/assert_tokenized.h
@@ -0,0 +1,24 @@
+// Copyright 2022 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+#pragma once
+
+#include "pw_assert_tokenized/handler.h"
+#include "pw_tokenizer/tokenize.h"
+
+// The tokens generated by this expression are not stored in the ELF in a way
+// that the pw_tokenizer tooling can extract. Tokenized file names must be
+// generated offline separately.
+#define PW_ASSERT_HANDLE_FAILURE(expression) \
+ pw_assert_tokenized_HandleAssertFailure(PW_TOKENIZER_STRING_TOKEN(__FILE__), \
+ __LINE__);
diff --git a/pw_assert_tokenized/public/pw_assert_tokenized/check_tokenized.h b/pw_assert_tokenized/public/pw_assert_tokenized/check_tokenized.h
new file mode 100644
index 0000000..591a194
--- /dev/null
+++ b/pw_assert_tokenized/public/pw_assert_tokenized/check_tokenized.h
@@ -0,0 +1,40 @@
+// Copyright 2022 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+#pragma once
+
+#include "pw_assert_tokenized/handler.h"
+#include "pw_tokenizer/tokenize.h"
+
+#define _PW_ASSERT_TOKENIZED_TO_HANDLER(str) \
+ do { \
+ constexpr uint32_t token = \
+ PW_TOKENIZE_STRING("Check failure in " __FILE__ ": " str); \
+ pw_assert_tokenized_HandleCheckFailure(token, __LINE__); \
+ } while (0)
+
+#define PW_HANDLE_CRASH(...) _PW_ASSERT_TOKENIZED_TO_HANDLER(#__VA_ARGS__)
+
+#define PW_HANDLE_ASSERT_FAILURE(condition_string, message, ...) \
+ _PW_ASSERT_TOKENIZED_TO_HANDLER(condition_string ", " message)
+
+#define PW_HANDLE_ASSERT_BINARY_COMPARE_FAILURE(arg_a_str, \
+ arg_a_val, \
+ comparison_op_str, \
+ arg_b_str, \
+ arg_b_val, \
+ type_fmt, \
+ message, \
+ ...) \
+ _PW_ASSERT_TOKENIZED_TO_HANDLER( \
+ arg_a_str " " comparison_op_str " " arg_b_str ", " message #__VA_ARGS__)
diff --git a/pw_assert_tokenized/public/pw_assert_tokenized/handler.h b/pw_assert_tokenized/public/pw_assert_tokenized/handler.h
new file mode 100644
index 0000000..dcdba50
--- /dev/null
+++ b/pw_assert_tokenized/public/pw_assert_tokenized/handler.h
@@ -0,0 +1,29 @@
+// Copyright 2022 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+#pragma once
+
+#include <stdint.h>
+
+#include "pw_preprocessor/compiler.h"
+#include "pw_preprocessor/util.h"
+
+PW_EXTERN_C_START
+
+PW_NO_RETURN void pw_assert_tokenized_HandleAssertFailure(
+ uint32_t tokenized_file_name, int line_number);
+
+PW_NO_RETURN void pw_assert_tokenized_HandleCheckFailure(
+ uint32_t tokenized_message, int line_number);
+
+PW_EXTERN_C_END
diff --git a/pw_build/docs.rst b/pw_build/docs.rst
index d0c6628..322f796 100644
--- a/pw_build/docs.rst
+++ b/pw_build/docs.rst
@@ -470,6 +470,8 @@
├── file1.txt
└── renamed.txt
+.. _module-pw_build-relative-source-file-names:
+
pw_relative_source_file_names
-----------------------------
This template recursively walks the listed dependencies and collects the names
diff --git a/pw_log_tokenized/public/pw_log_tokenized/log_tokenized.h b/pw_log_tokenized/public/pw_log_tokenized/log_tokenized.h
index 42cc614..110d542 100644
--- a/pw_log_tokenized/public/pw_log_tokenized/log_tokenized.h
+++ b/pw_log_tokenized/public/pw_log_tokenized/log_tokenized.h
@@ -58,7 +58,7 @@
PW_LOG_TOKENIZED_ENCODE_MESSAGE( \
(_PW_LOG_TOKENIZED_LEVEL(_pw_log_tokenized_level) | \
_PW_LOG_TOKENIZED_MODULE(_pw_log_tokenized_module_token) | \
- _PW_LOG_TOKENIZED_FLAGS(flags) | _PW_LOG_TOKENIZED_LINE()), \
+ _PW_LOG_TOKENIZED_FLAGS(flags) | _PW_LOG_TOKENIZED_LINE(__LINE__)), \
PW_LOG_TOKENIZED_FORMAT_STRING(message), \
__VA_ARGS__); \
} while (0)
@@ -76,10 +76,10 @@
// If the line number field is present, shift it to its position. Set it to zero
// if the line number is too large for PW_LOG_TOKENIZED_LINE_BITS.
#if PW_LOG_TOKENIZED_LINE_BITS == 0
-#define _PW_LOG_TOKENIZED_LINE() ((uintptr_t)0)
+#define _PW_LOG_TOKENIZED_LINE(line) ((uintptr_t)0)
#else
-#define _PW_LOG_TOKENIZED_LINE() \
- ((uintptr_t)(__LINE__ < (1 << PW_LOG_TOKENIZED_LINE_BITS) ? __LINE__ : 0) \
+#define _PW_LOG_TOKENIZED_LINE(line) \
+ ((uintptr_t)(line < (1 << PW_LOG_TOKENIZED_LINE_BITS) ? line : 0) \
<< PW_LOG_TOKENIZED_LEVEL_BITS)
#endif // PW_LOG_TOKENIZED_LINE_BITS
