pw_log_tokenized: Log metadata updates
- Add an optional line number to the 32-bit log tokenized payload.
- Update default payload field widths.
Log level: 3 bits
Line number: 11 bits (up to 2047, 0 if larger)
Flags: 2 bits (implementation defined)
Module token: 16 bits
- Use the maximum log level to indicate that a log is an assert. This
preserves an additional bit which can be used for the flags or line
number.
- Store data as key-value pairs in the format string.
- Reorganize pw_log_tokenized tests and add C tests.
Change-Id: I1daa2e6ce40038f96857caeb38976cf48f620dc3
Requires: pigweed-internal:12920
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/47861
Reviewed-by: Keir Mierle <keir@google.com>
Commit-Queue: Wyatt Hepler <hepler@google.com>
diff --git a/pw_assert_log/assert_log.cc b/pw_assert_log/assert_log.cc
index 9d8b9ba..278fa49 100644
--- a/pw_assert_log/assert_log.cc
+++ b/pw_assert_log/assert_log.cc
@@ -19,11 +19,11 @@
extern "C" void pw_assert_HandleFailure(void) {
#if PW_ASSERT_ENABLE_DEBUG
PW_LOG(PW_LOG_LEVEL_CRITICAL,
- PW_LOG_ASSERT_FAILED_FLAG,
+ PW_LOG_DEFAULT_FLAGS,
"Crash: PW_ASSERT() or PW_DASSERT() failure");
#else
PW_LOG(PW_LOG_LEVEL_CRITICAL,
- PW_LOG_ASSERT_FAILED_FLAG,
+ PW_LOG_DEFAULT_FLAGS,
"Crash: PW_ASSERT() failure. Note: PW_DASSERT disabled");
#endif // PW_ASSERT_ENABLE_DEBUG
PW_UNREACHABLE;
diff --git a/pw_assert_log/public/pw_assert_log/assert_log.h b/pw_assert_log/public/pw_assert_log/assert_log.h
index d26d339..9745827 100644
--- a/pw_assert_log/public/pw_assert_log/assert_log.h
+++ b/pw_assert_log/public/pw_assert_log/assert_log.h
@@ -19,16 +19,13 @@
#include "pw_preprocessor/compiler.h"
#include "pw_preprocessor/util.h"
-// Use the highest available log flag to indicate an assert failure.
-#define PW_LOG_ASSERT_FAILED_FLAG (1u << (PW_LOG_FLAG_BITS - 1u))
-
// Die with a message with several attributes included. This crash frontend
// funnels everything into the logger, which must then handle the true crash
// behaviour.
#define PW_HANDLE_CRASH(message, ...) \
do { \
- PW_LOG(PW_LOG_LEVEL_CRITICAL, \
- PW_LOG_ASSERT_FAILED_FLAG, \
+ PW_LOG(PW_LOG_LEVEL_FATAL, \
+ PW_LOG_DEFAULT_FLAGS, \
__FILE__ ":%d: Crash: " message, \
__LINE__, \
__VA_ARGS__); \
@@ -40,8 +37,8 @@
// log, then crashing/rebooting the device.
#define PW_HANDLE_ASSERT_FAILURE(condition_string, message, ...) \
do { \
- PW_LOG(PW_LOG_LEVEL_CRITICAL, \
- PW_LOG_ASSERT_FAILED_FLAG, \
+ PW_LOG(PW_LOG_LEVEL_FATAL, \
+ PW_LOG_DEFAULT_FLAGS, \
__FILE__ ":%d: Check failed: " condition_string ". " message, \
__LINE__, \
__VA_ARGS__); \
@@ -64,8 +61,8 @@
type_fmt, \
message, ...) \
do { \
- PW_LOG(PW_LOG_LEVEL_CRITICAL, \
- PW_LOG_ASSERT_FAILED_FLAG, \
+ PW_LOG(PW_LOG_LEVEL_FATAL, \
+ PW_LOG_DEFAULT_FLAGS, \
__FILE__ ":%d: Check failed: " \
arg_a_str " (=" type_fmt ") " \
comparison_op_str " " \
diff --git a/pw_log/public/pw_log/levels.h b/pw_log/public/pw_log/levels.h
index a70abc3..8702837 100644
--- a/pw_log/public/pw_log/levels.h
+++ b/pw_log/public/pw_log/levels.h
@@ -24,6 +24,8 @@
#define PW_LOG_LEVEL_ERROR 4
#define PW_LOG_LEVEL_CRITICAL 5
+#define PW_LOG_LEVEL_FATAL 7
+
// Number of bits required to represent the log level
#define PW_LOG_LEVEL_BITS 3
#define PW_LOG_LEVEL_BITMASK 7 // 0b111
diff --git a/pw_log_tokenized/BUILD b/pw_log_tokenized/BUILD
index 41b0747..80b841d 100644
--- a/pw_log_tokenized/BUILD
+++ b/pw_log_tokenized/BUILD
@@ -27,6 +27,7 @@
hdrs = [
"public/pw_log_tokenized/config.h",
"public/pw_log_tokenized/log_tokenized.h",
+ "public/pw_log_tokenized/metadata.h",
"public_overrides/pw_log_backend/log_backend.h",
],
includes = [
@@ -59,9 +60,22 @@
)
pw_cc_test(
- name = "test",
+ name = "log_tokenized_test",
srcs = [
- "test.cc",
+ "log_tokenized_test.cc",
+ "log_tokenized_test_c.c",
+ "pw_log_tokenized_private/test_utils.h",
+ ],
+ deps = [
+ ":headers",
+ "//pw_unit_test",
+ ],
+)
+
+pw_cc_test(
+ name = "metadata_test",
+ srcs = [
+ "metadata_test.cc",
],
deps = [
":headers",
diff --git a/pw_log_tokenized/BUILD.gn b/pw_log_tokenized/BUILD.gn
index a89054d..2109673 100644
--- a/pw_log_tokenized/BUILD.gn
+++ b/pw_log_tokenized/BUILD.gn
@@ -45,18 +45,31 @@
":public_include_path",
]
public_deps = [
- "$dir_pw_log:facade",
+ ":config",
+ ":metadata",
"$dir_pw_tokenizer:global_handler_with_payload.facade",
- dir_pw_preprocessor,
- pw_log_tokenized_CONFIG,
]
public = [
- "public/pw_log_tokenized/config.h",
"public/pw_log_tokenized/log_tokenized.h",
"public_overrides/pw_log_backend/log_backend.h",
]
}
+pw_source_set("metadata") {
+ public_configs = [ ":public_include_path" ]
+ public_deps = [ ":config" ]
+ public = [ "public/pw_log_tokenized/metadata.h" ]
+}
+
+pw_source_set("config") {
+ public_configs = [ ":public_include_path" ]
+ public_deps = [
+ "$dir_pw_log:facade",
+ pw_log_tokenized_CONFIG,
+ ]
+ public = [ "public/pw_log_tokenized/config.h" ]
+}
+
# The log backend deps that might cause circular dependencies, since
# pw_log is so ubiquitous. These deps are kept separate so they can be
# depended on from elsewhere.
@@ -82,12 +95,27 @@
}
pw_test_group("tests") {
- tests = [ ":log_tokenized_test" ]
+ tests = [
+ ":log_tokenized_test",
+ ":metadata_test",
+ ]
}
pw_test("log_tokenized_test") {
- sources = [ "test.cc" ]
- deps = [ ":pw_log_tokenized" ]
+ sources = [
+ "log_tokenized_test.cc",
+ "log_tokenized_test_c.c",
+ "pw_log_tokenized_private/test_utils.h",
+ ]
+ deps = [
+ ":pw_log_tokenized",
+ dir_pw_preprocessor,
+ ]
+}
+
+pw_test("metadata_test") {
+ sources = [ "metadata_test.cc" ]
+ deps = [ ":metadata" ]
}
pw_doc_group("docs") {
diff --git a/pw_log_tokenized/docs.rst b/pw_log_tokenized/docs.rst
index 0871c6b..1aa6267 100644
--- a/pw_log_tokenized/docs.rst
+++ b/pw_log_tokenized/docs.rst
@@ -39,6 +39,95 @@
See the documentation for :ref:`module-pw_tokenizer` for further details.
+Metadata in the format string
+-----------------------------
+With tokenized logging, the log format string is converted to a 32-bit token.
+Regardless of how long the format string is, it's always represented by a 32-bit
+token. Because of this, metadata can be packed into the tokenized string with
+no cost.
+
+``pw_log_tokenized`` uses a simple key-value format to encode metadata in a
+format string. Each field starts with the ``■`` (U+25A0 "Black Square")
+character, followed by the key name, the ``♦`` (U+2666 "Black Diamond Suit")
+character, and then the value. The string is encoded as UTF-8. Key names are
+comprised of alphanumeric ASCII characters and underscore and start with a
+letter.
+
+.. code-block::
+
+ "■key1♦contents1■key2♦contents2■key3♦contents3"
+
+This format makes the message easily machine parseable and human readable. It is
+extremely unlikely to conflict with log message contents due to the characters
+used.
+
+``pw_log_tokenized`` uses three fields: ``msg``, ``module``, and ``file``.
+Implementations may add other fields, but they will be ignored by the
+``pw_log_tokenized`` tooling.
+
+.. code-block::
+
+ "■msg♦Hyperdrive %d set to %f■module♦engine■file♦propulsion/hyper.cc"
+
+Using key-value pairs allows placing the fields in any order.
+``pw_log_tokenized`` places the message first. This is prefered when tokenizing
+C code because the tokenizer only hashes a fixed number of characters. If the
+file were first, the long path might take most of the hashed characters,
+increasing the odds of a collision with other strings in that file. In C++, all
+characters in the string are hashed, so the order is not important.
+
+Metadata in the tokenizer payload argument
+-------------------------------------------
+``pw_log_tokenized`` packs runtime-accessible metadata into a 32-bit integer
+which is passed as the "payload" argument for ``pw_log_tokenizer``'s global
+handler with payload facade. Packing this metadata into a single word rather
+than separate arguments reduces the code size significantly.
+
+Four items are packed into the payload argument:
+
+- Log level -- Used for runtime log filtering by level.
+- Line number -- Used to track where a log message originated.
+- Log flags -- Implementation-defined log flags.
+- Tokenized :c:macro:`PW_LOG_MODULE_NAME` -- Used for runtime log filtering by
+ module.
+
+Configuring metadata bit fields
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The number of bits to use for each metadata field is configurable through macros
+in ``pw_log/config.h``. The field widths must sum to 32 bits. A field with zero
+bits allocated is excluded from the log metadata.
+
+.. c:macro:: PW_LOG_TOKENIZED_LEVEL_BITS
+
+ Bits to allocate for the log level. Defaults to :c:macro:`PW_LOG_LEVEL_BITS`
+ (3).
+
+.. c:macro:: PW_LOG_TOKENIZED_LINE_BITS
+
+ Bits to allocate for the line number. Defaults to 11 (up to line 2047). If the
+ line number is too large to be represented by this field, line is reported as
+ 0.
+
+ Including the line number can slightly increase code size. Without the line
+ number, the log metadata argument is the same for all logs with the same level
+ and flags. With the line number, each metadata value is unique and must be
+ encoded as a separate word in the binary. Systems with extreme space
+ constraints may exclude line numbers by setting this macro to 0.
+
+ It is possible to include line numbers in tokenized log format strings, but
+ that is discouraged because line numbers change whenever a file is edited.
+ Passing the line number with the metadata is a lightweight way to include it.
+
+.. c:macro:: PW_LOG_TOKENIZED_FLAG_BITS
+
+ Bits to use for implementation-defined flags. Defaults to 2.
+
+.. c:macro:: PW_LOG_TOKENIZED_MODULE_BITS
+
+ Bits to use for the tokenized version of :c:macro:`PW_LOG_MODULE_NAME`.
+ Defaults to 16, which gives a ~1% probability of a collision with 37 module
+ names.
+
Using a custom macro
--------------------
Applications may use their own macro instead of
diff --git a/pw_log_tokenized/log_tokenized_test.cc b/pw_log_tokenized/log_tokenized_test.cc
new file mode 100644
index 0000000..e926749
--- /dev/null
+++ b/pw_log_tokenized/log_tokenized_test.cc
@@ -0,0 +1,140 @@
+// Copyright 2021 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+#define PW_LOG_MODULE_NAME "log module name!"
+
+// Configure the module so that the test runs against known values.
+#undef PW_LOG_TOKENIZED_LEVEL_BITS
+#undef PW_LOG_TOKENIZED_MODULE_BITS
+#undef PW_LOG_TOKENIZED_FLAG_BITS
+#undef PW_LOG_TOKENIZED_LINE_BITS
+
+#define PW_LOG_TOKENIZED_LEVEL_BITS 3
+#define PW_LOG_TOKENIZED_MODULE_BITS 16
+#define PW_LOG_TOKENIZED_FLAG_BITS 2
+#define PW_LOG_TOKENIZED_LINE_BITS 11
+
+#include "pw_log_tokenized/log_tokenized.h"
+
+#include "gtest/gtest.h"
+#include "pw_log_tokenized_private/test_utils.h"
+
+namespace pw::log_tokenized {
+namespace {
+
+TEST(LogTokenized, FormatString) {
+ PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD(63, 1023, "hello %d", 1);
+ EXPECT_STREQ(last_log.format_string,
+ "■msg♦hello %d■module♦log module name!■file♦" __FILE__);
+}
+
+constexpr uintptr_t kModuleToken =
+ PW_TOKENIZER_STRING_TOKEN(PW_LOG_MODULE_NAME) &
+ ((1u << PW_LOG_TOKENIZED_MODULE_BITS) - 1);
+
+TEST(LogTokenized, LogMetadata_LevelTooLarge_Clamps) {
+ auto check_metadata = [] {
+ Metadata metadata = Metadata(last_log.metadata);
+ EXPECT_EQ(metadata.level(), 7u);
+ EXPECT_EQ(metadata.flags(), 0u);
+ EXPECT_EQ(metadata.module(), kModuleToken);
+ EXPECT_TRUE(metadata.line_number() == 55u || metadata.line_number() == 45u);
+ };
+
+ PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD(8, 0, "hello");
+ check_metadata();
+
+ pw_log_tokenized_Test_LogMetadata_LevelTooLarge_Clamps();
+ check_metadata();
+}
+
+TEST(LogTokenized, LogMetadata_TooManyFlags_Truncates) {
+ auto check_metadata = [] {
+ Metadata metadata = Metadata(last_log.metadata);
+ EXPECT_EQ(metadata.level(), 1u);
+ EXPECT_EQ(metadata.flags(), 0b11u);
+ EXPECT_EQ(metadata.module(), kModuleToken);
+ EXPECT_TRUE(metadata.line_number() == 71u || metadata.line_number() == 49u);
+ };
+
+ PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD(1, 0xFFFFFFFF, "hello");
+ check_metadata();
+
+ pw_log_tokenized_Test_LogMetadata_TooManyFlags_Truncates();
+ check_metadata();
+}
+
+TEST(LogTokenized, LogMetadata_VariousValues) {
+ auto check_metadata = [] {
+ Metadata metadata = Metadata(last_log.metadata);
+ EXPECT_EQ(metadata.level(), 6u);
+ EXPECT_EQ(metadata.flags(), 3u);
+ EXPECT_EQ(metadata.module(), kModuleToken);
+ EXPECT_EQ(last_log.arg_count, 1u);
+ EXPECT_TRUE(metadata.line_number() == 88u || metadata.line_number() == 53u);
+ };
+
+ PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD(6, 3, "hello%s", "?");
+ check_metadata();
+
+ pw_log_tokenized_Test_LogMetadata_LogMetadata_VariousValues();
+ check_metadata();
+}
+
+TEST(LogTokenized, LogMetadata_Zero) {
+ auto check_metadata = [] {
+ Metadata metadata = Metadata(last_log.metadata);
+ EXPECT_EQ(metadata.level(), 0u);
+ EXPECT_EQ(metadata.flags(), 0u);
+ EXPECT_EQ(metadata.module(), kModuleToken);
+ EXPECT_EQ(last_log.arg_count, 0u);
+ EXPECT_TRUE(metadata.line_number() == 106u ||
+ metadata.line_number() == 57u);
+ };
+
+ PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD(0, 0, "hello");
+ check_metadata();
+
+ pw_log_tokenized_Test_LogMetadata_LogMetadata_Zero();
+ check_metadata();
+}
+
+TEST(LogTokenized, LogMetadata_MaxValues) {
+#line 2047
+ PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD(7, 3, "hello %d", 1);
+
+ Metadata metadata = Metadata(last_log.metadata);
+ EXPECT_EQ(metadata.line_number(), 2047u);
+ EXPECT_EQ(metadata.level(), 7u);
+ EXPECT_EQ(metadata.flags(), 3u);
+ EXPECT_EQ(metadata.module(), kModuleToken);
+ EXPECT_EQ(last_log.arg_count, 1u);
+}
+
+TEST(LogTokenized, LogMetadata_LineNumberTooLarge_IsZero) {
+#line 2048 // At 11 bits, the largest representable line is 2047
+ PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD(7, 3, "hello %d", 1);
+ EXPECT_EQ(Metadata(last_log.metadata).line_number(), 0u);
+
+#line 2049
+ PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD(7, 3, "hello %d", 1);
+ EXPECT_EQ(Metadata(last_log.metadata).line_number(), 0u);
+
+#line 99999
+ PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD(7, 3, "hello %d", 1);
+ EXPECT_EQ(Metadata(last_log.metadata).line_number(), 0u);
+}
+
+} // namespace
+} // namespace pw::log_tokenized
diff --git a/pw_log_tokenized/log_tokenized_test_c.c b/pw_log_tokenized/log_tokenized_test_c.c
new file mode 100644
index 0000000..c0b28b7
--- /dev/null
+++ b/pw_log_tokenized/log_tokenized_test_c.c
@@ -0,0 +1,58 @@
+// Copyright 2021 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+#define PW_LOG_MODULE_NAME "log module name!"
+
+// Configure the module so that the test runs against known values.
+#undef PW_LOG_TOKENIZED_LEVEL_BITS
+#undef PW_LOG_TOKENIZED_MODULE_BITS
+#undef PW_LOG_TOKENIZED_FLAG_BITS
+#undef PW_LOG_TOKENIZED_LINE_BITS
+
+#define PW_LOG_TOKENIZED_LEVEL_BITS 3
+#define PW_LOG_TOKENIZED_MODULE_BITS 16
+#define PW_LOG_TOKENIZED_FLAG_BITS 2
+#define PW_LOG_TOKENIZED_LINE_BITS 11
+
+#include "pw_log_tokenized/log_tokenized.h"
+#include "pw_log_tokenized_private/test_utils.h"
+
+pw_log_tokenized_CapturedLog last_log;
+
+void pw_log_tokenized_CaptureArgs(uintptr_t payload,
+ size_t arg_count,
+ const char* message,
+ ...) {
+ last_log.metadata = payload;
+ last_log.format_string = message;
+ last_log.arg_count = arg_count;
+}
+
+// These functions correspond to tests in log_tokenized_test.cc. The tests call
+// these functions and check the results.
+void pw_log_tokenized_Test_LogMetadata_LevelTooLarge_Clamps(void) {
+ PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD(8, 0, "hello");
+}
+
+void pw_log_tokenized_Test_LogMetadata_TooManyFlags_Truncates(void) {
+ PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD(1, 0xFFFFFFFF, "hello");
+}
+
+void pw_log_tokenized_Test_LogMetadata_LogMetadata_VariousValues(void) {
+ PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD(6, 3, "hello%s", "?");
+}
+
+void pw_log_tokenized_Test_LogMetadata_LogMetadata_Zero(void) {
+ PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD(0, 0, "hello");
+}
diff --git a/pw_log_tokenized/metadata_test.cc b/pw_log_tokenized/metadata_test.cc
new file mode 100644
index 0000000..3a45cee
--- /dev/null
+++ b/pw_log_tokenized/metadata_test.cc
@@ -0,0 +1,67 @@
+// Copyright 2021 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+#include "pw_log_tokenized/metadata.h"
+
+#include "gtest/gtest.h"
+
+namespace pw::log_tokenized {
+namespace {
+
+TEST(Metadata, NoLineBits) {
+ using NoLineBits = internal::GenericMetadata<6, 0, 10, 16>;
+
+ constexpr NoLineBits test1 = NoLineBits::Set<0, 0, 0>();
+ static_assert(test1.level() == 0);
+ static_assert(test1.module() == 0);
+ static_assert(test1.flags() == 0);
+ static_assert(test1.line_number() == 0);
+
+ constexpr NoLineBits test2 = NoLineBits::Set<3, 2, 1>();
+ static_assert(test2.level() == 3);
+ static_assert(test2.module() == 2);
+ static_assert(test2.flags() == 1);
+ static_assert(test2.line_number() == 0);
+
+ constexpr NoLineBits test3 = NoLineBits::Set<63, 65535, 1023>();
+ static_assert(test3.level() == 63);
+ static_assert(test3.module() == 65535);
+ static_assert(test3.flags() == 1023);
+ static_assert(test3.line_number() == 0);
+}
+
+TEST(Metadata, NoFlagBits) {
+ using NoFlagBits = internal::GenericMetadata<3, 13, 0, 16>;
+
+ constexpr NoFlagBits test1 = NoFlagBits::Set<0, 0, 0, 0>();
+ static_assert(test1.level() == 0);
+ static_assert(test1.module() == 0);
+ static_assert(test1.flags() == 0);
+ static_assert(test1.line_number() == 0);
+
+ constexpr NoFlagBits test2 = NoFlagBits::Set<3, 2, 0, 1>();
+ static_assert(test2.level() == 3);
+ static_assert(test2.module() == 2);
+ static_assert(test2.flags() == 0);
+ static_assert(test2.line_number() == 1);
+
+ constexpr NoFlagBits test3 = NoFlagBits::Set<7, 65535, 0, (1 << 13) - 1>();
+ static_assert(test3.level() == 7);
+ static_assert(test3.module() == 65535);
+ static_assert(test3.flags() == 0);
+ static_assert(test3.line_number() == (1 << 13) - 1);
+}
+
+} // namespace
+} // namespace pw::log_tokenized
diff --git a/pw_log_tokenized/public/pw_log_tokenized/config.h b/pw_log_tokenized/public/pw_log_tokenized/config.h
index 9c92d9f..de33510 100644
--- a/pw_log_tokenized/public/pw_log_tokenized/config.h
+++ b/pw_log_tokenized/public/pw_log_tokenized/config.h
@@ -15,39 +15,63 @@
#include <assert.h>
+#include "pw_log/levels.h"
#include "pw_log/options.h"
-#include "pw_preprocessor/concat.h"
// This macro takes the PW_LOG format string and optionally transforms it. By
-// default, the PW_LOG_MODULE_NAME is prepended to the string if present.
+// default, pw_log_tokenized specifies three fields as key-value pairs.
#ifndef PW_LOG_TOKENIZED_FORMAT_STRING
-#define PW_LOG_TOKENIZED_FORMAT_STRING(string) \
- PW_CONCAT(PW_LOG_TOKENIZED_FMT_, PW_LOG_MODULE_NAME_DEFINED)(string)
+#define _PW_LOG_TOKENIZED_FIELD(name, contents) "■" name "♦" contents
-#define PW_LOG_TOKENIZED_FMT_0(string) string
-#define PW_LOG_TOKENIZED_FMT_1(string) PW_LOG_MODULE_NAME " " string
+#define PW_LOG_TOKENIZED_FORMAT_STRING(string) \
+ _PW_LOG_TOKENIZED_FIELD("msg", string) \
+ _PW_LOG_TOKENIZED_FIELD("module", PW_LOG_MODULE_NAME) \
+ _PW_LOG_TOKENIZED_FIELD("file", __FILE__)
#endif // PW_LOG_TOKENIZED_FORMAT_STRING
-// The log level, module token, and flag bits are packed into the tokenizer's
-// payload argument, which is typically 32 bits. These macros specify the number
-// of bits to use for each field.
+// The log level, line number, flag bits, and module token are packed into the
+// tokenizer's payload argument, which is typically 32 bits. These macros
+// specify the number of bits to use for each field. A field with zero bits is
+// excluded.
+
+// Bits to allocate for the log level.
#ifndef PW_LOG_TOKENIZED_LEVEL_BITS
-#define PW_LOG_TOKENIZED_LEVEL_BITS 6
+#define PW_LOG_TOKENIZED_LEVEL_BITS PW_LOG_LEVEL_BITS
#endif // PW_LOG_TOKENIZED_LEVEL_BITS
+// Bits to allocate for the line number. Defaults to 11 (up to line 2047). If
+// the line number is too large to be represented by this field, line is
+// reported as 0.
+//
+// Including the line number can slightly increase code size. Without the line
+// number, the log metadata argument is the same for all logs with the same
+// level and flags. With the line number, each metadata value is unique and must
+// be encoded as a separate word in the binary. Systems with extreme space
+// constraints may exclude line numbers by setting this macro to 0.
+//
+// It is possible to include line numbers in tokenized log format strings, but
+// that is discouraged because line numbers change whenever a file is edited.
+// Passing the line number with the metadata is a lightweight way to include it.
+#ifndef PW_LOG_TOKENIZED_LINE_BITS
+#define PW_LOG_TOKENIZED_LINE_BITS 11
+#endif // PW_LOG_TOKENIZED_LINE_BITS
+
+// Bits to use for implementation-defined flags.
+#ifndef PW_LOG_TOKENIZED_FLAG_BITS
+#define PW_LOG_TOKENIZED_FLAG_BITS 2
+#endif // PW_LOG_TOKENIZED_FLAG_BITS
+
+// Bits to use for the tokenized version of PW_LOG_MODULE_NAME. Defaults to 16,
+// which gives a ~1% probability of a collision with 37 module names.
#ifndef PW_LOG_TOKENIZED_MODULE_BITS
#define PW_LOG_TOKENIZED_MODULE_BITS 16
#endif // PW_LOG_TOKENIZED_MODULE_BITS
-#ifndef PW_LOG_TOKENIZED_FLAG_BITS
-#define PW_LOG_TOKENIZED_FLAG_BITS 10
-#endif // PW_LOG_TOKENIZED_FLAG_BITS
-
-static_assert((PW_LOG_TOKENIZED_LEVEL_BITS + PW_LOG_TOKENIZED_MODULE_BITS +
- PW_LOG_TOKENIZED_FLAG_BITS) == 32,
- "Log metadata must fit in a 32-bit integer");
+static_assert((PW_LOG_TOKENIZED_LEVEL_BITS + PW_LOG_TOKENIZED_LINE_BITS +
+ PW_LOG_TOKENIZED_FLAG_BITS + PW_LOG_TOKENIZED_MODULE_BITS) == 32,
+ "Log metadata fields must use 32 bits");
// The macro to use to tokenize the log and its arguments. Defaults to
// PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD. Projects may define their own
diff --git a/pw_log_tokenized/public/pw_log_tokenized/log_tokenized.h b/pw_log_tokenized/public/pw_log_tokenized/log_tokenized.h
index 34acb74..42cc614 100644
--- a/pw_log_tokenized/public/pw_log_tokenized/log_tokenized.h
+++ b/pw_log_tokenized/public/pw_log_tokenized/log_tokenized.h
@@ -18,6 +18,11 @@
#include "pw_log_tokenized/config.h"
#include "pw_tokenizer/tokenize_to_global_handler_with_payload.h"
+// TODO(hepler): Remove this include.
+#ifdef __cplusplus
+#include "pw_log_tokenized/metadata.h"
+#endif // __cplusplus
+
// This macro implements PW_LOG using
// PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD or an equivalent alternate macro
// provided by PW_LOG_TOKENIZED_ENCODE_MESSAGE. The log level, module token, and
@@ -45,76 +50,54 @@
#define PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD( \
level, flags, message, ...) \
do { \
- _PW_TOKENIZER_CONST uintptr_t _pw_log_module_token = \
+ _PW_TOKENIZER_CONST uintptr_t _pw_log_tokenized_module_token = \
PW_TOKENIZE_STRING_MASK("pw_log_module_names", \
((1u << PW_LOG_TOKENIZED_MODULE_BITS) - 1u), \
PW_LOG_MODULE_NAME); \
+ const uintptr_t _pw_log_tokenized_level = level; \
PW_LOG_TOKENIZED_ENCODE_MESSAGE( \
- ((uintptr_t)(level) | \
- (_pw_log_module_token << PW_LOG_TOKENIZED_LEVEL_BITS) | \
- ((uintptr_t)(flags) \
- << (PW_LOG_TOKENIZED_LEVEL_BITS + PW_LOG_TOKENIZED_MODULE_BITS))), \
+ (_PW_LOG_TOKENIZED_LEVEL(_pw_log_tokenized_level) | \
+ _PW_LOG_TOKENIZED_MODULE(_pw_log_tokenized_module_token) | \
+ _PW_LOG_TOKENIZED_FLAGS(flags) | _PW_LOG_TOKENIZED_LINE()), \
PW_LOG_TOKENIZED_FORMAT_STRING(message), \
__VA_ARGS__); \
} while (0)
-#ifdef __cplusplus
+// If the level field is present, clamp it to the maximum value.
+#if PW_LOG_TOKENIZED_LEVEL_BITS == 0
+#define _PW_LOG_TOKENIZED_LEVEL(value) ((uintptr_t)0)
+#else
+#define _PW_LOG_TOKENIZED_LEVEL(value) \
+ (value < ((uintptr_t)1 << PW_LOG_TOKENIZED_LEVEL_BITS) \
+ ? value \
+ : ((uintptr_t)1 << PW_LOG_TOKENIZED_LEVEL_BITS) - 1)
+#endif // PW_LOG_TOKENIZED_LEVEL_BITS
-namespace pw {
-namespace log_tokenized {
-namespace internal {
+// If the line number field is present, shift it to its position. Set it to zero
+// if the line number is too large for PW_LOG_TOKENIZED_LINE_BITS.
+#if PW_LOG_TOKENIZED_LINE_BITS == 0
+#define _PW_LOG_TOKENIZED_LINE() ((uintptr_t)0)
+#else
+#define _PW_LOG_TOKENIZED_LINE() \
+ ((uintptr_t)(__LINE__ < (1 << PW_LOG_TOKENIZED_LINE_BITS) ? __LINE__ : 0) \
+ << PW_LOG_TOKENIZED_LEVEL_BITS)
+#endif // PW_LOG_TOKENIZED_LINE_BITS
-// This class, which is aliased to pw::log_tokenized::Metadata below, is used to
-// access the log metadata packed into the tokenizer's payload argument.
-template <unsigned kLevelBits,
- unsigned kModuleBits,
- unsigned kFlagBits,
- typename T = uintptr_t>
-class GenericMetadata {
- public:
- template <T log_level, T module, T flags>
- static constexpr GenericMetadata Set() {
- static_assert(log_level < (1 << kLevelBits), "The level is too large!");
- static_assert(module < (1 << kModuleBits), "The module is too large!");
- static_assert(flags < (1 << kFlagBits), "The flags are too large!");
+// If the flags field is present, mask it and shift it to its position.
+#if PW_LOG_TOKENIZED_FLAG_BITS == 0
+#define _PW_LOG_TOKENIZED_FLAGS(value) ((uintptr_t)0)
+#else
+#define _PW_LOG_TOKENIZED_FLAGS(value) \
+ (((uintptr_t)(value) & (((uintptr_t)1 << PW_LOG_TOKENIZED_FLAG_BITS) - 1)) \
+ << (PW_LOG_TOKENIZED_LEVEL_BITS + PW_LOG_TOKENIZED_LINE_BITS))
+#endif // PW_LOG_TOKENIZED_FLAG_BITS
- return GenericMetadata(log_level | (module << kLevelBits) |
- (flags << (kModuleBits + kLevelBits)));
- }
-
- constexpr GenericMetadata(T value) : bits_(value) {}
-
- // The log level of this message.
- constexpr T level() const { return bits_ & Mask<kLevelBits>(); }
-
- // The 16 bit tokenized version of the module name (PW_LOG_MODULE_NAME).
- constexpr T module() const {
- return (bits_ >> kLevelBits) & Mask<kModuleBits>();
- }
-
- // The flags provided to the log call.
- constexpr T flags() const {
- return (bits_ >> (kLevelBits + kModuleBits)) & Mask<kFlagBits>();
- }
-
- private:
- template <int bits>
- static constexpr T Mask() {
- return (T(1) << bits) - 1;
- }
-
- T bits_;
-
- static_assert(kLevelBits + kModuleBits + kFlagBits <= sizeof(bits_) * 8);
-};
-
-} // namespace internal
-
-using Metadata = internal::GenericMetadata<PW_LOG_TOKENIZED_LEVEL_BITS,
- PW_LOG_TOKENIZED_MODULE_BITS,
- PW_LOG_TOKENIZED_FLAG_BITS>;
-
-} // namespace log_tokenized
-} // namespace pw
-
-#endif // __cpluplus
+// If the module field is present, shift it to its position.
+#if PW_LOG_TOKENIZED_MODULE_BITS == 0
+#define _PW_LOG_TOKENIZED_MODULE(value) ((uintptr_t)0)
+#else
+#define _PW_LOG_TOKENIZED_MODULE(value) \
+ ((uintptr_t)(value) << ((PW_LOG_TOKENIZED_LEVEL_BITS + \
+ PW_LOG_TOKENIZED_LINE_BITS + \
+ PW_LOG_TOKENIZED_FLAG_BITS)))
+#endif // PW_LOG_TOKENIZED_MODULE_BITS
diff --git a/pw_log_tokenized/public/pw_log_tokenized/metadata.h b/pw_log_tokenized/public/pw_log_tokenized/metadata.h
new file mode 100644
index 0000000..86e6364
--- /dev/null
+++ b/pw_log_tokenized/public/pw_log_tokenized/metadata.h
@@ -0,0 +1,99 @@
+// Copyright 2021 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+#pragma once
+
+#include <cstdint>
+
+#include "pw_log_tokenized/config.h"
+
+namespace pw {
+namespace log_tokenized {
+namespace internal {
+
+// Internal class for managing the metadata bit fields.
+template <typename T, unsigned kBits, unsigned kShift>
+struct BitField {
+ public:
+ static constexpr T Get(T value) { return (value >> kShift) & kMask; }
+ static constexpr T Shift(T value) {
+ return (value <= kMask ? value : T(0)) << kShift;
+ }
+
+ private:
+ static constexpr T kMask = (T(1) << kBits) - 1;
+};
+
+template <typename T, unsigned kShift>
+class BitField<T, 0, kShift> {
+ public:
+ static constexpr T Get(T) { return 0; }
+ static constexpr T Shift(T) { return 0; }
+};
+
+// This class, which is aliased to pw::log_tokenized::Metadata below, is used to
+// access the log metadata packed into the tokenizer's payload argument.
+template <unsigned kLevelBits,
+ unsigned kLineBits,
+ unsigned kFlagBits,
+ unsigned kModuleBits,
+ typename T = uintptr_t>
+class GenericMetadata {
+ public:
+ template <T log_level = 0, T module = 0, T flags = 0, T line = 0>
+ static constexpr GenericMetadata Set() {
+ static_assert(log_level < (1 << kLevelBits), "The level is too large!");
+ static_assert(line < (1 << kLineBits), "The line number is too large!");
+ static_assert(flags < (1 << kFlagBits), "The flags are too large!");
+ static_assert(module < (1 << kModuleBits), "The module is too large!");
+
+ return GenericMetadata(Level::Shift(log_level) | Module::Shift(module) |
+ Flags::Shift(flags) | Line::Shift(line));
+ }
+
+ constexpr GenericMetadata(T value) : bits_(value) {}
+
+ // The log level of this message.
+ constexpr T level() const { return Level::Get(bits_); }
+
+ // The line number of the log call. The first line in a file is 1. If the line
+ // number is 0, it was too large to be stored.
+ constexpr T line_number() const { return Line::Get(bits_); }
+
+ // The flags provided to the log call.
+ constexpr T flags() const { return Flags::Get(bits_); }
+
+ // The 16 bit tokenized version of the module name (PW_LOG_MODULE_NAME).
+ constexpr T module() const { return Module::Get(bits_); }
+
+ private:
+ using Level = BitField<T, kLevelBits, 0>;
+ using Line = BitField<T, kLineBits, kLevelBits>;
+ using Flags = BitField<T, kFlagBits, kLevelBits + kLineBits>;
+ using Module = BitField<T, kModuleBits, kLevelBits + kLineBits + kFlagBits>;
+
+ T bits_;
+
+ static_assert(kLevelBits + kLineBits + kFlagBits + kModuleBits <=
+ sizeof(bits_) * 8);
+};
+
+} // namespace internal
+
+using Metadata = internal::GenericMetadata<PW_LOG_TOKENIZED_LEVEL_BITS,
+ PW_LOG_TOKENIZED_LINE_BITS,
+ PW_LOG_TOKENIZED_FLAG_BITS,
+ PW_LOG_TOKENIZED_MODULE_BITS>;
+
+} // namespace log_tokenized
+} // namespace pw
diff --git a/pw_log_tokenized/pw_log_tokenized_private/test_utils.h b/pw_log_tokenized/pw_log_tokenized_private/test_utils.h
new file mode 100644
index 0000000..dce3a3d
--- /dev/null
+++ b/pw_log_tokenized/pw_log_tokenized_private/test_utils.h
@@ -0,0 +1,52 @@
+// Copyright 2021 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "pw_log_tokenized/log_tokenized.h"
+#include "pw_preprocessor/arguments.h"
+#include "pw_preprocessor/compiler.h"
+#include "pw_preprocessor/util.h"
+
+// Create a fake version of the tokenization macro.
+#undef PW_LOG_TOKENIZED_ENCODE_MESSAGE
+#define PW_LOG_TOKENIZED_ENCODE_MESSAGE(payload, message, ...) \
+ pw_log_tokenized_CaptureArgs(payload, \
+ PW_MACRO_ARG_COUNT(__VA_ARGS__), \
+ message PW_COMMA_ARGS(__VA_ARGS__))
+
+PW_EXTERN_C_START
+
+typedef struct {
+ uintptr_t metadata;
+ const char* format_string;
+ size_t arg_count;
+} pw_log_tokenized_CapturedLog;
+
+extern pw_log_tokenized_CapturedLog last_log;
+
+void pw_log_tokenized_CaptureArgs(uintptr_t payload,
+ size_t arg_count,
+ const char* message,
+ ...) PW_PRINTF_FORMAT(3, 4);
+
+// C versions of tests.
+void pw_log_tokenized_Test_LogMetadata_LevelTooLarge_Clamps(void);
+void pw_log_tokenized_Test_LogMetadata_TooManyFlags_Truncates(void);
+void pw_log_tokenized_Test_LogMetadata_LogMetadata_VariousValues(void);
+void pw_log_tokenized_Test_LogMetadata_LogMetadata_Zero(void);
+
+PW_EXTERN_C_END
diff --git a/pw_log_tokenized/test.cc b/pw_log_tokenized/test.cc
deleted file mode 100644
index 7a843f4..0000000
--- a/pw_log_tokenized/test.cc
+++ /dev/null
@@ -1,100 +0,0 @@
-// Copyright 2021 The Pigweed Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not
-// use this file except in compliance with the License. You may obtain a copy of
-// the License at
-//
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-// License for the specific language governing permissions and limitations under
-// the License.
-
-#define PW_LOG_MODULE_NAME "This is the log module name!"
-
-// Create a fake version of the tokenization macro.
-#undef PW_LOG_TOKENIZED_ENCODE_MESSAGE
-#define PW_LOG_TOKENIZED_ENCODE_MESSAGE(payload, message, ...) \
- CaptureTokenizerArgs(payload, \
- PW_MACRO_ARG_COUNT(__VA_ARGS__), \
- message PW_COMMA_ARGS(__VA_ARGS__))
-
-#include <cstring>
-#include <string_view>
-#include <tuple>
-
-#include "gtest/gtest.h"
-#include "pw_log_tokenized/log_tokenized.h"
-#include "pw_preprocessor/arguments.h"
-#include "pw_preprocessor/compiler.h"
-
-namespace pw::log_tokenized {
-namespace {
-
-struct {
- Metadata metadata = Metadata(0);
- const char* format_string = "";
- size_t arg_count = 0;
-} last_log{};
-
-void CaptureTokenizerArgs(pw_tokenizer_Payload payload,
- size_t arg_count,
- const char* message,
- ...) PW_PRINTF_FORMAT(3, 4);
-
-void CaptureTokenizerArgs(pw_tokenizer_Payload payload,
- size_t arg_count,
- const char* message,
- ...) {
- last_log.metadata = payload;
- last_log.format_string = message;
- last_log.arg_count = arg_count;
-}
-
-constexpr uintptr_t kModuleToken =
- PW_TOKENIZER_STRING_TOKEN(PW_LOG_MODULE_NAME) &
- ((1u << PW_LOG_TOKENIZED_MODULE_BITS) - 1);
-
-constexpr Metadata test1 = Metadata::Set<0, 0, 0>();
-static_assert(test1.level() == 0);
-static_assert(test1.module() == 0);
-static_assert(test1.flags() == 0);
-
-constexpr Metadata test2 = Metadata::Set<3, 2, 1>();
-static_assert(test2.level() == 3);
-static_assert(test2.module() == 2);
-static_assert(test2.flags() == 1);
-
-constexpr Metadata test3 = Metadata::Set<63, 65535, 1023>();
-static_assert(test3.level() == 63);
-static_assert(test3.module() == 65535);
-static_assert(test3.flags() == 1023);
-
-TEST(LogTokenized, LogMetadata_Zero) {
- PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD(0, 0, "hello");
- EXPECT_EQ(last_log.metadata.level(), 0u);
- EXPECT_EQ(last_log.metadata.flags(), 0u);
- EXPECT_EQ(last_log.metadata.module(), kModuleToken);
- EXPECT_EQ(last_log.arg_count, 0u);
-}
-
-TEST(LogTokenized, LogMetadata_DifferentValues) {
- PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD(55, 36, "hello%s", "?");
- EXPECT_EQ(last_log.metadata.level(), 55u);
- EXPECT_EQ(last_log.metadata.flags(), 36u);
- EXPECT_EQ(last_log.metadata.module(), kModuleToken);
- EXPECT_EQ(last_log.arg_count, 1u);
-}
-
-TEST(LogTokenized, LogMetadata_MaxValues) {
- PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD(63, 1023, "hello %d", 1);
- EXPECT_EQ(last_log.metadata.level(), 63u);
- EXPECT_EQ(last_log.metadata.flags(), 1023u);
- EXPECT_EQ(last_log.metadata.module(), kModuleToken);
- EXPECT_EQ(last_log.arg_count, 1u);
-}
-
-} // namespace
-} // namespace pw::log_tokenized
diff --git a/pw_tokenizer/docs.rst b/pw_tokenizer/docs.rst
index 8967f12..168bc4a 100644
--- a/pw_tokenizer/docs.rst
+++ b/pw_tokenizer/docs.rst
@@ -975,8 +975,9 @@
.. attention::
Do not encode line numbers in tokenized strings. This results in a huge
number of lines being added to the database, since every time code moves,
- new strings are tokenized. If line numbers are desired in a tokenized
- string, add a ``"%d"`` to the string and pass ``__LINE__`` as an argument.
+ new strings are tokenized. If :ref:`module-pw_log_tokenized` is used, line
+ numbers are encoded in the log metadata. Line numbers may also be included by
+ by adding ``"%d"`` to the format string and passing ``__LINE__``.
Database management
-------------------
