pw_tokenizer: Custom tokenization macro support
- Make the pw::tokenizer::EncodeArgs function and pw_tokenizer_ArgTypes
typedef public.
- Update the pw::tokenizer::EncodedMessage class to make it simpler to
use. Also, use memcpy instead of aliasing the token.
Change-Id: Ic99930ccf3e002e4fbef5112f6b2d37c910e6ee9
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/39984
Pigweed-Auto-Submit: Wyatt Hepler <hepler@google.com>
Reviewed-by: Keir Mierle <keir@google.com>
Commit-Queue: Wyatt Hepler <hepler@google.com>
diff --git a/pw_log_tokenized/docs.rst b/pw_log_tokenized/docs.rst
index f64f524..3c100ed 100644
--- a/pw_log_tokenized/docs.rst
+++ b/pw_log_tokenized/docs.rst
@@ -39,13 +39,18 @@
See the documentation for :ref:`module-pw_tokenizer` for further details.
-Applications may select a different macro than
+Using a custom macro
+--------------------
+Applications may use their own macro instead of
``PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD`` by setting the
``PW_LOG_TOKENIZED_ENCODE_MESSAGE`` config macro. This macro should take
arguments equivalent to ``PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD``:
.. c:function:: PW_LOG_TOKENIZED_ENCODE_MESSAGE(pw_tokenizer_Payload log_metadata, const char* message, ...)
+For instructions on how to implement a custom tokenization macro, see
+:ref:`module-pw_tokenizer-custom-macro`.
+
Build targets
-------------
The GN build for ``pw_log_tokenized`` has two targets: ``pw_log_tokenized`` and
@@ -57,8 +62,7 @@
Python package
==============
-``pw_log_tokenized`` includes a Python package for decoding tokenized messages
-logs.
+``pw_log_tokenized`` includes a Python package for decoding tokenized logs.
pw_log_tokenized
----------------
diff --git a/pw_tokenizer/argument_types_test.cc b/pw_tokenizer/argument_types_test.cc
index 67c1d17..cf3b22b 100644
--- a/pw_tokenizer/argument_types_test.cc
+++ b/pw_tokenizer/argument_types_test.cc
@@ -187,38 +187,38 @@
TEST(ArgumentTypes, MultipleArgs) {
// clang-format off
- static_assert(PW_TOKENIZER_ARG_TYPES(1) == 1); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2) == 2); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3) == 3); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4) == 4); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5) == 5); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6) == 6); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7) == 7); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8) == 8); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9) == 9); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) == 10); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11) == 11); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12) == 12); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13) == 13); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14) == 14); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1) == 1);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2) == 2);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3) == 3);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4) == 4);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5) == 5);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6) == 6);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7) == 7);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8) == 8);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9) == 9);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) == 10);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11) == 11);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12) == 12);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13) == 13);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14) == 14);
#if PW_TOKENIZER_CFG_ARG_TYPES_SIZE_BYTES >= 8
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14) == 14); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15) == 15); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16) == 16); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17) == 17); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18) == 18); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19) == 19); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20) == 20); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21) == 21); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22) == 22); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23) == 23); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24) == 24); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25) == 25); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26) == 26); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) == 27); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28) == 28); // NOLINT
- static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29) == 29); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14) == 14);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15) == 15);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16) == 16);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17) == 17);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18) == 18);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19) == 19);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20) == 20);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21) == 21);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22) == 22);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23) == 23);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24) == 24);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25) == 25);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26) == 26);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) == 27);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28) == 28);
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29) == 29);
#endif // PW_TOKENIZER_CFG_ARG_TYPES_SIZE_BYTES
// clang-format on
}
diff --git a/pw_tokenizer/argument_types_test_c.c b/pw_tokenizer/argument_types_test_c.c
index 2030587..849357d 100644
--- a/pw_tokenizer/argument_types_test_c.c
+++ b/pw_tokenizer/argument_types_test_c.c
@@ -72,10 +72,10 @@
static char char_array[16];
// Define the test functions that are called by the C++ unit test.
-#define DEFINE_TEST_FUNCTION(name, ...) \
- _pw_tokenizer_ArgTypes pw_TestTokenizer##name(void) { \
- (void)char_array; \
- return PW_TOKENIZER_ARG_TYPES(__VA_ARGS__); \
+#define DEFINE_TEST_FUNCTION(name, ...) \
+ pw_tokenizer_ArgTypes pw_TestTokenizer##name(void) { \
+ (void)char_array; \
+ return PW_TOKENIZER_ARG_TYPES(__VA_ARGS__); \
}
DEFINE_TEST_FUNCTION(NoArgs);
diff --git a/pw_tokenizer/docs.rst b/pw_tokenizer/docs.rst
index eec6773..ee9d154 100644
--- a/pw_tokenizer/docs.rst
+++ b/pw_tokenizer/docs.rst
@@ -233,8 +233,97 @@
widely expanded macros, such as a logging macro, because it will result in
larger code size than its alternatives.
-Example: binary logging
-^^^^^^^^^^^^^^^^^^^^^^^
+.. _module-pw_tokenizer-custom-macro:
+
+Tokenize with a custom macro
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Projects may need more flexbility than the standard ``pw_tokenizer`` macros
+provide. To support this, projects may define custom tokenization macros. This
+requires the use of two low-level ``pw_tokenizer`` macros:
+
+.. c:macro:: PW_TOKENIZE_FORMAT_STRING(domain, mask, format, ...)
+
+ Tokenizes a format string and sets the ``_pw_tokenizer_token`` variable to the
+ token. Must be used in its own scope, since the same variable is used in every
+ invocation.
+
+ The tokenized string uses the specified :ref:`tokenization domain
+ <module-pw_tokenizer-domains>`. Use ``PW_TOKENIZER_DEFAULT_DOMAIN`` for the
+ default. The token also may be masked; use ``UINT32_MAX`` to keep all bits.
+
+.. c:macro:: PW_TOKENIZER_ARG_TYPES(...)
+
+ Converts a series of arguments to a compact format that replaces the format
+ string literal.
+
+Use these two macros within the custom tokenization macro to call a function
+that does the encoding. The following example implements a custom tokenization
+macro for use with :ref:`module-pw_log_tokenized`.
+
+.. code-block:: cpp
+
+ #include "pw_tokenizer/tokenize.h"
+
+ #ifndef __cplusplus
+ extern "C" {
+ #endif
+
+ void EncodeTokenizedMessage(pw_tokenizer_Payload metadata,
+ pw_tokenizer_Token token,
+ pw_tokenizer_ArgTypes types,
+ ...);
+
+ #ifndef __cplusplus
+ } // extern "C"
+ #endif
+
+ #define PW_LOG_TOKENIZED_ENCODE_MESSAGE(metadata, format, ...) \
+ do { \
+ _PW_TOKENIZE_FORMAT_STRING( \
+ PW_TOKENIZER_DEFAULT_DOMAIN, UINT32_MAX, format, __VA_ARGS__); \
+ EncodeTokenizedMessage(payload, \
+ _pw_tokenizer_token, \
+ PW_TOKENIZER_ARG_TYPES(__VA_ARGS__) \
+ PW_COMMA_ARGS(__VA_ARGS__)); \
+ } while (0)
+
+In this example, the ``EncodeTokenizedMessage`` function would handle encoding
+and processing the message. Encoding is done by the
+``pw::tokenizer::EncodedMessage`` class or ``pw::tokenizer::EncodeArgs``
+function from ``pw_tokenizer/encode_args.h``. The encoded message can then be
+transmitted or stored as needed.
+
+.. code-block:: cpp
+
+ #include "pw_log_tokenized/log_tokenized.h"
+ #include "pw_tokenizer/encode_args.h"
+
+ void HandleTokenizedMessage(pw::log_tokenized::Metadata metadata,
+ std::span<std::byte> message);
+
+ extern "C" void EncodeTokenizedMessage(const pw_tokenizer_Payload metadata,
+ const pw_tokenizer_Token token,
+ const pw_tokenizer_ArgTypes types,
+ ...) {
+ va_list args;
+ va_start(args, types);
+ pw::tokenizer::EncodedMessage encoded_message(token, types, args);
+ va_end(args);
+
+ HandleTokenizedMessage(metadata, encoded_message);
+ }
+
+.. admonition:: When to use a custom macro
+
+ Use existing tokenization macros whenever possible. A custom macro may be
+ needed to support use cases like the following:
+
+ * Variations of ``PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD`` that take
+ different arguments.
+ * Supporting global handler macros that use different handler functions.
+
+Binary logging with pw_tokenizer
+--------------------------------
String tokenization is perfect for logging. Consider the following log macro,
which gathers the file, line number, and log message. It calls the ``RecordLog``
function, which formats the log string, collects a timestamp, and transmits the
@@ -363,6 +452,8 @@
calculated values will differ between C and C++ for strings longer than
``PW_TOKENIZER_CFG_C_HASH_LENGTH`` characters.
+.. _module-pw_tokenizer-domains:
+
Tokenization domains
--------------------
``pw_tokenizer`` supports having multiple tokenization domains. Domains are a
diff --git a/pw_tokenizer/encode_args.cc b/pw_tokenizer/encode_args.cc
index 041d51f..93cd6c4 100644
--- a/pw_tokenizer/encode_args.cc
+++ b/pw_tokenizer/encode_args.cc
@@ -38,15 +38,15 @@
static_assert(0b10u == static_cast<uint8_t>(ArgType::kDouble));
static_assert(0b11u == static_cast<uint8_t>(ArgType::kString));
-size_t EncodeInt(int value, const std::span<uint8_t>& output) {
+size_t EncodeInt(int value, const std::span<std::byte>& output) {
return varint::Encode(value, std::as_writable_bytes(output));
}
-size_t EncodeInt64(int64_t value, const std::span<uint8_t>& output) {
+size_t EncodeInt64(int64_t value, const std::span<std::byte>& output) {
return varint::Encode(value, std::as_writable_bytes(output));
}
-size_t EncodeFloat(float value, const std::span<uint8_t>& output) {
+size_t EncodeFloat(float value, const std::span<std::byte>& output) {
if (output.size() < sizeof(value)) {
return 0;
}
@@ -54,7 +54,7 @@
return sizeof(value);
}
-size_t EncodeString(const char* string, const std::span<uint8_t>& output) {
+size_t EncodeString(const char* string, const std::span<std::byte>& output) {
// The top bit of the status byte indicates if the string was truncated.
static constexpr size_t kMaxStringLength = 0x7Fu;
@@ -71,17 +71,17 @@
// Scan the string to find out how many bytes to copy.
size_t bytes_to_copy = 0;
- uint8_t overflow_bit = 0;
+ std::byte overflow_bit = std::byte(0);
while (string[bytes_to_copy] != '\0') {
if (bytes_to_copy == max_bytes) {
- overflow_bit = '\x80';
+ overflow_bit = std::byte('\x80');
break;
}
bytes_to_copy += 1;
}
- output[0] = bytes_to_copy | overflow_bit;
+ output[0] = static_cast<std::byte>(bytes_to_copy) | overflow_bit;
std::memcpy(output.data() + 1, string, bytes_to_copy);
return bytes_to_copy + 1; // include the status byte in the total
@@ -89,9 +89,9 @@
} // namespace
-size_t EncodeArgs(_pw_tokenizer_ArgTypes types,
+size_t EncodeArgs(pw_tokenizer_ArgTypes types,
va_list args,
- std::span<uint8_t> output) {
+ std::span<std::byte> output) {
size_t arg_count = types & PW_TOKENIZER_TYPE_COUNT_MASK;
types >>= PW_TOKENIZER_TYPE_COUNT_SIZE_BITS;
diff --git a/pw_tokenizer/public/pw_tokenizer/encode_args.h b/pw_tokenizer/public/pw_tokenizer/encode_args.h
index d16c3bf..19f4e88 100644
--- a/pw_tokenizer/public/pw_tokenizer/encode_args.h
+++ b/pw_tokenizer/public/pw_tokenizer/encode_args.h
@@ -13,9 +13,9 @@
// the License.
#pragma once
-#include <array>
#include <cstdarg>
#include <cstddef>
+#include <cstring>
#include <span>
#include "pw_tokenizer/config.h"
@@ -25,12 +25,64 @@
namespace pw {
namespace tokenizer {
-// Buffer for encoding a tokenized string and arguments.
-struct EncodedMessage {
- pw_tokenizer_Token token;
- std::array<uint8_t,
- PW_TOKENIZER_CFG_ENCODING_BUFFER_SIZE_BYTES - sizeof(token)>
- args;
+// Encodes a tokenized string's arguments to a buffer. The
+// pw_tokenizer_ArgTypes parameter specifies the argument types, in place of a
+// format string.
+//
+// Most tokenization implementations may use the EncodedMessage class below.
+size_t EncodeArgs(pw_tokenizer_ArgTypes types,
+ va_list args,
+ std::span<std::byte> output);
+
+// Encodes a tokenized message to a fixed size buffer. The size of the buffer is
+// determined by the PW_TOKENIZER_CFG_ENCODING_BUFFER_SIZE_BYTES config macro.
+//
+// This class is used to encode tokenized messages passed in from the
+// tokenization macros. The macros provided by pw_tokenizer use this class, and
+// projects that elect to define their own versions of the tokenization macros
+// should use it when possible.
+//
+// To use the pw::Tokenizer::EncodedMessage, construct it with the token,
+// argument types, and va_list from the variadic arguments:
+//
+// void SendLogMessage(std::span<std::byte> log_data);
+//
+// extern "C" void TokenizeToSendLogMessage(pw_tokenizer_Token token,
+// pw_tokenizer_ArgTypes types,
+// ...) {
+// va_list args;
+// va_start(args, types);
+// EncodedMessage encoded_message(token, types, args);
+// va_end(args);
+//
+// SendLogMessage(encoded_message); // EncodedMessage converts to std::span
+// }
+//
+class EncodedMessage {
+ public:
+ // Encodes a tokenized message to an internal buffer.
+ EncodedMessage(pw_tokenizer_Token token,
+ pw_tokenizer_ArgTypes types,
+ va_list args) {
+ std::memcpy(data_, &token, sizeof(token));
+ args_size_ = EncodeArgs(
+ types, args, std::span<std::byte>(data_).subspan(sizeof(token)));
+ }
+
+ // The binary-encoded tokenized message.
+ const std::byte* data() const { return data_; }
+
+ // Returns the data() as a pointer to uint8_t instead of std::byte.
+ const uint8_t* data_as_uint8() const {
+ return reinterpret_cast<const uint8_t*>(data());
+ }
+
+ // The size of the encoded tokenized message in bytes.
+ size_t size() const { return sizeof(pw_tokenizer_Token) + args_size_; }
+
+ private:
+ std::byte data_[PW_TOKENIZER_CFG_ENCODING_BUFFER_SIZE_BYTES];
+ size_t args_size_;
};
static_assert(PW_TOKENIZER_CFG_ENCODING_BUFFER_SIZE_BYTES >=
@@ -38,17 +90,5 @@
"PW_TOKENIZER_CFG_ENCODING_BUFFER_SIZE_BYTES must be at least "
"large enough for a token (4 bytes)");
-static_assert(offsetof(EncodedMessage, args) == sizeof(EncodedMessage::token) &&
- PW_TOKENIZER_CFG_ENCODING_BUFFER_SIZE_BYTES ==
- sizeof(EncodedMessage),
- "EncodedMessage should not have padding bytes between members");
-
-// Encodes a tokenized string's arguments to a buffer. The
-// _pw_tokenizer_ArgTypes parameter specifies the argument types, in place of a
-// format string.
-size_t EncodeArgs(_pw_tokenizer_ArgTypes types,
- va_list args,
- std::span<uint8_t> output);
-
} // namespace tokenizer
} // namespace pw
diff --git a/pw_tokenizer/public/pw_tokenizer/internal/argument_types.h b/pw_tokenizer/public/pw_tokenizer/internal/argument_types.h
index 1f431e4..79f631d 100644
--- a/pw_tokenizer/public/pw_tokenizer/internal/argument_types.h
+++ b/pw_tokenizer/public/pw_tokenizer/internal/argument_types.h
@@ -31,7 +31,7 @@
#define PW_TOKENIZER_TYPE_COUNT_SIZE_BITS 4u
#define PW_TOKENIZER_TYPE_COUNT_MASK 0x0Fu
-typedef uint32_t _pw_tokenizer_ArgTypes;
+typedef uint32_t pw_tokenizer_ArgTypes;
#elif PW_TOKENIZER_CFG_ARG_TYPES_SIZE_BYTES == 8
@@ -42,7 +42,7 @@
#define PW_TOKENIZER_TYPE_COUNT_SIZE_BITS 6u
#define PW_TOKENIZER_TYPE_COUNT_MASK 0x1Fu // only 5 bits will be needed
-typedef uint64_t _pw_tokenizer_ArgTypes;
+typedef uint64_t pw_tokenizer_ArgTypes;
#else
@@ -52,7 +52,7 @@
// The tokenized string encoding function is a variadic function that works
// similarly to printf. Instead of a format string, however, the argument types
-// are packed into a _pw_tokenizer_ArgTypes.
+// are packed into a pw_tokenizer_ArgTypes.
//
// The four supported argument types are represented by two-bit argument codes.
// Just four types are required because only printf-compatible arguments are
@@ -62,10 +62,10 @@
// char* values cannot be printed as pointers with %p. These arguments are
// always encoded as strings. To format a char* as an address, cast it to void*
// or an integer.
-#define PW_TOKENIZER_ARG_TYPE_INT ((_pw_tokenizer_ArgTypes)0)
-#define PW_TOKENIZER_ARG_TYPE_INT64 ((_pw_tokenizer_ArgTypes)1)
-#define PW_TOKENIZER_ARG_TYPE_DOUBLE ((_pw_tokenizer_ArgTypes)2)
-#define PW_TOKENIZER_ARG_TYPE_STRING ((_pw_tokenizer_ArgTypes)3)
+#define PW_TOKENIZER_ARG_TYPE_INT ((pw_tokenizer_ArgTypes)0)
+#define PW_TOKENIZER_ARG_TYPE_INT64 ((pw_tokenizer_ArgTypes)1)
+#define PW_TOKENIZER_ARG_TYPE_DOUBLE ((pw_tokenizer_ArgTypes)2)
+#define PW_TOKENIZER_ARG_TYPE_STRING ((pw_tokenizer_ArgTypes)3)
// Select the int argument type based on the size of the type. Values smaller
// than int are promoted to int.
@@ -89,7 +89,7 @@
// This function selects the matching type enum for supported argument types.
template <typename T>
-constexpr _pw_tokenizer_ArgTypes VarargsType() {
+constexpr pw_tokenizer_ArgTypes VarargsType() {
using ArgType = std::decay_t<T>;
if constexpr (std::is_floating_point<ArgType>()) {
@@ -116,26 +116,26 @@
template <typename T, bool kDontCare1, bool kDontCare2>
struct SelectVarargsType<T, true, kDontCare1, kDontCare2> {
- static constexpr _pw_tokenizer_ArgTypes kValue = PW_TOKENIZER_ARG_TYPE_DOUBLE;
+ static constexpr pw_tokenizer_ArgTypes kValue = PW_TOKENIZER_ARG_TYPE_DOUBLE;
};
template <typename T, bool kDontCare>
struct SelectVarargsType<T, false, true, kDontCare> {
- static constexpr _pw_tokenizer_ArgTypes kValue = PW_TOKENIZER_ARG_TYPE_STRING;
+ static constexpr pw_tokenizer_ArgTypes kValue = PW_TOKENIZER_ARG_TYPE_STRING;
};
template <typename T>
struct SelectVarargsType<T, false, false, true> {
- static constexpr _pw_tokenizer_ArgTypes kValue = PW_TOKENIZER_ARG_TYPE_INT64;
+ static constexpr pw_tokenizer_ArgTypes kValue = PW_TOKENIZER_ARG_TYPE_INT64;
};
template <typename T>
struct SelectVarargsType<T, false, false, false> {
- static constexpr _pw_tokenizer_ArgTypes kValue = PW_TOKENIZER_ARG_TYPE_INT;
+ static constexpr pw_tokenizer_ArgTypes kValue = PW_TOKENIZER_ARG_TYPE_INT;
};
template <typename T>
-constexpr _pw_tokenizer_ArgTypes VarargsType() {
+constexpr pw_tokenizer_ArgTypes VarargsType() {
return SelectVarargsType<typename std::decay<T>::type>::kValue;
}
@@ -174,8 +174,8 @@
#endif // __cplusplus
-// Encodes the types of the provided arguments as a _pw_tokenizer_ArgTypes
-// value. Depending on the size of _pw_tokenizer_ArgTypes, the bottom 4 or 6
+// Encodes the types of the provided arguments as a pw_tokenizer_ArgTypes
+// value. Depending on the size of pw_tokenizer_ArgTypes, the bottom 4 or 6
// bits store the number of arguments and the remaining bits store the types,
// two bits per type.
//
@@ -184,4 +184,4 @@
#define PW_TOKENIZER_ARG_TYPES(...) \
PW_DELEGATE_BY_ARG_COUNT(_PW_TOKENIZER_TYPES_, __VA_ARGS__)
-#define _PW_TOKENIZER_TYPES_0() ((_pw_tokenizer_ArgTypes)0)
+#define _PW_TOKENIZER_TYPES_0() ((pw_tokenizer_ArgTypes)0)
diff --git a/pw_tokenizer/public/pw_tokenizer/tokenize.h b/pw_tokenizer/public/pw_tokenizer/tokenize.h
index 7b2fa49..cf21cd6 100644
--- a/pw_tokenizer/public/pw_tokenizer/tokenize.h
+++ b/pw_tokenizer/public/pw_tokenizer/tokenize.h
@@ -115,15 +115,15 @@
domain, UINT32_MAX, buffer, buffer_size_pointer, format, __VA_ARGS__)
// Same as PW_TOKENIZE_TO_BUFFER_DOMAIN, but applies a mask to the token.
-#define PW_TOKENIZE_TO_BUFFER_MASK( \
- domain, mask, buffer, buffer_size_pointer, format, ...) \
- do { \
- _PW_TOKENIZE_FORMAT_STRING(domain, mask, format, __VA_ARGS__); \
- _pw_tokenizer_ToBuffer(buffer, \
- buffer_size_pointer, \
- _pw_tokenizer_token, \
- PW_TOKENIZER_ARG_TYPES(__VA_ARGS__) \
- PW_COMMA_ARGS(__VA_ARGS__)); \
+#define PW_TOKENIZE_TO_BUFFER_MASK( \
+ domain, mask, buffer, buffer_size_pointer, format, ...) \
+ do { \
+ PW_TOKENIZE_FORMAT_STRING(domain, mask, format, __VA_ARGS__); \
+ _pw_tokenizer_ToBuffer(buffer, \
+ buffer_size_pointer, \
+ _pw_tokenizer_token, \
+ PW_TOKENIZER_ARG_TYPES(__VA_ARGS__) \
+ PW_COMMA_ARGS(__VA_ARGS__)); \
} while (0)
// Encodes a tokenized string and arguments to a buffer on the stack. The
@@ -166,7 +166,7 @@
// Same as PW_TOKENIZE_TO_CALLBACK_DOMAIN, but applies a mask to the token.
#define PW_TOKENIZE_TO_CALLBACK_MASK(domain, mask, callback, format, ...) \
do { \
- _PW_TOKENIZE_FORMAT_STRING(domain, mask, format, __VA_ARGS__); \
+ PW_TOKENIZE_FORMAT_STRING(domain, mask, format, __VA_ARGS__); \
_pw_tokenizer_ToCallback(callback, \
_pw_tokenizer_token, \
PW_TOKENIZER_ARG_TYPES(__VA_ARGS__) \
@@ -180,13 +180,13 @@
void _pw_tokenizer_ToBuffer(void* buffer,
size_t* buffer_size_bytes, // input and output arg
pw_tokenizer_Token token,
- _pw_tokenizer_ArgTypes types,
+ pw_tokenizer_ArgTypes types,
...);
void _pw_tokenizer_ToCallback(void (*callback)(const uint8_t* encoded_message,
size_t size_bytes),
pw_tokenizer_Token token,
- _pw_tokenizer_ArgTypes types,
+ pw_tokenizer_ArgTypes types,
...);
// This empty function allows the compiler to check the format string.
@@ -204,9 +204,9 @@
// This macro takes a printf-style format string and corresponding arguments. It
// checks that the arguments are correct, stores the format string in a special
-// section, and calculates the string's token at compile time.
+// section, and calculates the string's token at compile time. This
// clang-format off
-#define _PW_TOKENIZE_FORMAT_STRING(domain, mask, format, ...) \
+#define PW_TOKENIZE_FORMAT_STRING(domain, mask, format, ...) \
if (0) { /* Do not execute to prevent double evaluation of the arguments. */ \
pw_tokenizer_CheckFormatString(format PW_COMMA_ARGS(__VA_ARGS__)); \
} \
diff --git a/pw_tokenizer/public/pw_tokenizer/tokenize_to_global_handler.h b/pw_tokenizer/public/pw_tokenizer/tokenize_to_global_handler.h
index 72a5e7d..ca7e0b9 100644
--- a/pw_tokenizer/public/pw_tokenizer/tokenize_to_global_handler.h
+++ b/pw_tokenizer/public/pw_tokenizer/tokenize_to_global_handler.h
@@ -54,7 +54,7 @@
// token.
#define PW_TOKENIZE_TO_GLOBAL_HANDLER_MASK(domain, mask, format, ...) \
do { \
- _PW_TOKENIZE_FORMAT_STRING(domain, mask, format, __VA_ARGS__); \
+ PW_TOKENIZE_FORMAT_STRING(domain, mask, format, __VA_ARGS__); \
_pw_tokenizer_ToGlobalHandler(_pw_tokenizer_token, \
PW_TOKENIZER_ARG_TYPES(__VA_ARGS__) \
PW_COMMA_ARGS(__VA_ARGS__)); \
@@ -71,7 +71,7 @@
// This function encodes the tokenized strings. Do not call it directly;
// instead, use the PW_TOKENIZE_TO_GLOBAL_HANDLER macro.
void _pw_tokenizer_ToGlobalHandler(pw_tokenizer_Token token,
- _pw_tokenizer_ArgTypes types,
+ pw_tokenizer_ArgTypes types,
...);
PW_EXTERN_C_END
diff --git a/pw_tokenizer/public/pw_tokenizer/tokenize_to_global_handler_with_payload.h b/pw_tokenizer/public/pw_tokenizer/tokenize_to_global_handler_with_payload.h
index 2577e48..1faace3 100644
--- a/pw_tokenizer/public/pw_tokenizer/tokenize_to_global_handler_with_payload.h
+++ b/pw_tokenizer/public/pw_tokenizer/tokenize_to_global_handler_with_payload.h
@@ -56,7 +56,7 @@
#define PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD_MASK( \
domain, mask, payload, format, ...) \
do { \
- _PW_TOKENIZE_FORMAT_STRING(domain, mask, format, __VA_ARGS__); \
+ PW_TOKENIZE_FORMAT_STRING(domain, mask, format, __VA_ARGS__); \
_pw_tokenizer_ToGlobalHandlerWithPayload( \
payload, \
_pw_tokenizer_token, \
@@ -79,7 +79,7 @@
// instead, use the PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD macro.
void _pw_tokenizer_ToGlobalHandlerWithPayload(pw_tokenizer_Payload payload,
pw_tokenizer_Token token,
- _pw_tokenizer_ArgTypes types,
+ pw_tokenizer_ArgTypes types,
...);
PW_EXTERN_C_END
diff --git a/pw_tokenizer/pw_tokenizer_private/argument_types_test.h b/pw_tokenizer/pw_tokenizer_private/argument_types_test.h
index b616392..71f3ef2 100644
--- a/pw_tokenizer/pw_tokenizer_private/argument_types_test.h
+++ b/pw_tokenizer/pw_tokenizer_private/argument_types_test.h
@@ -20,27 +20,27 @@
PW_EXTERN_C_START
-_pw_tokenizer_ArgTypes pw_TestTokenizerNoArgs(void);
+pw_tokenizer_ArgTypes pw_TestTokenizerNoArgs(void);
-_pw_tokenizer_ArgTypes pw_TestTokenizerChar(void);
-_pw_tokenizer_ArgTypes pw_TestTokenizerUint8(void);
-_pw_tokenizer_ArgTypes pw_TestTokenizerUint16(void);
-_pw_tokenizer_ArgTypes pw_TestTokenizerInt32(void);
-_pw_tokenizer_ArgTypes pw_TestTokenizerInt64(void);
-_pw_tokenizer_ArgTypes pw_TestTokenizerUint64(void);
-_pw_tokenizer_ArgTypes pw_TestTokenizerFloat(void);
-_pw_tokenizer_ArgTypes pw_TestTokenizerDouble(void);
-_pw_tokenizer_ArgTypes pw_TestTokenizerString(void);
-_pw_tokenizer_ArgTypes pw_TestTokenizerMutableString(void);
+pw_tokenizer_ArgTypes pw_TestTokenizerChar(void);
+pw_tokenizer_ArgTypes pw_TestTokenizerUint8(void);
+pw_tokenizer_ArgTypes pw_TestTokenizerUint16(void);
+pw_tokenizer_ArgTypes pw_TestTokenizerInt32(void);
+pw_tokenizer_ArgTypes pw_TestTokenizerInt64(void);
+pw_tokenizer_ArgTypes pw_TestTokenizerUint64(void);
+pw_tokenizer_ArgTypes pw_TestTokenizerFloat(void);
+pw_tokenizer_ArgTypes pw_TestTokenizerDouble(void);
+pw_tokenizer_ArgTypes pw_TestTokenizerString(void);
+pw_tokenizer_ArgTypes pw_TestTokenizerMutableString(void);
-_pw_tokenizer_ArgTypes pw_TestTokenizerIntFloat(void);
-_pw_tokenizer_ArgTypes pw_TestTokenizerUint64Char(void);
-_pw_tokenizer_ArgTypes pw_TestTokenizerStringString(void);
-_pw_tokenizer_ArgTypes pw_TestTokenizerUint16Int(void);
-_pw_tokenizer_ArgTypes pw_TestTokenizerFloatString(void);
+pw_tokenizer_ArgTypes pw_TestTokenizerIntFloat(void);
+pw_tokenizer_ArgTypes pw_TestTokenizerUint64Char(void);
+pw_tokenizer_ArgTypes pw_TestTokenizerStringString(void);
+pw_tokenizer_ArgTypes pw_TestTokenizerUint16Int(void);
+pw_tokenizer_ArgTypes pw_TestTokenizerFloatString(void);
-_pw_tokenizer_ArgTypes pw_TestTokenizerNull(void);
-_pw_tokenizer_ArgTypes pw_TestTokenizerPointer(void);
-_pw_tokenizer_ArgTypes pw_TestTokenizerPointerPointer(void);
+pw_tokenizer_ArgTypes pw_TestTokenizerNull(void);
+pw_tokenizer_ArgTypes pw_TestTokenizerPointer(void);
+pw_tokenizer_ArgTypes pw_TestTokenizerPointerPointer(void);
PW_EXTERN_C_END
diff --git a/pw_tokenizer/tokenize.cc b/pw_tokenizer/tokenize.cc
index f34a81d..d515f0c 100644
--- a/pw_tokenizer/tokenize.cc
+++ b/pw_tokenizer/tokenize.cc
@@ -54,7 +54,7 @@
#endif // __APPLE__
constexpr Metadata metadata[] PW_TOKENIZER_INFO_SECTION = {
- {"hash_length_bytes", PW_TOKENIZER_CFG_C_HASH_LENGTH},
+ {"c_hash_length_bytes", PW_TOKENIZER_CFG_C_HASH_LENGTH},
{"sizeof_long", sizeof(long)}, // %l conversion specifier
{"sizeof_intmax_t", sizeof(intmax_t)}, // %j conversion specifier
{"sizeof_size_t", sizeof(size_t)}, // %z conversion specifier
@@ -66,7 +66,7 @@
extern "C" void _pw_tokenizer_ToBuffer(void* buffer,
size_t* buffer_size_bytes,
Token token,
- _pw_tokenizer_ArgTypes types,
+ pw_tokenizer_ArgTypes types,
...) {
if (*buffer_size_bytes < sizeof(token)) {
*buffer_size_bytes = 0;
@@ -80,8 +80,8 @@
const size_t encoded_bytes = EncodeArgs(
types,
args,
- std::span<uint8_t>(static_cast<uint8_t*>(buffer) + sizeof(token),
- *buffer_size_bytes - sizeof(token)));
+ std::span<std::byte>(static_cast<std::byte*>(buffer) + sizeof(token),
+ *buffer_size_bytes - sizeof(token)));
va_end(args);
*buffer_size_bytes = sizeof(token) + encoded_bytes;
@@ -90,18 +90,14 @@
extern "C" void _pw_tokenizer_ToCallback(
void (*callback)(const uint8_t* encoded_message, size_t size_bytes),
Token token,
- _pw_tokenizer_ArgTypes types,
+ pw_tokenizer_ArgTypes types,
...) {
- EncodedMessage encoded;
- encoded.token = token;
-
va_list args;
va_start(args, types);
- const size_t encoded_bytes = EncodeArgs(types, args, encoded.args);
+ EncodedMessage encoded(token, types, args);
va_end(args);
- callback(reinterpret_cast<const uint8_t*>(&encoded),
- sizeof(encoded.token) + encoded_bytes);
+ callback(encoded.data_as_uint8(), encoded.size());
}
} // namespace tokenizer
diff --git a/pw_tokenizer/tokenize_test.cc b/pw_tokenizer/tokenize_test.cc
index 393d8f7..ff87f51 100644
--- a/pw_tokenizer/tokenize_test.cc
+++ b/pw_tokenizer/tokenize_test.cc
@@ -401,6 +401,14 @@
EXPECT_TRUE(std::all_of(buffer_, std::end(buffer_), is_untouched));
}
+TEST_F(TokenizeToBuffer, CharArray) {
+ size_t message_size = sizeof(buffer_);
+ PW_TOKENIZE_TO_BUFFER(buffer_, &message_size, __func__);
+ constexpr auto expected = ExpectedData(__func__);
+ ASSERT_EQ(expected.size(), message_size);
+ EXPECT_EQ(std::memcmp(expected.data(), buffer_, expected.size()), 0);
+}
+
TEST_F(TokenizeToBuffer, C_StringShortFloat) {
size_t size = sizeof(buffer_);
pw_tokenizer_ToBufferTest_StringShortFloat(buffer_, &size);
@@ -517,6 +525,13 @@
EXPECT_EQ(std::memcmp(expected.data(), message_, expected.size()), 0);
}
+TEST_F(TokenizeToCallback, CharArray) {
+ PW_TOKENIZE_TO_CALLBACK(SetMessage, __func__);
+ constexpr auto expected = ExpectedData(__func__);
+ ASSERT_EQ(expected.size(), message_size_bytes_);
+ EXPECT_EQ(std::memcmp(expected.data(), message_, expected.size()), 0);
+}
+
TEST_F(TokenizeToCallback, C_SequentialZigZag) {
pw_tokenizer_ToCallbackTest_SequentialZigZag(SetMessage);
diff --git a/pw_tokenizer/tokenize_to_global_handler.cc b/pw_tokenizer/tokenize_to_global_handler.cc
index 78b79a0..5ac2755 100644
--- a/pw_tokenizer/tokenize_to_global_handler.cc
+++ b/pw_tokenizer/tokenize_to_global_handler.cc
@@ -20,18 +20,14 @@
namespace tokenizer {
extern "C" void _pw_tokenizer_ToGlobalHandler(pw_tokenizer_Token token,
- _pw_tokenizer_ArgTypes types,
+ pw_tokenizer_ArgTypes types,
...) {
- EncodedMessage encoded;
- encoded.token = token;
-
va_list args;
va_start(args, types);
- const size_t encoded_bytes = EncodeArgs(types, args, encoded.args);
+ EncodedMessage encoded(token, types, args);
va_end(args);
- pw_tokenizer_HandleEncodedMessage(reinterpret_cast<const uint8_t*>(&encoded),
- sizeof(encoded.token) + encoded_bytes);
+ pw_tokenizer_HandleEncodedMessage(encoded.data_as_uint8(), encoded.size());
}
} // namespace tokenizer
diff --git a/pw_tokenizer/tokenize_to_global_handler_with_payload.cc b/pw_tokenizer/tokenize_to_global_handler_with_payload.cc
index 0380aef..44b02c6 100644
--- a/pw_tokenizer/tokenize_to_global_handler_with_payload.cc
+++ b/pw_tokenizer/tokenize_to_global_handler_with_payload.cc
@@ -22,20 +22,15 @@
extern "C" void _pw_tokenizer_ToGlobalHandlerWithPayload(
const pw_tokenizer_Payload payload,
pw_tokenizer_Token token,
- _pw_tokenizer_ArgTypes types,
+ pw_tokenizer_ArgTypes types,
...) {
- EncodedMessage encoded;
- encoded.token = token;
-
va_list args;
va_start(args, types);
- const size_t encoded_bytes = EncodeArgs(types, args, encoded.args);
+ EncodedMessage encoded(token, types, args);
va_end(args);
pw_tokenizer_HandleEncodedMessageWithPayload(
- payload,
- reinterpret_cast<const uint8_t*>(&encoded),
- sizeof(encoded.token) + encoded_bytes);
+ payload, encoded.data_as_uint8(), encoded.size());
}
} // namespace tokenizer
