blob: f3a61df46290547649d2cb647a7c4e36b90c7a94 [file] [log] [blame]
// Copyright 2022 The Pigweed Authors
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.
#include <cstring>
#include <fstream>
#include <pw_assert/assert.h>
#include <pw_tokenizer/detokenize.h>
#include <pw_tokenizer/tokenize.h>
#include <string_view>
#include <vector>
#include <zephyr/kernel.h>
static std::vector<uint8_t> ReadWholeFile(const char *path) {
// Open the file
std::ifstream file(path, std::ios::binary);
// Stop eating new lines in binary mode
// Get the file size
std::streampos file_size;
file.seekg(0, std::ios::end);
file_size = file.tellg();
file.seekg(0, std::ios::beg);
// Reserve capacity
std::vector<uint8_t> data;
// Read the data
data.insert(data.begin(), std::istream_iterator<uint8_t>(file),
return data;
pw::tokenizer::Detokenizer OpenDatabase(const char *path) {
std::vector<uint8_t> data = ReadWholeFile(path);
pw::tokenizer::TokenDatabase database =
// This checks if the file contained a valid database.
return pw::tokenizer::Detokenizer(database);
constexpr uint32_t kHelloWorldToken = PW_TOKENIZE_STRING("Hello tokenized world!");
void main(void) {
char expected_string[1024];
uint8_t buffer[1024];
size_t size_bytes = sizeof(buffer);
pw::tokenizer::Detokenizer detokenizer = OpenDatabase("database.bin");
sprintf(expected_string, "token=%u\n", kHelloWorldToken);
PW_TOKENIZE_TO_BUFFER(buffer, &size_bytes, "token=%u\n", kHelloWorldToken);
printk("tokenized buffer size is %u bytes\n", size_bytes);
auto detokenized_string = detokenizer.Detokenize(buffer, size_bytes);
PW_ASSERT(strcmp(detokenized_string.BestString().c_str(), expected_string) ==
printk("detokenized message size is %u bytes\n",