pw_tokenizer: Support 256-byte hashes in C
- Generate hash macro for 256 B and update code and tests to support it.
- Rename 'mark_removals' to 'mark_removed'.
Change-Id: I451dc3123b6a9b829fcc19bb345967df5cf2cdb1
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/33862
Reviewed-by: Keir Mierle <keir@google.com>
Commit-Queue: Wyatt Hepler <hepler@google.com>
diff --git a/pw_tokenizer/BUILD b/pw_tokenizer/BUILD
index 350d463..f05d088 100644
--- a/pw_tokenizer/BUILD
+++ b/pw_tokenizer/BUILD
@@ -33,6 +33,7 @@
"public/pw_tokenizer/internal/argument_types_macro_4_byte.h",
"public/pw_tokenizer/internal/argument_types_macro_8_byte.h",
"public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_128_hash_macro.h",
+ "public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_256_hash_macro.h",
"public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_80_hash_macro.h",
"public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_96_hash_macro.h",
"public/pw_tokenizer/internal/tokenize_string.h",
diff --git a/pw_tokenizer/BUILD.gn b/pw_tokenizer/BUILD.gn
index e4d182b..1774974 100644
--- a/pw_tokenizer/BUILD.gn
+++ b/pw_tokenizer/BUILD.gn
@@ -86,6 +86,7 @@
"public/pw_tokenizer/internal/argument_types_macro_4_byte.h",
"public/pw_tokenizer/internal/argument_types_macro_8_byte.h",
"public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_128_hash_macro.h",
+ "public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_256_hash_macro.h",
"public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_80_hash_macro.h",
"public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_96_hash_macro.h",
"public/pw_tokenizer/internal/tokenize_string.h",
diff --git a/pw_tokenizer/hash_test.cc b/pw_tokenizer/hash_test.cc
index de6681e..3895b13 100644
--- a/pw_tokenizer/hash_test.cc
+++ b/pw_tokenizer/hash_test.cc
@@ -22,6 +22,7 @@
#include "gtest/gtest.h"
#include "pw_preprocessor/util.h"
#include "pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_128_hash_macro.h"
+#include "pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_256_hash_macro.h"
#include "pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_80_hash_macro.h"
#include "pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_96_hash_macro.h"
#include "pw_tokenizer_private/generated_hash_test_cases.h"
@@ -89,33 +90,28 @@
StringLength("123456") + k2 * '1' + k3 * '2' + k4 * '3' + k5 * '4');
}
+#define _CHECK_HASH_LENGTH(string, length) \
+ static_assert(PwTokenizer65599FixedLengthHash( \
+ std::string_view(string, sizeof(string) - 1), length) == \
+ PW_TOKENIZER_65599_FIXED_LENGTH_##length##_HASH(string), \
+ #length "-byte hash mismatch!")
+
// Use std::string_view so that \0 can appear in strings.
-#define TEST_SUPPORTED_HASHES(string_literal) \
- static_assert( \
- PwTokenizer65599FixedLengthHash( \
- std::string_view(string_literal, sizeof(string_literal) - 1), 80) == \
- PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH(string_literal), \
- "80-byte hash mismatch!"); \
- static_assert( \
- PwTokenizer65599FixedLengthHash( \
- std::string_view(string_literal, sizeof(string_literal) - 1), 96) == \
- PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH(string_literal), \
- "96-byte hash mismatch!"); \
- static_assert( \
- PwTokenizer65599FixedLengthHash( \
- std::string_view(string_literal, sizeof(string_literal) - 1), \
- 128) == PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH(string_literal), \
- "128-byte hash mismatch!"); \
- static_assert( \
- PwTokenizer65599FixedLengthHash( \
- std::string_view(string_literal, sizeof(string_literal) - 1), \
- sizeof(string_literal) - 1) == Hash(string_literal), \
- "Hash function mismatch!"); \
- EXPECT_EQ(PwTokenizer65599FixedLengthHash( \
- std::string_view(string_literal, sizeof(string_literal) - 1), \
- sizeof(string_literal) - 1), \
- pw_tokenizer_65599FixedLengthHash(string_literal, \
- sizeof(string_literal) - 1, \
+#define TEST_SUPPORTED_HASHES(string_literal) \
+ _CHECK_HASH_LENGTH(string_literal, 80); \
+ _CHECK_HASH_LENGTH(string_literal, 96); \
+ _CHECK_HASH_LENGTH(string_literal, 128); \
+ _CHECK_HASH_LENGTH(string_literal, 256); \
+ static_assert( \
+ PwTokenizer65599FixedLengthHash( \
+ std::string_view(string_literal, sizeof(string_literal) - 1), \
+ sizeof(string_literal) - 1) == Hash(string_literal), \
+ "Hash function mismatch!"); \
+ EXPECT_EQ(PwTokenizer65599FixedLengthHash( \
+ std::string_view(string_literal, sizeof(string_literal) - 1), \
+ sizeof(string_literal) - 1), \
+ pw_tokenizer_65599FixedLengthHash(string_literal, \
+ sizeof(string_literal) - 1, \
sizeof(string_literal) - 1))
TEST(HashMacro, Empty) { TEST_SUPPORTED_HASHES(""); }
diff --git a/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_128_hash_macro.h b/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_128_hash_macro.h
index 1c4eb82..27411d1 100644
--- a/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_128_hash_macro.h
+++ b/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_128_hash_macro.h
@@ -1,4 +1,4 @@
-// Copyright 2020 The Pigweed Authors
+// Copyright 2021 The Pigweed Authors
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
diff --git a/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_256_hash_macro.h b/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_256_hash_macro.h
new file mode 100644
index 0000000..d713bae
--- /dev/null
+++ b/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_256_hash_macro.h
@@ -0,0 +1,289 @@
+// Copyright 2021 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+// AUTOGENERATED - DO NOT EDIT
+//
+// This file was generated by generate_hash_macro.py.
+// To make changes, update the script and run it to regenerate the files.
+#pragma once
+
+#include <stdint.h>
+
+// 256-character version of the tokenizer hash function.
+//
+// The argument must be a string literal. It is concatenated with "" to ensure
+// that this is the case.
+//
+// clang-format off
+
+#define PW_TOKENIZER_65599_FIXED_LENGTH_256_HASH(str) \
+ (uint32_t)(sizeof(str "") - 1 + /* The argument must be a string literal. */ \
+ 0x0001003fu * (uint8_t)str[0] + \
+ 0x007e0f81u * (uint8_t)( 1 < sizeof(str) ? str[ 1] : 0) + \
+ 0x2e86d0bfu * (uint8_t)( 2 < sizeof(str) ? str[ 2] : 0) + \
+ 0x43ec5f01u * (uint8_t)( 3 < sizeof(str) ? str[ 3] : 0) + \
+ 0x162c613fu * (uint8_t)( 4 < sizeof(str) ? str[ 4] : 0) + \
+ 0xd62aee81u * (uint8_t)( 5 < sizeof(str) ? str[ 5] : 0) + \
+ 0xa311b1bfu * (uint8_t)( 6 < sizeof(str) ? str[ 6] : 0) + \
+ 0xd319be01u * (uint8_t)( 7 < sizeof(str) ? str[ 7] : 0) + \
+ 0xb156c23fu * (uint8_t)( 8 < sizeof(str) ? str[ 8] : 0) + \
+ 0x6698cd81u * (uint8_t)( 9 < sizeof(str) ? str[ 9] : 0) + \
+ 0x0d1b92bfu * (uint8_t)( 10 < sizeof(str) ? str[ 10] : 0) + \
+ 0xcc881d01u * (uint8_t)( 11 < sizeof(str) ? str[ 11] : 0) + \
+ 0x7280233fu * (uint8_t)( 12 < sizeof(str) ? str[ 12] : 0) + \
+ 0x50c7ac81u * (uint8_t)( 13 < sizeof(str) ? str[ 13] : 0) + \
+ 0x8da473bfu * (uint8_t)( 14 < sizeof(str) ? str[ 14] : 0) + \
+ 0x4f377c01u * (uint8_t)( 15 < sizeof(str) ? str[ 15] : 0) + \
+ 0xfaa8843fu * (uint8_t)( 16 < sizeof(str) ? str[ 16] : 0) + \
+ 0x33b78b81u * (uint8_t)( 17 < sizeof(str) ? str[ 17] : 0) + \
+ 0x45ac54bfu * (uint8_t)( 18 < sizeof(str) ? str[ 18] : 0) + \
+ 0x7a27db01u * (uint8_t)( 19 < sizeof(str) ? str[ 19] : 0) + \
+ 0xeacfe53fu * (uint8_t)( 20 < sizeof(str) ? str[ 20] : 0) + \
+ 0xae686a81u * (uint8_t)( 21 < sizeof(str) ? str[ 21] : 0) + \
+ 0x563335bfu * (uint8_t)( 22 < sizeof(str) ? str[ 22] : 0) + \
+ 0x6c593a01u * (uint8_t)( 23 < sizeof(str) ? str[ 23] : 0) + \
+ 0xe3f6463fu * (uint8_t)( 24 < sizeof(str) ? str[ 24] : 0) + \
+ 0x5fda4981u * (uint8_t)( 25 < sizeof(str) ? str[ 25] : 0) + \
+ 0xe03916bfu * (uint8_t)( 26 < sizeof(str) ? str[ 26] : 0) + \
+ 0x44cb9901u * (uint8_t)( 27 < sizeof(str) ? str[ 27] : 0) + \
+ 0x871ba73fu * (uint8_t)( 28 < sizeof(str) ? str[ 28] : 0) + \
+ 0xe70d2881u * (uint8_t)( 29 < sizeof(str) ? str[ 29] : 0) + \
+ 0x04bdf7bfu * (uint8_t)( 30 < sizeof(str) ? str[ 30] : 0) + \
+ 0x227ef801u * (uint8_t)( 31 < sizeof(str) ? str[ 31] : 0) + \
+ 0x7540083fu * (uint8_t)( 32 < sizeof(str) ? str[ 32] : 0) + \
+ 0xe3010781u * (uint8_t)( 33 < sizeof(str) ? str[ 33] : 0) + \
+ 0xe4c1d8bfu * (uint8_t)( 34 < sizeof(str) ? str[ 34] : 0) + \
+ 0x24735701u * (uint8_t)( 35 < sizeof(str) ? str[ 35] : 0) + \
+ 0x4f63693fu * (uint8_t)( 36 < sizeof(str) ? str[ 36] : 0) + \
+ 0xf2b5e681u * (uint8_t)( 37 < sizeof(str) ? str[ 37] : 0) + \
+ 0xa144b9bfu * (uint8_t)( 38 < sizeof(str) ? str[ 38] : 0) + \
+ 0x69a8b601u * (uint8_t)( 39 < sizeof(str) ? str[ 39] : 0) + \
+ 0xb685ca3fu * (uint8_t)( 40 < sizeof(str) ? str[ 40] : 0) + \
+ 0xb52bc581u * (uint8_t)( 41 < sizeof(str) ? str[ 41] : 0) + \
+ 0x5b469abfu * (uint8_t)( 42 < sizeof(str) ? str[ 42] : 0) + \
+ 0x111f1501u * (uint8_t)( 43 < sizeof(str) ? str[ 43] : 0) + \
+ 0x4ba72b3fu * (uint8_t)( 44 < sizeof(str) ? str[ 44] : 0) + \
+ 0xc962a481u * (uint8_t)( 45 < sizeof(str) ? str[ 45] : 0) + \
+ 0x33c77bbfu * (uint8_t)( 46 < sizeof(str) ? str[ 46] : 0) + \
+ 0x39d67401u * (uint8_t)( 47 < sizeof(str) ? str[ 47] : 0) + \
+ 0xafc78c3fu * (uint8_t)( 48 < sizeof(str) ? str[ 48] : 0) + \
+ 0xce5a8381u * (uint8_t)( 49 < sizeof(str) ? str[ 49] : 0) + \
+ 0x4bc75cbfu * (uint8_t)( 50 < sizeof(str) ? str[ 50] : 0) + \
+ 0x02ced301u * (uint8_t)( 51 < sizeof(str) ? str[ 51] : 0) + \
+ 0x83e6ed3fu * (uint8_t)( 52 < sizeof(str) ? str[ 52] : 0) + \
+ 0x63136281u * (uint8_t)( 53 < sizeof(str) ? str[ 53] : 0) + \
+ 0xc4463dbfu * (uint8_t)( 54 < sizeof(str) ? str[ 54] : 0) + \
+ 0x8b083201u * (uint8_t)( 55 < sizeof(str) ? str[ 55] : 0) + \
+ 0x69054e3fu * (uint8_t)( 56 < sizeof(str) ? str[ 56] : 0) + \
+ 0x268d4181u * (uint8_t)( 57 < sizeof(str) ? str[ 57] : 0) + \
+ 0xbe441ebfu * (uint8_t)( 58 < sizeof(str) ? str[ 58] : 0) + \
+ 0xf1829101u * (uint8_t)( 59 < sizeof(str) ? str[ 59] : 0) + \
+ 0x0022af3fu * (uint8_t)( 60 < sizeof(str) ? str[ 60] : 0) + \
+ 0xb7c82081u * (uint8_t)( 61 < sizeof(str) ? str[ 61] : 0) + \
+ 0x5ac0ffbfu * (uint8_t)( 62 < sizeof(str) ? str[ 62] : 0) + \
+ 0x553df001u * (uint8_t)( 63 < sizeof(str) ? str[ 63] : 0) + \
+ 0xea3f103fu * (uint8_t)( 64 < sizeof(str) ? str[ 64] : 0) + \
+ 0xb5c3ff81u * (uint8_t)( 65 < sizeof(str) ? str[ 65] : 0) + \
+ 0xbabce0bfu * (uint8_t)( 66 < sizeof(str) ? str[ 66] : 0) + \
+ 0xd53a4f01u * (uint8_t)( 67 < sizeof(str) ? str[ 67] : 0) + \
+ 0xc85a713fu * (uint8_t)( 68 < sizeof(str) ? str[ 68] : 0) + \
+ 0xbf80de81u * (uint8_t)( 69 < sizeof(str) ? str[ 69] : 0) + \
+ 0xff37c1bfu * (uint8_t)( 70 < sizeof(str) ? str[ 70] : 0) + \
+ 0x9077ae01u * (uint8_t)( 71 < sizeof(str) ? str[ 71] : 0) + \
+ 0x3b74d23fu * (uint8_t)( 72 < sizeof(str) ? str[ 72] : 0) + \
+ 0x73febd81u * (uint8_t)( 73 < sizeof(str) ? str[ 73] : 0) + \
+ 0x4931a2bfu * (uint8_t)( 74 < sizeof(str) ? str[ 74] : 0) + \
+ 0xa5f60d01u * (uint8_t)( 75 < sizeof(str) ? str[ 75] : 0) + \
+ 0xe48e333fu * (uint8_t)( 76 < sizeof(str) ? str[ 76] : 0) + \
+ 0x723d9c81u * (uint8_t)( 77 < sizeof(str) ? str[ 77] : 0) + \
+ 0xb9aa83bfu * (uint8_t)( 78 < sizeof(str) ? str[ 78] : 0) + \
+ 0x34b56c01u * (uint8_t)( 79 < sizeof(str) ? str[ 79] : 0) + \
+ 0x64a6943fu * (uint8_t)( 80 < sizeof(str) ? str[ 80] : 0) + \
+ 0x593d7b81u * (uint8_t)( 81 < sizeof(str) ? str[ 81] : 0) + \
+ 0x71a264bfu * (uint8_t)( 82 < sizeof(str) ? str[ 82] : 0) + \
+ 0x5bb5cb01u * (uint8_t)( 83 < sizeof(str) ? str[ 83] : 0) + \
+ 0x5cbdf53fu * (uint8_t)( 84 < sizeof(str) ? str[ 84] : 0) + \
+ 0xc7fe5a81u * (uint8_t)( 85 < sizeof(str) ? str[ 85] : 0) + \
+ 0x921945bfu * (uint8_t)( 86 < sizeof(str) ? str[ 86] : 0) + \
+ 0x39f72a01u * (uint8_t)( 87 < sizeof(str) ? str[ 87] : 0) + \
+ 0x6dd4563fu * (uint8_t)( 88 < sizeof(str) ? str[ 88] : 0) + \
+ 0x5d803981u * (uint8_t)( 89 < sizeof(str) ? str[ 89] : 0) + \
+ 0x3c0f26bfu * (uint8_t)( 90 < sizeof(str) ? str[ 90] : 0) + \
+ 0xee798901u * (uint8_t)( 91 < sizeof(str) ? str[ 91] : 0) + \
+ 0x38e9b73fu * (uint8_t)( 92 < sizeof(str) ? str[ 92] : 0) + \
+ 0xb8c31881u * (uint8_t)( 93 < sizeof(str) ? str[ 93] : 0) + \
+ 0x908407bfu * (uint8_t)( 94 < sizeof(str) ? str[ 94] : 0) + \
+ 0x983ce801u * (uint8_t)( 95 < sizeof(str) ? str[ 95] : 0) + \
+ 0x5efe183fu * (uint8_t)( 96 < sizeof(str) ? str[ 96] : 0) + \
+ 0x78c6f781u * (uint8_t)( 97 < sizeof(str) ? str[ 97] : 0) + \
+ 0xb077e8bfu * (uint8_t)( 98 < sizeof(str) ? str[ 98] : 0) + \
+ 0x56414701u * (uint8_t)( 99 < sizeof(str) ? str[ 99] : 0) + \
+ 0x8111793fu * (uint8_t)(100 < sizeof(str) ? str[100] : 0) + \
+ 0x3c8bd681u * (uint8_t)(101 < sizeof(str) ? str[101] : 0) + \
+ 0xbceac9bfu * (uint8_t)(102 < sizeof(str) ? str[102] : 0) + \
+ 0x4786a601u * (uint8_t)(103 < sizeof(str) ? str[103] : 0) + \
+ 0x4023da3fu * (uint8_t)(104 < sizeof(str) ? str[104] : 0) + \
+ 0xa311b581u * (uint8_t)(105 < sizeof(str) ? str[105] : 0) + \
+ 0xd6dcaabfu * (uint8_t)(106 < sizeof(str) ? str[106] : 0) + \
+ 0x8b0d0501u * (uint8_t)(107 < sizeof(str) ? str[107] : 0) + \
+ 0x3d353b3fu * (uint8_t)(108 < sizeof(str) ? str[108] : 0) + \
+ 0x4b589481u * (uint8_t)(109 < sizeof(str) ? str[109] : 0) + \
+ 0x1f4d8bbfu * (uint8_t)(110 < sizeof(str) ? str[110] : 0) + \
+ 0x3fd46401u * (uint8_t)(111 < sizeof(str) ? str[111] : 0) + \
+ 0x19459c3fu * (uint8_t)(112 < sizeof(str) ? str[112] : 0) + \
+ 0xd4607381u * (uint8_t)(113 < sizeof(str) ? str[113] : 0) + \
+ 0xb73d6cbfu * (uint8_t)(114 < sizeof(str) ? str[114] : 0) + \
+ 0x84dcc301u * (uint8_t)(115 < sizeof(str) ? str[115] : 0) + \
+ 0x7554fd3fu * (uint8_t)(116 < sizeof(str) ? str[116] : 0) + \
+ 0xdd295281u * (uint8_t)(117 < sizeof(str) ? str[117] : 0) + \
+ 0xbfac4dbfu * (uint8_t)(118 < sizeof(str) ? str[118] : 0) + \
+ 0x79262201u * (uint8_t)(119 < sizeof(str) ? str[119] : 0) + \
+ 0xf2635e3fu * (uint8_t)(120 < sizeof(str) ? str[120] : 0) + \
+ 0x04b33181u * (uint8_t)(121 < sizeof(str) ? str[121] : 0) + \
+ 0x599a2ebfu * (uint8_t)(122 < sizeof(str) ? str[122] : 0) + \
+ 0x3bb08101u * (uint8_t)(123 < sizeof(str) ? str[123] : 0) + \
+ 0x3170bf3fu * (uint8_t)(124 < sizeof(str) ? str[124] : 0) + \
+ 0xe9fe1081u * (uint8_t)(125 < sizeof(str) ? str[125] : 0) + \
+ 0xa6070fbfu * (uint8_t)(126 < sizeof(str) ? str[126] : 0) + \
+ 0xeb7be001u * (uint8_t)(127 < sizeof(str) ? str[127] : 0) + \
+ 0xd37d203fu * (uint8_t)(128 < sizeof(str) ? str[128] : 0) + \
+ 0x2c09ef81u * (uint8_t)(129 < sizeof(str) ? str[129] : 0) + \
+ 0xc5f2f0bfu * (uint8_t)(130 < sizeof(str) ? str[130] : 0) + \
+ 0xa7883f01u * (uint8_t)(131 < sizeof(str) ? str[131] : 0) + \
+ 0x7988813fu * (uint8_t)(132 < sizeof(str) ? str[132] : 0) + \
+ 0x69d6ce81u * (uint8_t)(133 < sizeof(str) ? str[133] : 0) + \
+ 0xda5dd1bfu * (uint8_t)(134 < sizeof(str) ? str[134] : 0) + \
+ 0x8ed59e01u * (uint8_t)(135 < sizeof(str) ? str[135] : 0) + \
+ 0xc492e23fu * (uint8_t)(136 < sizeof(str) ? str[136] : 0) + \
+ 0x4264ad81u * (uint8_t)(137 < sizeof(str) ? str[137] : 0) + \
+ 0x0447b2bfu * (uint8_t)(138 < sizeof(str) ? str[138] : 0) + \
+ 0xc063fd01u * (uint8_t)(139 < sizeof(str) ? str[139] : 0) + \
+ 0x559c433fu * (uint8_t)(140 < sizeof(str) ? str[140] : 0) + \
+ 0x54b38c81u * (uint8_t)(141 < sizeof(str) ? str[141] : 0) + \
+ 0x64b093bfu * (uint8_t)(142 < sizeof(str) ? str[142] : 0) + \
+ 0x5b335c01u * (uint8_t)(143 < sizeof(str) ? str[143] : 0) + \
+ 0xcda4a43fu * (uint8_t)(144 < sizeof(str) ? str[144] : 0) + \
+ 0x3fc36b81u * (uint8_t)(145 < sizeof(str) ? str[145] : 0) + \
+ 0x1c9874bfu * (uint8_t)(146 < sizeof(str) ? str[146] : 0) + \
+ 0x7e43bb01u * (uint8_t)(147 < sizeof(str) ? str[147] : 0) + \
+ 0xcdac053fu * (uint8_t)(148 < sizeof(str) ? str[148] : 0) + \
+ 0xa2944a81u * (uint8_t)(149 < sizeof(str) ? str[149] : 0) + \
+ 0x4cff55bfu * (uint8_t)(150 < sizeof(str) ? str[150] : 0) + \
+ 0x48951a01u * (uint8_t)(151 < sizeof(str) ? str[151] : 0) + \
+ 0xf6b2663fu * (uint8_t)(152 < sizeof(str) ? str[152] : 0) + \
+ 0x1c262981u * (uint8_t)(153 < sizeof(str) ? str[153] : 0) + \
+ 0x16e536bfu * (uint8_t)(154 < sizeof(str) ? str[154] : 0) + \
+ 0xd9277901u * (uint8_t)(155 < sizeof(str) ? str[155] : 0) + \
+ 0xe9b7c73fu * (uint8_t)(156 < sizeof(str) ? str[156] : 0) + \
+ 0x4b790881u * (uint8_t)(157 < sizeof(str) ? str[157] : 0) + \
+ 0x9b4a17bfu * (uint8_t)(158 < sizeof(str) ? str[158] : 0) + \
+ 0x4efad801u * (uint8_t)(159 < sizeof(str) ? str[159] : 0) + \
+ 0x47bc283fu * (uint8_t)(160 < sizeof(str) ? str[160] : 0) + \
+ 0xcf8ce781u * (uint8_t)(161 < sizeof(str) ? str[161] : 0) + \
+ 0xfb2df8bfu * (uint8_t)(162 < sizeof(str) ? str[162] : 0) + \
+ 0xc90f3701u * (uint8_t)(163 < sizeof(str) ? str[163] : 0) + \
+ 0xb1bf893fu * (uint8_t)(164 < sizeof(str) ? str[164] : 0) + \
+ 0x4761c681u * (uint8_t)(165 < sizeof(str) ? str[165] : 0) + \
+ 0x5790d9bfu * (uint8_t)(166 < sizeof(str) ? str[166] : 0) + \
+ 0x66649601u * (uint8_t)(167 < sizeof(str) ? str[167] : 0) + \
+ 0xc8c1ea3fu * (uint8_t)(168 < sizeof(str) ? str[168] : 0) + \
+ 0x51f7a581u * (uint8_t)(169 < sizeof(str) ? str[169] : 0) + \
+ 0xd172babfu * (uint8_t)(170 < sizeof(str) ? str[170] : 0) + \
+ 0x45faf501u * (uint8_t)(171 < sizeof(str) ? str[171] : 0) + \
+ 0x2dc34b3fu * (uint8_t)(172 < sizeof(str) ? str[172] : 0) + \
+ 0x8e4e8481u * (uint8_t)(173 < sizeof(str) ? str[173] : 0) + \
+ 0x89d39bbfu * (uint8_t)(174 < sizeof(str) ? str[174] : 0) + \
+ 0x86d25401u * (uint8_t)(175 < sizeof(str) ? str[175] : 0) + \
+ 0x81c3ac3fu * (uint8_t)(176 < sizeof(str) ? str[176] : 0) + \
+ 0x9b666381u * (uint8_t)(177 < sizeof(str) ? str[177] : 0) + \
+ 0xa1b37cbfu * (uint8_t)(178 < sizeof(str) ? str[178] : 0) + \
+ 0x47eab301u * (uint8_t)(179 < sizeof(str) ? str[179] : 0) + \
+ 0x65c30d3fu * (uint8_t)(180 < sizeof(str) ? str[180] : 0) + \
+ 0x183f4281u * (uint8_t)(181 < sizeof(str) ? str[181] : 0) + \
+ 0x3a125dbfu * (uint8_t)(182 < sizeof(str) ? str[182] : 0) + \
+ 0xa8441201u * (uint8_t)(183 < sizeof(str) ? str[183] : 0) + \
+ 0x7ac16e3fu * (uint8_t)(184 < sizeof(str) ? str[184] : 0) + \
+ 0xa3d92181u * (uint8_t)(185 < sizeof(str) ? str[185] : 0) + \
+ 0x73f03ebfu * (uint8_t)(186 < sizeof(str) ? str[186] : 0) + \
+ 0xc6de7101u * (uint8_t)(187 < sizeof(str) ? str[187] : 0) + \
+ 0x61becf3fu * (uint8_t)(188 < sizeof(str) ? str[188] : 0) + \
+ 0xdd340081u * (uint8_t)(189 < sizeof(str) ? str[189] : 0) + \
+ 0x704d1fbfu * (uint8_t)(190 < sizeof(str) ? str[190] : 0) + \
+ 0xc2b9d001u * (uint8_t)(191 < sizeof(str) ? str[191] : 0) + \
+ 0xbbbb303fu * (uint8_t)(192 < sizeof(str) ? str[192] : 0) + \
+ 0x634fdf81u * (uint8_t)(193 < sizeof(str) ? str[193] : 0) + \
+ 0x502900bfu * (uint8_t)(194 < sizeof(str) ? str[194] : 0) + \
+ 0xbad62f01u * (uint8_t)(195 < sizeof(str) ? str[195] : 0) + \
+ 0x29b6913fu * (uint8_t)(196 < sizeof(str) ? str[196] : 0) + \
+ 0xd52cbe81u * (uint8_t)(197 < sizeof(str) ? str[197] : 0) + \
+ 0x3483e1bfu * (uint8_t)(198 < sizeof(str) ? str[198] : 0) + \
+ 0xce338e01u * (uint8_t)(199 < sizeof(str) ? str[199] : 0) + \
+ 0x4cb0f23fu * (uint8_t)(200 < sizeof(str) ? str[200] : 0) + \
+ 0xd1ca9d81u * (uint8_t)(201 < sizeof(str) ? str[201] : 0) + \
+ 0x3e5dc2bfu * (uint8_t)(202 < sizeof(str) ? str[202] : 0) + \
+ 0x1bd1ed01u * (uint8_t)(203 < sizeof(str) ? str[203] : 0) + \
+ 0xc5aa533fu * (uint8_t)(204 < sizeof(str) ? str[204] : 0) + \
+ 0xf8297c81u * (uint8_t)(205 < sizeof(str) ? str[205] : 0) + \
+ 0x8eb6a3bfu * (uint8_t)(206 < sizeof(str) ? str[206] : 0) + \
+ 0xc2b14c01u * (uint8_t)(207 < sizeof(str) ? str[207] : 0) + \
+ 0x35a2b43fu * (uint8_t)(208 < sizeof(str) ? str[208] : 0) + \
+ 0xe7495b81u * (uint8_t)(209 < sizeof(str) ? str[209] : 0) + \
+ 0x468e84bfu * (uint8_t)(210 < sizeof(str) ? str[210] : 0) + \
+ 0xe1d1ab01u * (uint8_t)(211 < sizeof(str) ? str[211] : 0) + \
+ 0x3d9a153fu * (uint8_t)(212 < sizeof(str) ? str[212] : 0) + \
+ 0x3e2a3a81u * (uint8_t)(213 < sizeof(str) ? str[213] : 0) + \
+ 0x86e565bfu * (uint8_t)(214 < sizeof(str) ? str[214] : 0) + \
+ 0x98330a01u * (uint8_t)(215 < sizeof(str) ? str[215] : 0) + \
+ 0x7e90763fu * (uint8_t)(216 < sizeof(str) ? str[216] : 0) + \
+ 0x9bcc1981u * (uint8_t)(217 < sizeof(str) ? str[217] : 0) + \
+ 0x70bb46bfu * (uint8_t)(218 < sizeof(str) ? str[218] : 0) + \
+ 0x04d56901u * (uint8_t)(219 < sizeof(str) ? str[219] : 0) + \
+ 0x9985d73fu * (uint8_t)(220 < sizeof(str) ? str[220] : 0) + \
+ 0x9f2ef881u * (uint8_t)(221 < sizeof(str) ? str[221] : 0) + \
+ 0x251027bfu * (uint8_t)(222 < sizeof(str) ? str[222] : 0) + \
+ 0x46b8c801u * (uint8_t)(223 < sizeof(str) ? str[223] : 0) + \
+ 0x2f7a383fu * (uint8_t)(224 < sizeof(str) ? str[224] : 0) + \
+ 0xe752d781u * (uint8_t)(225 < sizeof(str) ? str[225] : 0) + \
+ 0xc4e408bfu * (uint8_t)(226 < sizeof(str) ? str[226] : 0) + \
+ 0x7cdd2701u * (uint8_t)(227 < sizeof(str) ? str[227] : 0) + \
+ 0xe16d993fu * (uint8_t)(228 < sizeof(str) ? str[228] : 0) + \
+ 0x1337b681u * (uint8_t)(229 < sizeof(str) ? str[229] : 0) + \
+ 0x7136e9bfu * (uint8_t)(230 < sizeof(str) ? str[230] : 0) + \
+ 0xc6428601u * (uint8_t)(231 < sizeof(str) ? str[231] : 0) + \
+ 0x505ffa3fu * (uint8_t)(232 < sizeof(str) ? str[232] : 0) + \
+ 0xc1dd9581u * (uint8_t)(233 < sizeof(str) ? str[233] : 0) + \
+ 0x4b08cabfu * (uint8_t)(234 < sizeof(str) ? str[234] : 0) + \
+ 0x41e8e501u * (uint8_t)(235 < sizeof(str) ? str[235] : 0) + \
+ 0x1d515b3fu * (uint8_t)(236 < sizeof(str) ? str[236] : 0) + \
+ 0x92447481u * (uint8_t)(237 < sizeof(str) ? str[237] : 0) + \
+ 0x7359abbfu * (uint8_t)(238 < sizeof(str) ? str[238] : 0) + \
+ 0x0ed04401u * (uint8_t)(239 < sizeof(str) ? str[239] : 0) + \
+ 0xe941bc3fu * (uint8_t)(240 < sizeof(str) ? str[240] : 0) + \
+ 0x236c5381u * (uint8_t)(241 < sizeof(str) ? str[241] : 0) + \
+ 0x0b298cbfu * (uint8_t)(242 < sizeof(str) ? str[242] : 0) + \
+ 0x4bf8a301u * (uint8_t)(243 < sizeof(str) ? str[243] : 0) + \
+ 0x55311d3fu * (uint8_t)(244 < sizeof(str) ? str[244] : 0) + \
+ 0x14553281u * (uint8_t)(245 < sizeof(str) ? str[245] : 0) + \
+ 0x33786dbfu * (uint8_t)(246 < sizeof(str) ? str[246] : 0) + \
+ 0x18620201u * (uint8_t)(247 < sizeof(str) ? str[247] : 0) + \
+ 0x021f7e3fu * (uint8_t)(248 < sizeof(str) ? str[248] : 0) + \
+ 0x03ff1181u * (uint8_t)(249 < sizeof(str) ? str[249] : 0) + \
+ 0x0d464ebfu * (uint8_t)(250 < sizeof(str) ? str[250] : 0) + \
+ 0x930c6101u * (uint8_t)(251 < sizeof(str) ? str[251] : 0) + \
+ 0x910cdf3fu * (uint8_t)(252 < sizeof(str) ? str[252] : 0) + \
+ 0x9169f081u * (uint8_t)(253 < sizeof(str) ? str[253] : 0) + \
+ 0xb9932fbfu * (uint8_t)(254 < sizeof(str) ? str[254] : 0) + \
+ 0xdaf7c001u * (uint8_t)(255 < sizeof(str) ? str[255] : 0))
+
+// clang-format on
diff --git a/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_80_hash_macro.h b/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_80_hash_macro.h
index 857c61f..8725ed1 100644
--- a/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_80_hash_macro.h
+++ b/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_80_hash_macro.h
@@ -1,4 +1,4 @@
-// Copyright 2020 The Pigweed Authors
+// Copyright 2021 The Pigweed Authors
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
diff --git a/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_96_hash_macro.h b/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_96_hash_macro.h
index 799044c..c63dfed 100644
--- a/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_96_hash_macro.h
+++ b/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_96_hash_macro.h
@@ -1,4 +1,4 @@
-// Copyright 2020 The Pigweed Authors
+// Copyright 2021 The Pigweed Authors
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
diff --git a/pw_tokenizer/public/pw_tokenizer/internal/tokenize_string.h b/pw_tokenizer/public/pw_tokenizer/internal/tokenize_string.h
index 06cefd7..783b84d 100644
--- a/pw_tokenizer/public/pw_tokenizer/internal/tokenize_string.h
+++ b/pw_tokenizer/public/pw_tokenizer/internal/tokenize_string.h
@@ -117,6 +117,11 @@
#include "pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_128_hash_macro.h"
#define PW_TOKENIZER_STRING_TOKEN PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH
+#elif PW_TOKENIZER_CFG_C_HASH_LENGTH == 256
+
+#include "pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_256_hash_macro.h"
+#define PW_TOKENIZER_STRING_TOKEN PW_TOKENIZER_65599_FIXED_LENGTH_256_HASH
+
#else // unsupported hash length
// Only hash lengths for which there is a corresponding macro header
@@ -125,6 +130,9 @@
// be added to this file.
#error "Unsupported value for PW_TOKENIZER_CFG_C_HASH_LENGTH"
+// Define a dummy macro to give clearer compilation errors.
+#define PW_TOKENIZER_STRING_TOKEN(unused) 0u
+
#endif // PW_TOKENIZER_CFG_C_HASH_LENGTH
#endif // __cpp_constexpr >= 201304L && defined(__cpp_inline_variables)
diff --git a/pw_tokenizer/py/database_test.py b/pw_tokenizer/py/database_test.py
index de863e3..290c831 100755
--- a/pw_tokenizer/py/database_test.py
+++ b/pw_tokenizer/py/database_test.py
@@ -198,10 +198,10 @@
self.assertEqual(db_with_custom_token.splitlines(),
self._csv.read_text().splitlines())
- def test_mark_removals(self):
+ def test_mark_removed(self):
self._csv.write_text(CSV_ALL_DOMAINS)
- run_cli('mark_removals', '--database', self._csv, '--date',
+ run_cli('mark_removed', '--database', self._csv, '--date',
'1998-09-04', self._elf)
# Add the removal date to the four tokens not in the default domain
@@ -223,7 +223,7 @@
self._csv.write_text(CSV_ALL_DOMAINS)
# Mark everything not in TEST_DOMAIN as removed.
- run_cli('mark_removals', '--database', self._csv,
+ run_cli('mark_removed', '--database', self._csv,
f'{self._elf}#TEST_DOMAIN')
# Delete all entries except those in TEST_DOMAIN.
diff --git a/pw_tokenizer/py/generate_hash_macro.py b/pw_tokenizer/py/generate_hash_macro.py
index 4c144e1..7120b59 100755
--- a/pw_tokenizer/py/generate_hash_macro.py
+++ b/pw_tokenizer/py/generate_hash_macro.py
@@ -19,7 +19,7 @@
HASH_CONSTANT = 65599
HASH_NAME = 'pw_tokenizer_65599_fixed_length'
-HASH_LENGTHS = 80, 96, 128
+HASH_LENGTHS = 80, 96, 128, 256
FILE_HEADER = """\
// Copyright {year} The Pigweed Authors
diff --git a/pw_tokenizer/py/pw_tokenizer/database.py b/pw_tokenizer/py/pw_tokenizer/database.py
index 9e7075e..39eb185 100755
--- a/pw_tokenizer/py/pw_tokenizer/database.py
+++ b/pw_tokenizer/py/pw_tokenizer/database.py
@@ -296,8 +296,8 @@
len(token_database) - initial, token_database.path)
-def _handle_mark_removals(token_database, databases, date):
- marked_removed = token_database.mark_removals(
+def _handle_mark_removed(token_database, databases, date):
+ marked_removed = token_database.mark_removed(
(entry for entry in tokens.Database.merged(*databases).entries()
if not entry.date_removed), date)
@@ -519,15 +519,15 @@
'marked as removed.'))
subparser.set_defaults(handler=_handle_add)
- # The 'mark_removals' command marks removed entries to match a set of ELFs.
+ # The 'mark_removed' command marks removed entries to match a set of ELFs.
subparser = subparsers.add_parser(
- 'mark_removals',
+ 'mark_removed',
parents=[option_db, option_tokens],
help=(
'Updates a database with tokenized strings from a set of strings. '
'Strings not present in the set remain in the database but are '
'marked as removed. New strings are NOT added.'))
- subparser.set_defaults(handler=_handle_mark_removals)
+ subparser.set_defaults(handler=_handle_mark_removed)
subparser.add_argument(
'--date',
type=year_month_day,
diff --git a/pw_tokenizer/py/pw_tokenizer/tokens.py b/pw_tokenizer/py/pw_tokenizer/tokens.py
index f7206b1..825aec9 100644
--- a/pw_tokenizer/py/pw_tokenizer/tokens.py
+++ b/pw_tokenizer/py/pw_tokenizer/tokens.py
@@ -161,7 +161,7 @@
if len(entries) > 1:
yield token, entries
- def mark_removals(
+ def mark_removed(
self,
all_entries: Iterable[TokenizedStringEntry],
removal_date: Optional[datetime] = None
diff --git a/pw_tokenizer/py/tokens_test.py b/pw_tokenizer/py/tokens_test.py
index 8c71a3b..1f67d5f 100755
--- a/pw_tokenizer/py/tokens_test.py
+++ b/pw_tokenizer/py/tokens_test.py
@@ -324,8 +324,8 @@
self.assertEqual(len(db.entries()), 18)
self.assertEqual(len(db.token_to_entries), 17)
- def test_mark_removals(self):
- """Tests that date_removed field is set by mark_removals."""
+ def test_mark_removed(self):
+ """Tests that date_removed field is set by mark_removed."""
db = tokens.Database.from_strings(
['MILK', 'apples', 'oranges', 'CHEESE', 'pears'])
@@ -333,7 +333,7 @@
all(entry.date_removed is None for entry in db.entries()))
date_1 = datetime.datetime(1, 2, 3)
- db.mark_removals(_entries('apples', 'oranges', 'pears'), date_1)
+ db.mark_removed(_entries('apples', 'oranges', 'pears'), date_1)
self.assertEqual(
db.token_to_entries[default_hash('MILK')][0].date_removed, date_1)
@@ -342,7 +342,7 @@
date_1)
now = datetime.datetime.now()
- db.mark_removals(_entries('MILK', 'CHEESE', 'pears'))
+ db.mark_removed(_entries('MILK', 'CHEESE', 'pears'))
# New strings are not added or re-added in mark_removed().
self.assertGreaterEqual(
