pw_tokenizer: Update default Python hash length
Update the default Python tokenization hash length to match the default
value in C++ (128). Previously, the default was accidentally set to 96
in Python, which would result in C++ and Python producing different
hashes for strings longer than 96 characters.
Change-Id: Id0039e8bf131d98a28ae7c62fdacc648b115fea3
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/19144
Reviewed-by: Armando Montanez <amontanez@google.com>
Commit-Queue: Wyatt Hepler <hepler@google.com>
diff --git a/pw_tokenizer/public/pw_tokenizer/config.h b/pw_tokenizer/public/pw_tokenizer/config.h
index 2e49fa3..fede273 100644
--- a/pw_tokenizer/public/pw_tokenizer/config.h
+++ b/pw_tokenizer/public/pw_tokenizer/config.h
@@ -49,6 +49,9 @@
// (pw_tokenizer/internal/mash_macro_#.h) are supported. Additional macros may
// be generated with the generate_hash_macro.py function. New macro headers must
// then be added to pw_tokenizer/internal/hash.h.
+//
+// This MUST match the value of DEFAULT_HASH_LENGTH in
+// pw_tokenizer/py/pw_tokenizer/tokens.py.
#ifndef PW_TOKENIZER_CFG_HASH_LENGTH
#define PW_TOKENIZER_CFG_HASH_LENGTH 128
#endif // PW_TOKENIZER_CFG_HASH_LENGTH
diff --git a/pw_tokenizer/py/pw_tokenizer/tokens.py b/pw_tokenizer/py/pw_tokenizer/tokens.py
index 4a416bf..685d074 100644
--- a/pw_tokenizer/py/pw_tokenizer/tokens.py
+++ b/pw_tokenizer/py/pw_tokenizer/tokens.py
@@ -26,7 +26,9 @@
DATE_FORMAT = '%Y-%m-%d'
-DEFAULT_HASH_LENGTH = 96
+# The default hash length to use. This MUST match the default value of
+# PW_TOKENIZER_CFG_HASH_LENGTH in pw_tokenizer/public/pw_tokenizer/config.h.
+DEFAULT_HASH_LENGTH = 128
TOKENIZER_HASH_CONSTANT = 65599
_LOG = logging.getLogger('pw_tokenizer')
