Improve dict validation (#140)
* Refactor dict validation using regular expression for simplification and improvement.
After the change, the validation will:
- Disallow `"\"`, `""`, and any non-space suffix after the entry ends (e.g. `"entry"suffix`).
- Disallow unesacped '"' in enties. Raw '"' can indeed be used in libFuzzer due to implementation details,
but it is undocumented (https://llvm.org/docs/LibFuzzer.html#dictionaries), implementation-dependent, and confusing.
* Fix a bad entry in example dictionaries.
* Show file paths and line numbers with errors.
diff --git a/examples/dictionaries/valid_part1.dict b/examples/dictionaries/valid_part1.dict
index 913cbad..29b6d96 100644
--- a/examples/dictionaries/valid_part1.dict
+++ b/examples/dictionaries/valid_part1.dict
@@ -6,7 +6,7 @@
# Use \\ for backslash and \" for quotes.
kw2="\"ac\\dc\""
# Use \xAB for hex values
-kw3="\xF7\xF8""
+kw3="\xF7\xF8"
# the name of the keyword followed by '=' may be omitted:
"foo\x0Abar"
-"ab\""
\ No newline at end of file
+"ab\""
diff --git a/fuzzing/tools/dict_validation.py b/fuzzing/tools/dict_validation.py
index 5a169e1..db3c89b 100644
--- a/fuzzing/tools/dict_validation.py
+++ b/fuzzing/tools/dict_validation.py
@@ -17,47 +17,22 @@
Validates the fuzzing dictionary.
"""
-from string import hexdigits
+import re
-
-def validate_entry(entry):
- """Validates a single fuzzing dictionary entry.
-
- Args:
- entry: a string containing a single entry.
-
- Returns:
- True if the argument is a valid fuzzing dictionary entry,
- otherwise False.
- """
-
- # Use set to contain hex digits to decrease the query time complexity
- hex_set = set(hexdigits)
- pos, end = 0, len(entry) - 1
- while pos < end:
- pos += 1
- chr = entry[pos]
-
- if not (chr.isprintable() or chr.isspace()):
- return False
-
- # Handle '\\'
- if chr == '\\':
- if pos + 1 <= end and (entry[pos + 1] == '\\' or
- entry[pos + 1] == '"'):
- pos += 1
- continue
-
- # Handle '\xAB'
- if pos + 3 <= end and entry[pos + 1] == 'x' and entry[
- pos + 2] in hex_set and entry[pos + 3] in hex_set:
- pos += 3
- continue
-
- return False
-
- return True
-
+_DICTIONARY_LINE_RE = re.compile(
+ r'''[^"]* # Skip an arbitrary prefix (not used by libFuzzer).
+ " # Must be enclosed in quotes.
+ (
+ [^\\\"] # One or more non-escape characters...
+ |
+ \\( # ...or an escape sequence...
+ [\\\"] # ...consisting of either `\` or `"`...
+ |
+ x[0-9a-f]{2} # ...or a hexa number, e.g. '\x0f'
+ )
+ )+
+ "''',
+ flags=re.IGNORECASE | re.VERBOSE)
def validate_line(line):
"""Validates a single line in the fuzzing dictionary entry.
@@ -72,15 +47,5 @@
line = line.strip()
if not line or line.startswith('#'):
return True
- if len(line) < 2 or line[-1] != '"':
- return False
-
- left = 0
- # Find the opening "
- while left < len(line) - 1 and line[left] != '"':
- left += 1
-
- if left >= len(line) - 1:
- return False
-
- return validate_entry(line[left:])
+ else:
+ return re.fullmatch(_DICTIONARY_LINE_RE, line) is not None
diff --git a/fuzzing/tools/dict_validation_test.py b/fuzzing/tools/dict_validation_test.py
index 47eb7f4..078f8c5 100644
--- a/fuzzing/tools/dict_validation_test.py
+++ b/fuzzing/tools/dict_validation_test.py
@@ -28,7 +28,6 @@
self.assertTrue(validate_line('":path"'))
self.assertTrue(validate_line('"keep-alive"'))
self.assertTrue(validate_line('"te"'))
- self.assertTrue(validate_line('"ab""'))
def test_escaped_words(self):
self.assertTrue(validate_line('kw2="\\"ac\\\\dc\\""'))
@@ -38,9 +37,23 @@
def test_invalid_escaped_words(self):
self.assertFalse(validate_line('"\\A"'))
+ def test_unfinished_escape(self):
+ self.assertFalse(validate_line('"\\"'))
+ self.assertFalse(validate_line('"\\x"'))
+ self.assertFalse(validate_line('"\\x1"'))
+
+ def test_invalid_unescaped_words(self):
+ self.assertFalse(validate_line('"""'))
+
def test_comment(self):
self.assertTrue(validate_line('# valid dictionary entries'))
+ def test_suffix_after_entry(self):
+ self.assertFalse(validate_line('"entry"suffix'))
+
+ def test_empty_entry(self):
+ self.assertFalse(validate_line('""'))
+
def test_empty_string(self):
self.assertTrue(validate_line(''))
diff --git a/fuzzing/tools/validate_dict.py b/fuzzing/tools/validate_dict.py
index 28f9065..d561e68 100644
--- a/fuzzing/tools/validate_dict.py
+++ b/fuzzing/tools/validate_dict.py
@@ -33,10 +33,11 @@
def validate_dict(dict_path, output_stream):
with open(dict_path, 'r') as dict:
- for line in dict.readlines():
+ for index, line in enumerate(dict.readlines()):
line = line.strip()
if not validate_line(line):
- print("ERROR: invalid dictionary entry '%s'" % line,
+ print("ERROR: invalid dictionary entry '%s' in %s:%d" %
+ (line, dict_path, index + 1),
file=stderr)
return False
if output_stream:
