New test script check-test-cases.py
This script checks test case descriptions in test_suite_*.data and
ssl-opt.sh.
It reports the following issues:
* Error: forbidden character in a test case description.
* Error: Duplicate test description.
* Warning: Test description is too long.
diff --git a/tests/scripts/check-test-cases.py b/tests/scripts/check-test-cases.py
new file mode 100755
index 0000000..688ad25
--- /dev/null
+++ b/tests/scripts/check-test-cases.py
@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+
+"""Sanity checks for test data.
+"""
+
+# Copyright (C) 2019, Arm Limited, All Rights Reserved
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This file is part of Mbed TLS (https://tls.mbed.org)
+
+import glob
+import os
+import re
+import sys
+
+class Results:
+ def __init__(self):
+ self.errors = 0
+ self.warnings = 0
+
+ def error(self, file_name, line_number, fmt, *args):
+ sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n').
+ format(file_name, line_number, *args))
+ self.errors += 1
+
+ def warning(self, file_name, line_number, fmt, *args):
+ sys.stderr.write(('{}:{}:Warning:' + fmt + '\n')
+ .format(file_name, line_number, args))
+ self.warnings += 1
+
+def collect_test_directories():
+ if os.path.isdir('tests'):
+ tests_dir = 'tests'
+ elif os.path.isdir('suites'):
+ tests_dir = '.'
+ elif os.path.isdir('../suites'):
+ tests_dir = '..'
+ directories = [tests_dir]
+ crypto_tests_dir = os.path.normpath(os.path.join(tests_dir,
+ '../crypto/tests'))
+ if os.path.isdir(crypto_tests_dir):
+ directories.append(crypto_tests_dir)
+ return directories
+
+def check_test_suite(results, data_file_name):
+ in_paragraph = False
+ descriptions = {}
+ line_number = 0
+ with open(data_file_name) as data_file:
+ for line in data_file:
+ line_number += 1
+ line = line.rstrip('\r\n')
+ if not line:
+ in_paragraph = False
+ continue
+ if line.startswith('#'):
+ continue
+ if not in_paragraph:
+ # This is a test case description line.
+ if line in descriptions:
+ results.error(data_file_name, line_number,
+ 'Duplicate description (also line {}): {}',
+ descriptions[line], line)
+ else:
+ if re.search(r'[\t;]', line):
+ results.error(data_file_name, line_number,
+ 'Forbidden character in description')
+ if len(line) > 66:
+ results.warning(data_file_name, line_number,
+ 'Test description will be truncated')
+ descriptions[line] = line_number
+ in_paragraph = True
+
+def check_ssl_opt_sh(results, file_name):
+ descriptions = {}
+ line_number = 0
+ with open(file_name) as file_contents:
+ for line in file_contents:
+ line_number += 1
+ # Assume that all run_test calls have the same simple form
+ # with the test description entirely on the same line as the
+ # function name.
+ m = re.match(r'\s+run_test\s+"([^"])"', line)
+ if not m:
+ continue
+ description = m.group(1)
+ if description in descriptions:
+ results.error(data_file_name, line_number,
+ 'Duplicate description (also line {}): {}',
+ descriptions[line], line)
+ else:
+ if re.search(r'[\t;]', line):
+ results.error(data_file_name, line_number,
+ 'Forbidden character in description')
+ if len(line) > 66:
+ results.warning(data_file_name, line_number,
+ 'Test description will break visual alignment')
+ descriptions[line] = line_number
+
+def main():
+ test_directories = collect_test_directories()
+ results = Results()
+ for directory in test_directories:
+ for data_file_name in glob.glob(os.path.join(directory, 'suites',
+ '*.data')):
+ check_test_suite(results, data_file_name)
+ ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh')
+ if os.path.exists(ssl_opt_sh):
+ check_ssl_opt_sh(results, ssl_opt_sh)
+ if results.warnings or results.errors:
+ sys.stderr.write('{}: {} errors, {} warnings\n'
+ .format(sys.argv[0], results.errors, results.warnings))
+ sys.exit(1 if results.errors else 0)
+
+if __name__ == '__main__':
+ main()
