blob: f25b602c7dd5b1da48d52da0985895a1d41661e7 [file] [log] [blame]
Gilles Peskineba94b582019-09-16 19:18:40 +02001#!/usr/bin/env python3
2
3"""Sanity checks for test data.
4"""
5
6# Copyright (C) 2019, Arm Limited, All Rights Reserved
7# SPDX-License-Identifier: Apache-2.0
8#
9# Licensed under the Apache License, Version 2.0 (the "License"); you may
10# not use this file except in compliance with the License.
11# You may obtain a copy of the License at
12#
13# http://www.apache.org/licenses/LICENSE-2.0
14#
15# Unless required by applicable law or agreed to in writing, software
16# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
17# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18# See the License for the specific language governing permissions and
19# limitations under the License.
20#
21# This file is part of Mbed TLS (https://tls.mbed.org)
22
Gilles Peskine1fb7aea2019-12-02 14:26:04 +010023import argparse
Gilles Peskineba94b582019-09-16 19:18:40 +020024import glob
25import os
26import re
27import sys
28
29class Results:
Darryl Green18220612019-12-17 15:03:59 +000030 """Store file and line information about errors or warnings in test suites."""
Gilles Peskine1fb7aea2019-12-02 14:26:04 +010031
32 def __init__(self, options):
Gilles Peskineba94b582019-09-16 19:18:40 +020033 self.errors = 0
34 self.warnings = 0
Gilles Peskine1fb7aea2019-12-02 14:26:04 +010035 self.ignore_warnings = options.quiet
Gilles Peskineba94b582019-09-16 19:18:40 +020036
37 def error(self, file_name, line_number, fmt, *args):
38 sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n').
39 format(file_name, line_number, *args))
40 self.errors += 1
41
42 def warning(self, file_name, line_number, fmt, *args):
Gilles Peskine1fb7aea2019-12-02 14:26:04 +010043 if not self.ignore_warnings:
44 sys.stderr.write(('{}:{}:Warning:' + fmt + '\n')
45 .format(file_name, line_number, *args))
46 self.warnings += 1
Gilles Peskineba94b582019-09-16 19:18:40 +020047
48def collect_test_directories():
Darryl Green18220612019-12-17 15:03:59 +000049 """Get the relative path for the TLS and Crypto test directories."""
Gilles Peskineba94b582019-09-16 19:18:40 +020050 if os.path.isdir('tests'):
51 tests_dir = 'tests'
52 elif os.path.isdir('suites'):
53 tests_dir = '.'
54 elif os.path.isdir('../suites'):
55 tests_dir = '..'
56 directories = [tests_dir]
Gilles Peskineba94b582019-09-16 19:18:40 +020057 return directories
58
Gilles Peskine32b94212019-09-20 18:00:49 +020059def check_description(results, seen, file_name, line_number, description):
Darryl Green18220612019-12-17 15:03:59 +000060 """Check test case descriptions for errors."""
Gilles Peskine32b94212019-09-20 18:00:49 +020061 if description in seen:
62 results.error(file_name, line_number,
63 'Duplicate description (also line {})',
64 seen[description])
65 return
Gilles Peskinef12ad582019-09-20 18:02:01 +020066 if re.search(br'[\t;]', description):
Gilles Peskine32b94212019-09-20 18:00:49 +020067 results.error(file_name, line_number,
68 'Forbidden character \'{}\' in description',
Gilles Peskinef12ad582019-09-20 18:02:01 +020069 re.search(br'[\t;]', description).group(0).decode('ascii'))
Gilles Peskine57870e82019-09-20 18:02:30 +020070 if re.search(br'[^ -~]', description):
71 results.error(file_name, line_number,
72 'Non-ASCII character in description')
Gilles Peskine32b94212019-09-20 18:00:49 +020073 if len(description) > 66:
74 results.warning(file_name, line_number,
75 'Test description too long ({} > 66)',
76 len(description))
77 seen[description] = line_number
78
Gilles Peskined34e9e42020-06-25 16:16:25 +020079def walk_test_suite(function, results, descriptions, data_file_name):
80 """Iterate over the test cases in the given unit test data file.
81
82Call function(results, descriptions, data_file_name, line_number, description)
83on each description.
84"""
Gilles Peskineba94b582019-09-16 19:18:40 +020085 in_paragraph = False
Gilles Peskinef12ad582019-09-20 18:02:01 +020086 with open(data_file_name, 'rb') as data_file:
87 for line_number, line in enumerate(data_file, 1):
88 line = line.rstrip(b'\r\n')
Gilles Peskineba94b582019-09-16 19:18:40 +020089 if not line:
90 in_paragraph = False
91 continue
Gilles Peskinef12ad582019-09-20 18:02:01 +020092 if line.startswith(b'#'):
Gilles Peskineba94b582019-09-16 19:18:40 +020093 continue
94 if not in_paragraph:
95 # This is a test case description line.
Gilles Peskined34e9e42020-06-25 16:16:25 +020096 function(results, descriptions,
97 data_file_name, line_number, line)
Gilles Peskineba94b582019-09-16 19:18:40 +020098 in_paragraph = True
99
Gilles Peskined34e9e42020-06-25 16:16:25 +0200100def walk_ssl_opt_sh(function, results, descriptions, file_name):
101 """Iterate over the test cases in ssl-opt.sh or a file with a similar format.
102
103Call function(results, descriptions, file_name, line_number, description)
104on each description.
105"""
Gilles Peskinef12ad582019-09-20 18:02:01 +0200106 with open(file_name, 'rb') as file_contents:
107 for line_number, line in enumerate(file_contents, 1):
Gilles Peskineba94b582019-09-16 19:18:40 +0200108 # Assume that all run_test calls have the same simple form
109 # with the test description entirely on the same line as the
110 # function name.
Gilles Peskinef12ad582019-09-20 18:02:01 +0200111 m = re.match(br'\s*run_test\s+"((?:[^\\"]|\\.)*)"', line)
Gilles Peskineba94b582019-09-16 19:18:40 +0200112 if not m:
113 continue
114 description = m.group(1)
Gilles Peskined34e9e42020-06-25 16:16:25 +0200115 function(results, descriptions,
116 file_name, line_number, description)
117
118def walk_all(function, results):
119 """Iterate over all named test cases.
120
121Call function(results, {}, file_name, line_number, description)
122on each description.
123"""
124 test_directories = collect_test_directories()
125 for directory in test_directories:
126 for data_file_name in glob.glob(os.path.join(directory, 'suites',
127 '*.data')):
128 walk_test_suite(function, results, {}, data_file_name)
129 ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh')
130 if os.path.exists(ssl_opt_sh):
131 walk_ssl_opt_sh(function, results, {}, ssl_opt_sh)
Gilles Peskineba94b582019-09-16 19:18:40 +0200132
133def main():
Gilles Peskine1fb7aea2019-12-02 14:26:04 +0100134 parser = argparse.ArgumentParser(description=__doc__)
135 parser.add_argument('--quiet', '-q',
136 action='store_true',
137 help='Hide warnings')
138 parser.add_argument('--verbose', '-v',
139 action='store_false', dest='quiet',
140 help='Show warnings (default: on; undoes --quiet)')
141 options = parser.parse_args()
Gilles Peskine1fb7aea2019-12-02 14:26:04 +0100142 results = Results(options)
Gilles Peskined34e9e42020-06-25 16:16:25 +0200143 walk_all(check_description, results)
Gilles Peskine1fb7aea2019-12-02 14:26:04 +0100144 if (results.warnings or results.errors) and not options.quiet:
Gilles Peskineba94b582019-09-16 19:18:40 +0200145 sys.stderr.write('{}: {} errors, {} warnings\n'
146 .format(sys.argv[0], results.errors, results.warnings))
147 sys.exit(1 if results.errors else 0)
148
149if __name__ == '__main__':
150 main()