Gilles Peskine | ba94b58 | 2019-09-16 19:18:40 +0200 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | |
| 3 | """Sanity checks for test data. |
| 4 | """ |
| 5 | |
| 6 | # Copyright (C) 2019, Arm Limited, All Rights Reserved |
| 7 | # SPDX-License-Identifier: Apache-2.0 |
| 8 | # |
| 9 | # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| 10 | # not use this file except in compliance with the License. |
| 11 | # You may obtain a copy of the License at |
| 12 | # |
| 13 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 14 | # |
| 15 | # Unless required by applicable law or agreed to in writing, software |
| 16 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 17 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 18 | # See the License for the specific language governing permissions and |
| 19 | # limitations under the License. |
| 20 | # |
| 21 | # This file is part of Mbed TLS (https://tls.mbed.org) |
| 22 | |
Gilles Peskine | 1fb7aea | 2019-12-02 14:26:04 +0100 | [diff] [blame] | 23 | import argparse |
Gilles Peskine | ba94b58 | 2019-09-16 19:18:40 +0200 | [diff] [blame] | 24 | import glob |
| 25 | import os |
| 26 | import re |
| 27 | import sys |
| 28 | |
| 29 | class Results: |
Darryl Green | 1822061 | 2019-12-17 15:03:59 +0000 | [diff] [blame] | 30 | """Store file and line information about errors or warnings in test suites.""" |
Gilles Peskine | 1fb7aea | 2019-12-02 14:26:04 +0100 | [diff] [blame] | 31 | |
| 32 | def __init__(self, options): |
Gilles Peskine | ba94b58 | 2019-09-16 19:18:40 +0200 | [diff] [blame] | 33 | self.errors = 0 |
| 34 | self.warnings = 0 |
Gilles Peskine | 1fb7aea | 2019-12-02 14:26:04 +0100 | [diff] [blame] | 35 | self.ignore_warnings = options.quiet |
Gilles Peskine | ba94b58 | 2019-09-16 19:18:40 +0200 | [diff] [blame] | 36 | |
| 37 | def error(self, file_name, line_number, fmt, *args): |
| 38 | sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n'). |
| 39 | format(file_name, line_number, *args)) |
| 40 | self.errors += 1 |
| 41 | |
| 42 | def warning(self, file_name, line_number, fmt, *args): |
Gilles Peskine | 1fb7aea | 2019-12-02 14:26:04 +0100 | [diff] [blame] | 43 | if not self.ignore_warnings: |
| 44 | sys.stderr.write(('{}:{}:Warning:' + fmt + '\n') |
| 45 | .format(file_name, line_number, *args)) |
| 46 | self.warnings += 1 |
Gilles Peskine | ba94b58 | 2019-09-16 19:18:40 +0200 | [diff] [blame] | 47 | |
| 48 | def collect_test_directories(): |
Darryl Green | 1822061 | 2019-12-17 15:03:59 +0000 | [diff] [blame] | 49 | """Get the relative path for the TLS and Crypto test directories.""" |
Gilles Peskine | ba94b58 | 2019-09-16 19:18:40 +0200 | [diff] [blame] | 50 | if os.path.isdir('tests'): |
| 51 | tests_dir = 'tests' |
| 52 | elif os.path.isdir('suites'): |
| 53 | tests_dir = '.' |
| 54 | elif os.path.isdir('../suites'): |
| 55 | tests_dir = '..' |
| 56 | directories = [tests_dir] |
Gilles Peskine | ba94b58 | 2019-09-16 19:18:40 +0200 | [diff] [blame] | 57 | return directories |
| 58 | |
Gilles Peskine | 32b9421 | 2019-09-20 18:00:49 +0200 | [diff] [blame] | 59 | def check_description(results, seen, file_name, line_number, description): |
Darryl Green | 1822061 | 2019-12-17 15:03:59 +0000 | [diff] [blame] | 60 | """Check test case descriptions for errors.""" |
Gilles Peskine | 32b9421 | 2019-09-20 18:00:49 +0200 | [diff] [blame] | 61 | if description in seen: |
| 62 | results.error(file_name, line_number, |
| 63 | 'Duplicate description (also line {})', |
| 64 | seen[description]) |
| 65 | return |
Gilles Peskine | f12ad58 | 2019-09-20 18:02:01 +0200 | [diff] [blame] | 66 | if re.search(br'[\t;]', description): |
Gilles Peskine | 32b9421 | 2019-09-20 18:00:49 +0200 | [diff] [blame] | 67 | results.error(file_name, line_number, |
| 68 | 'Forbidden character \'{}\' in description', |
Gilles Peskine | f12ad58 | 2019-09-20 18:02:01 +0200 | [diff] [blame] | 69 | re.search(br'[\t;]', description).group(0).decode('ascii')) |
Gilles Peskine | 57870e8 | 2019-09-20 18:02:30 +0200 | [diff] [blame] | 70 | if re.search(br'[^ -~]', description): |
| 71 | results.error(file_name, line_number, |
| 72 | 'Non-ASCII character in description') |
Gilles Peskine | 32b9421 | 2019-09-20 18:00:49 +0200 | [diff] [blame] | 73 | if len(description) > 66: |
| 74 | results.warning(file_name, line_number, |
| 75 | 'Test description too long ({} > 66)', |
| 76 | len(description)) |
| 77 | seen[description] = line_number |
| 78 | |
Gilles Peskine | d34e9e4 | 2020-06-25 16:16:25 +0200 | [diff] [blame^] | 79 | def walk_test_suite(function, results, descriptions, data_file_name): |
| 80 | """Iterate over the test cases in the given unit test data file. |
| 81 | |
| 82 | Call function(results, descriptions, data_file_name, line_number, description) |
| 83 | on each description. |
| 84 | """ |
Gilles Peskine | ba94b58 | 2019-09-16 19:18:40 +0200 | [diff] [blame] | 85 | in_paragraph = False |
Gilles Peskine | f12ad58 | 2019-09-20 18:02:01 +0200 | [diff] [blame] | 86 | with open(data_file_name, 'rb') as data_file: |
| 87 | for line_number, line in enumerate(data_file, 1): |
| 88 | line = line.rstrip(b'\r\n') |
Gilles Peskine | ba94b58 | 2019-09-16 19:18:40 +0200 | [diff] [blame] | 89 | if not line: |
| 90 | in_paragraph = False |
| 91 | continue |
Gilles Peskine | f12ad58 | 2019-09-20 18:02:01 +0200 | [diff] [blame] | 92 | if line.startswith(b'#'): |
Gilles Peskine | ba94b58 | 2019-09-16 19:18:40 +0200 | [diff] [blame] | 93 | continue |
| 94 | if not in_paragraph: |
| 95 | # This is a test case description line. |
Gilles Peskine | d34e9e4 | 2020-06-25 16:16:25 +0200 | [diff] [blame^] | 96 | function(results, descriptions, |
| 97 | data_file_name, line_number, line) |
Gilles Peskine | ba94b58 | 2019-09-16 19:18:40 +0200 | [diff] [blame] | 98 | in_paragraph = True |
| 99 | |
Gilles Peskine | d34e9e4 | 2020-06-25 16:16:25 +0200 | [diff] [blame^] | 100 | def walk_ssl_opt_sh(function, results, descriptions, file_name): |
| 101 | """Iterate over the test cases in ssl-opt.sh or a file with a similar format. |
| 102 | |
| 103 | Call function(results, descriptions, file_name, line_number, description) |
| 104 | on each description. |
| 105 | """ |
Gilles Peskine | f12ad58 | 2019-09-20 18:02:01 +0200 | [diff] [blame] | 106 | with open(file_name, 'rb') as file_contents: |
| 107 | for line_number, line in enumerate(file_contents, 1): |
Gilles Peskine | ba94b58 | 2019-09-16 19:18:40 +0200 | [diff] [blame] | 108 | # Assume that all run_test calls have the same simple form |
| 109 | # with the test description entirely on the same line as the |
| 110 | # function name. |
Gilles Peskine | f12ad58 | 2019-09-20 18:02:01 +0200 | [diff] [blame] | 111 | m = re.match(br'\s*run_test\s+"((?:[^\\"]|\\.)*)"', line) |
Gilles Peskine | ba94b58 | 2019-09-16 19:18:40 +0200 | [diff] [blame] | 112 | if not m: |
| 113 | continue |
| 114 | description = m.group(1) |
Gilles Peskine | d34e9e4 | 2020-06-25 16:16:25 +0200 | [diff] [blame^] | 115 | function(results, descriptions, |
| 116 | file_name, line_number, description) |
| 117 | |
| 118 | def walk_all(function, results): |
| 119 | """Iterate over all named test cases. |
| 120 | |
| 121 | Call function(results, {}, file_name, line_number, description) |
| 122 | on each description. |
| 123 | """ |
| 124 | test_directories = collect_test_directories() |
| 125 | for directory in test_directories: |
| 126 | for data_file_name in glob.glob(os.path.join(directory, 'suites', |
| 127 | '*.data')): |
| 128 | walk_test_suite(function, results, {}, data_file_name) |
| 129 | ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh') |
| 130 | if os.path.exists(ssl_opt_sh): |
| 131 | walk_ssl_opt_sh(function, results, {}, ssl_opt_sh) |
Gilles Peskine | ba94b58 | 2019-09-16 19:18:40 +0200 | [diff] [blame] | 132 | |
| 133 | def main(): |
Gilles Peskine | 1fb7aea | 2019-12-02 14:26:04 +0100 | [diff] [blame] | 134 | parser = argparse.ArgumentParser(description=__doc__) |
| 135 | parser.add_argument('--quiet', '-q', |
| 136 | action='store_true', |
| 137 | help='Hide warnings') |
| 138 | parser.add_argument('--verbose', '-v', |
| 139 | action='store_false', dest='quiet', |
| 140 | help='Show warnings (default: on; undoes --quiet)') |
| 141 | options = parser.parse_args() |
Gilles Peskine | 1fb7aea | 2019-12-02 14:26:04 +0100 | [diff] [blame] | 142 | results = Results(options) |
Gilles Peskine | d34e9e4 | 2020-06-25 16:16:25 +0200 | [diff] [blame^] | 143 | walk_all(check_description, results) |
Gilles Peskine | 1fb7aea | 2019-12-02 14:26:04 +0100 | [diff] [blame] | 144 | if (results.warnings or results.errors) and not options.quiet: |
Gilles Peskine | ba94b58 | 2019-09-16 19:18:40 +0200 | [diff] [blame] | 145 | sys.stderr.write('{}: {} errors, {} warnings\n' |
| 146 | .format(sys.argv[0], results.errors, results.warnings)) |
| 147 | sys.exit(1 if results.errors else 0) |
| 148 | |
| 149 | if __name__ == '__main__': |
| 150 | main() |