Test script for config.py
Run config.py with various options and store the results in files.
This script also supports the now-removed config.pl.
This is a framework to run non-regression tests on config.py: run it
with the old version, run it with the new version, and compare the
output.
This is deliberately not a functional test suite so that we don't need
to maintain a set of known outputs. When something changes in
config.py (or config.h), run the script before, run it after, and
check manually whether any differences in the output are acceptable.
diff --git a/tests/scripts/test_config_script.py b/tests/scripts/test_config_script.py
new file mode 100755
index 0000000..0a93fa0
--- /dev/null
+++ b/tests/scripts/test_config_script.py
@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+
+"""Test helper for the Mbed TLS configuration file tool
+
+Run config.py with various parameters and write the results to files.
+
+This is a harness to help regression testing, not a functional tester.
+Sample usage:
+
+ test_config_script.py -d old
+ ## Modify config.py and/or config.h ##
+ test_config_script.py -d new
+ diff -ru old new
+"""
+
+## Copyright (C) 2019, ARM Limited, All Rights Reserved
+## SPDX-License-Identifier: Apache-2.0
+##
+## Licensed under the Apache License, Version 2.0 (the "License"); you may
+## not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+## WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+## This file is part of Mbed TLS (https://tls.mbed.org)
+
+import argparse
+import glob
+import os
+import re
+import shutil
+import subprocess
+
+OUTPUT_FILE_PREFIX = 'config-'
+
+def output_file_name(directory, stem, extension):
+ return os.path.join(directory,
+ '{}{}.{}'.format(OUTPUT_FILE_PREFIX,
+ stem, extension))
+
+def cleanup_directory(directory):
+ """Remove old output files."""
+ for extension in []:
+ pattern = output_file_name(directory, '*', extension)
+ filenames = glob.glob(pattern)
+ for filename in filenames:
+ os.remove(filename)
+
+def prepare_directory(directory):
+ """Create the output directory if it doesn't exist yet.
+
+ If there are old output files, remove them.
+ """
+ if os.path.exists(directory):
+ cleanup_directory(directory)
+ else:
+ os.makedirs(directory)
+
+def guess_presets_from_help(help_text):
+ """Figure out what presets the script supports.
+
+ help_text should be the output from running the script with --help.
+ """
+ # Try the output format from config.py
+ hits = re.findall(r'\{([-\w,]+)\}', help_text)
+ for hit in hits:
+ words = set(hit.split(','))
+ if 'get' in words and 'set' in words and 'unset' in words:
+ words.remove('get')
+ words.remove('set')
+ words.remove('unset')
+ return words
+ # Try the output format from config.pl
+ hits = re.findall(r'\n +([-\w]+) +- ', help_text)
+ if hits:
+ return hits
+ raise Exception("Unable to figure out supported presets. Pass the '-p' option.")
+
+def list_presets(options):
+ """Return the list of presets to test.
+
+ The list is taken from the command line if present, otherwise it is
+ extracted from running the config script with --help.
+ """
+ if options.presets:
+ return re.split(r'[ ,]+', options.presets)
+ else:
+ help_text = subprocess.run([options.script, '--help'],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT).stdout
+ return guess_presets_from_help(help_text.decode('ascii'))
+
+def run_one(options, args):
+ """Run the config script with the given arguments.
+
+ Write the following files:
+ * config-xxx.h: modified file.
+ * config-xxx.out: standard output.
+ * config-xxx.err: standard output.
+ * config-xxx.status: exit code.
+ """
+ stem = '-'.join(args)
+ data_filename = output_file_name(options.output_directory, stem, 'h')
+ stdout_filename = output_file_name(options.output_directory, stem, 'out')
+ stderr_filename = output_file_name(options.output_directory, stem, 'err')
+ status_filename = output_file_name(options.output_directory, stem, 'status')
+ shutil.copy(options.input_file, data_filename)
+ # Pass only the file basename, not the full path, to avoid getting the
+ # directory name in error messages, which would make comparisons
+ # between output directories more difficult.
+ cmd = [os.path.abspath(options.script),
+ '-f', os.path.basename(data_filename)]
+ with open(stdout_filename, 'wb') as out:
+ with open(stderr_filename, 'wb') as err:
+ status = subprocess.call(cmd + args,
+ cwd=options.output_directory,
+ stdin=subprocess.DEVNULL,
+ stdout=out, stderr=err)
+ with open(status_filename, 'w') as status_file:
+ status_file.write('{}\n'.format(status))
+
+### A list of symbols to test with set and unset.
+TEST_SYMBOLS = [
+ 'CUSTOM_OPTION',
+ 'DOES_NOT_EXIST',
+ 'MBEDTLS_AES_C',
+ 'MBEDTLS_NO_UDBL_DIVISION',
+ 'MBEDTLS_PLATFORM_ZEROIZE_ALT',
+]
+
+### A list of symbols to test with set with a value.
+TEST_SYMBOLS_WITH_VALUE = [
+ 'CUSTOM_VALUE',
+ 'MBEDTLS_MPI_MAX_SIZE',
+]
+
+def run_all(options):
+ """Run all the command lines to test."""
+ presets = list_presets(options)
+ for preset in presets:
+ run_one(options, [preset])
+ for symbol in TEST_SYMBOLS:
+ run_one(options, ['set', symbol])
+ run_one(options, ['--force', 'set', symbol])
+ run_one(options, ['unset', symbol])
+ for symbol in TEST_SYMBOLS_WITH_VALUE:
+ run_one(options, ['set', symbol, 'value'])
+ run_one(options, ['--force', 'set', symbol, 'value'])
+
+def main():
+ """Command line entry point."""
+ parser = argparse.ArgumentParser(description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument('-d', metavar='DIR',
+ dest='output_directory', required=True,
+ help="""Output directory.""")
+ parser.add_argument('-f', metavar='FILE',
+ dest='input_file', default='include/mbedtls/config.h',
+ help="""Config file (default: %(default)s).""")
+ parser.add_argument('-p', metavar='PRESET,...',
+ dest='presets',
+ help="""Presets to test (default: guessed from --help).""")
+ parser.add_argument('-s', metavar='FILE',
+ dest='script', default='scripts/config.py',
+ help="""Configuration script (default: %(default)s).""")
+ options = parser.parse_args()
+ prepare_directory(options.output_directory)
+ run_all(options)
+
+if __name__ == '__main__':
+ main()
