| """Common code for test data generation. |
| |
| This module defines classes that are of general use to automatically |
| generate .data files for unit tests, as well as a main function. |
| |
| These are used both by generate_psa_tests.py and generate_bignum_tests.py. |
| """ |
| |
| # Copyright The Mbed TLS Contributors |
| # SPDX-License-Identifier: Apache-2.0 |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| # not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import argparse |
| import os |
| import posixpath |
| import re |
| import inspect |
| |
| from abc import ABCMeta, abstractmethod |
| from typing import Callable, Dict, Iterable, Iterator, List, Type, TypeVar |
| |
| from . import build_tree |
| from . import test_case |
| |
| T = TypeVar('T') #pylint: disable=invalid-name |
| |
| |
| class BaseTest(metaclass=ABCMeta): |
| """Base class for test case generation. |
| |
| Attributes: |
| count: Counter for test cases from this class. |
| case_description: Short description of the test case. This may be |
| automatically generated using the class, or manually set. |
| dependencies: A list of dependencies required for the test case. |
| show_test_count: Toggle for inclusion of `count` in the test description. |
| test_function: Test function which the class generates cases for. |
| test_name: A common name or description of the test function. This can |
| be `test_function`, a clearer equivalent, or a short summary of the |
| test function's purpose. |
| """ |
| count = 0 |
| case_description = "" |
| dependencies = [] # type: List[str] |
| show_test_count = True |
| test_function = "" |
| test_name = "" |
| |
| def __new__(cls, *args, **kwargs): |
| # pylint: disable=unused-argument |
| cls.count += 1 |
| return super().__new__(cls) |
| |
| @abstractmethod |
| def arguments(self) -> List[str]: |
| """Get the list of arguments for the test case. |
| |
| Override this method to provide the list of arguments required for |
| the `test_function`. |
| |
| Returns: |
| List of arguments required for the test function. |
| """ |
| raise NotImplementedError |
| |
| def description(self) -> str: |
| """Create a test case description. |
| |
| Creates a description of the test case, including a name for the test |
| function, an optional case count, and a description of the specific |
| test case. This should inform a reader what is being tested, and |
| provide context for the test case. |
| |
| Returns: |
| Description for the test case. |
| """ |
| if self.show_test_count: |
| return "{} #{} {}".format( |
| self.test_name, self.count, self.case_description |
| ).strip() |
| else: |
| return "{} {}".format(self.test_name, self.case_description).strip() |
| |
| |
| def create_test_case(self) -> test_case.TestCase: |
| """Generate TestCase from the instance.""" |
| tc = test_case.TestCase() |
| tc.set_description(self.description()) |
| tc.set_function(self.test_function) |
| tc.set_arguments(self.arguments()) |
| tc.set_dependencies(self.dependencies) |
| |
| return tc |
| |
| @classmethod |
| @abstractmethod |
| def generate_function_tests(cls) -> Iterator[test_case.TestCase]: |
| """Generate test cases for the class test function. |
| |
| This will be called in classes where `test_function` is set. |
| Implementations should yield TestCase objects, by creating instances |
| of the class with appropriate input data, and then calling |
| `create_test_case()` on each. |
| """ |
| raise NotImplementedError |
| |
| |
| class BaseTarget: |
| #pylint: disable=too-few-public-methods |
| """Base target for test case generation. |
| |
| Child classes of this class represent an output file, and can be referred |
| to as file targets. These indicate where test cases will be written to for |
| all subclasses of the file target, which is set by `target_basename`. |
| |
| Attributes: |
| target_basename: Basename of file to write generated tests to. This |
| should be specified in a child class of BaseTarget. |
| """ |
| target_basename = "" |
| |
| @classmethod |
| def generate_tests(cls) -> Iterator[test_case.TestCase]: |
| """Generate test cases for the class and its subclasses. |
| |
| In classes with `test_function` set, `generate_function_tests()` is |
| called to generate test cases first. |
| |
| In all classes, this method will iterate over its subclasses, and |
| yield from `generate_tests()` in each. Calling this method on a class X |
| will yield test cases from all classes derived from X. |
| """ |
| if issubclass(cls, BaseTest) and not inspect.isabstract(cls): |
| #pylint: disable=no-member |
| yield from cls.generate_function_tests() |
| for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__): |
| yield from subclass.generate_tests() |
| |
| |
| class TestGenerator: |
| """Generate test cases and write to data files.""" |
| def __init__(self, options) -> None: |
| self.test_suite_directory = options.directory |
| # Update `targets` with an entry for each child class of BaseTarget. |
| # Each entry represents a file generated by the BaseTarget framework, |
| # and enables generating the .data files using the CLI. |
| self.targets.update({ |
| subclass.target_basename: subclass.generate_tests |
| for subclass in BaseTarget.__subclasses__() |
| if subclass.target_basename |
| }) |
| |
| def filename_for(self, basename: str) -> str: |
| """The location of the data file with the specified base name.""" |
| return posixpath.join(self.test_suite_directory, basename + '.data') |
| |
| def write_test_data_file(self, basename: str, |
| test_cases: Iterable[test_case.TestCase]) -> None: |
| """Write the test cases to a .data file. |
| |
| The output file is ``basename + '.data'`` in the test suite directory. |
| """ |
| filename = self.filename_for(basename) |
| test_case.write_data_file(filename, test_cases) |
| |
| # Note that targets whose names contain 'test_format' have their content |
| # validated by `abi_check.py`. |
| targets = {} # type: Dict[str, Callable[..., Iterable[test_case.TestCase]]] |
| |
| def generate_target(self, name: str, *target_args) -> None: |
| """Generate cases and write to data file for a target. |
| |
| For target callables which require arguments, override this function |
| and pass these arguments using super() (see PSATestGenerator). |
| """ |
| test_cases = self.targets[name](*target_args) |
| self.write_test_data_file(name, test_cases) |
| |
| def main(args, description: str, generator_class: Type[TestGenerator] = TestGenerator): |
| """Command line entry point.""" |
| parser = argparse.ArgumentParser(description=description) |
| parser.add_argument('--list', action='store_true', |
| help='List available targets and exit') |
| parser.add_argument('--list-for-cmake', action='store_true', |
| help='Print \';\'-separated list of available targets and exit') |
| # If specified explicitly, this option may be a path relative to the |
| # current directory when the script is invoked. The default value |
| # is relative to the mbedtls root, which we don't know yet. So we |
| # can't set a string as the default value here. |
| parser.add_argument('--directory', metavar='DIR', |
| help='Output directory (default: tests/suites)') |
| parser.add_argument('targets', nargs='*', metavar='TARGET', |
| help='Target file to generate (default: all; "-": none)') |
| options = parser.parse_args(args) |
| |
| # Change to the mbedtls root, to keep things simple. But first, adjust |
| # command line options that might be relative paths. |
| if options.directory is None: |
| options.directory = 'tests/suites' |
| else: |
| options.directory = os.path.abspath(options.directory) |
| build_tree.chdir_to_root() |
| |
| generator = generator_class(options) |
| if options.list: |
| for name in sorted(generator.targets): |
| print(generator.filename_for(name)) |
| return |
| # List in a cmake list format (i.e. ';'-separated) |
| if options.list_for_cmake: |
| print(';'.join(generator.filename_for(name) |
| for name in sorted(generator.targets)), end='') |
| return |
| if options.targets: |
| # Allow "-" as a special case so you can run |
| # ``generate_xxx_tests.py - $targets`` and it works uniformly whether |
| # ``$targets`` is empty or not. |
| options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) |
| for target in options.targets |
| if target != '-'] |
| else: |
| options.targets = sorted(generator.targets) |
| for target in options.targets: |
| generator.generate_target(target) |