| #!/usr/bin/env python3 |
| # Copyright (c) 2022 Project CHIP Authors |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import glob |
| import logging |
| import os |
| import shutil |
| import subprocess |
| import sys |
| import unittest |
| from dataclasses import dataclass, field |
| from typing import List, Optional |
| |
| import yaml |
| |
| TESTS_DIR = os.path.join(os.path.dirname(__file__), "tests") |
| CHIP_ROOT = os.path.abspath(os.path.join( |
| os.path.dirname(__file__), "../../..")) |
| |
| |
| @dataclass |
| class ProgramArguments: |
| stamp_file: Optional[str] = None |
| regenerate_golden: bool = False |
| output_directory: str = '' |
| |
| |
| PROGRAM_ARGUMENTS = None |
| |
| |
| @dataclass |
| class ExpectedOutput: |
| file_name: str |
| golden_path: str |
| |
| |
| @dataclass |
| class GeneratorTestCase: |
| template: str |
| outputs: List[ExpectedOutput] = field(default_factory=list) |
| |
| def add_outputs(self, yaml_outputs_dict): |
| for file_name, golden_path in yaml_outputs_dict.items(): |
| self.outputs.append(ExpectedOutput( |
| file_name=file_name, golden_path=golden_path)) |
| |
| |
| @dataclass |
| class GeneratorTest: |
| zap: str |
| context: ProgramArguments |
| test_cases: List[GeneratorTestCase] = field(default_factory=list) |
| |
| def add_test_cases(self, yaml_test_case_dict): |
| for json, outputs in yaml_test_case_dict.items(): |
| test_case = GeneratorTestCase(template=json) |
| test_case.add_outputs(outputs) |
| self.test_cases.append(test_case) |
| |
| def run_test_cases(self, checker: unittest.TestCase): |
| for test in self.test_cases: |
| with checker.subTest(template=test.template): |
| output_directory = os.path.join( |
| self.context.output_directory, |
| os.path.splitext(os.path.basename(self.zap))[0], |
| os.path.splitext(os.path.basename(test.template))[0] |
| ) |
| |
| # ensure a clean start as ALL outputs will be compared |
| if os.path.exists(output_directory): |
| shutil.rmtree(output_directory) |
| os.makedirs(output_directory, exist_ok=True) |
| |
| subprocess.check_call([ |
| f"{CHIP_ROOT}/scripts/tools/zap/generate.py", |
| "--parallel", |
| "--output-dir", |
| output_directory, |
| "--templates", |
| os.path.join(TESTS_DIR, test.template), |
| os.path.join(TESTS_DIR, self.zap), |
| ], cwd=output_directory) |
| |
| # Files generated, ready to check: |
| # - every output file MUST exist in the golden image list |
| # - every golden image file MUST exist in the output |
| # - content must match |
| |
| # test.outputs contain: |
| # - file_name |
| # - golden_path |
| expected_files = set([o.file_name for o in test.outputs]) |
| actual_files = set([ |
| name[len(output_directory)+1:] for name in glob.glob(f"{output_directory}/**/*", recursive=True) |
| ]) |
| |
| checker.assertEqual( |
| expected_files, actual_files, msg="Expected and actual generated file list MUST be identical.") |
| |
| # All files exist, ready to do the compare |
| for entry in test.outputs: |
| expected = os.path.join(TESTS_DIR, entry.golden_path) |
| actual = os.path.join(output_directory, entry.file_name) |
| |
| try: |
| subprocess.check_call(["diff", actual, expected]) |
| except: |
| if self.context.regenerate_golden: |
| print( |
| f"Copying updated golden image from {actual} to {expected}") |
| subprocess.check_call(["cp", actual, expected]) |
| else: |
| print("*"*80) |
| print("* Golden image regeneration seems to have failed.") |
| print("* Documentation regarding code-generation logic available at docs/code_generation.md") |
| print("*\n* Specifically to update golden images, you may want to run:") |
| print("*\n* ./scripts/tools/zap/test_generate.py --output out/gen --regenerate") |
| print("*\n" + "*"*80) |
| raise |
| |
| |
| def build_tests(yaml_data, context: ProgramArguments) -> List[GeneratorTest]: |
| """ |
| Transforms the YAML dictonary (Dict[str, Dict[str, Dict[str,str]]]) into |
| a generator test structure. |
| """ |
| result = [] |
| |
| for input_zap, test_cases in yaml_data.items(): |
| generator = GeneratorTest(zap=input_zap, context=context) |
| generator.add_test_cases(test_cases) |
| result.append(generator) |
| |
| return result |
| |
| |
| class TestGenerators(unittest.TestCase): |
| def test_generators(self): |
| with open(os.path.join(TESTS_DIR, "available_tests.yaml"), 'rt') as stream: |
| yaml_data = yaml.safe_load(stream) |
| |
| global PROGRAM_ARGUMENTS |
| for test in build_tests(yaml_data, context=PROGRAM_ARGUMENTS): |
| with self.subTest(zap=test.zap): |
| test.run_test_cases(self) |
| |
| |
| def process_arguments(): |
| """Parses sys.argv and extracts arguments that are specific to the script.""" |
| args = sys.argv[:] |
| |
| program_args = ProgramArguments() |
| |
| if '--regenerate' in args: |
| idx = args.index('--regenerate') |
| program_args.regenerate_golden = True |
| del args[idx] |
| elif 'ZAP_GENERATE_GOLDEN_REGENERATE' in os.environ: |
| # Allow `ZAP_GENERATE_GOLDEN_REGENERATE=1 ninja check` to also |
| # update golden images |
| program_args.regenerate_golden = True |
| |
| if '--stamp' in args: |
| idx = args.index('--stamp') |
| program_args.stamp_file = args[idx + 1] |
| del args[idx+1] |
| del args[idx] |
| |
| if '--output' in args: |
| idx = args.index('--output') |
| program_args.output_directory = args[idx + 1] |
| del args[idx+1] |
| del args[idx] |
| else: |
| raise Exception("`--output` argument is required") |
| |
| return program_args, args |
| |
| |
| if __name__ == '__main__': |
| process_args, unittest_args = process_arguments() |
| |
| if process_args.stamp_file: |
| if os.path.exists(process_args.stamp_file): |
| os.remove(process_args.stamp_file) |
| |
| PROGRAM_ARGUMENTS = process_args |
| |
| unittest.main(argv=unittest_args) |
| |
| if process_args.stamp_file: |
| open(process_args.stamp_file, "wb").close() |