| #!/usr/bin/env python3 |
| # Copyright (c) 2024 Intel Corporation |
| # |
| # SPDX-License-Identifier: Apache-2.0 |
| """ |
| Blackbox tests for twister's command line functions |
| """ |
| import importlib |
| import re |
| import mock |
| import os |
| import pytest |
| import sys |
| import json |
| |
| from conftest import TEST_DATA, ZEPHYR_BASE, testsuite_filename_mock, clear_log_in_test |
| from twisterlib.testplan import TestPlan |
| |
| |
| @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) |
| class TestCoverage: |
| TESTDATA_1 = [ |
| ( |
| os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'), |
| ['qemu_x86'], |
| [ |
| 'coverage.log', 'coverage.json', |
| 'coverage' |
| ], |
| ), |
| ] |
| TESTDATA_2 = [ |
| ( |
| os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'), |
| ['qemu_x86'], |
| [ |
| 'GCOV_COVERAGE_DUMP_START', 'GCOV_COVERAGE_DUMP_END' |
| ], |
| ), |
| ] |
| TESTDATA_3 = [ |
| ( |
| os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'), |
| ['qemu_x86'], |
| [ |
| 'coverage.log', 'coverage.json', |
| 'coverage' |
| ], |
| r'{"files": \[], "gcovr/format_version": ".*"}' |
| ), |
| ] |
| TESTDATA_4 = [ |
| ( |
| 'gcovr', |
| [ |
| 'coverage.log', 'coverage.json', |
| 'coverage', os.path.join('coverage','coverage.xml') |
| ], |
| 'xml' |
| ), |
| ( |
| 'gcovr', |
| [ |
| 'coverage.log', 'coverage.json', |
| 'coverage', os.path.join('coverage','coverage.sonarqube.xml') |
| ], |
| 'sonarqube' |
| ), |
| ( |
| 'gcovr', |
| [ |
| 'coverage.log', 'coverage.json', |
| 'coverage', os.path.join('coverage','coverage.txt') |
| ], |
| 'txt' |
| ), |
| ( |
| 'gcovr', |
| [ |
| 'coverage.log', 'coverage.json', |
| 'coverage', os.path.join('coverage','coverage.csv') |
| ], |
| 'csv' |
| ), |
| ( |
| 'gcovr', |
| [ |
| 'coverage.log', 'coverage.json', |
| 'coverage', os.path.join('coverage','coverage.coveralls.json') |
| ], |
| 'coveralls' |
| ), |
| ( |
| 'gcovr', |
| [ |
| 'coverage.log', 'coverage.json', |
| 'coverage', os.path.join('coverage','index.html') |
| ], |
| 'html' |
| ), |
| ( |
| 'lcov', |
| [ |
| 'coverage.log', 'coverage.info', |
| 'ztest.info', 'coverage', |
| os.path.join('coverage','index.html') |
| ], |
| 'html' |
| ), |
| ( |
| 'lcov', |
| [ |
| 'coverage.log', 'coverage.info', |
| 'ztest.info' |
| ], |
| 'lcov' |
| ), |
| ] |
| TESTDATA_5 = [ |
| ( |
| os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'), |
| ['qemu_x86'], |
| 'gcovr', |
| 'Running gcovr -r' |
| ), |
| ( |
| os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'), |
| ['qemu_x86'], |
| 'lcov', |
| 'Running lcov --gcov-tool' |
| ) |
| ] |
| TESTDATA_6 = [ |
| ( |
| os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'), |
| ['qemu_x86'], |
| ['The specified file does not exist.', r'\[Errno 13\] Permission denied:'], |
| ) |
| ] |
| TESTDATA_7 = [ |
| ( |
| os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'), |
| ['qemu_x86_64', 'qemu_x86'], |
| ['qemu_x86_64', 'qemu_x86', ['qemu_x86_64', 'qemu_x86']], |
| ) |
| ] |
| |
| @classmethod |
| def setup_class(cls): |
| apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister') |
| cls.loader = importlib.machinery.SourceFileLoader('__main__', apath) |
| cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader) |
| cls.twister_module = importlib.util.module_from_spec(cls.spec) |
| |
| @pytest.mark.parametrize( |
| 'test_path, test_platforms, file_name', |
| TESTDATA_1, |
| ids=[ |
| 'coverage', |
| ] |
| ) |
| def test_coverage(self, capfd, test_path, test_platforms, out_path, file_name): |
| args = ['-i','--outdir', out_path, '-T', test_path] + \ |
| ['--coverage', '--coverage-tool', 'gcovr'] + \ |
| [val for pair in zip( |
| ['-p'] * len(test_platforms), test_platforms |
| ) for val in pair] |
| |
| with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ |
| pytest.raises(SystemExit) as sys_exit: |
| self.loader.exec_module(self.twister_module) |
| |
| out, err = capfd.readouterr() |
| sys.stdout.write(out) |
| sys.stderr.write(err) |
| |
| assert str(sys_exit.value) == '0' |
| |
| for f_name in file_name: |
| path = os.path.join(out_path, f_name) |
| assert os.path.exists(path), f'file not found {f_name}' |
| |
| @pytest.mark.parametrize( |
| 'test_path, test_platforms, expected', |
| TESTDATA_2, |
| ids=[ |
| 'enable_coverage', |
| ] |
| ) |
| def test_enable_coverage(self, capfd, test_path, test_platforms, out_path, expected): |
| args = ['-i','--outdir', out_path, '-T', test_path] + \ |
| ['--enable-coverage', '-vv'] + \ |
| [val for pair in zip( |
| ['-p'] * len(test_platforms), test_platforms |
| ) for val in pair] |
| |
| with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ |
| pytest.raises(SystemExit) as sys_exit: |
| self.loader.exec_module(self.twister_module) |
| |
| out, err = capfd.readouterr() |
| sys.stdout.write(out) |
| sys.stderr.write(err) |
| |
| assert str(sys_exit.value) == '0' |
| |
| for line in expected: |
| match = re.search(line, err) |
| assert match, f'line not found: {line}' |
| |
| @pytest.mark.parametrize( |
| 'test_path, test_platforms, file_name, expected_content', |
| TESTDATA_3, |
| ids=[ |
| 'coverage_basedir', |
| ] |
| ) |
| def test_coverage_basedir(self, capfd, test_path, test_platforms, out_path, file_name, expected_content): |
| base_dir = os.path.join(TEST_DATA, "test_dir") |
| if os.path.exists(base_dir): |
| os.rmdir(base_dir) |
| os.mkdir(base_dir) |
| args = ['--outdir', out_path,'-i', '-T', test_path] + \ |
| ['--coverage', '--coverage-tool', 'gcovr', '-v', '--coverage-basedir', base_dir] + \ |
| [val for pair in zip( |
| ['-p'] * len(test_platforms), test_platforms |
| ) for val in pair] |
| |
| with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ |
| pytest.raises(SystemExit) as sys_exit: |
| self.loader.exec_module(self.twister_module) |
| |
| out, err = capfd.readouterr() |
| sys.stdout.write(out) |
| sys.stderr.write(err) |
| |
| assert str(sys_exit.value) == '0' |
| |
| for f_name in file_name: |
| path = os.path.join(out_path, f_name) |
| assert os.path.exists(path), f'file not found {f_name}' |
| if f_name == 'coverage.json': |
| with open(path, "r") as json_file: |
| json_content = json.load(json_file) |
| pattern = re.compile(expected_content) |
| assert pattern.match(json.dumps(json_content)) |
| if os.path.exists(base_dir): |
| os.rmdir(base_dir) |
| |
| @pytest.mark.parametrize( |
| 'cov_tool, file_name, cov_format', |
| TESTDATA_4, |
| ids=[ |
| 'coverage_format gcovr xml', |
| 'coverage_format gcovr sonarqube', |
| 'coverage_format gcovr txt', |
| 'coverage_format gcovr csv', |
| 'coverage_format gcovr coveralls', |
| 'coverage_format gcovr html', |
| 'coverage_format lcov html', |
| 'coverage_format lcov lcov', |
| ] |
| ) |
| def test_coverage_format(self, capfd, out_path, cov_tool, file_name, cov_format): |
| test_path = os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2') |
| test_platforms = ['qemu_x86'] |
| args = ['--outdir', out_path,'-i', '-T', test_path] + \ |
| ['--coverage', '--coverage-tool', cov_tool, '--coverage-formats', cov_format, '-v'] + \ |
| [val for pair in zip( |
| ['-p'] * len(test_platforms), test_platforms |
| ) for val in pair] |
| |
| with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ |
| pytest.raises(SystemExit) as sys_exit: |
| self.loader.exec_module(self.twister_module) |
| |
| out, err = capfd.readouterr() |
| sys.stdout.write(out) |
| sys.stderr.write(err) |
| |
| assert str(sys_exit.value) == '0' |
| |
| for f_name in file_name: |
| path = os.path.join(out_path, f_name) |
| assert os.path.exists(path), f'file not found {f_name}, probably format {cov_format} not work properly' |
| |
| |
| |
| @pytest.mark.parametrize( |
| 'test_path, test_platforms, cov_tool, expected_content', |
| TESTDATA_5, |
| ids=[ |
| 'coverage_tool gcovr', |
| 'coverage_tool lcov' |
| ] |
| ) |
| def test_coverage_tool(self, capfd, caplog, test_path, test_platforms, out_path, cov_tool, expected_content): |
| args = ['--outdir', out_path,'-i', '-T', test_path] + \ |
| ['--coverage', '--coverage-tool', cov_tool, '-v'] + \ |
| [val for pair in zip( |
| ['-p'] * len(test_platforms), test_platforms |
| ) for val in pair] |
| |
| with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ |
| pytest.raises(SystemExit) as sys_exit: |
| self.loader.exec_module(self.twister_module) |
| |
| out, err = capfd.readouterr() |
| sys.stdout.write(out) |
| sys.stderr.write(err) |
| |
| assert str(sys_exit.value) == '0' |
| |
| assert re.search(expected_content, caplog.text), f'{cov_tool} line not found' |
| |
| @pytest.mark.parametrize( |
| 'test_path, test_platforms, expected_content', |
| TESTDATA_6, |
| ids=[ |
| 'missing tool' |
| ] |
| ) |
| def test_gcov_tool(self, capfd, test_path, test_platforms, out_path, expected_content): |
| args = ['--outdir', out_path, '-i', '-T', test_path] + \ |
| ['--coverage', '--gcov-tool', TEST_DATA, '-v'] + \ |
| [val for pair in zip( |
| ['-p'] * len(test_platforms), test_platforms |
| ) for val in pair] |
| |
| with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ |
| pytest.raises(SystemExit) as sys_exit: |
| self.loader.exec_module(self.twister_module) |
| |
| out, err = capfd.readouterr() |
| sys.stdout.write(out) |
| sys.stderr.write(err) |
| |
| assert str(sys_exit.value) == '1' |
| for line in expected_content: |
| result = re.search(line, err) |
| assert result, f'missing information in log: {line}' |
| |
| @pytest.mark.parametrize( |
| 'test_path, test_platforms, cov_platform', |
| TESTDATA_7, |
| ids=[ |
| 'coverage platform' |
| ] |
| ) |
| def test_coverage_platform(self, capfd, test_path, test_platforms, out_path, cov_platform): |
| def search_cov(): |
| pattern = r'TOTAL\s+(\d+)' |
| coverage_file_path = os.path.join(out_path, 'coverage', 'coverage.txt') |
| with open(coverage_file_path, 'r') as file: |
| data = file.read() |
| match = re.search(pattern, data) |
| if match: |
| total = int(match.group(1)) |
| return total |
| else: |
| print('Error, pattern not found') |
| |
| run = [] |
| for element in cov_platform: |
| args = ['--outdir', out_path, '-i', '-T', test_path] + \ |
| ['--coverage', '--coverage-formats', 'txt', '-v'] + \ |
| [val for pair in zip( |
| ['-p'] * len(test_platforms), test_platforms |
| ) for val in pair] |
| |
| if isinstance(element, list): |
| for nested_element in element: |
| args += ['--coverage-platform', nested_element] |
| else: |
| args += ['--coverage-platform', element] |
| |
| with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ |
| pytest.raises(SystemExit) as sys_exit: |
| self.loader.exec_module(self.twister_module) |
| |
| assert str(sys_exit.value) == '0' |
| |
| run += [search_cov()] |
| |
| capfd.readouterr() |
| clear_log_in_test() |
| |
| assert run[2] > run[0], 'Broader coverage platform selection did not result in broader coverage' |
| assert run[2] > run[1], 'Broader coverage platform selection did not result in broader coverage' |