scripts/tests/twister_blackbox/test_coverage.py

#!/usr/bin/env python3
# Copyright (c) 2024 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
"""
Blackbox tests for twister's command line functions
"""
import importlib
import re
import mock
import os
import pytest
import sys
import json

from conftest import TEST_DATA, ZEPHYR_BASE, testsuite_filename_mock, clear_log_in_test
from twisterlib.testplan import TestPlan


@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
class TestCoverage:
    TESTDATA_1 = [
        (
            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'),
            ['qemu_x86'],
            [
                'coverage.log', 'coverage.json',
                'coverage'
            ],
        ),
    ]
    TESTDATA_2 = [
        (
            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'),
            ['qemu_x86'],
            [
                'GCOV_COVERAGE_DUMP_START', 'GCOV_COVERAGE_DUMP_END'
            ],
        ),
    ]
    TESTDATA_3 = [
        (
            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'),
            ['qemu_x86'],
            [
                'coverage.log', 'coverage.json',
                'coverage'
            ],
            r'{"files": \[], "gcovr/format_version": ".*"}'
        ),
    ]
    TESTDATA_4 = [
        (
            'gcovr',
            [
                'coverage.log', 'coverage.json',
                'coverage', os.path.join('coverage','coverage.xml')
            ],
            'xml'
        ),
        (
            'gcovr',
            [
                'coverage.log', 'coverage.json',
                'coverage', os.path.join('coverage','coverage.sonarqube.xml')
            ],
            'sonarqube'
        ),
        (
            'gcovr',
            [
                'coverage.log', 'coverage.json',
                'coverage', os.path.join('coverage','coverage.txt')
            ],
            'txt'
        ),
        (
            'gcovr',
            [
                'coverage.log', 'coverage.json',
                'coverage', os.path.join('coverage','coverage.csv')
            ],
            'csv'
        ),
        (
            'gcovr',
            [
                'coverage.log', 'coverage.json',
                'coverage', os.path.join('coverage','coverage.coveralls.json')
            ],
            'coveralls'
        ),
        (
            'gcovr',
            [
                'coverage.log', 'coverage.json',
                'coverage', os.path.join('coverage','index.html')
            ],
            'html'
        ),
        (
            'lcov',
            [
                'coverage.log', 'coverage.info',
                'ztest.info', 'coverage',
                os.path.join('coverage','index.html')
            ],
            'html'
        ),
        (
            'lcov',
            [
                'coverage.log', 'coverage.info',
                'ztest.info'
            ],
            'lcov'
        ),
    ]
    TESTDATA_5 = [
        (
            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'),
            ['qemu_x86'],
            'gcovr',
            'Running gcovr -r'
        ),
        (
            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'),
            ['qemu_x86'],
            'lcov',
            'Running lcov --gcov-tool'
        )
    ]
    TESTDATA_6 = [
        (
            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'),
            ['qemu_x86'],
            ['The specified file does not exist.', r'\[Errno 13\] Permission denied:'],
        )
    ]
    TESTDATA_7 = [
        (
            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2'),
            ['qemu_x86_64', 'qemu_x86'],
            ['qemu_x86_64', 'qemu_x86', ['qemu_x86_64', 'qemu_x86']],
        )
    ]

    @classmethod
    def setup_class(cls):
        apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
        cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
        cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
        cls.twister_module = importlib.util.module_from_spec(cls.spec)

    @pytest.mark.parametrize(
        'test_path, test_platforms, file_name',
        TESTDATA_1,
        ids=[
            'coverage',
        ]
    )
    def test_coverage(self, capfd, test_path, test_platforms, out_path, file_name):
        args = ['-i','--outdir', out_path, '-T', test_path] + \
               ['--coverage', '--coverage-tool', 'gcovr'] + \
               [val for pair in zip(
                   ['-p'] * len(test_platforms), test_platforms
               ) for val in pair]

        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
                pytest.raises(SystemExit) as sys_exit:
            self.loader.exec_module(self.twister_module)

        out, err = capfd.readouterr()
        sys.stdout.write(out)
        sys.stderr.write(err)

        assert str(sys_exit.value) == '0'

        for f_name in file_name:
            path = os.path.join(out_path, f_name)
            assert os.path.exists(path), f'file not found {f_name}'

    @pytest.mark.parametrize(
        'test_path, test_platforms, expected',
        TESTDATA_2,
        ids=[
            'enable_coverage',
        ]
    )
    def test_enable_coverage(self, capfd, test_path, test_platforms, out_path, expected):
        args = ['-i','--outdir', out_path, '-T', test_path] + \
               ['--enable-coverage', '-vv'] + \
               [val for pair in zip(
                   ['-p'] * len(test_platforms), test_platforms
               ) for val in pair]

        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
                pytest.raises(SystemExit) as sys_exit:
            self.loader.exec_module(self.twister_module)

        out, err = capfd.readouterr()
        sys.stdout.write(out)
        sys.stderr.write(err)

        assert str(sys_exit.value) == '0'

        for line in expected:
            match = re.search(line, err)
            assert match, f'line not found: {line}'

    @pytest.mark.parametrize(
        'test_path, test_platforms, file_name, expected_content',
        TESTDATA_3,
        ids=[
            'coverage_basedir',
        ]
    )
    def test_coverage_basedir(self, capfd, test_path, test_platforms, out_path, file_name, expected_content):
        base_dir = os.path.join(TEST_DATA, "test_dir")
        if os.path.exists(base_dir):
            os.rmdir(base_dir)
        os.mkdir(base_dir)
        args = ['--outdir', out_path,'-i', '-T', test_path] + \
               ['--coverage', '--coverage-tool', 'gcovr',  '-v', '--coverage-basedir', base_dir] + \
               [val for pair in zip(
                   ['-p'] * len(test_platforms), test_platforms
               ) for val in pair]

        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
                pytest.raises(SystemExit) as sys_exit:
            self.loader.exec_module(self.twister_module)

        out, err = capfd.readouterr()
        sys.stdout.write(out)
        sys.stderr.write(err)

        assert str(sys_exit.value) == '0'

        for f_name in file_name:
            path = os.path.join(out_path, f_name)
            assert os.path.exists(path), f'file not found {f_name}'
            if f_name == 'coverage.json':
                with open(path, "r") as json_file:
                    json_content = json.load(json_file)
                    pattern = re.compile(expected_content)
                    assert pattern.match(json.dumps(json_content))
        if os.path.exists(base_dir):
            os.rmdir(base_dir)

    @pytest.mark.parametrize(
        'cov_tool, file_name, cov_format',
        TESTDATA_4,
        ids=[
            'coverage_format gcovr xml',
            'coverage_format gcovr sonarqube',
            'coverage_format gcovr txt',
            'coverage_format gcovr csv',
            'coverage_format gcovr coveralls',
            'coverage_format gcovr html',
            'coverage_format lcov html',
            'coverage_format lcov lcov',
        ]
    )
    def test_coverage_format(self, capfd, out_path, cov_tool, file_name, cov_format):
        test_path = os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2')
        test_platforms = ['qemu_x86']
        args = ['--outdir', out_path,'-i', '-T', test_path] + \
               ['--coverage', '--coverage-tool', cov_tool, '--coverage-formats', cov_format, '-v'] + \
               [val for pair in zip(
                   ['-p'] * len(test_platforms), test_platforms
               ) for val in pair]

        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
                pytest.raises(SystemExit) as sys_exit:
            self.loader.exec_module(self.twister_module)

        out, err = capfd.readouterr()
        sys.stdout.write(out)
        sys.stderr.write(err)

        assert str(sys_exit.value) == '0'

        for f_name in file_name:
            path = os.path.join(out_path, f_name)
            assert os.path.exists(path), f'file not found {f_name}, probably format {cov_format} not work properly'



    @pytest.mark.parametrize(
        'test_path, test_platforms, cov_tool, expected_content',
        TESTDATA_5,
        ids=[
            'coverage_tool gcovr',
            'coverage_tool lcov'
        ]
    )
    def test_coverage_tool(self, capfd, caplog, test_path, test_platforms, out_path, cov_tool, expected_content):
        args = ['--outdir', out_path,'-i', '-T', test_path] + \
            ['--coverage', '--coverage-tool', cov_tool,  '-v'] + \
               [val for pair in zip(
                   ['-p'] * len(test_platforms), test_platforms
               ) for val in pair]

        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
                pytest.raises(SystemExit) as sys_exit:
            self.loader.exec_module(self.twister_module)

        out, err = capfd.readouterr()
        sys.stdout.write(out)
        sys.stderr.write(err)

        assert str(sys_exit.value) == '0'

        assert re.search(expected_content, caplog.text), f'{cov_tool} line not found'

    @pytest.mark.parametrize(
        'test_path, test_platforms, expected_content',
        TESTDATA_6,
        ids=[
            'missing tool'
        ]
    )
    def test_gcov_tool(self, capfd, test_path, test_platforms, out_path, expected_content):
        args = ['--outdir', out_path, '-i', '-T', test_path] + \
            ['--coverage', '--gcov-tool', TEST_DATA, '-v'] + \
               [val for pair in zip(
                   ['-p'] * len(test_platforms), test_platforms
               ) for val in pair]

        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
                pytest.raises(SystemExit) as sys_exit:
            self.loader.exec_module(self.twister_module)

        out, err = capfd.readouterr()
        sys.stdout.write(out)
        sys.stderr.write(err)

        assert str(sys_exit.value) == '1'
        for line in expected_content:
            result = re.search(line, err)
            assert result, f'missing information in log: {line}'

    @pytest.mark.parametrize(
        'test_path, test_platforms, cov_platform',
        TESTDATA_7,
        ids=[
            'coverage platform'
        ]
    )
    def test_coverage_platform(self, capfd, test_path, test_platforms, out_path, cov_platform):
        def search_cov():
            pattern = r'TOTAL\s+(\d+)'
            coverage_file_path = os.path.join(out_path, 'coverage', 'coverage.txt')
            with open(coverage_file_path, 'r') as file:
                data = file.read()
            match = re.search(pattern, data)
            if match:
                total = int(match.group(1))
                return total
            else:
                print('Error, pattern not found')

        run = []
        for element in cov_platform:
            args = ['--outdir', out_path, '-i', '-T', test_path] + \
               ['--coverage', '--coverage-formats', 'txt', '-v'] + \
               [val for pair in zip(
                   ['-p'] * len(test_platforms), test_platforms
               ) for val in pair]

            if isinstance(element, list):
                for nested_element in element:
                    args += ['--coverage-platform', nested_element]
            else:
                args += ['--coverage-platform', element]

            with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
                    pytest.raises(SystemExit) as sys_exit:
                self.loader.exec_module(self.twister_module)

            assert str(sys_exit.value) == '0'

            run += [search_cov()]

            capfd.readouterr()
            clear_log_in_test()

        assert run[2] > run[0], 'Broader coverage platform selection did not result in broader coverage'
        assert run[2] > run[1], 'Broader coverage platform selection did not result in broader coverage'