Lukasz Mrugala | 3fb11e2 | 2024-02-21 12:36:15 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | # Copyright (c) 2024 Intel Corporation |
| 3 | # |
| 4 | # SPDX-License-Identifier: Apache-2.0 |
| 5 | """ |
| 6 | Blackbox tests for twister's command line functions concerning addons to normal functions |
| 7 | """ |
| 8 | |
| 9 | import importlib |
| 10 | import mock |
| 11 | import os |
| 12 | import pkg_resources |
| 13 | import pytest |
| 14 | import re |
| 15 | import shutil |
| 16 | import subprocess |
| 17 | import sys |
| 18 | |
| 19 | from conftest import ZEPHYR_BASE, TEST_DATA, sample_filename_mock, testsuite_filename_mock |
| 20 | from twisterlib.testplan import TestPlan |
| 21 | |
| 22 | |
| 23 | class TestAddon: |
| 24 | @classmethod |
| 25 | def setup_class(cls): |
| 26 | apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister') |
| 27 | cls.loader = importlib.machinery.SourceFileLoader('__main__', apath) |
| 28 | cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader) |
| 29 | cls.twister_module = importlib.util.module_from_spec(cls.spec) |
| 30 | |
| 31 | @classmethod |
| 32 | def teardown_class(cls): |
| 33 | pass |
| 34 | |
| 35 | @pytest.mark.parametrize( |
| 36 | 'ubsan_flags, expected_exit_value', |
| 37 | [ |
| 38 | # No sanitiser, no problem |
| 39 | ([], '0'), |
| 40 | # Sanitiser catches a mistake, error is raised |
| 41 | (['--enable-ubsan'], '1') |
| 42 | ], |
| 43 | ids=['no sanitiser', 'ubsan'] |
| 44 | ) |
| 45 | @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) |
| 46 | def test_enable_ubsan(self, out_path, ubsan_flags, expected_exit_value): |
| 47 | test_platforms = ['native_sim'] |
| 48 | test_path = os.path.join(TEST_DATA, 'tests', 'san', 'ubsan') |
| 49 | args = ['-i', '--outdir', out_path, '-T', test_path] + \ |
| 50 | ubsan_flags + \ |
| 51 | [] + \ |
| 52 | [val for pair in zip( |
| 53 | ['-p'] * len(test_platforms), test_platforms |
| 54 | ) for val in pair] |
| 55 | |
| 56 | with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ |
| 57 | pytest.raises(SystemExit) as sys_exit: |
| 58 | self.loader.exec_module(self.twister_module) |
| 59 | |
| 60 | assert str(sys_exit.value) == expected_exit_value |
| 61 | |
| 62 | @pytest.mark.parametrize( |
| 63 | 'lsan_flags, expected_exit_value', |
| 64 | [ |
| 65 | # No sanitiser, no problem |
| 66 | ([], '0'), |
| 67 | # Sanitiser catches a mistake, error is raised |
| 68 | (['--enable-asan', '--enable-lsan'], '1') |
| 69 | ], |
| 70 | ids=['no sanitiser', 'lsan'] |
| 71 | ) |
| 72 | @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) |
| 73 | def test_enable_lsan(self, out_path, lsan_flags, expected_exit_value): |
| 74 | test_platforms = ['native_sim'] |
| 75 | test_path = os.path.join(TEST_DATA, 'tests', 'san', 'lsan') |
| 76 | args = ['-i', '--outdir', out_path, '-T', test_path] + \ |
| 77 | lsan_flags + \ |
| 78 | [] + \ |
| 79 | [val for pair in zip( |
| 80 | ['-p'] * len(test_platforms), test_platforms |
| 81 | ) for val in pair] |
| 82 | |
| 83 | with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ |
| 84 | pytest.raises(SystemExit) as sys_exit: |
| 85 | self.loader.exec_module(self.twister_module) |
| 86 | |
| 87 | assert str(sys_exit.value) == expected_exit_value |
| 88 | |
| 89 | @pytest.mark.parametrize( |
| 90 | 'asan_flags, expected_exit_value, expect_asan', |
| 91 | [ |
| 92 | # No sanitiser, no problem |
| 93 | # Note that on some runs it may fail, |
| 94 | # as the process is killed instead of ending normally. |
| 95 | # This is not 100% repeatable, so this test is removed for now. |
| 96 | # ([], '0', False), |
| 97 | # Sanitiser catches a mistake, error is raised |
| 98 | (['--enable-asan'], '1', True) |
| 99 | ], |
| 100 | ids=[ |
| 101 | #'no sanitiser', |
| 102 | 'asan' |
| 103 | ] |
| 104 | ) |
| 105 | @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) |
| 106 | def test_enable_asan(self, capfd, out_path, asan_flags, expected_exit_value, expect_asan): |
| 107 | test_platforms = ['native_sim'] |
| 108 | test_path = os.path.join(TEST_DATA, 'tests', 'san', 'asan') |
| 109 | args = ['-i', '--outdir', out_path, '-T', test_path] + \ |
| 110 | asan_flags + \ |
| 111 | [] + \ |
| 112 | [val for pair in zip( |
| 113 | ['-p'] * len(test_platforms), test_platforms |
| 114 | ) for val in pair] |
| 115 | |
| 116 | with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ |
| 117 | pytest.raises(SystemExit) as sys_exit: |
| 118 | self.loader.exec_module(self.twister_module) |
| 119 | |
| 120 | assert str(sys_exit.value) == expected_exit_value |
| 121 | |
| 122 | out, err = capfd.readouterr() |
| 123 | sys.stdout.write(out) |
| 124 | sys.stderr.write(err) |
| 125 | |
| 126 | asan_template = r'^==\d+==ERROR:\s+AddressSanitizer:' |
| 127 | assert expect_asan == bool(re.search(asan_template, err, re.MULTILINE)) |
| 128 | |
| 129 | @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) |
| 130 | def test_extra_test_args(self, capfd, out_path): |
| 131 | test_platforms = ['native_sim'] |
| 132 | test_path = os.path.join(TEST_DATA, 'tests', 'params', 'dummy') |
| 133 | args = ['-i', '--outdir', out_path, '-T', test_path] + \ |
| 134 | [] + \ |
| 135 | ['-vvv'] + \ |
| 136 | [val for pair in zip( |
| 137 | ['-p'] * len(test_platforms), test_platforms |
| 138 | ) for val in pair] + \ |
| 139 | ['--', '-list'] |
| 140 | |
| 141 | with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ |
| 142 | pytest.raises(SystemExit) as sys_exit: |
| 143 | self.loader.exec_module(self.twister_module) |
| 144 | |
| 145 | # Use of -list makes tests not run. |
| 146 | # Thus, the tests 'failed'. |
| 147 | assert str(sys_exit.value) == '1' |
| 148 | |
| 149 | out, err = capfd.readouterr() |
| 150 | sys.stdout.write(out) |
| 151 | sys.stderr.write(err) |
| 152 | |
| 153 | expected_test_names = [ |
| 154 | 'param_tests::test_assert1', |
| 155 | 'param_tests::test_assert2', |
| 156 | 'param_tests::test_assert3', |
| 157 | ] |
| 158 | assert all([testname in err for testname in expected_test_names]) |
| 159 | |
| 160 | @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) |
| 161 | def test_extra_args(self, caplog, out_path): |
| 162 | test_platforms = ['qemu_x86', 'frdm_k64f'] |
| 163 | path = os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2') |
| 164 | args = ['--outdir', out_path, '-T', path] + \ |
| 165 | ['--extra-args', 'USE_CCACHE=0', '--extra-args', 'DUMMY=1'] + \ |
| 166 | [] + \ |
| 167 | [val for pair in zip( |
| 168 | ['-p'] * len(test_platforms), test_platforms |
| 169 | ) for val in pair] |
| 170 | |
| 171 | with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ |
| 172 | pytest.raises(SystemExit) as sys_exit: |
| 173 | self.loader.exec_module(self.twister_module) |
| 174 | |
| 175 | assert str(sys_exit.value) == '0' |
| 176 | |
| 177 | with open(os.path.join(out_path, 'twister.log')) as f: |
| 178 | twister_log = f.read() |
| 179 | |
| 180 | pattern_cache = r'Calling cmake: [^\n]+ -DUSE_CCACHE=0 [^\n]+\n' |
| 181 | pattern_dummy = r'Calling cmake: [^\n]+ -DDUMMY=1 [^\n]+\n' |
| 182 | |
| 183 | assert ' -DUSE_CCACHE=0 ' in twister_log |
| 184 | res = re.search(pattern_cache, twister_log) |
| 185 | assert res |
| 186 | |
| 187 | assert ' -DDUMMY=1 ' in twister_log |
| 188 | res = re.search(pattern_dummy, twister_log) |
| 189 | assert res |
| 190 | |
| 191 | # This test is not side-effect free. |
| 192 | # It installs and uninstalls pytest-twister-harness using pip |
| 193 | # It uses pip to check whether that plugin is previously installed |
| 194 | # and reinstalls it if detected at the start of its run. |
| 195 | # However, it does NOT restore the original plugin, ONLY reinstalls it. |
| 196 | @pytest.mark.parametrize( |
| 197 | 'allow_flags, do_install, expected_exit_value, expected_logs', |
| 198 | [ |
| 199 | ([], True, '1', ['By default Twister should work without pytest-twister-harness' |
| 200 | ' plugin being installed, so please, uninstall it by' |
| 201 | ' `pip uninstall pytest-twister-harness` and' |
| 202 | ' `git clean -dxf scripts/pylib/pytest-twister-harness`.']), |
| 203 | (['--allow-installed-plugin'], True, '0', ['You work with installed version' |
| 204 | ' of pytest-twister-harness plugin.']), |
| 205 | ([], False, '0', []), |
| 206 | (['--allow-installed-plugin'], False, '0', []), |
| 207 | ], |
| 208 | ids=['installed, but not allowed', 'installed, allowed', |
| 209 | 'not installed, not allowed', 'not installed, but allowed'] |
| 210 | ) |
| 211 | @mock.patch.object(TestPlan, 'SAMPLE_FILENAME', sample_filename_mock) |
| 212 | def test_allow_installed_plugin(self, caplog, out_path, allow_flags, do_install, |
| 213 | expected_exit_value, expected_logs): |
| 214 | environment_twister_module = importlib.import_module('twisterlib.environment') |
| 215 | harness_twister_module = importlib.import_module('twisterlib.harness') |
| 216 | runner_twister_module = importlib.import_module('twisterlib.runner') |
| 217 | |
| 218 | pth_path = os.path.join(ZEPHYR_BASE, 'scripts', 'pylib', 'pytest-twister-harness') |
| 219 | check_installed_command = [sys.executable, '-m', 'pip', 'list'] |
| 220 | install_command = [sys.executable, '-m', 'pip', 'install', '--no-input', pth_path] |
| 221 | uninstall_command = [sys.executable, '-m', 'pip', 'uninstall', '--yes', |
| 222 | 'pytest-twister-harness'] |
| 223 | |
| 224 | def big_uninstall(): |
| 225 | pth_path = os.path.join(ZEPHYR_BASE, 'scripts', 'pylib', 'pytest-twister-harness') |
| 226 | |
| 227 | subprocess.run(uninstall_command, check=True,) |
| 228 | |
| 229 | # For our registration to work, we have to delete the installation cache |
| 230 | additional_cache_paths = [ |
| 231 | # Plugin cache |
| 232 | os.path.join(pth_path, 'src', 'pytest_twister_harness.egg-info'), |
| 233 | # Additional caches |
| 234 | os.path.join(pth_path, 'src', 'pytest_twister_harness', '__pycache__'), |
| 235 | os.path.join(pth_path, 'src', 'pytest_twister_harness', 'device', '__pycache__'), |
| 236 | os.path.join(pth_path, 'src', 'pytest_twister_harness', 'helpers', '__pycache__'), |
| 237 | os.path.join(pth_path, 'src', 'pytest_twister_harness', 'build'), |
| 238 | ] |
| 239 | |
| 240 | for additional_cache_path in additional_cache_paths: |
| 241 | if os.path.exists(additional_cache_path): |
| 242 | if os.path.isfile(additional_cache_path): |
| 243 | os.unlink(additional_cache_path) |
| 244 | else: |
| 245 | shutil.rmtree(additional_cache_path) |
| 246 | |
| 247 | # To refresh the PYTEST_PLUGIN_INSTALLED global variable |
| 248 | def refresh_plugin_installed_variable(): |
| 249 | pkg_resources._initialize_master_working_set() |
| 250 | importlib.reload(environment_twister_module) |
| 251 | importlib.reload(harness_twister_module) |
| 252 | importlib.reload(runner_twister_module) |
| 253 | |
| 254 | check_installed_result = subprocess.run(check_installed_command, check=True, |
| 255 | capture_output=True, text=True) |
| 256 | previously_installed = 'pytest-twister-harness' in check_installed_result.stdout |
| 257 | |
| 258 | # To ensure consistent test start |
| 259 | big_uninstall() |
| 260 | |
| 261 | if do_install: |
| 262 | subprocess.run(install_command, check=True) |
| 263 | |
| 264 | # Refresh before the test, no matter the testcase |
| 265 | refresh_plugin_installed_variable() |
| 266 | |
| 267 | test_platforms = ['native_sim'] |
| 268 | test_path = os.path.join(TEST_DATA, 'samples', 'pytest', 'shell') |
| 269 | args = ['-i', '--outdir', out_path, '-T', test_path] + \ |
| 270 | allow_flags + \ |
| 271 | [] + \ |
| 272 | [val for pair in zip( |
| 273 | ['-p'] * len(test_platforms), test_platforms |
| 274 | ) for val in pair] |
| 275 | |
| 276 | with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ |
| 277 | pytest.raises(SystemExit) as sys_exit: |
| 278 | self.loader.exec_module(self.twister_module) |
| 279 | |
| 280 | # To ensure consistent test exit, prevent dehermetisation |
| 281 | if do_install: |
| 282 | big_uninstall() |
| 283 | |
| 284 | # To restore previously-installed plugin as well as we can |
| 285 | if previously_installed: |
| 286 | subprocess.run(install_command, check=True) |
| 287 | |
| 288 | if previously_installed or do_install: |
| 289 | refresh_plugin_installed_variable() |
| 290 | |
| 291 | assert str(sys_exit.value) == expected_exit_value |
| 292 | |
| 293 | assert all([log in caplog.text for log in expected_logs]) |
| 294 | |
| 295 | @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) |
| 296 | def test_pytest_args(self, out_path): |
| 297 | test_platforms = ['native_sim'] |
| 298 | test_path = os.path.join(TEST_DATA, 'tests', 'pytest') |
| 299 | args = ['-i', '--outdir', out_path, '-T', test_path] + \ |
| 300 | ['--pytest-args=--custom-pytest-arg', '--pytest-args=foo', |
| 301 | '--pytest-args=--cmdopt', '--pytest-args=.'] + \ |
| 302 | [] + \ |
| 303 | [val for pair in zip( |
| 304 | ['-p'] * len(test_platforms), test_platforms |
| 305 | ) for val in pair] |
| 306 | |
| 307 | with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ |
| 308 | pytest.raises(SystemExit) as sys_exit: |
| 309 | self.loader.exec_module(self.twister_module) |
| 310 | |
| 311 | # YAML was modified so that the test will fail without command line override. |
| 312 | assert str(sys_exit.value) == '0' |
| 313 | |
| 314 | @pytest.mark.parametrize( |
| 315 | 'valgrind_flags, expected_exit_value', |
| 316 | [ |
| 317 | # No sanitiser, leak is ignored |
| 318 | ([], '0'), |
| 319 | # Sanitiser catches a mistake, error is raised |
| 320 | (['--enable-valgrind'], '1') |
| 321 | ], |
| 322 | ids=['no valgrind', 'valgrind'] |
| 323 | ) |
| 324 | @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) |
| 325 | def test_enable_valgrind(self, capfd, out_path, valgrind_flags, expected_exit_value): |
| 326 | test_platforms = ['native_sim'] |
| 327 | test_path = os.path.join(TEST_DATA, 'tests', 'san', 'val') |
| 328 | args = ['-i', '--outdir', out_path, '-T', test_path] + \ |
| 329 | valgrind_flags + \ |
| 330 | [] + \ |
| 331 | [val for pair in zip( |
| 332 | ['-p'] * len(test_platforms), test_platforms |
| 333 | ) for val in pair] |
| 334 | |
| 335 | with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ |
| 336 | pytest.raises(SystemExit) as sys_exit: |
| 337 | self.loader.exec_module(self.twister_module) |
| 338 | |
| 339 | assert str(sys_exit.value) == expected_exit_value |