blob: 9bc0d50b9526874ee8648a889acfeaee5e2cd0fc [file] [log] [blame]
Lukasz Mrugala3fb11e22024-02-21 12:36:15 +00001#!/usr/bin/env python3
2# Copyright (c) 2024 Intel Corporation
3#
4# SPDX-License-Identifier: Apache-2.0
5"""
6Blackbox tests for twister's command line functions concerning addons to normal functions
7"""
8
9import importlib
10import mock
11import os
12import pkg_resources
13import pytest
14import re
15import shutil
16import subprocess
17import sys
18
19from conftest import ZEPHYR_BASE, TEST_DATA, sample_filename_mock, testsuite_filename_mock
20from twisterlib.testplan import TestPlan
21
22
23class TestAddon:
24 @classmethod
25 def setup_class(cls):
26 apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
27 cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
28 cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
29 cls.twister_module = importlib.util.module_from_spec(cls.spec)
30
31 @classmethod
32 def teardown_class(cls):
33 pass
34
35 @pytest.mark.parametrize(
36 'ubsan_flags, expected_exit_value',
37 [
38 # No sanitiser, no problem
39 ([], '0'),
40 # Sanitiser catches a mistake, error is raised
41 (['--enable-ubsan'], '1')
42 ],
43 ids=['no sanitiser', 'ubsan']
44 )
45 @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
46 def test_enable_ubsan(self, out_path, ubsan_flags, expected_exit_value):
47 test_platforms = ['native_sim']
48 test_path = os.path.join(TEST_DATA, 'tests', 'san', 'ubsan')
49 args = ['-i', '--outdir', out_path, '-T', test_path] + \
50 ubsan_flags + \
51 [] + \
52 [val for pair in zip(
53 ['-p'] * len(test_platforms), test_platforms
54 ) for val in pair]
55
56 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
57 pytest.raises(SystemExit) as sys_exit:
58 self.loader.exec_module(self.twister_module)
59
60 assert str(sys_exit.value) == expected_exit_value
61
62 @pytest.mark.parametrize(
63 'lsan_flags, expected_exit_value',
64 [
65 # No sanitiser, no problem
66 ([], '0'),
67 # Sanitiser catches a mistake, error is raised
68 (['--enable-asan', '--enable-lsan'], '1')
69 ],
70 ids=['no sanitiser', 'lsan']
71 )
72 @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
73 def test_enable_lsan(self, out_path, lsan_flags, expected_exit_value):
74 test_platforms = ['native_sim']
75 test_path = os.path.join(TEST_DATA, 'tests', 'san', 'lsan')
76 args = ['-i', '--outdir', out_path, '-T', test_path] + \
77 lsan_flags + \
78 [] + \
79 [val for pair in zip(
80 ['-p'] * len(test_platforms), test_platforms
81 ) for val in pair]
82
83 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
84 pytest.raises(SystemExit) as sys_exit:
85 self.loader.exec_module(self.twister_module)
86
87 assert str(sys_exit.value) == expected_exit_value
88
89 @pytest.mark.parametrize(
90 'asan_flags, expected_exit_value, expect_asan',
91 [
92 # No sanitiser, no problem
93 # Note that on some runs it may fail,
94 # as the process is killed instead of ending normally.
95 # This is not 100% repeatable, so this test is removed for now.
96 # ([], '0', False),
97 # Sanitiser catches a mistake, error is raised
98 (['--enable-asan'], '1', True)
99 ],
100 ids=[
101 #'no sanitiser',
102 'asan'
103 ]
104 )
105 @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
106 def test_enable_asan(self, capfd, out_path, asan_flags, expected_exit_value, expect_asan):
107 test_platforms = ['native_sim']
108 test_path = os.path.join(TEST_DATA, 'tests', 'san', 'asan')
109 args = ['-i', '--outdir', out_path, '-T', test_path] + \
110 asan_flags + \
111 [] + \
112 [val for pair in zip(
113 ['-p'] * len(test_platforms), test_platforms
114 ) for val in pair]
115
116 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
117 pytest.raises(SystemExit) as sys_exit:
118 self.loader.exec_module(self.twister_module)
119
120 assert str(sys_exit.value) == expected_exit_value
121
122 out, err = capfd.readouterr()
123 sys.stdout.write(out)
124 sys.stderr.write(err)
125
126 asan_template = r'^==\d+==ERROR:\s+AddressSanitizer:'
127 assert expect_asan == bool(re.search(asan_template, err, re.MULTILINE))
128
129 @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
130 def test_extra_test_args(self, capfd, out_path):
131 test_platforms = ['native_sim']
132 test_path = os.path.join(TEST_DATA, 'tests', 'params', 'dummy')
133 args = ['-i', '--outdir', out_path, '-T', test_path] + \
134 [] + \
135 ['-vvv'] + \
136 [val for pair in zip(
137 ['-p'] * len(test_platforms), test_platforms
138 ) for val in pair] + \
139 ['--', '-list']
140
141 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
142 pytest.raises(SystemExit) as sys_exit:
143 self.loader.exec_module(self.twister_module)
144
145 # Use of -list makes tests not run.
146 # Thus, the tests 'failed'.
147 assert str(sys_exit.value) == '1'
148
149 out, err = capfd.readouterr()
150 sys.stdout.write(out)
151 sys.stderr.write(err)
152
153 expected_test_names = [
154 'param_tests::test_assert1',
155 'param_tests::test_assert2',
156 'param_tests::test_assert3',
157 ]
158 assert all([testname in err for testname in expected_test_names])
159
160 @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
161 def test_extra_args(self, caplog, out_path):
162 test_platforms = ['qemu_x86', 'frdm_k64f']
163 path = os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic', 'group2')
164 args = ['--outdir', out_path, '-T', path] + \
165 ['--extra-args', 'USE_CCACHE=0', '--extra-args', 'DUMMY=1'] + \
166 [] + \
167 [val for pair in zip(
168 ['-p'] * len(test_platforms), test_platforms
169 ) for val in pair]
170
171 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
172 pytest.raises(SystemExit) as sys_exit:
173 self.loader.exec_module(self.twister_module)
174
175 assert str(sys_exit.value) == '0'
176
177 with open(os.path.join(out_path, 'twister.log')) as f:
178 twister_log = f.read()
179
180 pattern_cache = r'Calling cmake: [^\n]+ -DUSE_CCACHE=0 [^\n]+\n'
181 pattern_dummy = r'Calling cmake: [^\n]+ -DDUMMY=1 [^\n]+\n'
182
183 assert ' -DUSE_CCACHE=0 ' in twister_log
184 res = re.search(pattern_cache, twister_log)
185 assert res
186
187 assert ' -DDUMMY=1 ' in twister_log
188 res = re.search(pattern_dummy, twister_log)
189 assert res
190
191 # This test is not side-effect free.
192 # It installs and uninstalls pytest-twister-harness using pip
193 # It uses pip to check whether that plugin is previously installed
194 # and reinstalls it if detected at the start of its run.
195 # However, it does NOT restore the original plugin, ONLY reinstalls it.
196 @pytest.mark.parametrize(
197 'allow_flags, do_install, expected_exit_value, expected_logs',
198 [
199 ([], True, '1', ['By default Twister should work without pytest-twister-harness'
200 ' plugin being installed, so please, uninstall it by'
201 ' `pip uninstall pytest-twister-harness` and'
202 ' `git clean -dxf scripts/pylib/pytest-twister-harness`.']),
203 (['--allow-installed-plugin'], True, '0', ['You work with installed version'
204 ' of pytest-twister-harness plugin.']),
205 ([], False, '0', []),
206 (['--allow-installed-plugin'], False, '0', []),
207 ],
208 ids=['installed, but not allowed', 'installed, allowed',
209 'not installed, not allowed', 'not installed, but allowed']
210 )
211 @mock.patch.object(TestPlan, 'SAMPLE_FILENAME', sample_filename_mock)
212 def test_allow_installed_plugin(self, caplog, out_path, allow_flags, do_install,
213 expected_exit_value, expected_logs):
214 environment_twister_module = importlib.import_module('twisterlib.environment')
215 harness_twister_module = importlib.import_module('twisterlib.harness')
216 runner_twister_module = importlib.import_module('twisterlib.runner')
217
218 pth_path = os.path.join(ZEPHYR_BASE, 'scripts', 'pylib', 'pytest-twister-harness')
219 check_installed_command = [sys.executable, '-m', 'pip', 'list']
220 install_command = [sys.executable, '-m', 'pip', 'install', '--no-input', pth_path]
221 uninstall_command = [sys.executable, '-m', 'pip', 'uninstall', '--yes',
222 'pytest-twister-harness']
223
224 def big_uninstall():
225 pth_path = os.path.join(ZEPHYR_BASE, 'scripts', 'pylib', 'pytest-twister-harness')
226
227 subprocess.run(uninstall_command, check=True,)
228
229 # For our registration to work, we have to delete the installation cache
230 additional_cache_paths = [
231 # Plugin cache
232 os.path.join(pth_path, 'src', 'pytest_twister_harness.egg-info'),
233 # Additional caches
234 os.path.join(pth_path, 'src', 'pytest_twister_harness', '__pycache__'),
235 os.path.join(pth_path, 'src', 'pytest_twister_harness', 'device', '__pycache__'),
236 os.path.join(pth_path, 'src', 'pytest_twister_harness', 'helpers', '__pycache__'),
237 os.path.join(pth_path, 'src', 'pytest_twister_harness', 'build'),
238 ]
239
240 for additional_cache_path in additional_cache_paths:
241 if os.path.exists(additional_cache_path):
242 if os.path.isfile(additional_cache_path):
243 os.unlink(additional_cache_path)
244 else:
245 shutil.rmtree(additional_cache_path)
246
247 # To refresh the PYTEST_PLUGIN_INSTALLED global variable
248 def refresh_plugin_installed_variable():
249 pkg_resources._initialize_master_working_set()
250 importlib.reload(environment_twister_module)
251 importlib.reload(harness_twister_module)
252 importlib.reload(runner_twister_module)
253
254 check_installed_result = subprocess.run(check_installed_command, check=True,
255 capture_output=True, text=True)
256 previously_installed = 'pytest-twister-harness' in check_installed_result.stdout
257
258 # To ensure consistent test start
259 big_uninstall()
260
261 if do_install:
262 subprocess.run(install_command, check=True)
263
264 # Refresh before the test, no matter the testcase
265 refresh_plugin_installed_variable()
266
267 test_platforms = ['native_sim']
268 test_path = os.path.join(TEST_DATA, 'samples', 'pytest', 'shell')
269 args = ['-i', '--outdir', out_path, '-T', test_path] + \
270 allow_flags + \
271 [] + \
272 [val for pair in zip(
273 ['-p'] * len(test_platforms), test_platforms
274 ) for val in pair]
275
276 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
277 pytest.raises(SystemExit) as sys_exit:
278 self.loader.exec_module(self.twister_module)
279
280 # To ensure consistent test exit, prevent dehermetisation
281 if do_install:
282 big_uninstall()
283
284 # To restore previously-installed plugin as well as we can
285 if previously_installed:
286 subprocess.run(install_command, check=True)
287
288 if previously_installed or do_install:
289 refresh_plugin_installed_variable()
290
291 assert str(sys_exit.value) == expected_exit_value
292
293 assert all([log in caplog.text for log in expected_logs])
294
295 @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
296 def test_pytest_args(self, out_path):
297 test_platforms = ['native_sim']
298 test_path = os.path.join(TEST_DATA, 'tests', 'pytest')
299 args = ['-i', '--outdir', out_path, '-T', test_path] + \
300 ['--pytest-args=--custom-pytest-arg', '--pytest-args=foo',
301 '--pytest-args=--cmdopt', '--pytest-args=.'] + \
302 [] + \
303 [val for pair in zip(
304 ['-p'] * len(test_platforms), test_platforms
305 ) for val in pair]
306
307 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
308 pytest.raises(SystemExit) as sys_exit:
309 self.loader.exec_module(self.twister_module)
310
311 # YAML was modified so that the test will fail without command line override.
312 assert str(sys_exit.value) == '0'
313
314 @pytest.mark.parametrize(
315 'valgrind_flags, expected_exit_value',
316 [
317 # No sanitiser, leak is ignored
318 ([], '0'),
319 # Sanitiser catches a mistake, error is raised
320 (['--enable-valgrind'], '1')
321 ],
322 ids=['no valgrind', 'valgrind']
323 )
324 @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
325 def test_enable_valgrind(self, capfd, out_path, valgrind_flags, expected_exit_value):
326 test_platforms = ['native_sim']
327 test_path = os.path.join(TEST_DATA, 'tests', 'san', 'val')
328 args = ['-i', '--outdir', out_path, '-T', test_path] + \
329 valgrind_flags + \
330 [] + \
331 [val for pair in zip(
332 ['-p'] * len(test_platforms), test_platforms
333 ) for val in pair]
334
335 with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
336 pytest.raises(SystemExit) as sys_exit:
337 self.loader.exec_module(self.twister_module)
338
339 assert str(sys_exit.value) == expected_exit_value