| #!/usr/bin/env python3 |
| # Copyright (c) 2023 Intel Corporation |
| # |
| # SPDX-License-Identifier: Apache-2.0 |
| """ |
| Tests for environment.py classes' methods |
| """ |
| |
| import mock |
| import os |
| import pytest |
| import shutil |
| |
| from contextlib import nullcontext |
| |
| import twisterlib.environment |
| |
| |
| TESTDATA_1 = [ |
| ( |
| None, |
| None, |
| None, |
| ['--short-build-path', '-k'], |
| '--short-build-path requires Ninja to be enabled' |
| ), |
| ( |
| 'nt', |
| None, |
| None, |
| ['--device-serial-pty', 'dummy'], |
| '--device-serial-pty is not supported on Windows OS' |
| ), |
| ( |
| None, |
| None, |
| None, |
| ['--west-runner=dummy'], |
| 'west-runner requires west-flash to be enabled' |
| ), |
| ( |
| None, |
| None, |
| None, |
| ['--west-flash=\"--board-id=dummy\"'], |
| 'west-flash requires device-testing to be enabled' |
| ), |
| ( |
| None, |
| { |
| 'exist': [], |
| 'missing': ['valgrind'] |
| }, |
| None, |
| ['--enable-valgrind'], |
| 'valgrind enabled but valgrind executable not found' |
| ), |
| ( |
| None, |
| None, |
| None, |
| [ |
| '--device-testing', |
| '--device-serial', |
| 'dummy', |
| ], |
| 'When --device-testing is used with --device-serial' \ |
| ' or --device-serial-pty, exactly one platform must' \ |
| ' be specified' |
| ), |
| ( |
| None, |
| None, |
| None, |
| [ |
| '--device-testing', |
| '--device-serial', |
| 'dummy', |
| '--platform', |
| 'dummy_platform1', |
| '--platform', |
| 'dummy_platform2' |
| ], |
| 'When --device-testing is used with --device-serial' \ |
| ' or --device-serial-pty, exactly one platform must' \ |
| ' be specified' |
| ), |
| # Note the underscore. |
| ( |
| None, |
| None, |
| None, |
| ['--device-flash-with-test'], |
| '--device-flash-with-test requires --device_testing' |
| ), |
| ( |
| None, |
| None, |
| None, |
| ['--shuffle-tests'], |
| '--shuffle-tests requires --subset' |
| ), |
| ( |
| None, |
| None, |
| None, |
| ['--shuffle-tests-seed', '0'], |
| '--shuffle-tests-seed requires --shuffle-tests' |
| ), |
| ( |
| None, |
| None, |
| None, |
| ['/dummy/unrecognised/arg'], |
| 'Unrecognized arguments found: \'/dummy/unrecognised/arg\'.' \ |
| ' Use -- to delineate extra arguments for test binary' \ |
| ' or pass -h for help.' |
| ), |
| ( |
| None, |
| None, |
| True, |
| [], |
| 'By default Twister should work without pytest-twister-harness' \ |
| ' plugin being installed, so please, uninstall it by' \ |
| ' `pip uninstall pytest-twister-harness` and' \ |
| ' `git clean -dxf scripts/pylib/pytest-twister-harness`.' |
| ), |
| ] |
| |
| |
| @pytest.mark.parametrize( |
| 'os_name, which_dict, pytest_plugin, args, expected_error', |
| TESTDATA_1, |
| ids=[ |
| 'short build path without ninja', |
| 'device-serial-pty on Windows', |
| 'west runner without west flash', |
| 'west-flash without device-testing', |
| 'valgrind without executable', |
| 'device serial without platform', |
| 'device serial with multiple platforms', |
| 'device flash with test without device testing', |
| 'shuffle-tests without subset', |
| 'shuffle-tests-seed without shuffle-tests', |
| 'unrecognised argument', |
| 'pytest-twister-harness installed' |
| ] |
| ) |
| def test_parse_arguments_errors( |
| caplog, |
| os_name, |
| which_dict, |
| pytest_plugin, |
| args, |
| expected_error |
| ): |
| def mock_which(name): |
| if name in which_dict['missing']: |
| return False |
| elif name in which_dict['exist']: |
| return which_dict['path'][which_dict['exist']] \ |
| if which_dict['path'][which_dict['exist']] \ |
| else f'dummy/path/{name}' |
| else: |
| return f'dummy/path/{name}' |
| |
| with mock.patch('sys.argv', ['twister'] + args): |
| parser = twisterlib.environment.add_parse_arguments() |
| |
| if which_dict: |
| which_dict['path'] = {name: shutil.which(name) \ |
| for name in which_dict['exist']} |
| which_mock = mock.Mock(side_effect=mock_which) |
| |
| with mock.patch('os.name', os_name) \ |
| if os_name is not None else nullcontext(), \ |
| mock.patch('shutil.which', which_mock) \ |
| if which_dict else nullcontext(), \ |
| mock.patch('twisterlib.environment' \ |
| '.PYTEST_PLUGIN_INSTALLED', pytest_plugin) \ |
| if pytest_plugin is not None else nullcontext(): |
| with pytest.raises(SystemExit) as exit_info: |
| twisterlib.environment.parse_arguments(parser, args) |
| |
| assert exit_info.value.code == 1 |
| assert expected_error in ' '.join(caplog.text.split()) |
| |
| |
| def test_parse_arguments_errors_size(): |
| """`options.size` is not an error, rather a different functionality.""" |
| |
| args = ['--size', 'dummy.elf'] |
| |
| with mock.patch('sys.argv', ['twister'] + args): |
| parser = twisterlib.environment.add_parse_arguments() |
| |
| mock_calc_parent = mock.Mock() |
| mock_calc_parent.child = mock.Mock(return_value=mock.Mock()) |
| |
| def mock_calc(*args, **kwargs): |
| return mock_calc_parent.child(args, kwargs) |
| |
| with mock.patch('twisterlib.size_calc.SizeCalculator', mock_calc): |
| with pytest.raises(SystemExit) as exit_info: |
| twisterlib.environment.parse_arguments(parser, args) |
| |
| assert exit_info.value.code == 0 |
| |
| mock_calc_parent.child.assert_has_calls([mock.call(('dummy.elf', []), {})]) |
| mock_calc_parent.child().size_report.assert_has_calls([mock.call()]) |
| |
| |
| def test_parse_arguments_warnings(caplog): |
| args = ['--allow-installed-plugin'] |
| |
| with mock.patch('sys.argv', ['twister'] + args): |
| parser = twisterlib.environment.add_parse_arguments() |
| |
| with mock.patch('twisterlib.environment.PYTEST_PLUGIN_INSTALLED', True): |
| twisterlib.environment.parse_arguments(parser, args) |
| |
| assert 'You work with installed version of' \ |
| ' pytest-twister-harness plugin.' in ' '.join(caplog.text.split()) |
| |
| |
| TESTDATA_2 = [ |
| (['--enable-size-report']), |
| (['--compare-report', 'dummy']), |
| ] |
| |
| |
| @pytest.mark.parametrize( |
| 'additional_args', |
| TESTDATA_2, |
| ids=['show footprint', 'compare report'] |
| ) |
| def test_parse_arguments(zephyr_base, additional_args): |
| args = ['--coverage', '--platform', 'dummy_platform'] + \ |
| additional_args + ['--', 'dummy_extra_1', 'dummy_extra_2'] |
| |
| with mock.patch('sys.argv', ['twister'] + args): |
| parser = twisterlib.environment.add_parse_arguments() |
| |
| options = twisterlib.environment.parse_arguments(parser, args) |
| |
| assert os.path.join(zephyr_base, 'tests') in options.testsuite_root |
| assert os.path.join(zephyr_base, 'samples') in options.testsuite_root |
| |
| assert options.enable_size_report |
| |
| assert options.enable_coverage |
| |
| assert options.coverage_platform == ['dummy_platform'] |
| |
| assert options.extra_test_args == ['dummy_extra_1', 'dummy_extra_2'] |
| |
| |
| TESTDATA_3 = [ |
| ( |
| None, |
| mock.Mock( |
| generator_cmd='make', |
| generator='Unix Makefiles', |
| test_roots=None, |
| board_roots=None, |
| outdir=None, |
| ) |
| ), |
| ( |
| mock.Mock( |
| ninja=True, |
| board_root=['dummy1', 'dummy2'], |
| testsuite_root=[ |
| os.path.join('dummy', 'path', "tests"), |
| os.path.join('dummy', 'path', "samples") |
| ], |
| outdir='dummy_abspath', |
| ), |
| mock.Mock( |
| generator_cmd='ninja', |
| generator='Ninja', |
| test_roots=[ |
| os.path.join('dummy', 'path', "tests"), |
| os.path.join('dummy', 'path', "samples") |
| ], |
| board_roots=['dummy1', 'dummy2'], |
| outdir='dummy_abspath', |
| ) |
| ), |
| ( |
| mock.Mock( |
| ninja=False, |
| board_root='dummy0', |
| testsuite_root=[ |
| os.path.join('dummy', 'path', "tests"), |
| os.path.join('dummy', 'path', "samples") |
| ], |
| outdir='dummy_abspath', |
| ), |
| mock.Mock( |
| generator_cmd='make', |
| generator='Unix Makefiles', |
| test_roots=[ |
| os.path.join('dummy', 'path', "tests"), |
| os.path.join('dummy', 'path', "samples") |
| ], |
| board_roots=['dummy0'], |
| outdir='dummy_abspath', |
| ) |
| ), |
| ] |
| |
| |
| @pytest.mark.parametrize( |
| 'options, expected_env', |
| TESTDATA_3, |
| ids=[ |
| 'no options', |
| 'ninja', |
| 'make' |
| ] |
| ) |
| def test_twisterenv_init(options, expected_env): |
| original_abspath = os.path.abspath |
| |
| def mocked_abspath(path): |
| if path == 'dummy_abspath': |
| return 'dummy_abspath' |
| elif isinstance(path, mock.Mock): |
| return None |
| else: |
| return original_abspath(path) |
| |
| with mock.patch('os.path.abspath', side_effect=mocked_abspath): |
| twister_env = twisterlib.environment.TwisterEnv(options=options) |
| |
| assert twister_env.generator_cmd == expected_env.generator_cmd |
| assert twister_env.generator == expected_env.generator |
| |
| assert twister_env.test_roots == expected_env.test_roots |
| |
| assert twister_env.board_roots == expected_env.board_roots |
| assert twister_env.outdir == expected_env.outdir |
| |
| |
| def test_twisterenv_discover(): |
| options = mock.Mock( |
| ninja=True |
| ) |
| |
| original_abspath = os.path.abspath |
| |
| def mocked_abspath(path): |
| if path == 'dummy_abspath': |
| return 'dummy_abspath' |
| elif isinstance(path, mock.Mock): |
| return None |
| else: |
| return original_abspath(path) |
| |
| with mock.patch('os.path.abspath', side_effect=mocked_abspath): |
| twister_env = twisterlib.environment.TwisterEnv(options=options) |
| |
| mock_datetime = mock.Mock( |
| now=mock.Mock( |
| return_value=mock.Mock( |
| isoformat=mock.Mock(return_value='dummy_time') |
| ) |
| ) |
| ) |
| |
| with mock.patch.object( |
| twisterlib.environment.TwisterEnv, |
| 'check_zephyr_version', |
| mock.Mock()) as mock_czv, \ |
| mock.patch.object( |
| twisterlib.environment.TwisterEnv, |
| 'get_toolchain', |
| mock.Mock()) as mock_gt, \ |
| mock.patch('twisterlib.environment.datetime', mock_datetime): |
| twister_env.discover() |
| |
| mock_czv.assert_called_once() |
| mock_gt.assert_called_once() |
| assert twister_env.run_date == 'dummy_time' |
| |
| |
| TESTDATA_4 = [ |
| ( |
| mock.Mock(returncode=0, stdout='dummy stdout version'), |
| mock.Mock(returncode=0, stdout='dummy stdout date'), |
| ['Zephyr version: dummy stdout version'], |
| 'dummy stdout version', |
| 'dummy stdout date' |
| ), |
| ( |
| mock.Mock(returncode=0, stdout=''), |
| mock.Mock(returncode=0, stdout='dummy stdout date'), |
| ['Could not determine version'], |
| 'Unknown', |
| 'dummy stdout date' |
| ), |
| ( |
| mock.Mock(returncode=1, stdout='dummy stdout version'), |
| mock.Mock(returncode=0, stdout='dummy stdout date'), |
| ['Could not determine version'], |
| 'Unknown', |
| 'dummy stdout date' |
| ), |
| ( |
| OSError, |
| mock.Mock(returncode=1), |
| ['Could not determine version'], |
| 'Unknown', |
| 'Unknown' |
| ), |
| ] |
| |
| |
| @pytest.mark.parametrize( |
| 'git_describe_return, git_show_return, expected_logs,' \ |
| ' expected_version, expected_commit_date', |
| TESTDATA_4, |
| ids=[ |
| 'valid', |
| 'no zephyr version on describe', |
| 'error on git describe', |
| 'execution error on git describe', |
| ] |
| ) |
| def test_twisterenv_check_zephyr_version( |
| caplog, |
| git_describe_return, |
| git_show_return, |
| expected_logs, |
| expected_version, |
| expected_commit_date |
| ): |
| def mock_run(command, *args, **kwargs): |
| if all([keyword in command for keyword in ['git', 'describe']]): |
| if isinstance(git_describe_return, type) and \ |
| issubclass(git_describe_return, Exception): |
| raise git_describe_return() |
| return git_describe_return |
| if all([keyword in command for keyword in ['git', 'show']]): |
| if isinstance(git_show_return, type) and \ |
| issubclass(git_show_return, Exception): |
| raise git_show_return() |
| return git_show_return |
| |
| options = mock.Mock( |
| ninja=True |
| ) |
| |
| original_abspath = os.path.abspath |
| |
| def mocked_abspath(path): |
| if path == 'dummy_abspath': |
| return 'dummy_abspath' |
| elif isinstance(path, mock.Mock): |
| return None |
| else: |
| return original_abspath(path) |
| |
| with mock.patch('os.path.abspath', side_effect=mocked_abspath): |
| twister_env = twisterlib.environment.TwisterEnv(options=options) |
| |
| with mock.patch('subprocess.run', mock.Mock(side_effect=mock_run)): |
| twister_env.check_zephyr_version() |
| print(expected_logs) |
| print(caplog.text) |
| assert twister_env.version == expected_version |
| assert twister_env.commit_date == expected_commit_date |
| assert all([expected_log in caplog.text for expected_log in expected_logs]) |
| |
| |
| TESTDATA_5 = [ |
| ( |
| False, |
| None, |
| None, |
| 'Unable to find `cmake` in path', |
| None |
| ), |
| ( |
| True, |
| 0, |
| b'somedummy\x1B[123-@d1770', |
| 'Finished running dummy/script/path', |
| { |
| 'returncode': 0, |
| 'msg': 'Finished running dummy/script/path', |
| 'stdout': 'somedummyd1770', |
| } |
| ), |
| ( |
| True, |
| 1, |
| b'another\x1B_dummy', |
| 'Cmake script failure: dummy/script/path', |
| { |
| 'returncode': 1, |
| 'returnmsg': 'anotherdummy' |
| } |
| ), |
| ] |
| |
| |
| @pytest.mark.parametrize( |
| 'find_cmake, return_code, out, expected_log, expected_result', |
| TESTDATA_5, |
| ids=[ |
| 'cmake not found', |
| 'regex sanitation 1', |
| 'regex sanitation 2' |
| ] |
| ) |
| def test_twisterenv_run_cmake_script( |
| caplog, |
| find_cmake, |
| return_code, |
| out, |
| expected_log, |
| expected_result |
| ): |
| def mock_which(name, *args, **kwargs): |
| return 'dummy/cmake/path' if find_cmake else None |
| |
| def mock_popen(command, *args, **kwargs): |
| return mock.Mock( |
| pid=0, |
| returncode=return_code, |
| communicate=mock.Mock( |
| return_value=(out, '') |
| ) |
| ) |
| |
| args = ['dummy/script/path', 'var1=val1'] |
| |
| with mock.patch('shutil.which', mock_which), \ |
| mock.patch('subprocess.Popen', mock.Mock(side_effect=mock_popen)), \ |
| pytest.raises(Exception) \ |
| if not find_cmake else nullcontext() as exception: |
| results = twisterlib.environment.TwisterEnv.run_cmake_script(args) |
| |
| assert 'Running cmake script dummy/script/path' in caplog.text |
| |
| assert expected_log in caplog.text |
| |
| if exception is not None: |
| return |
| |
| assert expected_result.items() <= results.items() |
| |
| |
| TESTDATA_6 = [ |
| ( |
| { |
| 'returncode': 0, |
| 'stdout': '{\"ZEPHYR_TOOLCHAIN_VARIANT\": \"dummy toolchain\"}' |
| }, |
| None, |
| 'Using \'dummy toolchain\' toolchain.' |
| ), |
| ( |
| {'returncode': 1}, |
| 2, |
| None |
| ), |
| ] |
| |
| |
| @pytest.mark.parametrize( |
| 'script_result, exit_value, expected_log', |
| TESTDATA_6, |
| ids=['valid', 'error'] |
| ) |
| def test_get_toolchain(caplog, script_result, exit_value, expected_log): |
| options = mock.Mock( |
| ninja=True |
| ) |
| |
| original_abspath = os.path.abspath |
| |
| def mocked_abspath(path): |
| if path == 'dummy_abspath': |
| return 'dummy_abspath' |
| elif isinstance(path, mock.Mock): |
| return None |
| else: |
| return original_abspath(path) |
| |
| with mock.patch('os.path.abspath', side_effect=mocked_abspath): |
| twister_env = twisterlib.environment.TwisterEnv(options=options) |
| |
| with mock.patch.object( |
| twisterlib.environment.TwisterEnv, |
| 'run_cmake_script', |
| mock.Mock(return_value=script_result)), \ |
| pytest.raises(SystemExit) \ |
| if exit_value is not None else nullcontext() as exit_info: |
| twister_env.get_toolchain() |
| |
| if exit_info is not None: |
| assert exit_info.value.code == exit_value |
| else: |
| assert expected_log in caplog.text |