| #!/usr/bin/env python3 |
| |
| # Copyright(c) 2023 Google LLC |
| # SPDX-License-Identifier: Apache-2.0 |
| |
| """ |
| This test file contains testsuites for the Harness classes of twister |
| """ |
| import mock |
| import sys |
| import os |
| import pytest |
| import re |
| import logging as logger |
| |
| # ZEPHYR_BASE = os.getenv("ZEPHYR_BASE") |
| from conftest import ZEPHYR_BASE |
| |
| sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister")) |
| |
| from twisterlib.harness import ( |
| Bsim, |
| Console, |
| Gtest, |
| Harness, |
| HarnessImporter, |
| Pytest, |
| PytestHarnessException, |
| Robot, |
| Test, |
| ) |
| from twisterlib.statuses import TwisterStatus |
| from twisterlib.testinstance import TestInstance |
| |
| GTEST_START_STATE = " RUN " |
| GTEST_PASS_STATE = " OK " |
| GTEST_SKIP_STATE = " DISABLED " |
| GTEST_FAIL_STATE = " FAILED " |
| SAMPLE_GTEST_START = ( |
| "[00:00:00.000,000] [0m<inf> label: [==========] Running all tests.[0m" |
| ) |
| SAMPLE_GTEST_FMT = ( |
| "[00:00:00.000,000] [0m<inf> label: [{state}] {suite}.{test} (0ms)[0m" |
| ) |
| SAMPLE_GTEST_FMT_FAIL_WITH_PARAM = ( |
| "[00:00:00.000,000] [0m<inf> label: " |
| + "[{state}] {suite}.{test}, where GetParam() = 8-byte object <0B-00 00-00 00-9A 80-F7> (0 ms total)[0m" |
| ) |
| SAMPLE_GTEST_END = ( |
| "[00:00:00.000,000] [0m<inf> label: [==========] Done running all tests.[0m" |
| ) |
| SAMPLE_GTEST_END_VARIANT = ( |
| "[00:00:00.000,000] [0m<inf> label: [----------] Global test environment tear-down[0m" |
| ) |
| |
| |
| def process_logs(harness, logs): |
| for line in logs: |
| harness.handle(line) |
| |
| |
| TEST_DATA_RECORDING = [ |
| ([""], "^START:(?P<foo>.*):END", [], None), |
| (["START:bar:STOP"], "^START:(?P<foo>.*):END", [], None), |
| (["START:bar:END"], "^START:(?P<foo>.*):END", [{"foo": "bar"}], None), |
| ( |
| ["START:bar:baz:END"], |
| "^START:(?P<foo>.*):(?P<boo>.*):END", |
| [{"foo": "bar", "boo": "baz"}], |
| None, |
| ), |
| ( |
| ["START:bar:baz:END", "START:may:jun:END"], |
| "^START:(?P<foo>.*):(?P<boo>.*):END", |
| [{"foo": "bar", "boo": "baz"}, {"foo": "may", "boo": "jun"}], |
| None, |
| ), |
| (["START:bar:END"], "^START:(?P<foo>.*):END", [{"foo": "bar"}], []), |
| (["START:bar:END"], "^START:(?P<foo>.*):END", [{"foo": "bar"}], ["boo"]), |
| ( |
| ["START:bad_json:END"], |
| "^START:(?P<foo>.*):END", |
| [ |
| { |
| "foo": { |
| "ERROR": { |
| "msg": "Expecting value: line 1 column 1 (char 0)", |
| "doc": "bad_json", |
| } |
| } |
| } |
| ], |
| ["foo"], |
| ), |
| (["START::END"], "^START:(?P<foo>.*):END", [{"foo": {}}], ["foo"]), |
| ( |
| ['START: {"one":1, "two":2} :END'], |
| "^START:(?P<foo>.*):END", |
| [{"foo": {"one": 1, "two": 2}}], |
| ["foo"], |
| ), |
| ( |
| ['START: {"one":1, "two":2} :STOP:oops:END'], |
| "^START:(?P<foo>.*):STOP:(?P<boo>.*):END", |
| [{"foo": {"one": 1, "two": 2}, "boo": "oops"}], |
| ["foo"], |
| ), |
| ( |
| ['START: {"one":1, "two":2} :STOP:{"oops":0}:END'], |
| "^START:(?P<foo>.*):STOP:(?P<boo>.*):END", |
| [{"foo": {"one": 1, "two": 2}, "boo": {"oops": 0}}], |
| ["foo", "boo"], |
| ), |
| ] |
| |
| |
| @pytest.mark.parametrize( |
| "lines, pattern, expected_records, as_json", |
| TEST_DATA_RECORDING, |
| ids=[ |
| "empty", |
| "no match", |
| "match 1 field", |
| "match 2 fields", |
| "match 2 records", |
| "as_json empty", |
| "as_json no such field", |
| "error parsing json", |
| "empty json value", |
| "simple json", |
| "plain field and json field", |
| "two json fields", |
| ], |
| ) |
| def test_harness_parse_record(lines, pattern, expected_records, as_json): |
| harness = Harness() |
| harness.record = {"regex": pattern} |
| harness.record_pattern = re.compile(pattern) |
| |
| harness.record_as_json = as_json |
| if as_json is not None: |
| harness.record["as_json"] = as_json |
| |
| assert not harness.recording |
| |
| for line in lines: |
| harness.parse_record(line) |
| |
| assert harness.recording == expected_records |
| |
| |
| TEST_DATA_1 = [ |
| ("RunID: 12345", False, False, False, TwisterStatus.NONE, True), |
| ("PROJECT EXECUTION SUCCESSFUL", False, False, False, TwisterStatus.PASS, False), |
| ("PROJECT EXECUTION SUCCESSFUL", True, False, False, TwisterStatus.FAIL, False), |
| ("PROJECT EXECUTION FAILED", False, False, False, TwisterStatus.FAIL, False), |
| ("ZEPHYR FATAL ERROR", False, True, False, TwisterStatus.NONE, False), |
| ("GCOV_COVERAGE_DUMP_START", None, None, True, TwisterStatus.NONE, False), |
| ("GCOV_COVERAGE_DUMP_END", None, None, False, TwisterStatus.NONE, False), |
| ] |
| |
| |
| @pytest.mark.parametrize( |
| "line, fault, fail_on_fault, cap_cov, exp_stat, exp_id", |
| TEST_DATA_1, |
| ids=[ |
| "match id", |
| "passed passed", |
| "passed failed", |
| "failed failed", |
| "fail on fault", |
| "GCOV START", |
| "GCOV END", |
| ], |
| ) |
| def test_harness_process_test(line, fault, fail_on_fault, cap_cov, exp_stat, exp_id): |
| # Arrange |
| harness = Harness() |
| harness.run_id = 12345 |
| harness.status = TwisterStatus.NONE |
| harness.fault = fault |
| harness.fail_on_fault = fail_on_fault |
| mock.patch.object(Harness, "parse_record", return_value=None) |
| |
| # Act |
| harness.process_test(line) |
| |
| # Assert |
| assert harness.matched_run_id == exp_id |
| assert harness.status == exp_stat |
| assert harness.capture_coverage == cap_cov |
| assert harness.recording == [] |
| |
| |
| def test_robot_configure(tmp_path): |
| # Arrange |
| mock_platform = mock.Mock() |
| mock_platform.name = "mock_platform" |
| mock_platform.normalized_name = "mock_platform" |
| |
| mock_testsuite = mock.Mock(id="id", testcases=[]) |
| mock_testsuite.name = "mock_testsuite" |
| mock_testsuite.harness_config = {} |
| |
| outdir = tmp_path / "gtest_out" |
| outdir.mkdir() |
| |
| instance = TestInstance( |
| testsuite=mock_testsuite, platform=mock_platform, outdir=outdir |
| ) |
| instance.testsuite.harness_config = { |
| "robot_testsuite": "/path/to/robot/test", |
| "robot_option": "test_option", |
| } |
| robot_harness = Robot() |
| |
| # Act |
| robot_harness.configure(instance) |
| |
| # Assert |
| assert robot_harness.instance == instance |
| assert robot_harness.path == "/path/to/robot/test" |
| assert robot_harness.option == "test_option" |
| |
| |
| def test_robot_handle(tmp_path): |
| # Arrange |
| mock_platform = mock.Mock() |
| mock_platform.name = "mock_platform" |
| mock_platform.normalized_name = "mock_platform" |
| |
| mock_testsuite = mock.Mock(id="id", testcases=[]) |
| mock_testsuite.name = "mock_testsuite" |
| mock_testsuite.harness_config = {} |
| |
| outdir = tmp_path / "gtest_out" |
| outdir.mkdir() |
| |
| instance = TestInstance( |
| testsuite=mock_testsuite, platform=mock_platform, outdir=outdir |
| ) |
| |
| handler = Robot() |
| handler.instance = instance |
| handler.id = "test_case_1" |
| |
| line = "Test case passed" |
| |
| # Act |
| handler.handle(line) |
| tc = instance.get_case_or_create("test_case_1") |
| |
| # Assert |
| assert instance.status == TwisterStatus.PASS |
| assert tc.status == TwisterStatus.PASS |
| |
| |
| TEST_DATA_2 = [ |
| ("", 0, TwisterStatus.PASS), |
| ("Robot test failure: sourcedir for mock_platform", 1, TwisterStatus.FAIL), |
| ] |
| |
| |
| @pytest.mark.parametrize( |
| "exp_out, returncode, expected_status", TEST_DATA_2, ids=["passed", "failed"] |
| ) |
| def test_robot_run_robot_test(tmp_path, caplog, exp_out, returncode, expected_status): |
| # Arrange |
| command = ["command"] |
| |
| handler = mock.Mock() |
| handler.sourcedir = "sourcedir" |
| handler.log = "handler.log" |
| |
| path = "path" |
| option = "option" |
| |
| mock_platform = mock.Mock() |
| mock_platform.name = "mock_platform" |
| mock_platform.normalized_name = "mock_platform" |
| |
| mock_testsuite = mock.Mock(id="id", testcases=[mock.Mock()]) |
| mock_testsuite.name = "mock_testsuite" |
| mock_testsuite.harness_config = {} |
| |
| outdir = tmp_path / "gtest_out" |
| outdir.mkdir() |
| |
| instance = TestInstance( |
| testsuite=mock_testsuite, platform=mock_platform, outdir=outdir |
| ) |
| instance.build_dir = "build_dir" |
| |
| open_mock = mock.mock_open() |
| |
| robot = Robot() |
| robot.path = path |
| robot.option = option |
| robot.instance = instance |
| proc_mock = mock.Mock( |
| returncode=returncode, communicate=mock.Mock(return_value=(b"output", None)) |
| ) |
| popen_mock = mock.Mock( |
| return_value=mock.Mock( |
| __enter__=mock.Mock(return_value=proc_mock), __exit__=mock.Mock() |
| ) |
| ) |
| |
| # Act |
| with mock.patch("subprocess.Popen", popen_mock) as mock.mock_popen, mock.patch( |
| "builtins.open", open_mock |
| ): |
| robot.run_robot_test(command, handler) |
| |
| # Assert |
| assert instance.status == expected_status |
| open_mock().write.assert_called_once_with("output") |
| assert exp_out in caplog.text |
| |
| |
| TEST_DATA_3 = [ |
| ("one_line", None), |
| ("multi_line", 2), |
| ] |
| |
| |
| @pytest.mark.parametrize( |
| "type, num_patterns", TEST_DATA_3, ids=["one line", "multi line"] |
| ) |
| def test_console_configure(tmp_path, type, num_patterns): |
| # Arrange |
| mock_platform = mock.Mock() |
| mock_platform.name = "mock_platform" |
| mock_platform.normalized_name = "mock_platform" |
| |
| mock_testsuite = mock.Mock(id="id", testcases=[]) |
| mock_testsuite.name = "mock_testsuite" |
| mock_testsuite.harness_config = {} |
| |
| outdir = tmp_path / "gtest_out" |
| outdir.mkdir() |
| |
| instance = TestInstance( |
| testsuite=mock_testsuite, platform=mock_platform, outdir=outdir |
| ) |
| instance.testsuite.harness_config = { |
| "type": type, |
| "regex": ["pattern1", "pattern2"], |
| } |
| console = Console() |
| |
| # Act |
| console.configure(instance) |
| |
| # Assert |
| if num_patterns == 2: |
| assert len(console.patterns) == num_patterns |
| assert [pattern.pattern for pattern in console.patterns] == [ |
| "pattern1", |
| "pattern2", |
| ] |
| else: |
| assert console.pattern.pattern == "pattern1" |
| |
| |
| TEST_DATA_4 = [ |
| ("one_line", True, TwisterStatus.PASS, "line", False, False), |
| ("multi_line", True, TwisterStatus.PASS, "line", False, False), |
| ("multi_line", False, TwisterStatus.PASS, "line", False, False), |
| ("invalid_type", False, TwisterStatus.NONE, "line", False, False), |
| ("invalid_type", False, TwisterStatus.NONE, "ERROR", True, False), |
| ("invalid_type", False, TwisterStatus.NONE, "COVERAGE_START", False, True), |
| ("invalid_type", False, TwisterStatus.NONE, "COVERAGE_END", False, False), |
| ] |
| |
| |
| @pytest.mark.parametrize( |
| "line_type, ordered_val, exp_state, line, exp_fault, exp_capture", |
| TEST_DATA_4, |
| ids=[ |
| "one line", |
| "multi line ordered", |
| "multi line not ordered", |
| "logger error", |
| "fail on fault", |
| "GCOV START", |
| "GCOV END", |
| ], |
| ) |
| def test_console_handle( |
| tmp_path, line_type, ordered_val, exp_state, line, exp_fault, exp_capture |
| ): |
| mock_platform = mock.Mock() |
| mock_platform.name = "mock_platform" |
| mock_platform.normalized_name = "mock_platform" |
| |
| mock_testsuite = mock.Mock(id="id", testcases=[]) |
| mock_testsuite.name = "mock_testsuite" |
| mock_testsuite.harness_config = {} |
| |
| outdir = tmp_path / "gtest_out" |
| outdir.mkdir() |
| |
| instance = TestInstance( |
| testsuite=mock_testsuite, platform=mock_platform, outdir=outdir |
| ) |
| |
| console = Console() |
| console.instance = instance |
| console.type = line_type |
| console.patterns = [re.compile("pattern1"), re.compile("pattern2")] |
| console.pattern = re.compile("pattern") |
| console.patterns_expected = 0 |
| console.status = TwisterStatus.NONE |
| console.fail_on_fault = True |
| console.FAULT = "ERROR" |
| console.GCOV_START = "COVERAGE_START" |
| console.GCOV_END = "COVERAGE_END" |
| console.record = {"regex": "RESULT: (.*)"} |
| console.fieldnames = [] |
| console.recording = [] |
| console.regex = ["regex1", "regex2"] |
| console.id = "test_case_1" |
| |
| instance.get_case_or_create("test_case_1") |
| instance.testsuite.id = "test_suite_1" |
| |
| console.next_pattern = 0 |
| console.ordered = ordered_val |
| line = line |
| console.handle(line) |
| |
| line1 = "pattern1" |
| line2 = "pattern2" |
| console.handle(line1) |
| console.handle(line2) |
| assert console.status == exp_state |
| with pytest.raises(Exception): |
| console.handle(line) |
| assert logger.error.called |
| assert console.fault == exp_fault |
| assert console.capture_coverage == exp_capture |
| |
| |
| TEST_DATA_5 = [("serial_pty", 0), (None, 0), (None, 1)] |
| |
| |
| @pytest.mark.parametrize( |
| "pty_value, hardware_value", |
| TEST_DATA_5, |
| ids=["hardware pty", "hardware", "non hardware"], |
| ) |
| def test_pytest__generate_parameters_for_hardware(tmp_path, pty_value, hardware_value): |
| # Arrange |
| mock_platform = mock.Mock() |
| mock_platform.name = "mock_platform" |
| mock_platform.normalized_name = "mock_platform" |
| |
| mock_testsuite = mock.Mock(id="id", testcases=[]) |
| mock_testsuite.name = "mock_testsuite" |
| mock_testsuite.harness_config = {} |
| |
| outdir = tmp_path / "gtest_out" |
| outdir.mkdir() |
| |
| instance = TestInstance( |
| testsuite=mock_testsuite, platform=mock_platform, outdir=outdir |
| ) |
| |
| handler = mock.Mock() |
| handler.instance = instance |
| |
| hardware = mock.Mock() |
| hardware.serial_pty = pty_value |
| hardware.serial = "serial" |
| hardware.baud = 115200 |
| hardware.runner = "runner" |
| hardware.runner_params = ["--runner-param1", "runner-param2"] |
| hardware.fixtures = ["fixture1:option1", "fixture2"] |
| |
| options = handler.options |
| options.west_flash = "args" |
| |
| hardware.probe_id = "123" |
| hardware.product = "product" |
| hardware.pre_script = "pre_script" |
| hardware.post_flash_script = "post_flash_script" |
| hardware.post_script = "post_script" |
| |
| pytest_test = Pytest() |
| pytest_test.configure(instance) |
| |
| # Act |
| if hardware_value == 0: |
| handler.get_hardware.return_value = hardware |
| command = pytest_test._generate_parameters_for_hardware(handler) |
| else: |
| handler.get_hardware.return_value = None |
| |
| # Assert |
| if hardware_value == 1: |
| with pytest.raises(PytestHarnessException) as exinfo: |
| pytest_test._generate_parameters_for_hardware(handler) |
| assert str(exinfo.value) == "Hardware is not available" |
| else: |
| assert "--device-type=hardware" in command |
| if pty_value == "serial_pty": |
| assert "--device-serial-pty=serial_pty" in command |
| else: |
| assert "--device-serial=serial" in command |
| assert "--device-serial-baud=115200" in command |
| assert "--runner=runner" in command |
| assert "--runner-params=--runner-param1" in command |
| assert "--runner-params=runner-param2" in command |
| assert "--west-flash-extra-args=args" in command |
| assert "--device-id=123" in command |
| assert "--device-product=product" in command |
| assert "--pre-script=pre_script" in command |
| assert "--post-flash-script=post_flash_script" in command |
| assert "--post-script=post_script" in command |
| assert "--twister-fixture=fixture1:option1" in command |
| assert "--twister-fixture=fixture2" in command |
| |
| |
| def test__update_command_with_env_dependencies(): |
| cmd = ["cmd"] |
| pytest_test = Pytest() |
| mock.patch.object(Pytest, "PYTEST_PLUGIN_INSTALLED", False) |
| |
| # Act |
| result_cmd, _ = pytest_test._update_command_with_env_dependencies(cmd) |
| |
| # Assert |
| assert result_cmd == ["cmd", "-p", "twister_harness.plugin"] |
| |
| |
| def test_pytest_run(tmp_path, caplog): |
| # Arrange |
| timeout = 10 |
| cmd = ["command"] |
| exp_out = "Support for handler handler_type not implemented yet" |
| |
| harness = Pytest() |
| harness = mock.create_autospec(harness) |
| |
| mock.patch.object(Pytest, "generate_command", return_value=cmd) |
| mock.patch.object(Pytest, "run_command") |
| |
| mock_platform = mock.Mock() |
| mock_platform.name = "mock_platform" |
| mock_platform.normalized_name = "mock_platform" |
| |
| mock_testsuite = mock.Mock( |
| id="id", testcases=[], source_dir="source_dir", harness_config={} |
| ) |
| mock_testsuite.name = "mock_testsuite" |
| mock_testsuite.harness_config = {} |
| |
| handler = mock.Mock(options=mock.Mock(verbose=0), type_str="handler_type") |
| |
| outdir = tmp_path / "gtest_out" |
| outdir.mkdir() |
| |
| instance = TestInstance( |
| testsuite=mock_testsuite, platform=mock_platform, outdir=outdir |
| ) |
| instance.handler = handler |
| |
| test_obj = Pytest() |
| test_obj.configure(instance) |
| |
| # Act |
| test_obj.pytest_run(timeout) |
| # Assert |
| assert test_obj.status == TwisterStatus.FAIL |
| assert exp_out in caplog.text |
| |
| |
| TEST_DATA_6 = [(None), ("Test")] |
| |
| |
| @pytest.mark.parametrize("name", TEST_DATA_6, ids=["no name", "provided name"]) |
| def test_get_harness(name): |
| # Arrange |
| harnessimporter = HarnessImporter() |
| harness_name = name |
| |
| # Act |
| harness_class = harnessimporter.get_harness(harness_name) |
| |
| # Assert |
| assert isinstance(harness_class, Test) |
| |
| |
| TEST_DATA_7 = [ |
| ( |
| "", |
| "Running TESTSUITE suite_name", |
| ["suite_name"], |
| TwisterStatus.NONE, |
| True, |
| TwisterStatus.NONE, |
| ), |
| ("", "START - test_testcase", [], TwisterStatus.STARTED, True, TwisterStatus.NONE), |
| ( |
| "", |
| "PASS - test_example in 0 seconds", |
| [], |
| TwisterStatus.PASS, |
| True, |
| TwisterStatus.NONE, |
| ), |
| ( |
| "", |
| "SKIP - test_example in 0 seconds", |
| [], |
| TwisterStatus.SKIP, |
| True, |
| TwisterStatus.NONE, |
| ), |
| ( |
| "", |
| "FAIL - test_example in 0 seconds", |
| [], |
| TwisterStatus.FAIL, |
| True, |
| TwisterStatus.NONE, |
| ), |
| ( |
| "not a ztest and no state for test_id", |
| "START - test_testcase", |
| [], |
| TwisterStatus.PASS, |
| False, |
| TwisterStatus.PASS, |
| ), |
| ( |
| "not a ztest and no state for test_id", |
| "START - test_testcase", |
| [], |
| TwisterStatus.FAIL, |
| False, |
| TwisterStatus.FAIL, |
| ), |
| ] |
| |
| |
| @pytest.mark.parametrize( |
| "exp_out, line, exp_suite_name, exp_status, ztest, state", |
| TEST_DATA_7, |
| ids=["testsuite", "testcase", "pass", "skip", "failed", "ztest pass", "ztest fail"], |
| ) |
| def test_test_handle( |
| tmp_path, caplog, exp_out, line, exp_suite_name, exp_status, ztest, state |
| ): |
| # Arrange |
| line = line |
| mock_platform = mock.Mock() |
| mock_platform.name = "mock_platform" |
| mock_platform.normalized_name = "mock_platform" |
| |
| mock_testsuite = mock.Mock(id="id", testcases=[]) |
| mock_testsuite.name = "mock_testsuite" |
| mock_testsuite.harness_config = {} |
| |
| outdir = tmp_path / "gtest_out" |
| outdir.mkdir() |
| |
| instance = TestInstance( |
| testsuite=mock_testsuite, platform=mock_platform, outdir=outdir |
| ) |
| |
| test_obj = Test() |
| test_obj.configure(instance) |
| test_obj.id = "test_id" |
| test_obj.ztest = ztest |
| test_obj.status = state |
| test_obj.id = "test_id" |
| # Act |
| test_obj.handle(line) |
| |
| # Assert |
| assert test_obj.detected_suite_names == exp_suite_name |
| assert exp_out in caplog.text |
| if not "Running" in line and exp_out == "": |
| assert test_obj.instance.testcases[0].status == exp_status |
| if "ztest" in exp_out: |
| assert test_obj.instance.testcases[1].status == exp_status |
| |
| |
| @pytest.fixture |
| def gtest(tmp_path): |
| mock_platform = mock.Mock() |
| mock_platform.name = "mock_platform" |
| mock_platform.normalized_name = "mock_platform" |
| mock_testsuite = mock.Mock() |
| mock_testsuite.name = "mock_testsuite" |
| mock_testsuite.detailed_test_id = True |
| mock_testsuite.id = "id" |
| mock_testsuite.testcases = [] |
| mock_testsuite.harness_config = {} |
| outdir = tmp_path / "gtest_out" |
| outdir.mkdir() |
| |
| instance = TestInstance( |
| testsuite=mock_testsuite, platform=mock_platform, outdir=outdir |
| ) |
| |
| harness = Gtest() |
| harness.configure(instance) |
| return harness |
| |
| |
| def test_gtest_start_test_no_suites_detected(gtest): |
| process_logs(gtest, [SAMPLE_GTEST_START]) |
| assert len(gtest.detected_suite_names) == 0 |
| assert gtest.status == TwisterStatus.NONE |
| |
| |
| def test_gtest_start_test(gtest): |
| process_logs( |
| gtest, |
| [ |
| SAMPLE_GTEST_START, |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_START_STATE, suite="suite_name", test="test_name" |
| ), |
| ], |
| ) |
| assert gtest.status == TwisterStatus.NONE |
| assert len(gtest.detected_suite_names) == 1 |
| assert gtest.detected_suite_names[0] == "suite_name" |
| assert gtest.instance.get_case_by_name("id.suite_name.test_name") is not None |
| assert ( |
| gtest.instance.get_case_by_name("id.suite_name.test_name").status |
| == TwisterStatus.STARTED |
| ) |
| |
| |
| def test_gtest_pass(gtest): |
| process_logs( |
| gtest, |
| [ |
| SAMPLE_GTEST_START, |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_START_STATE, suite="suite_name", test="test_name" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_PASS_STATE, suite="suite_name", test="test_name" |
| ), |
| ], |
| ) |
| assert gtest.status == TwisterStatus.NONE |
| assert len(gtest.detected_suite_names) == 1 |
| assert gtest.detected_suite_names[0] == "suite_name" |
| assert ( |
| gtest.instance.get_case_by_name("id.suite_name.test_name") != TwisterStatus.NONE |
| ) |
| assert ( |
| gtest.instance.get_case_by_name("id.suite_name.test_name").status |
| == TwisterStatus.PASS |
| ) |
| |
| |
| def test_gtest_failed(gtest): |
| process_logs( |
| gtest, |
| [ |
| SAMPLE_GTEST_START, |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_START_STATE, suite="suite_name", test="test_name" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_FAIL_STATE, suite="suite_name", test="test_name" |
| ), |
| ], |
| ) |
| assert gtest.status == TwisterStatus.NONE |
| assert len(gtest.detected_suite_names) == 1 |
| assert gtest.detected_suite_names[0] == "suite_name" |
| assert ( |
| gtest.instance.get_case_by_name("id.suite_name.test_name") != TwisterStatus.NONE |
| ) |
| assert ( |
| gtest.instance.get_case_by_name("id.suite_name.test_name").status |
| == TwisterStatus.FAIL |
| ) |
| |
| |
| def test_gtest_skipped(gtest): |
| process_logs( |
| gtest, |
| [ |
| SAMPLE_GTEST_START, |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_START_STATE, suite="suite_name", test="test_name" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_SKIP_STATE, suite="suite_name", test="test_name" |
| ), |
| ], |
| ) |
| assert gtest.status == TwisterStatus.NONE |
| assert len(gtest.detected_suite_names) == 1 |
| assert gtest.detected_suite_names[0] == "suite_name" |
| assert ( |
| gtest.instance.get_case_by_name("id.suite_name.test_name") != TwisterStatus.NONE |
| ) |
| assert ( |
| gtest.instance.get_case_by_name("id.suite_name.test_name").status |
| == TwisterStatus.SKIP |
| ) |
| |
| |
| def test_gtest_all_pass(gtest): |
| process_logs( |
| gtest, |
| [ |
| SAMPLE_GTEST_START, |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_START_STATE, suite="suite_name", test="test_name" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_PASS_STATE, suite="suite_name", test="test_name" |
| ), |
| SAMPLE_GTEST_END, |
| ], |
| ) |
| assert gtest.status == TwisterStatus.PASS |
| assert len(gtest.detected_suite_names) == 1 |
| assert gtest.detected_suite_names[0] == "suite_name" |
| assert ( |
| gtest.instance.get_case_by_name("id.suite_name.test_name") != TwisterStatus.NONE |
| ) |
| assert ( |
| gtest.instance.get_case_by_name("id.suite_name.test_name").status |
| == TwisterStatus.PASS |
| ) |
| |
| |
| def test_gtest_all_pass_with_variant(gtest): |
| process_logs( |
| gtest, |
| [ |
| SAMPLE_GTEST_START, |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_START_STATE, suite="suite_name", test="test_name" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_PASS_STATE, suite="suite_name", test="test_name" |
| ), |
| SAMPLE_GTEST_END_VARIANT, |
| ], |
| ) |
| assert gtest.status == "passed" |
| assert len(gtest.detected_suite_names) == 1 |
| assert gtest.detected_suite_names[0] == "suite_name" |
| assert gtest.instance.get_case_by_name("id.suite_name.test_name") is not None |
| assert gtest.instance.get_case_by_name("id.suite_name.test_name").status == "passed" |
| |
| |
| def test_gtest_one_skipped(gtest): |
| process_logs( |
| gtest, |
| [ |
| SAMPLE_GTEST_START, |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_START_STATE, suite="suite_name", test="test_name" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_PASS_STATE, suite="suite_name", test="test_name" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_START_STATE, suite="suite_name", test="test_name1" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_SKIP_STATE, suite="suite_name", test="test_name1" |
| ), |
| SAMPLE_GTEST_END, |
| ], |
| ) |
| assert gtest.status == TwisterStatus.PASS |
| assert len(gtest.detected_suite_names) == 1 |
| assert gtest.detected_suite_names[0] == "suite_name" |
| assert ( |
| gtest.instance.get_case_by_name("id.suite_name.test_name") != TwisterStatus.NONE |
| ) |
| assert ( |
| gtest.instance.get_case_by_name("id.suite_name.test_name").status |
| == TwisterStatus.PASS |
| ) |
| assert ( |
| gtest.instance.get_case_by_name("id.suite_name.test_name1") |
| != TwisterStatus.NONE |
| ) |
| assert ( |
| gtest.instance.get_case_by_name("id.suite_name.test_name1").status |
| == TwisterStatus.SKIP |
| ) |
| |
| |
| def test_gtest_one_fail(gtest): |
| process_logs( |
| gtest, |
| [ |
| SAMPLE_GTEST_START, |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_START_STATE, suite="suite_name", test="test0" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_PASS_STATE, suite="suite_name", test="test0" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_START_STATE, suite="suite_name", test="test1" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_FAIL_STATE, suite="suite_name", test="test1" |
| ), |
| SAMPLE_GTEST_END, |
| ], |
| ) |
| assert gtest.status == TwisterStatus.FAIL |
| assert len(gtest.detected_suite_names) == 1 |
| assert gtest.detected_suite_names[0] == "suite_name" |
| assert gtest.instance.get_case_by_name("id.suite_name.test0") != TwisterStatus.NONE |
| assert ( |
| gtest.instance.get_case_by_name("id.suite_name.test0").status |
| == TwisterStatus.PASS |
| ) |
| assert gtest.instance.get_case_by_name("id.suite_name.test1") != TwisterStatus.NONE |
| assert ( |
| gtest.instance.get_case_by_name("id.suite_name.test1").status |
| == TwisterStatus.FAIL |
| ) |
| |
| |
| def test_gtest_one_fail_with_variant(gtest): |
| process_logs( |
| gtest, |
| [ |
| SAMPLE_GTEST_START, |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_START_STATE, suite="suite_name", test="test0" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_PASS_STATE, suite="suite_name", test="test0" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_START_STATE, suite="suite_name", test="test1" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_FAIL_STATE, suite="suite_name", test="test1" |
| ), |
| SAMPLE_GTEST_END_VARIANT, |
| ], |
| ) |
| assert gtest.status == "failed" |
| assert len(gtest.detected_suite_names) == 1 |
| assert gtest.detected_suite_names[0] == "suite_name" |
| assert gtest.instance.get_case_by_name("id.suite_name.test0") is not None |
| assert gtest.instance.get_case_by_name("id.suite_name.test0").status == "passed" |
| assert gtest.instance.get_case_by_name("id.suite_name.test1") is not None |
| assert gtest.instance.get_case_by_name("id.suite_name.test1").status == "failed" |
| |
| |
| def test_gtest_one_fail_with_variant_and_param(gtest): |
| process_logs( |
| gtest, |
| [ |
| SAMPLE_GTEST_START, |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_START_STATE, suite="suite_name", test="test0" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_PASS_STATE, suite="suite_name", test="test0" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_START_STATE, suite="suite_name", test="test1" |
| ), |
| SAMPLE_GTEST_FMT_FAIL_WITH_PARAM.format( |
| state=GTEST_FAIL_STATE, suite="suite_name", test="test1" |
| ), |
| SAMPLE_GTEST_END_VARIANT, |
| ], |
| ) |
| assert gtest.status == "failed" |
| assert len(gtest.detected_suite_names) == 1 |
| assert gtest.detected_suite_names[0] == "suite_name" |
| assert gtest.instance.get_case_by_name("id.suite_name.test0") is not None |
| assert gtest.instance.get_case_by_name("id.suite_name.test0").status == "passed" |
| assert gtest.instance.get_case_by_name("id.suite_name.test1") is not None |
| assert gtest.instance.get_case_by_name("id.suite_name.test1").status == "failed" |
| |
| |
| def test_gtest_missing_result(gtest): |
| with pytest.raises( |
| AssertionError, |
| match=r"gTest error, id.suite_name.test0 didn't finish", |
| ): |
| process_logs( |
| gtest, |
| [ |
| SAMPLE_GTEST_START, |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_START_STATE, suite="suite_name", test="test0" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_START_STATE, suite="suite_name", test="test1" |
| ), |
| ], |
| ) |
| |
| |
| def test_gtest_mismatch_result(gtest): |
| with pytest.raises( |
| AssertionError, |
| match=r"gTest error, mismatched tests. Expected id.suite_name.test0 but got None", |
| ): |
| process_logs( |
| gtest, |
| [ |
| SAMPLE_GTEST_START, |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_START_STATE, suite="suite_name", test="test0" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_PASS_STATE, suite="suite_name", test="test1" |
| ), |
| ], |
| ) |
| |
| |
| def test_gtest_repeated_result(gtest): |
| with pytest.raises( |
| AssertionError, |
| match=r"gTest error, mismatched tests. Expected id.suite_name.test1 but got id.suite_name.test0", |
| ): |
| process_logs( |
| gtest, |
| [ |
| SAMPLE_GTEST_START, |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_START_STATE, suite="suite_name", test="test0" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_PASS_STATE, suite="suite_name", test="test0" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_START_STATE, suite="suite_name", test="test1" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_PASS_STATE, suite="suite_name", test="test0" |
| ), |
| ], |
| ) |
| |
| |
| def test_gtest_repeated_run(gtest): |
| with pytest.raises( |
| AssertionError, |
| match=r"gTest error, id.suite_name.test0 running twice", |
| ): |
| process_logs( |
| gtest, |
| [ |
| SAMPLE_GTEST_START, |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_START_STATE, suite="suite_name", test="test0" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_PASS_STATE, suite="suite_name", test="test0" |
| ), |
| SAMPLE_GTEST_FMT.format( |
| state=GTEST_START_STATE, suite="suite_name", test="test0" |
| ), |
| ], |
| ) |
| |
| |
| def test_bsim_build(monkeypatch, tmp_path): |
| mocked_instance = mock.Mock() |
| build_dir = tmp_path / "build_dir" |
| os.makedirs(build_dir) |
| mocked_instance.build_dir = str(build_dir) |
| mocked_instance.name = "platform_name/test/dummy.test" |
| mocked_instance.testsuite.harness_config = {} |
| |
| harness = Bsim() |
| harness.instance = mocked_instance |
| |
| monkeypatch.setenv("BSIM_OUT_PATH", str(tmp_path)) |
| os.makedirs(os.path.join(tmp_path, "bin"), exist_ok=True) |
| zephyr_exe_path = os.path.join(build_dir, "zephyr", "zephyr.exe") |
| os.makedirs(os.path.dirname(zephyr_exe_path), exist_ok=True) |
| with open(zephyr_exe_path, "w") as file: |
| file.write("TEST_EXE") |
| |
| harness.build() |
| |
| new_exe_path = os.path.join(tmp_path, "bin", "bs_platform_name_test_dummy_test") |
| assert os.path.exists(new_exe_path) |
| with open(new_exe_path, "r") as file: |
| exe_content = file.read() |
| assert "TEST_EXE" in exe_content |