[MatterYamlTests][darwin-framework-tool] Use a list of tests to run f… (#31792)
diff --git a/.github/workflows/darwin-tests.yaml b/.github/workflows/darwin-tests.yaml
index d4db7d1..288166d 100644
--- a/.github/workflows/darwin-tests.yaml
+++ b/.github/workflows/darwin-tests.yaml
@@ -106,6 +106,7 @@
run: |
./scripts/run_in_build_env.sh \
"./scripts/tests/run_test_suite.py \
+ --runner darwin_framework_tool_python \
--chip-tool ./out/darwin-x64-darwin-framework-tool-${BUILD_VARIANT_FRAMEWORK_TOOL}/darwin-framework-tool \
--target-skip-glob '{TestAccessControlConstraints}' \
run \
diff --git a/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/decoder.py b/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/decoder.py
index a7dd32b..cd77147 100644
--- a/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/decoder.py
+++ b/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/decoder.py
@@ -322,7 +322,7 @@
provided_field_name = provided_field_name[0].lower(
) + provided_field_name[1:]
- if provided_field_name in value:
+ if provided_field_name in value and provided_field_name != field_name:
value[field_name] = self.run(
specs,
value[provided_field_name],
diff --git a/scripts/tests/chiptest/__init__.py b/scripts/tests/chiptest/__init__.py
index 378e3df..dba11f1 100644
--- a/scripts/tests/chiptest/__init__.py
+++ b/scripts/tests/chiptest/__init__.py
@@ -160,6 +160,60 @@
}
+def _GetDarwinFrameworkToolUnsupportedTests() -> Set[str]:
+ """Tests that fail in darwin-framework-tool for some reason"""
+ return {
+ "DL_LockUnlock", # darwin-framework-tool does not currently support reading or subscribing to Events
+ "Test_AddNewFabricFromExistingFabric", # darwin-framework-tool does not support the GetCommissionerRootCertificate command.
+ # The name of the arguments once converted differs for chip-tool and darwin-framework-tool (attribute-ids vs attribute-id. See #31934)
+ "TestAttributesById",
+ "TestBasicInformation", # darwin-framework-tool does not support writing readonly attributes by name
+ "TestClusterComplexTypes", # Darwin framework has no way to represent a present but null optional nullable field.
+ # When reading TestFabricScoped in TestClusterMultiFabric, the result differs because of missing fields that have been declared in the YAML step with null value to workaround some limitation of the test harness (#29110)
+ "TestClusterMultiFabric",
+ "TestCommandsById", # darwin-framework-tool does not support writing readonly attributes by name
+ "TestDiagnosticLogs", # darwin-framework-tool does not implement a BDXTransferServerDelegate
+ "TestDiscovery", # darwin-framework-tool does not support dns-sd commands.
+ "TestEvents", # darwin-framework-tool does not currently support reading or subscribing to Events
+ "TestEventsById", # darwin-framework-tool does not currently support reading or subscribing to Events
+ "TestGroupMessaging", # darwin-framework-tool does not support group commands.
+ "TestIcdManagementCluster", # darwin-framework-tool does not support ICD registration
+ "TestUnitTestingClusterMei", # darwin-framework-tool does not currently support reading or subscribing to Events
+ "TestReadNoneSubscribeNone", # darwin-framework-tool does not supports those commands.
+
+ "Test_TC_ACE_1_6", # darwin-framework-tool does not support group commands.
+ "Test_TC_ACL_2_5", # darwin-framework-tool does not currently support reading or subscribing to Events
+ "Test_TC_ACL_2_6", # darwin-framework-tool does not currently support reading or subscribing to Events
+ "Test_TC_ACL_2_7", # darwin-framework-tool does not currently support reading or subscribing to Events
+ "Test_TC_ACL_2_8", # darwin-framework-tool does not currently support reading or subscribing to Events
+ "Test_TC_ACL_2_9", # darwin-framework-tool does not currently support reading or subscribing to Events
+ "Test_TC_ACL_2_10", # darwin-framework-tool does not currently support reading or subscribing to Events
+ "Test_TC_BINFO_2_1", # darwin-framework-tool does not support writing readonly attributes by name
+ "Test_TC_BINFO_2_2", # darwin-framework-tool does not currently support reading or subscribing to Events
+ # The name of the arguments once converted differs for chip-tool and darwin-framework-tool (attribute-ids vs attribute-id. See #31934)
+ "Test_TC_BRBINFO_2_1",
+ "Test_TC_DGGEN_2_3", # darwin-framework-tool does not currently support reading or subscribing to Events
+ "Test_TC_DRLK_2_1", # darwin-framework-tool does not support writing readonly attributes by name
+ "Test_TC_DGTHREAD_2_1", # Thread Network Diagnostics is not implemented under darwin.
+ "Test_TC_DGTHREAD_2_2", # Thread Network Diagnostics is not implemented under darwin.
+ "Test_TC_DGTHREAD_2_3", # Thread Network Diagnostics is not implemented under darwin.
+ "Test_TC_DGTHREAD_2_4", # Thread Network Diagnostics is not implemented under darwin.
+ "Test_TC_FLABEL_2_1", # darwin-framework-tool does not support writing readonly attributes by name
+ "Test_TC_GRPKEY_2_1", # darwin-framework-tool does not support writing readonly attributes by name
+ "Test_TC_LCFG_2_1", # darwin-framework-tool does not support writing readonly attributes by name
+ "Test_TC_OPCREDS_3_7", # darwin-framework-tool does not support the GetCommissionerRootCertificate command.
+ "Test_TC_OPSTATE_2_4", # darwin-framework-tool does not currently support reading or subscribing to Events
+ "Test_TC_SMOKECO_2_2", # darwin-framework-tool does not currently support reading or subscribing to Events
+ "Test_TC_SMOKECO_2_3", # darwin-framework-tool does not currently support reading or subscribing to Events
+ "Test_TC_SMOKECO_2_4", # darwin-framework-tool does not currently support reading or subscribing to Events
+ "Test_TC_SMOKECO_2_5", # darwin-framework-tool does not currently support reading or subscribing to Events
+ "Test_TC_SMOKECO_2_6", # darwin-framework-tool does not currently support reading or subscribing to Events
+ "Test_TC_SC_4_1", # darwin-framework-tool does not support dns-sd commands.
+ "Test_TC_SC_5_2", # darwin-framework-tool does not support group commands.
+ "Test_TC_S_2_3", # darwin-framework-tool does not support group commands.
+ }
+
+
def _GetChipReplUnsupportedTests() -> Set[str]:
"""Tests that fail in chip-repl for some reason"""
return {
@@ -284,7 +338,7 @@
)
-def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short_run_name: bool):
+def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, treat_dft_unsupported_as_in_development: bool, use_short_run_name: bool):
"""
use_short_run_name should be true if we want the run_name to be "Test_ABC" instead of "some/path/Test_ABC.yaml"
"""
@@ -294,6 +348,7 @@
extra_slow_tests = _GetExtraSlowTests()
in_development_tests = _GetInDevelopmentTests()
chip_repl_unsupported_tests = _GetChipReplUnsupportedTests()
+ treat_dft_unsupported_as_in_development_tests = _GetDarwinFrameworkToolUnsupportedTests()
purposeful_failure_tests = _GetPurposefulFailureTests()
for path in _AllYamlTests():
@@ -327,6 +382,9 @@
else:
run_name = str(path)
+ if treat_dft_unsupported_as_in_development and run_name in treat_dft_unsupported_as_in_development_tests:
+ tags.add(TestTag.IN_DEVELOPMENT)
+
yield TestDefinition(
run_name=run_name,
name=path.stem, # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2"
@@ -336,12 +394,17 @@
def AllReplYamlTests():
- for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True, use_short_run_name=False):
+ for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True, treat_dft_unsupported_as_in_development=False, use_short_run_name=False):
yield test
def AllChipToolYamlTests():
- for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, use_short_run_name=True):
+ for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, treat_dft_unsupported_as_in_development=False, use_short_run_name=True):
+ yield test
+
+
+def AllDarwinFrameworkToolYamlTests():
+ for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, treat_dft_unsupported_as_in_development=True, use_short_run_name=True):
yield test
diff --git a/scripts/tests/chiptest/test_definition.py b/scripts/tests/chiptest/test_definition.py
index da1b56a..19f0a7d 100644
--- a/scripts/tests/chiptest/test_definition.py
+++ b/scripts/tests/chiptest/test_definition.py
@@ -251,10 +251,9 @@
class TestRunTime(Enum):
- CHIP_TOOL_BUILTIN = auto() # run via chip-tool built-in test commands
CHIP_TOOL_PYTHON = auto() # use the python yaml test parser with chip-tool
+ DARWIN_FRAMEWORK_TOOL_PYTHON = auto() # use the python yaml test parser with chip-tool
CHIP_REPL_PYTHON = auto() # use the python yaml test runner
- DARWIN_FRAMEWORK_TOOL_BUILTIN = auto() # run via darwin-framework-tool built-in test commands
@dataclass
@@ -281,7 +280,7 @@
return ", ".join([t.to_s() for t in self.tags])
def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str,
- timeout_seconds: typing.Optional[int], dry_run=False, test_runtime: TestRunTime = TestRunTime.CHIP_TOOL_BUILTIN):
+ timeout_seconds: typing.Optional[int], dry_run=False, test_runtime: TestRunTime = TestRunTime.CHIP_TOOL_PYTHON):
"""
Executes the given test case using the provided runner for execution.
"""
@@ -333,8 +332,6 @@
# so it will be commissionable again.
app.factoryReset()
- tool_cmd = paths.chip_tool if test_runtime != TestRunTime.CHIP_TOOL_PYTHON else paths.chip_tool_with_python_cmd
-
if dry_run:
tool_storage_dir = None
tool_storage_args = []
@@ -350,38 +347,37 @@
app.start()
setupCode = app.setupCode
- pairing_cmd = tool_cmd + ['pairing', 'code', TEST_NODE_ID, setupCode]
- test_cmd = tool_cmd + ['tests', self.run_name] + ['--PICS', pics_file]
- if test_runtime == TestRunTime.CHIP_TOOL_PYTHON:
+ if test_runtime == TestRunTime.CHIP_REPL_PYTHON:
+ chip_repl_yaml_tester_cmd = paths.chip_repl_yaml_tester_cmd
+ python_cmd = chip_repl_yaml_tester_cmd + \
+ ['--setup-code', setupCode] + ['--yaml-path', self.run_name] + ["--pics-file", pics_file]
+ if dry_run:
+ logging.info(" ".join(python_cmd))
+ else:
+ runner.RunSubprocess(python_cmd, name='CHIP_REPL_YAML_TESTER',
+ dependencies=[apps_register], timeout_seconds=timeout_seconds)
+ else:
+ pairing_cmd = paths.chip_tool_with_python_cmd + ['pairing', 'code', TEST_NODE_ID, setupCode]
+ test_cmd = paths.chip_tool_with_python_cmd + ['tests', self.run_name] + ['--PICS', pics_file]
server_args = ['--server_path', paths.chip_tool[-1]] + \
['--server_arguments', 'interactive server' +
(' ' if len(tool_storage_args) else '') + ' '.join(tool_storage_args)]
pairing_cmd += server_args
test_cmd += server_args
- elif test_runtime == TestRunTime.CHIP_TOOL_BUILTIN:
- pairing_cmd += tool_storage_args
- test_cmd += tool_storage_args
- if dry_run:
- # Some of our command arguments have spaces in them, so if we are
- # trying to log commands people can run we should quote those.
- def quoter(arg): return f"'{arg}'" if ' ' in arg else arg
- logging.info(" ".join(map(quoter, pairing_cmd)))
- logging.info(" ".join(map(quoter, test_cmd)))
- elif test_runtime == TestRunTime.CHIP_REPL_PYTHON:
- chip_repl_yaml_tester_cmd = paths.chip_repl_yaml_tester_cmd
- python_cmd = chip_repl_yaml_tester_cmd + \
- ['--setup-code', app.setupCode] + ['--yaml-path', self.run_name] + ["--pics-file", pics_file]
- runner.RunSubprocess(python_cmd, name='CHIP_REPL_YAML_TESTER',
- dependencies=[apps_register], timeout_seconds=timeout_seconds)
- else:
- runner.RunSubprocess(pairing_cmd,
- name='PAIR', dependencies=[apps_register])
-
- runner.RunSubprocess(
- test_cmd,
- name='TEST', dependencies=[apps_register],
- timeout_seconds=timeout_seconds)
+ if dry_run:
+ # Some of our command arguments have spaces in them, so if we are
+ # trying to log commands people can run we should quote those.
+ def quoter(arg): return f"'{arg}'" if ' ' in arg else arg
+ logging.info(" ".join(map(quoter, pairing_cmd)))
+ logging.info(" ".join(map(quoter, test_cmd)))
+ else:
+ runner.RunSubprocess(pairing_cmd,
+ name='PAIR', dependencies=[apps_register])
+ runner.RunSubprocess(
+ test_cmd,
+ name='TEST', dependencies=[apps_register],
+ timeout_seconds=timeout_seconds)
except Exception:
logging.error("!!!!!!!!!!!!!!!!!!!! ERROR !!!!!!!!!!!!!!!!!!!!!!")
diff --git a/scripts/tests/run_test_suite.py b/scripts/tests/run_test_suite.py
index 03773e4..d4d94b2 100755
--- a/scripts/tests/run_test_suite.py
+++ b/scripts/tests/run_test_suite.py
@@ -123,8 +123,8 @@
)
@click.option(
'--runner',
- type=click.Choice(['codegen', 'chip_repl_python', 'chip_tool_python'], case_sensitive=False),
- default='codegen',
+ type=click.Choice(['chip_repl_python', 'chip_tool_python', 'darwin_framework_tool_python'], case_sensitive=False),
+ default='chip_tool_python',
help='Run YAML tests using the specified runner.')
@click.option(
'--chip-tool',
@@ -138,18 +138,18 @@
log_fmt = '%(levelname)-7s %(message)s'
coloredlogs.install(level=__LOG_LEVELS__[log_level], fmt=log_fmt)
- runtime = TestRunTime.CHIP_TOOL_BUILTIN
+ runtime = TestRunTime.CHIP_TOOL_PYTHON
if runner == 'chip_repl_python':
runtime = TestRunTime.CHIP_REPL_PYTHON
- elif runner == 'chip_tool_python':
- runtime = TestRunTime.CHIP_TOOL_PYTHON
- elif chip_tool is not None and os.path.basename(chip_tool) == "darwin-framework-tool":
- runtime = TestRunTime.DARWIN_FRAMEWORK_TOOL_BUILTIN
+ elif runner == 'darwin_framework_tool_python':
+ runtime = TestRunTime.DARWIN_FRAMEWORK_TOOL_PYTHON
if chip_tool is None and not runtime == TestRunTime.CHIP_REPL_PYTHON:
- # non yaml tests REQUIRE chip-tool. Yaml tests should not require chip-tool
paths_finder = PathsFinder()
- chip_tool = paths_finder.get('chip-tool')
+ if runtime == TestRunTime.CHIP_TOOL_PYTHON:
+ chip_tool = paths_finder.get('chip-tool')
+ else:
+ chip_tool = paths_finder.get('darwin-framework-tool')
if include_tags:
include_tags = set([TestTag.__members__[t] for t in include_tags])
@@ -160,10 +160,10 @@
# Figures out selected test that match the given name(s)
if runtime == TestRunTime.CHIP_REPL_PYTHON:
all_tests = [test for test in chiptest.AllReplYamlTests()]
- elif runtime == TestRunTime.CHIP_TOOL_PYTHON and os.path.basename(chip_tool) != "darwin-framework-tool":
- all_tests = [test for test in chiptest.AllChipToolYamlTests()]
+ elif runtime == TestRunTime.DARWIN_FRAMEWORK_TOOL_PYTHON:
+ all_tests = [test for test in chiptest.AllDarwinFrameworkToolYamlTests()]
else:
- all_tests = [test for test in chiptest.AllChipToolTests(chip_tool)]
+ all_tests = [test for test in chiptest.AllChipToolYamlTests()]
tests = all_tests
@@ -178,7 +178,7 @@
TestTag.PURPOSEFUL_FAILURE,
}
- if runtime != TestRunTime.CHIP_TOOL_PYTHON:
+ if runtime == TestRunTime.CHIP_REPL_PYTHON:
exclude_tags.add(TestTag.CHIP_TOOL_PYTHON_ONLY)
if 'all' not in target:
@@ -319,7 +319,7 @@
chip_repl_yaml_tester = paths_finder.get('yamltest_with_chip_repl_tester.py')
if chip_tool_with_python is None:
- if context.obj.chip_tool and os.path.basename(context.obj.chip_tool) == "darwin-framework-tool":
+ if context.obj.runtime == TestRunTime.DARWIN_FRAMEWORK_TOOL_PYTHON:
chip_tool_with_python = paths_finder.get('darwinframeworktool.py')
else:
chip_tool_with_python = paths_finder.get('chiptool.py')
