sanitycheck: introduce integration testing
Use --integration testing for CI to limit execution and builds on
platforms that actually provide most of the coverage instead of blindly
building/running on all available platforms.
Signed-off-by: Anas Nashif <anas.nashif@intel.com>
diff --git a/scripts/sanity_chk/sanitylib.py b/scripts/sanity_chk/sanitylib.py
index dc49f6e..2b526e8 100644
--- a/scripts/sanity_chk/sanitylib.py
+++ b/scripts/sanity_chk/sanitylib.py
@@ -1271,12 +1271,6 @@
d[k] = v
for k, v in self.tests[name].items():
- if k not in valid_keys:
- raise ConfigurationError(
- self.filename,
- "Unknown config key '%s' in definition for '%s'" %
- (k, name))
-
if k in d:
if isinstance(d[k], str):
# By default, we just concatenate string values of keys
@@ -1444,6 +1438,7 @@
self.depends_on = None
self.min_flash = -1
self.extra_sections = None
+ self.integration_platforms = []
@staticmethod
def get_unique(testcase_root, workdir, name):
@@ -2294,6 +2289,7 @@
"arch_whitelist": {"type": "set"},
"arch_exclude": {"type": "set"},
"extra_sections": {"type": "list", "default": []},
+ "integration_platforms": {"type": "list", "default": []},
"platform_exclude": {"type": "set"},
"platform_whitelist": {"type": "set"},
"toolchain_exclude": {"type": "set"},
@@ -2366,6 +2362,9 @@
# hardcoded for now
self.connected_hardware = []
+ # run integration tests only
+ self.integration = False
+
def get_platform_instances(self, platform):
filtered_dict = {k:v for k,v in self.instances.items() if k.startswith(platform + "/")}
return filtered_dict
@@ -2636,6 +2635,7 @@
tc.depends_on = tc_dict["depends_on"]
tc.min_flash = tc_dict["min_flash"]
tc.extra_sections = tc_dict["extra_sections"]
+ tc.integration_platforms = tc_dict["integration_platforms"]
tc.parse_subcases(tc_path)
@@ -2759,6 +2759,10 @@
discards[instance] = "Not runnable on device"
continue
+ if self.integration and tc.integration_platforms and plat.name not in tc.integration_platforms:
+ discards[instance] = "Not part of integration platforms"
+ continue
+
if tc.skip:
discards[instance] = "Skip filter"
continue
