sanitycheck: fixed handling of retries
When re-running failed tests, do not go through any filtering, just load
the failed tests directly and execute them. The filtering was done based
on default platforms and any tests that were failing on non-default
platforms were ignored.
Fixes #13956
Signed-off-by: Anas Nashif <anas.nashif@intel.com>
diff --git a/scripts/sanitycheck b/scripts/sanitycheck
index 4d23058..57d2af4 100755
--- a/scripts/sanitycheck
+++ b/scripts/sanitycheck
@@ -1814,17 +1814,17 @@
def create_overlay(self, platform):
file = os.path.join(self.outdir, "overlay.conf")
os.makedirs(self.outdir, exist_ok=True)
- f = open(file, "w")
- content = ""
+ with open(file, "w") as f:
+ content = ""
- if len(self.test.extra_configs) > 0:
- content = "\n".join(self.test.extra_configs)
- if options.enable_coverage:
- if platform in options.coverage_platform:
- content = content + "\nCONFIG_COVERAGE=y"
+ if len(self.test.extra_configs) > 0:
+ content = "\n".join(self.test.extra_configs)
- f.write(content)
- f.close()
+ if options.enable_coverage:
+ if platform in options.coverage_platform:
+ content = content + "\nCONFIG_COVERAGE=y"
+
+ f.write(content)
def calculate_sizes(self):
"""Get the RAM/ROM sizes of a test case.
@@ -1942,8 +1942,15 @@
self.instances = {}
- def get_last_failed(self):
+ def get_platform(self, name):
+ selected_platform = None
+ for platform in self.platforms:
+ if platform.name == name:
+ selected_platform = platform
+ break
+ return selected_platform
+ def get_last_failed(self):
try:
if not os.path.exists(LAST_SANITY):
raise SanityRuntimeError("Couldn't find last sanity run.")
@@ -1954,13 +1961,16 @@
result = []
with open(LAST_SANITY, "r") as fp:
cr = csv.DictReader(fp)
+ instance_list = []
for row in cr:
if row["passed"] == "True":
continue
test = row["test"]
- platform = row["platform"]
- result.append((test, platform))
- return result
+ platform = self.get_platform(row["platform"])
+ instance = TestInstance(self.testcases[test], platform, self.outdir)
+ instance.create_overlay(platform.name)
+ instance_list.append(instance)
+ self.add_instances(instance_list)
def load_from_file(self, file):
try:
@@ -1976,14 +1986,9 @@
instance_list = []
for row in cr:
name = os.path.join(row[0], row[1])
- platforms = self.arches[row[3]].platforms
- myp = None
- for platform in platforms:
- if platform.name == row[2]:
- selected_platform = platform
- break
- instance = TestInstance(self.testcases[name], selected_platform, self.outdir)
- instance.create_overlay(selected_platform.name)
+ platform = self.get_platform(row[2])
+ instance = TestInstance(self.testcases[name], platform, self.outdir)
+ instance.create_overlay(platform.name)
instance_list.append(instance)
self.add_instances(instance_list)
@@ -2007,7 +2012,6 @@
instances = []
discards = {}
platform_filter = options.platform
- last_failed = options.only_failed
testcase_filter = run_individual_tests
arch_filter = options.arch
tag_filter = options.tag
@@ -2022,9 +2026,6 @@
verbose(" exclude_tag: " + str(exclude_tag))
verbose(" config_filter: " + str(config_filter))
- if last_failed:
- failed_tests = self.get_last_failed()
-
default_platforms = False
if all_plats:
@@ -2062,10 +2063,6 @@
if testcase_filter and tc_name not in testcase_filter:
continue
- if last_failed and (
- tc.name, plat.name) not in failed_tests:
- continue
-
if arch_filter and arch_name not in arch_filter:
continue
@@ -2214,11 +2211,6 @@
discards[instance] = "Testcase name filter"
continue
- if last_failed and (
- tc.name, plat.name) not in failed_tests:
- discards[instance] = "Passed or skipped during last run"
- continue
-
if arch_filter and arch_name not in arch_filter:
discards[instance] = "Command line testcase arch filter"
continue
@@ -3240,6 +3232,8 @@
discards = []
if options.load_tests:
ts.load_from_file(options.load_tests)
+ elif options.only_failed:
+ ts.get_last_failed()
else:
discards = ts.apply_filters()
