initial commit for next-gen sanity checks
The online help ./scripts/sanitycheck --help describes usage.
Most users will simply want to run with no arguments.
Change-Id: Icedbbfc22599a64a6e3dbbb808ff3276db06f2e0
Signed-off-by: Andrew Boie <andrew.p.boie@intel.com>
diff --git a/.gitignore b/.gitignore
index bec6497..78585e0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,3 +18,4 @@
xml
html/
doc/latex/
+sanity-out/
diff --git a/samples/bluetooth/beacon/testcase.ini b/samples/bluetooth/beacon/testcase.ini
new file mode 100644
index 0000000..7946e54
--- /dev/null
+++ b/samples/bluetooth/beacon/testcase.ini
@@ -0,0 +1,12 @@
+[test_x86]
+tags = bluetooth
+build_only = true
+arch_whitelist = x86
+# FIXME Doesn't work for ia32_pci
+config_whitelist = CONFIG_PLATFORM="ia32"
+
+[test_arm]
+tags = bluetooth
+build_only = true
+arch_whitelist = arm
+
diff --git a/samples/bluetooth/central/testcase.ini b/samples/bluetooth/central/testcase.ini
new file mode 100644
index 0000000..84e8d8e
--- /dev/null
+++ b/samples/bluetooth/central/testcase.ini
@@ -0,0 +1,7 @@
+[test]
+tags = bluetooth
+build_only = true
+arch_whitelist = x86
+# FIXME Doesn't work for ia32_pci
+config_whitelist = CONFIG_PLATFORM="ia32"
+
diff --git a/samples/bluetooth/init/testcase.ini b/samples/bluetooth/init/testcase.ini
new file mode 100644
index 0000000..4ea5f6e
--- /dev/null
+++ b/samples/bluetooth/init/testcase.ini
@@ -0,0 +1,7 @@
+[test]
+tags = bluetooth
+build_only = true
+arch_whitelist = x86
+# Doesn't work for ia32_pci
+config_whitelist = CONFIG_PLATFORM="ia32"
+
diff --git a/samples/bluetooth/peripheral/testcase.ini b/samples/bluetooth/peripheral/testcase.ini
new file mode 100644
index 0000000..7946e54
--- /dev/null
+++ b/samples/bluetooth/peripheral/testcase.ini
@@ -0,0 +1,12 @@
+[test_x86]
+tags = bluetooth
+build_only = true
+arch_whitelist = x86
+# FIXME Doesn't work for ia32_pci
+config_whitelist = CONFIG_PLATFORM="ia32"
+
+[test_arm]
+tags = bluetooth
+build_only = true
+arch_whitelist = arm
+
diff --git a/samples/bluetooth/shell/testcase.ini b/samples/bluetooth/shell/testcase.ini
new file mode 100644
index 0000000..7946e54
--- /dev/null
+++ b/samples/bluetooth/shell/testcase.ini
@@ -0,0 +1,12 @@
+[test_x86]
+tags = bluetooth
+build_only = true
+arch_whitelist = x86
+# FIXME Doesn't work for ia32_pci
+config_whitelist = CONFIG_PLATFORM="ia32"
+
+[test_arm]
+tags = bluetooth
+build_only = true
+arch_whitelist = arm
+
diff --git a/samples/bluetooth/tester/testcase.ini b/samples/bluetooth/tester/testcase.ini
new file mode 100644
index 0000000..9c84ca2
--- /dev/null
+++ b/samples/bluetooth/tester/testcase.ini
@@ -0,0 +1,4 @@
+[test]
+tags = bluetooth
+build_only = true
+platform_whitelist = ti_lm3s6965
diff --git a/samples/microkernel/apps/hello_world/testcase.ini b/samples/microkernel/apps/hello_world/testcase.ini
new file mode 100644
index 0000000..8ee37ed
--- /dev/null
+++ b/samples/microkernel/apps/hello_world/testcase.ini
@@ -0,0 +1,4 @@
+[test]
+build_only = true
+tags = apps
+
diff --git a/samples/microkernel/apps/nfc_hello/testcase.ini b/samples/microkernel/apps/nfc_hello/testcase.ini
new file mode 100644
index 0000000..f839a31
--- /dev/null
+++ b/samples/microkernel/apps/nfc_hello/testcase.ini
@@ -0,0 +1,7 @@
+[test]
+build_only = true
+tags = apps
+arch_whitelist = x86
+# Doesn't work for ia32_pci
+config_whitelist = CONFIG_PLATFORM="ia32"
+
diff --git a/samples/microkernel/apps/philosophers/testcase.ini b/samples/microkernel/apps/philosophers/testcase.ini
new file mode 100644
index 0000000..8ee37ed
--- /dev/null
+++ b/samples/microkernel/apps/philosophers/testcase.ini
@@ -0,0 +1,4 @@
+[test]
+build_only = true
+tags = apps
+
diff --git a/samples/microkernel/benchmark/app_kernel/src/Makefile b/samples/microkernel/benchmark/app_kernel/src/Makefile
index ec93a2c..609033a 100644
--- a/samples/microkernel/benchmark/app_kernel/src/Makefile
+++ b/samples/microkernel/benchmark/app_kernel/src/Makefile
@@ -1,5 +1,5 @@
ccflags-y += -I$(CURDIR)/misc/generated/sysgen
-ccflags-y += -I$(CURDIR)/../../latency_measure/src
+ccflags-y += -I$(srctree)/samples/microkernel/benchmark/latency_measure/src
obj-y := fifo_b.o mailbox_b.o master.o mempool_b.o \
nop_b.o pipe_r.o sema_r.o event_b.o \
diff --git a/samples/microkernel/benchmark/app_kernel/testcase.ini b/samples/microkernel/benchmark/app_kernel/testcase.ini
new file mode 100644
index 0000000..4a7ab9e
--- /dev/null
+++ b/samples/microkernel/benchmark/app_kernel/testcase.ini
@@ -0,0 +1,6 @@
+[test]
+tags = benchmark
+arch_whitelist = x86
+# On my machine, takes about 110 to run, 180 to be safe
+timeout = 180
+
diff --git a/samples/microkernel/benchmark/boot_time/testcase.ini b/samples/microkernel/benchmark/boot_time/testcase.ini
new file mode 100644
index 0000000..3609d30
--- /dev/null
+++ b/samples/microkernel/benchmark/boot_time/testcase.ini
@@ -0,0 +1,4 @@
+[test]
+tags = benchmark
+arch_whitelist = x86
+
diff --git a/samples/microkernel/benchmark/footprint/testcase.ini b/samples/microkernel/benchmark/footprint/testcase.ini
new file mode 100644
index 0000000..ed1318c
--- /dev/null
+++ b/samples/microkernel/benchmark/footprint/testcase.ini
@@ -0,0 +1,17 @@
+[footprint-min]
+tags = footprint
+extra_args = TEST=min
+build_only = true
+
+[footprint-reg]
+tags = footprint
+extra_args = TEST=reg
+build_only = true
+arch_whitelist = x86
+
+[footprint-max]
+tags = footprint
+extra_args = TEST=max
+build_only = true
+arch_whitelist = x86
+
diff --git a/samples/microkernel/benchmark/latency_measure/testcase.ini b/samples/microkernel/benchmark/latency_measure/testcase.ini
new file mode 100644
index 0000000..3609d30
--- /dev/null
+++ b/samples/microkernel/benchmark/latency_measure/testcase.ini
@@ -0,0 +1,4 @@
+[test]
+tags = benchmark
+arch_whitelist = x86
+
diff --git a/samples/microkernel/benchmark/sys_kernel/testcase.ini b/samples/microkernel/benchmark/sys_kernel/testcase.ini
new file mode 100644
index 0000000..3609d30
--- /dev/null
+++ b/samples/microkernel/benchmark/sys_kernel/testcase.ini
@@ -0,0 +1,4 @@
+[test]
+tags = benchmark
+arch_whitelist = x86
+
diff --git a/samples/microkernel/test/test_bluetooth/testcase.ini b/samples/microkernel/test/test_bluetooth/testcase.ini
new file mode 100644
index 0000000..ee3f7b9
--- /dev/null
+++ b/samples/microkernel/test/test_bluetooth/testcase.ini
@@ -0,0 +1,3 @@
+[test-bluetooth]
+tags = bluetooth
+
diff --git a/samples/microkernel/test/test_critical/testcase.ini b/samples/microkernel/test/test_critical/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/microkernel/test/test_critical/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/microkernel/test/test_events/testcase.ini b/samples/microkernel/test/test_events/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/microkernel/test/test_events/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/microkernel/test/test_fifo/testcase.ini b/samples/microkernel/test/test_fifo/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/microkernel/test/test_fifo/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/microkernel/test/test_libs/testcase.ini b/samples/microkernel/test/test_libs/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/microkernel/test/test_libs/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/microkernel/test/test_mail/testcase.ini b/samples/microkernel/test/test_mail/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/microkernel/test/test_mail/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/microkernel/test/test_map/testcase.ini b/samples/microkernel/test/test_map/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/microkernel/test/test_map/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/microkernel/test/test_mutex/testcase.ini b/samples/microkernel/test/test_mutex/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/microkernel/test/test_mutex/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/microkernel/test/test_pipe/testcase.ini b/samples/microkernel/test/test_pipe/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/microkernel/test/test_pipe/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/microkernel/test/test_pool/testcase.ini b/samples/microkernel/test/test_pool/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/microkernel/test/test_pool/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/microkernel/test/test_rand32/testcase.ini b/samples/microkernel/test/test_rand32/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/microkernel/test/test_rand32/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/microkernel/test/test_sema/testcase.ini b/samples/microkernel/test/test_sema/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/microkernel/test/test_sema/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/microkernel/test/test_sprintf/testcase.ini b/samples/microkernel/test/test_sprintf/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/microkernel/test/test_sprintf/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/microkernel/test/test_stackprot/testcase.ini b/samples/microkernel/test/test_stackprot/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/microkernel/test/test_stackprot/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/microkernel/test/test_static_idt/testcase.ini b/samples/microkernel/test/test_static_idt/testcase.ini
new file mode 100644
index 0000000..6356d91
--- /dev/null
+++ b/samples/microkernel/test/test_static_idt/testcase.ini
@@ -0,0 +1,4 @@
+[test]
+tags = core
+config_whitelist = CONFIG_ISA_IA32
+
diff --git a/samples/microkernel/test/test_task/testcase.ini b/samples/microkernel/test/test_task/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/microkernel/test/test_task/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/microkernel/test/test_task_irq/testcase.ini b/samples/microkernel/test/test_task_irq/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/microkernel/test/test_task_irq/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/microkernel/test/test_tickless/testcase.ini b/samples/microkernel/test/test_tickless/testcase.ini
new file mode 100644
index 0000000..3592865
--- /dev/null
+++ b/samples/microkernel/test/test_tickless/testcase.ini
@@ -0,0 +1,4 @@
+[test]
+tags = core
+config_whitelist = !CONFIG_PLATFORM_TI_LM3S6965_QEMU
+
diff --git a/samples/microkernel/test/test_timer/testcase.ini b/samples/microkernel/test/test_timer/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/microkernel/test/test_timer/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/microkernel/test/test_xip/testcase.ini b/samples/microkernel/test/test_xip/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/microkernel/test/test_xip/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/nanokernel/apps/bluetooth/init/tescase.ini b/samples/nanokernel/apps/bluetooth/init/tescase.ini
new file mode 100644
index 0000000..7a40035
--- /dev/null
+++ b/samples/nanokernel/apps/bluetooth/init/tescase.ini
@@ -0,0 +1,4 @@
+[test]
+tags = bluetooth apps
+build_only = true
+
diff --git a/samples/nanokernel/apps/bluetooth/shell/tescase.ini b/samples/nanokernel/apps/bluetooth/shell/tescase.ini
new file mode 100644
index 0000000..7a40035
--- /dev/null
+++ b/samples/nanokernel/apps/bluetooth/shell/tescase.ini
@@ -0,0 +1,4 @@
+[test]
+tags = bluetooth apps
+build_only = true
+
diff --git a/samples/nanokernel/apps/hello_world/testcase.ini b/samples/nanokernel/apps/hello_world/testcase.ini
new file mode 100644
index 0000000..8ee37ed
--- /dev/null
+++ b/samples/nanokernel/apps/hello_world/testcase.ini
@@ -0,0 +1,4 @@
+[test]
+build_only = true
+tags = apps
+
diff --git a/samples/nanokernel/apps/philosophers/testcase.ini b/samples/nanokernel/apps/philosophers/testcase.ini
new file mode 100644
index 0000000..8ee37ed
--- /dev/null
+++ b/samples/nanokernel/apps/philosophers/testcase.ini
@@ -0,0 +1,4 @@
+[test]
+build_only = true
+tags = apps
+
diff --git a/samples/nanokernel/benchmark/boot_time/testcase.ini b/samples/nanokernel/benchmark/boot_time/testcase.ini
new file mode 100644
index 0000000..3609d30
--- /dev/null
+++ b/samples/nanokernel/benchmark/boot_time/testcase.ini
@@ -0,0 +1,4 @@
+[test]
+tags = benchmark
+arch_whitelist = x86
+
diff --git a/samples/nanokernel/benchmark/footprint/testcase.ini b/samples/nanokernel/benchmark/footprint/testcase.ini
new file mode 100644
index 0000000..ed1318c
--- /dev/null
+++ b/samples/nanokernel/benchmark/footprint/testcase.ini
@@ -0,0 +1,17 @@
+[footprint-min]
+tags = footprint
+extra_args = TEST=min
+build_only = true
+
+[footprint-reg]
+tags = footprint
+extra_args = TEST=reg
+build_only = true
+arch_whitelist = x86
+
+[footprint-max]
+tags = footprint
+extra_args = TEST=max
+build_only = true
+arch_whitelist = x86
+
diff --git a/samples/nanokernel/benchmark/latency_measure/testcase.ini b/samples/nanokernel/benchmark/latency_measure/testcase.ini
new file mode 100644
index 0000000..3609d30
--- /dev/null
+++ b/samples/nanokernel/benchmark/latency_measure/testcase.ini
@@ -0,0 +1,4 @@
+[test]
+tags = benchmark
+arch_whitelist = x86
+
diff --git a/samples/nanokernel/benchmark/sys_kernel/testcase.ini b/samples/nanokernel/benchmark/sys_kernel/testcase.ini
new file mode 100644
index 0000000..3609d30
--- /dev/null
+++ b/samples/nanokernel/benchmark/sys_kernel/testcase.ini
@@ -0,0 +1,4 @@
+[test]
+tags = benchmark
+arch_whitelist = x86
+
diff --git a/samples/nanokernel/test/test_arm_m3_irq_vector_table/testcase.ini b/samples/nanokernel/test/test_arm_m3_irq_vector_table/testcase.ini
new file mode 100644
index 0000000..d7b14e1
--- /dev/null
+++ b/samples/nanokernel/test/test_arm_m3_irq_vector_table/testcase.ini
@@ -0,0 +1,4 @@
+[test]
+tags = core
+config_whitelist = CONFIG_CPU_CORTEX_M3_M4
+
diff --git a/samples/nanokernel/test/test_bluetooth/testcase.ini b/samples/nanokernel/test/test_bluetooth/testcase.ini
new file mode 100644
index 0000000..ee3f7b9
--- /dev/null
+++ b/samples/nanokernel/test/test_bluetooth/testcase.ini
@@ -0,0 +1,3 @@
+[test-bluetooth]
+tags = bluetooth
+
diff --git a/samples/nanokernel/test/test_context/testcase.ini b/samples/nanokernel/test/test_context/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/nanokernel/test/test_context/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/nanokernel/test/test_fifo/testcase.ini b/samples/nanokernel/test/test_fifo/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/nanokernel/test/test_fifo/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/nanokernel/test/test_lifo/testcase.ini b/samples/nanokernel/test/test_lifo/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/nanokernel/test/test_lifo/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/nanokernel/test/test_sema/testcase.ini b/samples/nanokernel/test/test_sema/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/nanokernel/test/test_sema/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/nanokernel/test/test_stack/testcase.ini b/samples/nanokernel/test/test_stack/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/nanokernel/test/test_stack/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/nanokernel/test/test_stackprot/testcase.ini b/samples/nanokernel/test/test_stackprot/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/nanokernel/test/test_stackprot/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/nanokernel/test/test_static_idt/testcase.ini b/samples/nanokernel/test/test_static_idt/testcase.ini
new file mode 100644
index 0000000..6356d91
--- /dev/null
+++ b/samples/nanokernel/test/test_static_idt/testcase.ini
@@ -0,0 +1,4 @@
+[test]
+tags = core
+config_whitelist = CONFIG_ISA_IA32
+
diff --git a/samples/nanokernel/test/test_timer/testcase.ini b/samples/nanokernel/test/test_timer/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/nanokernel/test/test_timer/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/samples/nanokernel/test/test_xip/testcase.ini b/samples/nanokernel/test/test_xip/testcase.ini
new file mode 100644
index 0000000..2e4e885
--- /dev/null
+++ b/samples/nanokernel/test/test_xip/testcase.ini
@@ -0,0 +1,3 @@
+[test]
+tags = core
+
diff --git a/scripts/sanity_chk/.gitignore b/scripts/sanity_chk/.gitignore
new file mode 100644
index 0000000..ff5caac
--- /dev/null
+++ b/scripts/sanity_chk/.gitignore
@@ -0,0 +1,2 @@
+last_sanity.csv
+
diff --git a/scripts/sanity_chk/arches/arm.ini b/scripts/sanity_chk/arches/arm.ini
new file mode 100644
index 0000000..ed67930
--- /dev/null
+++ b/scripts/sanity_chk/arches/arm.ini
@@ -0,0 +1,9 @@
+[arch]
+name = arm
+platforms = basic_cortex_m3 fsl_frdm_k64f
+
+[basic_cortex_m3]
+qemu_support = true
+
+[fsl_frdm_k64f]
+
diff --git a/scripts/sanity_chk/arches/x86.ini b/scripts/sanity_chk/arches/x86.ini
new file mode 100644
index 0000000..392bd41
--- /dev/null
+++ b/scripts/sanity_chk/arches/x86.ini
@@ -0,0 +1,12 @@
+[arch]
+name = x86
+platforms = basic_minuteia basic_atom galileo
+
+[basic_atom]
+qemu_support = true
+
+[basic_minuteia]
+qemu_support = true
+
+[galileo]
+
diff --git a/scripts/sanity_chk/sanity_last_release.csv b/scripts/sanity_chk/sanity_last_release.csv
new file mode 100644
index 0000000..1697b6a
--- /dev/null
+++ b/scripts/sanity_chk/sanity_last_release.csv
@@ -0,0 +1,229 @@
+test,arch,platform,passed,status,extra_args,qemu,qemu_time,ram_size,rom_size
+nanokernel/test/test_stackprot/test,x86,basic_atom,True,,,True,0.3900439739227295,11707,7491
+nanokernel/test/test_fifo/test,x86,basic_atom,True,,,True,1.578080177307129,45810,17638
+microkernel/test/test_events/test,x86,basic_minuteia,True,,,True,0.6130318641662598,25960,16376
+nanokernel/test/test_fifo/test,arm,fsl_frdm_k64f,True,,,False,,28688,16181
+bluetooth/beacon/test_arm,arm,basic_cortex_m3,True,,,True,,15160,33191
+nanokernel/test/test_sema/test,x86,galileo,True,,,False,,47044,17476
+microkernel/test/test_task_irq/test,arm,fsl_frdm_k64f,True,,,False,,11020,14872
+microkernel/benchmark/footprint/footprint-min,arm,basic_cortex_m3,True,,TEST=min,True,,1796,6316
+bluetooth/init/test,x86,basic_minuteia,True,,,True,,51880,38948
+nanokernel/test/test_lifo/test,x86,galileo,True,,,False,,48012,18260
+microkernel/test/test_mutex/test,x86,basic_atom,True,,,True,1.4354000091552734,33175,14355
+nanokernel/benchmark/latency_measure/test,x86,basic_minuteia,True,,,True,0.4863758087158203,32955,17631
+nanokernel/test/test_static_idt/test,x86,basic_atom,True,,,True,0.43580198287963867,10856,7156
+nanokernel/benchmark/sys_kernel/test,x86,basic_minuteia,True,,,True,0.5170221328735352,24626,16982
+nanokernel/benchmark/boot_time/test,x86,basic_minuteia,True,,,True,0.39171504974365234,9717,6041
+nanokernel/test/test_bluetooth/test-bluetooth,x86,galileo,True,,,False,,44816,34924
+microkernel/test/test_pool/test,arm,basic_cortex_m3,True,,,True,3.669074058532715,27020,15462
+nanokernel/test/test_sema/test,x86,basic_atom,True,,,True,1.8950130939483643,44676,15372
+microkernel/test/test_timer/test,x86,basic_atom,True,,,True,2.8563740253448486,53070,25238
+microkernel/test/test_tickless/test,x86,galileo,True,,,False,,23518,14650
+bluetooth/shell/test_arm,arm,basic_cortex_m3,True,,,True,,19068,40203
+microkernel/benchmark/boot_time/test,x86,basic_minuteia,True,,,True,0.38483095169067383,17389,11853
+nanokernel/test/test_lifo/test,x86,basic_minuteia,True,,,True,7.8495118618011475,45016,15572
+microkernel/test/test_map/test,x86,galileo,True,,,False,,28939,18023
+microkernel/test/test_sema/test,arm,basic_cortex_m3,True,,,True,12.122911930084229,19228,16717
+microkernel/test/test_static_idt/test,x86,basic_atom,True,,,True,0.38988304138183594,21112,12528
+nanokernel/test/test_stackprot/test,x86,galileo,True,,,False,,14339,9859
+nanokernel/test/test_fifo/test,arm,basic_cortex_m3,True,,,True,7.265285968780518,28672,14811
+microkernel/test/test_static_idt/test,x86,galileo,True,,,False,,23348,14500
+microkernel/test/test_libs/test,arm,basic_cortex_m3,True,,,True,0.00885319709777832,10762,13360
+microkernel/apps/hello_world/test,x86,galileo,True,,,False,,27317,18453
+nanokernel/apps/hello_world/test,x86,galileo,True,,,False,,13899,8431
+microkernel/apps/philosophers/test,arm,basic_cortex_m3,True,,,True,,14180,11295
+nanokernel/test/test_timer/test,arm,basic_cortex_m3,True,,,True,29.05277395248413,7580,8289
+microkernel/test/test_task/test,arm,fsl_frdm_k64f,True,,,False,,10880,15069
+nanokernel/benchmark/boot_time/test,x86,basic_atom,True,,,True,0.40399789810180664,10121,6405
+microkernel/test/test_libs/test,x86,basic_atom,True,,,True,0.4623570442199707,23773,15157
+microkernel/test/test_bluetooth/test-bluetooth,x86,basic_minuteia,True,,,True,0.45185089111328125,50388,37464
+microkernel/test/test_xip/test,x86,basic_minuteia,True,,,True,0.9812290668487549,9356,10556
+microkernel/test/test_pipe/test,x86,basic_atom,True,,,True,1.9525740146636963,39160,27600
+microkernel/benchmark/footprint/footprint-reg,x86,galileo,True,,TEST=reg,False,,18707,15711
+nanokernel/test/test_timer/test,x86,basic_minuteia,True,,,True,30.641869068145752,16675,9399
+microkernel/test/test_rand32/test,x86,galileo,True,,,False,,20064,13272
+nanokernel/test/test_timer/test,x86,basic_atom,True,,,True,5.522861957550049,17075,9755
+microkernel/test/test_sprintf/test,x86,basic_minuteia,True,,,True,0.4806699752807617,27705,21229
+microkernel/test/test_map/test,arm,basic_cortex_m3,True,,,True,0.1430048942565918,12804,14181
+microkernel/test/test_stackprot/test,x86,galileo,True,,,False,,25270,16418
+nanokernel/test/test_stack/test,x86,galileo,True,,,False,,22460,12740
+microkernel/test/test_mutex/test,arm,basic_cortex_m3,True,,,True,6.846920967102051,21656,12791
+microkernel/test/test_libs/test,x86,basic_minuteia,True,,,True,0.46005892753601074,23081,14517
+microkernel/apps/philosophers/test,arm,fsl_frdm_k64f,True,,,False,,14196,12517
+microkernel/test/test_events/test,arm,fsl_frdm_k64f,True,,,False,,11800,15588
+microkernel/test/test_fifo/test,x86,basic_atom,True,,,True,0.4695100784301758,26785,18153
+nanokernel/benchmark/latency_measure/test,x86,basic_atom,True,,,True,0.4914400577545166,33375,17975
+microkernel/test/test_tickless/test,x86,basic_atom,True,,,True,1.0034639835357666,21326,12734
+nanokernel/test/test_bluetooth/test-bluetooth,x86,basic_minuteia,True,,,True,0.4623138904571533,42048,32464
+nanokernel/test/test_context/test,x86,galileo,True,,,False,,36164,14604
+microkernel/test/test_stackprot/test,arm,fsl_frdm_k64f,True,,,False,,10728,14365
+nanokernel/test/test_stackprot/test,arm,fsl_frdm_k64f,True,,,False,,4492,7952
+microkernel/apps/philosophers/test,x86,galileo,True,,,False,,26573,14657
+nanokernel/benchmark/footprint/footprint-min,x86,basic_atom,True,,TEST=min,True,,2208,1852
+nanokernel/apps/philosophers/test,x86,basic_atom,True,,,True,,15953,6537
+microkernel/test/test_sema/test,x86,galileo,True,,,False,,37651,21471
+microkernel/test/test_libs/test,x86,galileo,True,,,False,,25865,16985
+microkernel/test/test_task_irq/test,x86,basic_minuteia,True,,,True,0.4734950065612793,27018,18458
+microkernel/test/test_tickless/test,x86,basic_minuteia,True,,,True,4.3573009967803955,20738,12194
+microkernel/test/test_map/test,x86,basic_atom,True,,,True,0.4864978790283203,26739,16087
+microkernel/test/test_fifo/test,arm,basic_cortex_m3,True,,,True,0.15673398971557617,10788,15897
+nanokernel/test/test_bluetooth/test-bluetooth,arm,basic_cortex_m3,True,,,True,0.00786900520324707,9992,26561
+nanokernel/test/test_xip/test,arm,fsl_frdm_k64f,True,,,False,,3472,6061
+nanokernel/test/test_stack/test,arm,basic_cortex_m3,True,,,True,1.1097431182861328,9724,8524
+bluetooth/central/test,x86,basic_minuteia,True,,,True,,52204,39272
+nanokernel/test/test_lifo/test,arm,basic_cortex_m3,True,,,True,7.264161109924316,30092,13720
+nanokernel/benchmark/footprint/footprint-reg,x86,basic_atom,True,,TEST=reg,True,,6514,4866
+microkernel/benchmark/boot_time/test,x86,galileo,True,,,False,,19893,14053
+microkernel/benchmark/app_kernel/test,x86,basic_atom,True,,,True,18.793134927749634,76772,44772
+microkernel/apps/hello_world/test,arm,basic_cortex_m3,True,,,True,,10716,11116
+nanokernel/test/test_timer/test,x86,galileo,True,,,False,,19543,11959
+microkernel/test/test_pool/test,x86,basic_atom,True,,,True,1.0459659099578857,41669,17509
+bluetooth/peripheral/test_x86,x86,basic_minuteia,True,,,True,,56617,43625
+nanokernel/test/test_stackprot/test,x86,basic_minuteia,True,,,True,0.4640929698944092,11027,6855
+microkernel/test/test_static_idt/test,x86,basic_minuteia,True,,,True,0.509680986404419,20476,11944
+microkernel/test/test_stackprot/test,x86,basic_atom,True,,,True,0.49658799171447754,22902,14314
+nanokernel/test/test_lifo/test,arm,fsl_frdm_k64f,True,,,False,,30108,15090
+microkernel/test/test_mail/test,x86,basic_minuteia,True,,,True,0.6057891845703125,35433,24749
+nanokernel/benchmark/sys_kernel/test,x86,galileo,True,,,False,,27122,19170
+nanokernel/benchmark/footprint/footprint-max,x86,basic_atom,True,,TEST=max,True,,15893,11673
+microkernel/benchmark/footprint/footprint-min,x86,basic_atom,True,,TEST=min,True,,5800,5076
+bluetooth/beacon/test_x86,x86,basic_minuteia,True,,,True,,52576,39644
+nanokernel/benchmark/boot_time/test,x86,galileo,True,,,False,,12329,8349
+nanokernel/test/test_static_idt/test,x86,basic_minuteia,True,,,True,0.4573800563812256,10460,6804
+microkernel/test/test_map/test,x86,basic_minuteia,True,,,True,0.4831821918487549,26063,15463
+nanokernel/test/test_stackprot/test,arm,basic_cortex_m3,True,,,True,0.010055065155029297,4476,6692
+microkernel/test/test_libs/test,arm,fsl_frdm_k64f,True,,,False,,10778,14514
+nanokernel/test/test_xip/test,arm,basic_cortex_m3,True,,,True,0.006029844284057617,3456,4831
+microkernel/test/test_xip/test,x86,basic_atom,True,,,True,0.8974871635437012,9616,11116
+microkernel/test/test_critical/test,x86,basic_minuteia,True,,,True,15.503901958465576,19631,13123
+microkernel/test/test_mail/test,arm,basic_cortex_m3,True,,,True,0.27579283714294434,13112,22403
+microkernel/test/test_events/test,x86,galileo,True,,,False,,28780,18880
+microkernel/test/test_task/test,arm,basic_cortex_m3,True,,,True,3.028717041015625,10864,13843
+microkernel/test/test_bluetooth/test-bluetooth,x86,galileo,True,,,False,,53164,39924
+nanokernel/test/test_xip/test,x86,basic_minuteia,True,,,True,1.2589969635009766,5296,5349
+microkernel/test/test_sprintf/test,x86,basic_atom,True,,,True,0.4770331382751465,31429,24901
+nanokernel/apps/hello_world/test,x86,basic_minuteia,True,,,True,,11031,5871
+microkernel/apps/nfc_hello/test,x86,basic_minuteia,True,,,True,,14018,10626
+microkernel/test/test_timer/test,arm,fsl_frdm_k64f,True,,,False,,30100,24391
+microkernel/benchmark/footprint/footprint-reg,x86,basic_atom,True,,TEST=reg,True,,16391,13659
+microkernel/test/test_pipe/test,x86,galileo,True,,,False,,40856,29072
+microkernel/benchmark/app_kernel/test,x86,galileo,True,,,False,,77672,45756
+nanokernel/apps/philosophers/test,x86,basic_minuteia,True,,,True,,15541,6185
+nanokernel/test/test_sema/test,arm,basic_cortex_m3,True,,,True,9.185144901275635,29844,12642
+nanokernel/test/test_xip/test,x86,basic_atom,True,,,True,1.2263851165771484,5340,5705
+microkernel/test/test_mutex/test,x86,galileo,True,,,False,,35427,16343
+nanokernel/benchmark/footprint/footprint-min,x86,basic_minuteia,True,,TEST=min,True,,2024,1720
+nanokernel/test/test_bluetooth/test-bluetooth,x86,basic_atom,True,,,True,0.44973301887512207,42776,33004
+microkernel/test/test_timer/test,arm,basic_cortex_m3,True,,,True,15.362816095352173,30084,23013
+nanokernel/test/test_context/test,x86,basic_minuteia,True,,,True,6.956724166870117,33292,12040
+microkernel/benchmark/sys_kernel/test,x86,galileo,True,,,False,,52082,26450
+nanokernel/test/test_arm_m3_irq_vector_table/test,arm,fsl_frdm_k64f,True,,,False,,3492,6421
+microkernel/test/test_mail/test,x86,basic_atom,True,,,True,0.5486729145050049,36045,25309
+bluetooth/shell/test_arm,arm,fsl_frdm_k64f,True,,,False,,19072,40973
+bluetooth/init/test,x86,basic_atom,True,,,True,,52840,39740
+microkernel/test/test_mutex/test,x86,basic_minuteia,True,,,True,7.501186847686768,32543,13775
+microkernel/apps/nfc_hello/test,x86,basic_atom,True,,,True,,14474,11010
+microkernel/test/test_task_irq/test,x86,galileo,True,,,False,,29906,21030
+microkernel/test/test_mutex/test,arm,fsl_frdm_k64f,True,,,False,,21672,14013
+microkernel/test/test_pool/test,x86,galileo,True,,,False,,43545,19269
+bluetooth/shell/test_x86,x86,basic_minuteia,True,,,True,,57193,42473
+microkernel/test/test_pipe/test,x86,basic_minuteia,True,,,True,10.45518708229065,38072,26604
+nanokernel/benchmark/footprint/footprint-reg,x86,basic_minuteia,True,,TEST=reg,True,,6122,4518
+nanokernel/test/test_arm_m3_irq_vector_table/test,arm,basic_cortex_m3,True,,,True,0.010872840881347656,3476,5191
+microkernel/test/test_sema/test,x86,basic_atom,True,,,True,2.331605911254883,35511,19587
+nanokernel/test/test_stack/test,arm,fsl_frdm_k64f,True,,,False,,9740,9746
+microkernel/test/test_stackprot/test,x86,basic_minuteia,True,,,True,0.42488694190979004,21942,13406
+microkernel/test/test_tickless/test,arm,fsl_frdm_k64f,True,,,False,,10720,12548
+microkernel/test/test_pool/test,arm,fsl_frdm_k64f,True,,,False,,27036,16688
+bluetooth/beacon/test_x86,x86,basic_atom,True,,,True,,53544,40444
+nanokernel/test/test_timer/test,arm,fsl_frdm_k64f,True,,,False,,7596,9511
+nanokernel/apps/philosophers/test,arm,basic_cortex_m3,True,,,True,,9660,5619
+microkernel/benchmark/footprint/footprint-max,x86,basic_atom,True,,TEST=max,True,,46366,37714
+microkernel/test/test_xip/test,arm,fsl_frdm_k64f,True,,,False,,7608,11056
+microkernel/test/test_stackprot/test,arm,basic_cortex_m3,True,,,True,0.011138916015625,10712,13109
+bluetooth/central/test,x86,basic_atom,True,,,True,,53168,40068
+nanokernel/test/test_fifo/test,x86,galileo,True,,,False,,48134,19746
+microkernel/test/test_critical/test,arm,fsl_frdm_k64f,True,,,False,,8968,13386
+microkernel/test/test_bluetooth/test-bluetooth,arm,fsl_frdm_k64f,True,,,False,,15168,32535
+microkernel/apps/hello_world/test,x86,basic_minuteia,True,,,True,,24441,15893
+microkernel/benchmark/app_kernel/test,x86,basic_minuteia,True,,,True,1.7877001762390137,75300,43692
+microkernel/apps/philosophers/test,x86,basic_minuteia,True,,,True,,23701,12101
+microkernel/benchmark/sys_kernel/test,x86,basic_minuteia,True,,,True,0.6504800319671631,49586,24262
+nanokernel/apps/hello_world/test,arm,fsl_frdm_k64f,True,,,False,,5480,6498
+microkernel/test/test_pool/test,x86,basic_minuteia,True,,,True,4.2199461460113525,40669,16709
+microkernel/apps/hello_world/test,x86,basic_atom,True,,,True,,25137,16537
+microkernel/benchmark/footprint/footprint-min,arm,fsl_frdm_k64f,True,,TEST=min,False,,1796,7408
+microkernel/test/test_pipe/test,arm,basic_cortex_m3,True,,,True,9.724919080734253,14848,24094
+nanokernel/test/test_context/test,arm,fsl_frdm_k64f,True,,,False,,21660,10989
+bluetooth/peripheral/test_x86,x86,basic_atom,True,,,True,,57685,44521
+microkernel/test/test_mail/test,x86,galileo,True,,,False,,38209,27209
+microkernel/test/test_task_irq/test,x86,basic_atom,True,,,True,0.4650437831878662,27790,19178
+nanokernel/test/test_sema/test,x86,basic_minuteia,True,,,True,9.942577123641968,44232,14972
+bluetooth/beacon/test_arm,arm,fsl_frdm_k64f,True,,,False,,15164,33805
+microkernel/benchmark/boot_time/test,x86,basic_atom,True,,,True,0.45662379264831543,18017,12329
+microkernel/test/test_events/test,x86,basic_atom,True,,,True,0.5145089626312256,26568,16932
+microkernel/test/test_sprintf/test,arm,basic_cortex_m3,True,,,True,0.012573957443237305,8604,17885
+microkernel/apps/hello_world/test,arm,fsl_frdm_k64f,True,,,False,,10732,12338
+microkernel/test/test_xip/test,x86,galileo,True,,,False,,9656,13112
+nanokernel/benchmark/latency_measure/test,x86,galileo,True,,,False,,35395,19763
+microkernel/test/test_xip/test,arm,basic_cortex_m3,True,,,True,0.0038878917694091797,7592,9834
+microkernel/benchmark/footprint/footprint-max,x86,galileo,True,,TEST=max,False,,48142,39226
+microkernel/benchmark/latency_measure/test,x86,basic_minuteia,True,,,True,0.57499098777771,48543,28171
+microkernel/benchmark/footprint/footprint-min,x86,basic_minuteia,True,,TEST=min,True,,5588,4956
+nanokernel/test/test_lifo/test,x86,basic_atom,True,,,True,1.595146894454956,45652,16156
+nanokernel/benchmark/sys_kernel/test,x86,basic_atom,True,,,True,0.5303559303283691,25098,17310
+nanokernel/benchmark/footprint/footprint-min,arm,basic_cortex_m3,True,,TEST=min,True,,600,2054
+nanokernel/test/test_bluetooth/test-bluetooth,arm,fsl_frdm_k64f,True,,,False,,10008,27715
+microkernel/test/test_task_irq/test,arm,basic_cortex_m3,True,,,True,0.020154953002929688,11004,13662
+nanokernel/test/test_stack/test,x86,basic_minuteia,True,,,True,1.5868020057678223,19648,10236
+microkernel/test/test_sprintf/test,x86,galileo,True,,,False,,30585,23793
+microkernel/test/test_rand32/test,arm,basic_cortex_m3,True,,,True,0.01945209503173828,8608,10078
+microkernel/test/test_sema/test,x86,basic_minuteia,True,,,True,13.0003981590271,34827,18963
+microkernel/test/test_timer/test,x86,basic_minuteia,True,,,True,16.345667839050293,52238,24482
+nanokernel/test/test_static_idt/test,x86,galileo,True,,,False,,13340,9376
+microkernel/test/test_task/test,x86,basic_minuteia,True,,,True,3.4601569175720215,24416,15848
+nanokernel/test/test_context/test,x86,basic_atom,True,,,True,1.449923038482666,33780,12484
+microkernel/benchmark/footprint/footprint-max,x86,basic_minuteia,True,,TEST=max,True,,45162,36570
+microkernel/benchmark/footprint/footprint-reg,x86,basic_minuteia,True,,TEST=reg,True,,15831,13151
+nanokernel/apps/hello_world/test,x86,basic_atom,True,,,True,,11427,6223
+bluetooth/peripheral/test_arm,arm,fsl_frdm_k64f,True,,,False,,16256,37390
+nanokernel/apps/philosophers/test,arm,fsl_frdm_k64f,True,,,False,,9676,6841
+nanokernel/test/test_sema/test,arm,fsl_frdm_k64f,True,,,False,,29860,13864
+microkernel/test/test_bluetooth/test-bluetooth,arm,basic_cortex_m3,True,,,True,0.009977102279663086,15152,31381
+microkernel/test/test_rand32/test,x86,basic_minuteia,True,,,True,0.5146400928497314,17192,10716
+microkernel/benchmark/sys_kernel/test,x86,basic_atom,True,,,True,2.2390949726104736,50254,24802
+nanokernel/benchmark/footprint/footprint-min,arm,fsl_frdm_k64f,True,,TEST=min,False,,600,3154
+nanokernel/test/test_context/test,arm,basic_cortex_m3,True,,,True,6.3456408977508545,21644,9763
+microkernel/test/test_fifo/test,x86,basic_minuteia,True,,,True,0.4909679889678955,26113,17533
+nanokernel/apps/hello_world/test,arm,basic_cortex_m3,True,,,True,,5464,5276
+nanokernel/benchmark/footprint/footprint-min,x86,galileo,True,,TEST=min,False,,2144,1856
+nanokernel/apps/philosophers/test,x86,galileo,True,,,False,,18413,8749
+microkernel/test/test_fifo/test,arm,fsl_frdm_k64f,True,,,False,,10804,17127
+microkernel/benchmark/latency_measure/test,x86,galileo,True,,,False,,50975,30295
+nanokernel/test/test_fifo/test,x86,basic_minuteia,True,,,True,7.889343023300171,45138,17058
+microkernel/test/test_fifo/test,x86,galileo,True,,,False,,28997,20101
+microkernel/test/test_map/test,arm,fsl_frdm_k64f,True,,,False,,12820,15403
+microkernel/test/test_task/test,x86,galileo,True,,,False,,27232,18348
+microkernel/test/test_rand32/test,arm,fsl_frdm_k64f,True,,,False,,8624,11300
+nanokernel/benchmark/footprint/footprint-max,x86,basic_minuteia,True,,TEST=max,True,,15045,10881
+microkernel/test/test_sema/test,arm,fsl_frdm_k64f,True,,,False,,19244,17939
+microkernel/test/test_sprintf/test,arm,fsl_frdm_k64f,True,,,False,,8620,19101
+bluetooth/shell/test_x86,x86,basic_atom,True,,,True,,58301,43281
+microkernel/test/test_task/test,x86,basic_atom,True,,,True,1.00689697265625,25056,16436
+microkernel/benchmark/latency_measure/test,x86,basic_atom,True,,,True,2.2400739192962646,49127,28671
+nanokernel/benchmark/footprint/footprint-max,x86,galileo,True,,TEST=max,False,,17857,13385
+microkernel/test/test_critical/test,x86,basic_atom,True,,,True,3.27829909324646,20247,13687
+microkernel/test/test_pipe/test,arm,fsl_frdm_k64f,True,,,False,,14864,25262
+nanokernel/benchmark/footprint/footprint-reg,x86,galileo,True,,TEST=reg,False,,8990,7078
+nanokernel/test/test_xip/test,x86,galileo,True,,,False,,5588,7913
+microkernel/test/test_critical/test,x86,galileo,True,,,False,,22507,15683
+microkernel/test/test_timer/test,x86,galileo,True,,,False,,55238,27174
+microkernel/test/test_rand32/test,x86,basic_atom,True,,,True,0.49033498764038086,17804,11276
+microkernel/benchmark/footprint/footprint-min,x86,galileo,True,,TEST=min,False,,5696,5080
+microkernel/test/test_mail/test,arm,fsl_frdm_k64f,True,,,False,,13128,23559
+nanokernel/test/test_stack/test,x86,basic_atom,True,,,True,0.6568129062652588,20188,10576
+microkernel/test/test_events/test,arm,basic_cortex_m3,True,,,True,0.3083319664001465,11784,14366
+microkernel/test/test_bluetooth/test-bluetooth,x86,basic_atom,True,,,True,0.36398792266845703,51348,38248
+microkernel/test/test_critical/test,arm,basic_cortex_m3,True,,,True,14.526043891906738,8952,12164
+bluetooth/peripheral/test_arm,arm,basic_cortex_m3,True,,,True,,16252,36774
+microkernel/apps/philosophers/test,x86,basic_atom,True,,,True,,24329,12677
diff --git a/scripts/sanitycheck b/scripts/sanitycheck
new file mode 100755
index 0000000..1c41568
--- /dev/null
+++ b/scripts/sanitycheck
@@ -0,0 +1,1509 @@
+#!/usr/bin/env python
+"""Zephyr Sanity Tests
+
+This script scans for the set of unit test applications in the git
+repository and attempts to execute them. By default, it tries to
+build each test case on one platform per architecture, using a precedence
+list defined in an archtecture configuration file, and if possible
+run the tests in the QEMU emulator.
+
+Test cases are detected by the presence of a 'testcase.ini' file in
+the application's project directory. This file may contain one or
+more blocks, each identifying a test scenario. The title of the block
+is a name for the test case, which only needs to be unique for the
+test cases specified in that testcase.ini file. The full canonical
+name for each test case is <path to test case under samples/>/<block>.
+
+Each testcase.ini block can define the following key/value pairs:
+
+ tags = <list of tags> (required)
+ A set of string tags for the testcase. Usually pertains to
+ functional domains but can be anything. Command line invocations
+ of this script can filter the set of tests to run based on tag.
+
+ extra_args = <list of extra arguments>
+ Extra arguments to pass to Make when building or running the
+ test case.
+
+ build_only = <True|False>
+ If true, don't try to run the test under QEMU even if the
+ selected platform supports it.
+
+ timeout = <number of seconds>
+ Length of time to run test in QEMU before automatically killing it.
+ Default to 60 seconds.
+
+ arch_whitelist = <list of arches, such as x86, arm, arc>
+ Set of architectures that this test case should only be run for.
+
+ platform_whitelist = <list of platforms>
+ Set of platforms that this test case should only be run for for.
+
+ config_whitelist = <list of config options>
+ Config options can either be config names like CONFIG_FOO which
+ match if the configuration is defined to any value, or key/value
+ pairs like CONFIG_FOO=bar which match if it is set to a specific
+ value. May prepend a '!' to invert the match.
+
+Architectures and platforms are defined in an archtecture configuration
+file which are stored by default in scripts/sanity_chk/arches/. These
+each define an [arch] block with the following key/value pairs:
+
+ name = <arch name>
+ The name of the arch. Example: x86
+
+ platforms = <list of supported platforms in order of precedence>
+ List of supported platforms for this arch. The ordering here
+ is used to select a default platform to build for that arch.
+
+For every platform defined, there must be a corresponding block for it
+in the arch configuration file. This block can be empty if there are
+no special definitions for that arch. Options are:
+
+ qemu_support = <True|False> (default False)
+ Indicates whether binaries for this platform can run under QEMU
+
+ microkernel_support = <True|False> (default True)
+ Indicates whether this platform supports microkernel or just nanokernel
+
+The set of test cases that actually run depends on directives in the
+testcase and archtecture .ini file and options passed in on the command
+line. If there is every any confusion, running with -v or --discard-report
+can help show why particular test cases were skipped.
+
+Metrics (such as pass/fail state and binary size) for the last code
+release are stored in scripts/sanity_chk/sanity_last_release.csv.
+To update this, pass the --all --release options.
+
+Most everyday users will run with no arguments.
+"""
+
+import argparse
+import os
+import sys
+import ConfigParser
+import re
+import tempfile
+import subprocess
+import multiprocessing
+import select
+import shutil
+import signal
+import threading
+import time
+import csv
+
+if "ZEPHYR_BASE" not in os.environ:
+ sys.stderr.write("$ZEPHYR_BASE environment variable undefined")
+ exit(1)
+ZEPHYR_BASE = os.environ["ZEPHYR_BASE"]
+VERBOSE = 0
+LAST_SANITY = os.path.join(ZEPHYR_BASE, "scripts", "sanity_chk",
+ "last_sanity.csv")
+RELEASE_DATA = os.path.join(ZEPHYR_BASE, "scripts", "sanity_chk",
+ "sanity_last_release.csv")
+PARALLEL = multiprocessing.cpu_count() * 2
+
+if os.isatty(sys.stdout.fileno()):
+ TERMINAL = True
+ COLOR_NORMAL = '\033[0m'
+ COLOR_RED = '\033[91m'
+ COLOR_GREEN = '\033[92m'
+ COLOR_YELLOW = '\033[93m'
+else:
+ TERMINAL = False
+ COLOR_NORMAL = ""
+ COLOR_RED = ""
+ COLOR_GREEN = ""
+ COLOR_YELLOW = ""
+
+class SanityCheckException(Exception):
+ pass
+
+class SanityRuntimeError(SanityCheckException):
+ pass
+
+class ConfigurationError(SanityCheckException):
+ def __init__(self, cfile, message):
+ self.cfile = cfile
+ self.message = message
+
+ def __str__(self):
+ return repr(self.cfile + ": " + self.message)
+
+class MakeError(SanityCheckException):
+ pass
+
+class BuildError(MakeError):
+ pass
+
+class ExecutionError(MakeError):
+ pass
+
+# Debug Functions
+
+def debug(what):
+ if VERBOSE >= 1:
+ print what
+
+def error(what):
+ sys.stderr.write(COLOR_RED + what + COLOR_NORMAL + "\n")
+
+def verbose(what):
+ if VERBOSE >= 2:
+ print what
+
+def info(what):
+ sys.stdout.write(what + "\n")
+
+# Utility functions
+class QEMUHandler:
+ """Spawns a thread to monitor QEMU output from pipes
+
+ We pass QEMU_PIPE to 'make qemu' and monitor the pipes for output.
+ We need to do this as once qemu starts, it runs forever until killed.
+ Test cases emit special messages to the console as they run, we check
+ for these to collect whether the test passed or failed.
+ """
+ RUN_PASSED = "PROJECT EXECUTION SUCCESSFUL"
+ RUN_FAILED = "PROJECT EXECUTION FAILED"
+
+ @staticmethod
+ def _thread(handler, timeout, outdir, logfile, fifo_fn, pid_fn, results):
+ fifo_in = fifo_fn + ".in"
+ fifo_out = fifo_fn + ".out"
+
+ # These in/out nodes are named from QEMU's perspective, not ours
+ if os.path.exists(fifo_in):
+ os.unlink(fifo_in)
+ os.mkfifo(fifo_in)
+ if os.path.exists(fifo_out):
+ os.unlink(fifo_out)
+ os.mkfifo(fifo_out)
+
+ # We don't do anything with out_fp but we need to open it for
+ # writing so that QEMU doesn't block, due to the way pipes work
+ out_fp = open(fifo_in, "wb")
+ # Disable internal buffering, we don't
+ # want read() or poll() to ever block if there is data in there
+ in_fp = open(fifo_out, "rb", buffering=0)
+ log_out_fp = open(logfile, "w")
+
+ start_time = time.time()
+ timeout_time = start_time + timeout
+ p = select.poll()
+ p.register(in_fp, select.POLLIN)
+
+ metrics = {}
+ line = ""
+ while True:
+ this_timeout = int((timeout_time - time.time()) * 1000)
+ if this_timeout < 0 or not p.poll(this_timeout):
+ out_state = "timeout"
+ break
+
+ c = in_fp.read(1)
+ if c == "":
+ # EOF, this shouldn't happen unless QEMU crashes
+ out_state = "unexpected eof"
+ break
+ line = line + c
+ if c != "\n":
+ continue
+
+ # If we get here, line contains a full line of data output from QEMU
+ log_out_fp.write(line)
+ log_out_fp.flush()
+ line = line.strip()
+ verbose("QEMU: %s" % line)
+
+ if line == QEMUHandler.RUN_PASSED:
+ out_state = "passed"
+ break
+
+ if line == QEMUHandler.RUN_FAILED:
+ out_state = "failed"
+ break
+
+ # TODO: Add support for getting numerical performance data
+ # from test cases. Will involve extending test case reporting
+ # APIs. Add whatever gets reported to the metrics dictionary
+ line = ""
+
+ metrics["qemu_time"] = time.time() - start_time
+ verbose("QEMU complete (%s) after %f seconds" %
+ (out_state, metrics["qemu_time"]))
+ handler.set_state(out_state, metrics)
+
+ log_out_fp.close()
+ out_fp.close()
+ in_fp.close()
+
+ pid = int(open(pid_fn).read())
+ os.unlink(pid_fn)
+ os.kill(pid, signal.SIGTERM)
+ os.unlink(fifo_in)
+ os.unlink(fifo_out)
+
+
+ def __init__(self, name, outdir, log_fn, timeout):
+ """Constructor
+
+ @param name Arbitrary name of the created thread
+ @param outdir Working directory, shoudl be where qemu.pid gets created
+ by kbuild
+ @param log_fn Absolute path to write out QEMU's log data
+ @param timeout Kill the QEMU process if it doesn't finish up within
+ the given number of seconds
+ """
+ # Create pipe to get QEMU's serial output
+ self.results = {}
+ self.state = "waiting"
+ self.lock = threading.Lock()
+
+ # We pass this to QEMU which looks for fifos with .in and .out
+ # suffixes.
+ self.fifo_fn = os.path.join(outdir, "qemu-fifo")
+
+ self.pid_fn = os.path.join(outdir, "qemu.pid")
+ if os.path.exists(self.pid_fn):
+ os.unlink(self.pid_fn)
+
+ self.log_fn = log_fn
+ self.thread = threading.Thread(name=name, target=QEMUHandler._thread,
+ args=(self, timeout, outdir, self.log_fn,
+ self.fifo_fn, self.pid_fn,
+ self.results))
+ self.thread.daemon = True
+ verbose("Spawning QEMU process for %s" % name)
+ self.thread.start()
+
+ def set_state(self, state, metrics):
+ self.lock.acquire()
+ self.state = state
+ self.metrics = metrics
+ self.lock.release()
+
+ def get_state(self):
+ self.lock.acquire()
+ ret = (self.state, self.metrics)
+ self.lock.release()
+ return ret
+
+ def get_fifo(self):
+ return self.fifo_fn
+
+
+class SizeCalculator:
+ def __init__(self, filename_stem):
+ """Constructor
+
+ @param filename_stem Path to the output binary, minus file extension.
+ The <filename_stem>.elf is parsed by objdump. Either .elf or .bin
+ is sized for the ROM size depending on whether XIP is supported
+ """
+ elf_filename = filename_stem + ".elf"
+
+ # Make sure this is an ELF binary
+ with open(elf_filename, "rb") as f:
+ magic = f.read(4)
+
+ if (magic != "\x7fELF"):
+ raise SanityRuntimeError("%s is not an ELF binary" % elf_filename)
+
+ # Search for CONFIG_XIP in the ELF's list of symbols using NM and AWK.
+ # GREP can not be used as it returns an error if the symbol is not found.
+ is_xip_command = "nm " + elf_filename + " | awk '/CONFIG_XIP/ { print $3 }'"
+ is_xip_output = subprocess.check_output(is_xip_command, shell=True)
+ self.is_xip = (len(is_xip_output) != 0)
+
+ self.elf_filename = elf_filename
+ self.sections = {}
+ self.xip_rom_size = 0
+ self.xip_ram_size = 0
+ self.ram_size = 0
+
+ self._calculate_sizes()
+
+ def get_ram_size(self):
+ """Get the amount of RAM the application will use up on the device
+
+ @return amount of RAM, in bytes
+ """
+ if self.is_xip:
+ return self.xip_ram_size
+ else:
+ return self.ram_size
+
+ def get_rom_size(self):
+ """Get the size of the data that this application uses on device's flash
+
+ @return amount of ROM, in bytes
+ """
+ return self.xip_rom_size
+
+ def unrecognized_sections(self):
+ """Get a list of sections inside the binary that weren't recognized
+
+ @return list of unrecogized section names
+ """
+ slist = []
+ for k, v in self.sections.iteritems():
+ if not v["recognized"]:
+ slist.append(k)
+ return slist
+
+ def _calculate_sizes(self):
+ """ Calculate RAM and ROM usage by section """
+ objdump_command = "objdump -h " + self.elf_filename
+ objdump_output = subprocess.check_output(objdump_command,
+ shell=True).splitlines()
+
+ for line in objdump_output:
+ words = line.split()
+
+ if (len(words) == 0): # Skip lines that are too short
+ continue
+
+ index = words[0]
+ if (not index[0].isdigit()): # Skip lines that do not start
+ continue # with a digit
+
+ name = words[1] # Skip lines with section names
+ if (name[0] == '.'): # starting with '.'
+ continue
+
+ size = int(words[2], 16)
+ phys_addr = int(words[4], 16)
+
+ # Add section to memory use totals (for both non-XIP and XIP scenarios)
+ #
+ # In an XIP image, the following sections are placed into ROM:
+ # text, ctors, rodata and datas
+ # In an XIP image, the following sections are placed into RAM:
+ # datas, bss and noinit
+ # In a non-XIP image, the following sections are placed into RAM
+ # text, ctors, rodata, datas, bss and noinit
+ # Unrecognized section names are not included in the calculations.
+
+ self.ram_size += size
+ recognized = True
+
+ if ((name == "text") or (name == "ctors") or (name == "rodata")):
+ self.xip_rom_size += size
+ elif (name == "datas"):
+ self.xip_rom_size += size
+ self.xip_ram_size += size
+ elif ((name == "bss") or (name == "noinit")):
+ self.xip_ram_size += size
+ else:
+ recognized = False
+ self.ram_size -= size # Undo the calculation
+
+ self.sections[name] = {"phys_addr" : phys_addr, "size" : size,
+ "recognized" : recognized}
+
+
+class MakeGoal:
+ """Metadata class representing one of the sub-makes called by MakeGenerator
+
+ MakeGenerator returns a dictionary of these which can then be associdated
+ with TestInstances to get a complete picture of what happened during a test.
+ MakeGenerator is used for tasks outside of building tests (such as
+ defconfigs) which is why MakeGoal is a separate class from TestInstance.
+ """
+ def __init__(self, name, text, qemu, make_log, build_log, run_log,
+ qemu_log):
+ self.name = name
+ self.text = text
+ self.qemu = qemu
+ self.make_log = make_log
+ self.build_log = build_log
+ self.run_log = run_log
+ self.qemu_log = qemu_log
+ self.make_state = "waiting"
+ self.failed = False
+ self.finished = False
+ self.reason = None
+ self.metrics = {}
+
+ def get_error_log(self):
+ if self.make_state == "waiting":
+ # Shouldn't ever see this; breakage in the main Makefile itself.
+ return self.make_log
+ elif self.make_state == "building":
+ # Failure when calling the sub-make to build the code
+ return self.build_log
+ elif self.make_state == "running":
+ # Failure in sub-make for "make qemu", qemu probably failed to start
+ return self.run_log
+ elif self.make_state == "finished":
+ # QEMU finished, but timed out or otherwise wasn't successful
+ return self.qemu_log
+
+ def fail(self, reason):
+ self.failed = True
+ self.finished = True
+ self.reason = reason
+
+ def success(self):
+ self.finished = True
+
+ def __str__(self):
+ if self.finished:
+ if self.failed:
+ return "[%s] failed (%s: see %s)" % (self.name, self.reason,
+ self.get_error_log())
+ else:
+ return "[%s] passed" % self.name
+ else:
+ return "[%s] in progress (%s)" % (self.name, self.make_state)
+
+
+class MakeGenerator:
+ """Generates a Makefile which just calls a bunch of sub-make sessions
+
+ In any given test suite we may need to build dozens if not hundreds of
+ test cases. The cleanest way to parallelize this is to just let Make
+ do the parallelization, sharing the jobserver among all the different
+ sub-make targets.
+ """
+
+ GOAL_HEADER_TMPL = """.PHONY: {goal}
+{goal}:
+"""
+
+ MAKE_RULE_TMPL = """\t@echo sanity_test_{phase} {goal} >&2
+\t$(MAKE) -C {directory} O={outdir} V={verb} {args} >{logfile} 2>&1
+"""
+
+ GOAL_FOOTER_TMPL = "\t@echo sanity_test_finished {goal} >&2\n\n"
+
+ re_make = re.compile("sanity_test_([A-Za-z0-9]+) (.+)|$|make[:] \*\*\* [[](.+)[]] Error.+$")
+
+ def __init__(self, base_outdir):
+ """MakeGenerator constructor
+
+ @param base_outdir Intended to be the base out directory. A make.log
+ file will be created here which contains the output of the
+ top-level Make session, as well as the dynamic control Makefile
+ @param verbose If true, pass V=1 to all the sub-makes which greatly
+ increases their verbosity
+ """
+ self.goals = {}
+ if not os.path.exists(base_outdir):
+ os.makedirs(base_outdir)
+ self.logfile = os.path.join(base_outdir, "make.log")
+ self.makefile = os.path.join(base_outdir, "Makefile")
+
+ def _get_rule_header(self, name):
+ return MakeGenerator.GOAL_HEADER_TMPL.format(goal=name)
+
+ def _get_sub_make(self, name, phase, workdir, outdir, logfile, args):
+ verb = "1" if VERBOSE else "0"
+ args = " ".join(args)
+ return MakeGenerator.MAKE_RULE_TMPL.format(phase=phase, goal=name,
+ outdir=outdir,
+ directory=workdir, verb=verb,
+ args=args, logfile=logfile)
+
+ def _get_rule_footer(self, name):
+ return MakeGenerator.GOAL_FOOTER_TMPL.format(goal=name)
+
+ def _add_goal(self, outdir):
+ if not os.path.exists(outdir):
+ os.makedirs(outdir)
+
+ def add_build_goal(self, name, directory, outdir, args):
+ """Add a goal to invoke a Kbuild session
+
+ @param name A unique string name for this build goal. The results
+ dictionary returned by execute() will be keyed by this name.
+ @param directory Absolute path to working directory, will be passed
+ to make -C
+ @param outdir Absolute path to output directory, will be passed to
+ Kbuild via -O=<path>
+ @param args Extra command line arguments to pass to 'make', typically
+ environment variables or specific Make goals
+ """
+ self._add_goal(outdir)
+ build_logfile = os.path.join(outdir, "build.log")
+ text = (self._get_rule_header(name) +
+ self._get_sub_make(name, "building", directory,
+ outdir, build_logfile, args) +
+ self._get_rule_footer(name))
+ self.goals[name] = MakeGoal(name, text, None, self.logfile, build_logfile,
+ None, None)
+
+ def add_qemu_goal(self, name, directory, outdir, args, timeout=30):
+ """Add a goal to build a Zephyr project and then run it under QEMU
+
+ The generated make goal invokes Make twice, the first time it will
+ build the default goal, and the second will invoke the 'qemu' goal.
+ The output of the QEMU session will be monitored, and terminated
+ either upon pass/fail result of the test program, or the timeout
+ is reached.
+
+ @param name A unique string name for this build goal. The results
+ dictionary returned by execute() will be keyed by this name.
+ @param directory Absolute path to working directory, will be passed
+ to make -C
+ @param outdir Absolute path to output directory, will be passed to
+ Kbuild via -O=<path>
+ @param args Extra command line arguments to pass to 'make', typically
+ environment variables. Do not pass specific Make goals here.
+ @param timeout Maximum length of time QEMU session should be allowed
+ to run before automatically killing it. Default is 30 seconds.
+ """
+
+ self._add_goal(outdir)
+ build_logfile = os.path.join(outdir, "build.log")
+ run_logfile = os.path.join(outdir, "run.log")
+ qemu_logfile = os.path.join(outdir, "qemu.log")
+
+ q = QEMUHandler(name, outdir, qemu_logfile, timeout)
+ args.append("QEMU_PIPE=%s" % q.get_fifo())
+ text = (self._get_rule_header(name) +
+ self._get_sub_make(name, "building", directory,
+ outdir, build_logfile, args) +
+ self._get_sub_make(name, "running", directory,
+ outdir, run_logfile,
+ args + ["qemu"]) +
+ self._get_rule_footer(name))
+ self.goals[name] = MakeGoal(name, text, q, self.logfile, build_logfile,
+ run_logfile, qemu_logfile)
+
+
+ def add_test_instance(self, ti, build_only=False):
+ """Add a goal to build/test a TestInstance object
+
+ @param ti TestInstance object to build. The status dictionary returned
+ by execute() will be keyed by its .name field.
+ """
+ args = ti.test.extra_args[:]
+ args.extend(["ARCH=%s" % ti.platform.arch.name,
+ "PLATFORM_CONFIG=%s" % ti.platform.name])
+ if ti.platform.qemu_support and not ti.build_only and not build_only:
+ self.add_qemu_goal(ti.name, ti.test.code_location, ti.outdir,
+ args, ti.test.timeout)
+ else:
+ self.add_build_goal(ti.name, ti.test.code_location, ti.outdir, args)
+
+ def execute(self, callback_fn=None, context=None):
+ """Execute all the registered build goals
+
+ @param callback_fn If not None, a callback function will be called
+ as individual goals transition between states. This function
+ should accept two parameters: a string state and an arbitrary
+ context object, supplied here
+ @param context Context object to pass to the callback function.
+ Type and semantics are specific to that callback function.
+ @return A dictionary mapping goal names to final status.
+ """
+
+ with open(self.makefile, "w") as tf, \
+ open(os.devnull, "wb") as devnull, \
+ open(self.logfile, "w") as make_log:
+ # Create our dynamic Makefile and execute it.
+ # Watch stderr output which is where we will keep
+ # track of build state
+ for name, goal in self.goals.iteritems():
+ tf.write(goal.text)
+ tf.write("all: %s\n" % (" ".join(self.goals.keys())))
+ tf.flush()
+
+ # os.environ["CC"] = "ccache gcc" FIXME doesn't work
+
+ cmd = ["make", "-k", "-j", str(PARALLEL), "-f", tf.name, "all"]
+ p = subprocess.Popen(cmd, stderr=subprocess.PIPE,
+ stdout=devnull)
+
+ for line in iter(p.stderr.readline, b''):
+ make_log.write(line)
+ verbose("MAKE: " + repr(line.strip()))
+ m = MakeGenerator.re_make.match(line)
+ if not m:
+ continue
+
+ state, name, error = m.groups()
+ if error:
+ goal = self.goals[error]
+ else:
+ goal = self.goals[name]
+ goal.make_state = state
+
+
+ if error:
+ goal.fail("build_error")
+ else:
+ if state == "finished":
+ if goal.qemu:
+ thread_status, metrics = goal.qemu.get_state()
+ goal.metrics.update(metrics)
+ if thread_status == "passed":
+ goal.success()
+ else:
+ goal.fail(thread_status)
+ else:
+ goal.success()
+
+ if callback_fn:
+ callback_fn(context, self.goals, goal)
+
+ p.wait()
+ return self.goals
+
+
+# "list" - List of strings
+# "list:<type>" - List of <type>
+# "set" - Set of unordered, unique strings
+# "set:<type>" - Set of <type>
+# "float" - Floating point
+# "int" - Integer
+# "bool" - Boolean
+# "str" - String
+
+# XXX Be sure to update __doc__ if you change any of this!!
+
+arch_valid_keys = {"name" : {"type" : "str", "required" : True},
+ "platforms" : {"type" : "list", "required" : True}}
+
+platform_valid_keys = {"qemu_support" : {"type" : "bool", "default" : False},
+ "microkernel_support" : {"type" : "bool",
+ "default" : True}}
+
+testcase_valid_keys = {"tags" : {"type" : "set", "required" : True},
+ "extra_args" : {"type" : "list"},
+ "build_only" : {"type" : "bool", "default" : False},
+ "timeout" : {"type" : "int", "default" : 60},
+ "arch_whitelist" : {"type" : "set"},
+ "platform_whitelist" : {"type" : "set"},
+ "config_whitelist" : {"type" : "set"}}
+
+
+class SanityConfigParser:
+ """Class to read architecture and test case .ini files with semantic checking
+ """
+ def __init__(self, filename):
+ """Instantiate a new SanityConfigParser object
+
+ @param filename Source .ini file to read
+ """
+ cp = ConfigParser.SafeConfigParser()
+ cp.readfp(open(filename))
+ self.filename = filename
+ self.cp = cp
+
+ def _cast_value(self, value, typestr):
+ v = value.strip()
+ if typestr == "str":
+ return v
+
+ elif typestr == "float":
+ return float(v)
+
+ elif typestr == "int":
+ return int(v)
+
+ elif typestr == "bool":
+ v = v.lower()
+ if v == "true" or v == "1":
+ return True
+ elif v == "" or v == "false" or v == "0":
+ return False
+ raise ConfigurationError(self.filename,
+ "bad value for boolean: '%s'" % value)
+
+ elif typestr.startswith("list"):
+ vs = v.split()
+ if len(typestr) > 4 and typestr[4] == ":":
+ return [self._cast_value(vsi, typestr[5:]) for vsi in vs]
+ else:
+ return vs
+
+ elif typestr.startswith("set"):
+ vs = v.split()
+ if len(typestr) > 3 and typestr[3] == ":":
+ return set([self._cast_value(vsi, typestr[4:]) for vsi in vs])
+ else:
+ return set(vs)
+
+ else:
+ raise ConfigurationError(self.filename, "unknown type '%s'" % value)
+
+
+ def sections(self):
+ """Get the set of sections within the .ini file
+
+ @return a list of string section names"""
+ return self.cp.sections()
+
+ def get_section(self, section, valid_keys):
+ """Get a dictionary representing the keys/values within a section
+
+ @param section The section in the .ini file to retrieve data from
+ @param valid_keys A dictionary representing the intended semantics
+ for this section. Each key in this dictionary is a key that could
+ be specified, if a key is given in the .ini file which isn't in
+ here, it will generate an error. Each value in this dictionary
+ is another dictionary containing metadata:
+
+ "default" - Default value if not given
+ "type" - Data type to convert the text value to. Simple types
+ supported are "str", "float", "int", "bool" which will get
+ converted to respective Python data types. "set" and "list"
+ may also be specified which will split the value by
+ whitespace (but keep the elements as strings). finally,
+ "list:<type>" and "set:<type>" may be given which will
+ perform a type conversion after splitting the value up.
+ "required" - If true, raise an error if not defined. If false
+ and "default" isn't specified, a type conversion will be
+ done on an empty string
+ @return A dictionary containing the section key-value pairs with
+ type conversion and default values filled in per valid_keys
+ """
+
+ d = {}
+ cp = self.cp
+
+ if not cp.has_section(section):
+ raise ConfigurationError(self.filename, "Missing section '%s'" % section)
+
+ for k, v in cp.items(section):
+ if k not in valid_keys:
+ raise ConfigurationError(self.filename,
+ "Unknown config key '%s' in defintiion for '%s'"
+ % (k, section))
+ d[k] = v
+
+ for k, kinfo in valid_keys.iteritems():
+ if k not in d:
+ if "required" in kinfo:
+ required = kinfo["required"]
+ else:
+ required = False
+
+ if required:
+ raise ConfigurationError(self.filename,
+ "missing required value for '%s' in section '%s'"
+ % (k, section))
+ else:
+ if "default" in kinfo:
+ default = kinfo["default"]
+ else:
+ default = self._cast_value("", kinfo["type"])
+ d[k] = default
+ else:
+ try:
+ d[k] = self._cast_value(d[k], kinfo["type"])
+ except ValueError, ve:
+ raise ConfigurationError(self.filename,
+ "bad %s value '%s' for key '%s' in section '%s'"
+ % (kinfo["type"], d[k], k, section))
+
+ return d
+
+
+class Platform:
+ """Class representing metadata for a particular platform
+
+ Maps directly to PLATFORM_CONFIG when building"""
+ def __init__(self, arch, name, plat_dict):
+ """Constructor.
+
+ @param arch Architecture object for this platform
+ @param name String name for this platform, same as PLATFORM_CONFIG
+ @param plat_dict SanityConfigParser output on the relevant section
+ in the architecture configuration file which has lots of metadata.
+ See the Architecture class.
+ """
+ self.name = name
+ self.qemu_support = plat_dict["qemu_support"]
+ self.microkernel_support = plat_dict["microkernel_support"]
+ self.arch = arch
+ # Gets populated in a separate step
+ self.defconfig = {"micro" : None, "nano" : None}
+ pass
+
+ def set_defconfig(self, ktype, defconfig):
+ """Set defconfig information for a particular kernel type.
+
+ We do this in another step because all the defconfigs are generated
+ at once from a sub-make, see TestSuite constructor
+
+ @param ktype Kernel type, either "micro" or "nano"
+ @param defconfig Dictionary containing defconfig information
+ """
+ self.defconfig[ktype] = defconfig
+
+ def get_defconfig(self, ktype):
+ """Return a dictionary representing the key/value pairs expressed
+ in the kernel defconfig used for this arch/platform. Used to identify
+ platform features.
+
+ @param ktype Kernel type, either "micro" or "nano"
+ @return dictionary corresponding to the defconfig contents. unset
+ values will not be defined
+ """
+
+ if ktype == "micro" and not self.microkernel_support:
+ raise SanityRuntimeError("Invalid kernel type queried")
+
+ return self.defconfig[ktype]
+
+ def __repr__(self):
+ return "<%s on %s>" % (self.name, self.arch.name)
+
+
+class Architecture:
+ """Class representing metadata for a particular architecture
+ """
+ def __init__(self, cfile):
+ """Architecture constructor
+
+ @param cfile Path to Architecture configuration file, which gives
+ info about the arch and all the platforms for it
+ """
+ cp = SanityConfigParser(cfile)
+ self.platforms = []
+
+ arch = cp.get_section("arch", arch_valid_keys)
+
+ self.name = arch["name"]
+
+ for plat_name in arch["platforms"]:
+ verbose("Platform: %s" % plat_name)
+ plat_dict = cp.get_section(plat_name, platform_valid_keys)
+ self.platforms.append(Platform(self, plat_name, plat_dict))
+
+ def __repr__(self):
+ return "<arch %s>" % self.name
+
+
+class TestCase:
+ """Class representing a test application
+ """
+ makefile_re = re.compile("\s*KERNEL_TYPE\s*[?=]+\s*(micro|nano)\s*")
+
+ def __init__(self, testcase_root, workdir, name, tc_dict):
+ """TestCase constructor.
+
+ This gets called by TestSuite as it finds and reads testcase.ini files.
+ Multiple TestCase instances may be generated from a single testcase.ini,
+ each one corresponds to a section within that file.
+
+ Reads the Makefile inside the testcase directory to figure out the
+ kernel type for purposes of configuration filtering
+
+ We need to have a unique name for every single test case. Since
+ a testcase.ini can define multiple tests, the canonical name for
+ the test case is <workdir>/<name>.
+
+ @param testcase_root Absolute path to the root directory where
+ all the test cases live
+ @param workdir Relative path to the project directory for this
+ test application from the test_case root.
+ @param name Name of this test case, corresponding to the section name
+ in the test case configuration file. For many test cases that just
+ define one test, can be anything and is usually "test". This is
+ really only used to distinguish between different cases when
+ the testcase.ini defines multiple tests
+ @param tc_dict Dictionary with section values for this test case
+ from the testcase.ini file
+ """
+ self.code_location = os.path.join(testcase_root, workdir)
+ self.tags = tc_dict["tags"]
+ self.extra_args = tc_dict["extra_args"]
+ self.arch_whitelist = tc_dict["arch_whitelist"]
+ self.platform_whitelist = tc_dict["platform_whitelist"]
+ self.config_whitelist = tc_dict["config_whitelist"]
+ self.timeout = tc_dict["timeout"]
+ self.build_only = tc_dict["build_only"]
+ self.path = os.path.join(workdir, name)
+ self.name = self.path # for now
+ self.ktype = None
+
+ with open(os.path.join(testcase_root, workdir, "Makefile")) as makefile:
+ for line in makefile.readlines():
+ m = TestCase.makefile_re.match(line)
+ if m:
+ self.ktype = m.group(1)
+ break
+ if not self.ktype:
+ raise ConfigurationError(os.path.join(workdir, "Makefile"),
+ "KERNEL_TYPE not found")
+
+ def __repr__(self):
+ return self.name
+
+
+
+class TestInstance:
+ """Class representing the execution of a particular TestCase on a platform
+
+ @param test The TestCase object we want to build/execute
+ @param platform Platform object that we want to build and run against
+ @param base_outdir Base directory for all test results. The actual
+ out directory used is <outdir>/<platform>/<test case name>
+ """
+ def __init__(self, test, platform, base_outdir, build_only=False):
+ self.test = test
+ self.platform = platform
+ self.name = os.path.join(platform.name, test.path)
+ self.outdir = os.path.join(base_outdir, platform.name, test.path)
+ self.build_only = build_only or test.build_only
+
+ def calculate_sizes(self):
+ """Get the RAM/ROM sizes of a test case.
+
+ This can only be run after the instance has been executed by
+ MakeGenerator, otherwise there won't be any binaries to measure.
+
+ @return A SizeCalculator object
+ """
+ if self.test.ktype == "micro":
+ binary_stem = os.path.join(self.outdir, "microkernel")
+ else:
+ binary_stem = os.path.join(self.outdir, "nanokernel")
+
+ return SizeCalculator(binary_stem)
+
+ def __repr__(self):
+ return "<TestCase %s on %s>" % (self.test.name, self.platform.name)
+
+
+
+class TestSuite:
+ config_re = re.compile('(CONFIG_[A-Z0-9_]+)[=](.+)$')
+
+ def __init__(self, arch_root, testcase_root, outdir):
+ # Keep track of which test cases we've filtered out and why
+ discards = {}
+ self.arches = {}
+ self.testcases = {}
+ self.platforms = []
+ self.outdir = outdir
+ self.instances = {}
+ self.goals = None
+ self.discards = None
+
+ arch_root = os.path.abspath(arch_root)
+ testcase_root = os.path.abspath(testcase_root)
+ outdir = os.path.abspath(outdir)
+
+ debug("Reading test case configuration files under %s..." % testcase_root)
+ for dirpath, dirnames, filenames in os.walk(testcase_root,
+ topdown=True):
+ if "testcase.ini" in filenames:
+ verbose("Found test case in " + dirpath)
+ dirnames[:] = []
+ cp = SanityConfigParser(os.path.join(dirpath, "testcase.ini"))
+ workdir = os.path.relpath(dirpath, testcase_root)
+
+ for section in cp.sections():
+ tc_dict = cp.get_section(section, testcase_valid_keys)
+ tc = TestCase(testcase_root, workdir, section, tc_dict)
+ self.testcases[tc.name] = tc
+
+ debug("Reading architecture configuration files under %s..." % arch_root)
+ for dirpath, dirnames, filenames in os.walk(arch_root):
+ for filename in filenames:
+ if filename.endswith(".ini"):
+ fn = os.path.join(dirpath, filename)
+ verbose("Found arch configuration " + fn)
+ arch = Architecture(fn)
+ self.arches[arch.name] = arch
+ self.platforms.extend(arch.platforms)
+
+
+ # Now that we know the full set of arches/platforms, get the defconfig
+ # information from them by calling Make
+ info("Building platform defconfigs...")
+ dlist = {}
+ config_outdir = os.path.join(outdir, "configs")
+ mg = MakeGenerator(config_outdir)
+
+ for plat in self.platforms:
+ ktypes = ["nano"]
+ if plat.microkernel_support:
+ ktypes.append("micro")
+
+ for ktype in ktypes:
+ stem = ktype + "_" + plat.name
+
+ in_defconfig = stem + "_defconfig"
+ out_config = os.path.join(config_outdir, stem + "_config")
+ dlist[plat, ktype] = out_config
+
+ args = ["ARCH=" + plat.arch.name,
+ "KBUILD_DEFCONFIG=" + in_defconfig,
+ "KCONFIG_CONFIG=" + out_config, "defconfig"]
+ # FIXME would be nice to use a common outdir for this so that
+ # conf, gen_idt, etc aren't rebuilt for every plat/ktype combo,
+ # need a way to avoid different Make processe from clobbering
+ # each other since they all try to build them simultaneously
+ mg.add_build_goal(stem, ZEPHYR_BASE, os.path.join(config_outdir,
+ plat.name,
+ ktype), args)
+
+ results = mg.execute(None)
+
+ for k, out_config in dlist.iteritems():
+ plat, ktype = k
+ defconfig = {}
+ with open(out_config, "r") as fp:
+ for line in fp.readlines():
+ m = TestSuite.config_re.match(line)
+ if not m:
+ continue
+ defconfig[m.group(1)] = m.group(2).strip()
+ plat.set_defconfig(ktype, defconfig)
+
+ self.instances = {}
+
+ def get_last_failed(self):
+ if not os.path.exists(LAST_SANITY):
+ return []
+ result = []
+ with open(LAST_SANITY, "r") as fp:
+ cr = csv.DictReader(fp)
+ for row in cr:
+ if row["passed"] == "True":
+ continue
+ test = row["test"]
+ platform = row["platform"]
+ result.append((test, platform))
+ return result
+
+ def apply_filters(self, platform_filter, arch_filter, tag_filter,
+ config_filter, testcase_filter, last_failed):
+ instances = []
+ discards = {}
+ verbose("platform filter: " + str(platform_filter))
+ verbose(" arch_filter: " + str(arch_filter))
+ verbose(" tag_filter: " + str(tag_filter))
+ verbose(" config_filter: " + str(config_filter))
+
+ if last_failed:
+ failed_tests = self.get_last_failed()
+
+ if not platform_filter or "default" in platform_filter:
+ info("Selecting default platforms per test case")
+ default_platforms = True
+ platform_filter = []
+ else:
+ default_platforms = False
+
+ if "all" in platform_filter:
+ info("Selecting all possible platforms per test case")
+ platform_filter = []
+
+ for tc_name, tc in self.testcases.iteritems():
+ for arch_name, arch in self.arches.iteritems():
+ instance_list = []
+ for plat in arch.platforms:
+ instance = TestInstance(tc, plat, self.outdir)
+
+ if tag_filter and not tc.tags.intersection(tag_filter):
+ discards[instance] = "Command line testcase tag filter"
+ continue
+
+ if testcase_filter and tc_name not in testcase_filter:
+ discards[instance] = "Testcase name filter"
+ continue
+
+ if last_failed and (tc.name, plat.name) not in failed_tests:
+ discards[instance] = "Passed or skipped during last run"
+ continue
+
+ if arch_filter and arch_name not in arch_filter:
+ discards[instance] = "Command line testcase arch filter"
+ continue
+
+ if tc.arch_whitelist and arch.name not in tc.arch_whitelist:
+ discards[instance] = "Not in test case arch whitelist"
+ continue
+
+ if platform_filter and plat.name not in platform_filter:
+ discards[instance] = "Command line platform filter"
+ continue
+
+ if tc.platform_whitelist and plat.name not in tc.platform_whitelist:
+ discards[instance] = "Not in testcase platform whitelist"
+ continue
+
+ if not plat.microkernel_support and tc.ktype == "micro":
+ discards[instance] = "No microkernel support for platform"
+ continue
+
+ defconfig = plat.get_defconfig(tc.ktype)
+ config_pass = True
+ # FIXME this is kind of gross clean it up
+ for cw in tc.config_whitelist:
+ invert = (cw[0] == "!")
+ if invert:
+ cw = cw[1:]
+
+ if "=" in cw:
+ k, v = cw.split("=")
+ testval = k not in defconfig or defconfig[k] != v
+ if invert:
+ testval = not testval
+ if testval:
+ discards[instance] = "%s%s in platform defconfig" % (
+ cw, " not" if not invert else "")
+ config_pass = False
+ break
+ else:
+ testval = cw not in defconfig
+ if invert:
+ testval = not testval
+ if testval:
+ discards[instance] = "%s%s set in platform defconfig" % (
+ cw, " not" if not invert else "")
+ config_pass = False
+ break
+
+ if not config_pass:
+ continue
+
+ instance_list.append(instance)
+
+ if not instance_list:
+ # Every platform in this arch was rejected already
+ continue
+
+ if default_platforms:
+ self.add_instance(instance_list[0])
+ for instance in instance_list[1:]:
+ discards[instance] = "Not in default set for arch"
+ else:
+ for instance in instance_list:
+ self.add_instance(instance)
+ self.discards = discards
+ return discards
+
+ def add_instance(self, ti):
+ self.instances[ti.name] = ti
+
+ def execute(self, cb, cb_context, build_only):
+ mg = MakeGenerator(self.outdir)
+ for i in self.instances.values():
+ mg.add_test_instance(i, build_only)
+ self.goals = mg.execute(cb, cb_context)
+ for name, goal in self.goals.iteritems():
+ i = self.instances[name]
+ if goal.failed:
+ continue
+ sc = i.calculate_sizes()
+ goal.metrics["ram_size"] = sc.get_ram_size()
+ goal.metrics["rom_size"] = sc.get_rom_size()
+ return self.goals
+
+ def discard_report(self, filename):
+ if self.discards == None:
+ raise SanityRuntimeException("apply_filters() hasn't been run!")
+
+ with open(filename, "wb") as csvfile:
+ fieldnames = ["test", "arch", "platform", "reason"]
+ cw = csv.DictWriter(csvfile, fieldnames, lineterminator=os.linesep)
+ cw.writeheader()
+ for instance, reason in self.discards.iteritems():
+ rowdict = {"test" : i.test.name,
+ "arch" : i.platform.arch.name,
+ "platform" : i.platform.name,
+ "reason" : reason}
+ cw.writerow(rowdict)
+
+ def compare_metrics(self, filename):
+ # name, datatype, lower results better
+ interesting_metrics = [("ram_size", int, True),
+ ("rom_size", int, True)]
+
+ if self.goals == None:
+ raise SanityRuntimeException("execute() hasn't been run!")
+
+ if not os.path.exists(filename):
+ info("Cannot compare metrics, %s not found" % filename)
+ return []
+
+ results = []
+ saved_metrics = {}
+ with open(filename) as fp:
+ cr = csv.DictReader(fp)
+ for row in cr:
+ d = {}
+ for m, _, _ in interesting_metrics:
+ d[m] = row[m]
+ saved_metrics[(row["test"], row["platform"])] = d
+
+ for name, goal in self.goals.iteritems():
+ i = self.instances[name]
+ mkey = (i.test.name, i.platform.name)
+ if mkey not in saved_metrics:
+ continue
+ sm = saved_metrics[mkey]
+ for metric, mtype, lower_better in interesting_metrics:
+ if metric not in goal.metrics:
+ continue
+ if sm[metric] == "":
+ continue
+ delta = goal.metrics[metric] - mtype(sm[metric])
+ if ((lower_better and delta > 0) or
+ (not lower_better and delta < 0)):
+ results.append((i, metric, goal.metrics[metric], delta))
+ return results
+
+ def testcase_report(self, filename):
+ if self.goals == None:
+ raise SanityRuntimeException("execute() hasn't been run!")
+
+ with open(filename, "wb") as csvfile:
+ fieldnames = ["test", "arch", "platform", "passed", "status",
+ "extra_args", "qemu", "qemu_time", "ram_size",
+ "rom_size"]
+ cw = csv.DictWriter(csvfile, fieldnames, lineterminator=os.linesep)
+ cw.writeheader()
+ for name, goal in self.goals.iteritems():
+ i = self.instances[name]
+ rowdict = {"test" : i.test.name,
+ "arch" : i.platform.arch.name,
+ "platform" : i.platform.name,
+ "extra_args" : " ".join(i.test.extra_args),
+ "qemu" : i.platform.qemu_support}
+ if goal.failed:
+ rowdict["passed"] = False
+ rowdict["status"] = goal.reason
+ else:
+ rowdict["passed"] = True
+ if goal.qemu:
+ rowdict["qemu_time"] = goal.metrics["qemu_time"]
+ rowdict["ram_size"] = goal.metrics["ram_size"]
+ rowdict["rom_size"] = goal.metrics["rom_size"]
+ cw.writerow(rowdict)
+
+
+def parse_arguments():
+
+ parser = argparse.ArgumentParser(description = __doc__,
+ formatter_class = argparse.RawDescriptionHelpFormatter)
+
+ parser.add_argument("-p", "--platform", action="append",
+ help="Platform filter for testing. If unspecified, default to the "
+ "set of default platforms in the arch configuration files for "
+ "the selected arches. May also specify 'all' to match all "
+ "platforms for the selected arches. Multiple invocations "
+ "are treated as a logical 'or' relationship")
+ parser.add_argument("-a", "--arch", action="append",
+ help="Arch filter for testing. Takes precedence over --platform. "
+ "If unspecified, test all arches. Multiple invocations "
+ "are treated as a logical 'or' relationship")
+ parser.add_argument("-t", "--tag", action="append",
+ help="Specify tags to restrict which tests to run by tag value. "
+ "Default is to not do any tag filtering. Multiple invocations "
+ "are treated as a logical 'or' relationship")
+ parser.add_argument("-f", "--only-failed", action="store_true",
+ help="Run only those tests that failed the previous sanity check "
+ "invocation.")
+ parser.add_argument("-c", "--config", action="append",
+ help="Specify platform configuration values filtering. This can be "
+ "specified two ways: <config>=<value> or just <config>. The "
+ "defconfig for all platforms, for all kernel types will be "
+ "checked. For the <config>=<value> case, only match defconfig "
+ "that have that value defined. For the <config> case, match "
+ "defconfig that have that value assigned to any value. "
+ "Prepend a '!' to invert the match.")
+ parser.add_argument("-s", "--test", action="append",
+ help="Run only the specified test cases. These are named by "
+ "<path to test project relative to "
+ "--testcase-root>/<testcase.ini section name>")
+ parser.add_argument("-l", "--all", action="store_true",
+ help="Same as --platform all")
+
+ parser.add_argument("-o", "--testcase-report",
+ help="Output a CSV spreadsheet containing results of the test run")
+ parser.add_argument("-d", "--discard-report",
+ help="Output a CSV spreadhseet showing tests that were skipped "
+ "and why")
+ parser.add_argument("-y", "--dry-run", action="store_true",
+ help="Create the filtered list of test cases, but don't actually "
+ "run them. Useful if you're just interested in "
+ "--discard-report")
+
+ parser.add_argument("-r", "--release", action="store_true",
+ help="Update the benchmark database with the results of this test "
+ "run. Intended to be run by CI when tagging an official "
+ "release. This database is used as a basis for comparison "
+ "when looking for deltas in metrics such as footprint")
+ parser.add_argument("-w", "--warnings-as-errors", action="store_true",
+ help="Treat warning conditions as errors")
+ parser.add_argument("-v", "--verbose", action="count", default=0,
+ help="Emit debugging information, call multiple times to increase "
+ "verbosity")
+ parser.add_argument("-i", "--inline-logs", action="store_true",
+ help="Upon test failure, print relevant log data to stdout "
+ "instead of just a path to it")
+ parser.add_argument("-m", "--last-metrics", action="store_true",
+ help="Instead of comparing metrics from the last --release, "
+ "compare with the results of the previous sanity check "
+ "invocation")
+ parser.add_argument("-u", "--no-update", action="store_true",
+ help="do not update the results of the last run of the sanity "
+ "checks")
+ parser.add_argument("-b", "--build-only", action="store_true",
+ help="Only build the code, do not execute any of it in QEMU")
+ parser.add_argument("-j", "--jobs", type=int,
+ help="Number of cores to use when building, defaults to "
+ "number of CPUs * 2")
+ parser.add_argument("-H", "--footprint-threshold", type=float, default=5,
+ help="When checking test case footprint sizes, warn the user if "
+ "the new app size is greater then the specified percentage "
+ "from the last release. Default is 5. 0 to warn on any "
+ "increase on app size")
+
+ parser.add_argument("-O", "--outdir",
+ default="%s/sanity-out" % ZEPHYR_BASE,
+ help="Output directory for logs and binaries.")
+ parser.add_argument("-C", "--clean", action="store_true",
+ help="Delete the outdir before building")
+ parser.add_argument("-T", "--testcase-root",
+ default="%s/samples" % ZEPHYR_BASE,
+ help="Base directory to recursively search for test cases. All "
+ "testcase.ini files under here will be processed")
+ parser.add_argument("-A", "--arch-root",
+ default="%s/scripts/sanity_chk/arches" % ZEPHYR_BASE,
+ help="Directory to search for arch configuration files. All .ini "
+ "files in the directory will be processed.")
+
+ return parser.parse_args()
+
+def log_info(filename):
+ filename = os.path.relpath(filename)
+ if INLINE_LOGS:
+ print "{:-^100}".format(filename)
+ with open(filename) as fp:
+ sys.stdout.write(fp.read())
+ print "{:-^100}".format(filename)
+ else:
+ print "\tsee: " + COLOR_YELLOW + filename + COLOR_NORMAL
+
+def terse_test_cb(instances, goals, goal):
+ total_tests = len(goals)
+ total_done = 0
+ total_failed = 0
+
+ for k, g in goals.iteritems():
+ if g.finished:
+ total_done += 1
+ if g.failed:
+ total_failed += 1
+
+ if goal.failed:
+ i = instances[goal.name]
+ info("\n\n{:<25} {:<50} {}FAILED{}: {}".format(i.platform.name,
+ i.test.name, COLOR_RED, COLOR_NORMAL, goal.reason))
+ log_info(goal.get_error_log())
+ info("")
+
+ sys.stdout.write("\rtotal complete: %s%3d/%3d%s failed: %s%3d%s" % (
+ COLOR_GREEN, total_done, total_tests, COLOR_NORMAL,
+ COLOR_RED if total_failed > 0 else COLOR_NORMAL,
+ total_failed, COLOR_NORMAL))
+ sys.stdout.flush()
+
+def chatty_test_cb(instances, goals, goal):
+ i = instances[goal.name]
+
+ if VERBOSE < 2 and not goal.finished:
+ return
+
+ if goal.failed:
+ status = COLOR_RED + "FAILED" + COLOR_NORMAL + ": " + goal.reason
+ elif goal.finished:
+ status = COLOR_GREEN + "PASSED" + COLOR_NORMAL
+ else:
+ status = goal.make_state
+
+ info("{:<25} {:<50} {}".format(i.platform.name, i.test.name, status))
+ if goal.failed:
+ log_info(goal.get_error_log())
+
+def main():
+ global VERBOSE, INLINE_LOGS, PARALLEL
+ args = parse_arguments()
+ VERBOSE += args.verbose
+ INLINE_LOGS = args.inline_logs
+ if args.jobs:
+ PARALLEL = args.jobs
+ if args.all:
+ args.platform = ["all"]
+
+ if os.path.exists(args.outdir) and args.clean:
+ info("Cleaning output directory " + args.outdir)
+ shutil.rmtree(args.outdir)
+
+ ts = TestSuite(args.arch_root, args.testcase_root, args.outdir)
+ discards = ts.apply_filters(args.platform, args.arch, args.tag, args.config,
+ args.test, args.only_failed)
+
+ if args.discard_report:
+ ts.discard_report(args.discard_report)
+
+ if VERBOSE:
+ for i, reason in discards.iteritems():
+ debug("{:<25} {:<50} {}SKIPPED{}: {}".format(i.platform.name,
+ i.test.name, COLOR_YELLOW, COLOR_NORMAL, reason))
+
+ info("%d tests selected, %d tests discarded due to filters" %
+ (len(ts.instances), len(discards)))
+
+ if args.dry_run:
+ return
+
+ if VERBOSE or not TERMINAL:
+ goals = ts.execute(chatty_test_cb, ts.instances, args.build_only)
+ else:
+ goals = ts.execute(terse_test_cb, ts.instances, args.build_only)
+ print
+
+ deltas = ts.compare_metrics(LAST_SANITY if args.last_metrics
+ else RELEASE_DATA)
+ warnings = 0
+ if deltas:
+ for i, metric, value, delta in deltas:
+ percentage = (float(delta) / float(value - delta))
+ if percentage < (args.footprint_threshold / 100.0):
+ continue
+
+ info("{:<25} {:<50} {}WARNING{}: {} is now {} {:+.2%}".format(
+ i.platform.name, i.test.name, COLOR_YELLOW, COLOR_NORMAL,
+ metric, value, percentage))
+ warnings += 1
+
+ if warnings:
+ info("Deltas based on metrics from last %s" %
+ ("release" if not args.last_metrics else "run"))
+
+ failed = 0
+ for name, goal in goals.iteritems():
+ if goal.failed:
+ failed += 1
+
+ info("%s%d of %d%s tests passed with %s%d%s warnings" %
+ (COLOR_RED if failed else COLOR_GREEN, len(goals) - failed,
+ len(goals), COLOR_NORMAL, COLOR_YELLOW if warnings else COLOR_NORMAL,
+ warnings, COLOR_NORMAL))
+
+ if args.testcase_report:
+ ts.testcase_report(args.testcase_report)
+ if not args.no_update:
+ ts.testcase_report(LAST_SANITY)
+ if args.release:
+ ts.testcase_report(RELEASE_DATA)
+
+ if failed or (warnings and args.warnings_as_errors):
+ sys.exit(1)
+
+if __name__ == "__main__":
+ main()
+
