Tweak test ordering in all.sh
diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh
index 1e11d78..d01dc25 100755
--- a/tests/scripts/all.sh
+++ b/tests/scripts/all.sh
@@ -6,8 +6,8 @@
# CMake configuration. After this script is run, the CMake cache is lost and
# CMake is not initialised any more!
#
-# Assumes gcc, clang (recent enough for using ASan) are available, as weel as
-# cmake. Also assumes valgrind is available if --memcheck is used.
+# Assumes gcc and clang (recent enough for using ASan) are available,
+# as well as cmake and valgrind.
# Abort on errors (and uninitiliased variables)
set -eu
@@ -54,54 +54,72 @@
echo "******************************************************************"
}
-# Step 1: various build types
+# The test ordering tries to optimize for the following criteria:
+# 1. Catch possible problems early, by running first test that run quickly
+# and/or are more likely to fail than others.
+# 2. Minimize total running time, by avoiding useless rebuilds
+#
+# Indicative running times are given for reference.
-msg "Unix make, default compiler and flags"
-cleanup
-make
-
-msg "cmake, gcc with lots of warnings"
+msg "build: cmake, gcc with lots of warnings" # ~ 1 min
cleanup
CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Check .
make
-msg "cmake, clang with lots of warnings"
+msg "test: main suites with valgrind" # ~ 2 min 10s
+make memcheck
+
+msg "build: with ASan" # ~ 1 min
+cleanup
+cmake -D CMAKE_BUILD_TYPE:String=ASan .
+make
+
+msg "test: ssl-opt.sh (ASan build)" # ~ 1 min 10s
+cd tests
+./ssl-opt.sh
+cd ..
+
+msg "test: main suites and selftest (ASan build)" # ~ 10s + 30s
+make test
+programs/test/selftest
+
+msg "test: ref-configs (ASan build)" # ~ 4 min 45 s
+tests/scripts/test-ref-configs.pl
+
+# Most issues are likely to be caught at this point
+
+msg "build: with ASan (rebuild after ref-configs)" # ~ 1 min
+make
+
+msg "test: compat.sh (ASan build)" # ~ 7 min 30s
+cd tests
+./compat.sh
+cd ..
+
+msg "build: cmake, clang with lots of warnings" # ~ 40s
cleanup
CC=clang cmake -D CMAKE_BUILD_TYPE:String=Check .
make
-# Step 2: Full tests, with ASan
-
-msg "ASan build and full tests"
+msg "build: Unix make, -O2" # ~ 30s
cleanup
-cmake -D CMAKE_BUILD_TYPE:String=ASan .
make
-make test
-programs/test/selftest
-cd tests
-./compat.sh
-./ssl-opt.sh
-cd ..
-tests/scripts/test-ref-configs.pl
-# Step 3: using valgrind's memcheck
-
-msg "Release build, test suites with valgrind's memcheck"
-cleanup
-# optimized build to compensate a bit for valgrind slowdown
-cmake -D CMAKE_BUILD_TYPE:String=Release .
-make
-make memcheck
+# Optional parts that take a long time to run
if [ "$MEMORY" -gt 0 ]; then
+ msg "test: ssl-opt --memcheck (-02 build)" # ~ 8 min
cd tests
./ssl-opt.sh --memcheck
- [ "$MEMORY" -gt 1 ] && ./compat.sh --memcheck
cd ..
- # no test-ref-configs: doesn't have a memcheck option (yet?)
-fi
-# Done
+ if [ "$MEMORY" -gt 1 ]; then
+ msg "test: compat --memcheck (-02 build)" # ~ 42 min
+ cd tests
+ ./compat.sh --memcheck
+ cd ..
+ fi
+fi
echo "Done."
cleanup
