test: exit with nonzero value when not all tests completed successfully

If any of the tests in our test system is not passing the execution
of the test suite completes with nonzero exit value.

It is better to rely on the exit value of the test system instead
of some arbitrary strings in test output (or use both).

diff --git a/test/aggregate-results.sh b/test/aggregate-results.sh
index 732d6ca7f53ca8bf332f1179199ec132fe6abe04..b016edb9117108523642b17862b20846f4fbc89c 100755 (executable)
--- a/test/aggregate-results.sh
+++ b/test/aggregate-results.sh
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -eu
+
 fixed=0
 success=0
 failed=0
@@ -79,3 +81,10 @@ if [ "$skipped" != "0" ]; then
     tests=$(pluralize "test" $skipped)
     echo "$skipped $tests skipped."
 fi
+
+if [ $success -gt 0 -a $fixed -eq 0 -a $failed -eq 0 -a $skipped -eq 0 ]
+then
+    exit 0
+else
+    exit 1
+fi

diff --git a/test/notmuch-test b/test/notmuch-test
index 6db797955bdb54d852b060f0bae3083335672759..aa28bb05e71160c937865c55eb5add3f4ffc02d5 100755 (executable)
--- a/test/notmuch-test
+++ b/test/notmuch-test
@@ -97,6 +97,9 @@ trap - HUP INT TERM
 
 # Report results
 ./aggregate-results.sh test-results/*
+ev=$?
 
 # Clean up
 rm -rf test-results corpus.mail
+
+exit $ev