From: Carl Worth Date: Fri, 17 Sep 2010 20:53:47 +0000 (-0700) Subject: test: Cleanup the test output X-Git-Tag: 0.4~99 X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=222926abe1e221a6239ccb9436968af43e1b6fb2;p=notmuch.git test: Cleanup the test output This makes the new, git-derived test suite report results in a manner similar to the original notmuch test suite. Notable changes include: * No more initial '*' on every line * Only colorize a single word * Don't print useless test numbers * Use "PASS" in place of "ok" * Begin sentences with a capital letter * Print test descriptions for each block * Separate each block of tests with a blank line * Don't summarize counts between each block --- diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 00000000..87364a27 --- /dev/null +++ b/test/.gitignore @@ -0,0 +1 @@ +test-results diff --git a/test/aggregate-results.sh b/test/aggregate-results.sh index d5bab75d..cc5b83fb 100755 --- a/test/aggregate-results.sh +++ b/test/aggregate-results.sh @@ -27,8 +27,55 @@ do done <"$file" done -printf "%-8s%d\n" fixed $fixed -printf "%-8s%d\n" success $success -printf "%-8s%d\n" failed $failed -printf "%-8s%d\n" broken $broken -printf "%-8s%d\n" total $total +pluralize () { + case $2 in + 1) + case $1 in + test) + echo test ;; + failure) + echo failure ;; + esac + ;; + *) + case $1 in + test) + echo tests ;; + failure) + echo failures ;; + esac + ;; + esac +} + +echo "Notmuch test suite complete." +if [ "$fixed" = "0" ] && [ "$broken" = "0" ]; then + tests=$(pluralize "test" $total) + printf "All $total $tests " + if [ "$broken" = "0" ]; then + echo "passed." + else + failures=$(pluralize "failure" $broken) + echo "behaved as expected ($broken expected $failures)." + fi; +else + echo "$success/$total tests passed." + if [ "$broken" != "0" ]; then + tests=$(pluralize "test" $broken) + echo "$broken broken $tests failed as expected." + fi + if [ "$fixed" != "0" ]; then + tests=$(pluralize "test" $fixed) + echo "$fixed broken $tests now fixed." + fi + if [ "$failed" != "0" ]; then + tests=$(pluralize "test" $failed) + echo "$failed $tests failed." + fi +fi + +skipped=$(($total - $fixed - $success - $failed - $broken)) +if [ "$skipped" != "0" ]; then + tests=$(pluralize "test" $skipped) + echo "$skipped $tests skipped." +fi diff --git a/test/notmuch-test b/test/notmuch-test index 8ff21cc6..61b9280a 100755 --- a/test/notmuch-test +++ b/test/notmuch-test @@ -20,4 +20,4 @@ done ./aggregate-results.sh test-results/t*-* # Clean up -rm -r test-results +#rm -r test-results diff --git a/test/t0000-basic.sh b/test/t0000-basic.sh index 511e200a..9c495b60 100755 --- a/test/t0000-basic.sh +++ b/test/t0000-basic.sh @@ -3,7 +3,7 @@ # Copyright (c) 2005 Junio C Hamano # -test_description='Tests the test framework itself.' +test_description='the test framework itself.' ################################################################ # It appears that people try to run tests without building... diff --git a/test/test-lib.sh b/test/test-lib.sh index 5ae916cb..35ff3d98 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -130,16 +130,17 @@ if test -n "$color"; then *) test -n "$quiet" && return;; esac shift - printf "* %s" "$*" + printf " " + printf "$@" tput sgr0 - echo ) } else say_color() { test -z "$1" && test -n "$quiet" && return shift - echo "* $*" + printf " " + printf "$@" } fi @@ -158,10 +159,12 @@ error "Test script did not set test_description." if test "$help" = "t" then - echo "$test_description" + echo "Tests ${test_description}" exit 0 fi +echo "Testing ${test_description}" + exec 5>&1 if test "$verbose" = "t" then @@ -439,12 +442,14 @@ test_have_prereq () { test_ok_ () { test_success=$(($test_success + 1)) - say_color "" " ok $test_count: $@" + say_color pass "%-6s" "PASS" + echo " $@" } test_failure_ () { test_failure=$(($test_failure + 1)) - say_color error "FAIL $test_count: $1" + say_color error "%-6s" "FAIL" + echo " $1" shift echo "$@" | sed -e 's/^/ /' test "$immediate" = "" || { GIT_EXIT_OK=t; exit 1; } @@ -452,12 +457,14 @@ test_failure_ () { test_known_broken_ok_ () { test_fixed=$(($test_fixed+1)) - say_color "" " FIXED $test_count: $@" + say_color pass "%-6s" "FIXED" + echo " $@" } test_known_broken_failure_ () { test_broken=$(($test_broken+1)) - say_color skip " still broken $test_count: $@" + say_color pass "%-6s" "BROKEN" + echo " $@" } test_debug () { @@ -490,7 +497,8 @@ test_skip () { case "$to_skip" in t) say_color skip >&3 "skipping test: $@" - say_color skip "skip $test_count: $1" + say_color skip "%-6s" "SKIP" + echo " $1" : true ;; *) @@ -684,32 +692,14 @@ test_done () { echo "failed $test_failure" >> $test_results_path echo "" >> $test_results_path - if test "$test_fixed" != 0 - then - say_color pass "fixed $test_fixed known breakage(s)" - fi - if test "$test_broken" != 0 - then - say_color error "still have $test_broken known breakage(s)" - msg="remaining $(($test_count-$test_broken)) test(s)" + echo + + if [ "$test_failure" = "0" ]; then + rm -rf "$remove_tmp" + exit 0 else - msg="$test_count test(s)" + exit 1 fi - case "$test_failure" in - 0) - say_color pass "passed all $msg" - - test -d "$remove_trash" && - cd "$(dirname "$remove_trash")" && - rm -rf "$(basename "$remove_trash")" - - exit 0 ;; - - *) - say_color error "failed $test_failure among $msg" - exit 1 ;; - - esac } find_notmuch_path () @@ -808,21 +798,21 @@ fi export PATH # Test repository -test="trash directory.$(basename "$0" .sh)" +test="tmp.$(basename "$0" .sh)" test -n "$root" && test="$root/$test" case "$test" in -/*) TRASH_DIRECTORY="$test" ;; - *) TRASH_DIRECTORY="$TEST_DIRECTORY/$test" ;; +/*) TMP_DIRECTORY="$test" ;; + *) TMP_DIRECTORY="$TEST_DIRECTORY/$test" ;; esac -test ! -z "$debug" || remove_trash=$TRASH_DIRECTORY +test ! -z "$debug" || remove_tmp=$TMP_DIRECTORY rm -fr "$test" || { GIT_EXIT_OK=t echo >&5 "FATAL: Cannot prepare test area" exit 1 } -MAIL_DIR="${TRASH_DIRECTORY}/mail" -export NOTMUCH_CONFIG="${TRASH_DIRECTORY}/notmuch-config" +MAIL_DIR="${TMP_DIRECTORY}/mail" +export NOTMUCH_CONFIG="${TMP_DIRECTORY}/notmuch-config" mkdir -p "${test}" mkdir "$MAIL_DIR"