From ac9dbb47de734afb0ddd4d0b2eb8f5027b1a7f60 Mon Sep 17 00:00:00 2001 From: Michal Sojka Date: Sun, 14 Nov 2010 22:54:28 +0100 Subject: [PATCH] test: Better handling of stdout and stderr Git-style tests (test_expect_success etc.) suppress stdout and stderr unless -v is given. Notmuch-style tests (created by test_begin_subtest and test_expect_equal) do not have this behavior so implement it the same. Additionally, for both test styles, the test-lib.sh is changed so that the content of suppressed stdout and stderr is shown in case of failed test. Finally a test for this functionality is added to basic tests. --- test/basic | 21 ++++++++++++++++- test/test-lib.sh | 21 ++++++++++++----- test/test-verbose | 27 ++++++++++++++++++++++ test/test.expected-output/test-verbose-no | 20 ++++++++++++++++ test/test.expected-output/test-verbose-yes | 24 +++++++++++++++++++ 5 files changed, 106 insertions(+), 7 deletions(-) create mode 100755 test/test-verbose create mode 100644 test/test.expected-output/test-verbose-no create mode 100644 test/test.expected-output/test-verbose-yes diff --git a/test/basic b/test/basic index 725c7538..309779ca 100755 --- a/test/basic +++ b/test/basic @@ -52,9 +52,28 @@ test_expect_code 2 'failure to clean up causes the test to fail' ' # Ensure that all tests are being run test_begin_subtest 'Ensure that all available tests will be run by notmuch-test' tests_in_suite=$(grep TESTS= ../notmuch-test | sed -e "s/TESTS=\"\(.*\)\"/\1/" | tr " " "\n" | sort) -available=$(ls -1 ../ | grep -v -E "^(aggregate-results.sh|Makefile|Makefile.local|notmuch-test|README|test-lib.sh|test-results|tmp.*|valgrind|corpus*|emacs.expected-output|smtp-dummy|smtp-dummy.c)" | sort) +available=$(ls -1 ../ | grep -v -E "^(aggregate-results.sh|Makefile|Makefile.local|notmuch-test|README|test-lib.sh|test-results|tmp.*|valgrind|corpus*|emacs.expected-output|smtp-dummy|smtp-dummy.c|test-verbose|test.expected-output)" | sort) test_expect_equal "$tests_in_suite" "$available" +EXPECTED=../test.expected-output +suppress_diff_date() { + sed -e 's/\(.*\-\-\- test-verbose\.4\.\expected\).*/\1/' \ + -e 's/\(.*\+\+\+ test-verbose\.4\.\output\).*/\1/' +} + +test_begin_subtest "Ensure that test output is suppressed unless the test fails" +output=$(cd ..; ./test-verbose 2>&1 | suppress_diff_date) +expected=$(cat $EXPECTED/test-verbose-no | suppress_diff_date) +test_expect_equal "$output" "$expected" + +test_begin_subtest "Ensure that -v does not suppress test output" +output=$(cd ..; ./test-verbose -v 2>&1 | suppress_diff_date) +expected=$(cat $EXPECTED/test-verbose-yes | suppress_diff_date) +# Do not include the results of test-verbose in totals +rm $TEST_DIRECTORY/test-results/test-verbose-* +test_expect_equal "$output" "$expected" + + ################################################################ # Test mail store prepared in test-lib.sh diff --git a/test/test-lib.sh b/test/test-lib.sh index 418eaa7f..68c9cf8a 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -165,12 +165,6 @@ fi echo $(basename "$0"): "Testing ${test_description}" exec 5>&1 -if test "$verbose" = "t" -then - exec 4>&2 3>&1 -else - exec 4>/dev/null 3>/dev/null -fi test_failure=0 test_count=0 @@ -403,6 +397,11 @@ add_email_corpus () test_begin_subtest () { test_subtest_name="$1" + # Remember stdout and stderr file descriptios and redirect test + # output to the previously prepared file descriptors 3 and 4 (see + # bellow) + if test "$verbose" != "t"; then exec 4>test.output 3>&4; fi + exec 6>&1 7>&2 >&3 2>&4 } # Pass test if two arguments match @@ -413,6 +412,7 @@ test_begin_subtest () # name. test_expect_equal () { + exec 1>&6 2>&7 # Restore stdout and stderr test "$#" = 3 && { prereq=$1; shift; } || prereq= test "$#" = 2 || error "bug in the test script: not 2 or 3 parameters to test_expect_equal" @@ -508,6 +508,7 @@ test_failure_ () { echo " $1" shift echo "$@" | sed -e 's/^/ /' + if test "$verbose" != "t"; then cat test.output; fi test "$immediate" = "" || { GIT_EXIT_OK=t; exit 1; } } @@ -529,6 +530,7 @@ test_debug () { test_run_ () { test_cleanup=: + if test "$verbose" != "t"; then exec 4>test.output 3>&4; fi eval >&3 2>&4 "$1" eval_ret=$? eval >&3 2>&4 "$test_cleanup" @@ -918,6 +920,13 @@ EOF # in subprocesses like git equals our $PWD (for pathname comparisons). cd -P "$test" || error "Cannot setup test environment" +if test "$verbose" = "t" +then + exec 4>&2 3>&1 +else + exec 4>test.output 3>&4 +fi + this_test=${0##*/} for skp in $NOTMUCH_SKIP_TESTS do diff --git a/test/test-verbose b/test/test-verbose new file mode 100755 index 00000000..f29a9c7d --- /dev/null +++ b/test/test-verbose @@ -0,0 +1,27 @@ +#!/bin/bash + +test_description='the verbosity options of the test framework itself.' + +. ./test-lib.sh + +test_expect_success 'print something in test_expect_success and pass' ' + echo "hello stdout" && + echo "hello stderr" >&2 && + true +' +test_expect_success 'print something in test_expect_success and fail' ' + echo "hello stdout" && + echo "hello stderr" >&2 && + false +' +test_begin_subtest 'print something between test_begin_subtest and test_expect_equal and pass' +echo "hello stdout" +echo "hello stderr" >&2 +test_expect_equal "a" "a" + +test_begin_subtest 'print something test_begin_subtest and test_expect_equal and fail' +echo "hello stdout" +echo "hello stderr" >&2 +test_expect_equal "a" "b" + +test_done diff --git a/test/test.expected-output/test-verbose-no b/test/test.expected-output/test-verbose-no new file mode 100644 index 00000000..0bca7540 --- /dev/null +++ b/test/test.expected-output/test-verbose-no @@ -0,0 +1,20 @@ +test-verbose: Testing the verbosity options of the test framework itself. + PASS print something in test_expect_success and pass + FAIL print something in test_expect_success and fail + + echo "hello stdout" && + echo "hello stderr" >&2 && + false + +hello stdout +hello stderr + PASS print something between test_begin_subtest and test_expect_equal and pass + FAIL print something test_begin_subtest and test_expect_equal and fail + --- test-verbose.4.expected 2010-11-14 21:41:12.738189710 +0000 + +++ test-verbose.4.output 2010-11-14 21:41:12.738189710 +0000 + @@ -1 +1 @@ + -b + +a +hello stdout +hello stderr + diff --git a/test/test.expected-output/test-verbose-yes b/test/test.expected-output/test-verbose-yes new file mode 100644 index 00000000..ebe51874 --- /dev/null +++ b/test/test.expected-output/test-verbose-yes @@ -0,0 +1,24 @@ +test-verbose: Testing the verbosity options of the test framework itself. +hello stdout +hello stderr + PASS print something in test_expect_success and pass +hello stdout +hello stderr + FAIL print something in test_expect_success and fail + + echo "hello stdout" && + echo "hello stderr" >&2 && + false + +hello stdout +hello stderr + PASS print something between test_begin_subtest and test_expect_equal and pass +hello stdout +hello stderr + FAIL print something test_begin_subtest and test_expect_equal and fail + --- test-verbose.4.expected 2010-11-14 21:41:06.650023289 +0000 + +++ test-verbose.4.output 2010-11-14 21:41:06.650023289 +0000 + @@ -1 +1 @@ + -b + +a + -- 2.26.2