From: Michal Sojka <sojkam1@fel.cvut.cz>
To: notmuch@notmuchmail.org
Subject: [PATCH] test: Better handling of stdout and stderr
Date: Wed, 10 Nov 2010 00:40:35 +0100
Message-Id: <1289346035-24904-1-git-send-email-sojkam1@fel.cvut.cz>
Precedence: list

Git-style tests suppress stdout and stderr unless -v is given.
cworth-style tests (created by test_begin_subtest, test_expect_equal)
do not have this behavior so implement it the same.

Additionally, for both test styles, the test-lib.sh is changed so that
the content of suppressed stdout and stderr is shown in case of failed
test.

Finally a test for this functionality is added to basic tests
---
 test/basic                                 |   19 ++++++++++++++++++-
 test/test-lib.sh                           |   21 +++++++++++++++------
 test/test-verbose                          |   27 +++++++++++++++++++++++++++
 test/test.expected-output/test-verbose-no  |   20 ++++++++++++++++++++
 test/test.expected-output/test-verbose-yes |   24 ++++++++++++++++++++++++
 5 files changed, 104 insertions(+), 7 deletions(-)
 create mode 100755 test/test-verbose
 create mode 100644 test/test.expected-output/test-verbose-no
 create mode 100644 test/test.expected-output/test-verbose-yes

diff --git a/test/basic b/test/basic
index 725c753..091dda0 100755
--- a/test/basic
+++ b/test/basic
@@ -52,9 +52,26 @@ test_expect_code 2 'failure to clean up causes the test to fail' '
 # Ensure that all tests are being run
 test_begin_subtest 'Ensure that all available tests will be run by notmuch-test'
 tests_in_suite=$(grep TESTS= ../notmuch-test | sed -e "s/TESTS=\"\(.*\)\"/\1/" | tr " " "\n" | sort)
-available=$(ls -1 ../ | grep -v -E "^(aggregate-results.sh|Makefile|Makefile.local|notmuch-test|README|test-lib.sh|test-results|tmp.*|valgrind|corpus*|emacs.expected-output|smtp-dummy|smtp-dummy.c)" | sort)
+available=$(ls -1 ../ | grep -v -E "^(aggregate-results.sh|Makefile|Makefile.local|notmuch-test|README|test-lib.sh|test-results|tmp.*|valgrind|corpus*|emacs.expected-output|smtp-dummy|smtp-dummy.c|test-verbose|test.expected-output)" | sort)
 test_expect_equal "$tests_in_suite" "$available"
 
+EXPECTED=../test.expected-output
+suppress_diff_date() {
+    sed -e 's/\(.*\-\-\- test\.4\.\expected\).*/\1/' \
+	-e 's/\(.*\+\+\+ test\.4\.\output\).*/\1/'
+}
+
+test_begin_subtest "Ensure that test output is suppressed unless the test fails"
+output=$(cd ..; ./test-verbose 2>&1 | suppress_diff_date)
+expected=$(cat $EXPECTED/test-verbose-no | suppress_diff_date)
+test_expect_equal "$output" "$expected"
+
+test_begin_subtest "Ensure that -v does not suppress test output"
+output=$(cd ..; ./test-verbose -v 2>&1 | suppress_diff_date)
+expected=$(cat $EXPECTED/test-verbose-yes | suppress_diff_date)
+test_expect_equal "$output" "$expected"
+
+
 ################################################################
 # Test mail store prepared in test-lib.sh
 
diff --git a/test/test-lib.sh b/test/test-lib.sh
index 8ecc9a0..3f69ab3 100644
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -165,12 +165,6 @@ fi
 echo $(basename "$0"): "Testing ${test_description}"
 
 exec 5>&1
-if test "$verbose" = "t"
-then
-	exec 4>&2 3>&1
-else
-	exec 4>/dev/null 3>/dev/null
-fi
 
 test_failure=0
 test_count=0
@@ -393,6 +387,11 @@ add_email_corpus ()
 test_begin_subtest ()
 {
     test_subtest_name="$1"
+    # Remember stdout and stderr file descriptios and redirect test
+    # output to the previously prepared file descriptors 3 and 4 (see
+    # bellow)
+    if test "$verbose" != "t"; then exec 4>test.output 3>&4; fi
+    exec 6>&1 7>&2 >&3 2>&4
 }
 
 # Pass test if two arguments match
@@ -403,6 +402,7 @@ test_begin_subtest ()
 # name.
 test_expect_equal ()
 {
+	exec 1>&6 2>&7		# Restore stdout and stderr
 	test "$#" = 3 && { prereq=$1; shift; } || prereq=
 	test "$#" = 2 ||
 	error "bug in the test script: not 2 or 3 parameters to test_expect_equal"
@@ -498,6 +498,7 @@ test_failure_ () {
 	echo " $1"
 	shift
 	echo "$@" | sed -e 's/^/	/'
+	if test "$verbose" != "t"; then cat test.output; fi
 	test "$immediate" = "" || { GIT_EXIT_OK=t; exit 1; }
 }
 
@@ -519,6 +520,7 @@ test_debug () {
 
 test_run_ () {
 	test_cleanup=:
+	if test "$verbose" != "t"; then exec 4>test.output 3>&4; fi
 	eval >&3 2>&4 "$1"
 	eval_ret=$?
 	eval >&3 2>&4 "$test_cleanup"
@@ -908,6 +910,13 @@ EOF
 # in subprocesses like git equals our $PWD (for pathname comparisons).
 cd -P "$test" || error "Cannot setup test environment"
 
+if test "$verbose" = "t"
+then
+	exec 4>&2 3>&1
+else
+	exec 4>test.output 3>&4
+fi
+
 this_test=${0##*/}
 this_test=${this_test%%-*}
 for skp in $NOTMUCH_SKIP_TESTS
diff --git a/test/test-verbose b/test/test-verbose
new file mode 100755
index 0000000..ea52f93
--- /dev/null
+++ b/test/test-verbose
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+test_description='the verbosity options of the test framework itself.'
+
+. ./test-lib.sh
+
+test_expect_success 'print something in git-style test and pass' '
+  echo "hello stdout" &&
+  echo "hello stderr" >&2 &&
+  true
+'
+test_expect_success 'print something in git-style test and fail' '
+  echo "hello stdout" &&
+  echo "hello stderr" >&2 &&
+  false
+'
+test_begin_subtest 'print something in cworth-style test and pass'
+echo "hello stdout"
+echo "hello stderr" >&2
+test_expect_equal "a" "a"
+
+test_begin_subtest 'print something in cworth-style test and fail'
+echo "hello stdout"
+echo "hello stderr" >&2
+test_expect_equal "a" "b"
+
+test_done
diff --git a/test/test.expected-output/test-verbose-no b/test/test.expected-output/test-verbose-no
new file mode 100644
index 0000000..04fe71d
--- /dev/null
+++ b/test/test.expected-output/test-verbose-no
@@ -0,0 +1,20 @@
+test-verbose: Testing the verbosity options of the test framework itself.
+ PASS   print something in git-style test and pass
+ FAIL   print something in git-style test and fail
+	
+	  echo "hello stdout" &&
+	  echo "hello stderr" >&2 &&
+	  false
+	
+hello stdout
+hello stderr
+ PASS   print something in cworth-style test and pass
+ FAIL   print something in cworth-style test and fail
+	--- test.4.expected	2010-11-09 23:18:55.597013913 +0000
+	+++ test.4.output	2010-11-09 23:18:55.597013913 +0000
+	@@ -1 +1 @@
+	-b
+	+a
+hello stdout
+hello stderr
+
diff --git a/test/test.expected-output/test-verbose-yes b/test/test.expected-output/test-verbose-yes
new file mode 100644
index 0000000..a84a574
--- /dev/null
+++ b/test/test.expected-output/test-verbose-yes
@@ -0,0 +1,24 @@
+test-verbose: Testing the verbosity options of the test framework itself.
+hello stdout
+hello stderr
+ PASS   print something in git-style test and pass
+hello stdout
+hello stderr
+ FAIL   print something in git-style test and fail
+	
+	  echo "hello stdout" &&
+	  echo "hello stderr" >&2 &&
+	  false
+	
+hello stdout
+hello stderr
+ PASS   print something in cworth-style test and pass
+hello stdout
+hello stderr
+ FAIL   print something in cworth-style test and fail
+	--- test.4.expected	2010-11-09 23:19:05.756552603 +0000
+	+++ test.4.output	2010-11-09 23:19:05.756552603 +0000
+	@@ -1 +1 @@
+	-b
+	+a
+
-- 
1.7.1