1 Return-Path: <wsh@resox.2x.cz>
\r
2 X-Original-To: notmuch@notmuchmail.org
\r
3 Delivered-To: notmuch@notmuchmail.org
\r
4 Received: from localhost (localhost [127.0.0.1])
\r
5 by olra.theworths.org (Postfix) with ESMTP id 4990D40DBD3
\r
6 for <notmuch@notmuchmail.org>; Tue, 9 Nov 2010 15:41:27 -0800 (PST)
\r
7 X-Virus-Scanned: Debian amavisd-new at olra.theworths.org
\r
11 X-Spam-Status: No, score=-1.9 tagged_above=-999 required=5
\r
12 tests=[BAYES_00=-1.9] autolearn=ham
\r
13 Received: from olra.theworths.org ([127.0.0.1])
\r
14 by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024)
\r
15 with ESMTP id VhG74+BDVPT0 for <notmuch@notmuchmail.org>;
\r
16 Tue, 9 Nov 2010 15:41:16 -0800 (PST)
\r
17 Received: from smtp.nextra.cz (smtp.nextra.cz [212.65.193.3])
\r
18 by olra.theworths.org (Postfix) with ESMTP id 0275240DBD6
\r
19 for <notmuch@notmuchmail.org>; Tue, 9 Nov 2010 15:41:15 -0800 (PST)
\r
20 Received: from resox.2x.cz (unknown [213.29.198.144])
\r
21 by smtp.nextra.cz (Postfix) with ESMTP id 5EBD6893A8;
\r
22 Wed, 10 Nov 2010 00:41:12 +0100 (CET)
\r
23 Received: from wsh by resox.2x.cz with local (Exim 4.72)
\r
24 (envelope-from <wsh@resox.2x.cz>)
\r
25 id 1PFxoE-0006UA-28; Wed, 10 Nov 2010 00:41:02 +0100
\r
26 From: Michal Sojka <sojkam1@fel.cvut.cz>
\r
27 To: notmuch@notmuchmail.org
\r
28 Subject: [PATCH] test: Better handling of stdout and stderr
\r
29 Date: Wed, 10 Nov 2010 00:40:35 +0100
\r
30 Message-Id: <1289346035-24904-1-git-send-email-sojkam1@fel.cvut.cz>
\r
31 X-Mailer: git-send-email 1.7.1
\r
32 X-BeenThere: notmuch@notmuchmail.org
\r
33 X-Mailman-Version: 2.1.13
\r
35 List-Id: "Use and development of the notmuch mail system."
\r
36 <notmuch.notmuchmail.org>
\r
37 List-Unsubscribe: <http://notmuchmail.org/mailman/options/notmuch>,
\r
38 <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>
\r
39 List-Archive: <http://notmuchmail.org/pipermail/notmuch>
\r
40 List-Post: <mailto:notmuch@notmuchmail.org>
\r
41 List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>
\r
42 List-Subscribe: <http://notmuchmail.org/mailman/listinfo/notmuch>,
\r
43 <mailto:notmuch-request@notmuchmail.org?subject=subscribe>
\r
44 X-List-Received-Date: Tue, 09 Nov 2010 23:41:27 -0000
\r
46 Git-style tests suppress stdout and stderr unless -v is given.
\r
47 cworth-style tests (created by test_begin_subtest, test_expect_equal)
\r
48 do not have this behavior so implement it the same.
\r
50 Additionally, for both test styles, the test-lib.sh is changed so that
\r
51 the content of suppressed stdout and stderr is shown in case of failed
\r
54 Finally a test for this functionality is added to basic tests
\r
56 test/basic | 19 ++++++++++++++++++-
\r
57 test/test-lib.sh | 21 +++++++++++++++------
\r
58 test/test-verbose | 27 +++++++++++++++++++++++++++
\r
59 test/test.expected-output/test-verbose-no | 20 ++++++++++++++++++++
\r
60 test/test.expected-output/test-verbose-yes | 24 ++++++++++++++++++++++++
\r
61 5 files changed, 104 insertions(+), 7 deletions(-)
\r
62 create mode 100755 test/test-verbose
\r
63 create mode 100644 test/test.expected-output/test-verbose-no
\r
64 create mode 100644 test/test.expected-output/test-verbose-yes
\r
66 diff --git a/test/basic b/test/basic
\r
67 index 725c753..091dda0 100755
\r
70 @@ -52,9 +52,26 @@ test_expect_code 2 'failure to clean up causes the test to fail' '
\r
71 # Ensure that all tests are being run
\r
72 test_begin_subtest 'Ensure that all available tests will be run by notmuch-test'
\r
73 tests_in_suite=$(grep TESTS= ../notmuch-test | sed -e "s/TESTS=\"\(.*\)\"/\1/" | tr " " "\n" | sort)
\r
74 -available=$(ls -1 ../ | grep -v -E "^(aggregate-results.sh|Makefile|Makefile.local|notmuch-test|README|test-lib.sh|test-results|tmp.*|valgrind|corpus*|emacs.expected-output|smtp-dummy|smtp-dummy.c)" | sort)
\r
75 +available=$(ls -1 ../ | grep -v -E "^(aggregate-results.sh|Makefile|Makefile.local|notmuch-test|README|test-lib.sh|test-results|tmp.*|valgrind|corpus*|emacs.expected-output|smtp-dummy|smtp-dummy.c|test-verbose|test.expected-output)" | sort)
\r
76 test_expect_equal "$tests_in_suite" "$available"
\r
78 +EXPECTED=../test.expected-output
\r
79 +suppress_diff_date() {
\r
80 + sed -e 's/\(.*\-\-\- test\.4\.\expected\).*/\1/' \
\r
81 + -e 's/\(.*\+\+\+ test\.4\.\output\).*/\1/'
\r
84 +test_begin_subtest "Ensure that test output is suppressed unless the test fails"
\r
85 +output=$(cd ..; ./test-verbose 2>&1 | suppress_diff_date)
\r
86 +expected=$(cat $EXPECTED/test-verbose-no | suppress_diff_date)
\r
87 +test_expect_equal "$output" "$expected"
\r
89 +test_begin_subtest "Ensure that -v does not suppress test output"
\r
90 +output=$(cd ..; ./test-verbose -v 2>&1 | suppress_diff_date)
\r
91 +expected=$(cat $EXPECTED/test-verbose-yes | suppress_diff_date)
\r
92 +test_expect_equal "$output" "$expected"
\r
95 ################################################################
\r
96 # Test mail store prepared in test-lib.sh
\r
98 diff --git a/test/test-lib.sh b/test/test-lib.sh
\r
99 index 8ecc9a0..3f69ab3 100644
\r
100 --- a/test/test-lib.sh
\r
101 +++ b/test/test-lib.sh
\r
102 @@ -165,12 +165,6 @@ fi
\r
103 echo $(basename "$0"): "Testing ${test_description}"
\r
106 -if test "$verbose" = "t"
\r
110 - exec 4>/dev/null 3>/dev/null
\r
115 @@ -393,6 +387,11 @@ add_email_corpus ()
\r
116 test_begin_subtest ()
\r
118 test_subtest_name="$1"
\r
119 + # Remember stdout and stderr file descriptios and redirect test
\r
120 + # output to the previously prepared file descriptors 3 and 4 (see
\r
122 + if test "$verbose" != "t"; then exec 4>test.output 3>&4; fi
\r
123 + exec 6>&1 7>&2 >&3 2>&4
\r
126 # Pass test if two arguments match
\r
127 @@ -403,6 +402,7 @@ test_begin_subtest ()
\r
129 test_expect_equal ()
\r
131 + exec 1>&6 2>&7 # Restore stdout and stderr
\r
132 test "$#" = 3 && { prereq=$1; shift; } || prereq=
\r
134 error "bug in the test script: not 2 or 3 parameters to test_expect_equal"
\r
135 @@ -498,6 +498,7 @@ test_failure_ () {
\r
138 echo "$@" | sed -e 's/^/ /'
\r
139 + if test "$verbose" != "t"; then cat test.output; fi
\r
140 test "$immediate" = "" || { GIT_EXIT_OK=t; exit 1; }
\r
143 @@ -519,6 +520,7 @@ test_debug () {
\r
147 + if test "$verbose" != "t"; then exec 4>test.output 3>&4; fi
\r
150 eval >&3 2>&4 "$test_cleanup"
\r
151 @@ -908,6 +910,13 @@ EOF
\r
152 # in subprocesses like git equals our $PWD (for pathname comparisons).
\r
153 cd -P "$test" || error "Cannot setup test environment"
\r
155 +if test "$verbose" = "t"
\r
159 + exec 4>test.output 3>&4
\r
163 this_test=${this_test%%-*}
\r
164 for skp in $NOTMUCH_SKIP_TESTS
\r
165 diff --git a/test/test-verbose b/test/test-verbose
\r
166 new file mode 100755
\r
167 index 0000000..ea52f93
\r
169 +++ b/test/test-verbose
\r
173 +test_description='the verbosity options of the test framework itself.'
\r
177 +test_expect_success 'print something in git-style test and pass' '
\r
178 + echo "hello stdout" &&
\r
179 + echo "hello stderr" >&2 &&
\r
182 +test_expect_success 'print something in git-style test and fail' '
\r
183 + echo "hello stdout" &&
\r
184 + echo "hello stderr" >&2 &&
\r
187 +test_begin_subtest 'print something in cworth-style test and pass'
\r
188 +echo "hello stdout"
\r
189 +echo "hello stderr" >&2
\r
190 +test_expect_equal "a" "a"
\r
192 +test_begin_subtest 'print something in cworth-style test and fail'
\r
193 +echo "hello stdout"
\r
194 +echo "hello stderr" >&2
\r
195 +test_expect_equal "a" "b"
\r
198 diff --git a/test/test.expected-output/test-verbose-no b/test/test.expected-output/test-verbose-no
\r
199 new file mode 100644
\r
200 index 0000000..04fe71d
\r
202 +++ b/test/test.expected-output/test-verbose-no
\r
204 +test-verbose: Testing the verbosity options of the test framework itself.
\r
205 + PASS print something in git-style test and pass
\r
206 + FAIL print something in git-style test and fail
\r
208 + echo "hello stdout" &&
\r
209 + echo "hello stderr" >&2 &&
\r
214 + PASS print something in cworth-style test and pass
\r
215 + FAIL print something in cworth-style test and fail
\r
216 + --- test.4.expected 2010-11-09 23:18:55.597013913 +0000
\r
217 + +++ test.4.output 2010-11-09 23:18:55.597013913 +0000
\r
224 diff --git a/test/test.expected-output/test-verbose-yes b/test/test.expected-output/test-verbose-yes
\r
225 new file mode 100644
\r
226 index 0000000..a84a574
\r
228 +++ b/test/test.expected-output/test-verbose-yes
\r
230 +test-verbose: Testing the verbosity options of the test framework itself.
\r
233 + PASS print something in git-style test and pass
\r
236 + FAIL print something in git-style test and fail
\r
238 + echo "hello stdout" &&
\r
239 + echo "hello stderr" >&2 &&
\r
244 + PASS print something in cworth-style test and pass
\r
247 + FAIL print something in cworth-style test and fail
\r
248 + --- test.4.expected 2010-11-09 23:19:05.756552603 +0000
\r
249 + +++ test.4.output 2010-11-09 23:19:05.756552603 +0000
\r