From 82f7d0670e0db8f813a0bece05b0b9b3e0e7b265 Mon Sep 17 00:00:00 2001
From: Dmitry Kurochkin <dmitry.kurochkin@gmail.com>
Date: Mon, 4 Jul 2011 05:59:02 +0400
Subject: [PATCH] [PATCH 2/3] test: improve known broken tests support

---
 0f/f310cb8b122c08f3a80ef707e0495203f3a02c | 211 ++++++++++++++++++++++
 1 file changed, 211 insertions(+)
 create mode 100644 0f/f310cb8b122c08f3a80ef707e0495203f3a02c

diff --git a/0f/f310cb8b122c08f3a80ef707e0495203f3a02c b/0f/f310cb8b122c08f3a80ef707e0495203f3a02c
new file mode 100644
index 000000000..ea239fa73
--- /dev/null
+++ b/0f/f310cb8b122c08f3a80ef707e0495203f3a02c
@@ -0,0 +1,211 @@
+Return-Path: <dmitry.kurochkin@gmail.com>
+X-Original-To: notmuch@notmuchmail.org
+Delivered-To: notmuch@notmuchmail.org
+Received: from localhost (localhost [127.0.0.1])
+	by olra.theworths.org (Postfix) with ESMTP id 82D18431FB6
+	for <notmuch@notmuchmail.org>; Sun,  3 Jul 2011 19:00:33 -0700 (PDT)
+X-Virus-Scanned: Debian amavisd-new at olra.theworths.org
+X-Spam-Flag: NO
+X-Spam-Score: -0.799
+X-Spam-Level: 
+X-Spam-Status: No, score=-0.799 tagged_above=-999 required=5
+	tests=[DKIM_SIGNED=0.1, DKIM_VALID=-0.1, DKIM_VALID_AU=-0.1,
+	FREEMAIL_FROM=0.001, RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled
+Received: from olra.theworths.org ([127.0.0.1])
+	by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024)
+	with ESMTP id Q+R+u5PKW-N5 for <notmuch@notmuchmail.org>;
+	Sun,  3 Jul 2011 19:00:32 -0700 (PDT)
+Received: from mail-bw0-f53.google.com (mail-bw0-f53.google.com
+	[209.85.214.53]) (using TLSv1 with cipher RC4-SHA (128/128 bits))
+	(No client certificate requested)
+	by olra.theworths.org (Postfix) with ESMTPS id 2C7F5431FD0
+	for <notmuch@notmuchmail.org>; Sun,  3 Jul 2011 19:00:32 -0700 (PDT)
+Received: by mail-bw0-f53.google.com with SMTP id 12so4396763bwg.26
+	for <notmuch@notmuchmail.org>; Sun, 03 Jul 2011 19:00:31 -0700 (PDT)
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=gamma;
+	h=from:to:cc:subject:date:message-id:x-mailer:in-reply-to:references;
+	bh=Q69nW/X1GjQSSubDoOfpua8h4YUCUaNunihsreXWxuc=;
+	b=au1qIlLbKiNU/2ApM6rcr8LiaF/gWFWSMIcnIT9S7rn+x/o2hmekLJ+I+lTHVgJt9W
+	OnRyticL0SeaIZh/MA+cbX2aqYeeJB/jWYNhlyfeGbNcSH0FZr1OeAhQHfH2ztBIQBex
+	xKdoABoj6I8Fqn1wKsT5KH7/ubLUOlWah475E=
+Received: by 10.204.19.19 with SMTP id y19mr4970270bka.164.1309744831406;
+	Sun, 03 Jul 2011 19:00:31 -0700 (PDT)
+Received: from localhost ([91.144.186.21])
+	by mx.google.com with ESMTPS id e6sm5101585bka.23.2011.07.03.19.00.29
+	(version=TLSv1/SSLv3 cipher=OTHER);
+	Sun, 03 Jul 2011 19:00:30 -0700 (PDT)
+From: Dmitry Kurochkin <dmitry.kurochkin@gmail.com>
+To: notmuch@notmuchmail.org
+Subject: [PATCH 2/3] test: improve known broken tests support
+Date: Mon,  4 Jul 2011 05:59:02 +0400
+Message-Id: <1309744743-8556-2-git-send-email-dmitry.kurochkin@gmail.com>
+X-Mailer: git-send-email 1.7.5.4
+In-Reply-To: <1309744743-8556-1-git-send-email-dmitry.kurochkin@gmail.com>
+References: <1309744743-8556-1-git-send-email-dmitry.kurochkin@gmail.com>
+X-BeenThere: notmuch@notmuchmail.org
+X-Mailman-Version: 2.1.13
+Precedence: list
+List-Id: "Use and development of the notmuch mail system."
+	<notmuch.notmuchmail.org>
+List-Unsubscribe: <http://notmuchmail.org/mailman/options/notmuch>,
+	<mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>
+List-Archive: <http://notmuchmail.org/pipermail/notmuch>
+List-Post: <mailto:notmuch@notmuchmail.org>
+List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>
+List-Subscribe: <http://notmuchmail.org/mailman/listinfo/notmuch>,
+	<mailto:notmuch-request@notmuchmail.org?subject=subscribe>
+X-List-Received-Date: Mon, 04 Jul 2011 02:00:33 -0000
+
+There is existing support for broken tests.  But it is not convenient
+to use.  The primary issue is that we have to maintain a set of
+test_expect_*_failure functions which are equivalent to the normal
+test_expect_* counterparts except for what functions are called for
+result reporting.  The patch adds test_subtest_known_broken function
+which marks a subset as broken, making the normal test_expect_*
+functions behave as test_expect_*_failure.  All test_expect_*_failure
+functions are removed.  Test_known_broken_failure_ is changed to
+format details the same way as test_failure_ does.
+
+Another benefit of this change is that the diff when a broken test is
+fixed would be small and nice.
+
+Documentation is updated accordingly.
+---
+ test/README      |   17 ++++++++---------
+ test/test-lib.sh |   53 +++++++++++++++--------------------------------------
+ 2 files changed, 23 insertions(+), 47 deletions(-)
+
+diff --git a/test/README b/test/README
+index a245bf1..f926b9f 100644
+--- a/test/README
++++ b/test/README
+@@ -132,20 +132,19 @@ library for your script to use.
+    <script>.  If it yields success, test is considered
+    successful.  <message> should state what it is testing.
+ 
+- test_expect_failure <message> <script>
+-
+-   This is NOT the opposite of test_expect_success, but is used
+-   to mark a test that demonstrates a known breakage.  Unlike
+-   the usual test_expect_success tests, which say "ok" on
+-   success and "FAIL" on failure, this will say "FIXED" on
+-   success and "still broken" on failure.  Failures from these
+-   tests won't cause -i (immediate) to stop.
+-
+  test_begin_subtest <message>
+ 
+    Set the test description message for a subsequent test_expect_equal
+    invocation (see below).
+ 
++ test_subtest_known_broken
++
++   Mark the current test as broken.  Such tests are expected to fail.
++   Unlike the normal tests, which say "PASS" on success and "FAIL" on
++   failure, these will say "FIXED" on success and "BROKEN" on failure.
++   Failures from these tests won't cause -i (immediate) to stop.  This
++   must be called before any test_expect_* function.
++
+  test_expect_equal <output> <expected>
+ 
+    This is an often-used convenience function built on top of
+diff --git a/test/test-lib.sh b/test/test-lib.sh
+index 22e387e..0cd4170 100755
+--- a/test/test-lib.sh
++++ b/test/test-lib.sh
+@@ -424,6 +424,7 @@ test_begin_subtest ()
+ 	error "bug in test script: Missing test_expect_equal in ${BASH_SOURCE[1]}:${BASH_LINENO[0]}"
+     fi
+     test_subtest_name="$1"
++    test_subtest_known_broken_=
+     # Remember stdout and stderr file descriptors and redirect test
+     # output to the previously prepared file descriptors 3 and 4 (see
+     # below)
+@@ -484,29 +485,6 @@ test_expect_equal_file ()
+     fi
+ }
+ 
+-test_expect_equal_failure ()
+-{
+-	exec 1>&6 2>&7		# Restore stdout and stderr
+-	inside_subtest=
+-	test "$#" = 3 && { prereq=$1; shift; } || prereq=
+-	test "$#" = 2 ||
+-	error "bug in the test script: not 2 or 3 parameters to test_expect_equal"
+-
+-	output="$1"
+-	expected="$2"
+-	if ! test_skip "$@"
+-	then
+-		if [ "$output" = "$expected" ]; then
+-			test_known_broken_ok_ "$test_subtest_name"
+-		else
+-			test_known_broken_failure_ "$test_subtest_name"
+-			testname=$this_test.$test_count
+-			echo "$expected" > $testname.expected
+-			echo "$output" > $testname.output
+-		fi
+-    fi
+-}
+-
+ NOTMUCH_NEW ()
+ {
+     notmuch new | grep -v -E -e '^Processed [0-9]*( total)? file|Found [0-9]* total file'
+@@ -568,12 +546,20 @@ test_have_prereq () {
+ # the text_expect_* functions instead.
+ 
+ test_ok_ () {
++	if [ "$test_subtest_known_broken_" = 1 ]; then
++		test_known_broken_ok_ "$@"
++		return
++	fi
+ 	test_success=$(($test_success + 1))
+ 	say_color pass "%-6s" "PASS"
+ 	echo " $@"
+ }
+ 
+ test_failure_ () {
++	if [ "$test_subtest_known_broken_" = 1 ]; then
++		test_known_broken_failure_ "$@"
++		return
++	fi
+ 	test_failure=$(($test_failure + 1))
+ 	say_color error "%-6s" "FAIL"
+ 	echo " $1"
+@@ -592,7 +578,10 @@ test_known_broken_ok_ () {
+ test_known_broken_failure_ () {
+ 	test_broken=$(($test_broken+1))
+ 	say_color pass "%-6s" "BROKEN"
+-	echo " $@"
++	echo " $1"
++	shift
++	echo "$@" | sed -e 's/^/	/'
++	if test "$verbose" != "t"; then cat test.output; fi
+ }
+ 
+ test_debug () {
+@@ -636,20 +625,8 @@ test_skip () {
+ 	esac
+ }
+ 
+-test_expect_failure () {
+-	test "$#" = 3 && { prereq=$1; shift; } || prereq=
+-	test "$#" = 2 ||
+-	error "bug in the test script: not 2 or 3 parameters to test-expect-failure"
+-	if ! test_skip "$@"
+-	then
+-		test_run_ "$2"
+-		if [ "$?" = 0 -a "$eval_ret" = 0 ]
+-		then
+-			test_known_broken_ok_ "$1"
+-		else
+-			test_known_broken_failure_ "$1"
+-		fi
+-	fi
++test_subtest_known_broken () {
++	test_subtest_known_broken_=1
+ }
+ 
+ test_expect_success () {
+-- 
+1.7.5.4
+
-- 
2.26.2