From: Dmitry Kurochkin <dmitry.kurochkin@gmail.com>
To: notmuch@notmuchmail.org
Subject: [PATCH v2 04/10] test: add support for external executable
	dependencies
Date: Thu, 17 Nov 2011 17:05:57 +0400
Message-Id: <1321535163-4895-5-git-send-email-dmitry.kurochkin@gmail.com>
In-Reply-To: <1321535163-4895-1-git-send-email-dmitry.kurochkin@gmail.com>
References: <1321535163-4895-1-git-send-email-dmitry.kurochkin@gmail.com>
Precedence: list

There is existing support for general prerequisites in the test suite.
But it is not very convenient to use: every test case has to keep
track for it's dependencies and they have to be explicitly listed.

The patch aims to add better support for a particular type of external
dependencies: external executables.  The main idea is to replace
missing external binaries with shell functions that have the same
name.  These functions always fail and keep track of missing
dependencies for a subtest.  The result reporting functions later can
check that an external binaries are missing and correctly report SKIP
result instead of FAIL.  The primary benefit is that the test cases do
not need to declare their dependencies or be changed in any way.
---
 test/test-lib.sh |   49 +++++++++++++++++++++++++++++++++++++++++--------
 1 files changed, 41 insertions(+), 8 deletions(-)

diff --git a/test/test-lib.sh b/test/test-lib.sh
index f21e45e..625d19b 100755
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -526,40 +526,53 @@ notmuch_json_show_sanitize ()
 # - Implicitly by specifying the prerequisite tag in the calls to
 #   test_expect_{success,failure,code}.
 #
 # The single parameter is the prerequisite tag (a simple word, in all
 # capital letters by convention).
 
 test_set_prereq () {
 	satisfied="$satisfied$1 "
 }
 satisfied=" "
 
 test_have_prereq () {
 	case $satisfied in
 	*" $1 "*)
 		: yes, have it ;;
 	*)
 		! : nope ;;
 	esac
 }
 
+# declare prerequisite for the given external binary
+test_declare_external_prereq () {
+	binary="$1"
+	test "$#" = 2 && name=$2 || name="$binary(1)"
+
+	hash $binary 2>/dev/null || eval "
+$binary () {
+	echo -n \"\$test_subtest_missing_external_prereqs_\" | grep -e \" $name \" ||
+	test_subtest_missing_external_prereqs_=\"$test_subtest_missing_external_prereqs_ $name\"
+	false
+}"
+}
+
 # You are not expected to call test_ok_ and test_failure_ directly, use
 # the text_expect_* functions instead.
 
 test_ok_ () {
 	if test "$test_subtest_known_broken_" = "t"; then
 		test_known_broken_ok_ "$@"
 		return
 	fi
 	test_success=$(($test_success + 1))
 	say_color pass "%-6s" "PASS"
 	echo " $@"
 }
 
 test_failure_ () {
 	if test "$test_subtest_known_broken_" = "t"; then
 		test_known_broken_failure_ "$@"
 		return
 	fi
 	test_failure=$(($test_failure + 1))
 	test_failure_message_ "FAIL" "$@"
@@ -602,82 +615,101 @@ test_run_ () {
 	return 0
 }
 
 test_skip () {
 	test_count=$(($test_count+1))
 	to_skip=
 	for skp in $NOTMUCH_SKIP_TESTS
 	do
 		case $this_test.$test_count in
 		$skp)
 			to_skip=t
 		esac
 	done
 	if test -z "$to_skip" && test -n "$prereq" &&
 	   ! test_have_prereq "$prereq"
 	then
 		to_skip=t
 	fi
 	case "$to_skip" in
 	t)
-		test_reset_state_
-		say_color skip >&3 "skipping test: $@"
-		say_color skip "%-6s" "SKIP"
-		echo " $1"
-		: true
+		test_report_skip_ "$@"
 		;;
 	*)
-		false
+		test_check_missing_external_prereqs_ "$@"
 		;;
 	esac
 }
 
+test_check_missing_external_prereqs_ () {
+	if test -n "$test_subtest_missing_external_prereqs_"; then
+		say_color skip >&3 "missing prerequisites:"
+		echo "$test_subtest_missing_external_prereqs_" >&3
+		test_report_skip_ "$@"
+	else
+		false
+	fi
+}
+
+test_report_skip_ () {
+	test_reset_state_
+	say_color skip >&3 "skipping test: $@"
+	say_color skip "%-6s" "SKIP"
+	echo " $1"
+}
+
 test_subtest_known_broken () {
 	test_subtest_known_broken_=t
 }
 
 test_expect_success () {
 	test "$#" = 3 && { prereq=$1; shift; } || prereq=
 	test "$#" = 2 ||
 	error "bug in the test script: not 2 or 3 parameters to test-expect-success"
 	test_reset_state_
 	if ! test_skip "$@"
 	then
 		test_run_ "$2"
-		if [ "$?" = 0 -a "$eval_ret" = 0 ]
+		run_ret="$?"
+		# test_run_ may update missing external prerequisites
+		test_check_missing_external_prereqs_ "$@" ||
+		if [ "$run_ret" = 0 -a "$eval_ret" = 0 ]
 		then
 			test_ok_ "$1"
 		else
 			test_failure_ "$@"
 		fi
 	fi
 }
 
 test_expect_code () {
 	test "$#" = 4 && { prereq=$1; shift; } || prereq=
 	test "$#" = 3 ||
 	error "bug in the test script: not 3 or 4 parameters to test-expect-code"
 	test_reset_state_
 	if ! test_skip "$@"
 	then
 		test_run_ "$3"
-		if [ "$?" = 0 -a "$eval_ret" = "$1" ]
+		run_ret="$?"
+		# test_run_ may update missing external prerequisites,
+		test_check_missing_external_prereqs_ "$@" ||
+		if [ "$run_ret" = 0 -a "$eval_ret" = "$1" ]
 		then
 			test_ok_ "$2"
 		else
 			test_failure_ "$@"
 		fi
 	fi
 }
 
 # test_external runs external test scripts that provide continuous
 # test output about their progress, and succeeds/fails on
 # zero/non-zero exit code.  It outputs the test output on stdout even
 # in non-verbose mode, and announces the external script with "* run
 # <n>: ..." before running it.  When providing relative paths, keep in
 # mind that all scripts run in "trash directory".
 # Usage: test_external description command arguments...
 # Example: test_external 'Perl API' perl ../path/to/test.pl
 test_external () {
 	test "$#" = 4 && { prereq=$1; shift; } || prereq=
 	test "$#" = 3 ||
 	error >&5 "bug in the test script: not 3 or 4 parameters to test_external"
@@ -848,40 +880,41 @@ test_emacs () {
 		# user's TERM is given to dtach which assumes a minimally
 		# VT100-compatible terminal -- and emacs inherits that
 		TERM=$ORIGINAL_TERM dtach -n "$TMP_DIRECTORY/emacs-dtach-socket.$$" \
 			sh -c "stty rows 24 cols 80; exec '$TMP_DIRECTORY/run_emacs' \
 				--no-window-system \
 				--eval '(setq server-name \"$server_name\")' \
 				--eval '(server-start)' \
 				--eval '(orphan-watchdog $$)'" || return
 		EMACS_SERVER="$server_name"
 		# wait until the emacs server is up
 		until test_emacs '()' 2>/dev/null; do
 			sleep 1
 		done
 	fi
 
 	emacsclient --socket-name="$EMACS_SERVER" --eval "(progn $@)"
 }
 
 test_reset_state_ () {
 	test_subtest_known_broken_=
+	test_subtest_missing_external_prereqs_=
 }
 
 
 find_notmuch_path ()
 {
     dir="$1"
 
     while [ -n "$dir" ]; do
 	bin="$dir/notmuch"
 	if [ -x "$bin" ]; then
 	    echo "$dir"
 	    return
 	fi
 	dir="$(dirname "$dir")"
 	if [ "$dir" = "/" ]; then
 	    break
 	fi
     done
 }
 
-- 
1.7.7.2