From: David Bremner <david@tethera.net>
To: notmuch@notmuchmail.org
Subject: v3 of boolean folder: patches
Date: Sat,  8 Mar 2014 17:19:30 -0400
Message-Id: <1394313585-28422-1-git-send-email-david@tethera.net>
Precedence: list

This is a followup to 

     id:cover.1393105055.git.jani@nikula.org

The first 11 patches are very close to that series; in the last 4 I
have added some minimalist infrastructure to download and verify
pre-built test databases.

I decided that the benefits of being to really apply and test the
patch series outweighed the fact that one of patchs is about
150K. Apologies to those of you on GSM modems and the like.

Interdiff follows 

diff --git a/Makefile b/Makefile
index 0428160..97084b1 100644
--- a/Makefile
+++ b/Makefile
@@ -5,7 +5,8 @@ all:
 # List all subdirectories here. Each contains its own Makefile.local.
 # Use of '=', without '+=', seems to be required for out-of-tree
 # builds to work.
-subdirs = compat completion emacs lib man parse-time-string performance-test util test
+subdirs = compat completion emacs lib man parse-time-string performance-test util test \
+	test/test-databases
 
 # We make all targets depend on the Makefiles themselves.
 global_deps = Makefile Makefile.config Makefile.local \
diff --git a/devel/gen-testdb.sh b/devel/gen-testdb.sh
index c291dff..621b31e 100755
--- a/devel/gen-testdb.sh
+++ b/devel/gen-testdb.sh
@@ -81,6 +81,10 @@ shift `expr $OPTIND - 1`
 
 . ./test-lib.sh
 
+SHORT_CORPUS=$(basename ${CORPUS:-database})
+DBNAME=${SHORT_CORPUS}${SUFFIX}
+TARBALLNAME=${DBNAME}.tar.xz
+
 CORPUS=${CORPUS:-${TEST_DIRECTORY}/corpus}
 
 test_expect_code 0 "notmuch version specified on the command line" \
@@ -112,13 +116,16 @@ cp -a ${CORPUS} ${MAIL_DIR}
 test_expect_code 0 "index the corpus" \
     "notmuch new"
 
-# finally, wrap the resulting mail store and database in a tarball
-DBNAME=database${SUFFIX}
+# wrap the resulting mail store and database in a tarball
+
 cp -a ${MAIL_DIR} ${TMP_DIRECTORY}/${DBNAME}
-tar zcf ${TMP_DIRECTORY}/${DBNAME}.tar.gz -C ${TMP_DIRECTORY} ${DBNAME}
+tar Jcf ${TMP_DIRECTORY}/${TARBALLNAME} -C ${TMP_DIRECTORY} ${DBNAME}
 mkdir -p  ${TEST_DIRECTORY}/test-databases
-cp -a ${TMP_DIRECTORY}/${DBNAME}.tar.gz ${TEST_DIRECTORY}/test-databases
-test_expect_code 0 "create the output tarball ${DBNAME}.tar.gz" \
-    "test -f ${TEST_DIRECTORY}/test-databases/${DBNAME}.tar.gz"
+cp -a ${TMP_DIRECTORY}/${TARBALLNAME} ${TEST_DIRECTORY}/test-databases
+test_expect_code 0 "create the output tarball ${TARBALLNAME}" \
+    "test -f ${TEST_DIRECTORY}/test-databases/${TARBALLNAME}"
 
+# generate a checksum file
+test_expect_code 0 "compute checksum" \
+    "(cd ${TEST_DIRECTORY}/test-databases/ && sha256sum ${TARBALLNAME} > ${TARBALLNAME}.sha256)"
 test_done
diff --git a/test/Makefile.local b/test/Makefile.local
index 99324ba..bfabd15 100644
--- a/test/Makefile.local
+++ b/test/Makefile.local
@@ -2,6 +2,8 @@
 
 dir := test
 
+# save against changes in $(dir)
+test_src_dir := $(dir)
 extra_cflags += -I.
 
 smtp_dummy_srcs =		\
@@ -44,12 +46,12 @@ TEST_BINARIES=$(dir)/arg-test \
 
 test-binaries: $(TEST_BINARIES)
 
-test:	all test-binaries
-	@${dir}/notmuch-test $(OPTIONS)
+test:	all test-binaries fetch-test-databases
+	@${test_src_dir}/notmuch-test $(OPTIONS)
 
 check: test
 
 SRCS := $(SRCS) $(smtp_dummy_srcs)
 CLEAN += $(TEST_BINARIES) $(addsuffix .o,$(TEST_BINARIES)) \
 	 $(dir)/database-test.o \
-	 $(dir)/corpus.mail $(dir)/test-results $(dir)/tmp.*
+	 $(dir)/corpus.mail.* $(dir)/test-results $(dir)/tmp.*
diff --git a/test/README b/test/README
index 79a9b1b..81a1c82 100644
--- a/test/README
+++ b/test/README
@@ -64,6 +64,14 @@ The following command-line options are available when running tests:
 	Pointing this argument at a tmpfs filesystem can improve the
 	speed of the test suite for some users.
 
+Certain tests require precomputed databases to complete. You can fetch these
+databases with
+
+	make download-test-databases
+
+If you do not download the test databases, the relevant tests will be
+skipped.
+
 When invoking the test suite via "make test" any of the above options
 can be specified as follows:
 
diff --git a/test/T530-upgrade.sh b/test/T530-upgrade.sh
index cf9914e..e1e8ac5 100755
--- a/test/T530-upgrade.sh
+++ b/test/T530-upgrade.sh
@@ -3,7 +3,19 @@ test_description="database upgrade"
 
 . ./test-lib.sh
 
-tar zxf $TEST_DIRECTORY/test-databases/database-v1.tar.gz -C ${MAIL_DIR} --strip-components=1
+dbtarball=folders-v1.tar.xz
+
+# XXX: Accomplish the same with test lib helpers
+if [ ! -e ${TEST_DIRECTORY}/test-databases/${dbtarball} ]; then
+    test_subtest_missing_external_prereq_["${dbtarball} - fetch with 'make download-test-databases'"]=t
+fi
+
+test_expect_success \
+    'database checksum' \
+    '( cd $TEST_DIRECTORY/test-databases &&
+       sha256sum --quiet --check --status ${dbtarball}.sha256 )'
+
+tar xf $TEST_DIRECTORY/test-databases/${dbtarball} -C ${MAIL_DIR} --strip-components=1
 
 test_begin_subtest "folder: search does not work with old database version"
 output=$(notmuch search folder:foo)
diff --git a/test/test-databases/.gitignore b/test/test-databases/.gitignore
new file mode 100644
index 0000000..b5624b7
--- /dev/null
+++ b/test/test-databases/.gitignore
@@ -0,0 +1 @@
+*.tar.xz
diff --git a/test/test-databases/Makefile b/test/test-databases/Makefile
new file mode 100644
index 0000000..b250a8b
--- /dev/null
+++ b/test/test-databases/Makefile
@@ -0,0 +1,7 @@
+# See Makefile.local for the list of files to be compiled in this
+# directory.
+all:
+	$(MAKE) -C ../.. all
+
+.DEFAULT:
+	$(MAKE) -C ../.. $@
diff --git a/test/test-databases/Makefile.local b/test/test-databases/Makefile.local
new file mode 100644
index 0000000..e777ada
--- /dev/null
+++ b/test/test-databases/Makefile.local
@@ -0,0 +1,14 @@
+# -*- makefile -*-
+
+TEST_DATABASE_MIRROR=http://notmuchmail.org/releases/test-databases
+
+dir := test/test-databases
+
+test_databases := $(dir)/folders-v1.tar.xz
+
+%.tar.xz:
+	wget -nv -O $@ ${TEST_DATABASE_MIRROR}/$(notdir $@);
+
+download-test-databases: ${test_databases}
+
+DISTCLEAN := $(DISTCLEAN) ${test_databases}
diff --git a/test/test-databases/README b/test/test-databases/README
deleted file mode 100644
index af5defe..0000000
--- a/test/test-databases/README
+++ /dev/null
@@ -1,5 +0,0 @@
-Notmuch test databases
-======================
-
-This directory contains pre-generated databases with their source
-corpus, chiefly for the purpose of testing database upgrade.
diff --git a/test/test-databases/database-v1.tar.gz b/test/test-databases/database-v1.tar.gz
deleted file mode 100644
index bb4df4d..0000000
Binary files a/test/test-databases/database-v1.tar.gz and /dev/null differ
diff --git a/test/test-databases/folders-v1.tar.xz.sha256 b/test/test-databases/folders-v1.tar.xz.sha256
new file mode 100644
index 0000000..01ad79d
--- /dev/null
+++ b/test/test-databases/folders-v1.tar.xz.sha256
@@ -0,0 +1 @@
+ace8a61216756b90a421e23d03910e1228bcb910e197c35e51e29f2cf57b37d9  folders-v1.tar.xz