[Patch v2 05/11] perf-test: cache unpacked corpus

diff --git a/c9/b70ffd350dbb8cc69b19436cd1e32018c0eb20 b/c9/b70ffd350dbb8cc69b19436cd1e32018c0eb20
new file mode 100644 (file)
index 0000000..8dcff36
--- /dev/null
+++ b/c9/b70ffd350dbb8cc69b19436cd1e32018c0eb20
@@ -0,0 +1,156 @@
+Return-Path: <bremner@tethera.net>\r
+X-Original-To: notmuch@notmuchmail.org\r
+Delivered-To: notmuch@notmuchmail.org\r
+Received: from localhost (localhost [127.0.0.1])\r
+       by olra.theworths.org (Postfix) with ESMTP id 8015F431E64\r
+       for <notmuch@notmuchmail.org>; Wed,  5 Dec 2012 19:02:13 -0800 (PST)\r
+X-Virus-Scanned: Debian amavisd-new at olra.theworths.org\r
+X-Spam-Flag: NO\r
+X-Spam-Score: 0\r
+X-Spam-Level: \r
+X-Spam-Status: No, score=0 tagged_above=-999 required=5 tests=[none]\r
+       autolearn=disabled\r
+Received: from olra.theworths.org ([127.0.0.1])\r
+       by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024)\r
+       with ESMTP id 0l1H+CRuxSsx for <notmuch@notmuchmail.org>;\r
+       Wed,  5 Dec 2012 19:02:11 -0800 (PST)\r
+Received: from tesseract.cs.unb.ca (tesseract.cs.unb.ca [131.202.240.238])\r
+       (using TLSv1 with cipher AES256-SHA (256/256 bits))\r
+       (No client certificate requested)\r
+       by olra.theworths.org (Postfix) with ESMTPS id 854F2431FB6\r
+       for <notmuch@notmuchmail.org>; Wed,  5 Dec 2012 19:02:11 -0800 (PST)\r
+Received: from fctnnbsc30w-142167090129.dhcp-dynamic.fibreop.nb.bellaliant.net\r
+       ([142.167.90.129] helo=zancas.localnet)\r
+       by tesseract.cs.unb.ca with esmtpsa\r
+       (TLS1.0:DHE_RSA_AES_128_CBC_SHA1:16) (Exim 4.72)\r
+       (envelope-from <bremner@tethera.net>)\r
+       id 1TgRiy-0002Cz-Pn; Wed, 05 Dec 2012 23:02:09 -0400\r
+Received: from bremner by zancas.localnet with local (Exim 4.80)\r
+       (envelope-from <bremner@tethera.net>)\r
+       id 1TgRit-0001Yf-A5; Wed, 05 Dec 2012 23:02:03 -0400\r
+From: david@tethera.net\r
+To: notmuch@notmuchmail.org\r
+Subject: [Patch v2 05/11] perf-test: cache unpacked corpus\r
+Date: Wed,  5 Dec 2012 23:01:42 -0400\r
+Message-Id: <1354762908-5788-6-git-send-email-david@tethera.net>\r
+X-Mailer: git-send-email 1.7.10.4\r
+In-Reply-To: <1354762908-5788-1-git-send-email-david@tethera.net>\r
+References: <1354762908-5788-1-git-send-email-david@tethera.net>\r
+X-Spam_bar: -\r
+Cc: David Bremner <bremner@debian.org>\r
+X-BeenThere: notmuch@notmuchmail.org\r
+X-Mailman-Version: 2.1.13\r
+Precedence: list\r
+List-Id: "Use and development of the notmuch mail system."\r
+       <notmuch.notmuchmail.org>\r
+List-Unsubscribe: <http://notmuchmail.org/mailman/options/notmuch>,\r
+       <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>\r
+List-Archive: <http://notmuchmail.org/pipermail/notmuch>\r
+List-Post: <mailto:notmuch@notmuchmail.org>\r
+List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>\r
+List-Subscribe: <http://notmuchmail.org/mailman/listinfo/notmuch>,\r
+       <mailto:notmuch-request@notmuchmail.org?subject=subscribe>\r
+X-List-Received-Date: Thu, 06 Dec 2012 03:02:13 -0000\r
+\r
+From: David Bremner <bremner@debian.org>\r
+\r
+Unpacking is not really the expensive step (compared to the initial\r
+notmuch new), but this is a pre-requisite to caching the database.\r
+---\r
+ performance-test/.gitignore       |    1 +\r
+ performance-test/Makefile.local   |    2 +-\r
+ performance-test/perf-test-lib.sh |   51 ++++++++++++++++++++++++-------------\r
+ 3 files changed, 36 insertions(+), 18 deletions(-)\r
+\r
+diff --git a/performance-test/.gitignore b/performance-test/.gitignore\r
+index 53f2697..796ed01 100644\r
+--- a/performance-test/.gitignore\r
++++ b/performance-test/.gitignore\r
+@@ -1 +1,2 @@\r
+ tmp.*/\r
++corpus/\r
+diff --git a/performance-test/Makefile.local b/performance-test/Makefile.local\r
+index 5d2acbd..63e4c3d 100644\r
+--- a/performance-test/Makefile.local\r
++++ b/performance-test/Makefile.local\r
+@@ -29,4 +29,4 @@ $(TXZFILE):\r
+ download-corpus:\r
+       wget -O ${TXZFILE} ${DEFAULT_URL}\r
+ \r
+-CLEAN := $(CLEAN) $(dir)/tmp.*\r
++CLEAN := $(CLEAN) $(dir)/tmp.* $(dir)/corpus\r
+diff --git a/performance-test/perf-test-lib.sh b/performance-test/perf-test-lib.sh\r
+index 415bf15..0c10718 100644\r
+--- a/performance-test/perf-test-lib.sh\r
++++ b/performance-test/perf-test-lib.sh\r
+@@ -39,33 +39,50 @@ add_email_corpus ()\r
+ {\r
+     rm -rf ${MAIL_DIR}\r
+ \r
+-    case "$1" in\r
+-      --small)\r
+-          arg="mail/enron/bailey-s"\r
++    case "$corpus_size" in\r
++      small)\r
++          mail_subdir="mail/enron/bailey-s"\r
++          check_for="${TEST_DIRECTORY}/corpus/$mail_subdir"\r
+           ;;\r
+-      --medium)\r
+-          arg="mail/notmuch-archive"\r
++      medium)\r
++          mail_subdir="mail/notmuch-archive"\r
++          check_for="${TEST_DIRECTORY}/corpus/$mail_subdir"\r
+           ;;\r
+       *)\r
+-          arg=mail\r
++          mail_subdir=mail\r
++          check_for="${TEST_DIRECTORY}/corpus/$mail_subdir/enron/wolfe-j"\r
+     esac\r
+ \r
+-    if command -v pixz > /dev/null; then\r
+-      XZ=pixz\r
+-    else\r
+-      XZ=xz\r
++    MAIL_CORPUS="${TEST_DIRECTORY}/corpus/$mail_subdir"\r
++    args=()\r
++    if [ ! -d "$check_for" ] ; then\r
++      args+=("notmuch-email-corpus/$mail_subdir")\r
+     fi\r
+ \r
+-    printf "Unpacking corpus\n"\r
+-    tar --checkpoint=.5000 --extract --strip-components=1 \\r
+-      --directory ${TMP_DIRECTORY} \\r
+-      --use-compress-program ${XZ} \\r
+-      --file ../download/notmuch-email-corpus-${PERFTEST_VERSION}.tar.xz \\r
+-      notmuch-email-corpus/"$arg"\r
++    if [[ ${#args[@]} > 0 ]]; then\r
++      if command -v pixz > /dev/null; then\r
++          XZ=pixz\r
++      else\r
++          XZ=xz\r
++      fi\r
++\r
++      printf "Unpacking corpus\n"\r
++      mkdir -p "${TEST_DIRECTORY}/corpus"\r
++\r
++      tar --checkpoint=.5000 --extract --strip-components=1 \\r
++          --directory ${TEST_DIRECTORY}/corpus \\r
++          --use-compress-program ${XZ} \\r
++          --file ../download/notmuch-email-corpus-${PERFTEST_VERSION}.tar.xz \\r
++          "${args[@]}"\r
++\r
++      printf "\n"\r
+ \r
+-    printf "\n"\r
++    fi\r
++\r
++    cp -lr $MAIL_CORPUS $MAIL_DIR\r
+ }\r
+ \r
++\r
+ print_header () {\r
+     printf "[v%4s %6s]          Wall(s)\tUsr(s)\tSys(s)\tRes(K)\tIn/Out(512B)\n" \\r
+          ${PERFTEST_VERSION} ${corpus_size}\r
+-- \r
+1.7.10.4\r
+\r