From 7fc10011fcc182d02ded4a7fda114655145685f8 Mon Sep 17 00:00:00 2001 From: david Date: Tue, 4 Dec 2012 21:17:03 +2000 Subject: [PATCH] [PATCH 2/3] perf-test: cache unpacked corpus --- 27/dedccd4997e0415d3805d8e2d333ae646aade9 | 155 ++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 27/dedccd4997e0415d3805d8e2d333ae646aade9 diff --git a/27/dedccd4997e0415d3805d8e2d333ae646aade9 b/27/dedccd4997e0415d3805d8e2d333ae646aade9 new file mode 100644 index 000000000..37bedaadb --- /dev/null +++ b/27/dedccd4997e0415d3805d8e2d333ae646aade9 @@ -0,0 +1,155 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id B2412431FD6 + for ; Mon, 3 Dec 2012 17:17:40 -0800 (PST) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Spam-Flag: NO +X-Spam-Score: 0 +X-Spam-Level: +X-Spam-Status: No, score=0 tagged_above=-999 required=5 tests=[none] + autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id j70a4mTR2m02 for ; + Mon, 3 Dec 2012 17:17:36 -0800 (PST) +Received: from tesseract.cs.unb.ca (tesseract.cs.unb.ca [131.202.240.238]) + (using TLSv1 with cipher AES256-SHA (256/256 bits)) + (No client certificate requested) + by olra.theworths.org (Postfix) with ESMTPS id D10A2431FBC + for ; Mon, 3 Dec 2012 17:17:35 -0800 (PST) +Received: from fctnnbsc30w-142167090129.dhcp-dynamic.fibreop.nb.bellaliant.net + ([142.167.90.129] helo=zancas.localnet) + by tesseract.cs.unb.ca with esmtpsa + (TLS1.0:DHE_RSA_AES_128_CBC_SHA1:16) (Exim 4.72) + (envelope-from ) + id 1Tfh8g-00081S-OF; Mon, 03 Dec 2012 21:17:34 -0400 +Received: from bremner by zancas.localnet with local (Exim 4.80) + (envelope-from ) + id 1Tfh8b-0002ka-9g; Mon, 03 Dec 2012 21:17:29 -0400 +From: david@tethera.net +To: notmuch@notmuchmail.org +Subject: [PATCH 2/3] perf-test: cache unpacked corpus +Date: Mon, 3 Dec 2012 21:17:03 -0400 +Message-Id: <1354583824-10520-2-git-send-email-david@tethera.net> +X-Mailer: git-send-email 1.7.10.4 +In-Reply-To: <1354583824-10520-1-git-send-email-david@tethera.net> +References: <1354583824-10520-1-git-send-email-david@tethera.net> +X-Spam_bar: - +Cc: David Bremner +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Tue, 04 Dec 2012 01:17:41 -0000 + +From: David Bremner + +Unpacking is not really the expensive step (compared to the initial +notmuch new), but this is a pre-requisite to caching the database. +--- + performance-test/.gitignore | 1 + + performance-test/Makefile.local | 2 +- + performance-test/perf-test-lib.sh | 51 +++++++++++++++++++++---------------- + 3 files changed, 31 insertions(+), 23 deletions(-) + +diff --git a/performance-test/.gitignore b/performance-test/.gitignore +index 53f2697..7e20f7c 100644 +--- a/performance-test/.gitignore ++++ b/performance-test/.gitignore +@@ -1 +1,2 @@ + tmp.*/ ++corpus.mail.*/ +diff --git a/performance-test/Makefile.local b/performance-test/Makefile.local +index 5d2acbd..eb713d0 100644 +--- a/performance-test/Makefile.local ++++ b/performance-test/Makefile.local +@@ -29,4 +29,4 @@ $(TXZFILE): + download-corpus: + wget -O ${TXZFILE} ${DEFAULT_URL} + +-CLEAN := $(CLEAN) $(dir)/tmp.* ++CLEAN := $(CLEAN) $(dir)/tmp.* $(dir)/corpus.mail.* +diff --git a/performance-test/perf-test-lib.sh b/performance-test/perf-test-lib.sh +index bba793d..9fbf874 100644 +--- a/performance-test/perf-test-lib.sh ++++ b/performance-test/perf-test-lib.sh +@@ -35,37 +35,44 @@ then + exit 1 + fi + ++CORPUS_DIR=${TEST_DIRECTORY}/corpus.mail.$corpus_size + add_email_corpus () + { + rm -rf ${MAIL_DIR} ++ if [ ! -d $CORPUS_DIR ]; then ++ case "$corpus_size" in ++ small) ++ arg="mail/enron/bailey-s" ++ ;; ++ medium) ++ arg="mail/notmuch-archive" ++ ;; ++ *) ++ arg=mail ++ esac + +- case "$1" in +- --small) +- arg="mail/enron/bailey-s" +- ;; +- --medium) +- arg="mail/notmuch-archive" +- ;; +- *) +- arg=mail +- esac ++ if command -v pixz > /dev/null; then ++ XZ=pixz ++ else ++ XZ=xz ++ fi + +- if command -v pixz > /dev/null; then +- XZ=pixz +- else +- XZ=xz +- fi ++ printf "Unpacking corpus\n" ++ mkdir $CORPUS_DIR ++ ++ tar --checkpoint=.5000 --extract --strip-components=2 \ ++ --directory $CORPUS_DIR \ ++ --use-compress-program ${XZ} \ ++ --file ../download/notmuch-email-corpus-${PERFTEST_VERSION}.tar.xz \ ++ notmuch-email-corpus/"$arg" + +- printf "Unpacking corpus\n" +- tar --checkpoint=.5000 --extract --strip-components=1 \ +- --directory ${TMP_DIRECTORY} \ +- --use-compress-program ${XZ} \ +- --file ../download/notmuch-email-corpus-${PERFTEST_VERSION}.tar.xz \ +- notmuch-email-corpus/"$arg" ++ printf "\n" + +- printf "\n" ++ fi ++ cp -lr $CORPUS_DIR $MAIL_DIR + } + ++ + print_header () { + printf "[v%4s] Wall(s)\tUsr(s)\tSys(s)\tRes(K)\tIn(512B)\tOut(512B)\n" \ + ${PERFTEST_VERSION} +-- +1.7.10.4 + -- 2.26.2