From f3f8ebb0afeccb2cad71e3d28c8acb342bb6a49f Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Wed, 5 Dec 2012 23:55:56 +1900 Subject: [PATCH] Re: [PATCH 2/3] perf-test: cache unpacked corpus --- 42/21286333dcea24a34e075de769749c18a39c28 | 187 ++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 42/21286333dcea24a34e075de769749c18a39c28 diff --git a/42/21286333dcea24a34e075de769749c18a39c28 b/42/21286333dcea24a34e075de769749c18a39c28 new file mode 100644 index 000000000..06bf6d2c2 --- /dev/null +++ b/42/21286333dcea24a34e075de769749c18a39c28 @@ -0,0 +1,187 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id 87E91431FB6 + for ; Tue, 4 Dec 2012 20:56:05 -0800 (PST) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Spam-Flag: NO +X-Spam-Score: -0.7 +X-Spam-Level: +X-Spam-Status: No, score=-0.7 tagged_above=-999 required=5 + tests=[RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id 3Jclia8FikoE for ; + Tue, 4 Dec 2012 20:56:01 -0800 (PST) +Received: from dmz-mailsec-scanner-2.mit.edu (DMZ-MAILSEC-SCANNER-2.MIT.EDU + [18.9.25.13]) + by olra.theworths.org (Postfix) with ESMTP id E89E0431FAE + for ; Tue, 4 Dec 2012 20:56:00 -0800 (PST) +X-AuditID: 1209190d-b7f266d00000092b-b6-50bed3e077ea +Received: from mailhub-auth-1.mit.edu ( [18.9.21.35]) + by dmz-mailsec-scanner-2.mit.edu (Symantec Messaging Gateway) with SMTP + id DF.28.02347.0E3DEB05; Tue, 4 Dec 2012 23:56:00 -0500 (EST) +Received: from outgoing.mit.edu (OUTGOING-AUTH.MIT.EDU [18.7.22.103]) + by mailhub-auth-1.mit.edu (8.13.8/8.9.2) with ESMTP id qB54txAA015022; + Tue, 4 Dec 2012 23:55:59 -0500 +Received: from awakening.csail.mit.edu (awakening.csail.mit.edu [18.26.4.91]) + (authenticated bits=0) + (User authenticated as amdragon@ATHENA.MIT.EDU) + by outgoing.mit.edu (8.13.6/8.12.4) with ESMTP id qB54tuMV019436 + (version=TLSv1/SSLv3 cipher=DHE-RSA-AES128-SHA bits=128 verify=NOT); + Tue, 4 Dec 2012 23:55:58 -0500 (EST) +Received: from amthrax by awakening.csail.mit.edu with local (Exim 4.80) + (envelope-from ) + id 1Tg71Y-0004ff-GU; Tue, 04 Dec 2012 23:55:56 -0500 +From: Austin Clements +To: david@tethera.net, notmuch@notmuchmail.org +Subject: Re: [PATCH 2/3] perf-test: cache unpacked corpus +In-Reply-To: <1354583824-10520-2-git-send-email-david@tethera.net> +References: <1354583824-10520-1-git-send-email-david@tethera.net> + <1354583824-10520-2-git-send-email-david@tethera.net> +User-Agent: Notmuch/0.14+100~gcb9b0b0 (http://notmuchmail.org) Emacs/23.4.1 + (i486-pc-linux-gnu) +Date: Tue, 04 Dec 2012 23:55:56 -0500 +Message-ID: <87vcchcd2b.fsf@awakening.csail.mit.edu> +MIME-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +X-Brightmail-Tracker: + H4sIAAAAAAAAA+NgFmpileLIzCtJLcpLzFFi42IR4hRV1n1weV+AwdlPEhYbl/1ktrjR2s1o + cf3mTGYHZo9fbXOZPZ6tusXsseXQe+YA5igum5TUnMyy1CJ9uwSujM2zV7MU3JGo2Hh5L1sD + 4wrhLkZODgkBE4mrL9+zQ9hiEhfurWfrYuTiEBLYxyhxY08LI4SznlFiytKz7BDOBSaJ29ve + MkM4SxgldsxcwAjSzyagL7Fi7STWLkYODhEBPYmFW2VAwswCahKT1pwBWyEsYCkxc8V/JhCb + U8BJ4tKkL2CtQgLlEjNm/mUEaRUViJe4vFQBJMwioCpx9uFOsHJeoEv/99yDsgUlTs58wgIx + Xkvixr+XTBMYBWchSc1CklrAyLSKUTYlt0o3NzEzpzg1Wbc4OTEvL7VI10gvN7NELzWldBMj + OHgleXcwvjuodIhRgINRiYdX0nhfgBBrYllxZe4hRkkOJiVR3rYjQCG+pPyUyozE4oz4otKc + 1OJDjBIczEoivP/sgHK8KYmVValF+TApaQ4WJXHeKyk3/YUE0hNLUrNTUwtSi2CyMhwcShK8 + psAoFRIsSk1PrUjLzClBSDNxcIIM5wEa7ghSw1tckJhbnJkOkT/FqMvR+6/rCaMQS15+XqqU + OK8cSJEASFFGaR7cHFjSecUoDvSWMK8GSBUPMGHBTXoFtIQJaMkLod0gS0oSEVJSDYwcs/0d + r5xcW3szZt+k+HW9y1wXaLoKFWW/6q/+WFkQyu38dPvHXHvrK59fTAyev56XgSHKr+mpK7Pa + X+9ulYa+Xx+2TK67fmiHqO/C8OU/JsirXls77QS31j6lLLuZ7ImPZ+uuspM9vl0hyk6j7v+T + bJPAlT9PmX1bvyTjcnR/PBf3yv0PlOKVWIozEg21mIuKEwGuaL+IFQMAAA== +Cc: David Bremner +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Wed, 05 Dec 2012 04:56:05 -0000 + +On Mon, 03 Dec 2012, david@tethera.net wrote: +> From: David Bremner +> +> Unpacking is not really the expensive step (compared to the initial +> notmuch new), but this is a pre-requisite to caching the database. +> --- +> performance-test/.gitignore | 1 + +> performance-test/Makefile.local | 2 +- +> performance-test/perf-test-lib.sh | 51 +++++++++++++++++++++---------------- +> 3 files changed, 31 insertions(+), 23 deletions(-) +> +> diff --git a/performance-test/.gitignore b/performance-test/.gitignore +> index 53f2697..7e20f7c 100644 +> --- a/performance-test/.gitignore +> +++ b/performance-test/.gitignore +> @@ -1 +1,2 @@ +> tmp.*/ +> +corpus.mail.*/ +> diff --git a/performance-test/Makefile.local b/performance-test/Makefile.local +> index 5d2acbd..eb713d0 100644 +> --- a/performance-test/Makefile.local +> +++ b/performance-test/Makefile.local +> @@ -29,4 +29,4 @@ $(TXZFILE): +> download-corpus: +> wget -O ${TXZFILE} ${DEFAULT_URL} +> +> -CLEAN := $(CLEAN) $(dir)/tmp.* +> +CLEAN := $(CLEAN) $(dir)/tmp.* $(dir)/corpus.mail.* +> diff --git a/performance-test/perf-test-lib.sh b/performance-test/perf-test-lib.sh +> index bba793d..9fbf874 100644 +> --- a/performance-test/perf-test-lib.sh +> +++ b/performance-test/perf-test-lib.sh +> @@ -35,37 +35,44 @@ then +> exit 1 +> fi +> +> +CORPUS_DIR=${TEST_DIRECTORY}/corpus.mail.$corpus_size +> add_email_corpus () +> { +> rm -rf ${MAIL_DIR} +> + if [ ! -d $CORPUS_DIR ]; then +> + case "$corpus_size" in +> + small) +> + arg="mail/enron/bailey-s" +> + ;; +> + medium) +> + arg="mail/notmuch-archive" +> + ;; +> + *) +> + arg=mail +> + esac +> +> - case "$1" in +> - --small) +> - arg="mail/enron/bailey-s" +> - ;; +> - --medium) +> - arg="mail/notmuch-archive" +> - ;; + +The README still refers to these arguments, so it should be updated, +too. + +> - *) +> - arg=mail +> - esac +> + if command -v pixz > /dev/null; then +> + XZ=pixz +> + else +> + XZ=xz +> + fi +> +> - if command -v pixz > /dev/null; then +> - XZ=pixz +> - else +> - XZ=xz +> - fi +> + printf "Unpacking corpus\n" +> + mkdir $CORPUS_DIR +> + +> + tar --checkpoint=.5000 --extract --strip-components=2 \ +> + --directory $CORPUS_DIR \ +> + --use-compress-program ${XZ} \ +> + --file ../download/notmuch-email-corpus-${PERFTEST_VERSION}.tar.xz \ +> + notmuch-email-corpus/"$arg" +> +> - printf "Unpacking corpus\n" +> - tar --checkpoint=.5000 --extract --strip-components=1 \ +> - --directory ${TMP_DIRECTORY} \ +> - --use-compress-program ${XZ} \ +> - --file ../download/notmuch-email-corpus-${PERFTEST_VERSION}.tar.xz \ +> - notmuch-email-corpus/"$arg" +> + printf "\n" +> +> - printf "\n" +> + fi +> + cp -lr $CORPUS_DIR $MAIL_DIR +> } +> +> + +> print_header () { +> printf "[v%4s] Wall(s)\tUsr(s)\tSys(s)\tRes(K)\tIn(512B)\tOut(512B)\n" \ +> ${PERFTEST_VERSION} +> -- +> 1.7.10.4 +> +> _______________________________________________ +> notmuch mailing list +> notmuch@notmuchmail.org +> http://notmuchmail.org/mailman/listinfo/notmuch -- 2.26.2