c9/b70ffd350dbb8cc69b19436cd1e32018c0eb20

   1 Return-Path: <bremner@tethera.net>\r
   2 X-Original-To: notmuch@notmuchmail.org\r
   3 Delivered-To: notmuch@notmuchmail.org\r
   4 Received: from localhost (localhost [127.0.0.1])\r
   5         by olra.theworths.org (Postfix) with ESMTP id 8015F431E64\r
   6         for <notmuch@notmuchmail.org>; Wed,  5 Dec 2012 19:02:13 -0800 (PST)\r
   7 X-Virus-Scanned: Debian amavisd-new at olra.theworths.org\r
   8 X-Spam-Flag: NO\r
   9 X-Spam-Score: 0\r
  10 X-Spam-Level: \r
  11 X-Spam-Status: No, score=0 tagged_above=-999 required=5 tests=[none]\r
  12         autolearn=disabled\r
  13 Received: from olra.theworths.org ([127.0.0.1])\r
  14         by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024)\r
  15         with ESMTP id 0l1H+CRuxSsx for <notmuch@notmuchmail.org>;\r
  16         Wed,  5 Dec 2012 19:02:11 -0800 (PST)\r
  17 Received: from tesseract.cs.unb.ca (tesseract.cs.unb.ca [131.202.240.238])\r
  18         (using TLSv1 with cipher AES256-SHA (256/256 bits))\r
  19         (No client certificate requested)\r
  20         by olra.theworths.org (Postfix) with ESMTPS id 854F2431FB6\r
  21         for <notmuch@notmuchmail.org>; Wed,  5 Dec 2012 19:02:11 -0800 (PST)\r
  22 Received: from fctnnbsc30w-142167090129.dhcp-dynamic.fibreop.nb.bellaliant.net\r
  23         ([142.167.90.129] helo=zancas.localnet)\r
  24         by tesseract.cs.unb.ca with esmtpsa\r
  25         (TLS1.0:DHE_RSA_AES_128_CBC_SHA1:16) (Exim 4.72)\r
  26         (envelope-from <bremner@tethera.net>)\r
  27         id 1TgRiy-0002Cz-Pn; Wed, 05 Dec 2012 23:02:09 -0400\r
  28 Received: from bremner by zancas.localnet with local (Exim 4.80)\r
  29         (envelope-from <bremner@tethera.net>)\r
  30         id 1TgRit-0001Yf-A5; Wed, 05 Dec 2012 23:02:03 -0400\r
  31 From: david@tethera.net\r
  32 To: notmuch@notmuchmail.org\r
  33 Subject: [Patch v2 05/11] perf-test: cache unpacked corpus\r
  34 Date: Wed,  5 Dec 2012 23:01:42 -0400\r
  35 Message-Id: <1354762908-5788-6-git-send-email-david@tethera.net>\r
  36 X-Mailer: git-send-email 1.7.10.4\r
  37 In-Reply-To: <1354762908-5788-1-git-send-email-david@tethera.net>\r
  38 References: <1354762908-5788-1-git-send-email-david@tethera.net>\r
  39 X-Spam_bar: -\r
  40 Cc: David Bremner <bremner@debian.org>\r
  41 X-BeenThere: notmuch@notmuchmail.org\r
  42 X-Mailman-Version: 2.1.13\r
  43 Precedence: list\r
  44 List-Id: "Use and development of the notmuch mail system."\r
  45         <notmuch.notmuchmail.org>\r
  46 List-Unsubscribe: <http://notmuchmail.org/mailman/options/notmuch>,\r
  47         <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>\r
  48 List-Archive: <http://notmuchmail.org/pipermail/notmuch>\r
  49 List-Post: <mailto:notmuch@notmuchmail.org>\r
  50 List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>\r
  51 List-Subscribe: <http://notmuchmail.org/mailman/listinfo/notmuch>,\r
  52         <mailto:notmuch-request@notmuchmail.org?subject=subscribe>\r
  53 X-List-Received-Date: Thu, 06 Dec 2012 03:02:13 -0000\r
  54 \r
  55 From: David Bremner <bremner@debian.org>\r
  56 \r
  57 Unpacking is not really the expensive step (compared to the initial\r
  58 notmuch new), but this is a pre-requisite to caching the database.\r
  59 ---\r
  60  performance-test/.gitignore       |    1 +\r
  61  performance-test/Makefile.local   |    2 +-\r
  62  performance-test/perf-test-lib.sh |   51 ++++++++++++++++++++++++-------------\r
  63  3 files changed, 36 insertions(+), 18 deletions(-)\r
  64 \r
  65 diff --git a/performance-test/.gitignore b/performance-test/.gitignore\r
  66 index 53f2697..796ed01 100644\r
  67 --- a/performance-test/.gitignore\r
  68 +++ b/performance-test/.gitignore\r
  69 @@ -1 +1,2 @@\r
  70  tmp.*/\r
  71 +corpus/\r
  72 diff --git a/performance-test/Makefile.local b/performance-test/Makefile.local\r
  73 index 5d2acbd..63e4c3d 100644\r
  74 --- a/performance-test/Makefile.local\r
  75 +++ b/performance-test/Makefile.local\r
  76 @@ -29,4 +29,4 @@ $(TXZFILE):\r
  77  download-corpus:\r
  78         wget -O ${TXZFILE} ${DEFAULT_URL}\r
  79  \r
  80 -CLEAN := $(CLEAN) $(dir)/tmp.*\r
  81 +CLEAN := $(CLEAN) $(dir)/tmp.* $(dir)/corpus\r
  82 diff --git a/performance-test/perf-test-lib.sh b/performance-test/perf-test-lib.sh\r
  83 index 415bf15..0c10718 100644\r
  84 --- a/performance-test/perf-test-lib.sh\r
  85 +++ b/performance-test/perf-test-lib.sh\r
  86 @@ -39,33 +39,50 @@ add_email_corpus ()\r
  87  {\r
  88      rm -rf ${MAIL_DIR}\r
  89  \r
  90 -    case "$1" in\r
  91 -       --small)\r
  92 -           arg="mail/enron/bailey-s"\r
  93 +    case "$corpus_size" in\r
  94 +       small)\r
  95 +           mail_subdir="mail/enron/bailey-s"\r
  96 +           check_for="${TEST_DIRECTORY}/corpus/$mail_subdir"\r
  97             ;;\r
  98 -       --medium)\r
  99 -           arg="mail/notmuch-archive"\r
 100 +       medium)\r
 101 +           mail_subdir="mail/notmuch-archive"\r
 102 +           check_for="${TEST_DIRECTORY}/corpus/$mail_subdir"\r
 103             ;;\r
 104         *)\r
 105 -           arg=mail\r
 106 +           mail_subdir=mail\r
 107 +           check_for="${TEST_DIRECTORY}/corpus/$mail_subdir/enron/wolfe-j"\r
 108      esac\r
 109  \r
 110 -    if command -v pixz > /dev/null; then\r
 111 -       XZ=pixz\r
 112 -    else\r
 113 -       XZ=xz\r
 114 +    MAIL_CORPUS="${TEST_DIRECTORY}/corpus/$mail_subdir"\r
 115 +    args=()\r
 116 +    if [ ! -d "$check_for" ] ; then\r
 117 +       args+=("notmuch-email-corpus/$mail_subdir")\r
 118      fi\r
 119  \r
 120 -    printf "Unpacking corpus\n"\r
 121 -    tar --checkpoint=.5000 --extract --strip-components=1 \\r
 122 -       --directory ${TMP_DIRECTORY} \\r
 123 -       --use-compress-program ${XZ} \\r
 124 -       --file ../download/notmuch-email-corpus-${PERFTEST_VERSION}.tar.xz \\r
 125 -       notmuch-email-corpus/"$arg"\r
 126 +    if [[ ${#args[@]} > 0 ]]; then\r
 127 +       if command -v pixz > /dev/null; then\r
 128 +           XZ=pixz\r
 129 +       else\r
 130 +           XZ=xz\r
 131 +       fi\r
 132 +\r
 133 +       printf "Unpacking corpus\n"\r
 134 +       mkdir -p "${TEST_DIRECTORY}/corpus"\r
 135 +\r
 136 +       tar --checkpoint=.5000 --extract --strip-components=1 \\r
 137 +           --directory ${TEST_DIRECTORY}/corpus \\r
 138 +           --use-compress-program ${XZ} \\r
 139 +           --file ../download/notmuch-email-corpus-${PERFTEST_VERSION}.tar.xz \\r
 140 +           "${args[@]}"\r
 141 +\r
 142 +       printf "\n"\r
 143  \r
 144 -    printf "\n"\r
 145 +    fi\r
 146 +\r
 147 +    cp -lr $MAIL_CORPUS $MAIL_DIR\r
 148  }\r
 149  \r
 150 +\r
 151  print_header () {\r
 152      printf "[v%4s %6s]          Wall(s)\tUsr(s)\tSys(s)\tRes(K)\tIn/Out(512B)\n" \\r
 153            ${PERFTEST_VERSION} ${corpus_size}\r
 154 -- \r
 155 1.7.10.4\r
 156 \r