--- /dev/null
+Return-Path: <pieter@praet.org>\r
+X-Original-To: notmuch@notmuchmail.org\r
+Delivered-To: notmuch@notmuchmail.org\r
+Received: from localhost (localhost [127.0.0.1])\r
+ by olra.theworths.org (Postfix) with ESMTP id 3EFB9431FD0\r
+ for <notmuch@notmuchmail.org>; Thu, 10 Nov 2011 17:34:34 -0800 (PST)\r
+X-Virus-Scanned: Debian amavisd-new at olra.theworths.org\r
+X-Spam-Flag: NO\r
+X-Spam-Score: -0.7\r
+X-Spam-Level: \r
+X-Spam-Status: No, score=-0.7 tagged_above=-999 required=5\r
+ tests=[RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled\r
+Received: from olra.theworths.org ([127.0.0.1])\r
+ by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024)\r
+ with ESMTP id eFAtHlloWcft for <notmuch@notmuchmail.org>;\r
+ Thu, 10 Nov 2011 17:34:33 -0800 (PST)\r
+Received: from mail-wy0-f181.google.com (mail-wy0-f181.google.com\r
+ [74.125.82.181]) (using TLSv1 with cipher RC4-SHA (128/128 bits))\r
+ (No client certificate requested)\r
+ by olra.theworths.org (Postfix) with ESMTPS id 20915431FB6\r
+ for <notmuch@notmuchmail.org>; Thu, 10 Nov 2011 17:34:33 -0800 (PST)\r
+Received: by wyg8 with SMTP id 8so3761231wyg.26\r
+ for <notmuch@notmuchmail.org>; Thu, 10 Nov 2011 17:34:32 -0800 (PST)\r
+Received: by 10.180.81.73 with SMTP id y9mr11590030wix.37.1320975271818;\r
+ Thu, 10 Nov 2011 17:34:31 -0800 (PST)\r
+Received: from localhost (26.48-242-81.adsl-dyn.isp.belgacom.be.\r
+ [81.242.48.26])\r
+ by mx.google.com with ESMTPS id co5sm5987687wib.8.2011.11.10.17.34.30\r
+ (version=TLSv1/SSLv3 cipher=OTHER);\r
+ Thu, 10 Nov 2011 17:34:31 -0800 (PST)\r
+From: Pieter Praet <pieter@praet.org>\r
+To: Austin Clements <amdragon@MIT.EDU>, notmuch@notmuchmail.org\r
+Subject: Re: [PATCH] Store "from" and "subject" headers in the database.\r
+In-Reply-To: <1320599856-24078-1-git-send-email-amdragon@mit.edu>\r
+References: <1320599856-24078-1-git-send-email-amdragon@mit.edu>\r
+User-Agent: Notmuch/0.9+76~g2fd88e6 (http://notmuchmail.org) Emacs/23.3.1\r
+ (x86_64-unknown-linux-gnu)\r
+Date: Fri, 11 Nov 2011 02:33:38 +0100\r
+Message-ID: <87obwjtpcd.fsf@praet.org>\r
+MIME-Version: 1.0\r
+Content-Type: text/plain; charset=us-ascii\r
+Cc: notmuch@kismala.com\r
+X-BeenThere: notmuch@notmuchmail.org\r
+X-Mailman-Version: 2.1.13\r
+Precedence: list\r
+List-Id: "Use and development of the notmuch mail system."\r
+ <notmuch.notmuchmail.org>\r
+List-Unsubscribe: <http://notmuchmail.org/mailman/options/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>\r
+List-Archive: <http://notmuchmail.org/pipermail/notmuch>\r
+List-Post: <mailto:notmuch@notmuchmail.org>\r
+List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>\r
+List-Subscribe: <http://notmuchmail.org/mailman/listinfo/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=subscribe>\r
+X-List-Received-Date: Fri, 11 Nov 2011 01:34:34 -0000\r
+\r
+On Sun, 6 Nov 2011 12:17:36 -0500, Austin Clements <amdragon@MIT.EDU> wrote:\r
+> This is a rebase and cleanup of Istvan Marko's patch from\r
+> id:m3pqnj2j7a.fsf@zsu.kismala.com\r
+> \r
+\r
+Fantastic performance improvement Austin! This should be merged in ASAP.\r
+\r
+BTW, compacting the db from time to time also has a significant impact:\r
+\r
+Running:\r
+ $ du -h .notmuch\r
+ $ sync && sudo /sbin/sysctl vm.drop_caches=3\r
+ $ time notmuch search "*" | wc -l\r
+\r
+On:\r
+ 1 - original database, compacted some time ago\r
+ 2 - fresh database generated before patching, non-compacted\r
+ 3 - fresh database generated after patching, non-compacted\r
+ 4 - fresh database generated after patching, compacted with\r
+ $ mv .notmuch/xapian .notmuch/xapian-fat\r
+ $ xapian-compact --no-renumber .notmuch/xapian-fat .notmuch/xapian\r
+\r
+Results:\r
+ | db | 1 | 2 | 3 | 4 |\r
+ |---------+-----------+----------+-----------+-----------|\r
+ | db size | 272M | 289M | 291M | 172M |\r
+ | amount | 9536 | 9540 | 9540 | 9540 |\r
+ |---------+-----------+----------+-----------+-----------|\r
+ | real | 1m42.221s | 2m3.193s | 0m30.762s | 0m10.505s |\r
+ | user | 0m8.379s | 0m8.133s | 0m4.043s | 0m3.353s |\r
+ | sys | 0m5.216s | 0m4.933s | 0m1.530s | 0m1.000s |\r
+\r
+\r
+> Search retrieves these headers for every message in the search\r
+> results. Previously, this required opening and parsing every message\r
+> file. Storing them directly in the database significantly reduces IO\r
+> and computation, speeding up search by between 50% and 10X.\r
+> \r
+> Taking full advantage of this requires a database rebuild, but it will\r
+> fall back to the old behavior for messages that do not have headers\r
+> stored in the database.\r
+> ---\r
+> lib/database.cc | 2 +-\r
+> lib/message.cc | 23 +++++++++++++++++++++--\r
+> lib/notmuch-private.h | 11 +++++++----\r
+> 3 files changed, 29 insertions(+), 7 deletions(-)\r
+> \r
+> diff --git a/lib/database.cc b/lib/database.cc\r
+> index fa632f8..e4ef14e 100644\r
+> --- a/lib/database.cc\r
+> +++ b/lib/database.cc\r
+> @@ -1725,7 +1725,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch,\r
+> goto DONE;\r
+> \r
+> date = notmuch_message_file_get_header (message_file, "date");\r
+> - _notmuch_message_set_date (message, date);\r
+> + _notmuch_message_set_header_values (message, date, from, subject);\r
+> \r
+> _notmuch_message_index_file (message, filename);\r
+> } else {\r
+> diff --git a/lib/message.cc b/lib/message.cc\r
+> index 8f22e02..ca7fbf2 100644\r
+> --- a/lib/message.cc\r
+> +++ b/lib/message.cc\r
+> @@ -412,6 +412,21 @@ _notmuch_message_ensure_message_file (notmuch_message_t *message)\r
+> const char *\r
+> notmuch_message_get_header (notmuch_message_t *message, const char *header)\r
+> {\r
+> + std::string value;\r
+> +\r
+> + /* Fetch header from the appropriate xapian value field if\r
+> + * available */\r
+> + if (strcasecmp (header, "from") == 0)\r
+> + value = message->doc.get_value (NOTMUCH_VALUE_FROM);\r
+> + else if (strcasecmp (header, "subject") == 0)\r
+> + value = message->doc.get_value (NOTMUCH_VALUE_SUBJECT);\r
+> + else if (strcasecmp (header, "message-id") == 0)\r
+> + value = message->doc.get_value (NOTMUCH_VALUE_MESSAGE_ID);\r
+> +\r
+> + if (!value.empty())\r
+> + return talloc_strdup (message, value.c_str ());\r
+> +\r
+> + /* Otherwise fall back to parsing the file */\r
+> _notmuch_message_ensure_message_file (message);\r
+> if (message->message_file == NULL)\r
+> return NULL;\r
+> @@ -795,8 +810,10 @@ notmuch_message_set_author (notmuch_message_t *message,\r
+> }\r
+> \r
+> void\r
+> -_notmuch_message_set_date (notmuch_message_t *message,\r
+> - const char *date)\r
+> +_notmuch_message_set_header_values (notmuch_message_t *message,\r
+> + const char *date,\r
+> + const char *from,\r
+> + const char *subject)\r
+> {\r
+> time_t time_value;\r
+> \r
+> @@ -809,6 +826,8 @@ _notmuch_message_set_date (notmuch_message_t *message,\r
+> \r
+> message->doc.add_value (NOTMUCH_VALUE_TIMESTAMP,\r
+> Xapian::sortable_serialise (time_value));\r
+> + message->doc.add_value (NOTMUCH_VALUE_FROM, from);\r
+> + message->doc.add_value (NOTMUCH_VALUE_SUBJECT, subject);\r
+> }\r
+> \r
+> /* Synchronize changes made to message->doc out into the database. */\r
+> diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h\r
+> index 0d3cc27..60a932f 100644\r
+> --- a/lib/notmuch-private.h\r
+> +++ b/lib/notmuch-private.h\r
+> @@ -93,7 +93,9 @@ NOTMUCH_BEGIN_DECLS\r
+> \r
+> typedef enum {\r
+> NOTMUCH_VALUE_TIMESTAMP = 0,\r
+> - NOTMUCH_VALUE_MESSAGE_ID\r
+> + NOTMUCH_VALUE_MESSAGE_ID,\r
+> + NOTMUCH_VALUE_FROM,\r
+> + NOTMUCH_VALUE_SUBJECT\r
+> } notmuch_value_t;\r
+> \r
+> /* Xapian (with flint backend) complains if we provide a term longer\r
+> @@ -269,9 +271,10 @@ void\r
+> _notmuch_message_ensure_thread_id (notmuch_message_t *message);\r
+> \r
+> void\r
+> -_notmuch_message_set_date (notmuch_message_t *message,\r
+> - const char *date);\r
+> -\r
+> +_notmuch_message_set_header_values (notmuch_message_t *message,\r
+> + const char *date,\r
+> + const char *from,\r
+> + const char *subject);\r
+> void\r
+> _notmuch_message_sync (notmuch_message_t *message);\r
+> \r
+> -- \r
+> 1.7.2.3\r
+> \r
+> _______________________________________________\r
+> notmuch mailing list\r
+> notmuch@notmuchmail.org\r
+> http://notmuchmail.org/mailman/listinfo/notmuch\r
+\r
+\r
+Peace\r
+\r
+-- \r
+Pieter\r
projects / notmuch-archives.git / commitdiff