[notmuch] [PATCH v2] Store the size of the file for each message

diff --git a/80/96d14f5037013426a63e3f5520f473c6c927f5 b/80/96d14f5037013426a63e3f5520f473c6c927f5
new file mode 100644 (file)
index 0000000..2b6dac4
--- /dev/null
+++ b/80/96d14f5037013426a63e3f5520f473c6c927f5
@@ -0,0 +1,216 @@
+Return-Path: <james@jameswestby.net>\r
+X-Original-To: notmuch@notmuchmail.org\r
+Delivered-To: notmuch@notmuchmail.org\r
+Received: from localhost (localhost [127.0.0.1])\r
+       by olra.theworths.org (Postfix) with ESMTP id DB168431FC0\r
+       for <notmuch@notmuchmail.org>; Fri, 18 Dec 2009 16:12:02 -0800 (PST)\r
+X-Virus-Scanned: Debian amavisd-new at olra.theworths.org\r
+Received: from olra.theworths.org ([127.0.0.1])\r
+       by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024)\r
+       with ESMTP id gubYVN8FosCK for <notmuch@notmuchmail.org>;\r
+       Fri, 18 Dec 2009 16:12:01 -0800 (PST)\r
+Received: from jameswestby.net (jameswestby.net [89.145.97.141])\r
+       by olra.theworths.org (Postfix) with ESMTP id 68580431FBF\r
+       for <notmuch@notmuchmail.org>; Fri, 18 Dec 2009 16:12:01 -0800 (PST)\r
+Received: from cpc4-aztw22-2-0-cust59.aztw.cable.virginmedia.com\r
+       ([94.169.116.60] helo=flash)\r
+       by jameswestby.net with esmtpa (Exim 4.69)\r
+       (envelope-from <james@jameswestby.net>)\r
+       id 1NLmvP-0006Ca-TL; Sat, 19 Dec 2009 00:12:00 +0000\r
+Received: by flash (Postfix, from userid 1000)\r
+       id 7075C6E546A; Sat, 19 Dec 2009 00:11:54 +0000 (GMT)\r
+From: James Westby <jw+debian@jameswestby.net>\r
+To: notmuch@notmuchmail.org\r
+Date: Sat, 19 Dec 2009 00:11:48 +0000\r
+Message-Id: <1261181508-22222-1-git-send-email-jw+debian@jameswestby.net>\r
+X-Mailer: git-send-email 1.6.3.3\r
+In-Reply-To: <871virzzjy.fsf@yoom.home.cworth.org>\r
+References: <871virzzjy.fsf@yoom.home.cworth.org>\r
+Subject: [notmuch] [PATCH v2] Store the size of the file for each message\r
+X-BeenThere: notmuch@notmuchmail.org\r
+X-Mailman-Version: 2.1.12\r
+Precedence: list\r
+List-Id: "Use and development of the notmuch mail system."\r
+       <notmuch.notmuchmail.org>\r
+List-Unsubscribe: <http://notmuchmail.org/mailman/options/notmuch>,\r
+       <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>\r
+List-Archive: <http://notmuchmail.org/pipermail/notmuch>\r
+List-Post: <mailto:notmuch@notmuchmail.org>\r
+List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>\r
+List-Subscribe: <http://notmuchmail.org/mailman/listinfo/notmuch>,\r
+       <mailto:notmuch-request@notmuchmail.org?subject=subscribe>\r
+X-List-Received-Date: Sat, 19 Dec 2009 00:12:03 -0000\r
+\r
+When indexing a message store the filesize along with it so that\r
+when we store all the filenames for a message-id we can know if\r
+any of them have different content cheaply.\r
+\r
+The value stored is defined to be the largest filesize of any\r
+of the files for that message.\r
+\r
+This changes the API for efficiency reasons. The size is often\r
+known to the caller, and so we save a second stat by asking them\r
+to provide it. If they don't know it they can pass -1 and the\r
+stat will be done for them.\r
+\r
+We store the filesize such that we can query a range. Thus it\r
+would be possible to query "filesize:0..100" if you somehow\r
+knew the raw message was less that 100 bytes.\r
+---\r
+\r
+  With new, improved, working, filesize:.. search.\r
+\r
+ lib/database.cc       |    7 +++++++\r
+ lib/message.cc        |   25 +++++++++++++++++++++++++\r
+ lib/notmuch-private.h |    8 +++++++-\r
+ lib/notmuch.h         |    5 +++++\r
+ notmuch-new.c         |    2 +-\r
+ 5 files changed, 45 insertions(+), 2 deletions(-)\r
+\r
+diff --git a/lib/database.cc b/lib/database.cc\r
+index b6c4d07..d834d94 100644\r
+--- a/lib/database.cc\r
++++ b/lib/database.cc\r
+@@ -463,6 +463,8 @@ notmuch_database_open (const char *path,\r
+     struct stat st;\r
+     int err;\r
+     unsigned int i;\r
++    Xapian::NumberValueRangeProcessor *filesize_proc = new Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_FILESIZE,\r
++                       "filesize:", true);\r
+ \r
+     if (asprintf (&notmuch_path, "%s/%s", path, ".notmuch") == -1) {\r
+       notmuch_path = NULL;\r
+@@ -508,6 +510,7 @@ notmuch_database_open (const char *path,\r
+       notmuch->query_parser->set_stemmer (Xapian::Stem ("english"));\r
+       notmuch->query_parser->set_stemming_strategy (Xapian::QueryParser::STEM_SOME);\r
+       notmuch->query_parser->add_valuerangeprocessor (notmuch->value_range_processor);\r
++      notmuch->query_parser->add_valuerangeprocessor (filesize_proc);\r
+ \r
+       for (i = 0; i < ARRAY_SIZE (BOOLEAN_PREFIX_EXTERNAL); i++) {\r
+           prefix_t *prefix = &BOOLEAN_PREFIX_EXTERNAL[i];\r
+@@ -889,6 +892,7 @@ _notmuch_database_link_message (notmuch_database_t *notmuch,\r
+ notmuch_status_t\r
+ notmuch_database_add_message (notmuch_database_t *notmuch,\r
+                             const char *filename,\r
++                            const off_t size,\r
+                             notmuch_message_t **message_ret)\r
+ {\r
+     notmuch_message_file_t *message_file;\r
+@@ -992,6 +996,9 @@ notmuch_database_add_message (notmuch_database_t *notmuch,\r
+       if (private_status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {\r
+           _notmuch_message_set_filename (message, filename);\r
+           _notmuch_message_add_term (message, "type", "mail");\r
++          ret = _notmuch_message_set_filesize (message, filename, size);\r
++          if (ret)\r
++              goto DONE;\r
+       } else {\r
+           ret = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;\r
+           goto DONE;\r
+diff --git a/lib/message.cc b/lib/message.cc\r
+index 49519f1..2bfc5ed 100644\r
+--- a/lib/message.cc\r
++++ b/lib/message.cc\r
+@@ -426,6 +426,31 @@ _notmuch_message_set_filename (notmuch_message_t *message,\r
+     message->doc.set_data (s);\r
+ }\r
+ \r
++notmuch_status_t\r
++_notmuch_message_set_filesize (notmuch_message_t *message,\r
++                             const char *filename,\r
++                             const off_t size)\r
++{\r
++    struct stat st;\r
++    off_t realsize = size;\r
++    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;\r
++\r
++    if (realsize < 0) {\r
++      if (stat (filename, &st)) {\r
++          ret = NOTMUCH_STATUS_FILE_ERROR;\r
++          goto DONE;\r
++      } else {\r
++          realsize = st.st_size;\r
++      }\r
++    }\r
++\r
++    message->doc.add_value (NOTMUCH_VALUE_FILESIZE,\r
++                       Xapian::sortable_serialise (realsize));\r
++\r
++  DONE:\r
++    return ret;\r
++}\r
++\r
+ const char *\r
+ notmuch_message_get_filename (notmuch_message_t *message)\r
+ {\r
+diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h\r
+index 116f63d..1ba3055 100644\r
+--- a/lib/notmuch-private.h\r
++++ b/lib/notmuch-private.h\r
+@@ -100,7 +100,8 @@ _internal_error (const char *format, ...) PRINTF_ATTRIBUTE (1, 2);\r
+ \r
+ typedef enum {\r
+     NOTMUCH_VALUE_TIMESTAMP = 0,\r
+-    NOTMUCH_VALUE_MESSAGE_ID\r
++    NOTMUCH_VALUE_MESSAGE_ID,\r
++    NOTMUCH_VALUE_FILESIZE\r
+ } notmuch_value_t;\r
+ \r
+ /* Xapian (with flint backend) complains if we provide a term longer\r
+@@ -193,6 +194,11 @@ void\r
+ _notmuch_message_set_filename (notmuch_message_t *message,\r
+                              const char *filename);\r
+ \r
++notmuch_status_t\r
++_notmuch_message_set_filesize (notmuch_message_t *message,\r
++                             const char *filename,\r
++                             const off_t size);\r
++\r
+ void\r
+ _notmuch_message_ensure_thread_id (notmuch_message_t *message);\r
+ \r
+diff --git a/lib/notmuch.h b/lib/notmuch.h\r
+index 60834fb..5d0d224 100644\r
+--- a/lib/notmuch.h\r
++++ b/lib/notmuch.h\r
+@@ -32,6 +32,7 @@\r
+ NOTMUCH_BEGIN_DECLS\r
+ \r
+ #include <time.h>\r
++#include <stdlib.h>\r
+ \r
+ #ifndef FALSE\r
+ #define FALSE 0\r
+@@ -241,6 +242,9 @@ notmuch_database_get_timestamp (notmuch_database_t *database,\r
+  * notmuch database will reference the filename, and will not copy the\r
+  * entire contents of the file.\r
+  *\r
++ * 'size' should be the number of bytes in the file, or -1 if you are\r
++ * not sure.\r
++ *\r
+  * If 'message' is not NULL, then, on successful return '*message'\r
+  * will be initialized to a message object that can be used for things\r
+  * such as adding tags to the just-added message. The user should call\r
+@@ -265,6 +269,7 @@ notmuch_database_get_timestamp (notmuch_database_t *database,\r
+ notmuch_status_t\r
+ notmuch_database_add_message (notmuch_database_t *database,\r
+                             const char *filename,\r
++                            const off_t size,\r
+                             notmuch_message_t **message);\r
+ \r
+ /* Find a message with the given message_id.\r
+diff --git a/notmuch-new.c b/notmuch-new.c\r
+index 9d20616..cea66c2 100644\r
+--- a/notmuch-new.c\r
++++ b/notmuch-new.c\r
+@@ -235,7 +235,7 @@ add_files_recursive (notmuch_database_t *notmuch,\r
+                   fflush (stdout);\r
+               }\r
+ \r
+-              status = notmuch_database_add_message (notmuch, next, &message);\r
++              status = notmuch_database_add_message (notmuch, next, st->st_size, &message);\r
+               switch (status) {\r
+                   /* success */\r
+                   case NOTMUCH_STATUS_SUCCESS:\r
+-- \r
+1.6.3.3\r
+\r