--- /dev/null
+Return-Path: <sascha-ml-email-notmuch-notmuch@silbe.org>\r
+X-Original-To: notmuch@notmuchmail.org\r
+Delivered-To: notmuch@notmuchmail.org\r
+Received: from localhost (localhost [127.0.0.1])\r
+ by olra.theworths.org (Postfix) with ESMTP id 02A2B431FBF\r
+ for <notmuch@notmuchmail.org>; Sun, 24 Jun 2012 09:29:45 -0700 (PDT)\r
+X-Virus-Scanned: Debian amavisd-new at olra.theworths.org\r
+X-Spam-Flag: NO\r
+X-Spam-Score: 0\r
+X-Spam-Level: \r
+X-Spam-Status: No, score=0 tagged_above=-999 required=5 tests=[none]\r
+ autolearn=disabled\r
+Received: from olra.theworths.org ([127.0.0.1])\r
+ by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024)\r
+ with ESMTP id YXhmglmXGvxG for <notmuch@notmuchmail.org>;\r
+ Sun, 24 Jun 2012 09:29:44 -0700 (PDT)\r
+Received: from smtp.chost.de (setoy.chost.de [217.160.209.225])\r
+ (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits))\r
+ (No client certificate requested)\r
+ by olra.theworths.org (Postfix) with ESMTPS id 9F1D2431FBC\r
+ for <notmuch@notmuchmail.org>; Sun, 24 Jun 2012 09:29:43 -0700 (PDT)\r
+Received: (qmail 14473 invoked by uid 5015); 24 Jun 2012 16:29:41 -0000\r
+Received: (nullmailer pid 3411 invoked by uid 123);\r
+ Sun, 24 Jun 2012 16:29:39 -0000\r
+Received: from twin.sascha.silbe.org (twin.sascha.silbe.org [192.168.1.2])\r
+ by flatty.sascha.silbe.org ([192.168.1.252])\r
+ with SMTP via TCP; 24 Jun 2012 16:29:39 -0000\r
+Received: (nullmailer pid 25951 invoked by uid 8193);\r
+ Sun, 24 Jun 2012 16:29:39 -0000\r
+From: Sascha Silbe <sascha-pgp@silbe.org>\r
+To: notmuch <notmuch@notmuchmail.org>\r
+Subject: [PATCH 3/3] new: don't read unchanged directories from disk\r
+Date: Sun, 24 Jun 2012 18:29:26 +0200\r
+Message-Id: <1340555366-25891-4-git-send-email-sascha-pgp@silbe.org>\r
+X-Mailer: git-send-email 1.7.10\r
+In-Reply-To: <1340555366-25891-1-git-send-email-sascha-pgp@silbe.org>\r
+References: <1340555366-25891-1-git-send-email-sascha-pgp@silbe.org>\r
+Mail-Followup-To: notmuch <notmuch@notmuchmail.org>\r
+X-BeenThere: notmuch@notmuchmail.org\r
+X-Mailman-Version: 2.1.13\r
+Precedence: list\r
+Reply-To: Sascha Silbe <sascha-ml-reply-to-2012-3@silbe.org>\r
+List-Id: "Use and development of the notmuch mail system."\r
+ <notmuch.notmuchmail.org>\r
+List-Unsubscribe: <http://notmuchmail.org/mailman/options/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>\r
+List-Archive: <http://notmuchmail.org/pipermail/notmuch>\r
+List-Post: <mailto:notmuch@notmuchmail.org>\r
+List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>\r
+List-Subscribe: <http://notmuchmail.org/mailman/listinfo/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=subscribe>\r
+X-List-Received-Date: Sun, 24 Jun 2012 16:29:45 -0000\r
+\r
+Previously, notmuch new listed all directories on disk, even if they\r
+were unchanged from the state recorded in the database. This could take\r
+a huge amount of time for large numbers of mails as it would list each\r
+individual mail.\r
+\r
+By iterating over the subdirectories recorded in the database we can\r
+avoid accessing the file system for each unchanged directory. If the\r
+modification time does not match we fall back to a full file system scan\r
+so new subdirectories will get picked up and scanned recursively.\r
+\r
+Timings for an Athlon BE-2300 with 4GiB RAM and a Samsung HD204UI hard\r
+disk containing a mail store of around 900k mails, for the "no new mail"\r
+case, three samples each:\r
+\r
+Hot cache (first run discarded):\r
+ Before After Speedup\r
+real mean 5.0s stdev 0.1s mean 1.8s stdev 0.1s 2.8\r
+user mean 2.4s stdev 0.1s mean 1.0s stdev 0.1s 2.4\r
+sys mean 2.6s stdev 0.0s mean 0.9s stdev 0.0s 2.9\r
+\r
+Cold cache on each run:\r
+ Before After Speedup\r
+real mean 433s stdev 1.2s mean 130s stdev 0.1s 3.3\r
+user mean 6.0s stdev 0.2s mean 2.5s stdev 0.0s 2.4\r
+sys mean 6.7s stdev 0.1s mean 2.8s stdev 0.1s 2.4\r
+\r
+Signed-off-by: Sascha Silbe <sascha-pgp@silbe.org>\r
+---\r
+ notmuch-new.c | 130 +++++++++++++++++++++++++++++++++++++++++++--------------\r
+ 1 file changed, 98 insertions(+), 32 deletions(-)\r
+\r
+diff --git a/notmuch-new.c b/notmuch-new.c\r
+index 938ae29..93feb5c 100644\r
+--- a/notmuch-new.c\r
++++ b/notmuch-new.c\r
+@@ -225,6 +225,33 @@ _entries_resemble_maildir (const char *path, struct dirent **entries, int count)\r
+ return 0;\r
+ }\r
+ \r
++/* Test if a directory recorded in the database looks like a Maildir directory.\r
++ *\r
++ * Search through the iterator of directory entries to see if we can find all\r
++ * three subdirectories typical for Maildir, that is "new", "cur", and "tmp".\r
++ *\r
++ * Return 1 if the directory looks like a Maildir and 0 otherwise.\r
++ */\r
++static int\r
++_subdirs_resemble_maildir (notmuch_filenames_t *db_subdirs)\r
++{\r
++ int found = 0;\r
++\r
++ while (notmuch_filenames_valid(db_subdirs) && found != 3)\r
++ {\r
++ const char *filename = notmuch_filenames_get(db_subdirs);\r
++\r
++ if (strcmp(filename, "new") == 0 || strcmp(filename, "cur") == 0 ||\r
++ strcmp(filename, "tmp") == 0)\r
++ {\r
++ found++;\r
++ }\r
++ notmuch_filenames_move_to_next (db_subdirs);\r
++ }\r
++\r
++ return (found == 3) ? 1 : 0 ;\r
++}\r
++\r
+ /* Test if the file/directory is to be ignored.\r
+ */\r
+ static notmuch_bool_t\r
+@@ -243,20 +270,22 @@ _entry_in_ignore_list (const char *entry, add_files_state_t *state)\r
+ *\r
+ * o Ask the filesystem for the mtime of 'path' (fs_mtime)\r
+ * o Ask the database for its timestamp of 'path' (db_mtime)\r
++ * o Ask the database for directories within 'path' (db_subdirs)\r
+ *\r
+- * o Ask the filesystem for files and directories within 'path'\r
+- * (via scandir and stored in fs_entries)\r
++ * o If fs_mtime is newer than db_mtime, ask the filesystem for\r
++ * files and directories within 'path' (via scandir and stored in\r
++ * fs_entries)\r
+ *\r
+- * o Pass 1: For each directory in fs_entries, recursively call into\r
+- * this same function.\r
++ * o Pass 1: For each subdirectory, recursively call into this same\r
++ * function. If fs_mtime is newer than db_mtime, scan fs_entries\r
++ * for subdirectories. Otherwise use the database (db_subdirs).\r
+ *\r
+ * o Compare fs_mtime to db_mtime. If they are equivalent, terminate\r
+ * the algorithm at this point, (this directory has not been\r
+ * updated in the filesystem since the last database scan of PASS\r
+ * 2).\r
+ *\r
+- * o Ask the database for files and directories within 'path'\r
+- * (db_files and db_subdirs)\r
++ * o Ask the database for files within 'path' (db_files)\r
+ *\r
+ * o Pass 2: Walk fs_entries simultaneously with db_files and\r
+ * db_subdirs. Look for one of three interesting cases:\r
+@@ -321,28 +350,48 @@ add_files (notmuch_database_t *notmuch,\r
+ goto DONE;\r
+ }\r
+ db_mtime = directory ? notmuch_directory_get_mtime (directory) : 0;\r
++ if (directory)\r
++ db_subdirs = notmuch_directory_get_child_directories (directory);\r
+ \r
+- /* If the database knows about this directory, then we sort based\r
+- * on strcmp to match the database sorting. Otherwise, we can do\r
+- * inode-based sorting for faster filesystem operation. */\r
+- num_fs_entries = scandir (path, &fs_entries, 0,\r
+- directory ?\r
+- dirent_sort_strcmp_name : dirent_sort_inode);\r
++ /* If the directory's modification time in the filesystem is the\r
++ * same as what we recorded in the database the last time we\r
++ * scanned it, then we can skip reading the entries from disk.\r
++ *\r
++ * We test for strict equality here to avoid a bug that can happen\r
++ * if the system clock jumps backward, (preventing new mail from\r
++ * being discovered until the clock catches up and the directory\r
++ * is modified again).\r
++ */\r
++ if (fs_mtime != db_mtime)\r
++ {\r
++ /* If the database knows about this directory, then we sort based\r
++ * on strcmp to match the database sorting. Otherwise, we can do\r
++ * inode-based sorting for faster filesystem operation. */\r
++ num_fs_entries = scandir (path, &fs_entries, 0,\r
++ directory ?\r
++ dirent_sort_strcmp_name : dirent_sort_inode);\r
++\r
++ if (num_fs_entries == -1) {\r
++ fprintf (stderr, "Error opening directory %s: %s\n",\r
++ path, strerror (errno));\r
++ /* We consider this a fatal error because, if a user moved a\r
++ * message from another directory that we were able to scan\r
++ * into this directory, skipping this directory will cause\r
++ * that message to be lost. */\r
++ ret = NOTMUCH_STATUS_FILE_ERROR;\r
++ goto DONE;\r
++ }\r
++ }\r
+ \r
+- if (num_fs_entries == -1) {\r
+- fprintf (stderr, "Error opening directory %s: %s\n",\r
+- path, strerror (errno));\r
+- /* We consider this a fatal error because, if a user moved a\r
+- * message from another directory that we were able to scan\r
+- * into this directory, skipping this directory will cause\r
+- * that message to be lost. */\r
+- ret = NOTMUCH_STATUS_FILE_ERROR;\r
+- goto DONE;\r
++ if (fs_entries)\r
++ is_maildir = _entries_resemble_maildir (path, fs_entries, num_fs_entries);\r
++ else\r
++ {\r
++ is_maildir = _subdirs_resemble_maildir (db_subdirs);\r
++ notmuch_filenames_rewind(db_subdirs);\r
+ }\r
+ \r
+ /* Pass 1: Recurse into all sub-directories. */\r
+- is_maildir = _entries_resemble_maildir (path, fs_entries, num_fs_entries);\r
+-\r
+ for (i = 0; i < num_fs_entries; i++) {\r
+ if (interrupted)\r
+ break;\r
+@@ -386,24 +435,41 @@ add_files (notmuch_database_t *notmuch,\r
+ next = NULL;\r
+ }\r
+ \r
++ /* Reading the directory from disk was skipped because it hasn't\r
++ * changed since the last time we scanned it. Recurse over the\r
++ * subdirectories using the database instead.\r
++ */\r
++ while (!fs_entries && notmuch_filenames_valid (db_subdirs))\r
++ {\r
++ const char *filename = notmuch_filenames_get (db_subdirs);\r
++ if (interrupted)\r
++ break;\r
++\r
++ next = talloc_asprintf (notmuch, "%s/%s", path, filename);\r
++ if (!_entry_in_ignore_list (filename, state) && stat (next, &st) == 0)\r
++ {\r
++ status = add_files (notmuch, next, state);\r
++ if (status) {\r
++ ret = status;\r
++ goto DONE;\r
++ }\r
++ }\r
++ talloc_free (next);\r
++ next = NULL;\r
++ notmuch_filenames_move_to_next (db_subdirs);\r
++ }\r
++\r
+ /* If the directory's modification time in the filesystem is the\r
+ * same as what we recorded in the database the last time we\r
+ * scanned it, then we can skip the second pass entirely.\r
+- *\r
+- * We test for strict equality here to avoid a bug that can happen\r
+- * if the system clock jumps backward, (preventing new mail from\r
+- * being discovered until the clock catches up and the directory\r
+- * is modified again).\r
+ */\r
+ if (directory && fs_mtime == db_mtime)\r
+ goto DONE;\r
+ \r
+ /* If the database has never seen this directory before, we can\r
+- * simply leave db_files and db_subdirs NULL. */\r
+- if (directory) {\r
++ * simply leave db_files NULL. */\r
++ if (directory)\r
+ db_files = notmuch_directory_get_child_files (directory);\r
+- db_subdirs = notmuch_directory_get_child_directories (directory);\r
+- }\r
+ \r
+ /* Pass 2: Scan for new files, removed files, and removed directories. */\r
+ for (i = 0; i < num_fs_entries; i++)\r
+-- \r
+1.7.10\r
+\r
projects / notmuch-archives.git / commitdiff