--- /dev/null
+Return-Path: <ethan.glasser.camp@gmail.com>\r
+X-Original-To: notmuch@notmuchmail.org\r
+Delivered-To: notmuch@notmuchmail.org\r
+Received: from localhost (localhost [127.0.0.1])\r
+ by olra.theworths.org (Postfix) with ESMTP id 159B5431FD4\r
+ for <notmuch@notmuchmail.org>; Sun, 1 Jul 2012 09:47:48 -0700 (PDT)\r
+X-Virus-Scanned: Debian amavisd-new at olra.theworths.org\r
+X-Spam-Flag: NO\r
+X-Spam-Score: 1.061\r
+X-Spam-Level: *\r
+X-Spam-Status: No, score=1.061 tagged_above=-999 required=5\r
+ tests=[DKIM_SIGNED=0.1, DKIM_VALID=-0.1, DKIM_VALID_AU=-0.1,\r
+ FREEMAIL_FROM=0.001, RCVD_IN_BL_SPAMCOP_NET=1.246,\r
+ RCVD_IN_DNSWL_LOW=-0.7, RCVD_IN_SORBS_WEB=0.614] autolearn=disabled\r
+Received: from olra.theworths.org ([127.0.0.1])\r
+ by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024)\r
+ with ESMTP id VkfYK+ryi0dV for <notmuch@notmuchmail.org>;\r
+ Sun, 1 Jul 2012 09:47:47 -0700 (PDT)\r
+Received: from mail-wi0-f169.google.com (mail-wi0-f169.google.com\r
+ [209.85.212.169]) (using TLSv1 with cipher RC4-SHA (128/128 bits))\r
+ (No client certificate requested)\r
+ by olra.theworths.org (Postfix) with ESMTPS id BB53A431FAF\r
+ for <notmuch@notmuchmail.org>; Sun, 1 Jul 2012 09:47:46 -0700 (PDT)\r
+Received: by wibhm2 with SMTP id hm2so2252707wib.2\r
+ for <notmuch@notmuchmail.org>; Sun, 01 Jul 2012 09:47:45 -0700 (PDT)\r
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20120113;\r
+ h=from:to:cc:subject:date:message-id:x-mailer:in-reply-to:references;\r
+ bh=L5VXoVxKCC9xdk9sRXrYZXjP8v7jJYaDqCn9aGHBbrg=;\r
+ b=RxYaXLeTxFLzHZvBj+0cnX8nNZc83uwoPGIj4mUllJc/gXvjZDUZYP4coyoPVV7XqG\r
+ tw7Hj/P+GbvVAppQBziQ+z61AQSOypyozltok1TJYHP58aT4hCwsymnkvhHbpU9nzVPc\r
+ h2Y7wsPqKQSe4jK0uNRdXsyv5VTmGvPpXu9JyBJruNJhfaVdAiuXiwZ9qvaC64MEOJ3t\r
+ xxxI8JmfngXokPgPdNjPlc+FNq+hiRH/iVDphCHLzI70fxH3oThgLtV5WCJDN3P6ZZiZ\r
+ ceyFlIeLIZ1Pk0ePSd7hFsxrPWpu8phBJFr5pLrPGCEyH+6jPv37s7UlMRPMF5KBXxzN\r
+ tgrA==\r
+Received: by 10.180.105.6 with SMTP id gi6mr17978731wib.4.1341161265405;\r
+ Sun, 01 Jul 2012 09:47:45 -0700 (PDT)\r
+Received: from localhost ([195.24.209.21])\r
+ by mx.google.com with ESMTPS id m4sm19729724wie.1.2012.07.01.09.47.30\r
+ (version=TLSv1/SSLv3 cipher=OTHER);\r
+ Sun, 01 Jul 2012 09:47:44 -0700 (PDT)\r
+From: Ethan Glasser-Camp <ethan.glasser.camp@gmail.com>\r
+To: notmuch@notmuchmail.org\r
+Subject: [PATCH v2 8/8] new: Add scan support for mbox:// URIs\r
+Date: Sun, 1 Jul 2012 12:39:50 -0400\r
+Message-Id: <1341160790-14525-9-git-send-email-ethan@betacantrips.com>\r
+X-Mailer: git-send-email 1.7.9.5\r
+In-Reply-To: <1341160790-14525-1-git-send-email-ethan@betacantrips.com>\r
+References:\r
+ <CAOJ+Ob0MSOez2MvD2fCgF7t32kFPk4g2+xCud88QmBLt_b5pOA@mail.gmail.com>\r
+ <1341160790-14525-1-git-send-email-ethan@betacantrips.com>\r
+Cc: Ethan Glasser-Camp <ethan@betacantrips.com>\r
+X-BeenThere: notmuch@notmuchmail.org\r
+X-Mailman-Version: 2.1.13\r
+Precedence: list\r
+List-Id: "Use and development of the notmuch mail system."\r
+ <notmuch.notmuchmail.org>\r
+List-Unsubscribe: <http://notmuchmail.org/mailman/options/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>\r
+List-Archive: <http://notmuchmail.org/pipermail/notmuch>\r
+List-Post: <mailto:notmuch@notmuchmail.org>\r
+List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>\r
+List-Subscribe: <http://notmuchmail.org/mailman/listinfo/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=subscribe>\r
+X-List-Received-Date: Sun, 01 Jul 2012 16:47:48 -0000\r
+\r
+This fixes the broken tests introduced by the last commit.\r
+\r
+Signed-off-by: Ethan Glasser-Camp <ethan@betacantrips.com>\r
+---\r
+More text was added to clarify how mbox scanning works.\r
+\r
+ notmuch-config.c | 4 +\r
+ notmuch-new.c | 304 +++++++++++++++++++++++++++++++++++++++++++++++++++++-\r
+ 2 files changed, 303 insertions(+), 5 deletions(-)\r
+\r
+diff --git a/notmuch-config.c b/notmuch-config.c\r
+index 387f855..e02b6a9 100644\r
+--- a/notmuch-config.c\r
++++ b/notmuch-config.c\r
+@@ -56,6 +56,10 @@ static const char new_config_comment[] =\r
+ "\t Each URL denotes a \"root\" which will be searched for mail files.\n"\r
+ "\t How this search is performed depends on the scheme of the URL (the\n"\r
+ "\t part before the first colon).\n"\r
++ "\n"\r
++ "\t\tmbox:///path scans all subdirectories starting at path for mbox\n"\r
++ "\t\t files, and scans all mbox files for all messages.\n"\r
++ "\n"\r
+ "\t The maildir located at database.path, above, will automatically be added.\n";\r
+ \r
+ static const char user_config_comment[] =\r
+diff --git a/notmuch-new.c b/notmuch-new.c\r
+index 5250562..061a1a8 100644\r
+--- a/notmuch-new.c\r
++++ b/notmuch-new.c\r
+@@ -19,6 +19,7 @@\r
+ */\r
+ \r
+ #include "notmuch-client.h"\r
++#include <uriparser/Uri.h>\r
+ \r
+ #include <unistd.h>\r
+ \r
+@@ -653,14 +654,307 @@ add_files (notmuch_database_t *notmuch,\r
+ return ret;\r
+ }\r
+ \r
++/* Scan an mbox file for messages.\r
++ *\r
++ * We assume that mboxes are append only -- this function does not\r
++ * check to see if messages have gone missing.\r
++ *\r
++ * The mtime of the mbox file is stored in a "directory" document in\r
++ * Xapian.\r
++ */\r
++/* FIXME: a certain amount of this code appears in add_files_recursive,\r
++ * and could be refactored\r
++ */\r
++static notmuch_status_t\r
++add_messages_mbox_file (notmuch_database_t *notmuch,\r
++ const char *path,\r
++ add_files_state_t *state)\r
++{\r
++ notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, status;\r
++ struct stat st;\r
++ time_t fs_mtime, db_mtime, stat_time;\r
++ FILE *mbox;\r
++ char *line, *path_uri = NULL, *message_uri = NULL;\r
++ int line_len;\r
++ size_t offset, end_offset, line_size = 0;\r
++ notmuch_directory_t *directory;\r
++ int content_length = -1, is_headers;\r
++\r
++ if (stat (path, &st)) {\r
++ fprintf (stderr, "Error reading mbox file %s: %s\n",\r
++ path, strerror (errno));\r
++ return NOTMUCH_STATUS_FILE_ERROR;\r
++ }\r
++\r
++ stat_time = time (NULL);\r
++ if (! S_ISREG (st.st_mode)) {\r
++ fprintf (stderr, "Error: %s is not a file.\n", path);\r
++ return NOTMUCH_STATUS_FILE_ERROR;\r
++ }\r
++\r
++ fs_mtime = st.st_mtime;\r
++\r
++ path_uri = talloc_asprintf (notmuch, "mbox://%s", path);\r
++ status = notmuch_database_get_directory (notmuch, path_uri, &directory);\r
++ if (status) {\r
++ ret = status;\r
++ goto DONE;\r
++ }\r
++ db_mtime = directory ? notmuch_directory_get_mtime (directory) : 0;\r
++\r
++ if (directory && db_mtime == fs_mtime) {\r
++ goto DONE;\r
++ }\r
++\r
++ mbox = fopen (path, "r");\r
++ if (mbox == NULL) {\r
++ fprintf (stderr, "Error: couldn't open %s for reading.\n",\r
++ path);\r
++ ret = NOTMUCH_STATUS_FILE_ERROR;\r
++ goto DONE;\r
++ }\r
++\r
++ line_len = getline (&line, &line_size, mbox);\r
++\r
++ if (line_len == -1) {\r
++ fprintf (stderr, "Error: reading from %s failed: %s\n",\r
++ path, strerror (errno));\r
++ ret = NOTMUCH_STATUS_FILE_ERROR;\r
++ goto DONE;\r
++ }\r
++\r
++ if (strncmp (line, "From ", 5) != 0) {\r
++ fprintf (stderr, "Note: Ignoring non-mbox file: %s\n",\r
++ path);\r
++ ret = NOTMUCH_STATUS_FILE_ERROR;\r
++ goto DONE;\r
++ }\r
++\r
++ /* Loop invariant: At the beginning of the loop, we have just read\r
++ * a From_ line, but haven't yet read any of the headers.\r
++ */\r
++ while (! feof (mbox)) {\r
++ is_headers = 1;\r
++ offset = ftell (mbox);\r
++ content_length = -1;\r
++\r
++ /* Read lines until we either get to the next From_ header, or\r
++ * we find a Content-Length header (mboxcl) and we run out of headers.\r
++ */\r
++ do {\r
++ /* Get the offset before we read, in case we got another From_ header. */\r
++ end_offset = ftell (mbox);\r
++\r
++ line_len = getline (&line, &line_size, mbox);\r
++\r
++ /* Check to see if this line is a content-length header,\r
++ * or the end of the headers. */\r
++ if (is_headers && strncasecmp (line, "Content-Length: ",\r
++ strlen ("Content-Length: ")) == 0)\r
++ content_length = strtol (line + strlen ("Content-Length: "),\r
++ NULL, 10);\r
++\r
++ if (is_headers && strlen (line) == 1 && *line == '\n') {\r
++ is_headers = 0;\r
++ /* If we got a content_length, skip the message body. */\r
++ if (content_length != -1) {\r
++ fseek (mbox, content_length, SEEK_CUR);\r
++ line_len = getline (&line, &line_size, mbox);\r
++\r
++ /* We should be at the end of the message. Sanity\r
++ * check: there should be a blank line, and then\r
++ * another From_ header. */\r
++ if (strlen (line) != 1 || *line != '\n') {\r
++ fprintf (stderr, "Warning: message with Content-Length not "\r
++ "immediately followed by blank line (%d)\n", offset);\r
++ }\r
++\r
++ end_offset = ftell (mbox);\r
++ line_len = getline (&line, &line_size, mbox);\r
++\r
++ if (line_len != -1 && strncmp (line, "From ", 5) != 0) {\r
++ fprintf (stderr, "Warning: message with Content-Length not "\r
++ "immediately followed by another message (%d)\n", offset);\r
++ fprintf (stderr, "Line was: %s", line);\r
++ }\r
++ }\r
++ }\r
++\r
++ } while (! feof (mbox) && strncmp (line, "From ", 5) != 0);\r
++\r
++ /* end_offset is now after the \n but before the From_. */\r
++ message_uri = talloc_asprintf (notmuch, "mbox://%s#%d+%d",\r
++ path, offset, (end_offset - 1) - offset);\r
++\r
++ _report_before_adding_file (state, message_uri);\r
++\r
++ status = _add_message (state, notmuch, message_uri);\r
++ if (status) {\r
++ ret = status;\r
++ goto DONE;\r
++ }\r
++\r
++ _report_added_file (state);\r
++\r
++ talloc_free (message_uri);\r
++ message_uri = NULL;\r
++ }\r
++\r
++ /* This is the same precaution we take in maildir. */\r
++ if (fs_mtime != stat_time)\r
++ _filename_list_add (state->directory_mtimes, path_uri)->mtime = fs_mtime;\r
++\r
++DONE:\r
++ if (line)\r
++ free (line);\r
++ if (path_uri)\r
++ talloc_free (path_uri);\r
++ if (message_uri)\r
++ talloc_free (message_uri);\r
++ if (directory)\r
++ notmuch_directory_destroy (directory);\r
++\r
++ return ret;\r
++}\r
++\r
++/*\r
++ * Examine path recursively as follows:\r
++ *\r
++ * - Recurse on each subdirectory, as in add_files.\r
++ *\r
++ * - Call add_messages_mbox_file on every non-directory.\r
++ */\r
++/* FIXME: this is almost entirely bits-and-pieces from\r
++ * add_files_recursive and could do with a refactor */\r
++static notmuch_status_t\r
++add_files_mbox (notmuch_database_t *notmuch,\r
++ const char *path,\r
++ add_files_state_t *state)\r
++{\r
++ struct dirent **fs_entries = NULL;\r
++ struct dirent *entry = NULL;\r
++ char *next = NULL;\r
++ int num_fs_entries = 0, i, entry_type;\r
++ struct stat st;\r
++ notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, status;\r
++\r
++ if (stat (path, &st)) {\r
++ fprintf (stderr, "Error reading directory %s: %s\n",\r
++ path, strerror (errno));\r
++ return NOTMUCH_STATUS_FILE_ERROR;\r
++ }\r
++\r
++ num_fs_entries = scandir (path, &fs_entries, 0,\r
++ dirent_sort_inode);\r
++\r
++ if (num_fs_entries == -1) {\r
++ fprintf (stderr, "Error opening directory %s: %s\n",\r
++ path, strerror (errno));\r
++ ret = NOTMUCH_STATUS_FILE_ERROR;\r
++ goto DONE;\r
++ }\r
++\r
++ for (i = 0; i < num_fs_entries; i++) {\r
++ if (interrupted)\r
++ break;\r
++\r
++ entry = fs_entries[i];\r
++\r
++ entry_type = dirent_type (path, entry);\r
++ if (entry_type == -1) {\r
++ /* Be pessimistic, e.g. so we don't lose lots of mail just\r
++ * because a user broke a symlink. */\r
++ fprintf (stderr, "Error reading file %s/%s: %s\n",\r
++ path, entry->d_name, strerror (errno));\r
++ return NOTMUCH_STATUS_FILE_ERROR;\r
++ } else if (entry_type != S_IFDIR) {\r
++ continue;\r
++ }\r
++\r
++ /* Ignore special directories to avoid infinite recursion.\r
++ * Also ignore the .notmuch directory, any "tmp" directory\r
++ * that appears within a maildir and files/directories\r
++ * the user has configured to be ignored.\r
++ */\r
++ if (strcmp (entry->d_name, ".") == 0 ||\r
++ strcmp (entry->d_name, "..") == 0 ||\r
++ strcmp (entry->d_name, ".notmuch") == 0 ||\r
++ _entry_in_ignore_list (entry->d_name, state))\r
++ {\r
++ continue;\r
++ }\r
++\r
++ next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name);\r
++ status = add_files_mbox (notmuch, next, state);\r
++ if (status) {\r
++ ret = status;\r
++ goto DONE;\r
++ }\r
++ talloc_free (next);\r
++ next = NULL;\r
++ }\r
++\r
++ /* Pass 2: Scan for new files, removed files, and removed directories. */\r
++ for (i = 0; i < num_fs_entries; i++)\r
++ {\r
++ if (interrupted)\r
++ break;\r
++\r
++ entry = fs_entries[i];\r
++\r
++ /* Ignore files & directories user has configured to be ignored */\r
++ if (_entry_in_ignore_list (entry->d_name, state))\r
++ continue;\r
++\r
++ /* Only add regular files (and symlinks to regular files). */\r
++ entry_type = dirent_type (path, entry);\r
++ if (entry_type == -1) {\r
++ fprintf (stderr, "Error reading file %s/%s: %s\n",\r
++ path, entry->d_name, strerror (errno));\r
++ return NOTMUCH_STATUS_FILE_ERROR;\r
++ } else if (entry_type != S_IFREG) {\r
++ continue;\r
++ }\r
++\r
++ next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name);\r
++ status = add_messages_mbox_file (notmuch, next, state);\r
++ talloc_free (next);\r
++ next = NULL;\r
++\r
++ if (status) {\r
++ ret = status;\r
++ goto DONE;\r
++ }\r
++ }\r
++\r
++DONE:\r
++ if (next)\r
++ talloc_free (next);\r
++ return ret;\r
++}\r
++\r
+ /* Call out to the appropriate add_files function, based on the URI. */\r
+ static notmuch_status_t\r
+-add_files_uri (unused(notmuch_database_t *notmuch),\r
+- unused(const char *uri),\r
+- unused(add_files_state_t *state))\r
++add_files_uri (notmuch_database_t *notmuch,\r
++ const char *uri,\r
++ add_files_state_t *state)\r
+ {\r
+- /* Stub for now */\r
+- return NOTMUCH_STATUS_SUCCESS;\r
++ UriUriA parsed;\r
++ UriParserStateA parser;\r
++ notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;\r
++ parser.uri = &parsed;\r
++ if (uriParseUriA (&parser, uri) != URI_SUCCESS)\r
++ goto DONE;\r
++\r
++ if (strncmp (parsed.scheme.first, "mbox",\r
++ parsed.scheme.afterLast - parsed.scheme.first) == 0) {\r
++ ret = add_files_mbox (notmuch, parsed.pathHead->text.first - 1, state);\r
++ goto DONE;\r
++ }\r
++\r
++DONE:\r
++ uriFreeUriMembersA (&parsed);\r
++ return ret;\r
+ }\r
+ \r
+ static void\r
+-- \r
+1.7.9.5\r
+\r
projects / notmuch-archives.git / commitdiff