1 Return-Path: <sojkam1@fel.cvut.cz>
\r
2 X-Original-To: notmuch@notmuchmail.org
\r
3 Delivered-To: notmuch@notmuchmail.org
\r
4 Received: from localhost (localhost [127.0.0.1])
\r
5 by olra.theworths.org (Postfix) with ESMTP id 46C1E418C25
\r
6 for <notmuch@notmuchmail.org>; Thu, 8 Apr 2010 08:13:40 -0700 (PDT)
\r
7 X-Virus-Scanned: Debian amavisd-new at olra.theworths.org
\r
11 X-Spam-Status: No, score=-1.9 tagged_above=-999 required=5
\r
12 tests=[BAYES_00=-1.9] autolearn=ham
\r
13 Received: from olra.theworths.org ([127.0.0.1])
\r
14 by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024)
\r
15 with ESMTP id HKu+AFdAdunZ for <notmuch@notmuchmail.org>;
\r
16 Thu, 8 Apr 2010 08:13:37 -0700 (PDT)
\r
17 Received: from max.feld.cvut.cz (max.feld.cvut.cz [147.32.192.36])
\r
18 by olra.theworths.org (Postfix) with ESMTP id 49CAC4196F0
\r
19 for <notmuch@notmuchmail.org>; Thu, 8 Apr 2010 08:13:37 -0700 (PDT)
\r
20 Received: from localhost (unknown [192.168.200.4])
\r
21 by max.feld.cvut.cz (Postfix) with ESMTP id 9D29D19F3391;
\r
22 Thu, 8 Apr 2010 17:13:36 +0200 (CEST)
\r
23 X-Virus-Scanned: IMAP AMAVIS
\r
24 Received: from max.feld.cvut.cz ([192.168.200.1])
\r
25 by localhost (styx.feld.cvut.cz [192.168.200.4]) (amavisd-new,
\r
27 with ESMTP id Je1mAHmgkph1; Thu, 8 Apr 2010 17:13:34 +0200 (CEST)
\r
28 Received: from imap.feld.cvut.cz (imap.feld.cvut.cz [147.32.192.34])
\r
29 by max.feld.cvut.cz (Postfix) with ESMTP id BC0D319F3340;
\r
30 Thu, 8 Apr 2010 17:13:34 +0200 (CEST)
\r
31 Received: from steelpick.2x.cz (k335-30.felk.cvut.cz [147.32.86.30])
\r
32 (Authenticated sender: sojkam1)
\r
33 by imap.feld.cvut.cz (Postfix) with ESMTPSA id B7AF515C062;
\r
34 Thu, 8 Apr 2010 17:13:34 +0200 (CEST)
\r
35 Received: from wsh by steelpick.2x.cz with local (Exim 4.71)
\r
36 (envelope-from <sojkam1@fel.cvut.cz>)
\r
37 id 1NztQE-0007sz-Gs; Thu, 08 Apr 2010 17:13:34 +0200
\r
38 From: Michal Sojka <sojkam1@fel.cvut.cz>
\r
39 To: notmuch@notmuchmail.org
\r
40 Subject: [PATCH] Add maildir-based mailstore
\r
41 Date: Thu, 8 Apr 2010 17:13:12 +0200
\r
42 Message-Id: <1270739592-30280-2-git-send-email-sojkam1@fel.cvut.cz>
\r
43 X-Mailer: git-send-email 1.7.0.2
\r
44 In-Reply-To: <1270739592-30280-1-git-send-email-sojkam1@fel.cvut.cz>
\r
45 References: <1270739592-30280-1-git-send-email-sojkam1@fel.cvut.cz>
\r
46 X-BeenThere: notmuch@notmuchmail.org
\r
47 X-Mailman-Version: 2.1.13
\r
49 List-Id: "Use and development of the notmuch mail system."
\r
50 <notmuch.notmuchmail.org>
\r
51 List-Unsubscribe: <http://notmuchmail.org/mailman/options/notmuch>,
\r
52 <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>
\r
53 List-Archive: <http://notmuchmail.org/pipermail/notmuch>
\r
54 List-Post: <mailto:notmuch@notmuchmail.org>
\r
55 List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>
\r
56 List-Subscribe: <http://notmuchmail.org/mailman/listinfo/notmuch>,
\r
57 <mailto:notmuch-request@notmuchmail.org?subject=subscribe>
\r
58 X-List-Received-Date: Thu, 08 Apr 2010 15:13:40 -0000
\r
60 This mailstore allows bi-directional synchronization between maildir
\r
61 flags and certain tags. The flag-to-tag mapping is defined by flag2tag
\r
64 The synchronization works this way:
\r
66 1) Whenever notmuch new is executed, the following happens:
\r
67 o New messages are tagged with inbox.
\r
68 o New messages without maildir info in the file name (typically
\r
69 files in new/) are tagged with unread.
\r
70 o For new or renamed messages with maildir info present in the
\r
71 file name, the tags defined in flag2tag are either added or
\r
72 removed depending on the flags from the file name.
\r
74 2) Whenever notmuch tag (or notmuch restore) is executed, a new set of
\r
75 flags based on the tags is constructed for every message and a new
\r
76 file name is prepared based on the old file name but with the new
\r
77 flags. If the old message file was in 'new' directory then this is
\r
78 replaced with 'cur' in the new file name. If the new and old file
\r
79 names differ, the file is renamed and notmuch database is updated
\r
82 The rename happens before the database is updated. In case of crash
\r
83 between rename and database update, the next run of notmuch new
\r
84 brings the database in sync with the mail store again.
\r
86 This mailstore is enabled by putting the following to your
\r
92 Signed-off-by: Michal Sojka <sojkam1@fel.cvut.cz>
\r
94 lib/database.cc | 7 ++
\r
95 lib/mailstore-files.c | 210 ++++++++++++++++++++++++++++++++++++++++++++++++-
\r
96 lib/mailstore.c | 1 +
\r
97 lib/message.cc | 41 +++++++++-
\r
98 lib/notmuch-private.h | 4 +
\r
100 6 files changed, 260 insertions(+), 4 deletions(-)
\r
102 diff --git a/lib/database.cc b/lib/database.cc
\r
103 index bd64ed3..4f3ce88 100644
\r
104 --- a/lib/database.cc
\r
105 +++ b/lib/database.cc
\r
106 @@ -1479,6 +1479,13 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
\r
108 _notmuch_message_add_filename (message, filename);
\r
110 + /* This is a new message or it has a new filename and as such,
\r
111 + * its tags in database either do not exists or might be out
\r
112 + * of date. Mailstore assigns the tags later in index_new(),
\r
113 + * but until then we should not synchronize the tags back to
\r
114 + * the mailstore. */
\r
115 + notmuch_message_set_flag(message, NOTMUCH_MESSAGE_FLAG_TAGS_INVALID, TRUE);
\r
117 /* Is this a newly created message object? */
\r
118 if (private_status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
\r
119 _notmuch_message_add_term (message, "type", "mail");
\r
120 diff --git a/lib/mailstore-files.c b/lib/mailstore-files.c
\r
121 index f2cb8d7..e3e346f 100644
\r
122 --- a/lib/mailstore-files.c
\r
123 +++ b/lib/mailstore-files.c
\r
125 #include "notmuch.h"
\r
126 #include "mailstore-private.h"
\r
127 #include <dirent.h>
\r
128 +#include <stdbool.h>
\r
130 +#define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
\r
132 +struct mailstore_priv {
\r
133 + void (*tag_new)(notmuch_message_t *message, const char *filename);
\r
134 + int (*tag_renamed)(notmuch_message_t *message, const char *filename);
\r
137 +struct maildir_flag_tag {
\r
143 +/* ASCII ordered table of Maildir flags and assiciated tags */
\r
144 +struct maildir_flag_tag flag2tag[] = {
\r
145 + { 'D', "draft", false},
\r
146 + { 'F', "flagged", false},
\r
147 + { 'P', "passed", false},
\r
148 + { 'R', "replied", false},
\r
149 + { 'S', "unread", true },
\r
150 + { 'T', "delete", false},
\r
153 typedef struct _filename_node {
\r
155 @@ -69,8 +93,9 @@ _filename_list_add (_filename_list_t *list,
\r
159 -tag_inbox_and_unread (notmuch_message_t *message)
\r
160 +tag_inbox_and_unread (notmuch_message_t *message, const char *filename)
\r
163 notmuch_message_add_tag (message, "inbox");
\r
164 notmuch_message_add_tag (message, "unread");
\r
166 @@ -116,6 +141,163 @@ _entries_resemble_maildir (struct dirent **entries, int count)
\r
171 +tag_from_maildir_flags (notmuch_message_t *message, const char *filename)
\r
173 + const char *flags, *p;
\r
175 + bool valid, unread;
\r
178 + flags = strstr(filename, ":2,");
\r
183 + /* Check that the letters are valid Maildir flags */
\r
186 + for (p=flags; valid && *p; p++) {
\r
194 + if (*p > f) f=*p;
\r
195 + else valid = false;
\r
202 + fprintf (stderr, "Warning: Invalid maildir flags in filename %s\n", filename);
\r
206 + notmuch_message_freeze(message);
\r
208 + for (i = 0; i < ARRAY_SIZE(flag2tag); i++) {
\r
209 + if ((strchr(flags, flag2tag[i].flag) != NULL) ^ flag2tag[i].inverse) {
\r
210 + notmuch_message_add_tag (message, flag2tag[i].tag);
\r
212 + notmuch_message_remove_tag (message, flag2tag[i].tag);
\r
215 + notmuch_message_thaw(message);
\r
217 + /* From now on, we can synchronize the tags from the database to
\r
218 + * the mailstore. */
\r
219 + notmuch_message_set_flag(message, NOTMUCH_MESSAGE_FLAG_TAGS_INVALID, FALSE);
\r
224 +get_new_flags(notmuch_message_t *message, char *flags)
\r
226 + notmuch_tags_t *tags;
\r
231 + for (i = 0; i < ARRAY_SIZE(flag2tag); i++)
\r
232 + flags[i] = flag2tag[i].inverse ? flag2tag[i].flag : '\0';
\r
234 + for (tags = notmuch_message_get_tags (message);
\r
235 + notmuch_tags_valid (tags);
\r
236 + notmuch_tags_move_to_next (tags))
\r
238 + tag = notmuch_tags_get (tags);
\r
239 + for (i = 0; i < ARRAY_SIZE(flag2tag); i++) {
\r
240 + if (strcmp(tag, flag2tag[i].tag) == 0)
\r
241 + flags[i] = flag2tag[i].inverse ? '\0' : flag2tag[i].flag;
\r
246 + for (i = 0; i < ARRAY_SIZE(flag2tag); i++) {
\r
254 +get_subdir (char *filename)
\r
256 + char *p, *subdir = NULL;
\r
258 + p = filename + strlen (filename) - 1;
\r
259 + while (p > filename + 3 && *p != '/')
\r
263 + if (subdir > filename && *(subdir - 1) != '/')
\r
269 +/* Store maildir-related tags as maildir flags */
\r
270 +static notmuch_private_status_t
\r
271 +maildir_sync_tags (notmuch_mailstore_t *mailstore,
\r
272 + notmuch_message_t *message)
\r
274 + char flags[ARRAY_SIZE(flag2tag)+1];
\r
275 + const char *filename, *p, *db_path;
\r
276 + char *filename_new, *subdir = NULL;
\r
278 + char *abs1, *abs2;
\r
281 + get_new_flags (message, flags);
\r
283 + filename = notmuch_message_get_filename (message);
\r
284 + /* TODO: Iterate over all file names. */
\r
285 + p = strstr(filename, ":2,");
\r
287 + p = filename + strlen(filename);
\r
289 + filename_new = talloc_size(message, (p-filename) + 3 + sizeof(flags));
\r
290 + if (unlikely (filename_new == NULL))
\r
291 + return NOTMUCH_STATUS_OUT_OF_MEMORY;
\r
292 + memcpy(filename_new, filename, p-filename);
\r
293 + filename_new[p-filename] = '\0';
\r
295 + /* If message is in new/ move it under cur/. */
\r
296 + subdir = get_subdir (filename_new);
\r
297 + if (subdir && memcmp (subdir, "new/", 4) == 0)
\r
298 + memcpy (subdir, "cur/", 4);
\r
300 + strcpy (filename_new+(p-filename), ":2,");
\r
301 + strcpy (filename_new+(p-filename)+3, flags);
\r
303 + if (strcmp (filename, filename_new) != 0) {
\r
304 + db_path = notmuch_database_get_path (mailstore->notmuch);
\r
305 + asprintf(&abs1, "%s/%s", db_path, filename);
\r
306 + asprintf(&abs2, "%s/%s", db_path, filename_new);
\r
307 + ret = rename (abs1, abs2);
\r
309 + perror (talloc_asprintf (message, "rename of %s to %s failed", abs1, abs2));
\r
314 + return _notmuch_message_rename (message, filename_new);
\r
315 + /* _notmuch_message_sync is our caller. Do not call it here. */
\r
317 + return NOTMUCH_STATUS_SUCCESS;
\r
321 +tag_inbox_and_maildir_flags (notmuch_message_t *message, const char *filename)
\r
323 + notmuch_message_add_tag (message, "inbox");
\r
324 + if (tag_from_maildir_flags(message, filename) != 0)
\r
325 + notmuch_message_add_tag (message, "unread");
\r
328 /* Examine 'path' recursively as follows:
\r
330 @@ -171,6 +353,7 @@ add_files_recursive (notmuch_mailstore_t *mailstore,
\r
332 notmuch_bool_t is_maildir, new_directory;
\r
333 _indexing_context_priv_t *priv = state->priv;
\r
334 + struct mailstore_priv *mailstore_priv = mailstore->priv;
\r
335 notmuch_database_t *notmuch = mailstore->notmuch;
\r
337 if (stat (path, &st)) {
\r
338 @@ -352,11 +535,12 @@ add_files_recursive (notmuch_mailstore_t *mailstore,
\r
340 case NOTMUCH_STATUS_SUCCESS:
\r
341 state->added_messages++;
\r
342 - tag_inbox_and_unread (message);
\r
343 + mailstore_priv->tag_new (message, next);
\r
345 /* Non-fatal issues (go on to next file) */
\r
346 case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
\r
347 - /* Stay silent on this one. */
\r
348 + if (mailstore_priv->tag_renamed)
\r
349 + mailstore_priv->tag_renamed (message, next);
\r
351 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
\r
352 fprintf (stderr, "Note: Ignoring non-mail file: %s\n",
\r
353 @@ -618,6 +802,10 @@ open_file(notmuch_mailstore_t *mailstore, const char *filename)
\r
357 +struct mailstore_priv files_priv = {
\r
358 + .tag_new = tag_inbox_and_unread,
\r
361 /* Original notmuch mail store */
\r
362 struct _notmuch_mailstore mailstore_files = {
\r
364 @@ -625,4 +813,20 @@ struct _notmuch_mailstore mailstore_files = {
\r
365 .index_new = index_new,
\r
366 .sync_tags = NULL, /* We cannot store tags in this mailstore. */
\r
367 .open_file = open_file,
\r
368 + .priv = &files_priv,
\r
371 +struct mailstore_priv maildir_priv = {
\r
372 + .tag_new = tag_inbox_and_maildir_flags,
\r
373 + .tag_renamed = tag_from_maildir_flags,
\r
376 +/* Similar to mailstore_files, but does bi-directional synchronization between certain tags and maildir flags */
\r
377 +struct _notmuch_mailstore mailstore_maildir = {
\r
378 + .type = "maildir",
\r
379 + .count_files = count_files,
\r
380 + .index_new = index_new,
\r
381 + .sync_tags = maildir_sync_tags,
\r
382 + .open_file = open_file,
\r
383 + .priv = &maildir_priv,
\r
385 diff --git a/lib/mailstore.c b/lib/mailstore.c
\r
386 index 709db72..5dbb47c 100644
\r
387 --- a/lib/mailstore.c
\r
388 +++ b/lib/mailstore.c
\r
391 static notmuch_mailstore_t *available[] = {
\r
393 + &mailstore_maildir,
\r
396 notmuch_mailstore_t *
\r
397 diff --git a/lib/message.cc b/lib/message.cc
\r
398 index c7eff7c..292e95e 100644
\r
399 --- a/lib/message.cc
\r
400 +++ b/lib/message.cc
\r
401 @@ -578,7 +578,8 @@ _notmuch_message_sync (notmuch_message_t *message)
\r
402 if (message->notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY)
\r
405 - if (message->notmuch->mailstore->sync_tags) {
\r
406 + if (message->notmuch->mailstore->sync_tags &&
\r
407 + !notmuch_message_get_flag(message, NOTMUCH_MESSAGE_FLAG_TAGS_INVALID)) {
\r
408 status = message->notmuch->mailstore->sync_tags (message->notmuch->mailstore,
\r
410 if (status != NOTMUCH_PRIVATE_STATUS_SUCCESS) {
\r
411 @@ -700,6 +701,44 @@ _notmuch_message_remove_term (notmuch_message_t *message,
\r
412 return NOTMUCH_PRIVATE_STATUS_SUCCESS;
\r
415 +/* Change the message filename stored in the database.
\r
417 + * This change will not be reflected in the database until the next
\r
418 + * call to _notmuch_message_sync.
\r
420 +notmuch_private_status_t
\r
421 +_notmuch_message_rename (notmuch_message_t *message,
\r
422 + const char *new_filename)
\r
424 + void *local = talloc_new (message);
\r
426 + Xapian::PostingIterator i, end;
\r
427 + Xapian::Document document;
\r
428 + notmuch_private_status_t pstatus;
\r
429 + notmuch_status_t status;
\r
430 + const char *old_filename;
\r
432 + old_filename = notmuch_message_get_filename(message);
\r
433 + old_filename = talloc_reference(local, old_filename);
\r
434 + if (unlikely(!old_filename))
\r
435 + return NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY;
\r
437 + status = _notmuch_message_add_filename (message, new_filename);
\r
439 + return (notmuch_private_status_t)status;
\r
441 + status = _notmuch_database_filename_to_direntry (local, message->notmuch,
\r
442 + old_filename, &direntry);
\r
444 + return (notmuch_private_status_t)status;
\r
446 + pstatus = _notmuch_message_remove_term (message, "file-direntry", direntry);
\r
448 + talloc_free (local);
\r
454 notmuch_message_add_tag (notmuch_message_t *message, const char *tag)
\r
456 diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
\r
457 index bab2090..c257365 100644
\r
458 --- a/lib/notmuch-private.h
\r
459 +++ b/lib/notmuch-private.h
\r
460 @@ -245,6 +245,10 @@ notmuch_status_t
\r
461 _notmuch_message_add_filename (notmuch_message_t *message,
\r
462 const char *filename);
\r
464 +notmuch_private_status_t
\r
465 +_notmuch_message_rename (notmuch_message_t *message,
\r
466 + const char *new_filename);
\r
469 _notmuch_message_ensure_thread_id (notmuch_message_t *message);
\r
471 diff --git a/lib/notmuch.h b/lib/notmuch.h
\r
472 index 54de0bd..942eef7 100644
\r
473 --- a/lib/notmuch.h
\r
474 +++ b/lib/notmuch.h
\r
475 @@ -769,6 +769,7 @@ notmuch_message_get_mailstore (notmuch_message_t *message);
\r
476 /* Message flags */
\r
477 typedef enum _notmuch_message_flag {
\r
478 NOTMUCH_MESSAGE_FLAG_MATCH,
\r
479 + NOTMUCH_MESSAGE_FLAG_TAGS_INVALID,
\r
480 } notmuch_message_flag_t;
\r
482 /* Get a value of a flag for the email corresponding to 'message'. */
\r