1 /* notmuch - Not much of an email program, (just index and search)
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see http://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
22 #define _GNU_SOURCE /* for getline */
28 /* This is separate from notmuch-private.h because we're trying to
29 * keep notmuch.c from looking into any internals, (which helps us
30 * develop notmuch.h into a plausible library interface).
44 #include <glib.h> /* g_strdup_printf */
46 #define unused(x) x __attribute__ ((unused))
48 /* There's no point in continuing when we've detected that we've done
49 * something wrong internally (as opposed to the user passing in a
52 * Note that __location__ comes from talloc.h.
54 #define INTERNAL_ERROR(format, ...) \
57 "Internal error: " format " (%s)\n", \
58 ##__VA_ARGS__, __location__); \
62 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
64 typedef int (*command_function_t) (int argc, char *argv[]);
66 typedef struct command {
68 command_function_t function;
73 int ignore_read_only_directories;
74 int saw_read_only_directory;
79 struct timeval tv_start;
83 chomp_newline (char *str)
85 if (str && str[strlen(str)-1] == '\n')
86 str[strlen(str)-1] = '\0';
89 /* Compute the number of seconds elapsed from start to end. */
91 tv_elapsed (struct timeval start, struct timeval end)
93 return ((end.tv_sec - start.tv_sec) +
94 (end.tv_usec - start.tv_usec) / 1e6);
98 print_formatted_seconds (double seconds)
104 printf ("almost no time");
108 if (seconds > 3600) {
109 hours = (int) seconds / 3600;
110 printf ("%dh ", hours);
111 seconds -= hours * 3600;
115 minutes = (int) seconds / 60;
116 printf ("%dm ", minutes);
117 seconds -= minutes * 60;
120 printf ("%ds", (int) seconds);
124 add_files_print_progress (add_files_state_t *state)
126 struct timeval tv_now;
127 double elapsed_overall, rate_overall;
129 gettimeofday (&tv_now, NULL);
131 elapsed_overall = tv_elapsed (state->tv_start, tv_now);
132 rate_overall = (state->processed_files) / elapsed_overall;
134 printf ("Processed %d", state->processed_files);
136 if (state->total_files) {
137 printf (" of %d files (", state->total_files);
138 print_formatted_seconds ((state->total_files - state->processed_files) /
140 printf (" remaining). \r");
142 printf (" files (%d files/sec.) \r", (int) rate_overall);
148 /* Examine 'path' recursively as follows:
150 * o Ask the filesystem for the mtime of 'path' (path_mtime)
152 * o Ask the database for its timestamp of 'path' (path_dbtime)
154 * o If 'path_mtime' > 'path_dbtime'
156 * o For each regular file in 'path' with mtime newer than the
157 * 'path_dbtime' call add_message to add the file to the
160 * o For each sub-directory of path, recursively call into this
163 * o Tell the database to update its time of 'path' to 'path_mtime'
165 * The 'struct stat *st' must point to a structure that has already
166 * been initialized for 'path' by calling stat().
168 static notmuch_status_t
169 add_files_recursive (notmuch_database_t *notmuch,
172 add_files_state_t *state)
175 struct dirent *e, *entry = NULL;
179 time_t path_mtime, path_dbtime;
180 notmuch_status_t status, ret = NOTMUCH_STATUS_SUCCESS;
182 /* If we're told to, we bail out on encountering a read-only
183 * directory, (with this being a clear clue from the user to
184 * Notmuch that new mail won't be arriving there and we need not
186 if (state->ignore_read_only_directories &&
187 (st->st_mode & S_IWUSR) == 0)
189 state->saw_read_only_directory = TRUE;
193 path_mtime = st->st_mtime;
195 path_dbtime = notmuch_database_get_timestamp (notmuch, path);
197 dir = opendir (path);
199 fprintf (stderr, "Error opening directory %s: %s\n",
200 path, strerror (errno));
201 ret = NOTMUCH_STATUS_FILE_ERROR;
205 entry_length = offsetof (struct dirent, d_name) +
206 pathconf (path, _PC_NAME_MAX) + 1;
207 entry = malloc (entry_length);
210 err = readdir_r (dir, entry, &e);
212 fprintf (stderr, "Error reading directory: %s\n",
214 ret = NOTMUCH_STATUS_FILE_ERROR;
221 /* If this directory hasn't been modified since the last
222 * add_files, then we only need to look further for
223 * sub-directories. */
224 if (path_mtime <= path_dbtime && entry->d_type != DT_DIR)
227 /* Ignore special directories to avoid infinite recursion.
228 * Also ignore the .notmuch directory.
230 /* XXX: Eventually we'll want more sophistication to let the
231 * user specify files to be ignored. */
232 if (strcmp (entry->d_name, ".") == 0 ||
233 strcmp (entry->d_name, "..") == 0 ||
234 strcmp (entry->d_name, ".notmuch") ==0)
239 next = g_strdup_printf ("%s/%s", path, entry->d_name);
241 if (stat (next, st)) {
242 fprintf (stderr, "Error reading %s: %s\n",
243 next, strerror (errno));
244 ret = NOTMUCH_STATUS_FILE_ERROR;
248 if (S_ISREG (st->st_mode)) {
249 /* If the file hasn't been modified since the last
250 * add_files, then we need not look at it. */
251 if (st->st_mtime > path_dbtime) {
252 state->processed_files++;
254 status = notmuch_database_add_message (notmuch, next);
257 case NOTMUCH_STATUS_SUCCESS:
258 state->added_messages++;
260 /* Non-fatal issues (go on to next file) */
261 case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
262 /* Stay silent on this one. */
264 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
265 fprintf (stderr, "Note: Ignoring non-mail file: %s\n",
268 /* Fatal issues. Don't process anymore. */
269 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
270 fprintf (stderr, "A Xapian error was encountered. Halting processing.\n");
274 INTERNAL_ERROR ("add_message returned unexpected value: %d", status);
277 if (state->processed_files % 1000 == 0)
278 add_files_print_progress (state);
280 } else if (S_ISDIR (st->st_mode)) {
281 status = add_files_recursive (notmuch, next, st, state);
282 if (status && ret == NOTMUCH_STATUS_SUCCESS)
290 status = notmuch_database_set_timestamp (notmuch, path, path_mtime);
291 if (status && ret == NOTMUCH_STATUS_SUCCESS)
305 /* This is the top-level entry point for add_files. It does a couple
306 * of error checks, and then calls into the recursive function,
307 * (avoiding the repeating of these error checks at every
308 * level---which would be useless becaues we already do a stat() at
309 * the level above). */
310 static notmuch_status_t
311 add_files (notmuch_database_t *notmuch,
313 add_files_state_t *state)
317 if (stat (path, &st)) {
318 fprintf (stderr, "Error reading directory %s: %s\n",
319 path, strerror (errno));
320 return NOTMUCH_STATUS_FILE_ERROR;
323 if (! S_ISDIR (st.st_mode)) {
324 fprintf (stderr, "Error: %s is not a directory.\n", path);
325 return NOTMUCH_STATUS_FILE_ERROR;
328 return add_files_recursive (notmuch, path, &st, state);
331 /* Recursively count all regular files in path and all sub-direcotries
332 * of path. The result is added to *count (which should be
333 * initialized to zero by the top-level caller before calling
336 count_files (const char *path, int *count)
339 struct dirent *entry, *e;
345 dir = opendir (path);
348 fprintf (stderr, "Warning: failed to open directory %s: %s\n",
349 path, strerror (errno));
353 entry_length = offsetof (struct dirent, d_name) +
354 pathconf (path, _PC_NAME_MAX) + 1;
355 entry = malloc (entry_length);
358 err = readdir_r (dir, entry, &e);
360 fprintf (stderr, "Error reading directory: %s\n",
369 /* Ignore special directories to avoid infinite recursion.
370 * Also ignore the .notmuch directory.
372 /* XXX: Eventually we'll want more sophistication to let the
373 * user specify files to be ignored. */
374 if (strcmp (entry->d_name, ".") == 0 ||
375 strcmp (entry->d_name, "..") == 0 ||
376 strcmp (entry->d_name, ".notmuch") == 0)
381 next = g_strdup_printf ("%s/%s", path, entry->d_name);
385 if (S_ISREG (st.st_mode)) {
387 if (*count % 1000 == 0) {
388 printf ("Found %d files so far.\r", *count);
391 } else if (S_ISDIR (st.st_mode)) {
392 count_files (next, count);
404 setup_command (unused (int argc), unused (char *argv[]))
406 notmuch_database_t *notmuch = NULL;
407 char *default_path, *mail_directory = NULL;
410 add_files_state_t add_files_state;
412 struct timeval tv_now;
413 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
415 printf ("Welcome to notmuch!\n\n");
417 printf ("The goal of notmuch is to help you manage and search your collection of\n"
418 "email, and to efficiently keep up with the flow of email as it comes in.\n\n");
420 printf ("Notmuch needs to know the top-level directory of your email archive,\n"
421 "(where you already have mail stored and where messages will be delivered\n"
422 "in the future). This directory can contain any number of sub-directories\n"
423 "and primarily just files with indvidual email messages (eg. maildir or mh\n"
424 "archives are perfect). If there are other, non-email files (such as\n"
425 "indexes maintained by other email programs) then notmuch will do its\n"
426 "best to detect those and ignore them.\n\n");
428 printf ("Mail storage that uses mbox format, (where one mbox file contains many\n"
429 "messages), will not work with notmuch. If that's how your mail is currently\n"
430 "stored, we recommend you first convert it to maildir format with a utility\n"
431 "such as mb2md. In that case, press Control-C now and run notmuch again\n"
432 "once the conversion is complete.\n\n");
435 default_path = notmuch_database_default_path ();
436 printf ("Top-level mail directory [%s]: ", default_path);
439 getline (&mail_directory, &line_size, stdin);
440 chomp_newline (mail_directory);
444 if (mail_directory == NULL || strlen (mail_directory) == 0) {
446 free (mail_directory);
447 mail_directory = default_path;
449 /* XXX: Instead of telling the user to use an environment
450 * variable here, we should really be writing out a configuration
451 * file and loading that on the next run. */
452 if (strcmp (mail_directory, default_path)) {
453 printf ("Note: Since you are not using the default path, you will want to set\n"
454 "the NOTMUCH_BASE environment variable to %s so that\n"
455 "future calls to notmuch commands will know where to find your mail.\n",
457 printf ("For example, if you are using bash for your shell, add:\n\n");
458 printf ("\texport NOTMUCH_BASE=%s\n\n", mail_directory);
459 printf ("to your ~/.bashrc file.\n\n");
464 notmuch = notmuch_database_create (mail_directory);
465 if (notmuch == NULL) {
466 fprintf (stderr, "Failed to create new notmuch database at %s\n",
468 ret = NOTMUCH_STATUS_FILE_ERROR;
472 printf ("OK. Let's take a look at the mail we can find in the directory\n");
473 printf ("%s ...\n", mail_directory);
476 count_files (mail_directory, &count);
478 printf ("Found %d total files. That's not much mail.\n\n", count);
480 printf ("Next, we'll inspect the messages and create a database of threads:\n");
482 add_files_state.ignore_read_only_directories = FALSE;
483 add_files_state.saw_read_only_directory = FALSE;
484 add_files_state.total_files = count;
485 add_files_state.processed_files = 0;
486 add_files_state.added_messages = 0;
487 gettimeofday (&add_files_state.tv_start, NULL);
489 ret = add_files (notmuch, mail_directory, &add_files_state);
491 gettimeofday (&tv_now, NULL);
492 elapsed = tv_elapsed (add_files_state.tv_start,
494 printf ("Processed %d %s in ", add_files_state.processed_files,
495 add_files_state.processed_files == 1 ?
496 "file" : "total files");
497 print_formatted_seconds (elapsed);
499 printf (" (%d files/sec.). \n",
500 (int) (add_files_state.processed_files / elapsed));
504 if (add_files_state.added_messages) {
505 printf ("Added %d %s to the database.\n\n",
506 add_files_state.added_messages,
507 add_files_state.added_messages == 1 ?
508 "message" : "unique messages");
511 printf ("When new mail is delivered to %s in the future,\n"
512 "run \"notmuch new\" to add it to the database.\n\n",
516 printf ("Note: At least one error was encountered: %s\n",
517 notmuch_status_to_string (ret));
522 free (mail_directory);
524 notmuch_database_close (notmuch);
530 new_command (unused (int argc), unused (char *argv[]))
532 notmuch_database_t *notmuch;
533 const char *mail_directory;
534 add_files_state_t add_files_state;
536 struct timeval tv_now;
539 notmuch = notmuch_database_open (NULL);
540 if (notmuch == NULL) {
545 mail_directory = notmuch_database_get_path (notmuch);
547 add_files_state.ignore_read_only_directories = TRUE;
548 add_files_state.saw_read_only_directory = FALSE;
549 add_files_state.total_files = 0;
550 add_files_state.processed_files = 0;
551 add_files_state.added_messages = 0;
552 gettimeofday (&add_files_state.tv_start, NULL);
554 ret = add_files (notmuch, mail_directory, &add_files_state);
556 gettimeofday (&tv_now, NULL);
557 elapsed = tv_elapsed (add_files_state.tv_start,
559 if (add_files_state.processed_files) {
560 printf ("Processed %d %s in ", add_files_state.processed_files,
561 add_files_state.processed_files == 1 ?
562 "file" : "total files");
563 print_formatted_seconds (elapsed);
565 printf (" (%d files/sec.). \n",
566 (int) (add_files_state.processed_files / elapsed));
571 if (add_files_state.added_messages) {
572 printf ("Added %d new %s to the database (not much, really).\n",
573 add_files_state.added_messages,
574 add_files_state.added_messages == 1 ?
575 "message" : "messages");
577 printf ("No new mail---and that's not much.\n");
580 if (elapsed > 1 && ! add_files_state.saw_read_only_directory) {
581 printf ("\nTip: If you have any sub-directories that are archives (that is,\n"
582 "they will never receive new mail), marking these directores as\n"
583 "read-only (chmod u-w /path/to/dir) will make \"notmuch new\"\n"
584 "much more efficient (it won't even look in those directories).\n");
588 printf ("\nNote: At least one error was encountered: %s\n",
589 notmuch_status_to_string (ret));
594 notmuch_database_close (notmuch);
600 search_command (int argc, char *argv[])
602 void *local = talloc_new (NULL);
603 notmuch_database_t *notmuch = NULL;
604 notmuch_query_t *query;
605 notmuch_results_t *results;
606 notmuch_message_t *message;
607 notmuch_tags_t *tags;
610 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
612 notmuch = notmuch_database_open (NULL);
613 if (notmuch == NULL) {
618 /* XXX: Should add xtalloc wrappers here and use them. */
619 query_str = talloc_strdup (local, "");
621 for (i = 0; i < argc; i++) {
623 query_str = talloc_asprintf_append (query_str, " ");
625 query_str = talloc_asprintf_append (query_str, "%s", argv[i]);
628 query = notmuch_query_create (notmuch, query_str);
630 fprintf (stderr, "Out of memory\n");
635 for (results = notmuch_query_search (query);
636 notmuch_results_has_more (results);
637 notmuch_results_advance (results))
640 message = notmuch_results_get (results);
642 printf ("%s (", notmuch_message_get_message_id (message));
644 for (tags = notmuch_message_get_tags (message);
645 notmuch_tags_has_more (tags);
646 notmuch_tags_advance (tags))
651 printf ("%s", notmuch_tags_get (tags));
658 notmuch_message_destroy (message);
661 notmuch_query_destroy (query);
665 notmuch_database_close (notmuch);
672 show_command (unused (int argc), unused (char *argv[]))
674 fprintf (stderr, "Error: show is not implemented yet.\n");
679 dump_command (int argc, char *argv[])
682 notmuch_database_t *notmuch = NULL;
683 notmuch_query_t *query;
684 notmuch_results_t *results;
685 notmuch_message_t *message;
686 notmuch_tags_t *tags;
690 output = fopen (argv[0], "w");
691 if (output == NULL) {
692 fprintf (stderr, "Error opening %s for writing: %s\n",
693 argv[0], strerror (errno));
701 notmuch = notmuch_database_open (NULL);
702 if (notmuch == NULL) {
707 query = notmuch_query_create (notmuch, "");
709 fprintf (stderr, "Out of memory\n");
714 notmuch_query_set_sort (query, NOTMUCH_SORT_MESSAGE_ID);
716 for (results = notmuch_query_search (query);
717 notmuch_results_has_more (results);
718 notmuch_results_advance (results))
721 message = notmuch_results_get (results);
724 "%s (", notmuch_message_get_message_id (message));
726 for (tags = notmuch_message_get_tags (message);
727 notmuch_tags_has_more (tags);
728 notmuch_tags_advance (tags))
731 fprintf (output, " ");
733 fprintf (output, "%s", notmuch_tags_get (tags));
738 fprintf (output, ")\n");
740 notmuch_message_destroy (message);
743 notmuch_query_destroy (query);
747 notmuch_database_close (notmuch);
748 if (output != stdout)
755 restore_command (int argc, char *argv[])
758 notmuch_database_t *notmuch = NULL;
767 input = fopen (argv[0], "r");
769 fprintf (stderr, "Error opening %s for reading: %s\n",
770 argv[0], strerror (errno));
775 printf ("No filename given. Reading dump from stdin.\n");
779 notmuch = notmuch_database_open (NULL);
780 if (notmuch == NULL) {
785 /* Dump output is one line per message. We match a sequence of
786 * non-space characters for the message-id, then one or more
787 * spaces, then a list of space-separated tags as a sequence of
788 * characters within literal '(' and ')'. */
790 "^([^ ]+) \\(([^)]*)\\)$",
793 while ((line_len = getline (&line, &line_size, input)) != -1) {
795 char *message_id, *tags, *tag, *next;
796 notmuch_message_t *message;
797 notmuch_status_t status;
799 chomp_newline (line);
801 rerr = xregexec (®ex, line, 3, match, 0);
802 if (rerr == REG_NOMATCH)
804 fprintf (stderr, "Warning: Ignoring invalid input line: %s\n",
809 message_id = xstrndup (line + match[1].rm_so,
810 match[1].rm_eo - match[1].rm_so);
811 tags = xstrndup (line + match[2].rm_so,
812 match[2].rm_eo - match[2].rm_so);
816 message = notmuch_database_find_message (notmuch, message_id);
817 if (message == NULL) {
818 fprintf (stderr, "Warning: Cannot apply tags to missing message: %s (",
824 tag = strsep (&next, " ");
828 status = notmuch_message_add_tag (message, tag);
831 "Error applying tag %s to message %s:\n",
833 fprintf (stderr, "%s\n",
834 notmuch_status_to_string (status));
837 fprintf (stderr, "%s%s",
838 tag == tags ? "" : " ", tag);
843 notmuch_message_destroy (message);
845 fprintf (stderr, ")\n");
857 notmuch_database_close (notmuch);
862 command_t commands[] = {
863 { "setup", setup_command,
864 "Interactively setup notmuch for first use.\n\n"
865 "\t\tInvoking notmuch with no command argument will run setup if\n"
866 "\t\tthe setup command has not previously been completed." },
867 { "new", new_command,
868 "Find and import any new messages.\n\n"
869 "\t\tScans all sub-directories of the database, adding new files\n"
870 "\t\tthat are found. Note: \"notmuch new\" will skip any\n"
871 "\t\tread-only directories, so you can use that to mark\n"
872 "\t\tdirectories that will not receive any new mail."},
873 { "search", search_command,
874 "<search-term> [...]\n\n"
875 "\t\tSearch for threads matching the given search terms.\n"
876 "\t\tOnce we actually implement search we'll document the\n"
877 "\t\tsyntax here." },
878 { "show", show_command,
880 "\t\tShow the thread with the given thread ID (see 'search')." },
881 { "dump", dump_command,
883 "\t\tCreate a plain-text dump of the tags for each message\n"
884 "\t\twriting to the given filename, if any, or to stdout.\n"
885 "\t\tThese tags are the only data in the notmuch database\n"
886 "\t\tthat can't be recreated from the messages themselves.\n"
887 "\t\tThe output of notmuch dump is therefore the only\n"
888 "\t\tcritical thing to backup (and much more friendly to\n"
889 "\t\tincremental backup than the native database files." },
890 { "restore", restore_command,
892 "\t\tRestore the tags from the given dump file (see 'dump')." }
901 fprintf (stderr, "Usage: notmuch <command> [args...]\n");
902 fprintf (stderr, "\n");
903 fprintf (stderr, "Where <command> and [args...] are as follows:\n");
904 fprintf (stderr, "\n");
906 for (i = 0; i < ARRAY_SIZE (commands); i++) {
907 command = &commands[i];
909 fprintf (stderr, "\t%s\t%s\n\n", command->name, command->usage);
914 main (int argc, char *argv[])
920 return setup_command (0, NULL);
922 for (i = 0; i < ARRAY_SIZE (commands); i++) {
923 command = &commands[i];
925 if (strcmp (argv[1], command->name) == 0)
926 return (command->function) (argc - 2, &argv[2]);
929 /* Don't complain about "help" being an unknown command when we're
930 about to provide exactly what's wanted anyway. */
931 if (strcmp (argv[1], "help") == 0 ||
932 strcmp (argv[1], "--help") == 0)
934 fprintf (stderr, "The notmuch mail system.\n\n");
936 fprintf (stderr, "Error: Unknown command '%s'\n\n", argv[1]);