From 34327483dda6bb0da5d1bb5e231afa28dbd6cdc6 Mon Sep 17 00:00:00 2001 From: Ethan Glasser-Camp Date: Thu, 16 Feb 2012 17:02:00 +1900 Subject: [PATCH] [RFC PATCH 07/13] notmuch-new: pull out useful bits of add_files_recursive --- d6/beba0023234d3e142f7b3bf7facb6d2e4e31ad | 333 ++++++++++++++++++++++ 1 file changed, 333 insertions(+) create mode 100644 d6/beba0023234d3e142f7b3bf7facb6d2e4e31ad diff --git a/d6/beba0023234d3e142f7b3bf7facb6d2e4e31ad b/d6/beba0023234d3e142f7b3bf7facb6d2e4e31ad new file mode 100644 index 000000000..d0f04f279 --- /dev/null +++ b/d6/beba0023234d3e142f7b3bf7facb6d2e4e31ad @@ -0,0 +1,333 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id AB64D431E62 + for ; Wed, 15 Feb 2012 14:09:45 -0800 (PST) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Spam-Flag: NO +X-Spam-Score: -0.54 +X-Spam-Level: +X-Spam-Status: No, score=-0.54 tagged_above=-999 required=5 + tests=[DKIM_SIGNED=0.1, DKIM_VALID=-0.1, DKIM_VALID_AU=-0.1, + RCVD_IN_BL_SPAMCOP_NET=1.246, RCVD_IN_DNSWL_MED=-2.3, + RCVD_IN_SORBS_WEB=0.614] autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id JYgxVZaFZw+r for ; + Wed, 15 Feb 2012 14:09:44 -0800 (PST) +Received: from cliffclavin.cs.rpi.edu (cliffclavin.cs.rpi.edu + [128.113.126.25]) (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits)) + (No client certificate requested) by olra.theworths.org (Postfix) with ESMTPS + id 89E9941EB90 for ; Wed, 15 Feb 2012 14:09:44 -0800 + (PST) +X-Hash: + S|1e6fe4067216137514005175a8428c8dbfb39661|ccff6b3b9d4b3c6d4682b599774abdc5 +X-Countries: Cameroon, United States +X-SMTP-From: accepted [195.24.209.20] [195.24.209.20] + (localhost) {Cameroon} +DKIM-Signature: v=1; a=rsa-sha1; c=relaxed/relaxed; d=cs.rpi.edu; h=from + :to:cc:subject:date:message-id:in-reply-to:references; s= + default; i=glasse@cs.rpi.edu; t=1329343782; x=1329948582; l=7459; + bh=8Cc1hoc5j0AGxRBD+Gl5gEhg5BE=; b=oz09gGRXEWGVjSqcmGq9Tew + SQLBvC1fQTydWumrdxMFTgBAXRDhTAZotLp/7co0a818olWbiIhsXmjvIioVh9Ys + Cq3y5SCZ+i3a8AZCuXJDj8d0HS2JYm/0p4dovv6eZnTKiQVnYoiFVrKcyuUYiIfC + Z3OLXYZYcWvZp+Rwdc7g= +DomainKey-Signature: a=rsa-sha1; c=nofws; d=cs.rpi.edu; h=from:to:cc + :subject:date:message-id:in-reply-to:references; q=dns; s= + default; b=B5tH8TFV4j4s5tN9wbaaaLGaOYa2qn/cRAHl2hnHLnYdMwM1qZAUj + EIc606a0CA6j1M4K6jB3Zqywl6BwydEXQABE96dotp4Ymka59dh1Lxhhtlnv5X+m + ZdFFpXTWBMGc22WFqCHZ6AIgaDancPN/+z4+Usq8EwfxVqG6NZ9X2s= +X-Spam-Info: -2.7; ALL_TRUSTED,AWL,BAYES_00 +X-Spam-Scanned-By: cliffclavin.cs.rpi.edu using SpamAssassin 3.2.5 (hard limit + 15) +Authentication-Results: cliffclavin.cs.rpi.edu; + DKIM=neutral (none) header.from=glasse@cs.rpi.edu; + SPF=neutral (mfrom; + Mechanism '?all' matched) smtp.mail=glasse@cs.rpi.edu +X-Auth-Passed: cliffclavin.cs.rpi.edu:q1FM9Dkb008829 Auth:glasse +X-Virus-Scanned-By: cliffclavin.cs.rpi.edu +Received: from localhost ([195.24.209.20]) (authenticated bits=0) + by cliffclavin.cs.rpi.edu (8.14.3/8.14.3) with ESMTP id q1FM9Dkb008829 + (version=TLSv1/SSLv3 cipher=DHE-RSA-AES128-SHA bits=128 verify=NO); + Wed, 15 Feb 2012 17:09:30 -0500 (EST) + (envelope-from glasse@cs.rpi.edu) +From: Ethan Glasser-Camp +To: notmuch@notmuchmail.org +Subject: [RFC PATCH 07/13] notmuch-new: pull out useful bits of + add_files_recursive +Date: Wed, 15 Feb 2012 17:02:00 -0500 +Message-Id: <1329343326-16410-8-git-send-email-glasse@cs.rpi.edu> +X-Mailer: git-send-email 1.7.5.4 +In-Reply-To: <1329343326-16410-1-git-send-email-glasse@cs.rpi.edu> +References: <1329343326-16410-1-git-send-email-glasse@cs.rpi.edu> +X-Scanned-By: MIMEDefang 2.67 on 128.113.126.25 +Cc: Ethan Glasser-Camp +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Wed, 15 Feb 2012 22:09:45 -0000 + +From: Ethan Glasser-Camp + +This is part of notmuch-new refactor phase 1: make add_files stuff +safe for other backends. add_files_recursive is essentially a +maildir-crawling function that periodically adds files to the database +or adds filenames to remove_files or remove_directory lists. I don't +see an easy way to adapt add_files_recursive for other backends who +might not have concepts of directories with other directories inside +of them, so instead just provide an add_files method for each backend. + +This patch pulls some bits out of add_files_recursive which will be +useful for other backends: two reporting functions +_report_before_adding_file and _report_added_file, as well as +_add_message, which actually does the message adding. + +Signed-off-by: Ethan Glasser-Camp +--- + notmuch-new.c | 193 +++++++++++++++++++++++++++++++++++---------------------- + 1 files changed, 120 insertions(+), 73 deletions(-) + +diff --git a/notmuch-new.c b/notmuch-new.c +index 355dde4..dbdfbb6 100644 +--- a/notmuch-new.c ++++ b/notmuch-new.c +@@ -181,6 +181,123 @@ _entries_resemble_maildir (struct dirent **entries, int count) + return 0; + } + ++/* Progress-reporting function. ++ * ++ * Can be used by any mailstore-crawling function that wants to alert ++ * users what message it's about to add. Subsequent errors will be due ++ * to this message ;) ++ */ ++static void ++_report_before_adding_file (add_files_state_t *state, const char *filename) ++{ ++ state->processed_files++; ++ ++ if (state->verbose) { ++ if (state->output_is_a_tty) ++ printf("\r\033[K"); ++ ++ printf ("%i/%i: %s", ++ state->processed_files, ++ state->total_files, ++ filename); ++ ++ putchar((state->output_is_a_tty) ? '\r' : '\n'); ++ fflush (stdout); ++ } ++} ++ ++/* Progress-reporting function. ++ * ++ * Call this to respond to the signal handler for SIGALRM. ++ */ ++static void ++_report_added_file (add_files_state_t *state) ++{ ++ if (do_print_progress) { ++ do_print_progress = 0; ++ generic_print_progress ("Processed", "files", state->tv_start, ++ state->processed_files, state->total_files); ++ } ++} ++ ++ ++/* Atomically handles adding a message to the database. ++ * ++ * Should be used by any mailstore-crawling function that finds a new ++ * message to add. ++ */ ++static notmuch_status_t ++_add_message (add_files_state_t *state, notmuch_database_t *notmuch, ++ const char *filename) ++{ ++ notmuch_status_t status, ret = NOTMUCH_STATUS_SUCCESS; ++ notmuch_message_t *message; ++ const char **tag; ++ ++ status = notmuch_database_begin_atomic (notmuch); ++ if (status) { ++ ret = status; ++ goto DONE; ++ } ++ ++ status = notmuch_database_add_message (notmuch, filename, &message); ++ ++ switch (status) { ++ /* success */ ++ case NOTMUCH_STATUS_SUCCESS: ++ state->added_messages++; ++ notmuch_message_freeze (message); ++ for (tag=state->new_tags; *tag != NULL; tag++) ++ notmuch_message_add_tag (message, *tag); ++ if (state->synchronize_flags == TRUE) ++ notmuch_message_maildir_flags_to_tags (message); ++ notmuch_message_thaw (message); ++ break; ++ /* Non-fatal issues (go on to next file) */ ++ case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID: ++ if (state->synchronize_flags == TRUE) ++ notmuch_message_maildir_flags_to_tags (message); ++ break; ++ case NOTMUCH_STATUS_FILE_NOT_EMAIL: ++ fprintf (stderr, "Note: Ignoring non-mail file: %s\n", ++ filename); ++ break; ++ /* Fatal issues. Don't process anymore. */ ++ case NOTMUCH_STATUS_READ_ONLY_DATABASE: ++ case NOTMUCH_STATUS_XAPIAN_EXCEPTION: ++ case NOTMUCH_STATUS_OUT_OF_MEMORY: ++ fprintf (stderr, "Error: %s. Halting processing.\n", ++ notmuch_status_to_string (status)); ++ ret = status; ++ goto DONE; ++ default: ++ case NOTMUCH_STATUS_FILE_ERROR: ++ case NOTMUCH_STATUS_NULL_POINTER: ++ case NOTMUCH_STATUS_TAG_TOO_LONG: ++ case NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW: ++ case NOTMUCH_STATUS_UNBALANCED_ATOMIC: ++ case NOTMUCH_STATUS_LAST_STATUS: ++ INTERNAL_ERROR ("add_message returned unexpected value: %d", status); ++ ret = status; ++ goto DONE; ++ } ++ ++ status = notmuch_database_end_atomic (notmuch); ++ if (status) { ++ ret = status; ++ goto DONE; ++ } ++ ++ DONE: ++ if (message) { ++ notmuch_message_destroy (message); ++ message = NULL; ++ } ++ ++ return ret; ++} ++ ++ + /* Examine 'path' recursively as follows: + * + * o Ask the filesystem for the mtime of 'path' (fs_mtime) +@@ -232,7 +349,6 @@ add_files_recursive (notmuch_database_t *notmuch, + char *next = NULL; + time_t fs_mtime, db_mtime; + notmuch_status_t status, ret = NOTMUCH_STATUS_SUCCESS; +- notmuch_message_t *message = NULL; + struct dirent **fs_entries = NULL; + int i, num_fs_entries; + notmuch_directory_t *directory; +@@ -241,7 +357,6 @@ add_files_recursive (notmuch_database_t *notmuch, + time_t stat_time; + struct stat st; + notmuch_bool_t is_maildir, new_directory; +- const char **tag; + + if (stat (path, &st)) { + fprintf (stderr, "Error reading directory %s: %s\n", +@@ -439,83 +554,15 @@ add_files_recursive (notmuch_database_t *notmuch, + * in the database, so add it. */ + next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name); + +- state->processed_files++; +- +- if (state->verbose) { +- if (state->output_is_a_tty) +- printf("\r\033[K"); ++ _report_before_adding_file (state, next); + +- printf ("%i/%i: %s", +- state->processed_files, +- state->total_files, +- next); +- +- putchar((state->output_is_a_tty) ? '\r' : '\n'); +- fflush (stdout); +- } +- +- status = notmuch_database_begin_atomic (notmuch); ++ status = _add_message (state, notmuch, next); + if (status) { + ret = status; + goto DONE; + } + +- status = notmuch_database_add_message (notmuch, next, &message); +- switch (status) { +- /* success */ +- case NOTMUCH_STATUS_SUCCESS: +- state->added_messages++; +- notmuch_message_freeze (message); +- for (tag=state->new_tags; *tag != NULL; tag++) +- notmuch_message_add_tag (message, *tag); +- if (state->synchronize_flags == TRUE) +- notmuch_message_maildir_flags_to_tags (message); +- notmuch_message_thaw (message); +- break; +- /* Non-fatal issues (go on to next file) */ +- case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID: +- if (state->synchronize_flags == TRUE) +- notmuch_message_maildir_flags_to_tags (message); +- break; +- case NOTMUCH_STATUS_FILE_NOT_EMAIL: +- fprintf (stderr, "Note: Ignoring non-mail file: %s\n", +- next); +- break; +- /* Fatal issues. Don't process anymore. */ +- case NOTMUCH_STATUS_READ_ONLY_DATABASE: +- case NOTMUCH_STATUS_XAPIAN_EXCEPTION: +- case NOTMUCH_STATUS_OUT_OF_MEMORY: +- fprintf (stderr, "Error: %s. Halting processing.\n", +- notmuch_status_to_string (status)); +- ret = status; +- goto DONE; +- default: +- case NOTMUCH_STATUS_FILE_ERROR: +- case NOTMUCH_STATUS_NULL_POINTER: +- case NOTMUCH_STATUS_TAG_TOO_LONG: +- case NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW: +- case NOTMUCH_STATUS_UNBALANCED_ATOMIC: +- case NOTMUCH_STATUS_LAST_STATUS: +- INTERNAL_ERROR ("add_message returned unexpected value: %d", status); +- goto DONE; +- } +- +- status = notmuch_database_end_atomic (notmuch); +- if (status) { +- ret = status; +- goto DONE; +- } +- +- if (message) { +- notmuch_message_destroy (message); +- message = NULL; +- } +- +- if (do_print_progress) { +- do_print_progress = 0; +- generic_print_progress ("Processed", "files", state->tv_start, +- state->processed_files, state->total_files); +- } ++ _report_added_file (state); + + talloc_free (next); + next = NULL; +-- +1.7.5.4 + -- 2.26.2