From 2789db2a78aa11b888bfc0e6047fd1dff26bc9fc Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Sat, 25 Jan 2014 16:18:22 +1900 Subject: [PATCH] Re: [PATCH 1/5] lib: make folder: prefix literal --- f5/0480549b08bcd12b33b93f94ecbe35d2627db3 | 396 ++++++++++++++++++++++ 1 file changed, 396 insertions(+) create mode 100644 f5/0480549b08bcd12b33b93f94ecbe35d2627db3 diff --git a/f5/0480549b08bcd12b33b93f94ecbe35d2627db3 b/f5/0480549b08bcd12b33b93f94ecbe35d2627db3 new file mode 100644 index 000000000..9d4c112b9 --- /dev/null +++ b/f5/0480549b08bcd12b33b93f94ecbe35d2627db3 @@ -0,0 +1,396 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id BCE61431FC7 + for ; Fri, 24 Jan 2014 13:18:32 -0800 (PST) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Spam-Flag: NO +X-Spam-Score: -0.7 +X-Spam-Level: +X-Spam-Status: No, score=-0.7 tagged_above=-999 required=5 + tests=[RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id nqJ8vBnXO3Or for ; + Fri, 24 Jan 2014 13:18:26 -0800 (PST) +Received: from dmz-mailsec-scanner-2.mit.edu (dmz-mailsec-scanner-2.mit.edu + [18.9.25.13]) + (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits)) + (No client certificate requested) + by olra.theworths.org (Postfix) with ESMTPS id 2BB4F431FC3 + for ; Fri, 24 Jan 2014 13:18:26 -0800 (PST) +X-AuditID: 1209190d-f79776d000000ce9-5a-52e2d8a1ecc6 +Received: from mailhub-auth-3.mit.edu ( [18.9.21.43]) + (using TLS with cipher AES256-SHA (256/256 bits)) + (Client did not present a certificate) + by dmz-mailsec-scanner-2.mit.edu (Symantec Messaging Gateway) with SMTP + id D5.86.03305.1A8D2E25; Fri, 24 Jan 2014 16:18:25 -0500 (EST) +Received: from outgoing.mit.edu (outgoing-auth-1.mit.edu [18.9.28.11]) + by mailhub-auth-3.mit.edu (8.13.8/8.9.2) with ESMTP id s0OLIOWs026329; + Fri, 24 Jan 2014 16:18:24 -0500 +Received: from awakening.csail.mit.edu (awakening.csail.mit.edu [18.26.4.91]) + (authenticated bits=0) + (User authenticated as amdragon@ATHENA.MIT.EDU) + by outgoing.mit.edu (8.13.8/8.12.4) with ESMTP id s0OLINGu020312 + (version=TLSv1/SSLv3 cipher=DHE-RSA-AES128-SHA bits=128 verify=NOT); + Fri, 24 Jan 2014 16:18:24 -0500 +Received: from amthrax by awakening.csail.mit.edu with local (Exim 4.80) + (envelope-from ) + id 1W6o8s-0006Dj-UL; Fri, 24 Jan 2014 16:18:22 -0500 +From: Austin Clements +To: Jani Nikula , notmuch@notmuchmail.org +Subject: Re: [PATCH 1/5] lib: make folder: prefix literal +In-Reply-To: + +References: + +User-Agent: Notmuch/0.17~rc2+14~g06f47e0 (http://notmuchmail.org) Emacs/23.4.1 + (i486-pc-linux-gnu) +Date: Fri, 24 Jan 2014 16:18:22 -0500 +Message-ID: <87wqhpm63l.fsf@awakening.csail.mit.edu> +MIME-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +X-Brightmail-Tracker: + H4sIAAAAAAAAA+NgFrrPIsWRmVeSWpSXmKPExsUixCmqrbvwxqMgg8c7+SyapjtbXL85k9mB + yePW/dfsHs9W3WIOYIrisklJzcksSy3St0vgyng78xRjQU9ExanNP5gbGB+5dDFyckgImEj8 + mPKbDcIWk7hwbz2YLSQwm0mic1JAFyMXkL2RUeLE5cmsEM5pJokTF56wQzhLGCXurJsM1sIm + oCGxbf9yRhBbRMBK4tzD1ywgtrCApcSOU0tYQWxOgTCJP3e7oVbUSOz8c5kZxBYVSJI4PfU3 + UA0HB4uAqkTLZQeQMC/QdUvfb2OCsAUlTs58AjaSWUBL4sa/l0wTGAVmIUnNQpJawMi0ilE2 + JbdKNzcxM6c4NVm3ODkxLy+1SNdILzezRC81pXQTIzgcJXl3ML47qHSIUYCDUYmH90PwwyAh + 1sSy4srcQ4ySHExKorxTLz8KEuJLyk+pzEgszogvKs1JLT7EKMHBrCTCO3czUI43JbGyKrUo + HyYlzcGiJM57k8M+SEggPbEkNTs1tSC1CCYrw8GhJMH79RpQo2BRanpqRVpmTglCmomDE2Q4 + D9DwSSA1vMUFibnFmekQ+VOMilLivAtAEgIgiYzSPLheWLp4xSgO9Iowr+F1oCoeYKqB634F + NJgJaPCKsw9ABpckIqSkGhjX7+IPKXtiFzmvWdu1oDkj8dl+k/CftZMvXp6h8MZQ+MIUb73q + NF8jjwzBbR0PeB7d+BdiNvPetnqO81b1e99wr7p7Qnv/zb2aX3N5Xuk5ntzTfW/W/1vvBB4a + yfFFPn99TqL90B5+zYMcupWe5Zwn1DoEbz09ZeeovfF4pY7gIcejh/a+vrxEiaU4I9FQi7mo + OBEAkcueSvICAAA= +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Fri, 24 Jan 2014 21:18:32 -0000 + +On Thu, 09 Jan 2014, Jani Nikula wrote: +> In xapian terms, convert folder: prefix from probabilistic to boolean +> prefix. This change constitutes a database change: bump the database +> version and add database upgrade support. +> --- +> lib/database.cc | 39 ++++++++++++- +> lib/message.cc | 154 +++++++++++++++++++++++++------------------------- +> lib/notmuch-private.h | 3 + +> 3 files changed, 117 insertions(+), 79 deletions(-) +> +> diff --git a/lib/database.cc b/lib/database.cc +> index f395061..145fd66 100644 +> --- a/lib/database.cc +> +++ b/lib/database.cc +> @@ -42,7 +42,7 @@ typedef struct { +> const char *prefix; +> } prefix_t; +> +> -#define NOTMUCH_DATABASE_VERSION 1 +> +#define NOTMUCH_DATABASE_VERSION 2 +> +> #define STRINGIFY(s) _SUB_STRINGIFY(s) +> #define _SUB_STRINGIFY(s) #s +> @@ -208,7 +208,8 @@ static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = { +> { "thread", "G" }, +> { "tag", "K" }, +> { "is", "K" }, +> - { "id", "Q" } +> + { "id", "Q" }, +> + { "folder", "P" }, +> }; +> +> static prefix_t PROBABILISTIC_PREFIX[]= { +> @@ -216,7 +217,6 @@ static prefix_t PROBABILISTIC_PREFIX[]= { +> { "to", "XTO" }, +> { "attachment", "XATTACHMENT" }, +> { "subject", "XSUBJECT"}, +> - { "folder", "XFOLDER"} +> }; +> +> const char * +> @@ -1167,6 +1167,39 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, +> } +> } +> +> + /* +> + * Prior to version 2, the "folder:" prefix was probabilistic and +> + * stemmed. Change it to the current boolean prefix. +> + */ +> + if (version < 2) { +> + notmuch_query_t *query = notmuch_query_create (notmuch, ""); +> + notmuch_messages_t *messages; +> + notmuch_message_t *message; +> + +> + count = 0; +> + total = notmuch_query_count_messages (query); +> + +> + for (messages = notmuch_query_search_messages (query); +> + notmuch_messages_valid (messages); +> + notmuch_messages_move_to_next (messages)) { +> + if (do_progress_notify) { +> + progress_notify (closure, (double) count / total); +> + do_progress_notify = 0; +> + } +> + +> + message = notmuch_messages_get (messages); +> + +> + _notmuch_message_upgrade_folder (message); +> + _notmuch_message_sync (message); +> + +> + notmuch_message_destroy (message); +> + +> + count++; +> + } +> + +> + notmuch_query_destroy (query); +> + } +> + + +Unless I'm missing something, the upgrade isn't done atomically. Should +it be? (The answer may be "no", since this process appears to be +idempotent; though it may simply be easier to reason about if it is +atomic.) + +Everything else in this patch LGTM (modulo my complaint about stripping +cur/new). + +> db->set_metadata ("version", STRINGIFY (NOTMUCH_DATABASE_VERSION)); +> db->flush (); +> +> diff --git a/lib/message.cc b/lib/message.cc +> index 1b46379..500aa26 100644 +> --- a/lib/message.cc +> +++ b/lib/message.cc +> @@ -505,89 +505,27 @@ _notmuch_message_add_filename (notmuch_message_t *message, +> _notmuch_message_add_term (message, "file-direntry", direntry); +> +> /* New terms allow user to search with folder: specification. */ +> - _notmuch_message_gen_terms (message, "folder", directory); +> + _notmuch_message_add_term (message, "folder", directory); +> +> talloc_free (local); +> +> return NOTMUCH_STATUS_SUCCESS; +> } +> +> -/* Remove a particular 'filename' from 'message'. +> - * +> - * This change will not be reflected in the database until the next +> - * call to _notmuch_message_sync. +> - * +> - * If this message still has other filenames, returns +> - * NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID. +> - * +> - * Note: This function does not remove a document from the database, +> - * even if the specified filename is the only filename for this +> - * message. For that functionality, see +> - * _notmuch_database_remove_message. */ +> -notmuch_status_t +> -_notmuch_message_remove_filename (notmuch_message_t *message, +> - const char *filename) +> +static void +> +_notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix) +> { +> - const char *direntry_prefix = _find_prefix ("file-direntry"); +> - int direntry_prefix_len = strlen (direntry_prefix); +> - const char *folder_prefix = _find_prefix ("folder"); +> - int folder_prefix_len = strlen (folder_prefix); +> - void *local = talloc_new (message); +> - char *zfolder_prefix = talloc_asprintf(local, "Z%s", folder_prefix); +> - int zfolder_prefix_len = strlen (zfolder_prefix); +> - char *direntry; +> - notmuch_private_status_t private_status; +> - notmuch_status_t status; +> - Xapian::TermIterator i, last; +> - +> - status = _notmuch_database_filename_to_direntry ( +> - local, message->notmuch, filename, NOTMUCH_FIND_LOOKUP, &direntry); +> - if (status || !direntry) +> - return status; +> + Xapian::TermIterator i; +> + size_t prefix_len = strlen (prefix); +> +> - /* Unlink this file from its parent directory. */ +> - private_status = _notmuch_message_remove_term (message, +> - "file-direntry", direntry); +> - status = COERCE_STATUS (private_status, +> - "Unexpected error from _notmuch_message_remove_term"); +> - if (status) +> - return status; +> - +> - /* Re-synchronize "folder:" terms for this message. This requires: +> - * 1. removing all "folder:" terms +> - * 2. removing all "folder:" stemmed terms +> - * 3. adding back terms for all remaining filenames of the message. */ +> - +> - /* 1. removing all "folder:" terms */ +> while (1) { +> i = message->doc.termlist_begin (); +> - i.skip_to (folder_prefix); +> + i.skip_to (prefix); +> +> /* Terminate loop when no terms remain with desired prefix. */ +> if (i == message->doc.termlist_end () || +> - strncmp ((*i).c_str (), folder_prefix, folder_prefix_len)) +> - { +> + strncmp ((*i).c_str (), prefix, prefix_len)) +> break; +> - } +> - +> - try { +> - message->doc.remove_term ((*i)); +> - } catch (const Xapian::InvalidArgumentError) { +> - /* Ignore failure to remove non-existent term. */ +> - } +> - } +> - +> - /* 2. removing all "folder:" stemmed terms */ +> - while (1) { +> - i = message->doc.termlist_begin (); +> - i.skip_to (zfolder_prefix); +> - +> - /* Terminate loop when no terms remain with desired prefix. */ +> - if (i == message->doc.termlist_end () || +> - strncmp ((*i).c_str (), zfolder_prefix, zfolder_prefix_len)) +> - { +> - break; +> - } +> +> try { +> message->doc.remove_term ((*i)); +> @@ -595,12 +533,18 @@ _notmuch_message_remove_filename (notmuch_message_t *message, +> /* Ignore failure to remove non-existent term. */ +> } +> } +> +} +> +> - /* 3. adding back terms for all remaining filenames of the message. */ +> - i = message->doc.termlist_begin (); +> - i.skip_to (direntry_prefix); +> +/* Add "folder:" terms for all filenames of the message. */ +> +static notmuch_status_t +> +_notmuch_message_add_folder_terms (void *ctx, notmuch_message_t *message) +> +{ +> + const char *direntry_prefix = _find_prefix ("file-direntry"); +> + int direntry_prefix_len = strlen (direntry_prefix); +> + Xapian::TermIterator i = message->doc.termlist_begin (); +> + notmuch_status_t status = NOTMUCH_STATUS_SUCCESS; +> +> - for (; i != message->doc.termlist_end (); i++) { +> + for (i.skip_to (direntry_prefix); i != message->doc.termlist_end (); i++) { +> unsigned int directory_id; +> const char *direntry, *directory; +> char *colon; +> @@ -620,18 +564,76 @@ _notmuch_message_remove_filename (notmuch_message_t *message, +> if (colon == NULL || *colon != ':') +> INTERNAL_ERROR ("malformed direntry"); +> +> - directory = _notmuch_database_get_directory_path (local, +> + directory = _notmuch_database_get_directory_path (ctx, +> message->notmuch, +> directory_id); +> - if (strlen (directory)) +> - _notmuch_message_gen_terms (message, "folder", directory); +> + _notmuch_message_add_term (message, "folder", directory); +> } +> +> + return status; +> +} +> + +> +/* Remove a particular 'filename' from 'message'. +> + * +> + * This change will not be reflected in the database until the next +> + * call to _notmuch_message_sync. +> + * +> + * If this message still has other filenames, returns +> + * NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID. +> + * +> + * Note: This function does not remove a document from the database, +> + * even if the specified filename is the only filename for this +> + * message. For that functionality, see +> + * _notmuch_database_remove_message. */ +> +notmuch_status_t +> +_notmuch_message_remove_filename (notmuch_message_t *message, +> + const char *filename) +> +{ +> + void *local = talloc_new (message); +> + char *direntry; +> + notmuch_private_status_t private_status; +> + notmuch_status_t status; +> + +> + status = _notmuch_database_filename_to_direntry ( +> + local, message->notmuch, filename, NOTMUCH_FIND_LOOKUP, &direntry); +> + if (status || !direntry) +> + return status; +> + +> + /* Unlink this file from its parent directory. */ +> + private_status = _notmuch_message_remove_term (message, +> + "file-direntry", direntry); +> + status = COERCE_STATUS (private_status, +> + "Unexpected error from _notmuch_message_remove_term"); +> + if (status) +> + return status; +> + +> + /* Remove all "folder:" terms from the message. */ +> + _notmuch_message_remove_terms (message, _find_prefix ("folder")); +> + +> + /* Add back "folder:" terms for all remaining filenames of the message. */ +> + status = _notmuch_message_add_folder_terms (local, message); +> + +> talloc_free (local); +> +> return status; +> } +> +> +/* Upgrade the "folder:" prefix from V1 to V2. */ +> +#define FOLDER_PREFIX_V1 "XFOLDER" +> +#define ZFOLDER_PREFIX_V1 "Z" FOLDER_PREFIX_V1 +> +void +> +_notmuch_message_upgrade_folder (notmuch_message_t *message) +> +{ +> + /* Remove all old "folder:" terms. */ +> + _notmuch_message_remove_terms (message, FOLDER_PREFIX_V1); +> + +> + /* Remove all old "folder:" stemmed terms. */ +> + _notmuch_message_remove_terms (message, ZFOLDER_PREFIX_V1); +> + +> + /* Add new boolean "folder:" terms. */ +> + _notmuch_message_add_folder_terms (message, message); +> +} +> + +> char * +> _notmuch_message_talloc_copy_data (notmuch_message_t *message) +> { +> diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h +> index af185c7..59eb2bc 100644 +> --- a/lib/notmuch-private.h +> +++ b/lib/notmuch-private.h +> @@ -263,6 +263,9 @@ _notmuch_message_gen_terms (notmuch_message_t *message, +> void +> _notmuch_message_upgrade_filename_storage (notmuch_message_t *message); +> +> +void +> +_notmuch_message_upgrade_folder (notmuch_message_t *message); +> + +> notmuch_status_t +> _notmuch_message_add_filename (notmuch_message_t *message, +> const char *filename); +> -- +> 1.8.5.2 +> +> _______________________________________________ +> notmuch mailing list +> notmuch@notmuchmail.org +> http://notmuchmail.org/mailman/listinfo/notmuch -- 2.26.2