Return-Path: <jani@nikula.org>
X-Original-To: notmuch@notmuchmail.org
Delivered-To: notmuch@notmuchmail.org
Received: from localhost (localhost [127.0.0.1])
	by olra.theworths.org (Postfix) with ESMTP id A4CE4431FC2
	for <notmuch@notmuchmail.org>; Thu,  9 Jan 2014 23:10:10 -0800 (PST)
X-Virus-Scanned: Debian amavisd-new at olra.theworths.org
X-Amavis-Alert: BAD HEADER SECTION, Duplicate header field: "References"
X-Spam-Flag: NO
X-Spam-Score: -0.7
X-Spam-Level: 
X-Spam-Status: No, score=-0.7 tagged_above=-999 required=5
	tests=[RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled
Received: from olra.theworths.org ([127.0.0.1])
	by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024)
	with ESMTP id 5ll8-SHA9A3P for <notmuch@notmuchmail.org>;
	Thu,  9 Jan 2014 23:10:01 -0800 (PST)
Received: from mail-ee0-f44.google.com (mail-ee0-f44.google.com
 [74.125.83.44])	(using TLSv1 with cipher RC4-SHA (128/128 bits))	(No client
 certificate requested)	by olra.theworths.org (Postfix) with ESMTPS id
 70E7C431FBC	for <notmuch@notmuchmail.org>; Thu,  9 Jan 2014 23:10:01 -0800
 (PST)
Received: by mail-ee0-f44.google.com with SMTP id b57so1710091eek.31
	for <notmuch@notmuchmail.org>; Thu, 09 Jan 2014 23:10:00 -0800 (PST)
X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
	d=1e100.net; s=20130820;
	h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to
	:references:in-reply-to:references;
	bh=3FgORGMngT3wLdUYKi8bcXxYsruTf6a9yKVojvc8/vo=;
	b=f8HrP5XqD1PbO2YpvcRPNi9ms6NTLhL1aUOSJEWfK6EcYWhAUgvoFn+kOhhPeOnYS3
	9ckwYE0JYUOJ3yaeseulLnHMVsUwxRHXdWlVHz5iqwNrTv5x8MdWOwcHAolGIUHjjbQM
	1b9XhYhARz4/oFO1uF8rKJqNscgroUYKXyAW58Y3387CTn/qRZ70JCsIaNd2kqcVp0Re
	sLuaCq7rs3dgDtZqwsnzMj8qGMmwCW5fOi3TNMS9/nRqYDwxnOlGXu3TuwRIEIk5VITW
	WQ0EcLnfejMDV7F2ztSJT5IepV6m6a6lH6CzU5lF5ALYiQryOiN1Q2h0XbptYczxF8Ip
	WuFg==
X-Gm-Message-State:
 ALoCoQmUrSsl3Iwe2SDOR2RsXMCN2wMICyS1wguyW1Oyr+08W0iLmlvieTclfs36s46TIvOMDT/J
X-Received: by 10.15.45.67 with SMTP id a43mr5624820eew.17.1389305929624;
	Thu, 09 Jan 2014 14:18:49 -0800 (PST)
Received: from localhost (dsl-hkibrasgw2-58c36f-91.dhcp.inet.fi.
	[88.195.111.91])
	by mx.google.com with ESMTPSA id a51sm9382988eeh.8.2014.01.09.14.18.47
	for <multiple recipients>
	(version=TLSv1.2 cipher=RC4-SHA bits=128/128);
	Thu, 09 Jan 2014 14:18:49 -0800 (PST)
From: Jani Nikula <jani@nikula.org>
To: notmuch@notmuchmail.org
Subject: [PATCH 1/5] lib: make folder: prefix literal
Date: Fri, 10 Jan 2014 00:18:32 +0200
Message-Id:
 <d735583dd1bb48a87530fe3d52a57abd4d17acb8.1389304779.git.jani@nikula.org>
X-Mailer: git-send-email 1.8.5.2
In-Reply-To: <cover.1389304779.git.jani@nikula.org>
References: <cover.1389304779.git.jani@nikula.org>
In-Reply-To: <cover.1389304779.git.jani@nikula.org>
References: <cover.1389304779.git.jani@nikula.org>
X-BeenThere: notmuch@notmuchmail.org
X-Mailman-Version: 2.1.13
Precedence: list
List-Id: "Use and development of the notmuch mail system."
	<notmuch.notmuchmail.org>
List-Unsubscribe: <http://notmuchmail.org/mailman/options/notmuch>,
	<mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>
List-Archive: <http://notmuchmail.org/pipermail/notmuch>
List-Post: <mailto:notmuch@notmuchmail.org>
List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>
List-Subscribe: <http://notmuchmail.org/mailman/listinfo/notmuch>,
	<mailto:notmuch-request@notmuchmail.org?subject=subscribe>
X-List-Received-Date: Fri, 10 Jan 2014 07:10:10 -0000

In xapian terms, convert folder: prefix from probabilistic to boolean
prefix. This change constitutes a database change: bump the database
version and add database upgrade support.
---
 lib/database.cc       |  39 ++++++++++++-
 lib/message.cc        | 154 +++++++++++++++++++++++++-------------------------
 lib/notmuch-private.h |   3 +
 3 files changed, 117 insertions(+), 79 deletions(-)

diff --git a/lib/database.cc b/lib/database.cc
index f395061..145fd66 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -42,7 +42,7 @@ typedef struct {
     const char *prefix;
 } prefix_t;
 
-#define NOTMUCH_DATABASE_VERSION 1
+#define NOTMUCH_DATABASE_VERSION 2
 
 #define STRINGIFY(s) _SUB_STRINGIFY(s)
 #define _SUB_STRINGIFY(s) #s
@@ -208,7 +208,8 @@ static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
     { "thread",			"G" },
     { "tag",			"K" },
     { "is",			"K" },
-    { "id",			"Q" }
+    { "id",			"Q" },
+    { "folder",			"P" },
 };
 
 static prefix_t PROBABILISTIC_PREFIX[]= {
@@ -216,7 +217,6 @@ static prefix_t PROBABILISTIC_PREFIX[]= {
     { "to",			"XTO" },
     { "attachment",		"XATTACHMENT" },
     { "subject",		"XSUBJECT"},
-    { "folder",			"XFOLDER"}
 };
 
 const char *
@@ -1167,6 +1167,39 @@ notmuch_database_upgrade (notmuch_database_t *notmuch,
 	}
     }
 
+    /*
+     * Prior to version 2, the "folder:" prefix was probabilistic and
+     * stemmed. Change it to the current boolean prefix.
+     */
+    if (version < 2) {
+	notmuch_query_t *query = notmuch_query_create (notmuch, "");
+	notmuch_messages_t *messages;
+	notmuch_message_t *message;
+
+	count = 0;
+	total = notmuch_query_count_messages (query);
+
+	for (messages = notmuch_query_search_messages (query);
+	     notmuch_messages_valid (messages);
+	     notmuch_messages_move_to_next (messages)) {
+	    if (do_progress_notify) {
+		progress_notify (closure, (double) count / total);
+		do_progress_notify = 0;
+	    }
+
+	    message = notmuch_messages_get (messages);
+
+	    _notmuch_message_upgrade_folder (message);
+	    _notmuch_message_sync (message);
+
+	    notmuch_message_destroy (message);
+
+	    count++;
+	}
+
+	notmuch_query_destroy (query);
+    }
+
     db->set_metadata ("version", STRINGIFY (NOTMUCH_DATABASE_VERSION));
     db->flush ();
 
diff --git a/lib/message.cc b/lib/message.cc
index 1b46379..500aa26 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -505,89 +505,27 @@ _notmuch_message_add_filename (notmuch_message_t *message,
     _notmuch_message_add_term (message, "file-direntry", direntry);
 
     /* New terms allow user to search with folder: specification. */
-    _notmuch_message_gen_terms (message, "folder", directory);
+    _notmuch_message_add_term (message, "folder", directory);
 
     talloc_free (local);
 
     return NOTMUCH_STATUS_SUCCESS;
 }
 
-/* Remove a particular 'filename' from 'message'.
- *
- * This change will not be reflected in the database until the next
- * call to _notmuch_message_sync.
- *
- * If this message still has other filenames, returns
- * NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID.
- *
- * Note: This function does not remove a document from the database,
- * even if the specified filename is the only filename for this
- * message. For that functionality, see
- * _notmuch_database_remove_message. */
-notmuch_status_t
-_notmuch_message_remove_filename (notmuch_message_t *message,
-				  const char *filename)
+static void
+_notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix)
 {
-    const char *direntry_prefix = _find_prefix ("file-direntry");
-    int direntry_prefix_len = strlen (direntry_prefix);
-    const char *folder_prefix = _find_prefix ("folder");
-    int folder_prefix_len = strlen (folder_prefix);
-    void *local = talloc_new (message);
-    char *zfolder_prefix = talloc_asprintf(local, "Z%s", folder_prefix);
-    int zfolder_prefix_len = strlen (zfolder_prefix);
-    char *direntry;
-    notmuch_private_status_t private_status;
-    notmuch_status_t status;
-    Xapian::TermIterator i, last;
-
-    status = _notmuch_database_filename_to_direntry (
-	local, message->notmuch, filename, NOTMUCH_FIND_LOOKUP, &direntry);
-    if (status || !direntry)
-	return status;
+    Xapian::TermIterator i;
+    size_t prefix_len = strlen (prefix);
 
-    /* Unlink this file from its parent directory. */
-    private_status = _notmuch_message_remove_term (message,
-						   "file-direntry", direntry);
-    status = COERCE_STATUS (private_status,
-			    "Unexpected error from _notmuch_message_remove_term");
-    if (status)
-	return status;
-
-    /* Re-synchronize "folder:" terms for this message. This requires:
-     *  1. removing all "folder:" terms
-     *  2. removing all "folder:" stemmed terms
-     *  3. adding back terms for all remaining filenames of the message. */
-
-    /* 1. removing all "folder:" terms */
     while (1) {
 	i = message->doc.termlist_begin ();
-	i.skip_to (folder_prefix);
+	i.skip_to (prefix);
 
 	/* Terminate loop when no terms remain with desired prefix. */
 	if (i == message->doc.termlist_end () ||
-	    strncmp ((*i).c_str (), folder_prefix, folder_prefix_len))
-	{
+	    strncmp ((*i).c_str (), prefix, prefix_len))
 	    break;
-	}
-
-	try {
-	    message->doc.remove_term ((*i));
-	} catch (const Xapian::InvalidArgumentError) {
-	    /* Ignore failure to remove non-existent term. */
-	}
-    }
-
-    /* 2. removing all "folder:" stemmed terms */
-    while (1) {
-	i = message->doc.termlist_begin ();
-	i.skip_to (zfolder_prefix);
-
-	/* Terminate loop when no terms remain with desired prefix. */
-	if (i == message->doc.termlist_end () ||
-	    strncmp ((*i).c_str (), zfolder_prefix, zfolder_prefix_len))
-	{
-	    break;
-	}
 
 	try {
 	    message->doc.remove_term ((*i));
@@ -595,12 +533,18 @@ _notmuch_message_remove_filename (notmuch_message_t *message,
 	    /* Ignore failure to remove non-existent term. */
 	}
     }
+}
 
-    /* 3. adding back terms for all remaining filenames of the message. */
-    i = message->doc.termlist_begin ();
-    i.skip_to (direntry_prefix);
+/* Add "folder:" terms for all filenames of the message. */
+static notmuch_status_t
+_notmuch_message_add_folder_terms (void *ctx, notmuch_message_t *message)
+{
+    const char *direntry_prefix = _find_prefix ("file-direntry");
+    int direntry_prefix_len = strlen (direntry_prefix);
+    Xapian::TermIterator i = message->doc.termlist_begin ();
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
 
-    for (; i != message->doc.termlist_end (); i++) {
+    for (i.skip_to (direntry_prefix); i != message->doc.termlist_end (); i++) {
 	unsigned int directory_id;
 	const char *direntry, *directory;
 	char *colon;
@@ -620,18 +564,76 @@ _notmuch_message_remove_filename (notmuch_message_t *message,
 	if (colon == NULL || *colon != ':')
 	    INTERNAL_ERROR ("malformed direntry");
 
-	directory = _notmuch_database_get_directory_path (local,
+	directory = _notmuch_database_get_directory_path (ctx,
 							  message->notmuch,
 							  directory_id);
-	if (strlen (directory))
-	    _notmuch_message_gen_terms (message, "folder", directory);
+	_notmuch_message_add_term (message, "folder", directory);
     }
 
+    return status;
+}
+
+/* Remove a particular 'filename' from 'message'.
+ *
+ * This change will not be reflected in the database until the next
+ * call to _notmuch_message_sync.
+ *
+ * If this message still has other filenames, returns
+ * NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID.
+ *
+ * Note: This function does not remove a document from the database,
+ * even if the specified filename is the only filename for this
+ * message. For that functionality, see
+ * _notmuch_database_remove_message. */
+notmuch_status_t
+_notmuch_message_remove_filename (notmuch_message_t *message,
+				  const char *filename)
+{
+    void *local = talloc_new (message);
+    char *direntry;
+    notmuch_private_status_t private_status;
+    notmuch_status_t status;
+
+    status = _notmuch_database_filename_to_direntry (
+	local, message->notmuch, filename, NOTMUCH_FIND_LOOKUP, &direntry);
+    if (status || !direntry)
+	return status;
+
+    /* Unlink this file from its parent directory. */
+    private_status = _notmuch_message_remove_term (message,
+						   "file-direntry", direntry);
+    status = COERCE_STATUS (private_status,
+			    "Unexpected error from _notmuch_message_remove_term");
+    if (status)
+	return status;
+
+    /* Remove all "folder:" terms from the message. */
+    _notmuch_message_remove_terms (message, _find_prefix ("folder"));
+
+    /* Add back "folder:" terms for all remaining filenames of the message. */
+    status = _notmuch_message_add_folder_terms (local, message);
+
     talloc_free (local);
 
     return status;
 }
 
+/* Upgrade the "folder:" prefix from V1 to V2. */
+#define FOLDER_PREFIX_V1	"XFOLDER"
+#define ZFOLDER_PREFIX_V1	"Z" FOLDER_PREFIX_V1
+void
+_notmuch_message_upgrade_folder (notmuch_message_t *message)
+{
+    /* Remove all old "folder:" terms. */
+    _notmuch_message_remove_terms (message, FOLDER_PREFIX_V1);
+
+    /* Remove all old "folder:" stemmed terms. */
+    _notmuch_message_remove_terms (message, ZFOLDER_PREFIX_V1);
+
+    /* Add new boolean "folder:" terms. */
+    _notmuch_message_add_folder_terms (message, message);
+}
+
 char *
 _notmuch_message_talloc_copy_data (notmuch_message_t *message)
 {
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index af185c7..59eb2bc 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -263,6 +263,9 @@ _notmuch_message_gen_terms (notmuch_message_t *message,
 void
 _notmuch_message_upgrade_filename_storage (notmuch_message_t *message);
 
+void
+_notmuch_message_upgrade_folder (notmuch_message_t *message);
+
 notmuch_status_t
 _notmuch_message_add_filename (notmuch_message_t *message,
 			       const char *filename);
-- 
1.8.5.2