From 07b0659ed3d7c414cfeacdab7d6f61e9779255c9 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Sun, 9 Mar 2014 23:40:27 +0200 Subject: [PATCH] [PATCH v4 06/13] lib: make folder: prefix literal --- e6/b86d6a43159508f0ccd9d2d14e8ead0ef6e473 | 377 ++++++++++++++++++++++ 1 file changed, 377 insertions(+) create mode 100644 e6/b86d6a43159508f0ccd9d2d14e8ead0ef6e473 diff --git a/e6/b86d6a43159508f0ccd9d2d14e8ead0ef6e473 b/e6/b86d6a43159508f0ccd9d2d14e8ead0ef6e473 new file mode 100644 index 000000000..3e44bdb58 --- /dev/null +++ b/e6/b86d6a43159508f0ccd9d2d14e8ead0ef6e473 @@ -0,0 +1,377 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id 1AC7C431FC2 + for ; Sun, 9 Mar 2014 14:41:24 -0700 (PDT) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Amavis-Alert: BAD HEADER SECTION, Duplicate header field: "References" +X-Spam-Flag: NO +X-Spam-Score: -0.7 +X-Spam-Level: +X-Spam-Status: No, score=-0.7 tagged_above=-999 required=5 + tests=[RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id siLmyBakeKKZ for ; + Sun, 9 Mar 2014 14:41:17 -0700 (PDT) +Received: from mail-la0-f53.google.com (mail-la0-f53.google.com + [209.85.215.53]) (using TLSv1 with cipher RC4-SHA (128/128 bits)) + (No client certificate requested) + by olra.theworths.org (Postfix) with ESMTPS id CF652431FBC + for ; Sun, 9 Mar 2014 14:40:53 -0700 (PDT) +Received: by mail-la0-f53.google.com with SMTP id b8so4086318lan.26 + for ; Sun, 09 Mar 2014 14:40:52 -0700 (PDT) +X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=1e100.net; s=20130820; + h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to + :references:in-reply-to:references; + bh=5P7cWv0MKfmrzhA3hqpPcC8poLNJT/d6bD3EPFCE+vo=; + b=TEaTWEB9KEE3hUkitsQ7yOipy7SwdEfThMDtdkUIMVDLN+nHLMqt1Ie7sL+CgFn3Rc + IRflk5iE3O0d5cIC0CWg3HkjevpThbAV4KJ53h6pHPgaCl7dYcReBp+Jd4uQXSbG5OQB + kYfI03C2cP64DGJuGNrPBPXFlLYaPmqSyk/nafnkmY4S3xvt6Zdrp2YWGOhHvERmtiZD + eCjkxG2yw53SZchF0CQmyNGZO++ibWYV49MXb+czsZezIHSeOFU2YjZQt/n0fqlLHuuw + gbVZStihYX2/14Oo0mCVZC6kwUKj2tGHarXwj0Uj7l4OZljGO0tV2uGW0gkYupLFKn5B + O+tQ== +X-Gm-Message-State: + ALoCoQnysQMx5s32H4YjQA6y0vwlMzFr7LOcWvfwPPTkJROheu48cDR3/CrEc18KPHbeBpJBw0Lv +X-Received: by 10.112.137.5 with SMTP id qe5mr20039501lbb.16.1394401252404; + Sun, 09 Mar 2014 14:40:52 -0700 (PDT) +Received: from localhost (dsl-hkibrasgw2-58c36f-91.dhcp.inet.fi. + [88.195.111.91]) + by mx.google.com with ESMTPSA id i7sm26898996lai.5.2014.03.09.14.40.50 + for + (version=TLSv1.2 cipher=RC4-SHA bits=128/128); + Sun, 09 Mar 2014 14:40:51 -0700 (PDT) +From: Jani Nikula +To: notmuch@notmuchmail.org +Subject: [PATCH v4 06/13] lib: make folder: prefix literal +Date: Sun, 9 Mar 2014 23:40:27 +0200 +Message-Id: + <9640c403666ae291e71c22f252eb05f606b14a5e.1394400503.git.jani@nikula.org> +X-Mailer: git-send-email 1.9.0 +In-Reply-To: +References: +In-Reply-To: +References: +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Sun, 09 Mar 2014 21:41:24 -0000 + +In xapian terms, convert folder: prefix from probabilistic to boolean +prefix, matching the paths, relative from the maildir root, of the +message files, ignoring the maildir new and cur leaf directories. + +folder:foo matches all message files in foo, foo/new, and foo/cur. + +folder:foo/new does *not* match message files in foo/new. + +folder:"" matches all message files in the top level maildir and its +new and cur subdirectories. + +This change constitutes a database change: bump the database version +and add database upgrade support for folder: terms. The upgrade also +adds path: terms. + +Finally, fix the folder search test for literal folder: search, as +some of the folder: matching capabilities are lost in the +probabilistic to boolean prefix change. +--- + lib/database.cc | 44 ++++++++++++++++++++++-- + lib/message.cc | 80 +++++++++++++++++++++++++++++++++++++------ + lib/notmuch-private.h | 3 ++ + test/T100-search-by-folder.sh | 24 +++++++++++-- + 4 files changed, 135 insertions(+), 16 deletions(-) + +diff --git a/lib/database.cc b/lib/database.cc +index 93cc7f57e9db..aef748f75b5d 100644 +--- a/lib/database.cc ++++ b/lib/database.cc +@@ -42,7 +42,7 @@ typedef struct { + const char *prefix; + } prefix_t; + +-#define NOTMUCH_DATABASE_VERSION 1 ++#define NOTMUCH_DATABASE_VERSION 2 + + #define STRINGIFY(s) _SUB_STRINGIFY(s) + #define _SUB_STRINGIFY(s) #s +@@ -210,6 +210,13 @@ static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = { + { "is", "K" }, + { "id", "Q" }, + { "path", "P" }, ++ /* ++ * Without the ":", since this is a multi-letter prefix, Xapian ++ * will add a colon itself if the first letter of the path is ++ * upper-case ASCII. Including the ":" forces there to always be a ++ * colon, which keeps our own logic simpler. ++ */ ++ { "folder", "XFOLDER:" }, + }; + + static prefix_t PROBABILISTIC_PREFIX[]= { +@@ -217,7 +224,6 @@ static prefix_t PROBABILISTIC_PREFIX[]= { + { "to", "XTO" }, + { "attachment", "XATTACHMENT" }, + { "subject", "XSUBJECT"}, +- { "folder", "XFOLDER"} + }; + + const char * +@@ -1168,6 +1174,40 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, + } + } + ++ /* ++ * Prior to version 2, the "folder:" prefix was probabilistic and ++ * stemmed. Change it to the current boolean prefix. Add "path:" ++ * prefixes while at it. ++ */ ++ if (version < 2) { ++ notmuch_query_t *query = notmuch_query_create (notmuch, ""); ++ notmuch_messages_t *messages; ++ notmuch_message_t *message; ++ ++ count = 0; ++ total = notmuch_query_count_messages (query); ++ ++ for (messages = notmuch_query_search_messages (query); ++ notmuch_messages_valid (messages); ++ notmuch_messages_move_to_next (messages)) { ++ if (do_progress_notify) { ++ progress_notify (closure, (double) count / total); ++ do_progress_notify = 0; ++ } ++ ++ message = notmuch_messages_get (messages); ++ ++ _notmuch_message_upgrade_folder (message); ++ _notmuch_message_sync (message); ++ ++ notmuch_message_destroy (message); ++ ++ count++; ++ } ++ ++ notmuch_query_destroy (query); ++ } ++ + db->set_metadata ("version", STRINGIFY (NOTMUCH_DATABASE_VERSION)); + db->flush (); + +diff --git a/lib/message.cc b/lib/message.cc +index 21abe8e12b9d..9243b769d2b0 100644 +--- a/lib/message.cc ++++ b/lib/message.cc +@@ -504,6 +504,56 @@ _notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix) + } + } + ++/* Return true if p points at "new" or "cur". */ ++static bool is_maildir (const char *p) ++{ ++ return strcmp (p, "cur") == 0 || strcmp (p, "new") == 0; ++} ++ ++/* Add "folder:" term for directory. */ ++static notmuch_status_t ++_notmuch_message_add_folder_terms (notmuch_message_t *message, ++ const char *directory) ++{ ++ char *folder, *last; ++ ++ folder = talloc_strdup (NULL, directory); ++ if (! folder) ++ return NOTMUCH_STATUS_OUT_OF_MEMORY; ++ ++ /* ++ * If the message file is in a leaf directory named "new" or ++ * "cur", presume maildir and index the parent directory. Thus a ++ * "folder:" prefix search matches messages in the specified ++ * maildir folder, i.e. in the specified directory and its "new" ++ * and "cur" subdirectories. ++ * ++ * Note that this means the "folder:" prefix can't be used for ++ * distinguishing between message files in "new" or "cur". The ++ * "path:" prefix needs to be used for that. ++ * ++ * Note the deliberate difference to _filename_is_in_maildir(). We ++ * don't want to index different things depending on the existence ++ * or non-existence of all maildir sibling directories "new", ++ * "cur", and "tmp". Doing so would be surprising, and difficult ++ * for the user to fix in case all subdirectories were not in ++ * place during indexing. ++ */ ++ last = strrchr (folder, '/'); ++ if (last) { ++ if (is_maildir (last + 1)) ++ *last = '\0'; ++ } else if (is_maildir (folder)) { ++ *folder = '\0'; ++ } ++ ++ _notmuch_message_add_term (message, "folder", folder); ++ ++ talloc_free (folder); ++ ++ return NOTMUCH_STATUS_SUCCESS; ++} ++ + #define RECURSIVE_SUFFIX "/**" + + /* Add "path:" terms for directory. */ +@@ -570,9 +620,8 @@ _notmuch_message_add_directory_terms (void *ctx, notmuch_message_t *message) + directory = _notmuch_database_get_directory_path (ctx, + message->notmuch, + directory_id); +- if (strlen (directory)) +- _notmuch_message_gen_terms (message, "folder", directory); + ++ _notmuch_message_add_folder_terms (message, directory); + _notmuch_message_add_path_terms (message, directory); + } + +@@ -610,9 +659,7 @@ _notmuch_message_add_filename (notmuch_message_t *message, + * notmuch_directory_get_child_files() . */ + _notmuch_message_add_term (message, "file-direntry", direntry); + +- /* New terms allow user to search with folder: specification. */ +- _notmuch_message_gen_terms (message, "folder", directory); +- ++ _notmuch_message_add_folder_terms (message, directory); + _notmuch_message_add_path_terms (message, directory); + + talloc_free (local); +@@ -637,8 +684,6 @@ _notmuch_message_remove_filename (notmuch_message_t *message, + const char *filename) + { + void *local = talloc_new (message); +- const char *folder_prefix = _find_prefix ("folder"); +- char *zfolder_prefix = talloc_asprintf(local, "Z%s", folder_prefix); + char *direntry; + notmuch_private_status_t private_status; + notmuch_status_t status; +@@ -659,10 +704,7 @@ _notmuch_message_remove_filename (notmuch_message_t *message, + /* Re-synchronize "folder:" and "path:" terms for this message. */ + + /* Remove all "folder:" terms. */ +- _notmuch_message_remove_terms (message, folder_prefix); +- +- /* Remove all "folder:" stemmed terms. */ +- _notmuch_message_remove_terms (message, zfolder_prefix); ++ _notmuch_message_remove_terms (message, _find_prefix ("folder")); + + /* Remove all "path:" terms. */ + _notmuch_message_remove_terms (message, _find_prefix ("path")); +@@ -675,6 +717,22 @@ _notmuch_message_remove_filename (notmuch_message_t *message, + return status; + } + ++/* Upgrade the "folder:" prefix from V1 to V2. */ ++#define FOLDER_PREFIX_V1 "XFOLDER" ++#define ZFOLDER_PREFIX_V1 "Z" FOLDER_PREFIX_V1 ++void ++_notmuch_message_upgrade_folder (notmuch_message_t *message) ++{ ++ /* Remove all old "folder:" terms. */ ++ _notmuch_message_remove_terms (message, FOLDER_PREFIX_V1); ++ ++ /* Remove all old "folder:" stemmed terms. */ ++ _notmuch_message_remove_terms (message, ZFOLDER_PREFIX_V1); ++ ++ /* Add new boolean "folder:" and "path:" terms. */ ++ _notmuch_message_add_directory_terms (message, message); ++} ++ + char * + _notmuch_message_talloc_copy_data (notmuch_message_t *message) + { +diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h +index af185c7c5ba8..59eb2bc285a5 100644 +--- a/lib/notmuch-private.h ++++ b/lib/notmuch-private.h +@@ -263,6 +263,9 @@ _notmuch_message_gen_terms (notmuch_message_t *message, + void + _notmuch_message_upgrade_filename_storage (notmuch_message_t *message); + ++void ++_notmuch_message_upgrade_folder (notmuch_message_t *message); ++ + notmuch_status_t + _notmuch_message_add_filename (notmuch_message_t *message, + const char *filename); +diff --git a/test/T100-search-by-folder.sh b/test/T100-search-by-folder.sh +index 5cc2ca8d388a..a7f63dd1750c 100755 +--- a/test/T100-search-by-folder.sh ++++ b/test/T100-search-by-folder.sh +@@ -3,6 +3,7 @@ test_description='"notmuch search" by folder: (with variations)' + . ./test-lib.sh + + add_message '[dir]=bad' '[subject]="To the bone"' ++add_message '[dir]=.' '[subject]="Top level"' + add_message '[dir]=bad/news' '[subject]="Bears"' + mkdir -p "${MAIL_DIR}/duplicate/bad/news" + cp "$gen_msg_filename" "${MAIL_DIR}/duplicate/bad/news" +@@ -12,29 +13,46 @@ add_message '[dir]=things/favorite' '[subject]="Raindrops, whiskers, kettles"' + add_message '[dir]=things/bad' '[subject]="Bites, stings, sad feelings"' + + test_begin_subtest "Single-world folder: specification (multiple results)" +-output=$(notmuch search folder:bad | notmuch_search_sanitize) ++output=$(notmuch search folder:bad folder:bad/news folder:things/bad | notmuch_search_sanitize) + test_expect_equal "$output" "thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; To the bone (inbox unread) + thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; Bears (inbox unread) + thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; Bites, stings, sad feelings (inbox unread)" + ++test_begin_subtest "Top level folder" ++output=$(notmuch search folder:'""' | notmuch_search_sanitize) ++test_expect_equal "$output" "thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; Top level (inbox unread)" ++ + test_begin_subtest "Two-word path to narrow results to one" + output=$(notmuch search folder:bad/news | notmuch_search_sanitize) + test_expect_equal "$output" "thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; Bears (inbox unread)" + ++test_begin_subtest "Folder search with --output=files" ++output=$(notmuch search --output=files folder:bad/news | notmuch_search_files_sanitize) ++test_expect_equal "$output" "MAIL_DIR/bad/news/msg-003 ++MAIL_DIR/duplicate/bad/news/msg-003" ++ + test_begin_subtest "After removing duplicate instance of matching path" + rm -r "${MAIL_DIR}/bad/news" + notmuch new + output=$(notmuch search folder:bad/news | notmuch_search_sanitize) ++test_expect_equal "$output" "" ++ ++test_begin_subtest "Folder search with --output=files part #2" ++output=$(notmuch search --output=files folder:duplicate/bad/news | notmuch_search_files_sanitize) ++test_expect_equal "$output" "MAIL_DIR/duplicate/bad/news/msg-003" ++ ++test_begin_subtest "After removing duplicate instance of matching path part #2" ++output=$(notmuch search folder:duplicate/bad/news | notmuch_search_sanitize) + test_expect_equal "$output" "thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; Bears (inbox unread)" + + test_begin_subtest "After rename, old path returns nothing" + mv "${MAIL_DIR}/duplicate/bad/news" "${MAIL_DIR}/duplicate/bad/olds" + notmuch new +-output=$(notmuch search folder:bad/news | notmuch_search_sanitize) ++output=$(notmuch search folder:duplicate/bad/news | notmuch_search_sanitize) + test_expect_equal "$output" "" + + test_begin_subtest "After rename, new path returns result" +-output=$(notmuch search folder:bad/olds | notmuch_search_sanitize) ++output=$(notmuch search folder:duplicate/bad/olds | notmuch_search_sanitize) + test_expect_equal "$output" "thread:XXX 2001-01-05 [1/1] Notmuch Test Suite; Bears (inbox unread)" + + test_done +-- +1.9.0 + -- 2.26.2