[PATCH v3 3/7] fix thread breakage via ghost-on-removal

diff --git a/dd/8289a5693b7461bf5ddb53362f6d5b9c967873 b/dd/8289a5693b7461bf5ddb53362f6d5b9c967873
new file mode 100644 (file)
index 0000000..f2249f4
--- /dev/null
+++ b/dd/8289a5693b7461bf5ddb53362f6d5b9c967873
@@ -0,0 +1,167 @@
+Return-Path: <dkg@fifthhorseman.net>\r
+X-Original-To: notmuch@notmuchmail.org\r
+Delivered-To: notmuch@notmuchmail.org\r
+Received: from localhost (localhost [127.0.0.1])\r
+ by arlo.cworth.org (Postfix) with ESMTP id D7A216DE02C6\r
+ for <notmuch@notmuchmail.org>; Fri,  8 Apr 2016 18:02:54 -0700 (PDT)\r
+X-Virus-Scanned: Debian amavisd-new at cworth.org\r
+X-Spam-Flag: NO\r
+X-Spam-Score: 0\r
+X-Spam-Level: \r
+X-Spam-Status: No, score=0 tagged_above=-999 required=5 tests=[none]\r
+ autolearn=disabled\r
+Received: from arlo.cworth.org ([127.0.0.1])\r
+ by localhost (arlo.cworth.org [127.0.0.1]) (amavisd-new, port 10024)\r
+ with ESMTP id x8hPoN1orWTA for <notmuch@notmuchmail.org>;\r
+ Fri,  8 Apr 2016 18:02:47 -0700 (PDT)\r
+Received: from che.mayfirst.org (che.mayfirst.org [209.234.253.108])\r
+ by arlo.cworth.org (Postfix) with ESMTP id C67FA6DE02CD\r
+ for <notmuch@notmuchmail.org>; Fri,  8 Apr 2016 18:02:38 -0700 (PDT)\r
+Received: from fifthhorseman.net (unknown [201.140.212.132])\r
+ by che.mayfirst.org (Postfix) with ESMTPSA id 89E7910080\r
+ for <notmuch@notmuchmail.org>; Fri,  8 Apr 2016 21:02:36 -0400 (EDT)\r
+Received: by fifthhorseman.net (Postfix, from userid 1000)\r
+ id F2EDF2003D; Fri,  8 Apr 2016 22:02:34 -0300 (ART)\r
+From: Daniel Kahn Gillmor <dkg@fifthhorseman.net>\r
+To: Notmuch Mail <notmuch@notmuchmail.org>\r
+Subject: [PATCH v3 3/7] fix thread breakage via ghost-on-removal\r
+Date: Fri,  8 Apr 2016 22:02:30 -0300\r
+Message-Id: <1460163754-22994-3-git-send-email-dkg@fifthhorseman.net>\r
+X-Mailer: git-send-email 2.8.0.rc3\r
+In-Reply-To: <1460163754-22994-1-git-send-email-dkg@fifthhorseman.net>\r
+References: <1459445693-3900-1-git-send-email-dkg@fifthhorseman.net>\r
+ <1460163754-22994-1-git-send-email-dkg@fifthhorseman.net>\r
+X-BeenThere: notmuch@notmuchmail.org\r
+X-Mailman-Version: 2.1.20\r
+Precedence: list\r
+List-Id: "Use and development of the notmuch mail system."\r
+ <notmuch.notmuchmail.org>\r
+List-Unsubscribe: <https://notmuchmail.org/mailman/options/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>\r
+List-Archive: <http://notmuchmail.org/pipermail/notmuch/>\r
+List-Post: <mailto:notmuch@notmuchmail.org>\r
+List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>\r
+List-Subscribe: <https://notmuchmail.org/mailman/listinfo/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=subscribe>\r
+X-List-Received-Date: Sat, 09 Apr 2016 01:02:54 -0000\r
+\r
+implement ghost-on-removal, the solution to T590-thread-breakage.sh\r
+that just adds a ghost message after removing each message.\r
+\r
+It leaks information about whether we've ever seen a given message id,\r
+but it's a fairly simple implementation.\r
+\r
+Note that _resolve_message_id_to_thread_id already introduces new\r
+message_ids to the database, so i think just searching for a given\r
+message ID may introduce the same metadata leakage.\r
+---\r
+ lib/message.cc               | 30 +++++++++++++++++++++++++++---\r
+ test/T590-thread-breakage.sh | 25 ++++++++++++-------------\r
+ 2 files changed, 39 insertions(+), 16 deletions(-)\r
+\r
+diff --git a/lib/message.cc b/lib/message.cc\r
+index 8d72ea2..435b78a 100644\r
+--- a/lib/message.cc\r
++++ b/lib/message.cc\r
+@@ -1037,20 +1037,44 @@ _notmuch_message_sync (notmuch_message_t *message)\r
+     message->modified = FALSE;\r
+ }\r
+ \r
+-/* Delete a message document from the database. */\r
++/* Delete a message document from the database, leaving a ghost\r
++ * message in its place */\r
+ notmuch_status_t\r
+ _notmuch_message_delete (notmuch_message_t *message)\r
+ {\r
+     notmuch_status_t status;\r
+     Xapian::WritableDatabase *db;\r
++    const char *mid, *tid;\r
++    notmuch_message_t *ghost;\r
++    notmuch_private_status_t private_status;\r
++    notmuch_database_t *notmuch;\r
++          \r
++    mid = notmuch_message_get_message_id (message);\r
++    tid = notmuch_message_get_thread_id (message);\r
++    notmuch = message->notmuch;\r
+ \r
+     status = _notmuch_database_ensure_writable (message->notmuch);\r
+     if (status)\r
+       return status;\r
+ \r
+-    db = static_cast <Xapian::WritableDatabase *> (message->notmuch->xapian_db);\r
++    db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);\r
+     db->delete_document (message->doc_id);\r
+-    return NOTMUCH_STATUS_SUCCESS;\r
++          \r
++    /* and reintroduce a ghost in its place */\r
++    ghost = _notmuch_message_create_for_message_id (notmuch, mid, &private_status);\r
++    if (private_status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {\r
++      private_status = _notmuch_message_initialize_ghost (ghost, tid);\r
++      if (! private_status)\r
++          _notmuch_message_sync (ghost);\r
++    } else if (private_status == NOTMUCH_PRIVATE_STATUS_SUCCESS) {\r
++      /* this is deeply weird, and we should not have gotten into\r
++         this state.  is there a better error message to return\r
++         here? */\r
++      return NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;\r
++    }\r
++\r
++    notmuch_message_destroy (ghost);\r
++    return COERCE_STATUS (private_status, "Error converting to ghost message");\r
+ }\r
+ \r
+ /* Transform a blank message into a ghost message.  The caller must\r
+diff --git a/test/T590-thread-breakage.sh b/test/T590-thread-breakage.sh\r
+index 24ffb42..22e5cbc 100755\r
+--- a/test/T590-thread-breakage.sh\r
++++ b/test/T590-thread-breakage.sh\r
+@@ -96,20 +96,11 @@ notmuch new >/dev/null\r
+ test_thread_count 1 'First message removed: still only one thread'\r
+ test_subject_count apple 0\r
+ test_subject_count banana 1\r
+-test_begin_subtest 'should be one ghost after first message removed'\r
+-test_subtest_known_broken\r
+-ghosts=$(../ghost-report ${MAIL_DIR}/.notmuch/xapian)\r
+-test_expect_equal "$ghosts" "1"\r
++test_ghost_count 1 'should be one ghost after first message removed'\r
+ \r
+ message_a\r
+ notmuch new >/dev/null\r
+-# this is known to fail (it shows 2 threads) because no "ghost\r
+-# message" was created for message A when it was removed from the\r
+-# index, despite message B still pointing to it.\r
+-test_begin_subtest 'First message reappears: should return to the same thread'\r
+-test_subtest_known_broken\r
+-count=$(notmuch count --output=threads)\r
+-test_expect_equal "$count" "1"\r
++test_thread_count 1 'First message reappears: should return to the same thread'\r
+ test_subject_count apple 1\r
+ test_subject_count banana 1\r
+ test_ghost_count 0\r
+@@ -119,13 +110,21 @@ notmuch new >/dev/null\r
+ test_thread_count 1 'Removing second message: still only one thread'\r
+ test_subject_count apple 1\r
+ test_subject_count banana 0\r
+-test_ghost_count 0 'No ghosts should remain after deletion of second message'\r
++test_begin_subtest 'No ghosts should remain after deletion of second message'\r
++# this is known to fail; we are leaking ghost messages deliberately\r
++test_subtest_known_broken\r
++ghosts=$(../ghost-report ${MAIL_DIR}/.notmuch/xapian)\r
++test_expect_equal "$ghosts" "0"\r
+ \r
+ rm -f ${MAIL_DIR}/cur/a\r
+ notmuch new >/dev/null\r
+ test_thread_count 0 'All messages gone: no threads'\r
+ test_subject_count apple 0\r
+ test_subject_count banana 0\r
+-test_ghost_count 0\r
++test_begin_subtest 'No ghosts should remain after full thread deletion'\r
++# this is known to fail; we are leaking ghost messages deliberately\r
++test_subtest_known_broken\r
++ghosts=$(../ghost-report ${MAIL_DIR}/.notmuch/xapian)\r
++test_expect_equal "$ghosts" "0"\r
+ \r
+ test_done\r
+-- \r
+2.8.0.rc3\r
+\r