--- /dev/null
+Return-Path: <aclements@csail.mit.edu>\r
+X-Original-To: notmuch@notmuchmail.org\r
+Delivered-To: notmuch@notmuchmail.org\r
+Received: from localhost (localhost [127.0.0.1])\r
+ by olra.theworths.org (Postfix) with ESMTP id 1F440431FD4\r
+ for <notmuch@notmuchmail.org>; Thu, 23 Oct 2014 05:31:13 -0700 (PDT)\r
+X-Virus-Scanned: Debian amavisd-new at olra.theworths.org\r
+X-Spam-Flag: NO\r
+X-Spam-Score: -2.3\r
+X-Spam-Level: \r
+X-Spam-Status: No, score=-2.3 tagged_above=-999 required=5\r
+ tests=[RCVD_IN_DNSWL_MED=-2.3] autolearn=disabled\r
+Received: from olra.theworths.org ([127.0.0.1])\r
+ by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024)\r
+ with ESMTP id JhhI9C9AN1ks for <notmuch@notmuchmail.org>;\r
+ Thu, 23 Oct 2014 05:31:07 -0700 (PDT)\r
+Received: from dmz-mailsec-scanner-3.mit.edu (dmz-mailsec-scanner-3.mit.edu\r
+ [18.9.25.14])\r
+ (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits))\r
+ (No client certificate requested)\r
+ by olra.theworths.org (Postfix) with ESMTPS id 568E5431FDF\r
+ for <notmuch@notmuchmail.org>; Thu, 23 Oct 2014 05:30:56 -0700 (PDT)\r
+X-AuditID: 1209190e-f79d46d000003643-84-5448f4ffc8d3\r
+Received: from mailhub-auth-4.mit.edu ( [18.7.62.39])\r
+ (using TLS with cipher AES256-SHA (256/256 bits))\r
+ (Client did not present a certificate)\r
+ by dmz-mailsec-scanner-3.mit.edu (Symantec Messaging Gateway) with SMTP\r
+ id FB.79.13891.005F8445; Thu, 23 Oct 2014 08:30:56 -0400 (EDT)\r
+Received: from outgoing.mit.edu (outgoing-auth-1.mit.edu [18.9.28.11])\r
+ by mailhub-auth-4.mit.edu (8.13.8/8.9.2) with ESMTP id s9NCUrD3030512; \r
+ Thu, 23 Oct 2014 08:30:53 -0400\r
+Received: from drake.dyndns.org\r
+ (216-15-114-40.c3-0.arl-ubr1.sbo-arl.ma.cable.rcn.com\r
+ [216.15.114.40]) (authenticated bits=0)\r
+ (User authenticated as amdragon@ATHENA.MIT.EDU)\r
+ by outgoing.mit.edu (8.13.8/8.12.4) with ESMTP id s9NCUmIb008961\r
+ (version=TLSv1/SSLv3 cipher=AES128-SHA bits=128 verify=NOT);\r
+ Thu, 23 Oct 2014 08:30:51 -0400\r
+Received: from amthrax by drake.dyndns.org with local (Exim 4.84)\r
+ (envelope-from <aclements@csail.mit.edu>)\r
+ id 1XhHXP-0007bq-AA; Thu, 23 Oct 2014 08:30:43 -0400\r
+From: Austin Clements <aclements@csail.mit.edu>\r
+To: notmuch@notmuchmail.org\r
+Subject: [PATCH v3 4/9] lib: Internal support for querying and creating ghost\r
+ messages\r
+Date: Thu, 23 Oct 2014 08:30:36 -0400\r
+Message-Id: <1414067441-29054-5-git-send-email-aclements@csail.mit.edu>\r
+X-Mailer: git-send-email 2.1.0\r
+In-Reply-To: <1414067441-29054-1-git-send-email-aclements@csail.mit.edu>\r
+References: <1414067441-29054-1-git-send-email-aclements@csail.mit.edu>\r
+X-Brightmail-Tracker:\r
+ H4sIAAAAAAAAA+NgFtrNIsWRmVeSWpSXmKPExsUixG6nrsvw1SPE4PVvFovVc3ksrt+cyezA\r
+ 5LFz1l12j2erbjEHMEVx2aSk5mSWpRbp2yVwZXRtlCyYZljR/uwKUwPjY/UuRk4OCQETiStr\r
+ 57BB2GISF+6tB7OFBGYzSeydotfFyAVkb2SU2H58OyOEc4dJYt60A2wQzhJGiXer37KCtLAJ\r
+ 6EusWDsJzBYRkJbYeXc2mM0s4Cjxef8isLHCAmESczfOYAKxWQRUJZ68X8MCYvMKuEvMnvSI\r
+ GeIMOYkNu/8zgticAh4Se3bcgTrJXWL784OsExj5FzAyrGKUTcmt0s1NzMwpTk3WLU5OzMtL\r
+ LdI11svNLNFLTSndxAgKI05Jvh2MXw8qHWIU4GBU4uGtSPYIEWJNLCuuzD3EKMnBpCTKG/MA\r
+ KMSXlJ9SmZFYnBFfVJqTWnyIUYKDWUmEN/4CUI43JbGyKrUoHyYlzcGiJM676QdfiJBAemJJ\r
+ anZqakFqEUxWhoNDSYKX8QtQo2BRanpqRVpmTglCmomDE2Q4D9BwEZAa3uKCxNzizHSI/ClG\r
+ RSlx3uTPQAkBkERGaR5cLyzOXzGKA70izKsG0s4DTBFw3a+ABjOBXL0BbHBJIkJKqoGxtGL5\r
+ q02XEqw/W3D83pF9RkV/dnPr1U+MP/qqnL49rZHa9fe/RqPUkfWXvmvVSTlV7FrBv3qxSYZC\r
+ /4HvVRO8F5/9kzhrje7OyYX+h3c3+5kHbXgWnmqWsDblej7zmt7iNZzsh2Jyrib+Fdlk7h02\r
+ yWzDrkkcgYc+tv87fsvg4o+mSEGVulQ7JZbijERDLeai4kQAueIBCc4CAAA=\r
+X-BeenThere: notmuch@notmuchmail.org\r
+X-Mailman-Version: 2.1.13\r
+Precedence: list\r
+List-Id: "Use and development of the notmuch mail system."\r
+ <notmuch.notmuchmail.org>\r
+List-Unsubscribe: <http://notmuchmail.org/mailman/options/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>\r
+List-Archive: <http://notmuchmail.org/pipermail/notmuch>\r
+List-Post: <mailto:notmuch@notmuchmail.org>\r
+List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>\r
+List-Subscribe: <http://notmuchmail.org/mailman/listinfo/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=subscribe>\r
+X-List-Received-Date: Thu, 23 Oct 2014 12:31:13 -0000\r
+\r
+From: Austin Clements <amdragon@mit.edu>\r
+\r
+This updates the message abstraction to support ghost messages: it\r
+adds a message flag that distinguishes regular messages from ghost\r
+messages, and an internal function for initializing a newly created\r
+(blank) message as a ghost message.\r
+---\r
+ lib/message.cc | 52 +++++++++++++++++++++++++++++++++++++++++++++++++--\r
+ lib/notmuch-private.h | 4 ++++\r
+ lib/notmuch.h | 9 ++++++++-\r
+ 3 files changed, 62 insertions(+), 3 deletions(-)\r
+\r
+diff --git a/lib/message.cc b/lib/message.cc\r
+index 55d2ff6..a7a13cc 100644\r
+--- a/lib/message.cc\r
++++ b/lib/message.cc\r
+@@ -39,6 +39,9 @@ struct visible _notmuch_message {\r
+ notmuch_message_file_t *message_file;\r
+ notmuch_message_list_t *replies;\r
+ unsigned long flags;\r
++ /* For flags that are initialized on-demand, lazy_flags indicates\r
++ * if each flag has been initialized. */\r
++ unsigned long lazy_flags;\r
+ \r
+ Xapian::Document doc;\r
+ Xapian::termcount termpos;\r
+@@ -99,6 +102,7 @@ _notmuch_message_create_for_document (const void *talloc_owner,\r
+ \r
+ message->frozen = 0;\r
+ message->flags = 0;\r
++ message->lazy_flags = 0;\r
+ \r
+ /* Each of these will be lazily created as needed. */\r
+ message->message_id = NULL;\r
+@@ -192,7 +196,7 @@ _notmuch_message_create (const void *talloc_owner,\r
+ *\r
+ * There is already a document with message ID 'message_id' in the\r
+ * database. The returned message can be used to query/modify the\r
+- * document.\r
++ * document. The message may be a ghost message.\r
+ *\r
+ * NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND:\r
+ *\r
+@@ -305,6 +309,7 @@ _notmuch_message_ensure_metadata (notmuch_message_t *message)\r
+ const char *thread_prefix = _find_prefix ("thread"),\r
+ *tag_prefix = _find_prefix ("tag"),\r
+ *id_prefix = _find_prefix ("id"),\r
++ *type_prefix = _find_prefix ("type"),\r
+ *filename_prefix = _find_prefix ("file-direntry"),\r
+ *replyto_prefix = _find_prefix ("replyto");\r
+ \r
+@@ -337,10 +342,25 @@ _notmuch_message_ensure_metadata (notmuch_message_t *message)\r
+ message->message_id =\r
+ _notmuch_message_get_term (message, i, end, id_prefix);\r
+ \r
++ /* Get document type */\r
++ assert (strcmp (id_prefix, type_prefix) < 0);\r
++ if (! NOTMUCH_TEST_BIT (message->lazy_flags, NOTMUCH_MESSAGE_FLAG_GHOST)) {\r
++ i.skip_to (type_prefix);\r
++ /* "T" is the prefix "type" fields. See\r
++ * BOOLEAN_PREFIX_INTERNAL. */\r
++ if (*i == "Tmail")\r
++ NOTMUCH_CLEAR_BIT (&message->flags, NOTMUCH_MESSAGE_FLAG_GHOST);\r
++ else if (*i == "Tghost")\r
++ NOTMUCH_SET_BIT (&message->flags, NOTMUCH_MESSAGE_FLAG_GHOST);\r
++ else\r
++ INTERNAL_ERROR ("Message without type term");\r
++ NOTMUCH_SET_BIT (&message->lazy_flags, NOTMUCH_MESSAGE_FLAG_GHOST);\r
++ }\r
++\r
+ /* Get filename list. Here we get only the terms. We lazily\r
+ * expand them to full file names when needed in\r
+ * _notmuch_message_ensure_filename_list. */\r
+- assert (strcmp (id_prefix, filename_prefix) < 0);\r
++ assert (strcmp (type_prefix, filename_prefix) < 0);\r
+ if (!message->filename_term_list && !message->filename_list)\r
+ message->filename_term_list =\r
+ _notmuch_database_get_terms_with_prefix (message, i, end,\r
+@@ -371,6 +391,11 @@ _notmuch_message_invalidate_metadata (notmuch_message_t *message,\r
+ message->tag_list = NULL;\r
+ }\r
+ \r
++ if (strcmp ("type", prefix_name) == 0) {\r
++ NOTMUCH_CLEAR_BIT (&message->flags, NOTMUCH_MESSAGE_FLAG_GHOST);\r
++ NOTMUCH_CLEAR_BIT (&message->lazy_flags, NOTMUCH_MESSAGE_FLAG_GHOST);\r
++ }\r
++\r
+ if (strcmp ("file-direntry", prefix_name) == 0) {\r
+ talloc_free (message->filename_term_list);\r
+ talloc_free (message->filename_list);\r
+@@ -869,6 +894,10 @@ notmuch_bool_t\r
+ notmuch_message_get_flag (notmuch_message_t *message,\r
+ notmuch_message_flag_t flag)\r
+ {\r
++ if (flag == NOTMUCH_MESSAGE_FLAG_GHOST &&\r
++ ! NOTMUCH_TEST_BIT (message->lazy_flags, flag))\r
++ _notmuch_message_ensure_metadata (message);\r
++\r
+ return NOTMUCH_TEST_BIT (message->flags, flag);\r
+ }\r
+ \r
+@@ -880,6 +909,7 @@ notmuch_message_set_flag (notmuch_message_t *message,\r
+ NOTMUCH_SET_BIT (&message->flags, flag);\r
+ else\r
+ NOTMUCH_CLEAR_BIT (&message->flags, flag);\r
++ NOTMUCH_SET_BIT (&message->lazy_flags, flag);\r
+ }\r
+ \r
+ time_t\r
+@@ -989,6 +1019,24 @@ _notmuch_message_delete (notmuch_message_t *message)\r
+ return NOTMUCH_STATUS_SUCCESS;\r
+ }\r
+ \r
++/* Transform a blank message into a ghost message. The caller must\r
++ * _notmuch_message_sync the message. */\r
++notmuch_private_status_t\r
++_notmuch_message_initialize_ghost (notmuch_message_t *message,\r
++ const char *thread_id)\r
++{\r
++ notmuch_private_status_t status;\r
++\r
++ status = _notmuch_message_add_term (message, "type", "ghost");\r
++ if (status)\r
++ return status;\r
++ status = _notmuch_message_add_term (message, "thread", thread_id);\r
++ if (status)\r
++ return status;\r
++\r
++ return NOTMUCH_PRIVATE_STATUS_SUCCESS;\r
++}\r
++\r
+ /* Ensure that 'message' is not holding any file object open. Future\r
+ * calls to various functions will still automatically open the\r
+ * message file as needed.\r
+diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h\r
+index 7250291..2f43c1d 100644\r
+--- a/lib/notmuch-private.h\r
++++ b/lib/notmuch-private.h\r
+@@ -308,6 +308,10 @@ _notmuch_message_sync (notmuch_message_t *message);\r
+ notmuch_status_t\r
+ _notmuch_message_delete (notmuch_message_t *message);\r
+ \r
++notmuch_private_status_t\r
++_notmuch_message_initialize_ghost (notmuch_message_t *message,\r
++ const char *thread_id);\r
++\r
+ void\r
+ _notmuch_message_close (notmuch_message_t *message);\r
+ \r
+diff --git a/lib/notmuch.h b/lib/notmuch.h\r
+index dae0416..92594b9 100644\r
+--- a/lib/notmuch.h\r
++++ b/lib/notmuch.h\r
+@@ -1221,7 +1221,14 @@ notmuch_message_get_filenames (notmuch_message_t *message);\r
+ */\r
+ typedef enum _notmuch_message_flag {\r
+ NOTMUCH_MESSAGE_FLAG_MATCH,\r
+- NOTMUCH_MESSAGE_FLAG_EXCLUDED\r
++ NOTMUCH_MESSAGE_FLAG_EXCLUDED,\r
++\r
++ /* This message is a "ghost message", meaning it has no filenames\r
++ * or content, but we know it exists because it was referenced by\r
++ * some other message. A ghost message has only a message ID and\r
++ * thread ID.\r
++ */\r
++ NOTMUCH_MESSAGE_FLAG_GHOST,\r
+ } notmuch_message_flag_t;\r
+ \r
+ /**\r
+-- \r
+2.1.0\r
+\r