--- /dev/null
+Return-Path: <aclements@csail.mit.edu>\r
+X-Original-To: notmuch@notmuchmail.org\r
+Delivered-To: notmuch@notmuchmail.org\r
+Received: from localhost (localhost [127.0.0.1])\r
+ by olra.theworths.org (Postfix) with ESMTP id 1F2F9431FC0\r
+ for <notmuch@notmuchmail.org>; Mon, 6 Oct 2014 16:17:43 -0700 (PDT)\r
+X-Virus-Scanned: Debian amavisd-new at olra.theworths.org\r
+X-Spam-Flag: NO\r
+X-Spam-Score: -2.3\r
+X-Spam-Level: \r
+X-Spam-Status: No, score=-2.3 tagged_above=-999 required=5\r
+ tests=[RCVD_IN_DNSWL_MED=-2.3] autolearn=disabled\r
+Received: from olra.theworths.org ([127.0.0.1])\r
+ by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024)\r
+ with ESMTP id GLMpMRwGzRw5 for <notmuch@notmuchmail.org>;\r
+ Mon, 6 Oct 2014 16:17:39 -0700 (PDT)\r
+Received: from dmz-mailsec-scanner-5.mit.edu (dmz-mailsec-scanner-5.mit.edu\r
+ [18.7.68.34])\r
+ by olra.theworths.org (Postfix) with ESMTP id 902B7431FBF\r
+ for <notmuch@notmuchmail.org>; Mon, 6 Oct 2014 16:17:30 -0700 (PDT)\r
+X-AuditID: 12074422-f79436d000000c21-12-5433230aed96\r
+Received: from mailhub-auth-4.mit.edu ( [18.7.62.39])\r
+ (using TLS with cipher AES256-SHA (256/256 bits))\r
+ (Client did not present a certificate)\r
+ by dmz-mailsec-scanner-5.mit.edu (Symantec Messaging Gateway) with SMTP\r
+ id E3.37.03105.A0323345; Mon, 6 Oct 2014 19:17:30 -0400 (EDT)\r
+Received: from outgoing.mit.edu (outgoing-auth-1.mit.edu [18.9.28.11])\r
+ by mailhub-auth-4.mit.edu (8.13.8/8.9.2) with ESMTP id s96NHPGu032086; \r
+ Mon, 6 Oct 2014 19:17:25 -0400\r
+Received: from drake.dyndns.org ([12.185.136.2]) (authenticated bits=0)\r
+ (User authenticated as amdragon@ATHENA.MIT.EDU)\r
+ by outgoing.mit.edu (8.13.8/8.12.4) with ESMTP id s96NHMCT014473\r
+ (version=TLSv1/SSLv3 cipher=AES128-SHA bits=128 verify=NOT);\r
+ Mon, 6 Oct 2014 19:17:24 -0400\r
+Received: from amthrax by drake.dyndns.org with local (Exim 4.84)\r
+ (envelope-from <aclements@csail.mit.edu>)\r
+ id 1XbHWs-0001Jq-71; Mon, 06 Oct 2014 17:17:22 -0600\r
+From: Austin Clements <aclements@csail.mit.edu>\r
+To: notmuch@notmuchmail.org\r
+Subject: [PATCH v2 09/12] lib: Implement upgrade to ghost messages feature\r
+Date: Mon, 6 Oct 2014 17:17:15 -0600\r
+Message-Id: <1412637438-4821-10-git-send-email-aclements@csail.mit.edu>\r
+X-Mailer: git-send-email 2.1.0\r
+In-Reply-To: <1412637438-4821-1-git-send-email-aclements@csail.mit.edu>\r
+References: <1412637438-4821-1-git-send-email-aclements@csail.mit.edu>\r
+X-Brightmail-Tracker:\r
+ H4sIAAAAAAAAA+NgFtrLIsWRmVeSWpSXmKPExsUixG6nrsulbBxi8Oi+mMWN1m5Gi+s3ZzI7\r
+ MHk8W3WL2WPLoffMAUxRXDYpqTmZZalF+nYJXBmrd+5mKdggW3Fr4nf2Bsap4l2MnBwSAiYS\r
+ J2/tYISwxSQu3FvP1sXIxSEkMJtJYsnF9ewQzgZGiR9/+1khnKVMEsvunmGEcJYwSmw8fpMZ\r
+ pJ9NQF9ixdpJrCC2iIC0xM67s8FsZgFLiatf25lAbGEBL4nfV5awdDFycLAIqErcmOcNEuYV\r
+ cJe4PqeZHeIMOYkNu/+DncQJFF9zZTLYGCEBN4nnP5+yTGDkX8DIsIpRNiW3Sjc3MTOnODVZ\r
+ tzg5MS8vtUjXVC83s0QvNaV0EyM4lFyUdjD+PKh0iFGAg1GJhzdih2GIEGtiWXFl7iFGSQ4m\r
+ JVHeOezGIUJ8SfkplRmJxRnxRaU5qcWHGCU4mJVEeLl+G4UI8aYkVlalFuXDpKQ5WJTEeTf9\r
+ 4AsREkhPLEnNTk0tSC2CycpwcChJ8HYoAg0VLEpNT61Iy8wpQUgzcXCCDOcBGn4IpIa3uCAx\r
+ tzgzHSJ/ilFRSpw3GCQhAJLIKM2D64XF+itGcaBXhHnng1TxANMEXPcroMFMQINN5+iDDC5J\r
+ REhJNTAy3+mI2M2196EUO5OewVInlrPhFzYKFK59nrRXdLdypswLu0atdZWP5m6zMxJa+b/n\r
+ 5aIZZ85qfF5+tu/ViQt7Jzx3eT09LbRm+X2mmpWef5LD83iuaYte/FW067esTO3tgLj5xern\r
+ Gk/vdGzO9lSo73fQXOeoNPNhIpfRlStX/x+/7bmCee0+JZbijERDLeai4kQA2ao+X9ACAAA=\r
+X-BeenThere: notmuch@notmuchmail.org\r
+X-Mailman-Version: 2.1.13\r
+Precedence: list\r
+List-Id: "Use and development of the notmuch mail system."\r
+ <notmuch.notmuchmail.org>\r
+List-Unsubscribe: <http://notmuchmail.org/mailman/options/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>\r
+List-Archive: <http://notmuchmail.org/pipermail/notmuch>\r
+List-Post: <mailto:notmuch@notmuchmail.org>\r
+List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>\r
+List-Subscribe: <http://notmuchmail.org/mailman/listinfo/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=subscribe>\r
+X-List-Received-Date: Mon, 06 Oct 2014 23:17:43 -0000\r
+\r
+From: Austin Clements <amdragon@mit.edu>\r
+\r
+Somehow this is the first upgrade pass that actually does *any* error\r
+checking, so this also adds the bit of necessary infrastructure to\r
+handle that.\r
+---\r
+ lib/database.cc | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++++--\r
+ 1 file changed, 64 insertions(+), 2 deletions(-)\r
+\r
+diff --git a/lib/database.cc b/lib/database.cc\r
+index fdcc526..1316529 100644\r
+--- a/lib/database.cc\r
++++ b/lib/database.cc\r
+@@ -1231,6 +1231,7 @@ notmuch_database_upgrade (notmuch_database_t *notmuch,\r
+ notmuch_bool_t timer_is_active = FALSE;\r
+ enum _notmuch_features target_features, new_features;\r
+ notmuch_status_t status;\r
++ notmuch_private_status_t private_status;\r
+ unsigned int count = 0, total = 0;\r
+ \r
+ status = _notmuch_database_ensure_writable (notmuch);\r
+@@ -1275,6 +1276,13 @@ notmuch_database_upgrade (notmuch_database_t *notmuch,\r
+ for (t = db->allterms_begin ("XTIMESTAMP"); t != t_end; t++)\r
+ ++total;\r
+ }\r
++ if (new_features & NOTMUCH_FEATURE_GHOSTS) {\r
++ /* The ghost message upgrade converts all thread_id_*\r
++ * metadata values into ghost message documents. */\r
++ t_end = db->metadata_keys_end ("thread_id_");\r
++ for (t = db->metadata_keys_begin ("thread_id_"); t != t_end; ++t)\r
++ ++total;\r
++ }\r
+ \r
+ /* Perform the upgrade in a transaction. */\r
+ db->begin_transaction (true);\r
+@@ -1378,10 +1386,64 @@ notmuch_database_upgrade (notmuch_database_t *notmuch,\r
+ }\r
+ }\r
+ \r
++ /* Perform metadata upgrades. */\r
++\r
++ /* Prior to NOTMUCH_FEATURE_GHOSTS, thread IDs for missing\r
++ * messages were stored as database metadata. Change these to\r
++ * ghost messages.\r
++ */\r
++ if (new_features & NOTMUCH_FEATURE_GHOSTS) {\r
++ notmuch_message_t *message;\r
++ std::string message_id, thread_id;\r
++\r
++ t_end = db->metadata_keys_end (NOTMUCH_METADATA_THREAD_ID_PREFIX);\r
++ for (t = db->metadata_keys_begin (NOTMUCH_METADATA_THREAD_ID_PREFIX);\r
++ t != t_end; ++t) {\r
++ if (do_progress_notify) {\r
++ progress_notify (closure, (double) count / total);\r
++ do_progress_notify = 0;\r
++ }\r
++\r
++ message_id = (*t).substr (\r
++ strlen (NOTMUCH_METADATA_THREAD_ID_PREFIX));\r
++ thread_id = db->get_metadata (*t);\r
++\r
++ /* Create ghost message */\r
++ message = _notmuch_message_create_for_message_id (\r
++ notmuch, message_id.c_str (), &private_status);\r
++ if (private_status == NOTMUCH_PRIVATE_STATUS_SUCCESS) {\r
++ /* Document already exists; ignore the stored thread ID */\r
++ } else if (private_status ==\r
++ NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {\r
++ private_status = _notmuch_message_initialize_ghost (\r
++ message, thread_id.c_str ());\r
++ if (! private_status)\r
++ _notmuch_message_sync (message);\r
++ }\r
++\r
++ if (private_status) {\r
++ fprintf (stderr,\r
++ "Upgrade failed while creating ghost messages.\n");\r
++ status = COERCE_STATUS (private_status, "Unexpected status from _notmuch_message_initialize_ghost");\r
++ goto DONE;\r
++ }\r
++\r
++ /* Clear saved metadata thread ID */\r
++ db->set_metadata (*t, "");\r
++\r
++ ++count;\r
++ }\r
++ }\r
++\r
++ status = NOTMUCH_STATUS_SUCCESS;\r
+ db->set_metadata ("features", _print_features (local, notmuch->features));\r
+ db->set_metadata ("version", STRINGIFY (NOTMUCH_DATABASE_VERSION));\r
+ \r
+- db->commit_transaction ();\r
++ DONE:\r
++ if (status == NOTMUCH_STATUS_SUCCESS)\r
++ db->commit_transaction ();\r
++ else\r
++ db->cancel_transaction ();\r
+ \r
+ if (timer_is_active) {\r
+ /* Now stop the timer. */\r
+@@ -1397,7 +1459,7 @@ notmuch_database_upgrade (notmuch_database_t *notmuch,\r
+ }\r
+ \r
+ talloc_free (local);\r
+- return NOTMUCH_STATUS_SUCCESS;\r
++ return status;\r
+ }\r
+ \r
+ notmuch_status_t\r
+-- \r
+2.1.0\r
+\r