lib: Split the database upgrade into two phases for safer operation.

The first phase copies data from the old format to the new format
without deleting anything. This allows an old notmuch to still use the
database if the upgrade process gets interrupted. The second phase
performs the deletion (after updating the database version number). If
the second phase is interrupted, there will be some unused data in the
database, but it shouldn't cause any actual harm.

diff --git a/lib/database.cc b/lib/database.cc
index d0262722cff27dce3086236ea7482e7ffaf968de..19f960e2d621d378f2254d8a1ddd5dc8554eec15 100644 (file)
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -722,7 +722,7 @@ notmuch_database_upgrade (notmuch_database_t *notmuch,
     }
 
     /* Before version 1, each message document had its filename in the
-     * data field. Move that into the new format by calling
+     * data field. Copy that into the new format by calling
      * notmuch_message_add_filename.
      */
     if (version < 1) {
@@ -730,6 +730,7 @@ notmuch_database_upgrade (notmuch_database_t *notmuch,
        notmuch_query_t *query = notmuch_query_create (notmuch, "");
        notmuch_messages_t *messages;
        notmuch_message_t *message;
+       char *filename;
 
        total = notmuch_query_count_messages (query);
 
@@ -744,15 +745,24 @@ notmuch_database_upgrade (notmuch_database_t *notmuch,
 
            message = notmuch_messages_get (messages);
 
-           _notmuch_message_upgrade_filename_storage (message);
+           filename = _notmuch_message_talloc_copy_data (message);
+           if (filename && *filename != '\0') {
+               _notmuch_message_add_filename (message, filename);
+               _notmuch_message_sync (message);
+           }
+           talloc_free (filename);
+
+           notmuch_message_destroy (message);
 
            count++;
        }
+
+       notmuch_query_destroy (query);
     }
 
     /* Also, before version 1 we stored directory timestamps in
      * XTIMESTAMP documents instead of the current XDIRECTORY
-     * documents. So convert those as well. */
+     * documents. So copy those as well. */
     if (version < 1) {
        Xapian::TermIterator t, t_end;
 
@@ -783,8 +793,6 @@ notmuch_database_upgrade (notmuch_database_t *notmuch,
                                                            term.c_str() + 10);
                notmuch_directory_set_mtime (directory, mtime);
                notmuch_directory_destroy (directory);
-
-               db->delete_document (*p);
            }
        }
     }
@@ -792,6 +800,66 @@ notmuch_database_upgrade (notmuch_database_t *notmuch,
     db->set_metadata ("version", STRINGIFY (NOTMUCH_DATABASE_VERSION));
     db->flush ();
 
+    /* Now that the upgrade is complete we can remove the old data
+     * and documents that are no longer needed. */
+    if (version < 1) {
+       unsigned int count = 0, total;
+       notmuch_query_t *query = notmuch_query_create (notmuch, "");
+       notmuch_messages_t *messages;
+       notmuch_message_t *message;
+       char *filename;
+
+       total = notmuch_query_count_messages (query);
+
+       for (messages = notmuch_query_search_messages (query);
+            notmuch_messages_has_more (messages);
+            notmuch_messages_advance (messages))
+       {
+           if (do_progress_notify) {
+               progress_notify (closure, count, total);
+               do_progress_notify = 0;
+           }
+
+           message = notmuch_messages_get (messages);
+
+           filename = _notmuch_message_talloc_copy_data (message);
+           if (filename && *filename != '\0') {
+               _notmuch_message_clear_data (message);
+               _notmuch_message_sync (message);
+           }
+           talloc_free (filename);
+
+           notmuch_message_destroy (message);
+
+           count++;
+       }
+
+       notmuch_query_destroy (query);
+    }
+
+    if (version < 1) {
+       Xapian::TermIterator t, t_end;
+
+       t_end = notmuch->xapian_db->allterms_end ("XTIMESTAMP");
+
+       for (t = notmuch->xapian_db->allterms_begin ("XTIMESTAMP");
+            t != t_end;
+            t++)
+       {
+           Xapian::PostingIterator p, p_end;
+           std::string term = *t;
+
+           p_end = notmuch->xapian_db->postlist_end (term);
+
+           for (p = notmuch->xapian_db->postlist_begin (term);
+                p != p_end;
+                p++)
+           {
+               db->delete_document (*p);
+           }
+       }
+    }
+
     if (timer_is_active) {
        /* Now stop the timer. */
        timerval.it_interval.tv_sec = 0;

diff --git a/lib/message.cc b/lib/message.cc
index baeaa46972867d10a1cc5cde801fac03c0817df1..1cda55a1e1b67b9573f7994018cc97bb73d08acc 100644 (file)
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -416,22 +416,16 @@ _notmuch_message_add_filename (notmuch_message_t *message,
     return NOTMUCH_STATUS_SUCCESS;
 }
 
-/* Move the filename from the data field (as it was in database format
- * version 0) to a file-direntry term instead (as in database format
- * version 1).
- */
-void
-_notmuch_message_upgrade_filename_storage (notmuch_message_t *message)
+char *
+_notmuch_message_talloc_copy_data (notmuch_message_t *message)
 {
-    char *filename;
+    return talloc_strdup (message, message->doc.get_data ().c_str ());
+}
 
-    filename = talloc_strdup (message, message->doc.get_data ().c_str ());
-    if (filename && *filename != '\0') {
-       _notmuch_message_add_filename (message, filename);
-       message->doc.set_data ("");
-       _notmuch_message_sync (message);
-    }
-    talloc_free (filename);
+void
+_notmuch_message_clear_data (notmuch_message_t *message)
+{
+    message->doc.set_data ("");
 }
 
 const char *

diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index f9adea728b4b8e83644f1b5c66245743c9ac5e86..c7fb0ef89312fa35ffda1cee7f22a9cd08da30c3 100644 (file)
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -258,6 +258,22 @@ _notmuch_message_sync (notmuch_message_t *message);
 void
 _notmuch_message_close (notmuch_message_t *message);
 
+/* Get a copy of the data in this message document.
+ *
+ * Caller should talloc_free the result when done.
+ *
+ * This function is intended to support database upgrade and really
+ * shouldn't be used otherwise. */
+char *
+_notmuch_message_talloc_copy_data (notmuch_message_t *message);
+
+/* Clear the data in this message document.
+ *
+ * This function is intended to support database upgrade and really
+ * shouldn't be used otherwise. */
+void
+_notmuch_message_clear_data (notmuch_message_t *message);
+
 /* index.cc */
 
 notmuch_status_t