From fd15b30b522cde51dd11cfcd47423835088b4346 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Wed, 30 Jul 2014 12:48:06 +2000 Subject: [PATCH] [PATCH v2 08/14] lib: Simplify upgrade code using a transaction --- 7e/6fb0dfb26578c9fbb02ba233cbab5daa77e652 | 191 ++++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 7e/6fb0dfb26578c9fbb02ba233cbab5daa77e652 diff --git a/7e/6fb0dfb26578c9fbb02ba233cbab5daa77e652 b/7e/6fb0dfb26578c9fbb02ba233cbab5daa77e652 new file mode 100644 index 000000000..2b0a6a173 --- /dev/null +++ b/7e/6fb0dfb26578c9fbb02ba233cbab5daa77e652 @@ -0,0 +1,191 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id 733C7431FAE + for ; Tue, 29 Jul 2014 09:48:47 -0700 (PDT) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Spam-Flag: NO +X-Spam-Score: -2.3 +X-Spam-Level: +X-Spam-Status: No, score=-2.3 tagged_above=-999 required=5 + tests=[RCVD_IN_DNSWL_MED=-2.3] autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id eeFEN+UzuBel for ; + Tue, 29 Jul 2014 09:48:41 -0700 (PDT) +Received: from dmz-mailsec-scanner-3.mit.edu (dmz-mailsec-scanner-3.mit.edu + [18.9.25.14]) + (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits)) + (No client certificate requested) + by olra.theworths.org (Postfix) with ESMTPS id 31D0F431FC9 + for ; Tue, 29 Jul 2014 09:48:22 -0700 (PDT) +X-AuditID: 1209190e-f79946d000007db1-cd-53d7d055d027 +Received: from mailhub-auth-3.mit.edu ( [18.9.21.43]) + (using TLS with cipher AES256-SHA (256/256 bits)) + (Client did not present a certificate) + by dmz-mailsec-scanner-3.mit.edu (Symantec Messaging Gateway) with SMTP + id A9.F5.32177.550D7D35; Tue, 29 Jul 2014 12:48:22 -0400 (EDT) +Received: from outgoing.mit.edu (outgoing-auth-1.mit.edu [18.9.28.11]) + by mailhub-auth-3.mit.edu (8.13.8/8.9.2) with ESMTP id s6TGmIuU030919; + Tue, 29 Jul 2014 12:48:18 -0400 +Received: from drake.dyndns.org (31-33-71.wireless.csail.mit.edu + [128.31.33.71]) (authenticated bits=0) + (User authenticated as amdragon@ATHENA.MIT.EDU) + by outgoing.mit.edu (8.13.8/8.12.4) with ESMTP id s6TGmFav030262 + (version=TLSv1/SSLv3 cipher=AES256-SHA bits=256 verify=NOT); + Tue, 29 Jul 2014 12:48:17 -0400 +Received: from amthrax by drake.dyndns.org with local (Exim 4.77) + (envelope-from ) + id 1XCAZS-0007Gw-Sd; Tue, 29 Jul 2014 12:48:14 -0400 +From: Austin Clements +To: notmuch@notmuchmail.org +Subject: [PATCH v2 08/14] lib: Simplify upgrade code using a transaction +Date: Tue, 29 Jul 2014 12:48:06 -0400 +Message-Id: <1406652492-27803-9-git-send-email-amdragon@mit.edu> +X-Mailer: git-send-email 2.0.0 +In-Reply-To: <1406652492-27803-1-git-send-email-amdragon@mit.edu> +References: <1406652492-27803-1-git-send-email-amdragon@mit.edu> +X-Brightmail-Tracker: + H4sIAAAAAAAAA+NgFtrNIsWRmVeSWpSXmKPExsUixCmqrRt24XqwwSV3i9VzeSyu35zJ7MDk + sXPWXXaPZ6tuMQcwRXHZpKTmZJalFunbJXBl7Jm1nLFgrVzFvPVNjA2ML8W7GDk5JARMJJ6f + O8ECYYtJXLi3nq2LkYtDSGA2k8SEw5eZIJyNjBK/172Eco4xSXRcvMMM4cxllHjYf44NpJ9N + QENi2/7ljCC2iIC0xM67s1lBbGYBR4nP+xeB1QgLeEgc+38VKM7BwSKgKnFvvhZImFfAQeLb + 36tsEGfISTTc+ARmcwK13rp3lwnEFgKqOTRrM8sERv4FjAyrGGVTcqt0cxMzc4pTk3WLkxPz + 8lKLdI31cjNL9FJTSjcxgsNIkm8H49eDSocYBTgYlXh4N8y9FizEmlhWXJl7iFGSg0lJlFdv + 3/VgIb6k/JTKjMTijPii0pzU4kOMEhzMSiK8fGeBcrwpiZVVqUX5MClpDhYlcd631lbBQgLp + iSWp2ampBalFMFkZDg4lCd7X54EaBYtS01Mr0jJzShDSTBycIMN5gIbvA6nhLS5IzC3OTIfI + n2JUlBLn3QSSEABJZJTmwfXC4vwVozjQK8K8B0GqeIApAq77FdBgJqDBrC5gg0sSEVJSDYzx + p/o3T7pr9YfDu203g8aDUqVeyfdbiwKK3Zb61AmcYVk6/83/ayGx9epWhcJrnBnvRLcX7P8s + oXFowe7jD84IpbKVP2A6uJFXimHadL1V2y6YJlx0PBXPHbDdat/Oc5Xfpv9ymFq0bunHLG3X + zRJ9SZtCWJsKdCWy1ATLS6e9iOi/KvdEp1mJpTgj0VCLuag4EQBt4z+fzgIAAA== +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Tue, 29 Jul 2014 16:48:47 -0000 + +Previously, the upgrade was organized as two passes -- an upgrade +pass, and a separate cleanup pass -- so the database was always in a +valid state. This change substantially simplifies this code by +performing the upgrade in a transaction and combining both passes in +to one. This 1) eliminates a lot of duplicate code between the +passes, 2) speeds up the upgrade process, 3) makes progress reporting +more accurate, 4) eliminates the potential for stale data if the +upgrade is interrupted during the cleanup pass, and 5) makes it easier +to reason about the safety of the upgrade code. +--- + lib/database.cc | 67 ++++++--------------------------------------------------- + 1 file changed, 7 insertions(+), 60 deletions(-) + +diff --git a/lib/database.cc b/lib/database.cc +index e650798..e4a1072 100644 +--- a/lib/database.cc ++++ b/lib/database.cc +@@ -1232,6 +1232,9 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, + timer_is_active = TRUE; + } + ++ /* Perform the upgrade in a transaction. */ ++ db->begin_transaction (true); ++ + /* Before version 1, each message document had its filename in the + * data field. Copy that into the new format by calling + * notmuch_message_add_filename. +@@ -1259,6 +1262,7 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, + filename = _notmuch_message_talloc_copy_data (message); + if (filename && *filename != '\0') { + _notmuch_message_add_filename (message, filename); ++ _notmuch_message_clear_data (message); + _notmuch_message_sync (message); + } + talloc_free (filename); +@@ -1306,6 +1310,8 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, + NOTMUCH_FIND_CREATE, &status); + notmuch_directory_set_mtime (directory, mtime); + notmuch_directory_destroy (directory); ++ ++ db->delete_document (*p); + } + } + } +@@ -1347,67 +1353,8 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, + notmuch->features |= NOTMUCH_FEATURES_CURRENT; + db->set_metadata ("features", _print_features (local, notmuch->features)); + db->set_metadata ("version", STRINGIFY (NOTMUCH_DATABASE_VERSION)); +- db->flush (); +- +- /* Now that the upgrade is complete we can remove the old data +- * and documents that are no longer needed. */ +- if (version < 1) { +- notmuch_query_t *query = notmuch_query_create (notmuch, ""); +- notmuch_messages_t *messages; +- notmuch_message_t *message; +- char *filename; +- +- for (messages = notmuch_query_search_messages (query); +- notmuch_messages_valid (messages); +- notmuch_messages_move_to_next (messages)) +- { +- if (do_progress_notify) { +- progress_notify (closure, (double) count / total); +- do_progress_notify = 0; +- } +- +- message = notmuch_messages_get (messages); +- +- filename = _notmuch_message_talloc_copy_data (message); +- if (filename && *filename != '\0') { +- _notmuch_message_clear_data (message); +- _notmuch_message_sync (message); +- } +- talloc_free (filename); +- +- notmuch_message_destroy (message); +- } + +- notmuch_query_destroy (query); +- } +- +- if (version < 1) { +- Xapian::TermIterator t, t_end; +- +- t_end = notmuch->xapian_db->allterms_end ("XTIMESTAMP"); +- +- for (t = notmuch->xapian_db->allterms_begin ("XTIMESTAMP"); +- t != t_end; +- t++) +- { +- Xapian::PostingIterator p, p_end; +- std::string term = *t; +- +- p_end = notmuch->xapian_db->postlist_end (term); +- +- for (p = notmuch->xapian_db->postlist_begin (term); +- p != p_end; +- p++) +- { +- if (do_progress_notify) { +- progress_notify (closure, (double) count / total); +- do_progress_notify = 0; +- } +- +- db->delete_document (*p); +- } +- } +- } ++ db->commit_transaction (); + + if (timer_is_active) { + /* Now stop the timer. */ +-- +2.0.0 + -- 2.26.2