[PATCH v2 15/20] insert: fsync new directory after rename

diff --git a/c1/5941a9775148bbff2454c811e126be9111978d b/c1/5941a9775148bbff2454c811e126be9111978d
new file mode 100644 (file)
index 0000000..452b5ee
--- /dev/null
+++ b/c1/5941a9775148bbff2454c811e126be9111978d
@@ -0,0 +1,168 @@
+Return-Path: <novalazy@gmail.com>\r
+X-Original-To: notmuch@notmuchmail.org\r
+Delivered-To: notmuch@notmuchmail.org\r
+Received: from localhost (localhost [127.0.0.1])\r
+       by olra.theworths.org (Postfix) with ESMTP id 87046431FAF\r
+       for <notmuch@notmuchmail.org>; Sat, 24 Nov 2012 17:18:30 -0800 (PST)\r
+X-Virus-Scanned: Debian amavisd-new at olra.theworths.org\r
+X-Spam-Flag: NO\r
+X-Spam-Score: -0.799\r
+X-Spam-Level: \r
+X-Spam-Status: No, score=-0.799 tagged_above=-999 required=5\r
+       tests=[DKIM_SIGNED=0.1, DKIM_VALID=-0.1, DKIM_VALID_AU=-0.1,\r
+       FREEMAIL_FROM=0.001, RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled\r
+Received: from olra.theworths.org ([127.0.0.1])\r
+       by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024)\r
+       with ESMTP id swS5-moHRvv5 for <notmuch@notmuchmail.org>;\r
+       Sat, 24 Nov 2012 17:18:30 -0800 (PST)\r
+Received: from mail-pb0-f53.google.com (mail-pb0-f53.google.com\r
+       [209.85.160.53]) (using TLSv1 with cipher RC4-SHA (128/128 bits))\r
+       (No client certificate requested)\r
+       by olra.theworths.org (Postfix) with ESMTPS id E4642431FAE\r
+       for <notmuch@notmuchmail.org>; Sat, 24 Nov 2012 17:18:29 -0800 (PST)\r
+Received: by mail-pb0-f53.google.com with SMTP id jt11so7944513pbb.26\r
+       for <notmuch@notmuchmail.org>; Sat, 24 Nov 2012 17:18:29 -0800 (PST)\r
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20120113;\r
+       h=from:to:cc:subject:date:message-id:x-mailer:in-reply-to:references;\r
+       bh=PuYwcem/vc/qBiHL8StfxxN4P+B9e7JuX2ucBDMO9aQ=;\r
+       b=oIHcDxe48tuwg1uxRPzWpa4JwzltJCICIS44hQL76wjVYUQV2GYte88YsDbYqOOQZN\r
+       EbF4E90zigi2uTkCY21skxxUdfpM5ilpzqk89Wfx0bW0/BVA/IEVIccK1eWtjb7OqNzv\r
+       e7kRLrLENmB5rNim4C55lTMPZO16G51LI4rq1byylXiIyeuZYaFjE6x6jp6vHpGY/9k9\r
+       P1NkkjKhX2IH/5uPK8qPN9+Wcvazb3aeemIEA9/gOdytfDLMi5K6UGOvwt/YNZDTyBfd\r
+       b62//U6i1NuB33o4cQYmogUol7x4lJkm1szIrMhH3r7xqibghVyu6Ise/+KXt9NZfnxD\r
+       Ehpw==\r
+Received: by 10.66.77.39 with SMTP id p7mr22254729paw.8.1353806309673;\r
+       Sat, 24 Nov 2012 17:18:29 -0800 (PST)\r
+Received: from localhost (215.42.233.220.static.exetel.com.au.\r
+       [220.233.42.215])\r
+       by mx.google.com with ESMTPS id t1sm6173552paw.11.2012.11.24.17.18.27\r
+       (version=TLSv1/SSLv3 cipher=OTHER);\r
+       Sat, 24 Nov 2012 17:18:28 -0800 (PST)\r
+From: Peter Wang <novalazy@gmail.com>\r
+To: notmuch@notmuchmail.org\r
+Subject: [PATCH v2 15/20] insert: fsync new directory after rename\r
+Date: Sun, 25 Nov 2012 12:16:41 +1100\r
+Message-Id: <1353806206-29133-16-git-send-email-novalazy@gmail.com>\r
+X-Mailer: git-send-email 1.7.12.1\r
+In-Reply-To: <1353806206-29133-1-git-send-email-novalazy@gmail.com>\r
+References: <1353806206-29133-1-git-send-email-novalazy@gmail.com>\r
+X-BeenThere: notmuch@notmuchmail.org\r
+X-Mailman-Version: 2.1.13\r
+Precedence: list\r
+List-Id: "Use and development of the notmuch mail system."\r
+       <notmuch.notmuchmail.org>\r
+List-Unsubscribe: <http://notmuchmail.org/mailman/options/notmuch>,\r
+       <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>\r
+List-Archive: <http://notmuchmail.org/pipermail/notmuch>\r
+List-Post: <mailto:notmuch@notmuchmail.org>\r
+List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>\r
+List-Subscribe: <http://notmuchmail.org/mailman/listinfo/notmuch>,\r
+       <mailto:notmuch-request@notmuchmail.org?subject=subscribe>\r
+X-List-Received-Date: Sun, 25 Nov 2012 01:18:30 -0000\r
+\r
+After moving the file from the 'tmp' to the 'new' directory,\r
+fsync on the 'new' directory for durability.\r
+---\r
+ notmuch-insert.c | 39 ++++++++++++++++++++++++++++++++-------\r
+ 1 file changed, 32 insertions(+), 7 deletions(-)\r
+\r
+diff --git a/notmuch-insert.c b/notmuch-insert.c\r
+index f09c579..831b322 100644\r
+--- a/notmuch-insert.c\r
++++ b/notmuch-insert.c\r
+@@ -143,10 +143,10 @@ maildir_create_folder (void *ctx, const char *dir)\r
+ /* Open a unique file in the Maildir 'tmp' directory.\r
+  * Returns the file descriptor on success, or -1 on failure.\r
+  * On success, file paths into the 'tmp' and 'new' directories are returned\r
+- * via tmppath and newpath. */\r
++ * via tmppath and newpath, and the path of the 'new' directory in newdir. */\r
+ static int\r
+ maildir_open_tmp_file (void *ctx, const char *dir,\r
+-                     char **tmppath, char **newpath)\r
++                     char **tmppath, char **newpath, char **newdir)\r
+ {\r
+     pid_t pid;\r
+     char hostname[256];\r
+@@ -183,8 +183,9 @@ maildir_open_tmp_file (void *ctx, const char *dir,\r
+       return -1;\r
+     }\r
+ \r
++    *newdir = talloc_asprintf (ctx, "%s/new", dir);\r
+     *newpath = talloc_asprintf (ctx, "%s/new/%s", dir, filename);\r
+-    if (! *newpath) {\r
++    if (! *newdir || ! *newpath) {\r
+       fprintf (stderr, "Out of memory\n");\r
+       close (fd);\r
+       unlink (*tmppath);\r
+@@ -204,14 +205,31 @@ maildir_open_tmp_file (void *ctx, const char *dir,\r
+  * http://wiki.dovecot.org/MailboxFormat/Maildir#Mail_delivery\r
+  */\r
+ static notmuch_bool_t\r
+-maildir_move_tmp_to_new (const char *tmppath, const char *newpath)\r
++maildir_move_tmp_to_new (const char *tmppath, const char *newpath,\r
++                       const char *newdir)\r
+ {\r
++    notmuch_bool_t ret;\r
++    int fd;\r
++\r
+     if (rename (tmppath, newpath) != 0) {\r
+       fprintf (stderr, "Error: rename() failed: %s\n", strerror (errno));\r
+       return FALSE;\r
+     }\r
+ \r
+-    return TRUE;\r
++    /* Sync the 'new' directory after rename for durability. */\r
++    ret = TRUE;\r
++    fd = open (newdir, O_RDONLY);\r
++    if (fd == -1) {\r
++      fprintf (stderr, "Error: open() dir failed: %s\n", strerror (errno));\r
++      ret = FALSE;\r
++    }\r
++    if (ret && fsync (fd) != 0) {\r
++      fprintf (stderr, "Error: fsync() dir failed: %s\n", strerror (errno));\r
++      ret = FALSE;\r
++    }\r
++    if (fd != -1)\r
++      close (fd);\r
++    return ret;\r
+ }\r
+ \r
+ /* Copy the contents of fdin into fdout. */\r
+@@ -307,6 +325,12 @@ add_file_to_database (notmuch_database_t *notmuch, const char *path,\r
+ \r
+     notmuch_message_thaw (message);\r
+ \r
++    /* notmuch_message_tags_to_maildir_flags may rename the message file\r
++     * once more, and does so without fsyncing the directory afterwards.\r
++     * rename() is atomic so after a crash the file should appear under\r
++     * the old or new name. notmuch new should be able to rename the file\r
++     * again if required, so another fsync is not required, I think.\r
++     */\r
+     notmuch_message_tags_to_maildir_flags (message);\r
+ \r
+     notmuch_message_destroy (message);\r
+@@ -321,10 +345,11 @@ insert_message (void *ctx, notmuch_database_t *notmuch, int fdin,\r
+ {\r
+     char *tmppath;\r
+     char *newpath;\r
++    char *newdir;\r
+     int fdout;\r
+     notmuch_bool_t ret;\r
+ \r
+-    fdout = maildir_open_tmp_file (ctx, dir, &tmppath, &newpath);\r
++    fdout = maildir_open_tmp_file (ctx, dir, &tmppath, &newpath, &newdir);\r
+     if (fdout < 0) {\r
+       return FALSE;\r
+     }\r
+@@ -335,7 +360,7 @@ insert_message (void *ctx, notmuch_database_t *notmuch, int fdin,\r
+     }\r
+     close (fdout);\r
+     if (ret) {\r
+-      ret = maildir_move_tmp_to_new (tmppath, newpath);\r
++      ret = maildir_move_tmp_to_new (tmppath, newpath, newdir);\r
+     }\r
+     if (!ret) {\r
+       unlink (tmppath);\r
+-- \r
+1.7.12.1\r
+\r