--- /dev/null
+Return-Path: <david@tethera.net>\r
+X-Original-To: notmuch@notmuchmail.org\r
+Delivered-To: notmuch@notmuchmail.org\r
+Received: from localhost (localhost [127.0.0.1])\r
+ by olra.theworths.org (Postfix) with ESMTP id 8EF5F431FBD\r
+ for <notmuch@notmuchmail.org>; Thu, 3 Apr 2014 20:13:18 -0700 (PDT)\r
+X-Virus-Scanned: Debian amavisd-new at olra.theworths.org\r
+X-Spam-Flag: NO\r
+X-Spam-Score: 0\r
+X-Spam-Level: \r
+X-Spam-Status: No, score=0 tagged_above=-999 required=5 tests=[none]\r
+ autolearn=disabled\r
+Received: from olra.theworths.org ([127.0.0.1])\r
+ by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024)\r
+ with ESMTP id YJRyfhJPe09T for <notmuch@notmuchmail.org>;\r
+ Thu, 3 Apr 2014 20:13:11 -0700 (PDT)\r
+Received: from mx.xen14.node3324.gplhost.com (gitolite.debian.net\r
+ [87.98.215.224]) (using TLSv1 with cipher AES256-SHA (256/256 bits))\r
+ (No client certificate requested)\r
+ by olra.theworths.org (Postfix) with ESMTPS id 51C54431FAF\r
+ for <notmuch@notmuchmail.org>; Thu, 3 Apr 2014 20:13:11 -0700 (PDT)\r
+Received: from remotemail by mx.xen14.node3324.gplhost.com with local (Exim\r
+ 4.72) (envelope-from <david@tethera.net>)\r
+ id 1WVtwF-0001DY-LD; Fri, 04 Apr 2014 02:33:03 +0000\r
+Received: (nullmailer pid 24864 invoked by uid 1000); Fri, 04 Apr 2014\r
+ 02:32:44 -0000\r
+From: David Bremner <david@tethera.net>\r
+To: notmuch@notmuchmail.org\r
+Subject: Re: [Patch v6 1/6] dump: support gzipped and atomic output\r
+In-Reply-To: <1396554083-3892-2-git-send-email-david@tethera.net>\r
+References: <1396554083-3892-1-git-send-email-david@tethera.net>\r
+ <1396554083-3892-2-git-send-email-david@tethera.net>\r
+User-Agent: Notmuch/0.17+56~gb9c3d8e (http://notmuchmail.org) Emacs/24.3.1\r
+ (x86_64-pc-linux-gnu)\r
+Date: Thu, 03 Apr 2014 23:32:44 -0300\r
+Message-ID: <877g75rfhv.fsf@maritornes.cs.unb.ca>\r
+MIME-Version: 1.0\r
+Content-Type: text/plain\r
+X-BeenThere: notmuch@notmuchmail.org\r
+X-Mailman-Version: 2.1.13\r
+Precedence: list\r
+List-Id: "Use and development of the notmuch mail system."\r
+ <notmuch.notmuchmail.org>\r
+List-Unsubscribe: <http://notmuchmail.org/mailman/options/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>\r
+List-Archive: <http://notmuchmail.org/pipermail/notmuch>\r
+List-Post: <mailto:notmuch@notmuchmail.org>\r
+List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>\r
+List-Subscribe: <http://notmuchmail.org/mailman/listinfo/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=subscribe>\r
+X-List-Received-Date: Fri, 04 Apr 2014 03:13:18 -0000\r
+\r
+David Bremner <david@tethera.net> writes:\r
+\r
+> The main goal is to support gzipped output for future internal\r
+> calls (e.g. from notmuch-new) to notmuch_database_dump.\r
+\r
+Apologies if this later turns out to be duplicate. I'm not sure what\r
+happened to the cover letter for this series.\r
+\r
+This supercedes \r
+\r
+id:1396401381-18128-1-git-send-email-david@tethera.net\r
+\r
+I tried to take care of Austin and Tomi's comments, but there were\r
+several messages so hopefully I didn't miss anything.\r
+\r
+I ended up merging the "gzipped output" and "atomic output" patches; I\r
+fought it for a bit, but I realized I would have to do some of the\r
+proposed changes twice and throw away the first one.\r
+\r
+diff from v5 follows:\r
+\r
+diff --git a/INSTALL b/INSTALL\r
+index df318fa..b543c50 100644\r
+--- a/INSTALL\r
++++ b/INSTALL\r
+@@ -67,6 +67,9 @@ Talloc, and zlib which are each described below:\r
+ by Xapian, so if you installed that you will already have\r
+ zlib. You may need to install the zlib headers separately.\r
+ \r
++ Notmuch needs the transparent write feature of zlib introduced\r
++ in version 1.2.5.2 (Dec. 2011).\r
++\r
+ zlib is available from http://zlib.net\r
+ \r
+ Building Documentation\r
+diff --git a/configure b/configure\r
+index d685ab3..1d624f7 100755\r
+--- a/configure\r
++++ b/configure\r
+@@ -340,9 +340,9 @@ else\r
+ errors=$((errors + 1))\r
+ fi\r
+ \r
+-printf "Checking for zlib development files... "\r
++printf "Checking for zlib (>= 1.2.5.2)... "\r
+ have_zlib=0\r
+-if pkg-config --exists zlib; then\r
++if pkg-config --atleast-version=1.2.5.2 zlib; then\r
+ printf "Yes.\n"\r
+ have_zlib=1\r
+ zlib_cflags=$(pkg-config --cflags zlib)\r
+diff --git a/notmuch-dump.c b/notmuch-dump.c\r
+index 05ed6b4..2a7252a 100644\r
+--- a/notmuch-dump.c\r
++++ b/notmuch-dump.c\r
+@@ -140,7 +140,7 @@ notmuch_database_dump (notmuch_database_t *notmuch,\r
+ tempname = talloc_asprintf (notmuch, "%s.XXXXXX", output_file_name);\r
+ outfd = mkstemp (tempname);\r
+ } else {\r
+- outfd = fileno (stdout);\r
++ outfd = dup (STDOUT_FILENO);\r
+ }\r
+ \r
+ if (outfd < 0) {\r
+@@ -153,6 +153,9 @@ notmuch_database_dump (notmuch_database_t *notmuch,\r
+ if (output == NULL) {\r
+ fprintf (stderr, "Error opening %s for (gzip) writing: %s\n",\r
+ name_for_error, strerror (errno));\r
++ if (close (outfd))\r
++ fprintf (stderr, "Error closing %s during shutdown: %s\n",\r
++ name_for_error, strerror (errno));\r
+ goto DONE;\r
+ }\r
+ \r
+@@ -165,22 +168,25 @@ notmuch_database_dump (notmuch_database_t *notmuch,\r
+ goto DONE;\r
+ }\r
+ \r
+- ret = fdatasync (outfd);\r
+- if (ret) {\r
+- perror ("fdatasync");\r
+- goto DONE;\r
+- }\r
+-\r
+ if (output_file_name) {\r
+- ret = gzclose_w (output);\r
+- if (ret != Z_OK) {\r
+- ret = EXIT_FAILURE;\r
++ ret = fdatasync (outfd);\r
++ if (ret) {\r
++ fprintf (stderr, "Error syncing %s to disk: %s\n",\r
++ name_for_error, strerror (errno));\r
+ goto DONE;\r
+ }\r
++ }\r
+ \r
++ if (gzclose_w (output) != Z_OK) {\r
++ ret = EXIT_FAILURE;\r
++ goto DONE;\r
++ }\r
++\r
++ if (output_file_name) {\r
+ ret = rename (tempname, output_file_name);\r
+ if (ret) {\r
+- perror ("rename");\r
++ fprintf (stderr, "Error renaming %s to %s: %s\n",\r
++ tempname, output_file_name, strerror (errno));\r
+ goto DONE;\r
+ }\r
+ \r
+diff --git a/notmuch-new.c b/notmuch-new.c\r
+index e0431c6..d269c7c 100644\r
+--- a/notmuch-new.c\r
++++ b/notmuch-new.c\r
+@@ -997,7 +997,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[])\r
+ * relatively small. */\r
+ \r
+ const char *backup_name =\r
+- talloc_asprintf (notmuch, "%s/backup-%04d-%02d-%02d-%02d:%02d:%02d.gz",\r
++ talloc_asprintf (notmuch, "%s/dump-%04d%02d%02dT%02d%02d%02d.gz",\r
+ dot_notmuch_path,\r
+ gm_time->tm_year + 1900,\r
+ gm_time->tm_mon + 1,\r
+@@ -1009,12 +1009,12 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[])\r
+ if (add_files_state.verbosity >= VERBOSITY_NORMAL) {\r
+ printf ("Welcome to a new version of notmuch! Your database will now be upgraded.\n");\r
+ printf ("This process is safe to interrupt.\n");\r
+- printf ("Backing up tags to %s\n", backup_name);\r
++ printf ("Backing up tags to %s...\n", backup_name);\r
+ }\r
+ \r
+ if (notmuch_database_dump (notmuch, backup_name, "",\r
+ DUMP_FORMAT_BATCH_TAG, TRUE)) {\r
+- fprintf (stderr, "Backup failed. Aborting upgrade");\r
++ fprintf (stderr, "Backup failed. Aborting upgrade.");\r
+ return EXIT_FAILURE;\r
+ }\r
+ \r
+diff --git a/notmuch-restore.c b/notmuch-restore.c\r
+index 86bce20..eb5b7b2 100644\r
+--- a/notmuch-restore.c\r
++++ b/notmuch-restore.c\r
+@@ -132,7 +132,6 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])\r
+ gzFile input;\r
+ char *line = NULL;\r
+ void *line_ctx = NULL;\r
+- size_t line_size;\r
+ ssize_t line_len;\r
+ \r
+ int ret = 0;\r
+@@ -166,8 +165,16 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])\r
+ \r
+ if (input_file_name)\r
+ input = gzopen (input_file_name, "r");\r
+- else\r
+- input = gzdopen (fileno (stdin), "r");\r
++ else {\r
++ int infd = dup (STDIN_FILENO);\r
++ if (infd < 0) {\r
++ fprintf (stderr, "Error duping stdin\n");\r
++ return EXIT_FAILURE;\r
++ }\r
++ input = gzdopen (infd, "r");\r
++ if (! input)\r
++ close (infd);\r
++ }\r
+ \r
+ if (input == NULL) {\r
+ fprintf (stderr, "Error opening %s for (gzip) reading: %s\n",\r
+@@ -189,7 +196,7 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])\r
+ do {\r
+ util_status_t status;\r
+ \r
+- status = gz_getline (line_ctx, &line, &line_size, &line_len, input);\r
++ status = gz_getline (line_ctx, &line, &line_len, input);\r
+ \r
+ /* empty input file not considered an error */\r
+ if (status == UTIL_EOF)\r
+@@ -262,7 +269,7 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])\r
+ if (ret)\r
+ break;\r
+ \r
+- } while (gz_getline (line_ctx, &line, &line_size, &line_len, input) == UTIL_SUCCESS);\r
++ } while (gz_getline (line_ctx, &line, &line_len, input) == UTIL_SUCCESS);\r
+ \r
+ if (line_ctx != NULL)\r
+ talloc_free (line_ctx);\r
+@@ -272,8 +279,7 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])\r
+ \r
+ notmuch_database_destroy (notmuch);\r
+ \r
+- if (input_file_name != NULL)\r
+- gzclose_r (input);\r
++ gzclose_r (input);\r
+ \r
+ return ret ? EXIT_FAILURE : EXIT_SUCCESS;\r
+ }\r
+diff --git a/test/T240-dump-restore.sh b/test/T240-dump-restore.sh\r
+index 50d4d48..efe463e 100755\r
+--- a/test/T240-dump-restore.sh\r
++++ b/test/T240-dump-restore.sh\r
+@@ -124,6 +124,15 @@ notmuch dump --format=batch-tag from:cworth | sed 's/^.*-- id://' | \\r
+ sort > OUTPUT.$test_count\r
+ test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count\r
+ \r
++test_begin_subtest "format=batch-tag, missing newline"\r
++printf "+a_tag_without_newline -- id:20091117232137.GA7669@griffis1.net" > IN\r
++notmuch restore --accumulate < IN\r
++notmuch dump id:20091117232137.GA7669@griffis1.net > OUT\r
++cat <<EOF > EXPECTED\r
+++a_tag_without_newline +inbox +unread -- id:20091117232137.GA7669@griffis1.net\r
++EOF\r
++test_expect_equal_file EXPECTED OUT\r
++\r
+ test_begin_subtest "format=batch-tag, # round-trip"\r
+ notmuch dump --format=sup | sort > EXPECTED.$test_count\r
+ notmuch dump --format=batch-tag | notmuch restore --format=batch-tag\r
+diff --git a/test/T530-upgrade.sh b/test/T530-upgrade.sh\r
+index 7a68ddf..7d5d5aa 100755\r
+--- a/test/T530-upgrade.sh\r
++++ b/test/T530-upgrade.sh\r
+@@ -37,7 +37,7 @@ Your notmuch database has now been upgraded to database format version 2.\r
+ No new mail."\r
+ \r
+ test_begin_subtest "tag backup matches pre-upgrade dump"\r
+-gunzip -c ${MAIL_DIR}/.notmuch/backup-*.gz | sort > backup-dump\r
++gunzip -c ${MAIL_DIR}/.notmuch/dump-*.gz | sort > backup-dump\r
+ test_expect_equal_file pre-upgrade-dump backup-dump\r
+ \r
+ test_begin_subtest "folder: no longer matches in the middle of path"\r
+diff --git a/util/zlib-extra.c b/util/zlib-extra.c\r
+index cb1eba0..922ab52 100644\r
+--- a/util/zlib-extra.c\r
++++ b/util/zlib-extra.c\r
+@@ -25,34 +25,36 @@\r
+ \r
+ /* mimic POSIX/glibc getline, but on a zlib gzFile stream, and using talloc */\r
+ util_status_t\r
+-gz_getline (void *talloc_ctx, char **bufptr, size_t *bufsiz, ssize_t *bytes_read,\r
+- gzFile stream)\r
++gz_getline (void *talloc_ctx, char **bufptr, ssize_t *bytes_read, gzFile stream)\r
+ {\r
+- size_t len = *bufsiz;\r
+ char *buf = *bufptr;\r
++ unsigned int len;\r
+ size_t offset = 0;\r
+ \r
+- if (len == 0 || buf == NULL) {\r
++ if (buf) {\r
++ len = talloc_array_length (buf);\r
++ } else {\r
+ /* same as getdelim from gnulib */\r
+ len = 120;\r
+- buf = talloc_size (talloc_ctx, len);\r
++ buf = talloc_array (talloc_ctx, char, len);\r
+ if (buf == NULL)\r
+ return UTIL_OUT_OF_MEMORY;\r
+ }\r
+ \r
+ while (1) {\r
+ if (! gzgets (stream, buf + offset, len - offset)) {\r
++ /* Null indicates EOF or error */\r
+ int zlib_status = 0;\r
+ (void) gzerror (stream, &zlib_status);\r
+ switch (zlib_status) {\r
+ case Z_OK:\r
+- /* follow getline behaviour */\r
+- *bytes_read = -1;\r
+- return UTIL_EOF;\r
+- break;\r
++ /* no data read before EOF */\r
++ if (offset == 0)\r
++ return UTIL_EOF;\r
++ else\r
++ goto SUCCESS;\r
+ case Z_ERRNO:\r
+ return UTIL_FILE;\r
+- break;\r
+ default:\r
+ return UTIL_ERROR;\r
+ }\r
+@@ -60,17 +62,16 @@ gz_getline (void *talloc_ctx, char **bufptr, size_t *bufsiz, ssize_t *bytes_read\r
+ \r
+ offset += strlen (buf + offset);\r
+ \r
+- if ( buf[offset - 1] == '\n' )\r
+- break;\r
++ if (buf[offset - 1] == '\n')\r
++ goto SUCCESS;\r
+ \r
+ len *= 2;\r
+ buf = talloc_realloc (talloc_ctx, buf, char, len);\r
+ if (buf == NULL)\r
+ return UTIL_OUT_OF_MEMORY;\r
+ }\r
+-\r
++ SUCCESS:\r
+ *bufptr = buf;\r
+- *bufsiz = len;\r
+ *bytes_read = offset;\r
+ return UTIL_SUCCESS;\r
+ }\r
+diff --git a/util/zlib-extra.h b/util/zlib-extra.h\r
+index ed46ac1..ed67115 100644\r
+--- a/util/zlib-extra.h\r
++++ b/util/zlib-extra.h\r
+@@ -1,11 +1,11 @@\r
+ #ifndef _ZLIB_EXTRA_H\r
+ #define _ZLIB_EXTRA_H\r
+ \r
+-#include <zlib.h>\r
+ #include "util.h"\r
++#include <zlib.h>\r
++\r
+ /* Like getline, but read from a gzFile. Allocation is with talloc */\r
+ util_status_t\r
+-gz_getline (void *ctx, char **lineptr, size_t *line_size, ssize_t *bytes_read,\r
+- gzFile stream);\r
++gz_getline (void *ctx, char **lineptr, ssize_t *bytes_read, gzFile stream);\r
+ \r
+ #endif\r
projects / notmuch-archives.git / commitdiff