--- /dev/null
+Return-Path: <bremner@tethera.net>\r
+X-Original-To: notmuch@notmuchmail.org\r
+Delivered-To: notmuch@notmuchmail.org\r
+Received: from localhost (localhost [127.0.0.1])\r
+ by olra.theworths.org (Postfix) with ESMTP id 1C79C431FBD\r
+ for <notmuch@notmuchmail.org>; Sat, 29 Mar 2014 06:46:46 -0700 (PDT)\r
+X-Virus-Scanned: Debian amavisd-new at olra.theworths.org\r
+X-Spam-Flag: NO\r
+X-Spam-Score: 0\r
+X-Spam-Level: \r
+X-Spam-Status: No, score=0 tagged_above=-999 required=5 tests=[none]\r
+ autolearn=disabled\r
+Received: from olra.theworths.org ([127.0.0.1])\r
+ by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024)\r
+ with ESMTP id hglS1tXUu2UB for <notmuch@notmuchmail.org>;\r
+ Sat, 29 Mar 2014 06:46:38 -0700 (PDT)\r
+Received: from yantan.tethera.net (yantan.tethera.net [199.188.72.155])\r
+ (using TLSv1 with cipher DHE-RSA-AES128-SHA (128/128 bits))\r
+ (No client certificate requested)\r
+ by olra.theworths.org (Postfix) with ESMTPS id 7AC74431FBC\r
+ for <notmuch@notmuchmail.org>; Sat, 29 Mar 2014 06:46:38 -0700 (PDT)\r
+Received: from remotemail by yantan.tethera.net with local (Exim 4.80)\r
+ (envelope-from <bremner@tethera.net>)\r
+ id 1WTtag-0006uK-Rs; Sat, 29 Mar 2014 10:46:30 -0300\r
+Received: (nullmailer pid 8942 invoked by uid 1000); Sat, 29 Mar 2014\r
+ 13:46:27 -0000\r
+From: David Bremner <david@tethera.net>\r
+To: notmuch@notmuchmail.org\r
+Subject: [PATCH] dump: support gzipped output\r
+Date: Sat, 29 Mar 2014 10:46:25 -0300\r
+Message-Id: <1396100785-8744-1-git-send-email-david@tethera.net>\r
+X-Mailer: git-send-email 1.9.0\r
+In-Reply-To: <1396056046-2247-1-git-send-email-david@tethera.net>\r
+References: <1396056046-2247-1-git-send-email-david@tethera.net>\r
+X-BeenThere: notmuch@notmuchmail.org\r
+X-Mailman-Version: 2.1.13\r
+Precedence: list\r
+List-Id: "Use and development of the notmuch mail system."\r
+ <notmuch.notmuchmail.org>\r
+List-Unsubscribe: <http://notmuchmail.org/mailman/options/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>\r
+List-Archive: <http://notmuchmail.org/pipermail/notmuch>\r
+List-Post: <mailto:notmuch@notmuchmail.org>\r
+List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>\r
+List-Subscribe: <http://notmuchmail.org/mailman/listinfo/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=subscribe>\r
+X-List-Received-Date: Sat, 29 Mar 2014 13:46:46 -0000\r
+\r
+The main goal is to support gzipped output for future internal\r
+calls (e.g. from notmuch-new) to notmuch_database_dump.\r
+\r
+The additional dependency is not very heavy since xapian already pulls\r
+in zlib.\r
+---\r
+\r
+I had a quick look at supporting gzipped input for restore; I think it\r
+just requires an implementation of getline that uses gzgetc or gzgets;\r
+the decompression can be completely transparent to the user based on\r
+magic number.\r
+\r
+ INSTALL | 17 ++++++++++----\r
+ Makefile.local | 2 +-\r
+ configure | 23 ++++++++++++++++---\r
+ doc/man1/notmuch-dump.rst | 3 +++\r
+ notmuch-client.h | 4 +++-\r
+ notmuch-dump.c | 57 ++++++++++++++++++++++++++++++-----------------\r
+ test/T240-dump-restore.sh | 12 ++++++++++\r
+ 7 files changed, 89 insertions(+), 29 deletions(-)\r
+\r
+diff --git a/INSTALL b/INSTALL\r
+index 690b0ef..2754e52 100644\r
+--- a/INSTALL\r
++++ b/INSTALL\r
+@@ -20,8 +20,8 @@ configure stage.\r
+ \r
+ Dependencies\r
+ ------------\r
+-Notmuch depends on three libraries: Xapian, GMime 2.4 or 2.6, and\r
+-Talloc which are each described below:\r
++Notmuch depends on four libraries: Xapian, GMime 2.4 or 2.6,\r
++Talloc, and zlib which are each described below:\r
+ \r
+ Xapian\r
+ ------\r
+@@ -60,6 +60,15 @@ Talloc which are each described below:\r
+ \r
+ Talloc is available from http://talloc.samba.org/\r
+ \r
++ zlib\r
++ ----\r
++\r
++ zlib is an extremely popular compression library. It is used\r
++ by Xapian, so if you installed that you will already have\r
++ zlib. You may need to install the zlib headers seperately.\r
++\r
++ zlib is available from http://zlib.net\r
++\r
+ Building Documentation\r
+ ----------------------\r
+ \r
+@@ -79,11 +88,11 @@ dependencies with a simple simple command line. For example:\r
+ \r
+ For Debian and similar:\r
+ \r
+- sudo apt-get install libxapian-dev libgmime-2.6-dev libtalloc-dev python-sphinx\r
++ sudo apt-get install libxapian-dev libgmime-2.6-dev libtalloc-dev zlib1g-dev python-sphinx\r
+ \r
+ For Fedora and similar:\r
+ \r
+- sudo yum install xapian-core-devel gmime-devel libtalloc-devel python-sphinx\r
++ sudo yum install xapian-core-devel gmime-devel libtalloc-devel zlib-devel python-sphinx\r
+ \r
+ On other systems, a similar command can be used, but the details of\r
+ the package names may be different.\r
+diff --git a/Makefile.local b/Makefile.local\r
+index cb7b106..e5a20a7 100644\r
+--- a/Makefile.local\r
++++ b/Makefile.local\r
+@@ -41,7 +41,7 @@ PV_FILE=bindings/python/notmuch/version.py\r
+ # Smash together user's values with our extra values\r
+ FINAL_CFLAGS = -DNOTMUCH_VERSION=$(VERSION) $(CPPFLAGS) $(CFLAGS) $(WARN_CFLAGS) $(extra_cflags) $(CONFIGURE_CFLAGS)\r
+ FINAL_CXXFLAGS = $(CPPFLAGS) $(CXXFLAGS) $(WARN_CXXFLAGS) $(extra_cflags) $(extra_cxxflags) $(CONFIGURE_CXXFLAGS)\r
+-FINAL_NOTMUCH_LDFLAGS = $(LDFLAGS) -Lutil -lutil -Llib -lnotmuch $(AS_NEEDED_LDFLAGS) $(GMIME_LDFLAGS) $(TALLOC_LDFLAGS)\r
++FINAL_NOTMUCH_LDFLAGS = $(LDFLAGS) -Lutil -lutil -Llib -lnotmuch $(AS_NEEDED_LDFLAGS) $(GMIME_LDFLAGS) $(TALLOC_LDFLAGS) $(ZLIB_LDFLAGS)\r
+ FINAL_NOTMUCH_LINKER = CC\r
+ ifneq ($(LINKER_RESOLVES_LIBRARY_DEPENDENCIES),1)\r
+ FINAL_NOTMUCH_LDFLAGS += $(CONFIGURE_LDFLAGS)\r
+diff --git a/configure b/configure\r
+index 1d430b9..89bb3f3 100755\r
+--- a/configure\r
++++ b/configure\r
+@@ -340,6 +340,18 @@ else\r
+ errors=$((errors + 1))\r
+ fi\r
+ \r
++printf "Checking for zlib development files... "\r
++have_zlib=0\r
++if pkg-config --exists zlib; then\r
++ printf "Yes.\n"\r
++ have_zlib=1\r
++ zlib_cflags=$(pkg-config --cflags zlib)\r
++ zlib_ldflags=$(pkg-config --libs zlib)\r
++else\r
++ printf "No.\n"\r
++ errors=$((errors + 1))\r
++fi\r
++\r
+ printf "Checking for talloc development files... "\r
+ if pkg-config --exists talloc; then\r
+ printf "Yes.\n"\r
+@@ -519,11 +531,11 @@ case a simple command will install everything you need. For example:\r
+ \r
+ On Debian and similar systems:\r
+ \r
+- sudo apt-get install libxapian-dev libgmime-2.6-dev libtalloc-dev\r
++ sudo apt-get install libxapian-dev libgmime-2.6-dev libtalloc-dev zlib1g-dev\r
+ \r
+ Or on Fedora and similar systems:\r
+ \r
+- sudo yum install xapian-core-devel gmime-devel libtalloc-devel\r
++ sudo yum install xapian-core-devel gmime-devel libtalloc-devel zlib-devel\r
+ \r
+ On other systems, similar commands can be used, but the details of the\r
+ package names may be different.\r
+@@ -844,6 +856,10 @@ XAPIAN_LDFLAGS = ${xapian_ldflags}\r
+ GMIME_CFLAGS = ${gmime_cflags}\r
+ GMIME_LDFLAGS = ${gmime_ldflags}\r
+ \r
++# Flags needed to compile and link against zlib\r
++ZLIB_CFLAGS = ${zlib_cflags}\r
++ZLIB_LDFLAGS = ${zlib_ldflags}\r
++\r
+ # Flags needed to compile and link against talloc\r
+ TALLOC_CFLAGS = ${talloc_cflags}\r
+ TALLOC_LDFLAGS = ${talloc_ldflags}\r
+@@ -882,6 +898,7 @@ CONFIGURE_CFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS) \\\r
+ -DUTIL_BYTE_ORDER=\$(UTIL_BYTE_ORDER)\r
+ \r
+ CONFIGURE_CXXFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS) \\\r
++ \$(ZLIB_CFLAGS) \\\r
+ \$(TALLOC_CFLAGS) -DHAVE_VALGRIND=\$(HAVE_VALGRIND) \\\r
+ \$(VALGRIND_CFLAGS) \$(XAPIAN_CXXFLAGS) \\\r
+ -DHAVE_STRCASESTR=\$(HAVE_STRCASESTR) \\\r
+@@ -892,5 +909,5 @@ CONFIGURE_CXXFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS) \\\r
+ -DHAVE_XAPIAN_COMPACT=\$(HAVE_XAPIAN_COMPACT) \\\r
+ -DUTIL_BYTE_ORDER=\$(UTIL_BYTE_ORDER)\r
+ \r
+-CONFIGURE_LDFLAGS = \$(GMIME_LDFLAGS) \$(TALLOC_LDFLAGS) \$(XAPIAN_LDFLAGS)\r
++CONFIGURE_LDFLAGS = \$(GMIME_LDFLAGS) \$(TALLOC_LDFLAGS) \$(ZLIB_LDFLAGS) \$(XAPIAN_LDFLAGS)\r
+ EOF\r
+diff --git a/doc/man1/notmuch-dump.rst b/doc/man1/notmuch-dump.rst\r
+index 17d1da5..d94cb4f 100644\r
+--- a/doc/man1/notmuch-dump.rst\r
++++ b/doc/man1/notmuch-dump.rst\r
+@@ -19,6 +19,9 @@ recreated from the messages themselves. The output of notmuch dump is\r
+ therefore the only critical thing to backup (and much more friendly to\r
+ incremental backup than the native database files.)\r
+ \r
++``--gzip``\r
++ Compress the output in a format compatible with **gzip(1)**.\r
++\r
+ ``--format=(sup|batch-tag)``\r
+ Notmuch restore supports two plain text dump formats, both with one\r
+ message-id per line, followed by a list of tags.\r
+diff --git a/notmuch-client.h b/notmuch-client.h\r
+index d110648..e1efbe0 100644\r
+--- a/notmuch-client.h\r
++++ b/notmuch-client.h\r
+@@ -450,7 +450,9 @@ typedef enum dump_formats {\r
+ int\r
+ notmuch_database_dump (notmuch_database_t *notmuch,\r
+ const char *output_file_name,\r
+- const char *query_str, dump_format_t output_format);\r
++ const char *query_str,\r
++ dump_format_t output_format,\r
++ notmuch_bool_t gzip_output);\r
+ \r
+ #include "command-line-arguments.h"\r
+ #endif\r
+diff --git a/notmuch-dump.c b/notmuch-dump.c\r
+index 21702d7..128a37d 100644\r
+--- a/notmuch-dump.c\r
++++ b/notmuch-dump.c\r
+@@ -21,10 +21,12 @@\r
+ #include "notmuch-client.h"\r
+ #include "hex-escape.h"\r
+ #include "string-util.h"\r
++#include <zlib.h>\r
++\r
+ \r
+ static int\r
+-database_dump_file (notmuch_database_t *notmuch, FILE *output,\r
+- const char *query_str, int output_format)\r
++database_dump_file (notmuch_database_t *notmuch, gzFile output,\r
++ const char *query_str, int output_format)\r
+ {\r
+ notmuch_query_t *query;\r
+ notmuch_messages_t *messages;\r
+@@ -69,7 +71,7 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output,\r
+ }\r
+ \r
+ if (output_format == DUMP_FORMAT_SUP) {\r
+- fprintf (output, "%s (", message_id);\r
++ gzprintf (output, "%s (", message_id);\r
+ }\r
+ \r
+ for (tags = notmuch_message_get_tags (message);\r
+@@ -78,12 +80,12 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output,\r
+ const char *tag_str = notmuch_tags_get (tags);\r
+ \r
+ if (! first)\r
+- fputs (" ", output);\r
++ gzputs (output, " ");\r
+ \r
+ first = 0;\r
+ \r
+ if (output_format == DUMP_FORMAT_SUP) {\r
+- fputs (tag_str, output);\r
++ gzputs (output, tag_str);\r
+ } else {\r
+ if (hex_encode (notmuch, tag_str,\r
+ &buffer, &buffer_size) != HEX_SUCCESS) {\r
+@@ -91,12 +93,12 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output,\r
+ tag_str);\r
+ return EXIT_FAILURE;\r
+ }\r
+- fprintf (output, "+%s", buffer);\r
++ gzprintf (output, "+%s", buffer);\r
+ }\r
+ }\r
+ \r
+ if (output_format == DUMP_FORMAT_SUP) {\r
+- fputs (")\n", output);\r
++ gzputs (output, ")\n");\r
+ } else {\r
+ if (make_boolean_term (notmuch, "id", message_id,\r
+ &buffer, &buffer_size)) {\r
+@@ -104,7 +106,7 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output,\r
+ message_id, strerror (errno));\r
+ return EXIT_FAILURE;\r
+ }\r
+- fprintf (output, " -- %s\n", buffer);\r
++ gzprintf (output, " -- %s\n", buffer);\r
+ }\r
+ \r
+ notmuch_message_destroy (message);\r
+@@ -121,24 +123,37 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output,\r
+ int\r
+ notmuch_database_dump (notmuch_database_t *notmuch,\r
+ const char *output_file_name,\r
+- const char *query_str, dump_format_t output_format)\r
++ const char *query_str,\r
++ dump_format_t output_format,\r
++ notmuch_bool_t gzip_output)\r
+ {\r
+- FILE *output = stdout;\r
++ gzFile output;\r
++ const char *mode = gzip_output ? "w9" : "wT";\r
++\r
+ int ret;\r
+ \r
+- if (output_file_name) {\r
+- output = fopen (output_file_name, "w");\r
+- if (output == NULL) {\r
+- fprintf (stderr, "Error opening %s for writing: %s\n",\r
+- output_file_name, strerror (errno));\r
+- return EXIT_FAILURE;\r
+- }\r
++ if (output_file_name)\r
++ output = gzopen (output_file_name, mode);\r
++ else\r
++ output = gzdopen (fileno (stdout), mode);\r
++\r
++ if (output == NULL) {\r
++ fprintf (stderr, "Error opening %s for (gzip) writing: %s\n",\r
++ output_file_name || "stdout", strerror (errno));\r
++ return EXIT_FAILURE;\r
+ }\r
+ \r
+ ret = database_dump_file (notmuch, output, query_str, output_format);\r
+ \r
+- if (output != stdout)\r
+- fclose (output);\r
++ /* unlike stdio, zlib needs explicit flushing */\r
++ if (gzflush (output, Z_FINISH)) {\r
++ fprintf (stderr, "Error flushing output: %s\n",\r
++ gzerror (output, NULL));\r
++ return EXIT_FAILURE;\r
++ }\r
++\r
++ if (output_file_name)\r
++ gzclose_w (output);\r
+ \r
+ return ret;\r
+ }\r
+@@ -158,6 +173,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])\r
+ int opt_index;\r
+ \r
+ int output_format = DUMP_FORMAT_BATCH_TAG;\r
++ notmuch_bool_t gzip_output = 0;\r
+ \r
+ notmuch_opt_desc_t options[] = {\r
+ { NOTMUCH_OPT_KEYWORD, &output_format, "format", 'f',\r
+@@ -165,6 +181,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])\r
+ { "batch-tag", DUMP_FORMAT_BATCH_TAG },\r
+ { 0, 0 } } },\r
+ { NOTMUCH_OPT_STRING, &output_file_name, "output", 'o', 0 },\r
++ { NOTMUCH_OPT_BOOLEAN, &gzip_output, "gzip", 'z', 0 },\r
+ { 0, 0, 0, 0, 0 }\r
+ };\r
+ \r
+@@ -181,7 +198,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])\r
+ }\r
+ \r
+ ret = notmuch_database_dump (notmuch, output_file_name, query_str,\r
+- output_format);\r
++ output_format, gzip_output);\r
+ \r
+ notmuch_database_destroy (notmuch);\r
+ \r
+diff --git a/test/T240-dump-restore.sh b/test/T240-dump-restore.sh\r
+index 0004438..d79aca8 100755\r
+--- a/test/T240-dump-restore.sh\r
++++ b/test/T240-dump-restore.sh\r
+@@ -68,6 +68,18 @@ test_begin_subtest "dump --output=outfile --"\r
+ notmuch dump --output=dump-1-arg-dash.actual --\r
+ test_expect_equal_file dump.expected dump-1-arg-dash.actual\r
+ \r
++# gzipped output\r
++\r
++test_begin_subtest "dump --gzip"\r
++notmuch dump --gzip > dump-gzip.gz\r
++gunzip dump-gzip.gz\r
++test_expect_equal_file dump.expected dump-gzip\r
++\r
++test_begin_subtest "dump --gzip --output=outfile"\r
++notmuch dump --gzip --output=dump-gzip-outfile.gz\r
++gunzip dump-gzip-outfile.gz\r
++test_expect_equal_file dump.expected dump-gzip-outfile\r
++\r
+ # Note, we assume all messages from cworth have a message-id\r
+ # containing cworth.org\r
+ \r
+-- \r
+1.9.0\r
+\r
projects / notmuch-archives.git / commitdiff