From 9e1e6b3f4a737eab9ecfeb9d161e88c40e3352f2 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 29 Mar 2014 22:20:46 +2100 Subject: [PATCH] [PATCH] RFC: impliment gzipped output for notmuch dump --- 5f/1f40dc6b66332192feaebdb8f90b9d0e8e0864 | 273 ++++++++++++++++++++++ 1 file changed, 273 insertions(+) create mode 100644 5f/1f40dc6b66332192feaebdb8f90b9d0e8e0864 diff --git a/5f/1f40dc6b66332192feaebdb8f90b9d0e8e0864 b/5f/1f40dc6b66332192feaebdb8f90b9d0e8e0864 new file mode 100644 index 000000000..449ebad8d --- /dev/null +++ b/5f/1f40dc6b66332192feaebdb8f90b9d0e8e0864 @@ -0,0 +1,273 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id 49EF4431FBC + for ; Fri, 28 Mar 2014 18:21:07 -0700 (PDT) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Spam-Flag: NO +X-Spam-Score: 0 +X-Spam-Level: +X-Spam-Status: No, score=0 tagged_above=-999 required=5 tests=[none] + autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id 156cfDDvftAf for ; + Fri, 28 Mar 2014 18:21:01 -0700 (PDT) +Received: from yantan.tethera.net (yantan.tethera.net [199.188.72.155]) + (using TLSv1 with cipher DHE-RSA-AES128-SHA (128/128 bits)) + (No client certificate requested) + by olra.theworths.org (Postfix) with ESMTPS id C4DC3431FAF + for ; Fri, 28 Mar 2014 18:21:01 -0700 (PDT) +Received: from remotemail by yantan.tethera.net with local (Exim 4.80) + (envelope-from ) + id 1WThx9-0004XZ-MW; Fri, 28 Mar 2014 22:20:55 -0300 +Received: (nullmailer pid 2727 invoked by uid 1000); Sat, 29 Mar 2014 + 01:20:51 -0000 +From: David Bremner +To: notmuch@notmuchmail.org +Subject: [PATCH] RFC: impliment gzipped output for notmuch dump +Date: Fri, 28 Mar 2014 22:20:46 -0300 +Message-Id: <1396056046-2247-1-git-send-email-david@tethera.net> +X-Mailer: git-send-email 1.9.0 +In-Reply-To: <874n2kipq4.fsf@qmul.ac.uk> +References: <874n2kipq4.fsf@qmul.ac.uk> +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Sat, 29 Mar 2014 01:21:07 -0000 + +--- + +This obviously needs at least one piece of tidying, and some tests. + +My motivation here is the ability to make gzipped backups from within +notmuch, e.g. as part of an upgrade procedure. + + + Makefile.local | 2 +- + configure | 19 ++++++++++++++++++- + notmuch-client.h | 4 +++- + notmuch-dump.c | 41 +++++++++++++++++++++++++++-------------- + 4 files changed, 49 insertions(+), 17 deletions(-) + +diff --git a/Makefile.local b/Makefile.local +index cb7b106..e5a20a7 100644 +--- a/Makefile.local ++++ b/Makefile.local +@@ -41,7 +41,7 @@ PV_FILE=bindings/python/notmuch/version.py + # Smash together user's values with our extra values + FINAL_CFLAGS = -DNOTMUCH_VERSION=$(VERSION) $(CPPFLAGS) $(CFLAGS) $(WARN_CFLAGS) $(extra_cflags) $(CONFIGURE_CFLAGS) + FINAL_CXXFLAGS = $(CPPFLAGS) $(CXXFLAGS) $(WARN_CXXFLAGS) $(extra_cflags) $(extra_cxxflags) $(CONFIGURE_CXXFLAGS) +-FINAL_NOTMUCH_LDFLAGS = $(LDFLAGS) -Lutil -lutil -Llib -lnotmuch $(AS_NEEDED_LDFLAGS) $(GMIME_LDFLAGS) $(TALLOC_LDFLAGS) ++FINAL_NOTMUCH_LDFLAGS = $(LDFLAGS) -Lutil -lutil -Llib -lnotmuch $(AS_NEEDED_LDFLAGS) $(GMIME_LDFLAGS) $(TALLOC_LDFLAGS) $(ZLIB_LDFLAGS) + FINAL_NOTMUCH_LINKER = CC + ifneq ($(LINKER_RESOLVES_LIBRARY_DEPENDENCIES),1) + FINAL_NOTMUCH_LDFLAGS += $(CONFIGURE_LDFLAGS) +diff --git a/configure b/configure +index 1d430b9..02ef785 100755 +--- a/configure ++++ b/configure +@@ -340,6 +340,18 @@ else + errors=$((errors + 1)) + fi + ++printf "Checking for zlib development files... " ++have_zlib=0 ++if pkg-config --exists zlib; then ++ printf "Yes.\n" ++ have_zlib=1 ++ zlib_cflags=$(pkg-config --cflags zlib) ++ zlib_ldflags=$(pkg-config --libs zlib) ++else ++ printf "No.\n" ++ errors=$((errors + 1)) ++fi ++ + printf "Checking for talloc development files... " + if pkg-config --exists talloc; then + printf "Yes.\n" +@@ -844,6 +856,10 @@ XAPIAN_LDFLAGS = ${xapian_ldflags} + GMIME_CFLAGS = ${gmime_cflags} + GMIME_LDFLAGS = ${gmime_ldflags} + ++# Flags needed to compile and link against zlib ++ZLIB_CFLAGS = ${zlib_cflags} ++ZLIB_LDFLAGS = ${zlib_ldflags} ++ + # Flags needed to compile and link against talloc + TALLOC_CFLAGS = ${talloc_cflags} + TALLOC_LDFLAGS = ${talloc_ldflags} +@@ -882,6 +898,7 @@ CONFIGURE_CFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS) \\ + -DUTIL_BYTE_ORDER=\$(UTIL_BYTE_ORDER) + + CONFIGURE_CXXFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS) \\ ++ \$(ZLIB_CFLAGS) \\ + \$(TALLOC_CFLAGS) -DHAVE_VALGRIND=\$(HAVE_VALGRIND) \\ + \$(VALGRIND_CFLAGS) \$(XAPIAN_CXXFLAGS) \\ + -DHAVE_STRCASESTR=\$(HAVE_STRCASESTR) \\ +@@ -892,5 +909,5 @@ CONFIGURE_CXXFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS) \\ + -DHAVE_XAPIAN_COMPACT=\$(HAVE_XAPIAN_COMPACT) \\ + -DUTIL_BYTE_ORDER=\$(UTIL_BYTE_ORDER) + +-CONFIGURE_LDFLAGS = \$(GMIME_LDFLAGS) \$(TALLOC_LDFLAGS) \$(XAPIAN_LDFLAGS) ++CONFIGURE_LDFLAGS = \$(GMIME_LDFLAGS) \$(TALLOC_LDFLAGS) \$(ZLIB_LDFLAGS) \$(XAPIAN_LDFLAGS) + EOF +diff --git a/notmuch-client.h b/notmuch-client.h +index d110648..e1efbe0 100644 +--- a/notmuch-client.h ++++ b/notmuch-client.h +@@ -450,7 +450,9 @@ typedef enum dump_formats { + int + notmuch_database_dump (notmuch_database_t *notmuch, + const char *output_file_name, +- const char *query_str, dump_format_t output_format); ++ const char *query_str, ++ dump_format_t output_format, ++ notmuch_bool_t gzip_output); + + #include "command-line-arguments.h" + #endif +diff --git a/notmuch-dump.c b/notmuch-dump.c +index 21702d7..029f90a 100644 +--- a/notmuch-dump.c ++++ b/notmuch-dump.c +@@ -21,10 +21,12 @@ + #include "notmuch-client.h" + #include "hex-escape.h" + #include "string-util.h" ++#include ++ + + static int +-database_dump_file (notmuch_database_t *notmuch, FILE *output, +- const char *query_str, int output_format) ++database_dump_file (notmuch_database_t *notmuch, gzFile output, ++ const char *query_str, int output_format) + { + notmuch_query_t *query; + notmuch_messages_t *messages; +@@ -69,7 +71,7 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output, + } + + if (output_format == DUMP_FORMAT_SUP) { +- fprintf (output, "%s (", message_id); ++ gzprintf (output, "%s (", message_id); + } + + for (tags = notmuch_message_get_tags (message); +@@ -78,12 +80,12 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output, + const char *tag_str = notmuch_tags_get (tags); + + if (! first) +- fputs (" ", output); ++ gzputs (output, " "); + + first = 0; + + if (output_format == DUMP_FORMAT_SUP) { +- fputs (tag_str, output); ++ gzputs (output, tag_str); + } else { + if (hex_encode (notmuch, tag_str, + &buffer, &buffer_size) != HEX_SUCCESS) { +@@ -91,12 +93,12 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output, + tag_str); + return EXIT_FAILURE; + } +- fprintf (output, "+%s", buffer); ++ gzprintf (output, "+%s", buffer); + } + } + + if (output_format == DUMP_FORMAT_SUP) { +- fputs (")\n", output); ++ gzputs (output, ")\n"); + } else { + if (make_boolean_term (notmuch, "id", message_id, + &buffer, &buffer_size)) { +@@ -104,7 +106,7 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output, + message_id, strerror (errno)); + return EXIT_FAILURE; + } +- fprintf (output, " -- %s\n", buffer); ++ gzprintf (output, " -- %s\n", buffer); + } + + notmuch_message_destroy (message); +@@ -121,24 +123,33 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output, + int + notmuch_database_dump (notmuch_database_t *notmuch, + const char *output_file_name, +- const char *query_str, dump_format_t output_format) ++ const char *query_str, ++ dump_format_t output_format, ++ notmuch_bool_t gzip_output) + { +- FILE *output = stdout; ++ gzFile output; ++ const char *mode = gzip_output ? "w9" : "wT"; ++ + int ret; + + if (output_file_name) { +- output = fopen (output_file_name, "w"); ++ output = gzopen (output_file_name, mode); + if (output == NULL) { + fprintf (stderr, "Error opening %s for writing: %s\n", + output_file_name, strerror (errno)); + return EXIT_FAILURE; + } ++ } else { ++ output = gzdopen (fileno (stdout), mode); + } + + ret = database_dump_file (notmuch, output, query_str, output_format); + +- if (output != stdout) +- fclose (output); ++ /* XXX check error return */ ++ gzflush (output, Z_FINISH); ++ ++ if (output_file_name) ++ gzclose_w (output); + + return ret; + } +@@ -158,6 +169,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[]) + int opt_index; + + int output_format = DUMP_FORMAT_BATCH_TAG; ++ notmuch_bool_t gzip_output = 0; + + notmuch_opt_desc_t options[] = { + { NOTMUCH_OPT_KEYWORD, &output_format, "format", 'f', +@@ -165,6 +177,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[]) + { "batch-tag", DUMP_FORMAT_BATCH_TAG }, + { 0, 0 } } }, + { NOTMUCH_OPT_STRING, &output_file_name, "output", 'o', 0 }, ++ { NOTMUCH_OPT_BOOLEAN, &gzip_output, "gzip", 'z', 0 }, + { 0, 0, 0, 0, 0 } + }; + +@@ -181,7 +194,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[]) + } + + ret = notmuch_database_dump (notmuch, output_file_name, query_str, +- output_format); ++ output_format, gzip_output); + + notmuch_database_destroy (notmuch); + +-- +1.9.0 + -- 2.26.2