Return-Path: X-Original-To: notmuch@notmuchmail.org Delivered-To: notmuch@notmuchmail.org Received: from localhost (localhost [127.0.0.1]) by olra.theworths.org (Postfix) with ESMTP id 19317431FBF for ; Sat, 29 Mar 2014 00:16:21 -0700 (PDT) X-Virus-Scanned: Debian amavisd-new at olra.theworths.org X-Spam-Flag: NO X-Spam-Score: 0 X-Spam-Level: X-Spam-Status: No, score=0 tagged_above=-999 required=5 tests=[none] autolearn=disabled Received: from olra.theworths.org ([127.0.0.1]) by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id AfuJms+9iLHJ for ; Sat, 29 Mar 2014 00:16:17 -0700 (PDT) Received: from guru.guru-group.fi (guru.guru-group.fi [46.183.73.34]) by olra.theworths.org (Postfix) with ESMTP id 6FAEA431FBC for ; Sat, 29 Mar 2014 00:16:17 -0700 (PDT) Received: from guru.guru-group.fi (localhost [IPv6:::1]) by guru.guru-group.fi (Postfix) with ESMTP id C169E1000E5; Sat, 29 Mar 2014 09:16:11 +0200 (EET) From: Tomi Ollila To: David Bremner , notmuch@notmuchmail.org Subject: Re: [PATCH] RFC: impliment gzipped output for notmuch dump In-Reply-To: <1396056046-2247-1-git-send-email-david@tethera.net> References: <874n2kipq4.fsf@qmul.ac.uk> <1396056046-2247-1-git-send-email-david@tethera.net> User-Agent: Notmuch/0.17+159~gb806a13 (http://notmuchmail.org) Emacs/24.3.1 (x86_64-unknown-linux-gnu) X-Face: HhBM'cA~ MIME-Version: 1.0 Content-Type: text/plain X-BeenThere: notmuch@notmuchmail.org X-Mailman-Version: 2.1.13 Precedence: list List-Id: "Use and development of the notmuch mail system." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 29 Mar 2014 07:16:21 -0000 On Sat, Mar 29 2014, David Bremner wrote: > --- > > This obviously needs at least one piece of tidying, and some tests. > > My motivation here is the ability to make gzipped backups from within > notmuch, e.g. as part of an upgrade procedure. +1 > > > Makefile.local | 2 +- > configure | 19 ++++++++++++++++++- > notmuch-client.h | 4 +++- > notmuch-dump.c | 41 +++++++++++++++++++++++++++-------------- > 4 files changed, 49 insertions(+), 17 deletions(-) > > diff --git a/Makefile.local b/Makefile.local > index cb7b106..e5a20a7 100644 > --- a/Makefile.local > +++ b/Makefile.local > @@ -41,7 +41,7 @@ PV_FILE=bindings/python/notmuch/version.py > # Smash together user's values with our extra values > FINAL_CFLAGS = -DNOTMUCH_VERSION=$(VERSION) $(CPPFLAGS) $(CFLAGS) $(WARN_CFLAGS) $(extra_cflags) $(CONFIGURE_CFLAGS) > FINAL_CXXFLAGS = $(CPPFLAGS) $(CXXFLAGS) $(WARN_CXXFLAGS) $(extra_cflags) $(extra_cxxflags) $(CONFIGURE_CXXFLAGS) > -FINAL_NOTMUCH_LDFLAGS = $(LDFLAGS) -Lutil -lutil -Llib -lnotmuch $(AS_NEEDED_LDFLAGS) $(GMIME_LDFLAGS) $(TALLOC_LDFLAGS) > +FINAL_NOTMUCH_LDFLAGS = $(LDFLAGS) -Lutil -lutil -Llib -lnotmuch $(AS_NEEDED_LDFLAGS) $(GMIME_LDFLAGS) $(TALLOC_LDFLAGS) $(ZLIB_LDFLAGS) > FINAL_NOTMUCH_LINKER = CC > ifneq ($(LINKER_RESOLVES_LIBRARY_DEPENDENCIES),1) > FINAL_NOTMUCH_LDFLAGS += $(CONFIGURE_LDFLAGS) > diff --git a/configure b/configure > index 1d430b9..02ef785 100755 > --- a/configure > +++ b/configure > @@ -340,6 +340,18 @@ else > errors=$((errors + 1)) > fi > > +printf "Checking for zlib development files... " > +have_zlib=0 > +if pkg-config --exists zlib; then > + printf "Yes.\n" > + have_zlib=1 > + zlib_cflags=$(pkg-config --cflags zlib) > + zlib_ldflags=$(pkg-config --libs zlib) > +else > + printf "No.\n" > + errors=$((errors + 1)) > +fi > + > printf "Checking for talloc development files... " > if pkg-config --exists talloc; then > printf "Yes.\n" > @@ -844,6 +856,10 @@ XAPIAN_LDFLAGS = ${xapian_ldflags} > GMIME_CFLAGS = ${gmime_cflags} > GMIME_LDFLAGS = ${gmime_ldflags} > > +# Flags needed to compile and link against zlib > +ZLIB_CFLAGS = ${zlib_cflags} > +ZLIB_LDFLAGS = ${zlib_ldflags} > + > # Flags needed to compile and link against talloc > TALLOC_CFLAGS = ${talloc_cflags} > TALLOC_LDFLAGS = ${talloc_ldflags} > @@ -882,6 +898,7 @@ CONFIGURE_CFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS) \\ > -DUTIL_BYTE_ORDER=\$(UTIL_BYTE_ORDER) > > CONFIGURE_CXXFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS) \\ > + \$(ZLIB_CFLAGS) \\ > \$(TALLOC_CFLAGS) -DHAVE_VALGRIND=\$(HAVE_VALGRIND) \\ > \$(VALGRIND_CFLAGS) \$(XAPIAN_CXXFLAGS) \\ > -DHAVE_STRCASESTR=\$(HAVE_STRCASESTR) \\ > @@ -892,5 +909,5 @@ CONFIGURE_CXXFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS) \\ > -DHAVE_XAPIAN_COMPACT=\$(HAVE_XAPIAN_COMPACT) \\ > -DUTIL_BYTE_ORDER=\$(UTIL_BYTE_ORDER) > > -CONFIGURE_LDFLAGS = \$(GMIME_LDFLAGS) \$(TALLOC_LDFLAGS) \$(XAPIAN_LDFLAGS) > +CONFIGURE_LDFLAGS = \$(GMIME_LDFLAGS) \$(TALLOC_LDFLAGS) \$(ZLIB_LDFLAGS) \$(XAPIAN_LDFLAGS) > EOF > diff --git a/notmuch-client.h b/notmuch-client.h > index d110648..e1efbe0 100644 > --- a/notmuch-client.h > +++ b/notmuch-client.h > @@ -450,7 +450,9 @@ typedef enum dump_formats { > int > notmuch_database_dump (notmuch_database_t *notmuch, > const char *output_file_name, > - const char *query_str, dump_format_t output_format); > + const char *query_str, > + dump_format_t output_format, > + notmuch_bool_t gzip_output); > > #include "command-line-arguments.h" > #endif > diff --git a/notmuch-dump.c b/notmuch-dump.c > index 21702d7..029f90a 100644 > --- a/notmuch-dump.c > +++ b/notmuch-dump.c > @@ -21,10 +21,12 @@ > #include "notmuch-client.h" > #include "hex-escape.h" > #include "string-util.h" > +#include > + > > static int > -database_dump_file (notmuch_database_t *notmuch, FILE *output, > - const char *query_str, int output_format) > +database_dump_file (notmuch_database_t *notmuch, gzFile output, > + const char *query_str, int output_format) > { > notmuch_query_t *query; > notmuch_messages_t *messages; > @@ -69,7 +71,7 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output, > } > > if (output_format == DUMP_FORMAT_SUP) { > - fprintf (output, "%s (", message_id); > + gzprintf (output, "%s (", message_id); > } > > for (tags = notmuch_message_get_tags (message); > @@ -78,12 +80,12 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output, > const char *tag_str = notmuch_tags_get (tags); > > if (! first) > - fputs (" ", output); > + gzputs (output, " "); > > first = 0; > > if (output_format == DUMP_FORMAT_SUP) { > - fputs (tag_str, output); > + gzputs (output, tag_str); > } else { > if (hex_encode (notmuch, tag_str, > &buffer, &buffer_size) != HEX_SUCCESS) { > @@ -91,12 +93,12 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output, > tag_str); > return EXIT_FAILURE; > } > - fprintf (output, "+%s", buffer); > + gzprintf (output, "+%s", buffer); > } > } > > if (output_format == DUMP_FORMAT_SUP) { > - fputs (")\n", output); > + gzputs (output, ")\n"); > } else { > if (make_boolean_term (notmuch, "id", message_id, > &buffer, &buffer_size)) { > @@ -104,7 +106,7 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output, > message_id, strerror (errno)); > return EXIT_FAILURE; > } > - fprintf (output, " -- %s\n", buffer); > + gzprintf (output, " -- %s\n", buffer); > } > > notmuch_message_destroy (message); > @@ -121,24 +123,33 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output, > int > notmuch_database_dump (notmuch_database_t *notmuch, > const char *output_file_name, > - const char *query_str, dump_format_t output_format) > + const char *query_str, > + dump_format_t output_format, > + notmuch_bool_t gzip_output) > { > - FILE *output = stdout; > + gzFile output; > + const char *mode = gzip_output ? "w9" : "wT"; > + > int ret; > > if (output_file_name) { > - output = fopen (output_file_name, "w"); > + output = gzopen (output_file_name, mode); > if (output == NULL) { > fprintf (stderr, "Error opening %s for writing: %s\n", > output_file_name, strerror (errno)); > return EXIT_FAILURE; > } > + } else { > + output = gzdopen (fileno (stdout), mode); > } > > ret = database_dump_file (notmuch, output, query_str, output_format); > > - if (output != stdout) > - fclose (output); > + /* XXX check error return */ > + gzflush (output, Z_FINISH); > + > + if (output_file_name) > + gzclose_w (output); > > return ret; > } > @@ -158,6 +169,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[]) > int opt_index; > > int output_format = DUMP_FORMAT_BATCH_TAG; > + notmuch_bool_t gzip_output = 0; > > notmuch_opt_desc_t options[] = { > { NOTMUCH_OPT_KEYWORD, &output_format, "format", 'f', > @@ -165,6 +177,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[]) > { "batch-tag", DUMP_FORMAT_BATCH_TAG }, > { 0, 0 } } }, > { NOTMUCH_OPT_STRING, &output_file_name, "output", 'o', 0 }, > + { NOTMUCH_OPT_BOOLEAN, &gzip_output, "gzip", 'z', 0 }, > { 0, 0, 0, 0, 0 } > }; > > @@ -181,7 +194,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[]) > } > > ret = notmuch_database_dump (notmuch, output_file_name, query_str, > - output_format); > + output_format, gzip_output); > > notmuch_database_destroy (notmuch); > > -- > 1.9.0 > > _______________________________________________ > notmuch mailing list > notmuch@notmuchmail.org > http://notmuchmail.org/mailman/listinfo/notmuch