From 41093a6f215007da642aafcdb6f7bf357aabb4f6 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Sat, 29 Mar 2014 11:25:12 +0200 Subject: [PATCH] Re: [PATCH] RFC: impliment gzipped output for notmuch dump --- 75/b1ff3bdcc0918d0a7051f1591d95a79061efe0 | 323 ++++++++++++++++++++++ 1 file changed, 323 insertions(+) create mode 100644 75/b1ff3bdcc0918d0a7051f1591d95a79061efe0 diff --git a/75/b1ff3bdcc0918d0a7051f1591d95a79061efe0 b/75/b1ff3bdcc0918d0a7051f1591d95a79061efe0 new file mode 100644 index 000000000..d28fa0f1b --- /dev/null +++ b/75/b1ff3bdcc0918d0a7051f1591d95a79061efe0 @@ -0,0 +1,323 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id 1A5CB431FBC + for ; Sat, 29 Mar 2014 02:25:27 -0700 (PDT) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Spam-Flag: NO +X-Spam-Score: -0.7 +X-Spam-Level: +X-Spam-Status: No, score=-0.7 tagged_above=-999 required=5 + tests=[RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id GA8I+embZj8Y for ; + Sat, 29 Mar 2014 02:25:20 -0700 (PDT) +Received: from mail-ee0-f43.google.com (mail-ee0-f43.google.com + [74.125.83.43]) (using TLSv1 with cipher RC4-SHA (128/128 bits)) (No client + certificate requested) by olra.theworths.org (Postfix) with ESMTPS id + 5059F431FAF for ; Sat, 29 Mar 2014 02:25:20 -0700 + (PDT) +Received: by mail-ee0-f43.google.com with SMTP id e53so4876282eek.30 + for ; Sat, 29 Mar 2014 02:25:17 -0700 (PDT) +X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=1e100.net; s=20130820; + h=x-gm-message-state:from:to:subject:in-reply-to:references + :user-agent:date:message-id:mime-version:content-type; + bh=+z21J9bCP/itUjQx1xsrXDUL6xEXLoT2amKlaIVsx3s=; + b=hUt4eKd8PbiF67AP/gCSMeBrOnDhHNU8wjny9dSRNExY7hVht5FxXp9HbzZhIcqZLt + 8AdyxLQ2yGaaH4cgETICZZ273hCcdpqRyh4ZlX1/Pg45JFko8YIe4sv0s0VbEgEbwtSM + bqOetCHlyrwWrB11exIcsmLNmWwmpkNKpzDse8SkSUt3RfrPmFtcgPTtNmw2uB4BJV83 + HHaUsY4czTgc6zofVZQkvRvCkRjGfziwIxcCebeakiv6hN25Apcm6JfxhS8udCmPt/jU + CIPzE+EXwZZGd0Z3ZNNY6s6ThQhjZXZ0ltoQfZxdthEjIxqtuqjpTAUtyl7/ZpGiCmMu + Grdw== +X-Gm-Message-State: + ALoCoQlaUGLVXMAghuKux+WfPe+LmNUMuwWHi4Bz+/Omkc2V3mThPE4wu54kOeTFPxllyeEhvvFg +X-Received: by 10.14.215.9 with SMTP id d9mr885559eep.64.1396085116412; + Sat, 29 Mar 2014 02:25:16 -0700 (PDT) +Received: from localhost (dsl-hkibrasgw2-58c36f-91.dhcp.inet.fi. + [88.195.111.91]) + by mx.google.com with ESMTPSA id g3sm17442780eet.35.2014.03.29.02.25.14 + for + (version=TLSv1.2 cipher=RC4-SHA bits=128/128); + Sat, 29 Mar 2014 02:25:15 -0700 (PDT) +From: Jani Nikula +To: David Bremner , notmuch@notmuchmail.org +Subject: Re: [PATCH] RFC: impliment gzipped output for notmuch dump +In-Reply-To: <1396056046-2247-1-git-send-email-david@tethera.net> +References: <874n2kipq4.fsf@qmul.ac.uk> + <1396056046-2247-1-git-send-email-david@tethera.net> +User-Agent: Notmuch/0.17+164~gcd5fd5a4837e (http://notmuchmail.org) + Emacs/24.3.1 (x86_64-pc-linux-gnu) +Date: Sat, 29 Mar 2014 11:25:12 +0200 +Message-ID: <87wqfdbblz.fsf@nikula.org> +MIME-Version: 1.0 +Content-Type: text/plain +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Sat, 29 Mar 2014 09:25:27 -0000 + +On Sat, 29 Mar 2014, David Bremner wrote: +> --- +> +> This obviously needs at least one piece of tidying, and some tests. +> +> My motivation here is the ability to make gzipped backups from within +> notmuch, e.g. as part of an upgrade procedure. + +Is it reasonable to require zlib? It's possible to have conditional +build for this, but it will be laborous to do it cleanly. I could be +persuaded either way, perhaps more easily to just requiring it. + +If you do require zlib, the missing dependencies error message from +configure should include a check for have_zlib and a note about it +similar to other required dependencies. + +Also, it would seem natural to support gzipped input in restore as +well... do you think it would be silly to support gzipped output only +internally until we've added gzipped input too? This could be in a later +release in the future. + + +BR, +Jani. + + + +> +> +> Makefile.local | 2 +- +> configure | 19 ++++++++++++++++++- +> notmuch-client.h | 4 +++- +> notmuch-dump.c | 41 +++++++++++++++++++++++++++-------------- +> 4 files changed, 49 insertions(+), 17 deletions(-) +> +> diff --git a/Makefile.local b/Makefile.local +> index cb7b106..e5a20a7 100644 +> --- a/Makefile.local +> +++ b/Makefile.local +> @@ -41,7 +41,7 @@ PV_FILE=bindings/python/notmuch/version.py +> # Smash together user's values with our extra values +> FINAL_CFLAGS = -DNOTMUCH_VERSION=$(VERSION) $(CPPFLAGS) $(CFLAGS) $(WARN_CFLAGS) $(extra_cflags) $(CONFIGURE_CFLAGS) +> FINAL_CXXFLAGS = $(CPPFLAGS) $(CXXFLAGS) $(WARN_CXXFLAGS) $(extra_cflags) $(extra_cxxflags) $(CONFIGURE_CXXFLAGS) +> -FINAL_NOTMUCH_LDFLAGS = $(LDFLAGS) -Lutil -lutil -Llib -lnotmuch $(AS_NEEDED_LDFLAGS) $(GMIME_LDFLAGS) $(TALLOC_LDFLAGS) +> +FINAL_NOTMUCH_LDFLAGS = $(LDFLAGS) -Lutil -lutil -Llib -lnotmuch $(AS_NEEDED_LDFLAGS) $(GMIME_LDFLAGS) $(TALLOC_LDFLAGS) $(ZLIB_LDFLAGS) +> FINAL_NOTMUCH_LINKER = CC +> ifneq ($(LINKER_RESOLVES_LIBRARY_DEPENDENCIES),1) +> FINAL_NOTMUCH_LDFLAGS += $(CONFIGURE_LDFLAGS) +> diff --git a/configure b/configure +> index 1d430b9..02ef785 100755 +> --- a/configure +> +++ b/configure +> @@ -340,6 +340,18 @@ else +> errors=$((errors + 1)) +> fi +> +> +printf "Checking for zlib development files... " +> +have_zlib=0 +> +if pkg-config --exists zlib; then +> + printf "Yes.\n" +> + have_zlib=1 +> + zlib_cflags=$(pkg-config --cflags zlib) +> + zlib_ldflags=$(pkg-config --libs zlib) +> +else +> + printf "No.\n" +> + errors=$((errors + 1)) +> +fi +> + +> printf "Checking for talloc development files... " +> if pkg-config --exists talloc; then +> printf "Yes.\n" +> @@ -844,6 +856,10 @@ XAPIAN_LDFLAGS = ${xapian_ldflags} +> GMIME_CFLAGS = ${gmime_cflags} +> GMIME_LDFLAGS = ${gmime_ldflags} +> +> +# Flags needed to compile and link against zlib +> +ZLIB_CFLAGS = ${zlib_cflags} +> +ZLIB_LDFLAGS = ${zlib_ldflags} +> + +> # Flags needed to compile and link against talloc +> TALLOC_CFLAGS = ${talloc_cflags} +> TALLOC_LDFLAGS = ${talloc_ldflags} +> @@ -882,6 +898,7 @@ CONFIGURE_CFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS) \\ +> -DUTIL_BYTE_ORDER=\$(UTIL_BYTE_ORDER) +> +> CONFIGURE_CXXFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS) \\ +> + \$(ZLIB_CFLAGS) \\ +> \$(TALLOC_CFLAGS) -DHAVE_VALGRIND=\$(HAVE_VALGRIND) \\ +> \$(VALGRIND_CFLAGS) \$(XAPIAN_CXXFLAGS) \\ +> -DHAVE_STRCASESTR=\$(HAVE_STRCASESTR) \\ +> @@ -892,5 +909,5 @@ CONFIGURE_CXXFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS) \\ +> -DHAVE_XAPIAN_COMPACT=\$(HAVE_XAPIAN_COMPACT) \\ +> -DUTIL_BYTE_ORDER=\$(UTIL_BYTE_ORDER) +> +> -CONFIGURE_LDFLAGS = \$(GMIME_LDFLAGS) \$(TALLOC_LDFLAGS) \$(XAPIAN_LDFLAGS) +> +CONFIGURE_LDFLAGS = \$(GMIME_LDFLAGS) \$(TALLOC_LDFLAGS) \$(ZLIB_LDFLAGS) \$(XAPIAN_LDFLAGS) +> EOF +> diff --git a/notmuch-client.h b/notmuch-client.h +> index d110648..e1efbe0 100644 +> --- a/notmuch-client.h +> +++ b/notmuch-client.h +> @@ -450,7 +450,9 @@ typedef enum dump_formats { +> int +> notmuch_database_dump (notmuch_database_t *notmuch, +> const char *output_file_name, +> - const char *query_str, dump_format_t output_format); +> + const char *query_str, +> + dump_format_t output_format, +> + notmuch_bool_t gzip_output); +> +> #include "command-line-arguments.h" +> #endif +> diff --git a/notmuch-dump.c b/notmuch-dump.c +> index 21702d7..029f90a 100644 +> --- a/notmuch-dump.c +> +++ b/notmuch-dump.c +> @@ -21,10 +21,12 @@ +> #include "notmuch-client.h" +> #include "hex-escape.h" +> #include "string-util.h" +> +#include +> + +> +> static int +> -database_dump_file (notmuch_database_t *notmuch, FILE *output, +> - const char *query_str, int output_format) +> +database_dump_file (notmuch_database_t *notmuch, gzFile output, +> + const char *query_str, int output_format) +> { +> notmuch_query_t *query; +> notmuch_messages_t *messages; +> @@ -69,7 +71,7 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output, +> } +> +> if (output_format == DUMP_FORMAT_SUP) { +> - fprintf (output, "%s (", message_id); +> + gzprintf (output, "%s (", message_id); +> } +> +> for (tags = notmuch_message_get_tags (message); +> @@ -78,12 +80,12 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output, +> const char *tag_str = notmuch_tags_get (tags); +> +> if (! first) +> - fputs (" ", output); +> + gzputs (output, " "); +> +> first = 0; +> +> if (output_format == DUMP_FORMAT_SUP) { +> - fputs (tag_str, output); +> + gzputs (output, tag_str); +> } else { +> if (hex_encode (notmuch, tag_str, +> &buffer, &buffer_size) != HEX_SUCCESS) { +> @@ -91,12 +93,12 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output, +> tag_str); +> return EXIT_FAILURE; +> } +> - fprintf (output, "+%s", buffer); +> + gzprintf (output, "+%s", buffer); +> } +> } +> +> if (output_format == DUMP_FORMAT_SUP) { +> - fputs (")\n", output); +> + gzputs (output, ")\n"); +> } else { +> if (make_boolean_term (notmuch, "id", message_id, +> &buffer, &buffer_size)) { +> @@ -104,7 +106,7 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output, +> message_id, strerror (errno)); +> return EXIT_FAILURE; +> } +> - fprintf (output, " -- %s\n", buffer); +> + gzprintf (output, " -- %s\n", buffer); +> } +> +> notmuch_message_destroy (message); +> @@ -121,24 +123,33 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output, +> int +> notmuch_database_dump (notmuch_database_t *notmuch, +> const char *output_file_name, +> - const char *query_str, dump_format_t output_format) +> + const char *query_str, +> + dump_format_t output_format, +> + notmuch_bool_t gzip_output) +> { +> - FILE *output = stdout; +> + gzFile output; +> + const char *mode = gzip_output ? "w9" : "wT"; +> + +> int ret; +> +> if (output_file_name) { +> - output = fopen (output_file_name, "w"); +> + output = gzopen (output_file_name, mode); +> if (output == NULL) { +> fprintf (stderr, "Error opening %s for writing: %s\n", +> output_file_name, strerror (errno)); +> return EXIT_FAILURE; +> } +> + } else { +> + output = gzdopen (fileno (stdout), mode); +> } +> +> ret = database_dump_file (notmuch, output, query_str, output_format); +> +> - if (output != stdout) +> - fclose (output); +> + /* XXX check error return */ +> + gzflush (output, Z_FINISH); +> + +> + if (output_file_name) +> + gzclose_w (output); +> +> return ret; +> } +> @@ -158,6 +169,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[]) +> int opt_index; +> +> int output_format = DUMP_FORMAT_BATCH_TAG; +> + notmuch_bool_t gzip_output = 0; +> +> notmuch_opt_desc_t options[] = { +> { NOTMUCH_OPT_KEYWORD, &output_format, "format", 'f', +> @@ -165,6 +177,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[]) +> { "batch-tag", DUMP_FORMAT_BATCH_TAG }, +> { 0, 0 } } }, +> { NOTMUCH_OPT_STRING, &output_file_name, "output", 'o', 0 }, +> + { NOTMUCH_OPT_BOOLEAN, &gzip_output, "gzip", 'z', 0 }, +> { 0, 0, 0, 0, 0 } +> }; +> +> @@ -181,7 +194,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[]) +> } +> +> ret = notmuch_database_dump (notmuch, output_file_name, query_str, +> - output_format); +> + output_format, gzip_output); +> +> notmuch_database_destroy (notmuch); +> +> -- +> 1.9.0 +> +> _______________________________________________ +> notmuch mailing list +> notmuch@notmuchmail.org +> http://notmuchmail.org/mailman/listinfo/notmuch + +-- +Jani -- 2.26.2