From 2c12c68da4ad9dfe7541b785b7ee4c4a7914c799 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Fri, 11 Apr 2014 09:24:38 +2000 Subject: [PATCH] Re: [Patch v7 4/6] restore: transparently support gzipped input --- bd/c2a56839740b9e12710037cd13d3bbd75c5f0b | 298 ++++++++++++++++++++++ 1 file changed, 298 insertions(+) create mode 100644 bd/c2a56839740b9e12710037cd13d3bbd75c5f0b diff --git a/bd/c2a56839740b9e12710037cd13d3bbd75c5f0b b/bd/c2a56839740b9e12710037cd13d3bbd75c5f0b new file mode 100644 index 000000000..5cf8dba77 --- /dev/null +++ b/bd/c2a56839740b9e12710037cd13d3bbd75c5f0b @@ -0,0 +1,298 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id 15DBA431FBD + for ; Thu, 10 Apr 2014 06:24:53 -0700 (PDT) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Spam-Flag: NO +X-Spam-Score: -0.7 +X-Spam-Level: +X-Spam-Status: No, score=-0.7 tagged_above=-999 required=5 + tests=[RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id keNYugqGW-o2 for ; + Thu, 10 Apr 2014 06:24:44 -0700 (PDT) +Received: from dmz-mailsec-scanner-7.mit.edu (dmz-mailsec-scanner-7.mit.edu + [18.7.68.36]) + (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits)) + (No client certificate requested) + by olra.theworths.org (Postfix) with ESMTPS id 9854D431FBC + for ; Thu, 10 Apr 2014 06:24:44 -0700 (PDT) +X-AuditID: 12074424-f79e26d000000c70-8b-53469b9a40a1 +Received: from mailhub-auth-3.mit.edu ( [18.9.21.43]) + (using TLS with cipher AES256-SHA (256/256 bits)) + (Client did not present a certificate) + by dmz-mailsec-scanner-7.mit.edu (Symantec Messaging Gateway) with SMTP + id A8.8B.03184.A9B96435; Thu, 10 Apr 2014 09:24:42 -0400 (EDT) +Received: from outgoing.mit.edu (outgoing-auth-1.mit.edu [18.9.28.11]) + by mailhub-auth-3.mit.edu (8.13.8/8.9.2) with ESMTP id s3ADOfiS028048; + Thu, 10 Apr 2014 09:24:42 -0400 +Received: from awakening.csail.mit.edu (awakening.csail.mit.edu [18.26.4.91]) + (authenticated bits=0) + (User authenticated as amdragon@ATHENA.MIT.EDU) + by outgoing.mit.edu (8.13.8/8.12.4) with ESMTP id s3ADOcn9015845 + (version=TLSv1/SSLv3 cipher=DHE-RSA-AES128-SHA bits=128 verify=NOT); + Thu, 10 Apr 2014 09:24:40 -0400 +Received: from amthrax by awakening.csail.mit.edu with local (Exim 4.80) + (envelope-from ) + id 1WYEy6-0003DR-Kb; Thu, 10 Apr 2014 09:24:38 -0400 +From: Austin Clements +To: David Bremner , notmuch@notmuchmail.org +Subject: Re: [Patch v7 4/6] restore: transparently support gzipped input +In-Reply-To: <1396712636-10640-5-git-send-email-david@tethera.net> +References: <1396712636-10640-1-git-send-email-david@tethera.net> + <1396712636-10640-5-git-send-email-david@tethera.net> +User-Agent: Notmuch/0.17+167~g3863755 (http://notmuchmail.org) Emacs/23.4.1 + (i486-pc-linux-gnu) +Date: Thu, 10 Apr 2014 09:24:38 -0400 +Message-ID: <8761mhxqp5.fsf@awakening.csail.mit.edu> +MIME-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +X-Brightmail-Tracker: + H4sIAAAAAAAAA+NgFrrHIsWRmVeSWpSXmKPExsUixCmqrTtrtluwwd+tphY3WrsZLa7fnMns + wOTxbNUtZo8th94zBzBFcdmkpOZklqUW6dslcGXcmLCAqeCWScW9k8uZGxjvqXcxcnJICJhI + LN54lh3CFpO4cG89WxcjF4eQwGwmiUVNc1hBEkICGxkl+icUQiROM0lMWryIHcJZwiix5PwV + sCo2AQ2JbfuXM4LYIgJ2Ej0XzgMVcXAIC3hIzH7qBmJyCjhJfFigAmIKCZRLvLmrCVIsKhAv + 0doP0cgioCqxfvZMsHt4gW47tf81E4QtKHFy5hMWEJtZQEvixr+XTBMYBWYhSc1CklrAyLSK + UTYlt0o3NzEzpzg1Wbc4OTEvL7VI11wvN7NELzWldBMjKBjZXVR2MDYfUjrEKMDBqMTDe3CZ + a7AQa2JZcWXuIUZJDiYlUV61SW7BQnxJ+SmVGYnFGfFFpTmpxYcYJTiYlUR4N04FyvGmJFZW + pRblw6SkOViUxHnfWlsFCwmkJ5akZqemFqQWwWRlODiUJHiXzAJqFCxKTU+tSMvMKUFIM3Fw + ggznARq+CKSGt7ggMbc4Mx0if4pRUUqc1wwkIQCSyCjNg+uFJYtXjOJArwjz7gCp4gEmGrju + V0CDmYAGp9q5gAwuSURISTUw7t50hoOjQFGiXlmwuHZLsZSAr7SgQkuZ/VrPTfUr3/h0bXnY + ZtZYUPd52oolMVpX5Tas9+jnE4tVDLrJJf41rHWN4sk3t9yzFI66XBGImFv6dfuWnzff9V1Y + uFHyp1TBCUaLlrq86mB3zllrq6afOma1MNti3s6yu/uzN7zosAmb+Gif0NMKJZbijERDLeai + 4kQAVK0GH/ECAAA= +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Thu, 10 Apr 2014 13:24:53 -0000 + +On Sat, 05 Apr 2014, David Bremner wrote: +> We rely completely on zlib to do the right thing in detecting gzipped +> input. Since our dump format is chosen to be 7 bit ascii, this should +> be fine. +> --- +> doc/man1/notmuch-restore.rst | 8 ++++ +> notmuch-restore.c | 93 +++++++++++++++++++++++++++++++++----------- +> test/T240-dump-restore.sh | 14 +++++++ +> 3 files changed, 92 insertions(+), 23 deletions(-) +> +> diff --git a/doc/man1/notmuch-restore.rst b/doc/man1/notmuch-restore.rst +> index d6cf19a..936b138 100644 +> --- a/doc/man1/notmuch-restore.rst +> +++ b/doc/man1/notmuch-restore.rst +> @@ -50,6 +50,14 @@ Supported options for **restore** include +> format, this heuristic, based the fact that batch-tag format +> contains no parentheses, should be accurate. +> +> +GZIPPED INPUT +> +============= +> + +> +\ **notmuch restore** will detect if the input is compressed in +> +**gzip(1)** format and automatically decompress it while reading. This +> +detection does not depend on file naming and in particular works for +> +standard input. +> + +> SEE ALSO +> ======== +> +> diff --git a/notmuch-restore.c b/notmuch-restore.c +> index c54d513..7abee0a 100644 +> --- a/notmuch-restore.c +> +++ b/notmuch-restore.c +> @@ -22,6 +22,7 @@ +> #include "hex-escape.h" +> #include "tag-util.h" +> #include "string-util.h" +> +#include "zlib-extra.h" +> +> static regex_t regex; +> +> @@ -128,10 +129,10 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[]) +> tag_op_list_t *tag_ops; +> +> char *input_file_name = NULL; +> - FILE *input = stdin; +> + const char *name_for_error = NULL; +> + gzFile input = NULL; +> char *line = NULL; +> void *line_ctx = NULL; +> - size_t line_size; +> ssize_t line_len; +> +> int ret = 0; +> @@ -157,39 +158,69 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[]) +> }; +> +> opt_index = parse_arguments (argc, argv, options, 1); +> - if (opt_index < 0) +> - return EXIT_FAILURE; +> + if (opt_index < 0) { +> + ret = EXIT_FAILURE; +> + goto DONE; +> + } +> + +> + name_for_error = input_file_name ? input_file_name : "stdin"; +> +> if (! accumulate) +> flags |= TAG_FLAG_REMOVE_ALL; +> +> - if (input_file_name) { +> - input = fopen (input_file_name, "r"); +> - if (input == NULL) { +> - fprintf (stderr, "Error opening %s for reading: %s\n", +> - input_file_name, strerror (errno)); +> - return EXIT_FAILURE; +> + errno = 0; +> + if (input_file_name) +> + input = gzopen (input_file_name, "r"); +> + else { +> + int infd = dup (STDIN_FILENO); +> + if (infd < 0) { +> + fprintf (stderr, "Error duping stdin: %s\n", +> + strerror (errno)); +> + ret = EXIT_FAILURE; +> + goto DONE; +> } +> + input = gzdopen (infd, "r"); +> + if (! input) +> + close (infd); +> + } +> + +> + if (input == NULL) { +> + fprintf (stderr, "Error opening %s for (gzip) reading: %s\n", +> + name_for_error, strerror (errno)); +> + ret = EXIT_FAILURE; +> + goto DONE; +> } +> +> if (opt_index < argc) { +> fprintf (stderr, "Unused positional parameter: %s\n", argv[opt_index]); +> - return EXIT_FAILURE; +> + ret = EXIT_FAILURE; +> + goto DONE; +> } +> +> tag_ops = tag_op_list_create (config); +> if (tag_ops == NULL) { +> fprintf (stderr, "Out of memory.\n"); +> - return EXIT_FAILURE; +> + ret = EXIT_FAILURE; +> + goto DONE; +> } +> +> do { +> - line_len = getline (&line, &line_size, input); +> + util_status_t status; +> + +> + status = gz_getline (line_ctx, &line, &line_len, input); +> +> /* empty input file not considered an error */ +> - if (line_len < 0) +> - return EXIT_SUCCESS; +> + if (status == UTIL_EOF) { +> + ret = EXIT_SUCCESS; +> + goto DONE; +> + } +> +> + if (status) { +> + fprintf (stderr, "Error reading (gzipped) input: %s\n", +> + gz_error_string(status, input)); +> + ret = EXIT_FAILURE; +> + goto DONE; +> + } +> } while ((line_len == 0) || +> (line[0] == '#') || +> /* the cast is safe because we checked about for line_len < 0 */ +> @@ -254,21 +285,37 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[]) +> if (ret) +> break; +> +> - } while ((line_len = getline (&line, &line_size, input)) != -1); +> + } while (! (ret = gz_getline (line_ctx, &line, &line_len, input))); +> + +> +> - if (line_ctx != NULL) +> - talloc_free (line_ctx); +> + /* EOF is normal loop termination condition, UTIL_SUCCESS is +> + * impossible here */ +> + if (ret == UTIL_EOF) { +> + ret = UTIL_SUCCESS; +> + } else { +> + fprintf (stderr, "Error reading (gzipped) input: %s\n", +> + gz_error_string (ret, input)); + +ret = EXIT_FAILURE; here? + +> + } +> + +> + /* currently this should not be after DONE: since we don't +> + * know if the xregcomp was reached +> + */ +> +> if (input_format == DUMP_FORMAT_SUP) +> regfree (®ex); +> +> - if (line) +> - free (line); +> + DONE: +> + if (line_ctx != NULL) +> + talloc_free (line_ctx); +> +> - notmuch_database_destroy (notmuch); +> + if (notmuch) +> + notmuch_database_destroy (notmuch); +> +> - if (input != stdin) +> - fclose (input); +> + if (input && gzclose_r (input)) { +> + fprintf (stderr, "Error closing %s: %s\n", +> + name_for_error, gzerror (input, NULL)); +> + ret = EXIT_FAILURE; +> + } +> +> return ret ? EXIT_FAILURE : EXIT_SUCCESS; +> } +> diff --git a/test/T240-dump-restore.sh b/test/T240-dump-restore.sh +> index b6d8602..efe463e 100755 +> --- a/test/T240-dump-restore.sh +> +++ b/test/T240-dump-restore.sh +> @@ -80,6 +80,20 @@ notmuch dump --gzip --output=dump-gzip-outfile.gz +> gunzip dump-gzip-outfile.gz +> test_expect_equal_file dump.expected dump-gzip-outfile +> +> +test_begin_subtest "restoring gzipped stdin" +> +notmuch dump --gzip --output=backup.gz +> +notmuch tag +new_tag '*' +> +notmuch restore < backup.gz +> +notmuch dump --output=dump.actual +> +test_expect_equal_file dump.expected dump.actual +> + +> +test_begin_subtest "restoring gzipped file" +> +notmuch dump --gzip --output=backup.gz +> +notmuch tag +new_tag '*' +> +notmuch restore --input=backup.gz +> +notmuch dump --output=dump.actual +> +test_expect_equal_file dump.expected dump.actual +> + +> # Note, we assume all messages from cworth have a message-id +> # containing cworth.org +> +> -- +> 1.9.0 +> +> _______________________________________________ +> notmuch mailing list +> notmuch@notmuchmail.org +> http://notmuchmail.org/mailman/listinfo/notmuch -- 2.26.2