Re: [Patch v6 4/6] restore: transparently support gzipped input

diff --git a/4a/a9cacf37b18b765420f44e34fba0e32b9ebd2b b/4a/a9cacf37b18b765420f44e34fba0e32b9ebd2b
new file mode 100644 (file)
index 0000000..91a673c
--- /dev/null
+++ b/4a/a9cacf37b18b765420f44e34fba0e32b9ebd2b
@@ -0,0 +1,250 @@
+Return-Path: <amdragon@mit.edu>\r
+X-Original-To: notmuch@notmuchmail.org\r
+Delivered-To: notmuch@notmuchmail.org\r
+Received: from localhost (localhost [127.0.0.1])\r
+       by olra.theworths.org (Postfix) with ESMTP id AFBB1431FBF\r
+       for <notmuch@notmuchmail.org>; Fri,  4 Apr 2014 14:56:35 -0700 (PDT)\r
+X-Virus-Scanned: Debian amavisd-new at olra.theworths.org\r
+X-Spam-Flag: NO\r
+X-Spam-Score: -0.7\r
+X-Spam-Level: \r
+X-Spam-Status: No, score=-0.7 tagged_above=-999 required=5\r
+       tests=[RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled\r
+Received: from olra.theworths.org ([127.0.0.1])\r
+       by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024)\r
+       with ESMTP id LapDbwHj2+Ig for <notmuch@notmuchmail.org>;\r
+       Fri,  4 Apr 2014 14:56:30 -0700 (PDT)\r
+Received: from dmz-mailsec-scanner-4.mit.edu (dmz-mailsec-scanner-4.mit.edu\r
+       [18.9.25.15])\r
+       (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits))\r
+       (No client certificate requested)\r
+       by olra.theworths.org (Postfix) with ESMTPS id 29E2B431FAF\r
+       for <notmuch@notmuchmail.org>; Fri,  4 Apr 2014 14:56:30 -0700 (PDT)\r
+X-AuditID: 1209190f-f790b6d000000c3a-62-533f2a8d4372\r
+Received: from mailhub-auth-2.mit.edu ( [18.7.62.36])\r
+       (using TLS with cipher AES256-SHA (256/256 bits))\r
+       (Client did not present a certificate)\r
+       by dmz-mailsec-scanner-4.mit.edu (Symantec Messaging Gateway) with SMTP\r
+       id 47.49.03130.D8A2F335; Fri,  4 Apr 2014 17:56:29 -0400 (EDT)\r
+Received: from outgoing.mit.edu (outgoing-auth-1.mit.edu [18.9.28.11])\r
+       by mailhub-auth-2.mit.edu (8.13.8/8.9.2) with ESMTP id s34LuRDG010027; \r
+       Fri, 4 Apr 2014 17:56:28 -0400\r
+Received: from awakening.csail.mit.edu (awakening.csail.mit.edu [18.26.4.91])\r
+       (authenticated bits=0)\r
+       (User authenticated as amdragon@ATHENA.MIT.EDU)\r
+       by outgoing.mit.edu (8.13.8/8.12.4) with ESMTP id s34LuP2r013654\r
+       (version=TLSv1/SSLv3 cipher=DHE-RSA-AES128-SHA bits=128 verify=NOT);\r
+       Fri, 4 Apr 2014 17:56:27 -0400\r
+Received: from amthrax by awakening.csail.mit.edu with local (Exim 4.80)\r
+       (envelope-from <amdragon@mit.edu>)\r
+       id 1WWC65-0004DI-N6; Fri, 04 Apr 2014 17:56:25 -0400\r
+Date: Fri, 4 Apr 2014 17:56:25 -0400\r
+From: Austin Clements <amdragon@MIT.EDU>\r
+To: David Bremner <david@tethera.net>\r
+Subject: Re: [Patch v6 4/6] restore: transparently support gzipped input\r
+Message-ID: <20140404215625.GA15472@mit.edu>\r
+References: <1396554083-3892-1-git-send-email-david@tethera.net>\r
+       <1396554083-3892-5-git-send-email-david@tethera.net>\r
+MIME-Version: 1.0\r
+Content-Type: text/plain; charset=us-ascii\r
+Content-Disposition: inline\r
+In-Reply-To: <1396554083-3892-5-git-send-email-david@tethera.net>\r
+User-Agent: Mutt/1.5.21 (2010-09-15)\r
+X-Brightmail-Tracker:\r
+ H4sIAAAAAAAAA+NgFmpkleLIzCtJLcpLzFFi42IRYrdT0e3Vsg826DrNY3GjtZvR4vrNmcwO\r
+       TB7PVt1i9thy6D1zAFMUl01Kak5mWWqRvl0CV0bP7gtsBVN0Kw5d3MPcwPhNqYuRk0NCwETi\r
+       /MTrTBC2mMSFe+vZQGwhgdlMEj96i7oYuYDsDYwST69+ZYdInGKSOLlBDCKxhFHi0NqprCAJ\r
+       FgEViUNznzGD2GwCGhLb9i9nBLFFBFQlrm6bDDaVWUBa4tvvZrBtwgIeErvfrAWr5xXQkdgw\r
+       aztQDQfQ0HKJp8tzIcKCEidnPmGBaNWSuPHvJRNICciY5f84QMKcAo4Sz658AJsiCnTBlJPb\r
+       2CYwCs1C0j0LSfcshO4FjMyrGGVTcqt0cxMzc4pTk3WLkxPz8lKLdE30cjNL9FJTSjcxgkKa\r
+       U5J/B+O3g0qHGAU4GJV4eDt22AULsSaWFVfmHmKU5GBSEuXtUrAPFuJLyk+pzEgszogvKs1J\r
+       LT7EKMHBrCTCK6MKlONNSaysSi3Kh0lJc7AoifO+tbYKFhJITyxJzU5NLUgtgsnKcHAoSfBO\r
+       0wRqFCxKTU+tSMvMKUFIM3FwggznARp+DqSGt7ggMbc4Mx0if4pRUUqcNwAkIQCSyCjNg+uF\r
+       pZxXjOJArwjzrgKp4gGmK7juV0CDmYAGN4TZgQwuSURISTUwZpysXdPuv0xss49+p9yZ52sy\r
+       mrO4/GIOs23pdze49Oj1wanNUSI+XsnZLXd/mDOeFdh+eGsiz2KRvdzCntp895mjV/bpfG6P\r
+       fMC9tV87+UGjisihX7pnVnxcMI/7ih73CrHCd/ObzNc0XA2znrN92ezfumdyrrxe1vBqq7yf\r
+       uNaJZUHlcdETlViKMxINtZiLihMBOQFFqxQDAAA=\r
+Cc: notmuch@notmuchmail.org\r
+X-BeenThere: notmuch@notmuchmail.org\r
+X-Mailman-Version: 2.1.13\r
+Precedence: list\r
+List-Id: "Use and development of the notmuch mail system."\r
+       <notmuch.notmuchmail.org>\r
+List-Unsubscribe: <http://notmuchmail.org/mailman/options/notmuch>,\r
+       <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>\r
+List-Archive: <http://notmuchmail.org/pipermail/notmuch>\r
+List-Post: <mailto:notmuch@notmuchmail.org>\r
+List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>\r
+List-Subscribe: <http://notmuchmail.org/mailman/listinfo/notmuch>,\r
+       <mailto:notmuch-request@notmuchmail.org?subject=subscribe>\r
+X-List-Received-Date: Fri, 04 Apr 2014 21:56:35 -0000\r
+\r
+Quoth David Bremner on Apr 03 at  4:41 pm:\r
+> We rely completely on zlib to do the right thing in detecting gzipped\r
+> input. Since our dump format is chosen to be 7 bit ascii, this should\r
+> be fine.\r
+> ---\r
+>  doc/man1/notmuch-restore.rst |  8 ++++++++\r
+>  notmuch-restore.c            | 41 ++++++++++++++++++++++++++---------------\r
+>  test/T240-dump-restore.sh    | 14 ++++++++++++++\r
+>  3 files changed, 48 insertions(+), 15 deletions(-)\r
+> \r
+> diff --git a/doc/man1/notmuch-restore.rst b/doc/man1/notmuch-restore.rst\r
+> index d6cf19a..936b138 100644\r
+> --- a/doc/man1/notmuch-restore.rst\r
+> +++ b/doc/man1/notmuch-restore.rst\r
+> @@ -50,6 +50,14 @@ Supported options for **restore** include\r
+>              format, this heuristic, based the fact that batch-tag format\r
+>              contains no parentheses, should be accurate.\r
+>  \r
+> +GZIPPED INPUT\r
+> +=============\r
+> +\r
+> +\ **notmuch restore** will detect if the input is compressed in\r
+> +**gzip(1)** format and automatically decompress it while reading. This\r
+> +detection does not depend on file naming and in particular works for\r
+> +standard input.\r
+> +\r
+>  SEE ALSO\r
+>  ========\r
+>  \r
+> diff --git a/notmuch-restore.c b/notmuch-restore.c\r
+> index c54d513..eb5b7b2 100644\r
+> --- a/notmuch-restore.c\r
+> +++ b/notmuch-restore.c\r
+> @@ -22,6 +22,7 @@\r
+>  #include "hex-escape.h"\r
+>  #include "tag-util.h"\r
+>  #include "string-util.h"\r
+> +#include "zlib-extra.h"\r
+>  \r
+>  static regex_t regex;\r
+>  \r
+> @@ -128,10 +129,9 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])\r
+>      tag_op_list_t *tag_ops;\r
+>  \r
+>      char *input_file_name = NULL;\r
+> -    FILE *input = stdin;\r
+> +    gzFile input;\r
+>      char *line = NULL;\r
+>      void *line_ctx = NULL;\r
+> -    size_t line_size;\r
+>      ssize_t line_len;\r
+>  \r
+>      int ret = 0;\r
+> @@ -163,13 +163,23 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])\r
+>      if (! accumulate)\r
+>      flags |= TAG_FLAG_REMOVE_ALL;\r
+>  \r
+> -    if (input_file_name) {\r
+> -    input = fopen (input_file_name, "r");\r
+> -    if (input == NULL) {\r
+> -        fprintf (stderr, "Error opening %s for reading: %s\n",\r
+> -                 input_file_name, strerror (errno));\r
+> +    if (input_file_name)\r
+> +    input = gzopen (input_file_name, "r");\r
+> +    else {\r
+> +    int infd = dup (STDIN_FILENO);\r
+> +    if (infd < 0) {\r
+> +        fprintf (stderr, "Error duping stdin\n");\r
+>          return EXIT_FAILURE;\r
+>      }\r
+> +    input = gzdopen (infd, "r");\r
+> +    if (! input)\r
+> +        close (infd);\r
+> +    }\r
+> +\r
+> +    if (input == NULL) {\r
+> +    fprintf (stderr, "Error opening %s for (gzip) reading: %s\n",\r
+> +             input_file_name ? input_file_name : "stdin", strerror (errno));\r
+\r
+There's a sketchy line about errno in the gz(d)open docs: "On error,\r
+gzopen() *may* set the global variable errno to indicate the error."\r
+(emphasis mine).  This suggests we should set errno to 0 before the\r
+calls to gz(d)open above.\r
+\r
+> +    return EXIT_FAILURE;\r
+>      }\r
+>  \r
+>      if (opt_index < argc) {\r
+> @@ -184,12 +194,17 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])\r
+>      }\r
+>  \r
+>      do {\r
+> -    line_len = getline (&line, &line_size, input);\r
+> +    util_status_t status;\r
+> +\r
+> +    status = gz_getline (line_ctx, &line, &line_len, input);\r
+>  \r
+>      /* empty input file not considered an error */\r
+> -    if (line_len < 0)\r
+> +    if (status == UTIL_EOF)\r
+>          return EXIT_SUCCESS;\r
+>  \r
+> +    if (status)\r
+\r
+Will this lead to a silent exit failure if there's a problem with\r
+decompression?  This suggests we should have a UTIL_GZERROR that tells\r
+the caller to consult gzerror for the error message.  (Though this is\r
+still an improvement over the original code, which would silently\r
+succeed when getline failed!)\r
+\r
+> +        return EXIT_FAILURE;\r
+> +\r
+>      } while ((line_len == 0) ||\r
+>           (line[0] == '#') ||\r
+>           /* the cast is safe because we checked about for line_len < 0 */\r
+> @@ -254,7 +269,7 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])\r
+>      if (ret)\r
+>          break;\r
+>  \r
+> -    }  while ((line_len = getline (&line, &line_size, input)) != -1);\r
+> +    }  while (gz_getline (line_ctx, &line, &line_len, input) == UTIL_SUCCESS);\r
+\r
+It looks like a gz_getline error here will cause restore to stop and\r
+claim that the restore was successful.  (The original code has this\r
+problem, too.)\r
+\r
+>  \r
+>      if (line_ctx != NULL)\r
+>      talloc_free (line_ctx);\r
+> @@ -262,13 +277,9 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])\r
+>      if (input_format == DUMP_FORMAT_SUP)\r
+>      regfree (&regex);\r
+>  \r
+> -    if (line)\r
+> -    free (line);\r
+> -\r
+>      notmuch_database_destroy (notmuch);\r
+>  \r
+> -    if (input != stdin)\r
+> -    fclose (input);\r
+> +    gzclose_r (input);\r
+>  \r
+>      return ret ? EXIT_FAILURE : EXIT_SUCCESS;\r
+>  }\r
+> diff --git a/test/T240-dump-restore.sh b/test/T240-dump-restore.sh\r
+> index b6d8602..efe463e 100755\r
+> --- a/test/T240-dump-restore.sh\r
+> +++ b/test/T240-dump-restore.sh\r
+> @@ -80,6 +80,20 @@ notmuch dump --gzip --output=dump-gzip-outfile.gz\r
+>  gunzip dump-gzip-outfile.gz\r
+>  test_expect_equal_file dump.expected dump-gzip-outfile\r
+>  \r
+> +test_begin_subtest "restoring gzipped stdin"\r
+> +notmuch dump --gzip --output=backup.gz\r
+> +notmuch tag +new_tag '*'\r
+> +notmuch restore < backup.gz\r
+> +notmuch dump --output=dump.actual\r
+> +test_expect_equal_file dump.expected dump.actual\r
+> +\r
+> +test_begin_subtest "restoring gzipped file"\r
+> +notmuch dump --gzip --output=backup.gz\r
+> +notmuch tag +new_tag '*'\r
+> +notmuch restore --input=backup.gz\r
+> +notmuch dump --output=dump.actual\r
+> +test_expect_equal_file dump.expected dump.actual\r
+> +\r
+>  # Note, we assume all messages from cworth have a message-id\r
+>  # containing cworth.org\r
+>  \r