--- /dev/null
+Return-Path: <tomi.ollila@iki.fi>\r
+X-Original-To: notmuch@notmuchmail.org\r
+Delivered-To: notmuch@notmuchmail.org\r
+Received: from localhost (localhost [127.0.0.1])\r
+ by arlo.cworth.org (Postfix) with ESMTP id 35BD86DE0217\r
+ for <notmuch@notmuchmail.org>; Sat, 7 May 2016 10:37:18 -0700 (PDT)\r
+X-Virus-Scanned: Debian amavisd-new at cworth.org\r
+X-Spam-Flag: NO\r
+X-Spam-Score: 0.6\r
+X-Spam-Level: \r
+X-Spam-Status: No, score=0.6 tagged_above=-999 required=5 tests=[AWL=-0.052,\r
+ SPF_NEUTRAL=0.652] autolearn=disabled\r
+Received: from arlo.cworth.org ([127.0.0.1])\r
+ by localhost (arlo.cworth.org [127.0.0.1]) (amavisd-new, port 10024)\r
+ with ESMTP id etCRJaI2VWbA for <notmuch@notmuchmail.org>;\r
+ Sat, 7 May 2016 10:37:09 -0700 (PDT)\r
+Received: from guru.guru-group.fi (guru.guru-group.fi [46.183.73.34])\r
+ by arlo.cworth.org (Postfix) with ESMTP id 3A20E6DE00EB\r
+ for <notmuch@notmuchmail.org>; Sat, 7 May 2016 10:37:09 -0700 (PDT)\r
+Received: from guru.guru-group.fi (localhost [IPv6:::1])\r
+ by guru.guru-group.fi (Postfix) with ESMTP id AA6B9100063;\r
+ Sat, 7 May 2016 20:37:07 +0300 (EEST)\r
+From: Tomi Ollila <tomi.ollila@iki.fi>\r
+To: David Bremner <david@tethera.net>, notmuch@notmuchmail.org\r
+Subject: Re: [Patch v3 07/11] CLI: add optional config data to dump output.\r
+In-Reply-To: <1462065879-29860-8-git-send-email-david@tethera.net>\r
+References: <1462065879-29860-1-git-send-email-david@tethera.net>\r
+ <1462065879-29860-8-git-send-email-david@tethera.net>\r
+User-Agent: Notmuch/0.22+9~gb26b5ad (http://notmuchmail.org) Emacs/24.3.1\r
+ (x86_64-unknown-linux-gnu)\r
+X-Face: HhBM'cA~<r"^Xv\KRN0P{vn'Y"Kd;zg_y3S[4)KSN~s?O\"QPoL\r
+ $[Xv_BD:i/F$WiEWax}R(MPS`^UaptOGD`*/=@\1lKoVa9tnrg0TW?"r7aRtgk[F\r
+ !)g;OY^,BjTbr)Np:%c_o'jj,Z\r
+Date: Sat, 07 May 2016 20:37:07 +0300\r
+Message-ID: <m2wpn5epmk.fsf@guru.guru-group.fi>\r
+MIME-Version: 1.0\r
+Content-Type: text/plain\r
+X-BeenThere: notmuch@notmuchmail.org\r
+X-Mailman-Version: 2.1.20\r
+Precedence: list\r
+List-Id: "Use and development of the notmuch mail system."\r
+ <notmuch.notmuchmail.org>\r
+List-Unsubscribe: <https://notmuchmail.org/mailman/options/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>\r
+List-Archive: <http://notmuchmail.org/pipermail/notmuch/>\r
+List-Post: <mailto:notmuch@notmuchmail.org>\r
+List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>\r
+List-Subscribe: <https://notmuchmail.org/mailman/listinfo/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=subscribe>\r
+X-List-Received-Date: Sat, 07 May 2016 17:37:18 -0000\r
+\r
+On Sun, May 01 2016, David Bremner <david@tethera.net> wrote:\r
+\r
+> Note that it changes the default dump output format, but doesn't break\r
+> existing notmuch-restore. It might break user scripts though.\r
+> ---\r
+> doc/man1/notmuch-dump.rst | 17 ++++++++++\r
+> notmuch-client.h | 8 +++++\r
+> notmuch-dump.c | 80 +++++++++++++++++++++++++++++++++++++++++++++--\r
+> notmuch-new.c | 2 +-\r
+> test/T150-tagging.sh | 8 ++---\r
+> test/T240-dump-restore.sh | 14 ++++-----\r
+> test/T590-libconfig.sh | 17 ++++++++++\r
+> test/test-lib.sh | 6 ++++\r
+> 8 files changed, 137 insertions(+), 15 deletions(-)\r
+>\r
+> diff --git a/doc/man1/notmuch-dump.rst b/doc/man1/notmuch-dump.rst\r
+> index a37c337..eda9e07 100644\r
+> --- a/doc/man1/notmuch-dump.rst\r
+> +++ b/doc/man1/notmuch-dump.rst\r
+> @@ -71,6 +71,23 @@ Supported options for **dump** include\r
+> characters. Note also that tags with spaces will not be\r
+> correctly restored with this format.\r
+> \r
+> + ``--include=(config|tags)``\r
+> +\r
+> + Control what kind of metadata is included in the output.\r
+> +\r
+> + **config**\r
+> +\r
+> + Output configuration data stored in the database. Each line\r
+> + starts with "#@ ", followed by a space seperated key-value\r
+> + pair. Both key and value are hex encoded if needed.\r
+> +\r
+> + **tags**\r
+> +\r
+> + Output per-message metadata, namely tags. See *format* above\r
+> + for description of the output.\r
+> +\r
+> + The default is to include both tags and configuration information\r
+> +\r
+> ``--output=``\ <filename>\r
+> Write output to given file instead of stdout.\r
+> \r
+> diff --git a/notmuch-client.h b/notmuch-client.h\r
+> index b3d0b66..ae6f124 100644\r
+> --- a/notmuch-client.h\r
+> +++ b/notmuch-client.h\r
+> @@ -446,11 +446,19 @@ typedef enum dump_formats {\r
+> DUMP_FORMAT_SUP\r
+> } dump_format_t;\r
+> \r
+> +typedef enum dump_includes {\r
+> + DUMP_INCLUDE_TAGS=1,\r
+> + DUMP_INCLUDE_CONFIG=2,\r
+> +} dump_include_t;\r
+> +\r
+> +#define NOTMUCH_DUMP_VERSION 2\r
+> +\r
+> int\r
+> notmuch_database_dump (notmuch_database_t *notmuch,\r
+> const char *output_file_name,\r
+> const char *query_str,\r
+> dump_format_t output_format,\r
+> + dump_include_t include,\r
+> notmuch_bool_t gzip_output);\r
+> \r
+> /* If status is non-zero (i.e. error) print appropriate\r
+> diff --git a/notmuch-dump.c b/notmuch-dump.c\r
+> index 829781f..a6cf810 100644\r
+> --- a/notmuch-dump.c\r
+> +++ b/notmuch-dump.c\r
+> @@ -23,16 +23,82 @@\r
+> #include "string-util.h"\r
+> #include <zlib.h>\r
+> \r
+> +static int\r
+> +database_dump_config (notmuch_database_t *notmuch, gzFile output)\r
+> +{\r
+> + notmuch_config_list_t *list;\r
+> + int ret = EXIT_FAILURE;\r
+> + char *buffer = NULL;\r
+> + size_t buffer_size = 0;\r
+> +\r
+> + if (print_status_database ("notmuch dump", notmuch,\r
+> + notmuch_database_get_config_list (notmuch, NULL, &list)))\r
+> + goto DONE;\r
+> +\r
+> + for (; notmuch_config_list_valid (list); notmuch_config_list_move_to_next (list)) {\r
+> + if (hex_encode (notmuch, notmuch_config_list_key (list),\r
+> + &buffer, &buffer_size) != HEX_SUCCESS) {\r
+> + fprintf (stderr, "Error: failed to hex-encode config key %s\n",\r
+> + notmuch_config_list_key (list));\r
+> + goto DONE;\r
+> + }\r
+> + gzprintf (output, "#@ %s", buffer);\r
+> +\r
+> + if (hex_encode (notmuch, notmuch_config_list_value (list),\r
+> + &buffer, &buffer_size) != HEX_SUCCESS) {\r
+> + fprintf (stderr, "Error: failed to hex-encode config value %s\n",\r
+> + notmuch_config_list_value (list) );\r
+> + goto DONE;\r
+> + }\r
+> +\r
+> + gzprintf (output, " %s\n", buffer);\r
+> + }\r
+> +\r
+> + ret = EXIT_SUCCESS;\r
+> +\r
+> + DONE:\r
+> + if (list)\r
+> + notmuch_config_list_destroy (list);\r
+> +\r
+> + if (buffer)\r
+> + talloc_free (buffer);\r
+> +\r
+> + return ret;\r
+> +}\r
+> +\r
+> +static void\r
+> +print_dump_header (gzFile output, int output_format, int include)\r
+> +{\r
+> + gzprintf (output, "#notmuch-dump %s:%d %s%s%s\n",\r
+> + (output_format == DUMP_FORMAT_SUP) ? "sup" : "batch-tag",\r
+> + NOTMUCH_DUMP_VERSION,\r
+> + (include & DUMP_INCLUDE_CONFIG) ? "config" : "",\r
+> + (include & DUMP_INCLUDE_TAGS) && (include & DUMP_INCLUDE_CONFIG) ? "," : "",\r
+> + (include & DUMP_INCLUDE_TAGS) ? "tags" : "");\r
+> +\r
+> +\r
+> +}\r
+> \r
+> static int\r
+> database_dump_file (notmuch_database_t *notmuch, gzFile output,\r
+> - const char *query_str, int output_format)\r
+> + const char *query_str, int output_format, int include)\r
+> {\r
+> notmuch_query_t *query;\r
+> notmuch_messages_t *messages;\r
+> notmuch_message_t *message;\r
+> notmuch_tags_t *tags;\r
+> \r
+> + print_dump_header (output, output_format, include);\r
+> +\r
+> + if (include & DUMP_INCLUDE_CONFIG) {\r
+> + if (print_status_database ("notmuch dump", notmuch,\r
+> + database_dump_config(notmuch,output)))\r
+> + return EXIT_FAILURE;\r
+> + }\r
+> +\r
+> + if (! (include & DUMP_INCLUDE_TAGS))\r
+> + return EXIT_SUCCESS;\r
+> +\r
+> if (! query_str)\r
+> query_str = "";\r
+> \r
+> @@ -130,6 +196,7 @@ notmuch_database_dump (notmuch_database_t *notmuch,\r
+> const char *output_file_name,\r
+> const char *query_str,\r
+> dump_format_t output_format,\r
+> + dump_include_t include,\r
+> notmuch_bool_t gzip_output)\r
+> {\r
+> gzFile output = NULL;\r
+> @@ -164,7 +231,7 @@ notmuch_database_dump (notmuch_database_t *notmuch,\r
+> goto DONE;\r
+> }\r
+> \r
+> - ret = database_dump_file (notmuch, output, query_str, output_format);\r
+> + ret = database_dump_file (notmuch, output, query_str, output_format, include);\r
+> if (ret) goto DONE;\r
+> \r
+> ret = gzflush (output, Z_FINISH);\r
+> @@ -226,6 +293,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])\r
+> int opt_index;\r
+> \r
+> int output_format = DUMP_FORMAT_BATCH_TAG;\r
+> + int include = 0;\r
+> notmuch_bool_t gzip_output = 0;\r
+> \r
+> notmuch_opt_desc_t options[] = {\r
+> @@ -233,6 +301,9 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])\r
+> (notmuch_keyword_t []){ { "sup", DUMP_FORMAT_SUP },\r
+> { "batch-tag", DUMP_FORMAT_BATCH_TAG },\r
+> { 0, 0 } } },\r
+> + { NOTMUCH_OPT_KEYWORD_FLAGS, &include, "include", 'I',\r
+> + (notmuch_keyword_t []){ { "config", DUMP_INCLUDE_CONFIG },\r
+> + { "tags", DUMP_INCLUDE_TAGS} } },\r
+> { NOTMUCH_OPT_STRING, &output_file_name, "output", 'o', 0 },\r
+> { NOTMUCH_OPT_BOOLEAN, &gzip_output, "gzip", 'z', 0 },\r
+> { NOTMUCH_OPT_INHERIT, (void *) ¬much_shared_options, NULL, 0, 0 },\r
+> @@ -245,6 +316,9 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])\r
+> \r
+> notmuch_process_shared_options (argv[0]);\r
+> \r
+> + if (include == 0)\r
+> + include = DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_TAGS;\r
+> +\r
+> if (opt_index < argc) {\r
+> query_str = query_string_from_args (notmuch, argc - opt_index, argv + opt_index);\r
+> if (query_str == NULL) {\r
+> @@ -254,7 +328,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])\r
+> }\r
+> \r
+> ret = notmuch_database_dump (notmuch, output_file_name, query_str,\r
+> - output_format, gzip_output);\r
+> + output_format, include, gzip_output);\r
+> \r
+> notmuch_database_destroy (notmuch);\r
+> \r
+> diff --git a/notmuch-new.c b/notmuch-new.c\r
+> index 04cb5ca..196a36e 100644\r
+> --- a/notmuch-new.c\r
+> +++ b/notmuch-new.c\r
+> @@ -1045,7 +1045,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[])\r
+> }\r
+> \r
+> if (notmuch_database_dump (notmuch, backup_name, "",\r
+> - DUMP_FORMAT_BATCH_TAG, TRUE)) {\r
+> + DUMP_FORMAT_BATCH_TAG, DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_TAGS, TRUE)) {\r
+> fprintf (stderr, "Backup failed. Aborting upgrade.");\r
+> return EXIT_FAILURE;\r
+> }\r
+> diff --git a/test/T150-tagging.sh b/test/T150-tagging.sh\r
+> index a451ffa..61d1311 100755\r
+> --- a/test/T150-tagging.sh\r
+> +++ b/test/T150-tagging.sh\r
+> @@ -188,7 +188,7 @@ cat <<EOF > EXPECTED\r
+> +%22%27%22%27%22%22%27%27 +inbox +tag4 +tag5 +unread -- id:msg-002@notmuch-test-suite\r
+> EOF\r
+> \r
+> -notmuch dump --format=batch-tag | sort > OUTPUT\r
+> +NOTMUCH_DUMP_TAGS > OUTPUT\r
+> notmuch restore --format=batch-tag < BACKUP\r
+> test_expect_equal_file EXPECTED OUTPUT\r
+> \r
+> @@ -209,7 +209,7 @@ cat <<EOF > EXPECTED\r
+> +%21@%23%20%24%25%5e%26%2a%29-_=+%5b%7b%5c%20%7c%3b%3a%27%20%22,.%3c%60%7e +inbox +tag5 +unread -- id:msg-001@notmuch-test-suite\r
+> EOF\r
+> \r
+> -notmuch dump --format=batch-tag | sort > OUTPUT\r
+> +NOTMUCH_DUMP_TAGS > OUTPUT\r
+> notmuch restore --format=batch-tag < BACKUP\r
+> test_expect_equal_file EXPECTED OUTPUT\r
+> \r
+> @@ -235,7 +235,7 @@ cat <<EOF > EXPECTED\r
+> +%2a@%7d%cf%b5%f4%85%80%adO3%da%a7 +=%e0%ac%95%c8%b3+%ef%aa%95%c8%a64w%c7%9d%c9%a2%cf%b3%d6%82%24B%c4%a9%c5%a1UX%ee%99%b0%27E7%ca%a4%d0%8b%5d +A%e1%a0%bc%de%8b%d5%b2V%d9%9b%f3%b5%a2%a3M%d8%a1u@%f0%a0%ac%948%7e%f0%ab%86%af%27 +L%df%85%ef%a1%a5m@%d3%96%c2%ab%d4%9f%ca%b8%f3%b3%a2%bf%c7%b1_u%d7%b4%c7%b1 +P%c4%98%2f +R +inbox +tag5 +unread +%7e%d1%8b%25%ec%a0%ae%d1%a0M%3b%e3%b6%b7%e9%a4%87%3c%db%9a%cc%a8%e1%96%9d +%c4%bf7%c7%ab9H%c4%99k%ea%91%bd%c3%8ck%e2%b3%8dk%c5%952V%e4%99%b2%d9%b3%e4%8b%bda%5b%24%c7%9b +%da%88=f%cc%b9I%ce%af%7b%c9%97%e3%b9%8bH%cb%92X%d2%8c6 +%dc%9crh%d2%86B%e5%97%a2%22t%ed%99%82d -- id:msg-001@notmuch-test-suite\r
+> EOF\r
+> \r
+> -notmuch dump --format=batch-tag | sort > OUTPUT\r
+> +NOTMUCH_DUMP_TAGS > OUTPUT\r
+> notmuch restore --format=batch-tag < BACKUP\r
+> test_expect_equal_file EXPECTED OUTPUT\r
+> \r
+> @@ -260,7 +260,7 @@ cat <<EOF > EXPECTED\r
+> +foo%3a%3abar%25 +found%3a%3ait +inbox +tag5 +unread +winner -- id:msg-001@notmuch-test-suite\r
+> EOF\r
+> \r
+> -notmuch dump --format=batch-tag | sort > OUTPUT\r
+> +NOTMUCH_DUMP_TAGS > OUTPUT\r
+> notmuch restore --format=batch-tag < BACKUP\r
+> test_expect_equal_file EXPECTED OUTPUT\r
+> \r
+> diff --git a/test/T240-dump-restore.sh b/test/T240-dump-restore.sh\r
+> index e6976ff..758d355 100755\r
+> --- a/test/T240-dump-restore.sh\r
+> +++ b/test/T240-dump-restore.sh\r
+> @@ -97,7 +97,7 @@ test_expect_equal_file dump.expected dump.actual\r
+> # Note, we assume all messages from cworth have a message-id\r
+> # containing cworth.org\r
+> \r
+> -grep 'cworth[.]org' dump.expected > dump-cworth.expected\r
+> +(head -1 dump.expected ; grep 'cworth[.]org' dump.expected) > dump-cworth.expected\r
+\r
+Instead of executing the commands in subshell, the 'group command' format\r
+could be used (and run in current shell). i.e. \r
+\r
+{ head -1 dump.expected ; grep 'cworth[.]org' dump.expected; } > dump-cworth.expected\r
+\r
+> test_begin_subtest "dump -- from:cworth"\r
+> notmuch dump -- from:cworth > dump-dash-cworth.actual\r
+> @@ -118,16 +118,16 @@ notmuch search --output=messages from:cworth | sed s/^id:// |\\r
+> test_expect_equal_file OUTPUT EXPECTED\r
+> \r
+> test_begin_subtest "format=batch-tag, dump sanity check."\r
+> -notmuch dump --format=sup from:cworth | cut -f1 -d' ' | \\r
+> +NOTMUCH_DUMP_TAGS --format=sup from:cworth | cut -f1 -d' ' | \\r
+> sort > EXPECTED.$test_count\r
+> -notmuch dump --format=batch-tag from:cworth | sed 's/^.*-- id://' | \\r
+> +NOTMUCH_DUMP_TAGS --format=batch-tag from:cworth | sed 's/^.*-- id://' | \\r
+> sort > OUTPUT.$test_count\r
+> test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count\r
+> \r
+> test_begin_subtest "format=batch-tag, missing newline"\r
+> printf "+a_tag_without_newline -- id:20091117232137.GA7669@griffis1.net" > IN\r
+> notmuch restore --accumulate < IN\r
+> -notmuch dump id:20091117232137.GA7669@griffis1.net > OUT\r
+> +NOTMUCH_DUMP_TAGS id:20091117232137.GA7669@griffis1.net > OUT\r
+> cat <<EOF > EXPECTED\r
+> +a_tag_without_newline +inbox +unread -- id:20091117232137.GA7669@griffis1.net\r
+> EOF\r
+> @@ -155,7 +155,7 @@ cat <<EOF >EXPECTED.$test_count\r
+> + -- id:20091117232137.GA7669@griffis1.net\r
+> EOF\r
+> notmuch restore --format=batch-tag < EXPECTED.$test_count\r
+> -notmuch dump --format=batch-tag id:20091117232137.GA7669@griffis1.net > OUTPUT.$test_count\r
+> +NOTMUCH_DUMP_TAGS --format=batch-tag id:20091117232137.GA7669@griffis1.net > OUTPUT.$test_count\r
+> test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count\r
+> \r
+> tag1='comic_swear=$&^%$^%\\//-+$^%$'\r
+> @@ -217,9 +217,9 @@ notmuch dump --format=batch-tag > OUTPUT.$test_count\r
+> test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count\r
+> \r
+> test_begin_subtest 'format=batch-tag, checking encoded output'\r
+> -notmuch dump --format=batch-tag -- from:cworth |\\r
+> +NOTMUCH_DUMP_TAGS --format=batch-tag -- from:cworth |\\r
+> awk "{ print \"+$enc1 +$enc2 +$enc3 -- \" \$5 }" > EXPECTED.$test_count\r
+\r
+The above I don't understand, $enc1, $enc2 and $enc3 should be expanded by\r
+the shell into the awk command -- I don't see these set anywhere, so to\r
+me it looks like these are empty strings...\r
+\r
+> -notmuch dump --format=batch-tag -- from:cworth > OUTPUT.$test_count\r
+> +NOTMUCH_DUMP_TAGS --format=batch-tag -- from:cworth > OUTPUT.$test_count\r
+> test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count\r
+> \r
+> test_begin_subtest 'restoring sane tags'\r
+> diff --git a/test/T590-libconfig.sh b/test/T590-libconfig.sh\r
+> index 8ca6883..5ea5300 100755\r
+> --- a/test/T590-libconfig.sh\r
+> +++ b/test/T590-libconfig.sh\r
+> @@ -115,4 +115,21 @@ testkey2 testvalue2\r
+> EOF\r
+> test_expect_equal_file EXPECTED OUTPUT\r
+> \r
+> +test_begin_subtest "dump config"\r
+> +cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}\r
+> +{\r
+> + RUN(notmuch_database_set_config (db, "key with spaces", "value, with, spaces!"));\r
+> +}\r
+> +EOF\r
+> +notmuch dump --include=config >OUTPUT\r
+> +cat <<'EOF' >EXPECTED\r
+> +#notmuch-dump batch-tag:2 config\r
+> +#@ aaabefore beforeval\r
+> +#@ key%20with%20spaces value,%20with,%20spaces%21\r
+> +#@ testkey1 testvalue1\r
+> +#@ testkey2 testvalue2\r
+> +#@ zzzafter afterval\r
+> +EOF\r
+> +test_expect_equal_file EXPECTED OUTPUT\r
+> +\r
+> test_done\r
+> diff --git a/test/test-lib.sh b/test/test-lib.sh\r
+> index 09f8731..68a8774 100644\r
+> --- a/test/test-lib.sh\r
+> +++ b/test/test-lib.sh\r
+> @@ -673,6 +673,12 @@ NOTMUCH_NEW ()\r
+> notmuch new "${@}" | grep -v -E -e '^Processed [0-9]*( total)? file|Found [0-9]* total file'\r
+> }\r
+> \r
+> +NOTMUCH_DUMP_TAGS ()\r
+> +{\r
+> + # this relies on the default format being batch-tag, otherwise some tests will break\r
+> + notmuch dump --include=tags "${@}" | sed '/^#/d' | sort\r
+> +}\r
+> +\r
+> notmuch_search_sanitize ()\r
+> {\r
+> perl -pe 's/("?thread"?: ?)("?)................("?)/\1\2XXX\3/'\r
+> -- \r
+> 2.8.0.rc3\r
+>\r
+> _______________________________________________\r
+> notmuch mailing list\r
+> notmuch@notmuchmail.org\r
+> https://notmuchmail.org/mailman/listinfo/notmuch\r