From cb0d517b065f38d235ba437c0948680e0a4e1550 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Thu, 26 May 2016 07:51:22 +2100 Subject: [PATCH] [Patch v6 3/7] CLI: add optional config data to dump output. --- 2c/b0c6a7747e19f0fc4f3a5e0bf6638145a0e1b0 | 405 ++++++++++++++++++++++ 1 file changed, 405 insertions(+) create mode 100644 2c/b0c6a7747e19f0fc4f3a5e0bf6638145a0e1b0 diff --git a/2c/b0c6a7747e19f0fc4f3a5e0bf6638145a0e1b0 b/2c/b0c6a7747e19f0fc4f3a5e0bf6638145a0e1b0 new file mode 100644 index 000000000..eda608543 --- /dev/null +++ b/2c/b0c6a7747e19f0fc4f3a5e0bf6638145a0e1b0 @@ -0,0 +1,405 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by arlo.cworth.org (Postfix) with ESMTP id 46CD46DE0931 + for ; Wed, 25 May 2016 03:52:04 -0700 (PDT) +X-Virus-Scanned: Debian amavisd-new at cworth.org +X-Spam-Flag: NO +X-Spam-Score: -0.012 +X-Spam-Level: +X-Spam-Status: No, score=-0.012 tagged_above=-999 required=5 + tests=[AWL=-0.001, SPF_PASS=-0.001, T_RP_MATCHES_RCVD=-0.01] + autolearn=disabled +Received: from arlo.cworth.org ([127.0.0.1]) + by localhost (arlo.cworth.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id h5tyTwJSPLZ1 for ; + Wed, 25 May 2016 03:51:55 -0700 (PDT) +Received: from fethera.tethera.net (fethera.tethera.net [198.245.60.197]) + by arlo.cworth.org (Postfix) with ESMTPS id 0C21E6DE02B5 + for ; Wed, 25 May 2016 03:51:39 -0700 (PDT) +Received: from remotemail by fethera.tethera.net with local (Exim 4.84) + (envelope-from ) + id 1b5WPR-0004UN-KJ; Wed, 25 May 2016 06:51:29 -0400 +Received: (nullmailer pid 23140 invoked by uid 1000); + Wed, 25 May 2016 10:51:31 -0000 +From: David Bremner +To: notmuch@notmuchmail.org +Subject: [Patch v6 3/7] CLI: add optional config data to dump output. +Date: Wed, 25 May 2016 07:51:22 -0300 +Message-Id: <1464173486-23032-4-git-send-email-david@tethera.net> +X-Mailer: git-send-email 2.8.1 +In-Reply-To: <1464173486-23032-1-git-send-email-david@tethera.net> +References: <1464173486-23032-1-git-send-email-david@tethera.net> +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.20 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Wed, 25 May 2016 10:52:04 -0000 + +Note that it changes the default dump output format, but doesn't break +existing notmuch-restore. It might break user scripts though. +--- + doc/man1/notmuch-dump.rst | 17 +++++++++++ + notmuch-client.h | 8 +++++ + notmuch-dump.c | 78 +++++++++++++++++++++++++++++++++++++++++++++-- + notmuch-new.c | 2 +- + test/T150-tagging.sh | 8 ++--- + test/T240-dump-restore.sh | 14 ++++----- + test/T590-libconfig.sh | 17 +++++++++++ + test/test-lib.sh | 6 ++++ + 8 files changed, 135 insertions(+), 15 deletions(-) + +diff --git a/doc/man1/notmuch-dump.rst b/doc/man1/notmuch-dump.rst +index a37c337..eda9e07 100644 +--- a/doc/man1/notmuch-dump.rst ++++ b/doc/man1/notmuch-dump.rst +@@ -71,6 +71,23 @@ Supported options for **dump** include + characters. Note also that tags with spaces will not be + correctly restored with this format. + ++ ``--include=(config|tags)`` ++ ++ Control what kind of metadata is included in the output. ++ ++ **config** ++ ++ Output configuration data stored in the database. Each line ++ starts with "#@ ", followed by a space seperated key-value ++ pair. Both key and value are hex encoded if needed. ++ ++ **tags** ++ ++ Output per-message metadata, namely tags. See *format* above ++ for description of the output. ++ ++ The default is to include both tags and configuration information ++ + ``--output=``\ + Write output to given file instead of stdout. + +diff --git a/notmuch-client.h b/notmuch-client.h +index b3d0b66..c130b93 100644 +--- a/notmuch-client.h ++++ b/notmuch-client.h +@@ -446,11 +446,19 @@ typedef enum dump_formats { + DUMP_FORMAT_SUP + } dump_format_t; + ++typedef enum dump_includes { ++ DUMP_INCLUDE_TAGS = 1, ++ DUMP_INCLUDE_CONFIG = 2, ++} dump_include_t; ++ ++#define NOTMUCH_DUMP_VERSION 2 ++ + int + notmuch_database_dump (notmuch_database_t *notmuch, + const char *output_file_name, + const char *query_str, + dump_format_t output_format, ++ dump_include_t include, + notmuch_bool_t gzip_output); + + /* If status is non-zero (i.e. error) print appropriate +diff --git a/notmuch-dump.c b/notmuch-dump.c +index 829781f..8e79511 100644 +--- a/notmuch-dump.c ++++ b/notmuch-dump.c +@@ -23,16 +23,80 @@ + #include "string-util.h" + #include + ++static int ++database_dump_config (notmuch_database_t *notmuch, gzFile output) ++{ ++ notmuch_config_list_t *list; ++ int ret = EXIT_FAILURE; ++ char *buffer = NULL; ++ size_t buffer_size = 0; ++ ++ if (print_status_database ("notmuch dump", notmuch, ++ notmuch_database_get_config_list (notmuch, NULL, &list))) ++ goto DONE; ++ ++ for (; notmuch_config_list_valid (list); notmuch_config_list_move_to_next (list)) { ++ if (hex_encode (notmuch, notmuch_config_list_key (list), ++ &buffer, &buffer_size) != HEX_SUCCESS) { ++ fprintf (stderr, "Error: failed to hex-encode config key %s\n", ++ notmuch_config_list_key (list)); ++ goto DONE; ++ } ++ gzprintf (output, "#@ %s", buffer); ++ ++ if (hex_encode (notmuch, notmuch_config_list_value (list), ++ &buffer, &buffer_size) != HEX_SUCCESS) { ++ fprintf (stderr, "Error: failed to hex-encode config value %s\n", ++ notmuch_config_list_value (list) ); ++ goto DONE; ++ } ++ ++ gzprintf (output, " %s\n", buffer); ++ } ++ ++ ret = EXIT_SUCCESS; ++ ++ DONE: ++ if (list) ++ notmuch_config_list_destroy (list); ++ ++ if (buffer) ++ talloc_free (buffer); ++ ++ return ret; ++} ++ ++static void ++print_dump_header (gzFile output, int output_format, int include) ++{ ++ gzprintf (output, "#notmuch-dump %s:%d %s%s%s\n", ++ (output_format == DUMP_FORMAT_SUP) ? "sup" : "batch-tag", ++ NOTMUCH_DUMP_VERSION, ++ (include & DUMP_INCLUDE_CONFIG) ? "config" : "", ++ (include & DUMP_INCLUDE_TAGS) && (include & DUMP_INCLUDE_CONFIG) ? "," : "", ++ (include & DUMP_INCLUDE_TAGS) ? "tags" : ""); ++} + + static int + database_dump_file (notmuch_database_t *notmuch, gzFile output, +- const char *query_str, int output_format) ++ const char *query_str, int output_format, int include) + { + notmuch_query_t *query; + notmuch_messages_t *messages; + notmuch_message_t *message; + notmuch_tags_t *tags; + ++ print_dump_header (output, output_format, include); ++ ++ if (include & DUMP_INCLUDE_CONFIG) { ++ if (print_status_database ("notmuch dump", notmuch, ++ database_dump_config(notmuch,output))) ++ return EXIT_FAILURE; ++ } ++ ++ if (! (include & DUMP_INCLUDE_TAGS)) ++ return EXIT_SUCCESS; ++ + if (! query_str) + query_str = ""; + +@@ -130,6 +194,7 @@ notmuch_database_dump (notmuch_database_t *notmuch, + const char *output_file_name, + const char *query_str, + dump_format_t output_format, ++ dump_include_t include, + notmuch_bool_t gzip_output) + { + gzFile output = NULL; +@@ -164,7 +229,7 @@ notmuch_database_dump (notmuch_database_t *notmuch, + goto DONE; + } + +- ret = database_dump_file (notmuch, output, query_str, output_format); ++ ret = database_dump_file (notmuch, output, query_str, output_format, include); + if (ret) goto DONE; + + ret = gzflush (output, Z_FINISH); +@@ -226,6 +291,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[]) + int opt_index; + + int output_format = DUMP_FORMAT_BATCH_TAG; ++ int include = 0; + notmuch_bool_t gzip_output = 0; + + notmuch_opt_desc_t options[] = { +@@ -233,6 +299,9 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[]) + (notmuch_keyword_t []){ { "sup", DUMP_FORMAT_SUP }, + { "batch-tag", DUMP_FORMAT_BATCH_TAG }, + { 0, 0 } } }, ++ { NOTMUCH_OPT_KEYWORD_FLAGS, &include, "include", 'I', ++ (notmuch_keyword_t []){ { "config", DUMP_INCLUDE_CONFIG }, ++ { "tags", DUMP_INCLUDE_TAGS} } }, + { NOTMUCH_OPT_STRING, &output_file_name, "output", 'o', 0 }, + { NOTMUCH_OPT_BOOLEAN, &gzip_output, "gzip", 'z', 0 }, + { NOTMUCH_OPT_INHERIT, (void *) ¬much_shared_options, NULL, 0, 0 }, +@@ -245,6 +314,9 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[]) + + notmuch_process_shared_options (argv[0]); + ++ if (include == 0) ++ include = DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_TAGS; ++ + if (opt_index < argc) { + query_str = query_string_from_args (notmuch, argc - opt_index, argv + opt_index); + if (query_str == NULL) { +@@ -254,7 +326,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[]) + } + + ret = notmuch_database_dump (notmuch, output_file_name, query_str, +- output_format, gzip_output); ++ output_format, include, gzip_output); + + notmuch_database_destroy (notmuch); + +diff --git a/notmuch-new.c b/notmuch-new.c +index 2d975eb..8d1545e 100644 +--- a/notmuch-new.c ++++ b/notmuch-new.c +@@ -1042,7 +1042,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[]) + } + + if (notmuch_database_dump (notmuch, backup_name, "", +- DUMP_FORMAT_BATCH_TAG, TRUE)) { ++ DUMP_FORMAT_BATCH_TAG, DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_TAGS, TRUE)) { + fprintf (stderr, "Backup failed. Aborting upgrade."); + return EXIT_FAILURE; + } +diff --git a/test/T150-tagging.sh b/test/T150-tagging.sh +index a451ffa..61d1311 100755 +--- a/test/T150-tagging.sh ++++ b/test/T150-tagging.sh +@@ -188,7 +188,7 @@ cat < EXPECTED + +%22%27%22%27%22%22%27%27 +inbox +tag4 +tag5 +unread -- id:msg-002@notmuch-test-suite + EOF + +-notmuch dump --format=batch-tag | sort > OUTPUT ++NOTMUCH_DUMP_TAGS > OUTPUT + notmuch restore --format=batch-tag < BACKUP + test_expect_equal_file EXPECTED OUTPUT + +@@ -209,7 +209,7 @@ cat < EXPECTED + +%21@%23%20%24%25%5e%26%2a%29-_=+%5b%7b%5c%20%7c%3b%3a%27%20%22,.%3c%60%7e +inbox +tag5 +unread -- id:msg-001@notmuch-test-suite + EOF + +-notmuch dump --format=batch-tag | sort > OUTPUT ++NOTMUCH_DUMP_TAGS > OUTPUT + notmuch restore --format=batch-tag < BACKUP + test_expect_equal_file EXPECTED OUTPUT + +@@ -235,7 +235,7 @@ cat < EXPECTED + +%2a@%7d%cf%b5%f4%85%80%adO3%da%a7 +=%e0%ac%95%c8%b3+%ef%aa%95%c8%a64w%c7%9d%c9%a2%cf%b3%d6%82%24B%c4%a9%c5%a1UX%ee%99%b0%27E7%ca%a4%d0%8b%5d +A%e1%a0%bc%de%8b%d5%b2V%d9%9b%f3%b5%a2%a3M%d8%a1u@%f0%a0%ac%948%7e%f0%ab%86%af%27 +L%df%85%ef%a1%a5m@%d3%96%c2%ab%d4%9f%ca%b8%f3%b3%a2%bf%c7%b1_u%d7%b4%c7%b1 +P%c4%98%2f +R +inbox +tag5 +unread +%7e%d1%8b%25%ec%a0%ae%d1%a0M%3b%e3%b6%b7%e9%a4%87%3c%db%9a%cc%a8%e1%96%9d +%c4%bf7%c7%ab9H%c4%99k%ea%91%bd%c3%8ck%e2%b3%8dk%c5%952V%e4%99%b2%d9%b3%e4%8b%bda%5b%24%c7%9b +%da%88=f%cc%b9I%ce%af%7b%c9%97%e3%b9%8bH%cb%92X%d2%8c6 +%dc%9crh%d2%86B%e5%97%a2%22t%ed%99%82d -- id:msg-001@notmuch-test-suite + EOF + +-notmuch dump --format=batch-tag | sort > OUTPUT ++NOTMUCH_DUMP_TAGS > OUTPUT + notmuch restore --format=batch-tag < BACKUP + test_expect_equal_file EXPECTED OUTPUT + +@@ -260,7 +260,7 @@ cat < EXPECTED + +foo%3a%3abar%25 +found%3a%3ait +inbox +tag5 +unread +winner -- id:msg-001@notmuch-test-suite + EOF + +-notmuch dump --format=batch-tag | sort > OUTPUT ++NOTMUCH_DUMP_TAGS > OUTPUT + notmuch restore --format=batch-tag < BACKUP + test_expect_equal_file EXPECTED OUTPUT + +diff --git a/test/T240-dump-restore.sh b/test/T240-dump-restore.sh +index e6976ff..bbfb09b 100755 +--- a/test/T240-dump-restore.sh ++++ b/test/T240-dump-restore.sh +@@ -97,7 +97,7 @@ test_expect_equal_file dump.expected dump.actual + # Note, we assume all messages from cworth have a message-id + # containing cworth.org + +-grep 'cworth[.]org' dump.expected > dump-cworth.expected ++{ head -1 dump.expected ; grep 'cworth[.]org' dump.expected; } > dump-cworth.expected + + test_begin_subtest "dump -- from:cworth" + notmuch dump -- from:cworth > dump-dash-cworth.actual +@@ -118,16 +118,16 @@ notmuch search --output=messages from:cworth | sed s/^id:// |\ + test_expect_equal_file OUTPUT EXPECTED + + test_begin_subtest "format=batch-tag, dump sanity check." +-notmuch dump --format=sup from:cworth | cut -f1 -d' ' | \ ++NOTMUCH_DUMP_TAGS --format=sup from:cworth | cut -f1 -d' ' | \ + sort > EXPECTED.$test_count +-notmuch dump --format=batch-tag from:cworth | sed 's/^.*-- id://' | \ ++NOTMUCH_DUMP_TAGS --format=batch-tag from:cworth | sed 's/^.*-- id://' | \ + sort > OUTPUT.$test_count + test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count + + test_begin_subtest "format=batch-tag, missing newline" + printf "+a_tag_without_newline -- id:20091117232137.GA7669@griffis1.net" > IN + notmuch restore --accumulate < IN +-notmuch dump id:20091117232137.GA7669@griffis1.net > OUT ++NOTMUCH_DUMP_TAGS id:20091117232137.GA7669@griffis1.net > OUT + cat < EXPECTED + +a_tag_without_newline +inbox +unread -- id:20091117232137.GA7669@griffis1.net + EOF +@@ -155,7 +155,7 @@ cat <EXPECTED.$test_count + + -- id:20091117232137.GA7669@griffis1.net + EOF + notmuch restore --format=batch-tag < EXPECTED.$test_count +-notmuch dump --format=batch-tag id:20091117232137.GA7669@griffis1.net > OUTPUT.$test_count ++NOTMUCH_DUMP_TAGS --format=batch-tag id:20091117232137.GA7669@griffis1.net > OUTPUT.$test_count + test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count + + tag1='comic_swear=$&^%$^%\\//-+$^%$' +@@ -217,9 +217,9 @@ notmuch dump --format=batch-tag > OUTPUT.$test_count + test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count + + test_begin_subtest 'format=batch-tag, checking encoded output' +-notmuch dump --format=batch-tag -- from:cworth |\ ++NOTMUCH_DUMP_TAGS --format=batch-tag -- from:cworth |\ + awk "{ print \"+$enc1 +$enc2 +$enc3 -- \" \$5 }" > EXPECTED.$test_count +-notmuch dump --format=batch-tag -- from:cworth > OUTPUT.$test_count ++NOTMUCH_DUMP_TAGS --format=batch-tag -- from:cworth > OUTPUT.$test_count + test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count + + test_begin_subtest 'restoring sane tags' +diff --git a/test/T590-libconfig.sh b/test/T590-libconfig.sh +index 8ca6883..5ea5300 100755 +--- a/test/T590-libconfig.sh ++++ b/test/T590-libconfig.sh +@@ -115,4 +115,21 @@ testkey2 testvalue2 + EOF + test_expect_equal_file EXPECTED OUTPUT + ++test_begin_subtest "dump config" ++cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} ++{ ++ RUN(notmuch_database_set_config (db, "key with spaces", "value, with, spaces!")); ++} ++EOF ++notmuch dump --include=config >OUTPUT ++cat <<'EOF' >EXPECTED ++#notmuch-dump batch-tag:2 config ++#@ aaabefore beforeval ++#@ key%20with%20spaces value,%20with,%20spaces%21 ++#@ testkey1 testvalue1 ++#@ testkey2 testvalue2 ++#@ zzzafter afterval ++EOF ++test_expect_equal_file EXPECTED OUTPUT ++ + test_done +diff --git a/test/test-lib.sh b/test/test-lib.sh +index 62e123d..e439862 100644 +--- a/test/test-lib.sh ++++ b/test/test-lib.sh +@@ -674,6 +674,12 @@ NOTMUCH_NEW () + notmuch new "${@}" | grep -v -E -e '^Processed [0-9]*( total)? file|Found [0-9]* total file' + } + ++NOTMUCH_DUMP_TAGS () ++{ ++ # this relies on the default format being batch-tag, otherwise some tests will break ++ notmuch dump --include=tags "${@}" | sed '/^#/d' | sort ++} ++ + notmuch_search_sanitize () + { + perl -pe 's/("?thread"?: ?)("?)................("?)/\1\2XXX\3/' +-- +2.8.1 + -- 2.26.2