From b5dbd0a4562729e2e60a4c2010c64d9a0b24c648 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 6 Aug 2016 22:52:37 +0900 Subject: [PATCH] [PATCH 7/9] CLI: add properties to dump output --- 73/f83b8238efb1dbd4460ca05b6412a74976e14a | 295 ++++++++++++++++++++++ 1 file changed, 295 insertions(+) create mode 100644 73/f83b8238efb1dbd4460ca05b6412a74976e14a diff --git a/73/f83b8238efb1dbd4460ca05b6412a74976e14a b/73/f83b8238efb1dbd4460ca05b6412a74976e14a new file mode 100644 index 000000000..08bd0c099 --- /dev/null +++ b/73/f83b8238efb1dbd4460ca05b6412a74976e14a @@ -0,0 +1,295 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by arlo.cworth.org (Postfix) with ESMTP id 3E88D6DE091B + for ; Sat, 6 Aug 2016 06:53:07 -0700 (PDT) +X-Virus-Scanned: Debian amavisd-new at cworth.org +X-Spam-Flag: NO +X-Spam-Score: -0.004 +X-Spam-Level: +X-Spam-Status: No, score=-0.004 tagged_above=-999 required=5 + tests=[AWL=-0.005, HEADER_FROM_DIFFERENT_DOMAINS=0.001] + autolearn=disabled +Received: from arlo.cworth.org ([127.0.0.1]) + by localhost (arlo.cworth.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id tC0-fn3Idznt for ; + Sat, 6 Aug 2016 06:52:59 -0700 (PDT) +Received: from fethera.tethera.net (fethera.tethera.net [198.245.60.197]) + by arlo.cworth.org (Postfix) with ESMTPS id 816EE6DE02AF + for ; Sat, 6 Aug 2016 06:52:55 -0700 (PDT) +Received: from remotemail by fethera.tethera.net with local (Exim 4.84_2) + (envelope-from ) + id 1bW22I-0007Du-Fu; Sat, 06 Aug 2016 09:53:10 -0400 +Received: (nullmailer pid 4137 invoked by uid 1000); + Sat, 06 Aug 2016 13:52:44 -0000 +From: David Bremner +To: notmuch@notmuchmail.org +Subject: [PATCH 7/9] CLI: add properties to dump output +Date: Sat, 6 Aug 2016 22:52:37 +0900 +Message-Id: <1470491559-3946-8-git-send-email-david@tethera.net> +X-Mailer: git-send-email 2.8.1 +In-Reply-To: <1470491559-3946-1-git-send-email-david@tethera.net> +References: <1470491559-3946-1-git-send-email-david@tethera.net> +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.20 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Sat, 06 Aug 2016 13:53:07 -0000 + +Part of providing extensibility via properties is to make sure that user +data is not lost. Thus we need to be able to dump and restore +properties. +--- + doc/man1/notmuch-dump.rst | 18 ++++++--- + notmuch-client.h | 3 ++ + notmuch-dump.c | 85 +++++++++++++++++++++++++++++++++++++++---- + notmuch-new.c | 2 +- + test/T610-message-property.sh | 21 +++++++++++ + 5 files changed, 116 insertions(+), 13 deletions(-) + +diff --git a/doc/man1/notmuch-dump.rst b/doc/man1/notmuch-dump.rst +index 94986a8..751850b 100644 +--- a/doc/man1/notmuch-dump.rst ++++ b/doc/man1/notmuch-dump.rst +@@ -71,7 +71,7 @@ Supported options for **dump** include + characters. Note also that tags with spaces will not be + correctly restored with this format. + +- ``--include=(config|tags)`` ++ ``--include=(config|properties|tags)`` + + Control what kind of metadata is included in the output. + +@@ -81,14 +81,22 @@ Supported options for **dump** include + starts with "#@ ", followed by a space seperated key-value + pair. Both key and value are hex encoded if needed. + ++ **properties** ++ ++ Output per-message (key,value) metadata. Each line starts ++ with "#= ", followed by a message id, and a space seperated ++ list of key=value pairs. pair. Ids, keys and values are hex ++ encoded if needed. ++ + **tags** + +- Output per-message metadata, namely tags. See *format* above ++ Output per-message boolean metadata, namely tags. See *format* above + for description of the output. + +- The default is to include both tags and configuration +- information. As of version 2 of the dump format, there is a +- header line of the following form ++ The default is to include all available types of data. The ++ option can be specified multiple times to select some subset. As ++ of version 2 of the dump format, there is a header line of the ++ following form + + | + | #notmuch-dump <*format*>:<*version*> <*included*> +diff --git a/notmuch-client.h b/notmuch-client.h +index ebc092b..9ce2aef 100644 +--- a/notmuch-client.h ++++ b/notmuch-client.h +@@ -449,8 +449,11 @@ typedef enum dump_formats { + typedef enum dump_includes { + DUMP_INCLUDE_TAGS = 1, + DUMP_INCLUDE_CONFIG = 2, ++ DUMP_INCLUDE_PROPERTIES = 4 + } dump_include_t; + ++#define DUMP_INCLUDE_DEFAULT (DUMP_INCLUDE_TAGS | DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_PROPERTIES) ++ + #define NOTMUCH_DUMP_VERSION 2 + + int +diff --git a/notmuch-dump.c b/notmuch-dump.c +index d80ed8b8..e7965ce 100644 +--- a/notmuch-dump.c ++++ b/notmuch-dump.c +@@ -69,12 +69,77 @@ database_dump_config (notmuch_database_t *notmuch, gzFile output) + static void + print_dump_header (gzFile output, int output_format, int include) + { +- gzprintf (output, "#notmuch-dump %s:%d %s%s%s\n", ++ const char *sep = ""; ++ ++ gzprintf (output, "#notmuch-dump %s:%d ", + (output_format == DUMP_FORMAT_SUP) ? "sup" : "batch-tag", +- NOTMUCH_DUMP_VERSION, +- (include & DUMP_INCLUDE_CONFIG) ? "config" : "", +- (include & DUMP_INCLUDE_TAGS) && (include & DUMP_INCLUDE_CONFIG) ? "," : "", +- (include & DUMP_INCLUDE_TAGS) ? "tags" : ""); ++ NOTMUCH_DUMP_VERSION); ++ ++ if (include & DUMP_INCLUDE_CONFIG) { ++ gzputs (output, "config"); ++ sep = ","; ++ } ++ if (include & DUMP_INCLUDE_PROPERTIES) { ++ gzprintf (output, "%sproperties", sep); ++ sep = ","; ++ } ++ if (include & DUMP_INCLUDE_TAGS) { ++ gzprintf (output, "%sproperties", sep); ++ } ++ gzputs (output, "\n"); ++} ++ ++static int ++dump_properties_message (void *ctx, ++ notmuch_message_t *message, ++ gzFile output, ++ char **buffer_p, size_t *size_p) ++{ ++ const char *message_id; ++ notmuch_message_properties_t *list; ++ notmuch_bool_t first = TRUE; ++ ++ message_id = notmuch_message_get_message_id (message); ++ ++ if (strchr (message_id, '\n')) { ++ fprintf (stderr, "Warning: skipping message id containing line break: \"%s\"\n", message_id); ++ return 0; ++ } ++ ++ for (list = notmuch_message_get_properties (message, "", FALSE); ++ notmuch_message_properties_valid (list); notmuch_message_properties_move_to_next (list)) { ++ const char *key, *val; ++ ++ if (first) { ++ if (hex_encode (ctx, message_id, buffer_p, size_p) != HEX_SUCCESS) { ++ fprintf (stderr, "Error: failed to hex-encode message-id %s\n", message_id); ++ return 1; ++ } ++ gzprintf (output, "#= %s", *buffer_p); ++ first = FALSE; ++ } ++ ++ key = notmuch_message_properties_key (list); ++ val = notmuch_message_properties_value (list); ++ ++ if (hex_encode (ctx, key, buffer_p, size_p) != HEX_SUCCESS) { ++ fprintf (stderr, "Error: failed to hex-encode key %s\n", key); ++ return 1; ++ } ++ gzprintf (output, " %s", *buffer_p); ++ ++ if (hex_encode (ctx, val, buffer_p, size_p) != HEX_SUCCESS) { ++ fprintf (stderr, "Error: failed to hex-encode value %s\n", val); ++ return 1; ++ } ++ gzprintf (output, "=%s", *buffer_p); ++ } ++ notmuch_message_properties_destroy (list); ++ ++ if (! first) ++ gzprintf (output, "\n", *buffer_p); ++ ++ return 0; + } + + static int +@@ -159,7 +224,7 @@ database_dump_file (notmuch_database_t *notmuch, gzFile output, + return EXIT_FAILURE; + } + +- if (! (include & DUMP_INCLUDE_TAGS)) ++ if (! (include & (DUMP_INCLUDE_TAGS | DUMP_INCLUDE_PROPERTIES))) + return EXIT_SUCCESS; + + if (! query_str) +@@ -189,6 +254,11 @@ database_dump_file (notmuch_database_t *notmuch, gzFile output, + &buffer, &buffer_size)) + return EXIT_FAILURE; + ++ if ((include & DUMP_INCLUDE_PROPERTIES) && ++ dump_properties_message (notmuch, message, output, ++ &buffer, &buffer_size)) ++ return EXIT_FAILURE; ++ + notmuch_message_destroy (message); + } + +@@ -312,6 +382,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[]) + { 0, 0 } } }, + { NOTMUCH_OPT_KEYWORD_FLAGS, &include, "include", 'I', + (notmuch_keyword_t []){ { "config", DUMP_INCLUDE_CONFIG }, ++ { "properties", DUMP_INCLUDE_PROPERTIES }, + { "tags", DUMP_INCLUDE_TAGS} } }, + { NOTMUCH_OPT_STRING, &output_file_name, "output", 'o', 0 }, + { NOTMUCH_OPT_BOOLEAN, &gzip_output, "gzip", 'z', 0 }, +@@ -326,7 +397,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[]) + notmuch_process_shared_options (argv[0]); + + if (include == 0) +- include = DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_TAGS; ++ include = DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_TAGS | DUMP_INCLUDE_PROPERTIES; + + if (opt_index < argc) { + query_str = query_string_from_args (notmuch, argc - opt_index, argv + opt_index); +diff --git a/notmuch-new.c b/notmuch-new.c +index 799fec2..c55dea7 100644 +--- a/notmuch-new.c ++++ b/notmuch-new.c +@@ -1042,7 +1042,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[]) + } + + if (notmuch_database_dump (notmuch, backup_name, "", +- DUMP_FORMAT_BATCH_TAG, DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_TAGS, TRUE)) { ++ DUMP_FORMAT_BATCH_TAG, DUMP_INCLUDE_DEFAULT, TRUE)) { + fprintf (stderr, "Backup failed. Aborting upgrade."); + return EXIT_FAILURE; + } +diff --git a/test/T610-message-property.sh b/test/T610-message-property.sh +index b5ddb7a..a9b76de 100755 +--- a/test/T610-message-property.sh ++++ b/test/T610-message-property.sh +@@ -89,6 +89,17 @@ testkey2 = NULL + EOF + test_expect_equal_file EXPECTED OUTPUT + ++test_begin_subtest "notmuch_message_remove_all_properties" ++cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} ++EXPECT0(notmuch_message_remove_all_properties (message, NULL)); ++print_properties (message, "", FALSE); ++EOF ++cat <<'EOF' >EXPECTED ++== stdout == ++== stderr == ++EOF ++test_expect_equal_file EXPECTED OUTPUT ++ + test_begin_subtest "notmuch_message_get_properties: empty list" + cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} + { +@@ -188,4 +199,14 @@ cat <<'EOF' >EXPECTED + EOF + test_expect_equal_file EXPECTED OUTPUT + ++test_begin_subtest "dump message properties" ++cat < PROPERTIES ++#= 4EFC743A.3060609@april.org fancy%20key%20with%20%c3%a1cc%c3%a8nts=import%20value%20with%20= testkey1=alice testkey1=bob testkey1=testvalue1 testkey1=testvalue2 testkey3=alice3 testkey3=bob3 testkey3=testvalue3 ++EOF ++cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR} ++EXPECT0(notmuch_message_add_property (message, "fancy key with áccènts", "import value with =")); ++EOF ++notmuch dump | grep '^#=' > OUTPUT ++test_expect_equal_file PROPERTIES OUTPUT ++ + test_done +-- +2.8.1 + -- 2.26.2