From: Tomi Ollila <tomi.ollila@iki.fi>
To: David Bremner <david@tethera.net>, notmuch@notmuchmail.org
Subject: Re: [Patch v3 07/11] CLI: add optional config data to dump output.
In-Reply-To: <1462065879-29860-8-git-send-email-david@tethera.net>
References: <1462065879-29860-1-git-send-email-david@tethera.net>
 <1462065879-29860-8-git-send-email-david@tethera.net>
User-Agent: Notmuch/0.22+9~gb26b5ad (http://notmuchmail.org) Emacs/24.3.1
 (x86_64-unknown-linux-gnu)
Date: Sat, 07 May 2016 20:37:07 +0300
Message-ID: <m2wpn5epmk.fsf@guru.guru-group.fi>
MIME-Version: 1.0
Content-Type: text/plain
Precedence: list

On Sun, May 01 2016, David Bremner <david@tethera.net> wrote:

> Note that it changes the default dump output format, but doesn't break
> existing notmuch-restore. It might break user scripts though.
> ---
>  doc/man1/notmuch-dump.rst | 17 ++++++++++
>  notmuch-client.h          |  8 +++++
>  notmuch-dump.c            | 80 +++++++++++++++++++++++++++++++++++++++++++++--
>  notmuch-new.c             |  2 +-
>  test/T150-tagging.sh      |  8 ++---
>  test/T240-dump-restore.sh | 14 ++++-----
>  test/T590-libconfig.sh    | 17 ++++++++++
>  test/test-lib.sh          |  6 ++++
>  8 files changed, 137 insertions(+), 15 deletions(-)
>
> diff --git a/doc/man1/notmuch-dump.rst b/doc/man1/notmuch-dump.rst
> index a37c337..eda9e07 100644
> --- a/doc/man1/notmuch-dump.rst
> +++ b/doc/man1/notmuch-dump.rst
> @@ -71,6 +71,23 @@ Supported options for **dump** include
>              characters. Note also that tags with spaces will not be
>              correctly restored with this format.
>  
> +    ``--include=(config|tags)``
> +
> +    Control what kind of metadata is included in the output.
> +
> +      **config**
> +
> +	Output configuration data stored in the database. Each line
> +	starts with "#@ ", followed by a space seperated key-value
> +	pair.  Both key and value are hex encoded if needed.
> +
> +      **tags**
> +
> +	Output per-message metadata, namely tags. See *format* above
> +	for description of the output.
> +
> +      The default is to include both tags and configuration information
> +
>      ``--output=``\ <filename>
>          Write output to given file instead of stdout.
>  
> diff --git a/notmuch-client.h b/notmuch-client.h
> index b3d0b66..ae6f124 100644
> --- a/notmuch-client.h
> +++ b/notmuch-client.h
> @@ -446,11 +446,19 @@ typedef enum dump_formats {
>      DUMP_FORMAT_SUP
>  } dump_format_t;
>  
> +typedef enum dump_includes {
> +    DUMP_INCLUDE_TAGS=1,
> +    DUMP_INCLUDE_CONFIG=2,
> +} dump_include_t;
> +
> +#define NOTMUCH_DUMP_VERSION 2
> +
>  int
>  notmuch_database_dump (notmuch_database_t *notmuch,
>  		       const char *output_file_name,
>  		       const char *query_str,
>  		       dump_format_t output_format,
> +		       dump_include_t include,
>  		       notmuch_bool_t gzip_output);
>  
>  /* If status is non-zero (i.e. error) print appropriate
> diff --git a/notmuch-dump.c b/notmuch-dump.c
> index 829781f..a6cf810 100644
> --- a/notmuch-dump.c
> +++ b/notmuch-dump.c
> @@ -23,16 +23,82 @@
>  #include "string-util.h"
>  #include <zlib.h>
>  
> +static int
> +database_dump_config (notmuch_database_t *notmuch, gzFile output)
> +{
> +    notmuch_config_list_t *list;
> +    int ret = EXIT_FAILURE;
> +    char *buffer = NULL;
> +    size_t buffer_size = 0;
> +
> +    if (print_status_database ("notmuch dump", notmuch,
> +			       notmuch_database_get_config_list (notmuch, NULL, &list)))
> +	goto DONE;
> +
> +    for (; notmuch_config_list_valid (list); notmuch_config_list_move_to_next (list)) {
> +	if (hex_encode (notmuch, notmuch_config_list_key (list),
> +			&buffer, &buffer_size) != HEX_SUCCESS) {
> +	    fprintf (stderr, "Error: failed to hex-encode config key %s\n",
> +		     notmuch_config_list_key (list));
> +	    goto DONE;
> +	}
> +	gzprintf (output, "#@ %s", buffer);
> +
> +	if (hex_encode (notmuch, notmuch_config_list_value (list),
> +			&buffer, &buffer_size) != HEX_SUCCESS) {
> +	    fprintf (stderr, "Error: failed to hex-encode config value %s\n",
> +		     notmuch_config_list_value (list) );
> +	    goto DONE;
> +	}
> +
> +	gzprintf (output, " %s\n", buffer);
> +    }
> +
> +    ret = EXIT_SUCCESS;
> +
> + DONE:
> +    if (list)
> +	notmuch_config_list_destroy (list);
> +
> +    if (buffer)
> +	talloc_free (buffer);
> +
> +    return ret;
> +}
> +
> +static void
> +print_dump_header (gzFile output, int output_format, int include)
> +{
> +    gzprintf (output, "#notmuch-dump %s:%d %s%s%s\n",
> +	      (output_format == DUMP_FORMAT_SUP) ? "sup" : "batch-tag",
> +	      NOTMUCH_DUMP_VERSION,
> +	      (include & DUMP_INCLUDE_CONFIG) ? "config" : "",
> +	      (include & DUMP_INCLUDE_TAGS) && (include & DUMP_INCLUDE_CONFIG) ? "," : "",
> +	      (include & DUMP_INCLUDE_TAGS) ? "tags" : "");
> +
> +
> +}
>  
>  static int
>  database_dump_file (notmuch_database_t *notmuch, gzFile output,
> -		    const char *query_str, int output_format)
> +		    const char *query_str, int output_format, int include)
>  {
>      notmuch_query_t *query;
>      notmuch_messages_t *messages;
>      notmuch_message_t *message;
>      notmuch_tags_t *tags;
>  
> +    print_dump_header (output, output_format, include);
> +
> +    if (include & DUMP_INCLUDE_CONFIG) {
> +	if (print_status_database ("notmuch dump", notmuch,
> +				   database_dump_config(notmuch,output)))
> +	    return EXIT_FAILURE;
> +    }
> +
> +    if (! (include & DUMP_INCLUDE_TAGS))
> +	return EXIT_SUCCESS;
> +
>      if (! query_str)
>  	query_str = "";
>  
> @@ -130,6 +196,7 @@ notmuch_database_dump (notmuch_database_t *notmuch,
>  		       const char *output_file_name,
>  		       const char *query_str,
>  		       dump_format_t output_format,
> +		       dump_include_t include,
>  		       notmuch_bool_t gzip_output)
>  {
>      gzFile output = NULL;
> @@ -164,7 +231,7 @@ notmuch_database_dump (notmuch_database_t *notmuch,
>  	goto DONE;
>      }
>  
> -    ret = database_dump_file (notmuch, output, query_str, output_format);
> +    ret = database_dump_file (notmuch, output, query_str, output_format, include);
>      if (ret) goto DONE;
>  
>      ret = gzflush (output, Z_FINISH);
> @@ -226,6 +293,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
>      int opt_index;
>  
>      int output_format = DUMP_FORMAT_BATCH_TAG;
> +    int include = 0;
>      notmuch_bool_t gzip_output = 0;
>  
>      notmuch_opt_desc_t options[] = {
> @@ -233,6 +301,9 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
>  	  (notmuch_keyword_t []){ { "sup", DUMP_FORMAT_SUP },
>  				  { "batch-tag", DUMP_FORMAT_BATCH_TAG },
>  				  { 0, 0 } } },
> +	{ NOTMUCH_OPT_KEYWORD_FLAGS, &include, "include", 'I',
> +	  (notmuch_keyword_t []){ { "config", DUMP_INCLUDE_CONFIG },
> +				  { "tags", DUMP_INCLUDE_TAGS} } },
>  	{ NOTMUCH_OPT_STRING, &output_file_name, "output", 'o', 0  },
>  	{ NOTMUCH_OPT_BOOLEAN, &gzip_output, "gzip", 'z', 0 },
>  	{ NOTMUCH_OPT_INHERIT, (void *) &notmuch_shared_options, NULL, 0, 0 },
> @@ -245,6 +316,9 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
>  
>      notmuch_process_shared_options (argv[0]);
>  
> +    if (include == 0)
> +	include = DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_TAGS;
> +
>      if (opt_index < argc) {
>  	query_str = query_string_from_args (notmuch, argc - opt_index, argv + opt_index);
>  	if (query_str == NULL) {
> @@ -254,7 +328,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
>      }
>  
>      ret = notmuch_database_dump (notmuch, output_file_name, query_str,
> -				 output_format, gzip_output);
> +				 output_format, include, gzip_output);
>  
>      notmuch_database_destroy (notmuch);
>  
> diff --git a/notmuch-new.c b/notmuch-new.c
> index 04cb5ca..196a36e 100644
> --- a/notmuch-new.c
> +++ b/notmuch-new.c
> @@ -1045,7 +1045,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[])
>  	    }
>  
>  	    if (notmuch_database_dump (notmuch, backup_name, "",
> -				       DUMP_FORMAT_BATCH_TAG, TRUE)) {
> +				       DUMP_FORMAT_BATCH_TAG, DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_TAGS, TRUE)) {
>  		fprintf (stderr, "Backup failed. Aborting upgrade.");
>  		return EXIT_FAILURE;
>  	    }
> diff --git a/test/T150-tagging.sh b/test/T150-tagging.sh
> index a451ffa..61d1311 100755
> --- a/test/T150-tagging.sh
> +++ b/test/T150-tagging.sh
> @@ -188,7 +188,7 @@ cat <<EOF > EXPECTED
>  +%22%27%22%27%22%22%27%27 +inbox +tag4 +tag5 +unread -- id:msg-002@notmuch-test-suite
>  EOF
>  
> -notmuch dump --format=batch-tag | sort > OUTPUT
> +NOTMUCH_DUMP_TAGS > OUTPUT
>  notmuch restore --format=batch-tag < BACKUP
>  test_expect_equal_file EXPECTED OUTPUT
>  
> @@ -209,7 +209,7 @@ cat <<EOF > EXPECTED
>  +%21@%23%20%24%25%5e%26%2a%29-_=+%5b%7b%5c%20%7c%3b%3a%27%20%22,.%3c%60%7e +inbox +tag5 +unread -- id:msg-001@notmuch-test-suite
>  EOF
>  
> -notmuch dump --format=batch-tag | sort > OUTPUT
> +NOTMUCH_DUMP_TAGS > OUTPUT
>  notmuch restore --format=batch-tag < BACKUP
>  test_expect_equal_file EXPECTED OUTPUT
>  
> @@ -235,7 +235,7 @@ cat <<EOF > EXPECTED
>  +%2a@%7d%cf%b5%f4%85%80%adO3%da%a7 +=%e0%ac%95%c8%b3+%ef%aa%95%c8%a64w%c7%9d%c9%a2%cf%b3%d6%82%24B%c4%a9%c5%a1UX%ee%99%b0%27E7%ca%a4%d0%8b%5d +A%e1%a0%bc%de%8b%d5%b2V%d9%9b%f3%b5%a2%a3M%d8%a1u@%f0%a0%ac%948%7e%f0%ab%86%af%27 +L%df%85%ef%a1%a5m@%d3%96%c2%ab%d4%9f%ca%b8%f3%b3%a2%bf%c7%b1_u%d7%b4%c7%b1 +P%c4%98%2f +R +inbox +tag5 +unread +%7e%d1%8b%25%ec%a0%ae%d1%a0M%3b%e3%b6%b7%e9%a4%87%3c%db%9a%cc%a8%e1%96%9d +%c4%bf7%c7%ab9H%c4%99k%ea%91%bd%c3%8ck%e2%b3%8dk%c5%952V%e4%99%b2%d9%b3%e4%8b%bda%5b%24%c7%9b +%da%88=f%cc%b9I%ce%af%7b%c9%97%e3%b9%8bH%cb%92X%d2%8c6 +%dc%9crh%d2%86B%e5%97%a2%22t%ed%99%82d -- id:msg-001@notmuch-test-suite
>  EOF
>  
> -notmuch dump --format=batch-tag | sort > OUTPUT
> +NOTMUCH_DUMP_TAGS > OUTPUT
>  notmuch restore --format=batch-tag < BACKUP
>  test_expect_equal_file EXPECTED OUTPUT
>  
> @@ -260,7 +260,7 @@ cat <<EOF > EXPECTED
>  +foo%3a%3abar%25 +found%3a%3ait +inbox +tag5 +unread +winner -- id:msg-001@notmuch-test-suite
>  EOF
>  
> -notmuch dump --format=batch-tag | sort > OUTPUT
> +NOTMUCH_DUMP_TAGS > OUTPUT
>  notmuch restore --format=batch-tag < BACKUP
>  test_expect_equal_file EXPECTED OUTPUT
>  
> diff --git a/test/T240-dump-restore.sh b/test/T240-dump-restore.sh
> index e6976ff..758d355 100755
> --- a/test/T240-dump-restore.sh
> +++ b/test/T240-dump-restore.sh
> @@ -97,7 +97,7 @@ test_expect_equal_file dump.expected dump.actual
>  # Note, we assume all messages from cworth have a message-id
>  # containing cworth.org
>  
> -grep 'cworth[.]org' dump.expected > dump-cworth.expected
> +(head -1 dump.expected ; grep 'cworth[.]org' dump.expected) > dump-cworth.expected

Instead of executing the commands in subshell, the 'group command' format
could be used (and run in current shell). i.e. 

{ head -1 dump.expected ; grep 'cworth[.]org' dump.expected; } > dump-cworth.expected

>  test_begin_subtest "dump -- from:cworth"
>  notmuch dump -- from:cworth > dump-dash-cworth.actual
> @@ -118,16 +118,16 @@ notmuch search --output=messages from:cworth | sed s/^id:// |\
>  test_expect_equal_file OUTPUT EXPECTED
>  
>  test_begin_subtest "format=batch-tag, dump sanity check."
> -notmuch dump --format=sup from:cworth | cut -f1 -d' ' | \
> +NOTMUCH_DUMP_TAGS --format=sup from:cworth | cut -f1 -d' ' | \
>      sort > EXPECTED.$test_count
> -notmuch dump --format=batch-tag from:cworth | sed 's/^.*-- id://' | \
> +NOTMUCH_DUMP_TAGS --format=batch-tag from:cworth | sed 's/^.*-- id://' | \
>      sort > OUTPUT.$test_count
>  test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
>  
>  test_begin_subtest "format=batch-tag, missing newline"
>  printf "+a_tag_without_newline -- id:20091117232137.GA7669@griffis1.net" > IN
>  notmuch restore --accumulate < IN
> -notmuch dump id:20091117232137.GA7669@griffis1.net > OUT
> +NOTMUCH_DUMP_TAGS id:20091117232137.GA7669@griffis1.net > OUT
>  cat <<EOF > EXPECTED
>  +a_tag_without_newline +inbox +unread -- id:20091117232137.GA7669@griffis1.net
>  EOF
> @@ -155,7 +155,7 @@ cat <<EOF >EXPECTED.$test_count
>  + -- id:20091117232137.GA7669@griffis1.net
>  EOF
>  notmuch restore --format=batch-tag < EXPECTED.$test_count
> -notmuch dump --format=batch-tag id:20091117232137.GA7669@griffis1.net > OUTPUT.$test_count
> +NOTMUCH_DUMP_TAGS --format=batch-tag id:20091117232137.GA7669@griffis1.net > OUTPUT.$test_count
>  test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
>  
>  tag1='comic_swear=$&^%$^%\\//-+$^%$'
> @@ -217,9 +217,9 @@ notmuch dump --format=batch-tag > OUTPUT.$test_count
>  test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
>  
>  test_begin_subtest 'format=batch-tag, checking encoded output'
> -notmuch dump --format=batch-tag -- from:cworth |\
> +NOTMUCH_DUMP_TAGS --format=batch-tag -- from:cworth |\
>  	 awk "{ print \"+$enc1 +$enc2 +$enc3 -- \" \$5 }" > EXPECTED.$test_count

The above I don't understand, $enc1, $enc2 and $enc3 should be expanded by
the shell into the awk command -- I don't see these set anywhere, so to
me it looks like these are empty strings...

> -notmuch dump --format=batch-tag -- from:cworth  > OUTPUT.$test_count
> +NOTMUCH_DUMP_TAGS --format=batch-tag -- from:cworth  > OUTPUT.$test_count
>  test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
>  
>  test_begin_subtest 'restoring sane tags'
> diff --git a/test/T590-libconfig.sh b/test/T590-libconfig.sh
> index 8ca6883..5ea5300 100755
> --- a/test/T590-libconfig.sh
> +++ b/test/T590-libconfig.sh
> @@ -115,4 +115,21 @@ testkey2 testvalue2
>  EOF
>  test_expect_equal_file EXPECTED OUTPUT
>  
> +test_begin_subtest "dump config"
> +cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
> +{
> +    RUN(notmuch_database_set_config (db, "key with spaces", "value, with, spaces!"));
> +}
> +EOF
> +notmuch dump --include=config >OUTPUT
> +cat <<'EOF' >EXPECTED
> +#notmuch-dump batch-tag:2 config
> +#@ aaabefore beforeval
> +#@ key%20with%20spaces value,%20with,%20spaces%21
> +#@ testkey1 testvalue1
> +#@ testkey2 testvalue2
> +#@ zzzafter afterval
> +EOF
> +test_expect_equal_file EXPECTED OUTPUT
> +
>  test_done
> diff --git a/test/test-lib.sh b/test/test-lib.sh
> index 09f8731..68a8774 100644
> --- a/test/test-lib.sh
> +++ b/test/test-lib.sh
> @@ -673,6 +673,12 @@ NOTMUCH_NEW ()
>      notmuch new "${@}" | grep -v -E -e '^Processed [0-9]*( total)? file|Found [0-9]* total file'
>  }
>  
> +NOTMUCH_DUMP_TAGS ()
> +{
> +    # this relies on the default format being batch-tag, otherwise some tests will break
> +    notmuch dump --include=tags "${@}" | sed '/^#/d' | sort
> +}
> +
>  notmuch_search_sanitize ()
>  {
>      perl -pe 's/("?thread"?: ?)("?)................("?)/\1\2XXX\3/'
> -- 
> 2.8.0.rc3
>
> _______________________________________________
> notmuch mailing list
> notmuch@notmuchmail.org
> https://notmuchmail.org/mailman/listinfo/notmuch