From: Mark Walters <markwalters1009@gmail.com>
To: David Bremner <david@tethera.net>, notmuch@notmuchmail.org
Subject: Re: [PATCH v3 09/10] random-dump.c: new test-binary to generate dump
	files
In-Reply-To: <1326591624-15493-10-git-send-email-david@tethera.net>
References: <874nwxbkhr.fsf@zancas.localnet>
	<1326591624-15493-1-git-send-email-david@tethera.net>
	<1326591624-15493-10-git-send-email-david@tethera.net>
User-Agent: Notmuch/0.11+154~ged6d37e (http://notmuchmail.org) Emacs/23.2.1
	(i486-pc-linux-gnu)
Date: Sun, 05 Feb 2012 01:04:13 +0000
Message-ID: <87pqduozua.fsf@qmul.ac.uk>
MIME-Version: 1.0
Content-Type: text/plain; charset=us-ascii
Cc: David Bremner <bremner@debian.org>
Precedence: list

On Sat, 14 Jan 2012 21:40:23 -0400, David Bremner <david@tethera.net> wrote:
> From: David Bremner <bremner@debian.org>
> 
> This binary creates a "torture test" dump file for the new dump
> format.
> ---
>  test/Makefile.local |    4 ++
>  test/basic          |    2 +-
>  test/random-dump.c  |  144 +++++++++++++++++++++++++++++++++++++++++++++++++++
>  3 files changed, 149 insertions(+), 1 deletions(-)
>  create mode 100644 test/random-dump.c
> 
> diff --git a/test/Makefile.local b/test/Makefile.local
> index ba697f4..b59f837 100644
> --- a/test/Makefile.local
> +++ b/test/Makefile.local
> @@ -16,6 +16,9 @@ $(dir)/arg-test: $(dir)/arg-test.o command-line-arguments.o util/libutil.a
>  $(dir)/hex-xcode: $(dir)/hex-xcode.o command-line-arguments.o util/libutil.a
>  	$(call quiet,CC) -I. $^ -o $@ -ltalloc
>  
> +$(dir)/random-dump:  $(dir)/random-dump.o command-line-arguments.o util/libutil.a
> +	$(call quiet,CC) -I. $^ -o $@ -ltalloc -lm
> +
>  $(dir)/smtp-dummy: $(smtp_dummy_modules)
>  	$(call quiet,CC) $^ -o $@
>  
> @@ -25,6 +28,7 @@ $(dir)/symbol-test: $(dir)/symbol-test.o
>  .PHONY: test check
>  
>  test-binaries: $(dir)/arg-test $(dir)/hex-xcode \
> +	$(dir)/random-dump \
>  	 $(dir)/smtp-dummy $(dir)/symbol-test
>  
>  test:	all test-binaries
> diff --git a/test/basic b/test/basic
> index af57026..e3a6cef 100755
> --- a/test/basic
> +++ b/test/basic
> @@ -54,7 +54,7 @@ test_begin_subtest 'Ensure that all available tests will be run by notmuch-test'
>  eval $(sed -n -e '/^TESTS="$/,/^"$/p' $TEST_DIRECTORY/notmuch-test)
>  tests_in_suite=$(for i in $TESTS; do echo $i; done | sort)
>  available=$(find "$TEST_DIRECTORY" -maxdepth 1 -type f -executable -printf '%f\n' | \
> -    sed -r -e "/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test|hex-xcode)$/d" | \
> +    sed -r -e "/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test|hex-xcode|random-dump)$/d" | \
>      sort)
>  test_expect_equal "$tests_in_suite" "$available"
>  
> diff --git a/test/random-dump.c b/test/random-dump.c
> new file mode 100644
> index 0000000..1949425
> --- /dev/null
> +++ b/test/random-dump.c
> @@ -0,0 +1,144 @@
> +/*
> +   Generate a random dump file in 'notmuch' format.
> +   Generated message-id's and tags are intentionally nasty.
> +
> +   We restrict ourselves to 7 bit message-ids, because generating
> +   random valid UTF-8 seems like work. And invalid UTF-8 can't be
> +   round-tripped via Xapian.
> +
> + */
> +
> +#include <stdlib.h>
> +#include <assert.h>
> +#include <talloc.h>
> +#include <string.h>
> +#include "math.h"
> +#include "hex-escape.h"
> +#include "command-line-arguments.h"
> +
> +static void
> +hex_out (void *ctx, char *buf)
> +{
> +    static char *encoded_buf = NULL;
> +    static size_t encoded_buf_size = 0;
> +
> +    if (hex_encode (ctx, buf, &encoded_buf, &encoded_buf_size) != HEX_SUCCESS) {
> +	fprintf (stderr, "Hex encoding failed");
> +	exit (1);
> +    }
> +
> +    fputs (encoded_buf, stdout);
> +}
> +
> +static void
> +random_chars (char *buf, int from, int stop, int max_char,
> +	      const char *blacklist)
> +{
> +    int i;
> +
> +    for (i = from; i < stop; i++) {
> +	do {
> +	    buf[i] = ' ' + (random () % (max_char - ' '));
> +	} while (blacklist && strchr (blacklist, buf[i]));
> +    }
> +}
> +
> +static void
> +random_tag (void *ctx, size_t len)
> +{
> +    static char *buf = NULL;
> +    static size_t buf_len = 0;
> +
> +    int use = (random () % (len - 1)) + 1;
> +
> +    if (len > buf_len) {
> +	buf = talloc_realloc (ctx, buf, char, len);
> +	buf_len = len;
> +    }
> +
> +    random_chars (buf, 0, use, 255, NULL);
> +
> +    buf[use] = '\0';
> +
> +    hex_out (ctx, buf);
> +}
> +
> +static void
> +random_message_id (void *ctx, size_t len)
> +{
> +    static char *buf = NULL;
> +    static size_t buf_len = 0;
> +
> +    int lhs_len = (random () % (len / 2 - 1)) + 1;
> +
> +    int rhs_len = (random () % len / 2) + 1;
> +
> +    const char *blacklist = "\n\r@<>[]()";
> +
> +    if (len > buf_len) {
> +	buf = talloc_realloc (ctx, buf, char, len);
> +	buf_len = len;
> +    }
> +
> +    random_chars (buf, 0, lhs_len, 127, blacklist);
> +
> +    buf[lhs_len] = '@';
> +
> +    random_chars (buf, lhs_len + 1, lhs_len + rhs_len + 1, 127, blacklist);
> +
> +    hex_out (ctx, buf);
> +}
> +
> +int
> +main (int argc, char **argv)
> +{
> +
> +    void *ctx = talloc_new (NULL);
> +    int num_lines = 500;
> +    int max_tags = 10;
> +    int message_id_len = 100;
> +    int tag_len = 50;
> +    int seed = 734569;
> +
> +    int pad_tag = 0, pad_mid = 0;
> +
> +    notmuch_opt_desc_t options[] = {
> +	{ NOTMUCH_OPT_INT, &num_lines, "num-lines", 'n', 0 },
> +	{ NOTMUCH_OPT_INT, &max_tags, "max-tags", 'm', 0 },
> +	{ NOTMUCH_OPT_INT, &message_id_len, "message-id-len", 'M', 0 },
> +	{ NOTMUCH_OPT_INT, &tag_len, "tag-len", 't', 0 },
> +	{ NOTMUCH_OPT_INT, &seed, "tag-len", 't', 0 },
> +	{ 0, 0, 0, 0, 0 }
> +    };
> +
> +    int opt_index = parse_arguments (argc, argv, options, 1);
> +
> +    if (opt_index < 0)
> +	exit (1);
> +
> +    pad_mid = ((int) log10 (num_lines) + 1);
> +    pad_tag = ((int) log10 (max_tags)) + 1;
> +
> +    srandom (seed);
> +
> +    int line;
> +    for (line = 0; line < num_lines; line++) {
> +
> +	printf ("%0*d-", pad_mid, line);
> +
> +	random_message_id (ctx, message_id_len);
> +
> +	int num_tags = random () % (max_tags + 1);
> +
> +	int j;
> +	for (j = 0; j < num_tags; j++) {
> +	    printf (" %0*d-", pad_tag, j);
> +	    random_tag (ctx, tag_len);
> +	}
> +	putchar ('\n');
> +    }
> +
> +    talloc_free (ctx);
> +
> +    return 0;
> +}

Hi

Just a thought on this and the next test. Could you add messages with
the random ids and tags from the above code to the Xapian database
directly by calling whatever notmuch-new calls. Then test by doing dump,
restore and dump and check the two dumps are equal? It might avoid your
gmime concern from the next patch and you could have arbitrary
(non-null) strings including all sorts of malformed utf-8.

I guess Xapian might do bizarre things on the malformed utf-8 but, if it
does, it might mean the correct place to fix it is in notmuch-new.

Best wishes

Mark