Return-Path: X-Original-To: notmuch@notmuchmail.org Delivered-To: notmuch@notmuchmail.org Received: from localhost (localhost [127.0.0.1]) by olra.theworths.org (Postfix) with ESMTP id BC35F431FAF for ; Sun, 15 Jan 2012 00:05:18 -0800 (PST) X-Virus-Scanned: Debian amavisd-new at olra.theworths.org X-Spam-Flag: NO X-Spam-Score: -0.699 X-Spam-Level: X-Spam-Status: No, score=-0.699 tagged_above=-999 required=5 tests=[HTML_MESSAGE=0.001, RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled Received: from olra.theworths.org ([127.0.0.1]) by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id m9UHXtLjTjfT for ; Sun, 15 Jan 2012 00:05:17 -0800 (PST) Received: from mail-pz0-f53.google.com (mail-pz0-f53.google.com [209.85.210.53]) (using TLSv1 with cipher RC4-SHA (128/128 bits)) (No client certificate requested) by olra.theworths.org (Postfix) with ESMTPS id 49D2A431FAE for ; Sun, 15 Jan 2012 00:05:17 -0800 (PST) Received: by dalz17 with SMTP id z17so2743369dal.26 for ; Sun, 15 Jan 2012 00:05:15 -0800 (PST) MIME-Version: 1.0 Received: by 10.68.213.38 with SMTP id np6mr16285599pbc.37.1326614715150; Sun, 15 Jan 2012 00:05:15 -0800 (PST) Received: by 10.68.58.234 with HTTP; Sun, 15 Jan 2012 00:05:15 -0800 (PST) Received: by 10.68.58.234 with HTTP; Sun, 15 Jan 2012 00:05:15 -0800 (PST) In-Reply-To: <1326591624-15493-10-git-send-email-david@tethera.net> References: <874nwxbkhr.fsf@zancas.localnet> <1326591624-15493-1-git-send-email-david@tethera.net> <1326591624-15493-10-git-send-email-david@tethera.net> Date: Sun, 15 Jan 2012 10:05:15 +0200 Message-ID: Subject: Re: [PATCH v3 09/10] random-dump.c: new test-binary to generate dump files From: Jani Nikula To: David Bremner Content-Type: multipart/alternative; boundary=e89a8ff1c322d9b16904b68c8ec8 Cc: notmuch@notmuchmail.org, David Bremner X-BeenThere: notmuch@notmuchmail.org X-Mailman-Version: 2.1.13 Precedence: list List-Id: "Use and development of the notmuch mail system." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 15 Jan 2012 08:05:18 -0000 --e89a8ff1c322d9b16904b68c8ec8 Content-Type: text/plain; charset=UTF-8 On Jan 15, 2012 3:41 AM, "David Bremner" wrote: > > From: David Bremner > > This binary creates a "torture test" dump file for the new dump > format. > --- > test/Makefile.local | 4 ++ > test/basic | 2 +- > test/random-dump.c | 144 +++++++++++++++++++++++++++++++++++++++++++++++++++ > 3 files changed, 149 insertions(+), 1 deletions(-) > create mode 100644 test/random-dump.c > > diff --git a/test/Makefile.local b/test/Makefile.local > index ba697f4..b59f837 100644 > --- a/test/Makefile.local > +++ b/test/Makefile.local > @@ -16,6 +16,9 @@ $(dir)/arg-test: $(dir)/arg-test.o command-line-arguments.o util/libutil.a > $(dir)/hex-xcode: $(dir)/hex-xcode.o command-line-arguments.o util/libutil.a > $(call quiet,CC) -I. $^ -o $@ -ltalloc > > +$(dir)/random-dump: $(dir)/random-dump.o command-line-arguments.o util/libutil.a > + $(call quiet,CC) -I. $^ -o $@ -ltalloc -lm > + > $(dir)/smtp-dummy: $(smtp_dummy_modules) > $(call quiet,CC) $^ -o $@ > > @@ -25,6 +28,7 @@ $(dir)/symbol-test: $(dir)/symbol-test.o > .PHONY: test check > > test-binaries: $(dir)/arg-test $(dir)/hex-xcode \ > + $(dir)/random-dump \ > $(dir)/smtp-dummy $(dir)/symbol-test > > test: all test-binaries > diff --git a/test/basic b/test/basic > index af57026..e3a6cef 100755 > --- a/test/basic > +++ b/test/basic > @@ -54,7 +54,7 @@ test_begin_subtest 'Ensure that all available tests will be run by notmuch-test' > eval $(sed -n -e '/^TESTS="$/,/^"$/p' $TEST_DIRECTORY/notmuch-test) > tests_in_suite=$(for i in $TESTS; do echo $i; done | sort) > available=$(find "$TEST_DIRECTORY" -maxdepth 1 -type f -executable -printf '%f\n' | \ > - sed -r -e "/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test|hex-xcode)$/d" | \ > + sed -r -e "/^(aggregate-results.sh|notmuch-test|smtp-dummy|test-verbose|symbol-test|arg-test|hex-xcode|random-dump)$/d" | \ > sort) > test_expect_equal "$tests_in_suite" "$available" > > diff --git a/test/random-dump.c b/test/random-dump.c > new file mode 100644 > index 0000000..1949425 > --- /dev/null > +++ b/test/random-dump.c > @@ -0,0 +1,144 @@ > +/* > + Generate a random dump file in 'notmuch' format. > + Generated message-id's and tags are intentionally nasty. > + > + We restrict ourselves to 7 bit message-ids, because generating > + random valid UTF-8 seems like work. And invalid UTF-8 can't be > + round-tripped via Xapian. > + > + */ > + > +#include > +#include > +#include > +#include > +#include "math.h" > +#include "hex-escape.h" > +#include "command-line-arguments.h" > + > +static void > +hex_out (void *ctx, char *buf) > +{ > + static char *encoded_buf = NULL; > + static size_t encoded_buf_size = 0; > + > + if (hex_encode (ctx, buf, &encoded_buf, &encoded_buf_size) != HEX_SUCCESS) { > + fprintf (stderr, "Hex encoding failed"); > + exit (1); > + } > + > + fputs (encoded_buf, stdout); > +} > + > +static void > +random_chars (char *buf, int from, int stop, int max_char, > + const char *blacklist) > +{ > + int i; > + > + for (i = from; i < stop; i++) { > + do { > + buf[i] = ' ' + (random () % (max_char - ' ')); > + } while (blacklist && strchr (blacklist, buf[i])); > + } > +} > + > +static void > +random_tag (void *ctx, size_t len) > +{ > + static char *buf = NULL; > + static size_t buf_len = 0; > + > + int use = (random () % (len - 1)) + 1; > + > + if (len > buf_len) { > + buf = talloc_realloc (ctx, buf, char, len); > + buf_len = len; > + } > + > + random_chars (buf, 0, use, 255, NULL); > + > + buf[use] = '\0'; > + > + hex_out (ctx, buf); > +} > + > +static void > +random_message_id (void *ctx, size_t len) > +{ > + static char *buf = NULL; > + static size_t buf_len = 0; > + > + int lhs_len = (random () % (len / 2 - 1)) + 1; > + > + int rhs_len = (random () % len / 2) + 1; > + > + const char *blacklist = "\n\r@<>[]()"; > + > + if (len > buf_len) { > + buf = talloc_realloc (ctx, buf, char, len); > + buf_len = len; > + } > + > + random_chars (buf, 0, lhs_len, 127, blacklist); > + > + buf[lhs_len] = '@'; > + > + random_chars (buf, lhs_len + 1, lhs_len + rhs_len + 1, 127, blacklist); > + > + hex_out (ctx, buf); > +} > + > +int > +main (int argc, char **argv) > +{ > + > + void *ctx = talloc_new (NULL); > + int num_lines = 500; > + int max_tags = 10; > + int message_id_len = 100; > + int tag_len = 50; > + int seed = 734569; > + > + int pad_tag = 0, pad_mid = 0; > + > + notmuch_opt_desc_t options[] = { > + { NOTMUCH_OPT_INT, &num_lines, "num-lines", 'n', 0 }, > + { NOTMUCH_OPT_INT, &max_tags, "max-tags", 'm', 0 }, > + { NOTMUCH_OPT_INT, &message_id_len, "message-id-len", 'M', 0 }, > + { NOTMUCH_OPT_INT, &tag_len, "tag-len", 't', 0 }, > + { NOTMUCH_OPT_INT, &seed, "tag-len", 't', 0 }, s/tag-len/seed/ > + { 0, 0, 0, 0, 0 } > + }; > + > + int opt_index = parse_arguments (argc, argv, options, 1); > + > + if (opt_index < 0) > + exit (1); > + > + pad_mid = ((int) log10 (num_lines) + 1); > + pad_tag = ((int) log10 (max_tags)) + 1; > + > + srandom (seed); > + > + int line; > + for (line = 0; line < num_lines; line++) { > + > + printf ("%0*d-", pad_mid, line); > + > + random_message_id (ctx, message_id_len); > + > + int num_tags = random () % (max_tags + 1); > + > + int j; > + for (j = 0; j < num_tags; j++) { > + printf (" %0*d-", pad_tag, j); > + random_tag (ctx, tag_len); > + } > + putchar ('\n'); > + } > + > + talloc_free (ctx); > + > + return 0; > +} > -- > 1.7.7.3 > > _______________________________________________ > notmuch mailing list > notmuch@notmuchmail.org > http://notmuchmail.org/mailman/listinfo/notmuch --e89a8ff1c322d9b16904b68c8ec8 Content-Type: text/html; charset=UTF-8 Content-Transfer-Encoding: quoted-printable


On Jan 15, 2012 3:41 AM, "David Bremner" <david@tethera.net> wrote:
>
> From: David Bremner <bremner@= debian.org>
>
> This binary creates a "torture test" dump file for the new d= ump
> format.
> ---
> =C2=A0test/Makefile.local | =C2=A0 =C2=A04 ++
> =C2=A0test/basic =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0| =C2=A0 =C2=A02 +-=
> =C2=A0test/random-dump.c =C2=A0| =C2=A0144 +++++++++++++++++++++++++++= ++++++++++++++++++++++++
> =C2=A03 files changed, 149 insertions(+), 1 deletions(-)
> =C2=A0create mode 100644 test/random-dump.c
>
> diff --git a/test/Makefile.local b/test/Makefile.local
> index ba697f4..b59f837 100644
> --- a/test/Makefile.local
> +++ b/test/Makefile.local
> @@ -16,6 +16,9 @@ $(dir)/arg-test: $(dir)/arg-test.o command-line-argu= ments.o util/libutil.a
> =C2=A0$(dir)/hex-xcode: $(dir)/hex-xcode.o command-line-arguments.o ut= il/libutil.a
> =C2=A0 =C2=A0 =C2=A0 =C2=A0$(call quiet,CC) -I. $^ -o $@ -ltalloc
>
> +$(dir)/random-dump: =C2=A0$(dir)/random-dump.o command-line-arguments= .o util/libutil.a
> + =C2=A0 =C2=A0 =C2=A0 $(call quiet,CC) -I. $^ -o $@ -ltalloc -lm
> +
> =C2=A0$(dir)/smtp-dummy: $(smtp_dummy_modules)
> =C2=A0 =C2=A0 =C2=A0 =C2=A0$(call quiet,CC) $^ -o $@
>
> @@ -25,6 +28,7 @@ $(dir)/symbol-test: $(dir)/symbol-test.o
> =C2=A0.PHONY: test check
>
> =C2=A0test-binaries: $(dir)/arg-test $(dir)/hex-xcode \
> + =C2=A0 =C2=A0 =C2=A0 $(dir)/random-dump \
> =C2=A0 =C2=A0 =C2=A0 =C2=A0 $(dir)/smtp-dummy $(dir)/symbol-test
>
> =C2=A0test: =C2=A0all test-binaries
> diff --git a/test/basic b/test/basic
> index af57026..e3a6cef 100755
> --- a/test/basic
> +++ b/test/basic
> @@ -54,7 +54,7 @@ test_begin_subtest 'Ensure that all available te= sts will be run by notmuch-test'
> =C2=A0eval $(sed -n -e '/^TESTS=3D"$/,/^"$/p' $TEST_= DIRECTORY/notmuch-test)
> =C2=A0tests_in_suite=3D$(for i in $TESTS; do echo $i; done | sort)
> =C2=A0available=3D$(find "$TEST_DIRECTORY" -maxdepth 1 -type= f -executable -printf '%f\n' | \
> - =C2=A0 =C2=A0sed -r -e "/^(aggregate-results.sh|notmuch-test|sm= tp-dummy|test-verbose|symbol-test|arg-test|hex-xcode)$/d" | \
> + =C2=A0 =C2=A0sed -r -e "/^(aggregate-results.sh|notmuch-test|sm= tp-dummy|test-verbose|symbol-test|arg-test|hex-xcode|random-dump)$/d" = | \
> =C2=A0 =C2=A0 sort)
> =C2=A0test_expect_equal "$tests_in_suite" "$available&q= uot;
>
> diff --git a/test/random-dump.c b/test/random-dump.c
> new file mode 100644
> index 0000000..1949425
> --- /dev/null
> +++ b/test/random-dump.c
> @@ -0,0 +1,144 @@
> +/*
> + =C2=A0 Generate a random dump file in 'notmuch' format.
> + =C2=A0 Generated message-id's and tags are intentionally nasty.<= br> > +
> + =C2=A0 We restrict ourselves to 7 bit message-ids, because generatin= g
> + =C2=A0 random valid UTF-8 seems like work. And invalid UTF-8 can'= ;t be
> + =C2=A0 round-tripped via Xapian.
> +
> + */
> +
> +#include <stdlib.h>
> +#include <assert.h>
> +#include <talloc.h>
> +#include <string.h>
> +#include "math.h"
> +#include "hex-escape.h"
> +#include "command-line-arguments.h"
> +
> +static void
> +hex_out (void *ctx, char *buf)
> +{
> + =C2=A0 =C2=A0static char *encoded_buf =3D NULL;
> + =C2=A0 =C2=A0static size_t encoded_buf_size =3D 0;
> +
> + =C2=A0 =C2=A0if (hex_encode (ctx, buf, &encoded_buf, &encode= d_buf_size) !=3D HEX_SUCCESS) {
> + =C2=A0 =C2=A0 =C2=A0 fprintf (stderr, "Hex encoding failed"= ;);
> + =C2=A0 =C2=A0 =C2=A0 exit (1);
> + =C2=A0 =C2=A0}
> +
> + =C2=A0 =C2=A0fputs (encoded_buf, stdout);
> +}
> +
> +static void
> +random_chars (char *buf, int from, int stop, int max_char,
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 const char *blacklist)
> +{
> + =C2=A0 =C2=A0int i;
> +
> + =C2=A0 =C2=A0for (i =3D from; i < stop; i++) {
> + =C2=A0 =C2=A0 =C2=A0 do {
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 buf[i] =3D ' ' + (random = () % (max_char - ' '));
> + =C2=A0 =C2=A0 =C2=A0 } while (blacklist && strchr (blacklist= , buf[i]));
> + =C2=A0 =C2=A0}
> +}
> +
> +static void
> +random_tag (void *ctx, size_t len)
> +{
> + =C2=A0 =C2=A0static char *buf =3D NULL;
> + =C2=A0 =C2=A0static size_t buf_len =3D 0;
> +
> + =C2=A0 =C2=A0int use =3D (random () % (len - 1)) + 1;
> +
> + =C2=A0 =C2=A0if (len > buf_len) {
> + =C2=A0 =C2=A0 =C2=A0 buf =3D talloc_realloc (ctx, buf, char, len); > + =C2=A0 =C2=A0 =C2=A0 buf_len =3D len;
> + =C2=A0 =C2=A0}
> +
> + =C2=A0 =C2=A0random_chars (buf, 0, use, 255, NULL);
> +
> + =C2=A0 =C2=A0buf[use] =3D '\0';
> +
> + =C2=A0 =C2=A0hex_out (ctx, buf);
> +}
> +
> +static void
> +random_message_id (void *ctx, size_t len)
> +{
> + =C2=A0 =C2=A0static char *buf =3D NULL;
> + =C2=A0 =C2=A0static size_t buf_len =3D 0;
> +
> + =C2=A0 =C2=A0int lhs_len =3D (random () % (len / 2 - 1)) + 1;
> +
> + =C2=A0 =C2=A0int rhs_len =3D (random () % len / 2) + 1;
> +
> + =C2=A0 =C2=A0const char *blacklist =3D "\n\r@<>[]()"= ;
> +
> + =C2=A0 =C2=A0if (len > buf_len) {
> + =C2=A0 =C2=A0 =C2=A0 buf =3D talloc_realloc (ctx, buf, char, len); > + =C2=A0 =C2=A0 =C2=A0 buf_len =3D len;
> + =C2=A0 =C2=A0}
> +
> + =C2=A0 =C2=A0random_chars (buf, 0, lhs_len, 127, blacklist);
> +
> + =C2=A0 =C2=A0buf[lhs_len] =3D '@';
> +
> + =C2=A0 =C2=A0random_chars (buf, lhs_len + 1, lhs_len + rhs_len + 1, = 127, blacklist);
> +
> + =C2=A0 =C2=A0hex_out (ctx, buf);
> +}
> +
> +int
> +main (int argc, char **argv)
> +{
> +
> + =C2=A0 =C2=A0void *ctx =3D talloc_new (NULL);
> + =C2=A0 =C2=A0int num_lines =3D 500;
> + =C2=A0 =C2=A0int max_tags =3D 10;
> + =C2=A0 =C2=A0int message_id_len =3D 100;
> + =C2=A0 =C2=A0int tag_len =3D 50;
> + =C2=A0 =C2=A0int seed =3D 734569;
> +
> + =C2=A0 =C2=A0int pad_tag =3D 0, pad_mid =3D 0;
> +
> + =C2=A0 =C2=A0notmuch_opt_desc_t options[] =3D {
> + =C2=A0 =C2=A0 =C2=A0 { NOTMUCH_OPT_INT, &num_lines, "num-li= nes", 'n', 0 },
> + =C2=A0 =C2=A0 =C2=A0 { NOTMUCH_OPT_INT, &max_tags, "max-tag= s", 'm', 0 },
> + =C2=A0 =C2=A0 =C2=A0 { NOTMUCH_OPT_INT, &message_id_len, "m= essage-id-len", 'M', 0 },
> + =C2=A0 =C2=A0 =C2=A0 { NOTMUCH_OPT_INT, &tag_len, "tag-len&= quot;, 't', 0 },
> + =C2=A0 =C2=A0 =C2=A0 { NOTMUCH_OPT_INT, &seed, "tag-len&quo= t;, 't', 0 },

s/tag-len/seed/

> + =C2=A0 =C2=A0 =C2=A0 { 0, 0, 0, 0, 0 }
> + =C2=A0 =C2=A0};
> +
> + =C2=A0 =C2=A0int opt_index =3D parse_arguments (argc, argv, options,= 1);
> +
> + =C2=A0 =C2=A0if (opt_index < 0)
> + =C2=A0 =C2=A0 =C2=A0 exit (1);
> +
> + =C2=A0 =C2=A0pad_mid =3D ((int) log10 (num_lines) + 1);
> + =C2=A0 =C2=A0pad_tag =3D ((int) log10 (max_tags)) + 1;
> +
> + =C2=A0 =C2=A0srandom (seed);
> +
> + =C2=A0 =C2=A0int line;
> + =C2=A0 =C2=A0for (line =3D 0; line < num_lines; line++) {
> +
> + =C2=A0 =C2=A0 =C2=A0 printf ("%0*d-", pad_mid, line);
> +
> + =C2=A0 =C2=A0 =C2=A0 random_message_id (ctx, message_id_len);
> +
> + =C2=A0 =C2=A0 =C2=A0 int num_tags =3D random () % (max_tags + 1); > +
> + =C2=A0 =C2=A0 =C2=A0 int j;
> + =C2=A0 =C2=A0 =C2=A0 for (j =3D 0; j < num_tags; j++) {
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 printf (" %0*d-", pad_t= ag, j);
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 random_tag (ctx, tag_len);
> + =C2=A0 =C2=A0 =C2=A0 }
> + =C2=A0 =C2=A0 =C2=A0 putchar ('\n');
> + =C2=A0 =C2=A0}
> +
> + =C2=A0 =C2=A0talloc_free (ctx);
> +
> + =C2=A0 =C2=A0return 0;
> +}
> --
> 1.7.7.3
>
> _______________________________________________
> notmuch mailing list
> notmuch@notmuchmail.org=
> http://not= muchmail.org/mailman/listinfo/notmuch

--e89a8ff1c322d9b16904b68c8ec8--