From: Mark Walters <markwalters1009@gmail.com>
To: notmuch@notmuchmail.org
Subject: Xapian Quote tags
User-Agent: Notmuch/0.14+236~g1d0044f (http://notmuchmail.org) Emacs/23.4.1
	(x86_64-pc-linux-gnu)
Date: Fri, 04 Jan 2013 13:55:52 +0000
Message-ID: <878v89hx1z.fsf@qmul.ac.uk>
MIME-Version: 1.0
Content-Type: text/plain; charset=us-ascii
Precedence: list


Hello

I would like to suggest that we Xapian quote the tags for notmuch
dump/restore. The general view on irc is that we probably want to do
this in the long term and I think it would be nice if we can avoid
changing the dump format a second time.

One problem is that our current line based parsing cannot cope with
Xapian quoted newlines (which stay newlines). So allow a line to start
with % which means hex-decode the whole line before passing to the main
parser. Thus the query tags etc still need to be Xapian encoded.

I attach a patch to show roughly what I mean. This is not complete: the
dump routine does not hex encode lines with newlines yet, tests man
pages etc are not updated and it is significantly unpolished. Also there
should be some consolidation between parse_boolean_term and the
xapian_decode routine.

Despite the above caveats it broadly seems to work.

Best wishes

Mark




>From d518e2be27ff7243ddc156699c2bfc38dec78b43 Mon Sep 17 00:00:00 2001
From: Mark Walters <markwalters1009@gmail.com>
Date: Fri, 4 Jan 2013 13:37:42 +0000
Subject: [PATCH] notmuch dump: xapian quote tags

---
 notmuch-dump.c |    6 +++---
 tag-util.c     |   53 ++++++++++++++++++++++++++++++++++++++++++++++-------
 2 files changed, 49 insertions(+), 10 deletions(-)

diff --git a/notmuch-dump.c b/notmuch-dump.c
index bf01a39..e94d870 100644
--- a/notmuch-dump.c
+++ b/notmuch-dump.c
@@ -120,9 +120,9 @@ notmuch_dump_command (unused (void *ctx), int argc, char *argv[])
 	    if (output_format == DUMP_FORMAT_SUP) {
 		fputs (tag_str, output);
 	    } else {
-		if (hex_encode (notmuch, tag_str,
-				&buffer, &buffer_size) != HEX_SUCCESS) {
-		    fprintf (stderr, "Error: failed to hex-encode tag %s\n",
+		if (make_boolean_term (notmuch, NULL, tag_str,
+				       &buffer, &buffer_size)) {
+		    fprintf (stderr, "Error: failed to xapian-encode tag %s\n",
 			     tag_str);
 		    return 1;
 		}
diff --git a/tag-util.c b/tag-util.c
index ca12b3b..7384afa 100644
--- a/tag-util.c
+++ b/tag-util.c
@@ -31,6 +31,38 @@ line_error (tag_parse_status_t status,
     return status;
 }
 
+static int
+xapian_decode_tag_inplace (char *str, char* tok, size_t *tok_len)
+{
+    char *pos = str;
+    char *out = str;
+
+    if (*pos == '"') {
+	int closed = 0;
+	/* Skip the opening quote, find the closing quote, and
+	 * un-double doubled internal quotes. */
+	for (++pos; *pos; ) {
+	    if (*pos == '"') {
+		++pos;
+		if (*pos != '"') {
+		    /* Found the closing quote. */
+		    closed = 1;
+		    break;
+		}
+	    }
+	    *out++ = *pos++;
+	}
+	if (! closed || *pos != ' ')
+	    return HEX_SYNTAX_ERROR;
+	*tok_len = pos - tok;
+    } else {
+	out = tok + (*tok_len)++;
+    }
+    /* Terminate token */
+    *out = '\0';
+    return HEX_SUCCESS;
+}
+
 tag_parse_status_t
 parse_tag_line (void *ctx, char *line,
 		tag_op_flag_t flags,
@@ -60,6 +92,15 @@ parse_tag_line (void *ctx, char *line,
 	goto DONE;
     }
 
+    if (*tok == '%') {
+	tok++;
+	if (hex_decode_inplace (tok) != HEX_SUCCESS) {
+	    ret = line_error (TAG_PARSE_INVALID, line_for_error,
+			      "hex decoding of line failed", "");
+	    goto DONE;
+	}
+    }
+
     tag_op_list_reset (tag_ops);
 
     /* Parse tags. */
@@ -89,23 +130,21 @@ parse_tag_line (void *ctx, char *line,
 	    goto DONE;
 	}
 
-	/* Terminate, and start next token after terminator. */
-	*(tok + tok_len++) = '\0';
-
 	remove = (*tok == '-');
 	tag = tok + 1;
 
-	/* Maybe refuse empty tags. */
+	/* Maybe refuse empty tags. Note a quoted empty tag is allowed. */
 	if (! (flags & TAG_FLAG_BE_GENEROUS) && *tag == '\0') {
 	    ret = line_error (TAG_PARSE_INVALID, line_for_error,
 			      "empty tag");
 	    goto DONE;
 	}
 
-	/* Decode tag. */
-	if (hex_decode_inplace (tag) != HEX_SUCCESS) {
+	/* Find real (quoted) end, terminate, and start next token
+	 * after terminator. */
+	if (xapian_decode_tag_inplace (tag, tok, &tok_len) != HEX_SUCCESS) {
 	    ret = line_error (TAG_PARSE_INVALID, line_for_error,
-			      "hex decoding of tag %s failed", tag);
+			      "xapian decoding of tag %s failed", tag);
 	    goto DONE;
 	}
 
-- 
1.7.9.1