From: Mark Walters <markwalters1009@gmail.com>
To: Austin Clements <amdragon@MIT.EDU>
Subject: Re: [RFC PATCH v3 00/11] notmuch-pick: an emacs threaded message view
	with split-pane
In-Reply-To: <20120214152114.GQ27039@mit.edu>
References: <1329072579-27340-1-git-send-email-markwalters1009@gmail.com>
	<1329096015-8078-1-git-send-email-markwalters1009@gmail.com>
	<87haytbnun.fsf@qmul.ac.uk> <20120214152114.GQ27039@mit.edu>
User-Agent: Notmuch/0.11.1+206~g3b67774 (http://notmuchmail.org) Emacs/23.2.1
	(i486-pc-linux-gnu)
Date: Wed, 15 Feb 2012 09:27:21 +0000
Message-ID: <87fwecbg5i.fsf@qmul.ac.uk>
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="=-=-="
Cc: notmuch@notmuchmail.org
Precedence: list

--=-=-=

On Tue, 14 Feb 2012 10:21:14 -0500, Austin Clements <amdragon@MIT.EDU> wrote:
> Quoth Mark Walters on Feb 14 at 12:28 pm:
> > Finally, if notmuch-pick were able to do work asynchronously (as
> > notmuch-search does now) then I think all the speed concerns would go
> > away. However, I am not sure how to do incremental json parsing.
> 
> For JSON search, at least, I think we've concluded that it should put
> newlines at strategic places in the JSON output (and never anywhere
> else) so that it's easy for a consumer to know when it has a complete
> JSON object and hand it to the JSON parser.  E.g.,
> 
> [{"thread":"X", timestamp: 42, ...}
> ,{"thread":"Y", timestamp: 24, ...}
> ]
> 
> This "framed JSON" works really well for the flat output of search.
> It's obviously trickier to apply to show's hierarchical output.  But,
> perhaps this will inspire you.
> 
> (One possibility: we could rework show's output to be non-hierarchical
> and use the above framing; that is, it could be a sequence of message
> objects that each indicate which earlier message object they're a
> reply to, probably restricted to DFS order.)

Here is a (very early version) of an async notmuch-pick based on the
above idea. This is on top of the previous patch series. There are two
attached patches: the first is a very small patch to notmuch-show.c to
print newlines between threads (since this is sufficient framing for
notmuch-pick; obviously it does not help notmuch-show at all).

The second patch implements the async handling in notmuch-pick.el. This
is probably very inefficient, and doubtless has significant bugs, but it
seems to mostly work and does display the first page of results almost
immediately even on very large queries so I think this is the right way
to go.

I am mostly posting it so people can play with the functionality but
cleanups, bugfixes, bug reports etc are gratefully received!

Best wishes

Mark


--=-=-=
Content-Type: text/x-diff
Content-Disposition: inline;
 filename=0001-cli-notmuch-show-with-framing-newlines-between-threa.patch

>From 5622a3ed7b486edc360ffa764756ce76f69ac032 Mon Sep 17 00:00:00 2001
From: Mark Walters <markwalters1009@gmail.com>
Date: Tue, 14 Feb 2012 19:33:56 +0000
Subject: [PATCH 1/2] cli: notmuch-show with framing newlines between threads in JSON

Add newlines between complete threads to make asynchronous parsing
of the JSON easier
---
 notmuch-show.c |    4 ++++
 1 files changed, 4 insertions(+), 0 deletions(-)

diff --git a/notmuch-show.c b/notmuch-show.c
index 3f8618b..a7cc684 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -912,6 +912,7 @@ do_show_messages (void *ctx,
     int first_set = 1;
 
     fputs (format->message_set_start, stdout);
+    fputs ("\n", stdout);
     messages = notmuch_query_search_messages (query);
 
     for (;
@@ -939,6 +940,7 @@ do_show_messages (void *ctx,
 
 	fputs (format->message_set_end, stdout);
 	fputs (format->message_set_end, stdout);
+	fputs ("\n", stdout);
     }
 
     fputs (format->message_set_end, stdout);
@@ -1029,6 +1031,7 @@ do_show (void *ctx,
     int first_toplevel = 1;
 
     fputs (format->message_set_start, stdout);
+    fputs ("\n", stdout);
 
     for (threads = notmuch_query_search_threads (query);
 	 notmuch_threads_valid (threads);
@@ -1047,6 +1050,7 @@ do_show (void *ctx,
 	first_toplevel = 0;
 
 	show_messages (ctx, format, messages, 0, params);
+	fputs ("\n", stdout);
 
 	notmuch_thread_destroy (thread);
 
-- 
1.7.2.3


--=-=-=
Content-Type: text/x-diff
Content-Disposition: inline;
 filename=0002-emacs-a-semi-working-async-pick.patch

>From c3bcd7d11b67a15a1760505c036e760e7028b125 Mon Sep 17 00:00:00 2001
From: Mark Walters <markwalters1009@gmail.com>
Date: Mon, 13 Feb 2012 23:31:45 +0000
Subject: [PATCH 2/2] emacs: a  semi-working async pick

---
 emacs/notmuch-pick.el |  111 ++++++++++++++++++++++++++++++++++++++-----------
 1 files changed, 87 insertions(+), 24 deletions(-)

diff --git a/emacs/notmuch-pick.el b/emacs/notmuch-pick.el
index 46eb720..f944d79 100644
--- a/emacs/notmuch-pick.el
+++ b/emacs/notmuch-pick.el
@@ -287,7 +287,7 @@ This command toggles the sort order for the current search."
 	(query-context notmuch-pick-query-context)
 	(buffer-name notmuch-pick-buffer-name))
     (erase-buffer)
-    (notmuch-pick-worker thread-id  query-context buffer-name)))
+    (notmuch-pick-worker thread-id  query-context (get-buffer buffer-name))))
 
 (defun notmuch-pick-toggle-view ()
   "Toggle showing threads or as isolated messages."
@@ -531,34 +531,97 @@ Complete list of currently available key bindings:
   (setq buffer-read-only t
 	truncate-lines t))
 
-(defun notmuch-pick-worker (thread-id &optional query-context buffer-name)
+(defvar notmuch-pick-process-filter-data nil
+  "Data that has not yet been processed.")
+(make-variable-buffer-local 'notmuch-pick-process-filter-data)
+
+(defun notmuch-pick-process-sentinel (proc msg)
+  "Add a message to let user know when \"notmuch pick\" exits"
+  (let ((buffer (process-buffer proc))
+	(status (process-status proc))
+	(exit-status (process-exit-status proc))
+	(never-found-target-thread nil))
+    (if (memq status '(exit signal))
+	(if (buffer-live-p buffer)
+	    (with-current-buffer buffer
+	      (save-excursion
+		(let ((inhibit-read-only t)
+		      (atbob (bobp)))
+		  (goto-char (point-max))
+		  (if (eq status 'signal)
+		      (insert "Incomplete search results (pick process was killed).\n"))
+		  (when (eq status 'exit)
+		    (if (not (string= notmuch-pick-process-filter-data "\n]"))
+			(insert (concat "Error: Unexpected output from notmuch pick:\n"
+					notmuch-pick-process-filter-data)))
+		    (insert "End of search results.")
+		    (unless (= exit-status 0)
+		      (insert (format " (process returned %d)" exit-status)))
+		    (insert "\n")))))))))
+
+(defun notmuch-pick-process-filter (proc string)
+  "Process and filter the output of \"notmuch show\" (for pick)"
+  (let ((buffer (process-buffer proc)))
+    (if (buffer-live-p buffer)
+	(with-current-buffer buffer
+	  (save-excursion
+	    (let ((line 0)
+		  (more t)
+		  (inhibit-read-only t)
+		  (string (concat notmuch-pick-process-filter-data string)))
+	      (while (string-match "\n.*\n" string)
+		(let ((frame (match-string 0 string))
+		      (frame-end (match-end 0)))
+		  (when (or (= (elt frame 1) ?\[) (= (elt frame 1) ?,))
+		    (let (json)
+		      (with-temp-buffer
+			(let ((json-object-type 'plist)
+			      (json-array-type 'list)
+			      (json-false 'nil))
+			  (insert frame)
+			  (goto-char (point-min))
+			  (search-forward "[")
+			  (backward-char)
+			  (setq json (json-read))))
+		      (goto-char (point-max))
+		      (notmuch-pick-insert-forest (list json))))
+		  (setq string (substring string (- frame-end 1)))))
+	      (setq notmuch-pick-process-filter-data string)))))))
+
+(defun notmuch-pick-worker (thread-id &optional query-context buffer)
   (interactive)
   (notmuch-pick-mode)
   (setq notmuch-pick-thread-id thread-id)
   (setq notmuch-pick-query-context query-context)
-  (setq notmuch-pick-buffer-name buffer-name)
+  (setq notmuch-pick-buffer-name (buffer-name buffer))
 
   (erase-buffer)
   (goto-char (point-min))
-  (save-excursion
-    (let* ((basic-args (list thread-id))
-	   (args (if query-context
-		     (append (list "\'") basic-args (list "and (" query-context ")\'"))
-		   (append (list "\'") basic-args (list "\'"))))
-	   (message-arg (if notmuch-pick-view-just-messages
-			    "--thread=none"
-			  "--thread=entire"))
-	   (sort-arg (if notmuch-pick-oldest-first
-			  "--sort=oldest-first"
-			"--sort=newest-first")))
-
-      (notmuch-pick-insert-forest (notmuch-query-get-threads args "--headers-only" message-arg sort-arg))
-      ;; If the query context reduced the results to nothing, run
-      ;; the basic query.
-      (when (and (eq (buffer-size) 0)
-		 query-context)
-	(notmuch-pick-insert-forest
-	 (notmuch-query-get-threads basic-args message-arg sort-arg))))))
+  (let* (args
+	 (basic-args thread-id)
+	 (search-args (concat
+		       basic-args (if query-context (concat " and (" query-context ")"))))
+	 (message-arg (if notmuch-pick-view-just-messages
+			  "--thread=none"
+			"--thread=entire"))
+	 (sort-arg (if notmuch-pick-oldest-first
+		       "--sort=oldest-first"
+		     "--sort=newest-first"))
+	 (proc (start-process
+		"notmuch-pick" buffer
+		notmuch-command "show" "--headers-only" "--format=json"
+		message-arg sort-arg search-args)))
+    (set-process-sentinel proc 'notmuch-pick-process-sentinel)
+    (set-process-filter proc 'notmuch-pick-process-filter)
+    (set-process-query-on-exit-flag proc nil)))
+;;      (notmuch-pick-insert-forest (notmuch-query-get-threads args "--headers-only" message-arg sort-arg))
+;;     (message "time3 (end forest): %s" (current-time))
+;; If the query context reduced the results to nothing, run
+;; the basic query.
+;;      (when (and (eq (buffer-size) 0)
+;;		 query-context)
+;;	(notmuch-pick-insert-forest
+;;	 (notmuch-query-get-threads basic-args message-arg sort-arg))))))
 
 (defun notmuch-pick (&optional query query-context buffer-name)
   "Run notmuch pick with the given `query' and display the results"
@@ -567,14 +630,14 @@ Complete list of currently available key bindings:
       (setq query (notmuch-read-query "Notmuch pick: ")))
   (let ((buffer (get-buffer-create (generate-new-buffer-name
 				    (or buffer-name
-					(concat "*notmuch-" query "*")))))
+					(concat "*notmuch-pick-" query "*")))))
 	(inhibit-read-only t))
 
     (switch-to-buffer buffer)
     ;; Don't track undo information for this buffer
     (set 'buffer-undo-list t)
 
-    (notmuch-pick-worker query query-context buffer-name)
+    (notmuch-pick-worker query query-context buffer)
 
     (setq truncate-lines t)))
 
-- 
1.7.2.3


--=-=-=--