--- /dev/null
+Return-Path: <amdragon@mit.edu>\r
+X-Original-To: notmuch@notmuchmail.org\r
+Delivered-To: notmuch@notmuchmail.org\r
+Received: from localhost (localhost [127.0.0.1])\r
+ by olra.theworths.org (Postfix) with ESMTP id 79B32431FC2\r
+ for <notmuch@notmuchmail.org>; Tue, 3 Jul 2012 15:21:16 -0700 (PDT)\r
+X-Virus-Scanned: Debian amavisd-new at olra.theworths.org\r
+X-Spam-Flag: NO\r
+X-Spam-Score: -0.7\r
+X-Spam-Level: \r
+X-Spam-Status: No, score=-0.7 tagged_above=-999 required=5\r
+ tests=[RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled\r
+Received: from olra.theworths.org ([127.0.0.1])\r
+ by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024)\r
+ with ESMTP id hHd5Yp1mWkuC for <notmuch@notmuchmail.org>;\r
+ Tue, 3 Jul 2012 15:21:12 -0700 (PDT)\r
+Received: from dmz-mailsec-scanner-5.mit.edu (DMZ-MAILSEC-SCANNER-5.MIT.EDU\r
+ [18.7.68.34])\r
+ by olra.theworths.org (Postfix) with ESMTP id 9E582431FC7\r
+ for <notmuch@notmuchmail.org>; Tue, 3 Jul 2012 15:21:06 -0700 (PDT)\r
+X-AuditID: 12074422-b7f1f6d00000090b-6f-4ff37052871c\r
+Received: from mailhub-auth-4.mit.edu ( [18.7.62.39])\r
+ by dmz-mailsec-scanner-5.mit.edu (Symantec Messaging Gateway) with SMTP\r
+ id BE.2D.02315.25073FF4; Tue, 3 Jul 2012 18:21:06 -0400 (EDT)\r
+Received: from outgoing.mit.edu (OUTGOING-AUTH.MIT.EDU [18.7.22.103])\r
+ by mailhub-auth-4.mit.edu (8.13.8/8.9.2) with ESMTP id q63ML523010809; \r
+ Tue, 3 Jul 2012 18:21:05 -0400\r
+Received: from drake.dyndns.org (26-4-182.dynamic.csail.mit.edu [18.26.4.182])\r
+ (authenticated bits=0)\r
+ (User authenticated as amdragon@ATHENA.MIT.EDU)\r
+ by outgoing.mit.edu (8.13.6/8.12.4) with ESMTP id q63ML1Aa023226\r
+ (version=TLSv1/SSLv3 cipher=AES256-SHA bits=256 verify=NOT);\r
+ Tue, 3 Jul 2012 18:21:02 -0400 (EDT)\r
+Received: from amthrax by drake.dyndns.org with local (Exim 4.77)\r
+ (envelope-from <amdragon@mit.edu>)\r
+ id 1SmBSv-0007g0-3E; Tue, 03 Jul 2012 18:21:01 -0400\r
+From: Austin Clements <amdragon@MIT.EDU>\r
+To: notmuch@notmuchmail.org\r
+Subject: [PATCH 8/8] emacs: Switch from text to JSON format for search results\r
+Date: Tue, 3 Jul 2012 18:20:59 -0400\r
+Message-Id: <1341354059-29396-9-git-send-email-amdragon@mit.edu>\r
+X-Mailer: git-send-email 1.7.10\r
+In-Reply-To: <1341354059-29396-1-git-send-email-amdragon@mit.edu>\r
+References: <1341354059-29396-1-git-send-email-amdragon@mit.edu>\r
+X-Brightmail-Tracker:\r
+ H4sIAAAAAAAAA+NgFrrEIsWRmVeSWpSXmKPExsUixG6nrhtU8NnfYPIuVosje2axWyw985/Z\r
+ 4vrNmcwWb1bOY3Vg8dg56y67x+GvC1k8nq26xezR9GMxawBLFJdNSmpOZllqkb5dAlfGzeM3\r
+ 2QqabSp+3XnH2MA4Vb+LkZNDQsBEYuKrJjYIW0ziwr31QDYXh5DAPkaJCxteMUI46xklHjWs\r
+ Y4FwTjJJbD/bDJWZyyixav06dpB+NgENiW37lzOC2CIC0hI7785mBbGZBXIlzrYeBosLC/hJ\r
+ nJjyAmwfi4CqxKatP8BsXgEHiV3/HzJB3CEv8fR+H1icU8BRYsXNsywgthBQzbYPHawTGPkX\r
+ MDKsYpRNya3SzU3MzClOTdYtTk7My0st0jXVy80s0UtNKd3ECA46F6UdjD8PKh1iFOBgVOLh\r
+ TVT47C/EmlhWXJl7iFGSg0lJlPdKHlCILyk/pTIjsTgjvqg0J7X4EKMEB7OSCO/qJKAcb0pi\r
+ ZVVqUT5MSpqDRUmc91rKTX8hgfTEktTs1NSC1CKYrAwHh5IEb2s+UKNgUWp6akVaZk4JQpqJ\r
+ gxNkOA/Q8GUgNbzFBYm5xZnpEPlTjIpS4rw7QRICIImM0jy4XlhSeMUoDvSKMO9RkCoeYEKB\r
+ 634FNJgJaHDe4k8gg0sSEVJSDYyqi0pKn39YExipbtvH6s99a2X31arC1vk/J51e9NH9ho/4\r
+ F3lfqUUTV3CKeb06OcvyyVQWNf3y39WfL3b8rxMsTzy+TinLuSjc+tkP7uJdC73k1l4ID3wV\r
+ yXvf5cWbYr+w1R0XNKWtJ2tc2Ww6de7+ynwJ3pUrTyfkrPYJ2rpuZuGajNh7PR5KLMUZiYZa\r
+ zEXFiQArsOpg5QIAAA==\r
+Cc: tomi.ollila@iki.fi\r
+X-BeenThere: notmuch@notmuchmail.org\r
+X-Mailman-Version: 2.1.13\r
+Precedence: list\r
+List-Id: "Use and development of the notmuch mail system."\r
+ <notmuch.notmuchmail.org>\r
+List-Unsubscribe: <http://notmuchmail.org/mailman/options/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>\r
+List-Archive: <http://notmuchmail.org/pipermail/notmuch>\r
+List-Post: <mailto:notmuch@notmuchmail.org>\r
+List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>\r
+List-Subscribe: <http://notmuchmail.org/mailman/listinfo/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=subscribe>\r
+X-List-Received-Date: Tue, 03 Jul 2012 22:21:16 -0000\r
+\r
+The JSON format eliminates the complex escaping issues that have\r
+plagued the text search format. This uses the incremental JSON parser\r
+so that, like the text parser, it can output search results\r
+incrementally.\r
+\r
+This slows down the parser by about ~4X, but puts us in a good\r
+position to optimize either by improving the JSON parser (evidence\r
+suggests this can reduce the overhead to ~40% over the text format) or\r
+by switching to S-expressions (evidence suggests this will more than\r
+double performance over the text parser). [1]\r
+\r
+This also fixes the incremental search parsing test.\r
+\r
+[1] id:"20110720205007.GB21316@mit.edu"\r
+---\r
+ emacs/notmuch.el | 113 ++++++++++++++++++++++++++++++++----------------------\r
+ test/emacs | 1 -\r
+ 2 files changed, 67 insertions(+), 47 deletions(-)\r
+\r
+diff --git a/emacs/notmuch.el b/emacs/notmuch.el\r
+index 084cec6..2a09a98 100644\r
+--- a/emacs/notmuch.el\r
++++ b/emacs/notmuch.el\r
+@@ -60,7 +60,7 @@\r
+ (require 'notmuch-message)\r
+ \r
+ (defcustom notmuch-search-result-format\r
+- `(("date" . "%s ")\r
++ `(("date" . "%12s ")\r
+ ("count" . "%-7s ")\r
+ ("authors" . "%-20s ")\r
+ ("subject" . "%s ")\r
+@@ -557,17 +557,14 @@ This function advances the next thread when finished."\r
+ (notmuch-search-tag '("-inbox"))\r
+ (notmuch-search-next-thread))\r
+ \r
+-(defvar notmuch-search-process-filter-data nil\r
+- "Data that has not yet been processed.")\r
+-(make-variable-buffer-local 'notmuch-search-process-filter-data)\r
+-\r
+ (defun notmuch-search-process-sentinel (proc msg)\r
+ "Add a message to let user know when \"notmuch search\" exits"\r
+ (let ((buffer (process-buffer proc))\r
+ (status (process-status proc))\r
+ (exit-status (process-exit-status proc))\r
+ (never-found-target-thread nil))\r
+- (if (memq status '(exit signal))\r
++ (when (memq status '(exit signal))\r
++ (kill-buffer (process-get proc 'parse-buf))\r
+ (if (buffer-live-p buffer)\r
+ (with-current-buffer buffer\r
+ (save-excursion\r
+@@ -577,8 +574,6 @@ This function advances the next thread when finished."\r
+ (if (eq status 'signal)\r
+ (insert "Incomplete search results (search process was killed).\n"))\r
+ (when (eq status 'exit)\r
+- (if notmuch-search-process-filter-data\r
+- (insert (concat "Error: Unexpected output from notmuch search:\n" notmuch-search-process-filter-data)))\r
+ (insert "End of search results.")\r
+ (unless (= exit-status 0)\r
+ (insert (format " (process returned %d)" exit-status)))\r
+@@ -757,45 +752,62 @@ non-authors is found, assume that all of the authors match."\r
+ (insert (apply #'format string objects))\r
+ (insert "\n")))\r
+ \r
++(defvar notmuch-search-process-state nil\r
++ "Parsing state of the search process filter.")\r
++\r
++(defvar notmuch-search-json-parser nil\r
++ "Incremental JSON parser for the search process filter.")\r
++\r
+ (defun notmuch-search-process-filter (proc string)\r
+ "Process and filter the output of \"notmuch search\""\r
+- (let ((buffer (process-buffer proc)))\r
+- (if (buffer-live-p buffer)\r
+- (with-current-buffer buffer\r
+- (let ((line 0)\r
+- (more t)\r
+- (inhibit-read-only t)\r
+- (string (concat notmuch-search-process-filter-data string)))\r
+- (setq notmuch-search-process-filter-data nil)\r
+- (while more\r
+- (while (and (< line (length string)) (= (elt string line) ?\n))\r
+- (setq line (1+ line)))\r
+- (if (string-match "^thread:\\([0-9A-Fa-f]*\\) \\([^][]*\\) \\[\\([0-9]*\\)/\\([0-9]*\\)\\] \\([^;]*\\); \\(.*\\) (\\([^()]*\\))$" string line)\r
+- (let* ((thread-id (match-string 1 string))\r
+- (tags-str (match-string 7 string))\r
+- (result (list :thread thread-id\r
+- :date_relative (match-string 2 string)\r
+- :matched (string-to-number\r
+- (match-string 3 string))\r
+- :total (string-to-number\r
+- (match-string 4 string))\r
+- :authors (match-string 5 string)\r
+- :subject (match-string 6 string)\r
+- :tags (if tags-str\r
+- (save-match-data\r
+- (split-string tags-str))))))\r
+- (if (/= (match-beginning 0) line)\r
+- (notmuch-search-show-error\r
+- (substring string line (match-beginning 0))))\r
+- (notmuch-search-show-result result)\r
+- (set 'line (match-end 0)))\r
+- (set 'more nil)\r
+- (while (and (< line (length string)) (= (elt string line) ?\n))\r
+- (setq line (1+ line)))\r
+- (if (< line (length string))\r
+- (setq notmuch-search-process-filter-data (substring string line)))\r
+- ))))\r
+- (delete-process proc))))\r
++ (let ((results-buf (process-buffer proc))\r
++ (parse-buf (process-get proc 'parse-buf))\r
++ (inhibit-read-only t)\r
++ done)\r
++ (if (not (buffer-live-p results-buf))\r
++ (delete-process proc)\r
++ (with-current-buffer parse-buf\r
++ ;; Insert new data\r
++ (save-excursion\r
++ (goto-char (point-max))\r
++ (insert string)))\r
++ (with-current-buffer results-buf\r
++ (while (not done)\r
++ (condition-case nil\r
++ (case notmuch-search-process-state\r
++ ((begin)\r
++ ;; Enter the results list\r
++ (if (eq (notmuch-json-begin-compound\r
++ notmuch-search-json-parser) 'retry)\r
++ (setq done t)\r
++ (setq notmuch-search-process-state 'result)))\r
++ ((result)\r
++ ;; Parse a result\r
++ (let ((result (notmuch-json-read notmuch-search-json-parser)))\r
++ (case result\r
++ ((retry) (setq done t))\r
++ ((end) (setq notmuch-search-process-state 'end))\r
++ (otherwise (notmuch-search-show-result result)))))\r
++ ((end)\r
++ ;; Any trailing data is unexpected\r
++ (with-current-buffer parse-buf\r
++ (skip-chars-forward " \t\r\n")\r
++ (if (eobp)\r
++ (setq done t)\r
++ (signal 'json-error nil)))))\r
++ (json-error\r
++ ;; Do our best to resynchronize and ensure forward\r
++ ;; progress\r
++ (notmuch-search-show-error\r
++ "%s"\r
++ (with-current-buffer parse-buf\r
++ (let ((bad (buffer-substring (line-beginning-position)\r
++ (line-end-position))))\r
++ (forward-line)\r
++ bad))))))\r
++ ;; Clear out what we've parsed\r
++ (with-current-buffer parse-buf\r
++ (delete-region (point-min) (point)))))))\r
+ \r
+ (defun notmuch-search-tag-all (&optional tag-changes)\r
+ "Add/remove tags from all messages in current search buffer.\r
+@@ -898,10 +910,19 @@ Other optional parameters are used as follows:\r
+ (let ((proc (start-process\r
+ "notmuch-search" buffer\r
+ notmuch-command "search"\r
++ "--format=json"\r
+ (if oldest-first\r
+ "--sort=oldest-first"\r
+ "--sort=newest-first")\r
+- query)))\r
++ query))\r
++ ;; Use a scratch buffer to accumulate partial output.\r
++ ;; This buffer will be killed by the sentinel, which\r
++ ;; should be called no matter how the process dies.\r
++ (parse-buf (generate-new-buffer " *notmuch search parse*")))\r
++ (set (make-local-variable 'notmuch-search-process-state) 'begin)\r
++ (set (make-local-variable 'notmuch-search-json-parser)\r
++ (notmuch-json-create-parser parse-buf))\r
++ (process-put proc 'parse-buf parse-buf)\r
+ (set-process-sentinel proc 'notmuch-search-process-sentinel)\r
+ (set-process-filter proc 'notmuch-search-process-filter)\r
+ (set-process-query-on-exit-flag proc nil))))\r
+diff --git a/test/emacs b/test/emacs\r
+index 293b12a..afe35ba 100755\r
+--- a/test/emacs\r
++++ b/test/emacs\r
+@@ -36,7 +36,6 @@ test_emacs '(notmuch-search "tag:inbox")\r
+ test_expect_equal_file OUTPUT $EXPECTED/notmuch-search-tag-inbox\r
+ \r
+ test_begin_subtest "Incremental parsing of search results"\r
+-test_subtest_known_broken\r
+ test_emacs "(ad-enable-advice 'notmuch-search-process-filter 'around 'pessimal)\r
+ (ad-activate 'notmuch-search-process-filter)\r
+ (notmuch-search \"tag:inbox\")\r
+-- \r
+1.7.10\r
+\r