From 731db1ed22af3509ca16d7e38ab02b6d82e3e579 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Wed, 4 Jul 2012 18:20:59 +2000 Subject: [PATCH] [PATCH 8/8] emacs: Switch from text to JSON format for search results --- 4b/82e57f494609f328a6a9bc2200d47d5d130c8a | 270 ++++++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100644 4b/82e57f494609f328a6a9bc2200d47d5d130c8a diff --git a/4b/82e57f494609f328a6a9bc2200d47d5d130c8a b/4b/82e57f494609f328a6a9bc2200d47d5d130c8a new file mode 100644 index 000000000..a220faba0 --- /dev/null +++ b/4b/82e57f494609f328a6a9bc2200d47d5d130c8a @@ -0,0 +1,270 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id 79B32431FC2 + for ; Tue, 3 Jul 2012 15:21:16 -0700 (PDT) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Spam-Flag: NO +X-Spam-Score: -0.7 +X-Spam-Level: +X-Spam-Status: No, score=-0.7 tagged_above=-999 required=5 + tests=[RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id hHd5Yp1mWkuC for ; + Tue, 3 Jul 2012 15:21:12 -0700 (PDT) +Received: from dmz-mailsec-scanner-5.mit.edu (DMZ-MAILSEC-SCANNER-5.MIT.EDU + [18.7.68.34]) + by olra.theworths.org (Postfix) with ESMTP id 9E582431FC7 + for ; Tue, 3 Jul 2012 15:21:06 -0700 (PDT) +X-AuditID: 12074422-b7f1f6d00000090b-6f-4ff37052871c +Received: from mailhub-auth-4.mit.edu ( [18.7.62.39]) + by dmz-mailsec-scanner-5.mit.edu (Symantec Messaging Gateway) with SMTP + id BE.2D.02315.25073FF4; Tue, 3 Jul 2012 18:21:06 -0400 (EDT) +Received: from outgoing.mit.edu (OUTGOING-AUTH.MIT.EDU [18.7.22.103]) + by mailhub-auth-4.mit.edu (8.13.8/8.9.2) with ESMTP id q63ML523010809; + Tue, 3 Jul 2012 18:21:05 -0400 +Received: from drake.dyndns.org (26-4-182.dynamic.csail.mit.edu [18.26.4.182]) + (authenticated bits=0) + (User authenticated as amdragon@ATHENA.MIT.EDU) + by outgoing.mit.edu (8.13.6/8.12.4) with ESMTP id q63ML1Aa023226 + (version=TLSv1/SSLv3 cipher=AES256-SHA bits=256 verify=NOT); + Tue, 3 Jul 2012 18:21:02 -0400 (EDT) +Received: from amthrax by drake.dyndns.org with local (Exim 4.77) + (envelope-from ) + id 1SmBSv-0007g0-3E; Tue, 03 Jul 2012 18:21:01 -0400 +From: Austin Clements +To: notmuch@notmuchmail.org +Subject: [PATCH 8/8] emacs: Switch from text to JSON format for search results +Date: Tue, 3 Jul 2012 18:20:59 -0400 +Message-Id: <1341354059-29396-9-git-send-email-amdragon@mit.edu> +X-Mailer: git-send-email 1.7.10 +In-Reply-To: <1341354059-29396-1-git-send-email-amdragon@mit.edu> +References: <1341354059-29396-1-git-send-email-amdragon@mit.edu> +X-Brightmail-Tracker: + H4sIAAAAAAAAA+NgFrrEIsWRmVeSWpSXmKPExsUixG6nrhtU8NnfYPIuVosje2axWyw985/Z + 4vrNmcwWb1bOY3Vg8dg56y67x+GvC1k8nq26xezR9GMxawBLFJdNSmpOZllqkb5dAlfGzeM3 + 2QqabSp+3XnH2MA4Vb+LkZNDQsBEYuKrJjYIW0ziwr31QDYXh5DAPkaJCxteMUI46xklHjWs + Y4FwTjJJbD/bDJWZyyixav06dpB+NgENiW37lzOC2CIC0hI7785mBbGZBXIlzrYeBosLC/hJ + nJjyAmwfi4CqxKatP8BsXgEHiV3/HzJB3CEv8fR+H1icU8BRYsXNsywgthBQzbYPHawTGPkX + MDKsYpRNya3SzU3MzClOTdYtTk7My0st0jXVy80s0UtNKd3ECA46F6UdjD8PKh1iFOBgVOLh + TVT47C/EmlhWXJl7iFGSg0lJlPdKHlCILyk/pTIjsTgjvqg0J7X4EKMEB7OSCO/qJKAcb0pi + ZVVqUT5MSpqDRUmc91rKTX8hgfTEktTs1NSC1CKYrAwHh5IEb2s+UKNgUWp6akVaZk4JQpqJ + gxNkOA/Q8GUgNbzFBYm5xZnpEPlTjIpS4rw7QRICIImM0jy4XlhSeMUoDvSKMO9RkCoeYEKB + 634FNJgJaHDe4k8gg0sSEVJSDYyqi0pKn39YExipbtvH6s99a2X31arC1vk/J51e9NH9ho/4 + F3lfqUUTV3CKeb06OcvyyVQWNf3y39WfL3b8rxMsTzy+TinLuSjc+tkP7uJdC73k1l4ID3wV + yXvf5cWbYr+w1R0XNKWtJ2tc2Ww6de7+ynwJ3pUrTyfkrPYJ2rpuZuGajNh7PR5KLMUZiYZa + zEXFiQArsOpg5QIAAA== +Cc: tomi.ollila@iki.fi +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Tue, 03 Jul 2012 22:21:16 -0000 + +The JSON format eliminates the complex escaping issues that have +plagued the text search format. This uses the incremental JSON parser +so that, like the text parser, it can output search results +incrementally. + +This slows down the parser by about ~4X, but puts us in a good +position to optimize either by improving the JSON parser (evidence +suggests this can reduce the overhead to ~40% over the text format) or +by switching to S-expressions (evidence suggests this will more than +double performance over the text parser). [1] + +This also fixes the incremental search parsing test. + +[1] id:"20110720205007.GB21316@mit.edu" +--- + emacs/notmuch.el | 113 ++++++++++++++++++++++++++++++++---------------------- + test/emacs | 1 - + 2 files changed, 67 insertions(+), 47 deletions(-) + +diff --git a/emacs/notmuch.el b/emacs/notmuch.el +index 084cec6..2a09a98 100644 +--- a/emacs/notmuch.el ++++ b/emacs/notmuch.el +@@ -60,7 +60,7 @@ + (require 'notmuch-message) + + (defcustom notmuch-search-result-format +- `(("date" . "%s ") ++ `(("date" . "%12s ") + ("count" . "%-7s ") + ("authors" . "%-20s ") + ("subject" . "%s ") +@@ -557,17 +557,14 @@ This function advances the next thread when finished." + (notmuch-search-tag '("-inbox")) + (notmuch-search-next-thread)) + +-(defvar notmuch-search-process-filter-data nil +- "Data that has not yet been processed.") +-(make-variable-buffer-local 'notmuch-search-process-filter-data) +- + (defun notmuch-search-process-sentinel (proc msg) + "Add a message to let user know when \"notmuch search\" exits" + (let ((buffer (process-buffer proc)) + (status (process-status proc)) + (exit-status (process-exit-status proc)) + (never-found-target-thread nil)) +- (if (memq status '(exit signal)) ++ (when (memq status '(exit signal)) ++ (kill-buffer (process-get proc 'parse-buf)) + (if (buffer-live-p buffer) + (with-current-buffer buffer + (save-excursion +@@ -577,8 +574,6 @@ This function advances the next thread when finished." + (if (eq status 'signal) + (insert "Incomplete search results (search process was killed).\n")) + (when (eq status 'exit) +- (if notmuch-search-process-filter-data +- (insert (concat "Error: Unexpected output from notmuch search:\n" notmuch-search-process-filter-data))) + (insert "End of search results.") + (unless (= exit-status 0) + (insert (format " (process returned %d)" exit-status))) +@@ -757,45 +752,62 @@ non-authors is found, assume that all of the authors match." + (insert (apply #'format string objects)) + (insert "\n"))) + ++(defvar notmuch-search-process-state nil ++ "Parsing state of the search process filter.") ++ ++(defvar notmuch-search-json-parser nil ++ "Incremental JSON parser for the search process filter.") ++ + (defun notmuch-search-process-filter (proc string) + "Process and filter the output of \"notmuch search\"" +- (let ((buffer (process-buffer proc))) +- (if (buffer-live-p buffer) +- (with-current-buffer buffer +- (let ((line 0) +- (more t) +- (inhibit-read-only t) +- (string (concat notmuch-search-process-filter-data string))) +- (setq notmuch-search-process-filter-data nil) +- (while more +- (while (and (< line (length string)) (= (elt string line) ?\n)) +- (setq line (1+ line))) +- (if (string-match "^thread:\\([0-9A-Fa-f]*\\) \\([^][]*\\) \\[\\([0-9]*\\)/\\([0-9]*\\)\\] \\([^;]*\\); \\(.*\\) (\\([^()]*\\))$" string line) +- (let* ((thread-id (match-string 1 string)) +- (tags-str (match-string 7 string)) +- (result (list :thread thread-id +- :date_relative (match-string 2 string) +- :matched (string-to-number +- (match-string 3 string)) +- :total (string-to-number +- (match-string 4 string)) +- :authors (match-string 5 string) +- :subject (match-string 6 string) +- :tags (if tags-str +- (save-match-data +- (split-string tags-str)))))) +- (if (/= (match-beginning 0) line) +- (notmuch-search-show-error +- (substring string line (match-beginning 0)))) +- (notmuch-search-show-result result) +- (set 'line (match-end 0))) +- (set 'more nil) +- (while (and (< line (length string)) (= (elt string line) ?\n)) +- (setq line (1+ line))) +- (if (< line (length string)) +- (setq notmuch-search-process-filter-data (substring string line))) +- )))) +- (delete-process proc)))) ++ (let ((results-buf (process-buffer proc)) ++ (parse-buf (process-get proc 'parse-buf)) ++ (inhibit-read-only t) ++ done) ++ (if (not (buffer-live-p results-buf)) ++ (delete-process proc) ++ (with-current-buffer parse-buf ++ ;; Insert new data ++ (save-excursion ++ (goto-char (point-max)) ++ (insert string))) ++ (with-current-buffer results-buf ++ (while (not done) ++ (condition-case nil ++ (case notmuch-search-process-state ++ ((begin) ++ ;; Enter the results list ++ (if (eq (notmuch-json-begin-compound ++ notmuch-search-json-parser) 'retry) ++ (setq done t) ++ (setq notmuch-search-process-state 'result))) ++ ((result) ++ ;; Parse a result ++ (let ((result (notmuch-json-read notmuch-search-json-parser))) ++ (case result ++ ((retry) (setq done t)) ++ ((end) (setq notmuch-search-process-state 'end)) ++ (otherwise (notmuch-search-show-result result))))) ++ ((end) ++ ;; Any trailing data is unexpected ++ (with-current-buffer parse-buf ++ (skip-chars-forward " \t\r\n") ++ (if (eobp) ++ (setq done t) ++ (signal 'json-error nil))))) ++ (json-error ++ ;; Do our best to resynchronize and ensure forward ++ ;; progress ++ (notmuch-search-show-error ++ "%s" ++ (with-current-buffer parse-buf ++ (let ((bad (buffer-substring (line-beginning-position) ++ (line-end-position)))) ++ (forward-line) ++ bad)))))) ++ ;; Clear out what we've parsed ++ (with-current-buffer parse-buf ++ (delete-region (point-min) (point))))))) + + (defun notmuch-search-tag-all (&optional tag-changes) + "Add/remove tags from all messages in current search buffer. +@@ -898,10 +910,19 @@ Other optional parameters are used as follows: + (let ((proc (start-process + "notmuch-search" buffer + notmuch-command "search" ++ "--format=json" + (if oldest-first + "--sort=oldest-first" + "--sort=newest-first") +- query))) ++ query)) ++ ;; Use a scratch buffer to accumulate partial output. ++ ;; This buffer will be killed by the sentinel, which ++ ;; should be called no matter how the process dies. ++ (parse-buf (generate-new-buffer " *notmuch search parse*"))) ++ (set (make-local-variable 'notmuch-search-process-state) 'begin) ++ (set (make-local-variable 'notmuch-search-json-parser) ++ (notmuch-json-create-parser parse-buf)) ++ (process-put proc 'parse-buf parse-buf) + (set-process-sentinel proc 'notmuch-search-process-sentinel) + (set-process-filter proc 'notmuch-search-process-filter) + (set-process-query-on-exit-flag proc nil)))) +diff --git a/test/emacs b/test/emacs +index 293b12a..afe35ba 100755 +--- a/test/emacs ++++ b/test/emacs +@@ -36,7 +36,6 @@ test_emacs '(notmuch-search "tag:inbox") + test_expect_equal_file OUTPUT $EXPECTED/notmuch-search-tag-inbox + + test_begin_subtest "Incremental parsing of search results" +-test_subtest_known_broken + test_emacs "(ad-enable-advice 'notmuch-search-process-filter 'around 'pessimal) + (ad-activate 'notmuch-search-process-filter) + (notmuch-search \"tag:inbox\") +-- +1.7.10 + -- 2.26.2