From: Austin Clements Date: Mon, 9 Jul 2012 21:42:41 +0000 (+2000) Subject: [PATCH v3 8/9] emacs: Switch from text to JSON format for search results X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=141f76e82d4dcfcf23c64428a6927994f2c3f42e;p=notmuch-archives.git [PATCH v3 8/9] emacs: Switch from text to JSON format for search results --- diff --git a/5f/8b4b3790066c49ad120d649022f8175c0d1fb1 b/5f/8b4b3790066c49ad120d649022f8175c0d1fb1 new file mode 100644 index 000000000..e85b42c29 --- /dev/null +++ b/5f/8b4b3790066c49ad120d649022f8175c0d1fb1 @@ -0,0 +1,277 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id F1467429E26 + for ; Mon, 9 Jul 2012 14:42:57 -0700 (PDT) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Spam-Flag: NO +X-Spam-Score: -0.7 +X-Spam-Level: +X-Spam-Status: No, score=-0.7 tagged_above=-999 required=5 + tests=[RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id oSiK4-+G36UT for ; + Mon, 9 Jul 2012 14:42:55 -0700 (PDT) +Received: from dmz-mailsec-scanner-3.mit.edu (DMZ-MAILSEC-SCANNER-3.MIT.EDU + [18.9.25.14]) + by olra.theworths.org (Postfix) with ESMTP id 4211C429E37 + for ; Mon, 9 Jul 2012 14:42:51 -0700 (PDT) +X-AuditID: 1209190e-b7fb56d0000008b2-59-4ffb5058c977 +Received: from mailhub-auth-2.mit.edu ( [18.7.62.36]) + by dmz-mailsec-scanner-3.mit.edu (Symantec Messaging Gateway) with SMTP + id 27.51.02226.8505BFF4; Mon, 9 Jul 2012 17:42:48 -0400 (EDT) +Received: from outgoing.mit.edu (OUTGOING-AUTH.MIT.EDU [18.7.22.103]) + by mailhub-auth-2.mit.edu (8.13.8/8.9.2) with ESMTP id q69Lglf7003849; + Mon, 9 Jul 2012 17:42:47 -0400 +Received: from drake.dyndns.org (26-4-176.dynamic.csail.mit.edu [18.26.4.176]) + (authenticated bits=0) + (User authenticated as amdragon@ATHENA.MIT.EDU) + by outgoing.mit.edu (8.13.6/8.12.4) with ESMTP id q69Lgi6q022690 + (version=TLSv1/SSLv3 cipher=AES256-SHA bits=256 verify=NOT); + Mon, 9 Jul 2012 17:42:47 -0400 (EDT) +Received: from amthrax by drake.dyndns.org with local (Exim 4.77) + (envelope-from ) + id 1SoLjA-0004g9-2a; Mon, 09 Jul 2012 17:42:44 -0400 +From: Austin Clements +To: notmuch@notmuchmail.org +Subject: [PATCH v3 8/9] emacs: Switch from text to JSON format for search + results +Date: Mon, 9 Jul 2012 17:42:41 -0400 +Message-Id: <1341870162-17782-9-git-send-email-amdragon@mit.edu> +X-Mailer: git-send-email 1.7.10 +In-Reply-To: <1341870162-17782-1-git-send-email-amdragon@mit.edu> +References: <1341354059-29396-1-git-send-email-amdragon@mit.edu> + <1341870162-17782-1-git-send-email-amdragon@mit.edu> +X-Brightmail-Tracker: + H4sIAAAAAAAAA+NgFjrGIsWRmVeSWpSXmKPExsUixG6nohsR8NvfYOsVPYvVc3ksrt+cyWzx + ZuU8Vgdmj52z7rJ7HP66kMXj2apbzAHMUVw2Kak5mWWpRfp2CVwZ+x6ZF1yyr5i18whTA+N8 + oy5GTg4JAROJlbPWs0DYYhIX7q1n62Lk4hAS2McocehlDzOEs55R4t3Oh0wQzkkmieN9Mxgh + nLmMEq/+tTGC9LMJaEhs278czBYRkJbYeXc2K4jNLBAnsWXKf7C4sECgxPldb9lAbBYBVYmG + PTPAangFHCSWbGtghbhDXuLp/T6wGk4BR4k5N3eA9QoJlEt8vvGPeQIj/wJGhlWMsim5Vbq5 + iZk5xanJusXJiXl5qUW6xnq5mSV6qSmlmxhBwcUpybeD8etBpUOMAhyMSjy80Uy//YVYE8uK + K3MPMUpyMCmJ8uZ5AoX4kvJTKjMSizPii0pzUosPMUpwMCuJ8N5yBcrxpiRWVqUW5cOkpDlY + lMR5r6Tc9BcSSE8sSc1OTS1ILYLJynBwKEnwTvMHahQsSk1PrUjLzClBSDNxcIIM5wEafg6k + hre4IDG3ODMdIn+KUVFKnPc6SEIAJJFRmgfXC4v+V4ziQK8I8+4AqeIBJg647ldAg5mABrf3 + /AIZXJKIkJJqYFzE3avAt6GoKcasJ+Ph+oiZ95weXBdT2d2yS/R3YS33HZEEMXu2gtsTucW6 + lqumuj36eIWl+VbwNOsUpXnRN0tP3dwgEPFEyM1E+8L/TjHVPo+ayjyvtVPS52qu0nggoZNr + UfN2X7fhEf+6h9x501S05/1arRsUvLHLManWV/bAwuTSs59llViKMxINtZiLihMBbJFpctkC AAA= +Cc: tomi.ollila@iki.fi +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Mon, 09 Jul 2012 21:42:58 -0000 + +The JSON format eliminates the complex escaping issues that have +plagued the text search format. This uses the incremental JSON parser +so that, like the text parser, it can output search results +incrementally. + +This slows down the parser by about ~4X, but puts us in a good +position to optimize either by improving the JSON parser (evidence +suggests this can reduce the overhead to ~40% over the text format) or +by switching to S-expressions (evidence suggests this will more than +double performance over the text parser). [1] + +This also fixes the incremental search parsing test. + +This has one minor side-effect on search result formatting. +Previously, the date field was always padded to a fixed width of 12 +characters because of how the text parser's regexp was written. The +JSON format doesn't do this. We could pad it out in Emacs before +formatting it, but, since all of the other fields are variable width, +we instead fix notmuch-search-result-format to take the variable-width +field and pad it out. For users who have customized this variable, +we'll mention in the NEWS how to fix this slight format change. + +[1] id:"20110720205007.GB21316@mit.edu" +--- + emacs/notmuch.el | 110 +++++++++++++++++++++++++++++++----------------------- + test/emacs | 1 - + 2 files changed, 64 insertions(+), 47 deletions(-) + +diff --git a/emacs/notmuch.el b/emacs/notmuch.el +index dfeaf35..fabb7c0 100644 +--- a/emacs/notmuch.el ++++ b/emacs/notmuch.el +@@ -60,7 +60,7 @@ + (require 'notmuch-message) + + (defcustom notmuch-search-result-format +- `(("date" . "%s ") ++ `(("date" . "%12s ") + ("count" . "%-7s ") + ("authors" . "%-20s ") + ("subject" . "%s ") +@@ -557,17 +557,14 @@ This function advances the next thread when finished." + (notmuch-search-tag '("-inbox")) + (notmuch-search-next-thread)) + +-(defvar notmuch-search-process-filter-data nil +- "Data that has not yet been processed.") +-(make-variable-buffer-local 'notmuch-search-process-filter-data) +- + (defun notmuch-search-process-sentinel (proc msg) + "Add a message to let user know when \"notmuch search\" exits" + (let ((buffer (process-buffer proc)) + (status (process-status proc)) + (exit-status (process-exit-status proc)) + (never-found-target-thread nil)) +- (if (memq status '(exit signal)) ++ (when (memq status '(exit signal)) ++ (kill-buffer (process-get proc 'parse-buf)) + (if (buffer-live-p buffer) + (with-current-buffer buffer + (save-excursion +@@ -577,8 +574,6 @@ This function advances the next thread when finished." + (if (eq status 'signal) + (insert "Incomplete search results (search process was killed).\n")) + (when (eq status 'exit) +- (if notmuch-search-process-filter-data +- (insert (concat "Error: Unexpected output from notmuch search:\n" notmuch-search-process-filter-data))) + (insert "End of search results.") + (unless (= exit-status 0) + (insert (format " (process returned %d)" exit-status))) +@@ -758,45 +753,59 @@ non-authors is found, assume that all of the authors match." + (insert (apply #'format string objects)) + (insert "\n"))) + ++(defvar notmuch-search-process-state nil ++ "Parsing state of the search process filter.") ++ ++(defvar notmuch-search-json-parser nil ++ "Incremental JSON parser for the search process filter.") ++ + (defun notmuch-search-process-filter (proc string) + "Process and filter the output of \"notmuch search\"" +- (let ((buffer (process-buffer proc))) +- (if (buffer-live-p buffer) +- (with-current-buffer buffer +- (let ((line 0) +- (more t) +- (inhibit-read-only t) +- (string (concat notmuch-search-process-filter-data string))) +- (setq notmuch-search-process-filter-data nil) +- (while more +- (while (and (< line (length string)) (= (elt string line) ?\n)) +- (setq line (1+ line))) +- (if (string-match "^thread:\\([0-9A-Fa-f]*\\) \\([^][]*\\) \\[\\([0-9]*\\)/\\([0-9]*\\)\\] \\([^;]*\\); \\(.*\\) (\\([^()]*\\))$" string line) +- (let* ((thread-id (match-string 1 string)) +- (tags-str (match-string 7 string)) +- (result (list :thread thread-id +- :date_relative (match-string 2 string) +- :matched (string-to-number +- (match-string 3 string)) +- :total (string-to-number +- (match-string 4 string)) +- :authors (match-string 5 string) +- :subject (match-string 6 string) +- :tags (if tags-str +- (save-match-data +- (split-string tags-str)))))) +- (if (/= (match-beginning 0) line) +- (notmuch-search-show-error +- (substring string line (match-beginning 0)))) +- (notmuch-search-show-result result) +- (set 'line (match-end 0))) +- (set 'more nil) +- (while (and (< line (length string)) (= (elt string line) ?\n)) +- (setq line (1+ line))) +- (if (< line (length string)) +- (setq notmuch-search-process-filter-data (substring string line))) +- )))) +- (delete-process proc)))) ++ (let ((results-buf (process-buffer proc)) ++ (parse-buf (process-get proc 'parse-buf)) ++ (inhibit-read-only t) ++ done) ++ (if (not (buffer-live-p results-buf)) ++ (delete-process proc) ++ (with-current-buffer parse-buf ++ ;; Insert new data ++ (save-excursion ++ (goto-char (point-max)) ++ (insert string))) ++ (with-current-buffer results-buf ++ (while (not done) ++ (condition-case nil ++ (case notmuch-search-process-state ++ ((begin) ++ ;; Enter the results list ++ (if (eq (notmuch-json-begin-compound ++ notmuch-search-json-parser) 'retry) ++ (setq done t) ++ (setq notmuch-search-process-state 'result))) ++ ((result) ++ ;; Parse a result ++ (let ((result (notmuch-json-read notmuch-search-json-parser))) ++ (case result ++ ((retry) (setq done t)) ++ ((end) (setq notmuch-search-process-state 'end)) ++ (otherwise (notmuch-search-show-result result))))) ++ ((end) ++ ;; Any trailing data is unexpected ++ (notmuch-json-eof notmuch-search-json-parser) ++ (setq done t))) ++ (json-error ++ ;; Do our best to resynchronize and ensure forward ++ ;; progress ++ (notmuch-search-show-error ++ "%s" ++ (with-current-buffer parse-buf ++ (let ((bad (buffer-substring (line-beginning-position) ++ (line-end-position)))) ++ (forward-line) ++ bad)))))) ++ ;; Clear out what we've parsed ++ (with-current-buffer parse-buf ++ (delete-region (point-min) (point))))))) + + (defun notmuch-search-tag-all (&optional tag-changes) + "Add/remove tags from all messages in current search buffer. +@@ -899,10 +908,19 @@ Other optional parameters are used as follows: + (let ((proc (start-process + "notmuch-search" buffer + notmuch-command "search" ++ "--format=json" + (if oldest-first + "--sort=oldest-first" + "--sort=newest-first") +- query))) ++ query)) ++ ;; Use a scratch buffer to accumulate partial output. ++ ;; This buffer will be killed by the sentinel, which ++ ;; should be called no matter how the process dies. ++ (parse-buf (generate-new-buffer " *notmuch search parse*"))) ++ (set (make-local-variable 'notmuch-search-process-state) 'begin) ++ (set (make-local-variable 'notmuch-search-json-parser) ++ (notmuch-json-create-parser parse-buf)) ++ (process-put proc 'parse-buf parse-buf) + (set-process-sentinel proc 'notmuch-search-process-sentinel) + (set-process-filter proc 'notmuch-search-process-filter) + (set-process-query-on-exit-flag proc nil)))) +diff --git a/test/emacs b/test/emacs +index 293b12a..afe35ba 100755 +--- a/test/emacs ++++ b/test/emacs +@@ -36,7 +36,6 @@ test_emacs '(notmuch-search "tag:inbox") + test_expect_equal_file OUTPUT $EXPECTED/notmuch-search-tag-inbox + + test_begin_subtest "Incremental parsing of search results" +-test_subtest_known_broken + test_emacs "(ad-enable-advice 'notmuch-search-process-filter 'around 'pessimal) + (ad-activate 'notmuch-search-process-filter) + (notmuch-search \"tag:inbox\") +-- +1.7.10 +