From b691f4ea2ecd0b10e958f1f422566d7fce94b765 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Fri, 6 Jul 2012 16:52:26 +2000 Subject: [PATCH] [PATCH v2 8/9] emacs: Switch from text to JSON format for search results --- ad/1c05bb2afe746ed5df2764130d07de698ff003 | 278 ++++++++++++++++++++++ 1 file changed, 278 insertions(+) create mode 100644 ad/1c05bb2afe746ed5df2764130d07de698ff003 diff --git a/ad/1c05bb2afe746ed5df2764130d07de698ff003 b/ad/1c05bb2afe746ed5df2764130d07de698ff003 new file mode 100644 index 000000000..a887ae45f --- /dev/null +++ b/ad/1c05bb2afe746ed5df2764130d07de698ff003 @@ -0,0 +1,278 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id 5A4D5431FC2 + for ; Thu, 5 Jul 2012 13:52:47 -0700 (PDT) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Spam-Flag: NO +X-Spam-Score: -0.7 +X-Spam-Level: +X-Spam-Status: No, score=-0.7 tagged_above=-999 required=5 + tests=[RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id NMz-pghLFrIF for ; + Thu, 5 Jul 2012 13:52:45 -0700 (PDT) +Received: from dmz-mailsec-scanner-4.mit.edu (DMZ-MAILSEC-SCANNER-4.MIT.EDU + [18.9.25.15]) + by olra.theworths.org (Postfix) with ESMTP id 84012431FC3 + for ; Thu, 5 Jul 2012 13:52:43 -0700 (PDT) +X-AuditID: 1209190f-b7f306d0000008b4-ec-4ff5fe9b1ede +Received: from mailhub-auth-1.mit.edu ( [18.9.21.35]) + by dmz-mailsec-scanner-4.mit.edu (Symantec Messaging Gateway) with SMTP + id FC.18.02228.B9EF5FF4; Thu, 5 Jul 2012 16:52:43 -0400 (EDT) +Received: from outgoing.mit.edu (OUTGOING-AUTH.MIT.EDU [18.7.22.103]) + by mailhub-auth-1.mit.edu (8.13.8/8.9.2) with ESMTP id q65Kqg4T024134; + Thu, 5 Jul 2012 16:52:42 -0400 +Received: from drake.dyndns.org (26-4-182.dynamic.csail.mit.edu [18.26.4.182]) + (authenticated bits=0) + (User authenticated as amdragon@ATHENA.MIT.EDU) + by outgoing.mit.edu (8.13.6/8.12.4) with ESMTP id q65Kqbd9027241 + (version=TLSv1/SSLv3 cipher=AES256-SHA bits=256 verify=NOT); + Thu, 5 Jul 2012 16:52:39 -0400 (EDT) +Received: from amthrax by drake.dyndns.org with local (Exim 4.77) + (envelope-from ) + id 1Smt2T-0004Xq-Mi; Thu, 05 Jul 2012 16:52:37 -0400 +From: Austin Clements +To: notmuch@notmuchmail.org +Subject: [PATCH v2 8/9] emacs: Switch from text to JSON format for search + results +Date: Thu, 5 Jul 2012 16:52:26 -0400 +Message-Id: <1341521547-15502-9-git-send-email-amdragon@mit.edu> +X-Mailer: git-send-email 1.7.10 +In-Reply-To: <1341521547-15502-1-git-send-email-amdragon@mit.edu> +References: <1341354059-29396-1-git-send-email-amdragon@mit.edu> + <1341521547-15502-1-git-send-email-amdragon@mit.edu> +X-Brightmail-Tracker: + H4sIAAAAAAAAA+NgFjrGIsWRmVeSWpSXmKPExsUixCmqrDv731d/g8alwhar5/JYXL85k9ni + zcp5rA7MHjtn3WX3OPx1IYvHs1W3mAOYo7hsUlJzMstSi/TtErgy9j0yL7hkXzFr5xGmBsb5 + Rl2MnBwSAiYSH5Y/ZYGwxSQu3FvP1sXIxSEksI9R4uWnX1DOekaJD3vvMEE4J5kkNsy7yQrh + zGWU+DN9FyNIP5uAhsS2/cvBbBEBaYmdd2ezgtjMAnESW6b8B4sLCwRKLJo0GWwfi4CqxP8l + e5hBbF4BB4l9V45C3SEv8fR+HxuIzSngKHFh4mKwXiGBcok/S/6xTGDkX8DIsIpRNiW3Sjc3 + MTOnODVZtzg5MS8vtUjXRC83s0QvNaV0EyM4uCT5dzB+O6h0iFGAg1GJh9cw94u/EGtiWXFl + 7iFGSQ4mJVHext9f/YX4kvJTKjMSizPii0pzUosPMUpwMCuJ8PZmAOV4UxIrq1KL8mFS0hws + SuK8V1Nu+gsJpCeWpGanphakFsFkZTg4lCR41YFRJCRYlJqeWpGWmVOCkGbi4AQZzgM0XAOk + hre4IDG3ODMdIn+KUVFKnFcaJCEAksgozYPrhUX/K0ZxoFeEeT/9BariASYOuO5XQIOZgAbn + Lf4EMrgkESEl1cDYWLvOq3FH9ZfA59qLr/n8FuYofZXnq8a5tOT71Z+tcxZl9x7bWxOZc6e+ + PfBETtzHQibf/fui+FsvRnd8d75+MuT7jSK20jNdF5gyP5w9rfZgIl9IW57Zir0cy1UWbZUr + l/8ck6PsZqgie75G9MDugmvG5+84aqpf5JrHYq76++2NbvZPfTVKLMUZiYZazEXFiQCAXJ2F + 2QIAAA== +Cc: tomi.ollila@iki.fi +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Thu, 05 Jul 2012 20:52:47 -0000 + +The JSON format eliminates the complex escaping issues that have +plagued the text search format. This uses the incremental JSON parser +so that, like the text parser, it can output search results +incrementally. + +This slows down the parser by about ~4X, but puts us in a good +position to optimize either by improving the JSON parser (evidence +suggests this can reduce the overhead to ~40% over the text format) or +by switching to S-expressions (evidence suggests this will more than +double performance over the text parser). [1] + +This also fixes the incremental search parsing test. + +This has one minor side-effect on search result formatting. +Previously, the date field was always padded to a fixed width of 12 +characters because of how the text parser's regexp was written. The +JSON format doesn't do this. We could pad it out in Emacs before +formatting it, but, since all of the other fields are variable width, +we instead fix notmuch-search-result-format to take the variable-width +field and pad it out. For users who have customized this variable, +we'll mention in the NEWS how to fix this slight format change. + +[1] id:"20110720205007.GB21316@mit.edu" +--- + emacs/notmuch.el | 110 +++++++++++++++++++++++++++++++----------------------- + test/emacs | 1 - + 2 files changed, 64 insertions(+), 47 deletions(-) + +diff --git a/emacs/notmuch.el b/emacs/notmuch.el +index dfeaf35..fabb7c0 100644 +--- a/emacs/notmuch.el ++++ b/emacs/notmuch.el +@@ -60,7 +60,7 @@ + (require 'notmuch-message) + + (defcustom notmuch-search-result-format +- `(("date" . "%s ") ++ `(("date" . "%12s ") + ("count" . "%-7s ") + ("authors" . "%-20s ") + ("subject" . "%s ") +@@ -557,17 +557,14 @@ This function advances the next thread when finished." + (notmuch-search-tag '("-inbox")) + (notmuch-search-next-thread)) + +-(defvar notmuch-search-process-filter-data nil +- "Data that has not yet been processed.") +-(make-variable-buffer-local 'notmuch-search-process-filter-data) +- + (defun notmuch-search-process-sentinel (proc msg) + "Add a message to let user know when \"notmuch search\" exits" + (let ((buffer (process-buffer proc)) + (status (process-status proc)) + (exit-status (process-exit-status proc)) + (never-found-target-thread nil)) +- (if (memq status '(exit signal)) ++ (when (memq status '(exit signal)) ++ (kill-buffer (process-get proc 'parse-buf)) + (if (buffer-live-p buffer) + (with-current-buffer buffer + (save-excursion +@@ -577,8 +574,6 @@ This function advances the next thread when finished." + (if (eq status 'signal) + (insert "Incomplete search results (search process was killed).\n")) + (when (eq status 'exit) +- (if notmuch-search-process-filter-data +- (insert (concat "Error: Unexpected output from notmuch search:\n" notmuch-search-process-filter-data))) + (insert "End of search results.") + (unless (= exit-status 0) + (insert (format " (process returned %d)" exit-status))) +@@ -758,45 +753,59 @@ non-authors is found, assume that all of the authors match." + (insert (apply #'format string objects)) + (insert "\n"))) + ++(defvar notmuch-search-process-state nil ++ "Parsing state of the search process filter.") ++ ++(defvar notmuch-search-json-parser nil ++ "Incremental JSON parser for the search process filter.") ++ + (defun notmuch-search-process-filter (proc string) + "Process and filter the output of \"notmuch search\"" +- (let ((buffer (process-buffer proc))) +- (if (buffer-live-p buffer) +- (with-current-buffer buffer +- (let ((line 0) +- (more t) +- (inhibit-read-only t) +- (string (concat notmuch-search-process-filter-data string))) +- (setq notmuch-search-process-filter-data nil) +- (while more +- (while (and (< line (length string)) (= (elt string line) ?\n)) +- (setq line (1+ line))) +- (if (string-match "^thread:\\([0-9A-Fa-f]*\\) \\([^][]*\\) \\[\\([0-9]*\\)/\\([0-9]*\\)\\] \\([^;]*\\); \\(.*\\) (\\([^()]*\\))$" string line) +- (let* ((thread-id (match-string 1 string)) +- (tags-str (match-string 7 string)) +- (result (list :thread thread-id +- :date_relative (match-string 2 string) +- :matched (string-to-number +- (match-string 3 string)) +- :total (string-to-number +- (match-string 4 string)) +- :authors (match-string 5 string) +- :subject (match-string 6 string) +- :tags (if tags-str +- (save-match-data +- (split-string tags-str)))))) +- (if (/= (match-beginning 0) line) +- (notmuch-search-show-error +- (substring string line (match-beginning 0)))) +- (notmuch-search-show-result result) +- (set 'line (match-end 0))) +- (set 'more nil) +- (while (and (< line (length string)) (= (elt string line) ?\n)) +- (setq line (1+ line))) +- (if (< line (length string)) +- (setq notmuch-search-process-filter-data (substring string line))) +- )))) +- (delete-process proc)))) ++ (let ((results-buf (process-buffer proc)) ++ (parse-buf (process-get proc 'parse-buf)) ++ (inhibit-read-only t) ++ done) ++ (if (not (buffer-live-p results-buf)) ++ (delete-process proc) ++ (with-current-buffer parse-buf ++ ;; Insert new data ++ (save-excursion ++ (goto-char (point-max)) ++ (insert string))) ++ (with-current-buffer results-buf ++ (while (not done) ++ (condition-case nil ++ (case notmuch-search-process-state ++ ((begin) ++ ;; Enter the results list ++ (if (eq (notmuch-json-begin-compound ++ notmuch-search-json-parser) 'retry) ++ (setq done t) ++ (setq notmuch-search-process-state 'result))) ++ ((result) ++ ;; Parse a result ++ (let ((result (notmuch-json-read notmuch-search-json-parser))) ++ (case result ++ ((retry) (setq done t)) ++ ((end) (setq notmuch-search-process-state 'end)) ++ (otherwise (notmuch-search-show-result result))))) ++ ((end) ++ ;; Any trailing data is unexpected ++ (notmuch-json-eof notmuch-search-json-parser) ++ (setq done t))) ++ (json-error ++ ;; Do our best to resynchronize and ensure forward ++ ;; progress ++ (notmuch-search-show-error ++ "%s" ++ (with-current-buffer parse-buf ++ (let ((bad (buffer-substring (line-beginning-position) ++ (line-end-position)))) ++ (forward-line) ++ bad)))))) ++ ;; Clear out what we've parsed ++ (with-current-buffer parse-buf ++ (delete-region (point-min) (point))))))) + + (defun notmuch-search-tag-all (&optional tag-changes) + "Add/remove tags from all messages in current search buffer. +@@ -899,10 +908,19 @@ Other optional parameters are used as follows: + (let ((proc (start-process + "notmuch-search" buffer + notmuch-command "search" ++ "--format=json" + (if oldest-first + "--sort=oldest-first" + "--sort=newest-first") +- query))) ++ query)) ++ ;; Use a scratch buffer to accumulate partial output. ++ ;; This buffer will be killed by the sentinel, which ++ ;; should be called no matter how the process dies. ++ (parse-buf (generate-new-buffer " *notmuch search parse*"))) ++ (set (make-local-variable 'notmuch-search-process-state) 'begin) ++ (set (make-local-variable 'notmuch-search-json-parser) ++ (notmuch-json-create-parser parse-buf)) ++ (process-put proc 'parse-buf parse-buf) + (set-process-sentinel proc 'notmuch-search-process-sentinel) + (set-process-filter proc 'notmuch-search-process-filter) + (set-process-query-on-exit-flag proc nil)))) +diff --git a/test/emacs b/test/emacs +index 293b12a..afe35ba 100755 +--- a/test/emacs ++++ b/test/emacs +@@ -36,7 +36,6 @@ test_emacs '(notmuch-search "tag:inbox") + test_expect_equal_file OUTPUT $EXPECTED/notmuch-search-tag-inbox + + test_begin_subtest "Incremental parsing of search results" +-test_subtest_known_broken + test_emacs "(ad-enable-advice 'notmuch-search-process-filter 'around 'pessimal) + (ad-activate 'notmuch-search-process-filter) + (notmuch-search \"tag:inbox\") +-- +1.7.10 + -- 2.26.2