[PATCH] WIP: regexp matching in 'subject' and 'from'

diff --git a/5f/d3e585a8aa84092eadf4a429d5faeb467a1ddc b/5f/d3e585a8aa84092eadf4a429d5faeb467a1ddc
new file mode 100644 (file)
index 0000000..cec3813
--- /dev/null
+++ b/5f/d3e585a8aa84092eadf4a429d5faeb467a1ddc
@@ -0,0 +1,342 @@
+Return-Path: <bremner@tethera.net>\r
+X-Original-To: notmuch@notmuchmail.org\r
+Delivered-To: notmuch@notmuchmail.org\r
+Received: from localhost (localhost [127.0.0.1])\r
+ by arlo.cworth.org (Postfix) with ESMTP id 02C226DE01BE\r
+ for <notmuch@notmuchmail.org>; Thu,  9 Jun 2016 19:28:35 -0700 (PDT)\r
+X-Virus-Scanned: Debian amavisd-new at cworth.org\r
+X-Spam-Flag: NO\r
+X-Spam-Score: -0.011\r
+X-Spam-Level: \r
+X-Spam-Status: No, score=-0.011 tagged_above=-999 required=5\r
+ tests=[AWL=-0.000, SPF_PASS=-0.001, T_RP_MATCHES_RCVD=-0.01]\r
+ autolearn=disabled\r
+Received: from arlo.cworth.org ([127.0.0.1])\r
+ by localhost (arlo.cworth.org [127.0.0.1]) (amavisd-new, port 10024)\r
+ with ESMTP id BskbBxEtfBwE for <notmuch@notmuchmail.org>;\r
+ Thu,  9 Jun 2016 19:28:26 -0700 (PDT)\r
+Received: from fethera.tethera.net (fethera.tethera.net [198.245.60.197])\r
+ by arlo.cworth.org (Postfix) with ESMTPS id 1EE566DE0130\r
+ for <notmuch@notmuchmail.org>; Thu,  9 Jun 2016 19:28:26 -0700 (PDT)\r
+Received: from remotemail by fethera.tethera.net with local (Exim 4.84)\r
+ (envelope-from <bremner@tethera.net>)\r
+ id 1bBCAz-00032U-41; Thu, 09 Jun 2016 22:28:01 -0400\r
+Received: (nullmailer pid 30971 invoked by uid 1000);\r
+ Fri, 10 Jun 2016 02:28:12 -0000\r
+From: David Bremner <david@tethera.net>\r
+To: David Bremner <david@tethera.net>,\r
+ Austin Clements <aclements@csail.mit.edu>\r
+Cc: sfischme@uwaterloo.ca, Gaute Hope <eg@gaute.vetsj.com>,\r
+ notmuch <notmuch@notmuchmail.org>\r
+Subject: [PATCH] WIP: regexp matching in 'subject' and 'from'\r
+Date: Thu,  9 Jun 2016 23:28:08 -0300\r
+Message-Id: <1465525688-30913-1-git-send-email-david@tethera.net>\r
+X-Mailer: git-send-email 2.8.1\r
+In-Reply-To: <1465265149-7174-1-git-send-email-david@tethera.net>\r
+References: <1465265149-7174-1-git-send-email-david@tethera.net>\r
+MIME-Version: 1.0\r
+Content-Type: text/plain; charset=UTF-8\r
+Content-Transfer-Encoding: 8bit\r
+X-BeenThere: notmuch@notmuchmail.org\r
+X-Mailman-Version: 2.1.20\r
+Precedence: list\r
+List-Id: "Use and development of the notmuch mail system."\r
+ <notmuch.notmuchmail.org>\r
+List-Unsubscribe: <https://notmuchmail.org/mailman/options/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>\r
+List-Archive: <http://notmuchmail.org/pipermail/notmuch/>\r
+List-Post: <mailto:notmuch@notmuchmail.org>\r
+List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>\r
+List-Subscribe: <https://notmuchmail.org/mailman/listinfo/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=subscribe>\r
+X-List-Received-Date: Fri, 10 Jun 2016 02:28:35 -0000\r
+\r
+the idea is that you can run\r
+\r
+% notmuch search subject_re:<your-favourite-regexp>\r
+% notmuch search from_re:<your-favourite-regexp>'\r
+\r
+or\r
+\r
+% notmuch search subject:"your usual phrase search"\r
+% notmuch search from:"usual phrase search"\r
+\r
+This should also work with bindings, since it extends the query parser.\r
+\r
+This is trivial to extend for other value slots, but currently the only\r
+value slots are date, message_id, from, subject, and last_mod. Date is\r
+already searchable, and message_id is not obviously useful to regex\r
+match.\r
+---\r
+\r
+This is more or less complete codewise, it fixes the know problems\r
+with the last version. Names of prefixes are debatable, and of course\r
+it needs doc and tests.  I don't see any reason not to do this at the moment,\r
+since it's basically free; no new terms are added to the database.\r
+\r
+ lib/Makefile.local     |   1 +\r
+ lib/database-private.h |   2 +\r
+ lib/database.cc        |  12 +++++-\r
+ lib/regexp-fields.cc   | 102 +++++++++++++++++++++++++++++++++++++++++++++++++\r
+ lib/regexp-fields.h    |  78 +++++++++++++++++++++++++++++++++++++\r
+ 5 files changed, 194 insertions(+), 1 deletion(-)\r
+ create mode 100644 lib/regexp-fields.cc\r
+ create mode 100644 lib/regexp-fields.h\r
+\r
+diff --git a/lib/Makefile.local b/lib/Makefile.local\r
+index beb9635..68771e6 100644\r
+--- a/lib/Makefile.local\r
++++ b/lib/Makefile.local\r
+@@ -51,6 +51,7 @@ libnotmuch_cxx_srcs =                \\r
+       $(dir)/query.cc         \\r
+       $(dir)/query-fp.cc      \\r
+       $(dir)/config.cc        \\r
++      $(dir)/regexp-fields.cc     \\r
+       $(dir)/thread.cc\r
+ \r
+ libnotmuch_modules := $(libnotmuch_c_srcs:.c=.o) $(libnotmuch_cxx_srcs:.cc=.o)\r
+diff --git a/lib/database-private.h b/lib/database-private.h\r
+index ca71a92..090fcdf 100644\r
+--- a/lib/database-private.h\r
++++ b/lib/database-private.h\r
+@@ -186,6 +186,8 @@ struct _notmuch_database {\r
+ #if HAVE_XAPIAN_FIELD_PROCESSOR\r
+     Xapian::FieldProcessor *date_field_processor;\r
+     Xapian::FieldProcessor *query_field_processor;\r
++    Xapian::FieldProcessor *subject_re_field_processor;\r
++    Xapian::FieldProcessor *from_re_field_processor;\r
+ #endif\r
+     Xapian::ValueRangeProcessor *last_mod_range_processor;\r
+ };\r
+diff --git a/lib/database.cc b/lib/database.cc\r
+index 86bf261..4049406 100644\r
+--- a/lib/database.cc\r
++++ b/lib/database.cc\r
+@@ -21,6 +21,7 @@\r
+ #include "database-private.h"\r
+ #include "parse-time-vrp.h"\r
+ #include "query-fp.h"\r
++#include "regexp-fields.h"\r
+ #include "string-util.h"\r
+ \r
+ #include <iostream>\r
+@@ -1008,6 +1009,10 @@ notmuch_database_open_verbose (const char *path,\r
+       notmuch->query_parser->add_boolean_prefix("date", notmuch->date_field_processor);\r
+       notmuch->query_field_processor = new QueryFieldProcessor (*notmuch->query_parser, notmuch);\r
+       notmuch->query_parser->add_boolean_prefix("query", notmuch->query_field_processor);\r
++      notmuch->subject_re_field_processor = new RegexpFieldProcessor (NOTMUCH_VALUE_SUBJECT, *notmuch->query_parser, notmuch);\r
++      notmuch->query_parser->add_boolean_prefix("subject_re", notmuch->subject_re_field_processor);\r
++      notmuch->from_re_field_processor = new RegexpFieldProcessor (NOTMUCH_VALUE_FROM, *notmuch->query_parser, notmuch);\r
++      notmuch->query_parser->add_boolean_prefix("from_re", notmuch->from_re_field_processor);\r
+ #endif\r
+       notmuch->last_mod_range_processor = new Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_LAST_MOD, "lastmod:");\r
+ \r
+@@ -1098,7 +1103,12 @@ notmuch_database_close (notmuch_database_t *notmuch)\r
+     notmuch->date_range_processor = NULL;\r
+     delete notmuch->last_mod_range_processor;\r
+     notmuch->last_mod_range_processor = NULL;\r
+-\r
++#ifdef HAVE_XAPIAN_FIELD_PROCESSOR\r
++    delete notmuch->from_re_field_processor;\r
++    notmuch->from_re_field_processor = NULL;\r
++    delete notmuch->subject_re_field_processor;\r
++    notmuch->subject_re_field_processor = NULL;\r
++#endif\r
+     return status;\r
+ }\r
+ \r
+diff --git a/lib/regexp-fields.cc b/lib/regexp-fields.cc\r
+new file mode 100644\r
+index 0000000..4bbebda\r
+--- /dev/null\r
++++ b/lib/regexp-fields.cc\r
+@@ -0,0 +1,102 @@\r
++/* query-fp.cc - "query:" field processor glue\r
++ *\r
++ * This file is part of notmuch.\r
++ *\r
++ * Copyright © 2015 Austin Clements\r
++ * Copyright © 2016 David Bremner\r
++ *\r
++ * This program is free software: you can redistribute it and/or modify\r
++ * it under the terms of the GNU General Public License as published by\r
++ * the Free Software Foundation, either version 3 of the License, or\r
++ * (at your option) any later version.\r
++ *\r
++ * This program is distributed in the hope that it will be useful,\r
++ * but WITHOUT ANY WARRANTY; without even the implied warranty of\r
++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\r
++ * GNU General Public License for more details.\r
++ *\r
++ * You should have received a copy of the GNU General Public License\r
++ * along with this program.  If not, see https://www.gnu.org/licenses/ .\r
++ *\r
++ * Author: Austin Clements <aclements@csail.mit.edu>\r
++ *                David Bremner <david@tethera.net>\r
++ */\r
++\r
++#include "regexp-fields.h"\r
++\r
++#ifdef HAVE_XAPIAN_FIELD_PROCESSOR\r
++RegexpPostingSource::RegexpPostingSource (Xapian::valueno slot, const std::string &regexp)\r
++    : slot_ (slot)\r
++{\r
++    int r = regcomp (&regexp_, regexp.c_str (), REG_EXTENDED | REG_NOSUB);\r
++\r
++    if (r != 0)\r
++      /* XXX Report a query syntax error using regerror */\r
++      throw "regcomp failed";\r
++}\r
++\r
++RegexpPostingSource::~RegexpPostingSource ()\r
++{\r
++    regfree (&regexp_);\r
++}\r
++\r
++void\r
++RegexpPostingSource::init (const Xapian::Database &db)\r
++{\r
++    db_ = db;\r
++    it_ = db_.valuestream_begin (slot_);\r
++    end_ = db.valuestream_end (slot_);\r
++    started_ = false;\r
++}\r
++\r
++Xapian::doccount\r
++RegexpPostingSource::get_termfreq_min () const\r
++{\r
++    return 0;\r
++}\r
++\r
++Xapian::doccount\r
++RegexpPostingSource::get_termfreq_est () const\r
++{\r
++    return get_termfreq_max () / 2;\r
++}\r
++\r
++Xapian::doccount\r
++RegexpPostingSource::get_termfreq_max () const\r
++{\r
++    return db_.get_value_freq (slot_);\r
++}\r
++\r
++Xapian::docid\r
++RegexpPostingSource::get_docid () const\r
++{\r
++    return it_.get_docid ();\r
++}\r
++\r
++bool\r
++RegexpPostingSource::at_end () const\r
++{\r
++    return it_ == end_;\r
++}\r
++\r
++void\r
++RegexpPostingSource::next (unused (double min_wt))\r
++{\r
++    if (started_ && ! at_end ())\r
++      ++it_;\r
++    started_ = true;\r
++\r
++    for (; ! at_end (); ++it_) {\r
++      std::string value = *it_;\r
++      if (regexec (&regexp_, value.c_str (), 0, NULL, 0) == 0)\r
++          break;\r
++    }\r
++}\r
++\r
++Xapian::Query\r
++RegexpFieldProcessor::operator() (const std::string & str)\r
++{\r
++    postings = new RegexpPostingSource (slot, str);\r
++    return Xapian::Query (postings);\r
++}\r
++#endif\r
+diff --git a/lib/regexp-fields.h b/lib/regexp-fields.h\r
+new file mode 100644\r
+index 0000000..a184cab\r
+--- /dev/null\r
++++ b/lib/regexp-fields.h\r
+@@ -0,0 +1,78 @@\r
++/* regex-fields.h - xapian glue for semi-bruteforce regexp search\r
++ *\r
++ * This file is part of notmuch.\r
++ *\r
++ * Copyright © 2015 Austin Clements\r
++ * Copyright © 2016 David Bremner\r
++ *\r
++ * This program is free software: you can redistribute it and/or modify\r
++ * it under the terms of the GNU General Public License as published by\r
++ * the Free Software Foundation, either version 3 of the License, or\r
++ * (at your option) any later version.\r
++ *\r
++ * This program is distributed in the hope that it will be useful,\r
++ * but WITHOUT ANY WARRANTY; without even the implied warranty of\r
++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\r
++ * GNU General Public License for more details.\r
++ *\r
++ * You should have received a copy of the GNU General Public License\r
++ * along with this program.  If not, see https://www.gnu.org/licenses/ .\r
++ *\r
++ * Author: Austin Clements <aclements@csail.mit.edu>\r
++ *                David Bremner <david@tethera.net>\r
++ */\r
++\r
++#ifndef NOTMUCH_REGEXP_FIELDS_H\r
++#define NOTMUCH_REGEXP_FIELDS_H\r
++#ifdef HAVE_XAPIAN_FIELD_PROCESSOR\r
++#include <sys/types.h>\r
++#include <regex.h>\r
++#include <xapian.h>\r
++#include "notmuch-private.h"\r
++\r
++/* A posting source that returns documents where a value matches a\r
++ * regexp.\r
++ */\r
++class RegexpPostingSource : public Xapian::PostingSource\r
++{\r
++ protected:\r
++    const Xapian::valueno slot_;\r
++    regex_t regexp_;\r
++    Xapian::Database db_;\r
++    bool started_;\r
++    Xapian::ValueIterator it_, end_;\r
++\r
++/* No copying */\r
++    RegexpPostingSource (const RegexpPostingSource &);\r
++    RegexpPostingSource &operator= (const RegexpPostingSource &);\r
++\r
++ public:\r
++    RegexpPostingSource (Xapian::valueno slot, const std::string &regexp);\r
++    ~RegexpPostingSource ();\r
++    void init (const Xapian::Database &db);\r
++    Xapian::doccount get_termfreq_min () const;\r
++    Xapian::doccount get_termfreq_est () const;\r
++    Xapian::doccount get_termfreq_max () const;\r
++    Xapian::docid get_docid () const;\r
++    bool at_end () const;\r
++    void next (unused (double min_wt));\r
++};\r
++\r
++\r
++class RegexpFieldProcessor : public Xapian::FieldProcessor {\r
++ protected:\r
++    Xapian::valueno slot;\r
++    Xapian::QueryParser &parser;\r
++    notmuch_database_t *notmuch;\r
++    RegexpPostingSource *postings = NULL;\r
++\r
++ public:\r
++    RegexpFieldProcessor (Xapian::valueno slot_, Xapian::QueryParser &parser_, notmuch_database_t *notmuch_)\r
++      : slot(slot_), parser(parser_), notmuch(notmuch_) { };\r
++\r
++    ~RegexpFieldProcessor () { delete postings; };\r
++\r
++    Xapian::Query operator()(const std::string & str);\r
++};\r
++#endif\r
++#endif /* NOTMUCH_REGEXP_FIELDS_H */\r
+-- \r
+2.8.1\r
+\r