[PATCH] WIP: regexp matching in subjects

diff --git a/e7/68ecfa992c3689768c031f7aaa3e8091f2b943 b/e7/68ecfa992c3689768c031f7aaa3e8091f2b943
new file mode 100644 (file)
index 0000000..afe3131
--- /dev/null
+++ b/e7/68ecfa992c3689768c031f7aaa3e8091f2b943
@@ -0,0 +1,378 @@
+Return-Path: <bremner@tethera.net>\r
+X-Original-To: notmuch@notmuchmail.org\r
+Delivered-To: notmuch@notmuchmail.org\r
+Received: from localhost (localhost [127.0.0.1])\r
+ by arlo.cworth.org (Postfix) with ESMTP id C682C6DE0130\r
+ for <notmuch@notmuchmail.org>; Mon,  6 Jun 2016 19:06:20 -0700 (PDT)\r
+X-Virus-Scanned: Debian amavisd-new at cworth.org\r
+X-Spam-Flag: NO\r
+X-Spam-Score: -0.011\r
+X-Spam-Level: \r
+X-Spam-Status: No, score=-0.011 tagged_above=-999 required=5\r
+ tests=[AWL=-0.000, SPF_PASS=-0.001, T_RP_MATCHES_RCVD=-0.01]\r
+ autolearn=disabled\r
+Received: from arlo.cworth.org ([127.0.0.1])\r
+ by localhost (arlo.cworth.org [127.0.0.1]) (amavisd-new, port 10024)\r
+ with ESMTP id rI7TzeRsZYMf for <notmuch@notmuchmail.org>;\r
+ Mon,  6 Jun 2016 19:06:10 -0700 (PDT)\r
+Received: from fethera.tethera.net (fethera.tethera.net [198.245.60.197])\r
+ by arlo.cworth.org (Postfix) with ESMTPS id C2C196DE00DA\r
+ for <notmuch@notmuchmail.org>; Mon,  6 Jun 2016 19:06:10 -0700 (PDT)\r
+Received: from remotemail by fethera.tethera.net with local (Exim 4.84)\r
+ (envelope-from <bremner@tethera.net>)\r
+ id 1bA6Om-0000AV-CX; Mon, 06 Jun 2016 22:05:44 -0400\r
+Received: (nullmailer pid 7230 invoked by uid 1000);\r
+ Tue, 07 Jun 2016 02:05:54 -0000\r
+From: David Bremner <david@tethera.net>\r
+To: Austin Clements <aclements@csail.mit.edu>,\r
+ David Bremner <david@tethera.net>\r
+Cc: sfischme@uwaterloo.ca, Gaute Hope <eg@gaute.vetsj.com>,\r
+ notmuch <notmuch@notmuchmail.org>\r
+Subject: [PATCH] WIP: regexp matching in subjects\r
+Date: Mon,  6 Jun 2016 23:05:49 -0300\r
+Message-Id: <1465265149-7174-1-git-send-email-david@tethera.net>\r
+X-Mailer: git-send-email 2.8.1\r
+In-Reply-To:\r
+ <CAH-f9WtC6CeVecfg8wFZUVc8K2rUfzsP72xo97sJX2y_mLW6-g@mail.gmail.com>\r
+References:\r
+ <CAH-f9WtC6CeVecfg8wFZUVc8K2rUfzsP72xo97sJX2y_mLW6-g@mail.gmail.com>\r
+MIME-Version: 1.0\r
+Content-Type: text/plain; charset=UTF-8\r
+Content-Transfer-Encoding: 8bit\r
+X-BeenThere: notmuch@notmuchmail.org\r
+X-Mailman-Version: 2.1.20\r
+Precedence: list\r
+List-Id: "Use and development of the notmuch mail system."\r
+ <notmuch.notmuchmail.org>\r
+List-Unsubscribe: <https://notmuchmail.org/mailman/options/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>\r
+List-Archive: <http://notmuchmail.org/pipermail/notmuch/>\r
+List-Post: <mailto:notmuch@notmuchmail.org>\r
+List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>\r
+List-Subscribe: <https://notmuchmail.org/mailman/listinfo/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=subscribe>\r
+X-List-Received-Date: Tue, 07 Jun 2016 02:06:20 -0000\r
+\r
+the idea is that you can run\r
+\r
+% notmuch search 'subject:rx:<your-favourite-regexp>'\r
+\r
+or\r
+\r
+% notmuch search subject:"your usual phrase search"\r
+\r
+This should also work with bindings.\r
+---\r
+\r
+Here is Austin's "hack", crammed into the field processor framework.\r
+I seem to have broken one of the existing subject search tests with my\r
+recursive query parsing. I didn't have time to figure out why, yet.\r
+\r
+ lib/Makefile.local     |  2 ++\r
+ lib/database-private.h |  1 +\r
+ lib/database.cc        |  5 +++\r
+ lib/regexp-ps.cc       | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++\r
+ lib/regexp-ps.h        | 37 ++++++++++++++++++++\r
+ lib/subject-fp.cc      | 41 ++++++++++++++++++++++\r
+ lib/subject-fp.h       | 43 +++++++++++++++++++++++\r
+ 7 files changed, 221 insertions(+)\r
+ create mode 100644 lib/regexp-ps.cc\r
+ create mode 100644 lib/regexp-ps.h\r
+ create mode 100644 lib/subject-fp.cc\r
+ create mode 100644 lib/subject-fp.h\r
+\r
+diff --git a/lib/Makefile.local b/lib/Makefile.local\r
+index beb9635..0e7311f 100644\r
+--- a/lib/Makefile.local\r
++++ b/lib/Makefile.local\r
+@@ -51,6 +51,8 @@ libnotmuch_cxx_srcs =                \\r
+       $(dir)/query.cc         \\r
+       $(dir)/query-fp.cc      \\r
+       $(dir)/config.cc        \\r
++      $(dir)/regexp-ps.cc     \\r
++      $(dir)/subject-fp.cc    \\r
+       $(dir)/thread.cc\r
+ \r
+ libnotmuch_modules := $(libnotmuch_c_srcs:.c=.o) $(libnotmuch_cxx_srcs:.cc=.o)\r
+diff --git a/lib/database-private.h b/lib/database-private.h\r
+index ca71a92..5de0b81 100644\r
+--- a/lib/database-private.h\r
++++ b/lib/database-private.h\r
+@@ -186,6 +186,7 @@ struct _notmuch_database {\r
+ #if HAVE_XAPIAN_FIELD_PROCESSOR\r
+     Xapian::FieldProcessor *date_field_processor;\r
+     Xapian::FieldProcessor *query_field_processor;\r
++    Xapian::FieldProcessor *subject_field_processor;\r
+ #endif\r
+     Xapian::ValueRangeProcessor *last_mod_range_processor;\r
+ };\r
+diff --git a/lib/database.cc b/lib/database.cc\r
+index 86bf261..adfbb81 100644\r
+--- a/lib/database.cc\r
++++ b/lib/database.cc\r
+@@ -21,6 +21,7 @@\r
+ #include "database-private.h"\r
+ #include "parse-time-vrp.h"\r
+ #include "query-fp.h"\r
++#include "subject-fp.h"\r
+ #include "string-util.h"\r
+ \r
+ #include <iostream>\r
+@@ -1008,6 +1009,8 @@ notmuch_database_open_verbose (const char *path,\r
+       notmuch->query_parser->add_boolean_prefix("date", notmuch->date_field_processor);\r
+       notmuch->query_field_processor = new QueryFieldProcessor (*notmuch->query_parser, notmuch);\r
+       notmuch->query_parser->add_boolean_prefix("query", notmuch->query_field_processor);\r
++      notmuch->subject_field_processor = new SubjectFieldProcessor (*notmuch->query_parser, notmuch);\r
++      notmuch->query_parser->add_boolean_prefix("subject", notmuch->subject_field_processor);\r
+ #endif\r
+       notmuch->last_mod_range_processor = new Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_LAST_MOD, "lastmod:");\r
+ \r
+@@ -1027,6 +1030,8 @@ notmuch_database_open_verbose (const char *path,\r
+ \r
+       for (i = 0; i < ARRAY_SIZE (PROBABILISTIC_PREFIX); i++) {\r
+           prefix_t *prefix = &PROBABILISTIC_PREFIX[i];\r
++          if (strcmp (prefix->name, "subject") == 0)\r
++              continue;\r
+           notmuch->query_parser->add_prefix (prefix->name, prefix->prefix);\r
+       }\r
+     } catch (const Xapian::Error &error) {\r
+diff --git a/lib/regexp-ps.cc b/lib/regexp-ps.cc\r
+new file mode 100644\r
+index 0000000..540c7d6\r
+--- /dev/null\r
++++ b/lib/regexp-ps.cc\r
+@@ -0,0 +1,92 @@\r
++/* query-fp.cc - "query:" field processor glue\r
++ *\r
++ * This file is part of notmuch.\r
++ *\r
++ * Copyright © 2016 David Bremner\r
++ *\r
++ * This program is free software: you can redistribute it and/or modify\r
++ * it under the terms of the GNU General Public License as published by\r
++ * the Free Software Foundation, either version 3 of the License, or\r
++ * (at your option) any later version.\r
++ *\r
++ * This program is distributed in the hope that it will be useful,\r
++ * but WITHOUT ANY WARRANTY; without even the implied warranty of\r
++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\r
++ * GNU General Public License for more details.\r
++ *\r
++ * You should have received a copy of the GNU General Public License\r
++ * along with this program.  If not, see https://www.gnu.org/licenses/ .\r
++ *\r
++ * Author: Austin Clements <aclements@csail.mit.edu>\r
++ *                David Bremner <david@tethera.net>\r
++ */\r
++\r
++#include "regexp-ps.h"\r
++\r
++RegexpPostingSource::RegexpPostingSource (Xapian::valueno slot, const std::string &regexp)\r
++    : slot_ (slot)\r
++{\r
++    int r = regcomp (&regexp_, regexp.c_str (), REG_EXTENDED | REG_NOSUB);\r
++\r
++    if (r != 0)\r
++      /* XXX Report a query syntax error using regerror */\r
++      throw "regcomp failed";\r
++}\r
++\r
++RegexpPostingSource::~RegexpPostingSource ()\r
++{\r
++    regfree (&regexp_);\r
++}\r
++\r
++void\r
++RegexpPostingSource::init (const Xapian::Database &db)\r
++{\r
++    db_ = db;\r
++    it_ = db_.valuestream_begin (slot_);\r
++    end_ = db.valuestream_end (slot_);\r
++    started_ = false;\r
++}\r
++\r
++Xapian::doccount\r
++RegexpPostingSource::get_termfreq_min () const\r
++{\r
++    return 0;\r
++}\r
++\r
++Xapian::doccount\r
++RegexpPostingSource::get_termfreq_est () const\r
++{\r
++    return get_termfreq_max () / 2;\r
++}\r
++\r
++Xapian::doccount\r
++RegexpPostingSource::get_termfreq_max () const\r
++{\r
++    return db_.get_value_freq (slot_);\r
++}\r
++\r
++Xapian::docid\r
++RegexpPostingSource::get_docid () const\r
++{\r
++    return it_.get_docid ();\r
++}\r
++\r
++bool\r
++RegexpPostingSource::at_end () const\r
++{\r
++    return it_ == end_;\r
++}\r
++\r
++void\r
++RegexpPostingSource::next (unused (double min_wt))\r
++{\r
++    if (started_ && ! at_end ())\r
++      ++it_;\r
++    started_ = true;\r
++\r
++    for (; ! at_end (); ++it_) {\r
++      std::string value = *it_;\r
++      if (regexec (&regexp_, value.c_str (), 0, NULL, 0) == 0)\r
++          break;\r
++    }\r
++}\r
+diff --git a/lib/regexp-ps.h b/lib/regexp-ps.h\r
+new file mode 100644\r
+index 0000000..a4553a7\r
+--- /dev/null\r
++++ b/lib/regexp-ps.h\r
+@@ -0,0 +1,37 @@\r
++#ifndef NOTMUCH_REGEX_PS_H\r
++#define NOTMUCH_REGEX_PS_H\r
++\r
++#include <sys/types.h>\r
++#include <regex.h>\r
++#include <xapian.h>\r
++#include "notmuch-private.h"\r
++\r
++/* A posting source that returns documents where a value matches a\r
++ * regexp.\r
++ */\r
++class RegexpPostingSource : public Xapian::PostingSource\r
++{\r
++protected:\r
++const Xapian::valueno slot_;\r
++regex_t regexp_;\r
++Xapian::Database db_;\r
++bool started_;\r
++Xapian::ValueIterator it_, end_;\r
++\r
++/* No copying */\r
++RegexpPostingSource (const RegexpPostingSource &);\r
++RegexpPostingSource &operator= (const RegexpPostingSource &);\r
++\r
++public:\r
++ RegexpPostingSource (Xapian::valueno slot, const std::string &regexp);\r
++~RegexpPostingSource ();\r
++void init (const Xapian::Database &db);\r
++Xapian::doccount get_termfreq_min () const;\r
++Xapian::doccount get_termfreq_est () const;\r
++Xapian::doccount get_termfreq_max () const;\r
++Xapian::docid get_docid () const;\r
++bool at_end () const;\r
++void next (unused (double min_wt));\r
++};\r
++\r
++#endif\r
+diff --git a/lib/subject-fp.cc b/lib/subject-fp.cc\r
+new file mode 100644\r
+index 0000000..1627721\r
+--- /dev/null\r
++++ b/lib/subject-fp.cc\r
+@@ -0,0 +1,41 @@\r
++/* subject-fp.cc - "subject:" field processor glue\r
++ *\r
++ * This file is part of notmuch.\r
++ *\r
++ * Copyright © 2016 David Bremner\r
++ *\r
++ * This program is free software: you can redistribute it and/or modify\r
++ * it under the terms of the GNU General Public License as published by\r
++ * the Free Software Foundation, either version 3 of the License, or\r
++ * (at your option) any later version.\r
++ *\r
++ * This program is distributed in the hope that it will be useful,\r
++ * but WITHOUT ANY WARRANTY; without even the implied warranty of\r
++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\r
++ * GNU General Public License for more details.\r
++ *\r
++ * You should have received a copy of the GNU General Public License\r
++ * along with this program.  If not, see https://www.gnu.org/licenses/ .\r
++ *\r
++ * Author: David Bremner <david@tethera.net>\r
++ */\r
++\r
++#include "database-private.h"\r
++#include "subject-fp.h"\r
++#include <iostream>\r
++\r
++#if HAVE_XAPIAN_FIELD_PROCESSOR\r
++\r
++Xapian::Query\r
++SubjectFieldProcessor::operator() (const std::string & str)\r
++{\r
++    std::string prefix = "rx:";\r
++\r
++    if (str.compare(0,prefix.size(),prefix)==0) {\r
++      postings = new RegexpPostingSource(NOTMUCH_VALUE_SUBJECT, str.substr(prefix.size()));\r
++      return Xapian::Query(postings);\r
++    } else {\r
++      return parser.parse_query (str, NOTMUCH_QUERY_PARSER_FLAGS, _find_prefix ("subject"));\r
++    }\r
++}\r
++#endif\r
+diff --git a/lib/subject-fp.h b/lib/subject-fp.h\r
+new file mode 100644\r
+index 0000000..ca622ba\r
+--- /dev/null\r
++++ b/lib/subject-fp.h\r
+@@ -0,0 +1,43 @@\r
++/* subject-fp.h - subject field processor glue\r
++ *\r
++ * This file is part of notmuch.\r
++ *\r
++ * Copyright © 2016 David Bremner\r
++ *\r
++ * This program is free software: you can redistribute it and/or modify\r
++ * it under the terms of the GNU General Public License as published by\r
++ * the Free Software Foundation, either version 3 of the License, or\r
++ * (at your option) any later version.\r
++ *\r
++ * This program is distributed in the hope that it will be useful,\r
++ * but WITHOUT ANY WARRANTY; without even the implied warranty of\r
++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\r
++ * GNU General Public License for more details.\r
++ *\r
++ * You should have received a copy of the GNU General Public License\r
++ * along with this program.  If not, see https://www.gnu.org/licenses/ .\r
++ *\r
++ * Author: David Bremner <david@tethera.net>\r
++ */\r
++\r
++#ifndef NOTMUCH_SUBJECT_FP_H\r
++#define NOTMUCH_SUBJECT_FP_H\r
++\r
++#include <xapian.h>\r
++#include "notmuch.h"\r
++#include "regexp-ps.h"\r
++\r
++#if HAVE_XAPIAN_FIELD_PROCESSOR\r
++class SubjectFieldProcessor : public Xapian::FieldProcessor {\r
++ protected:\r
++    Xapian::QueryParser &parser;\r
++    notmuch_database_t *notmuch;\r
++    RegexpPostingSource *postings = NULL;\r
++ public:\r
++    SubjectFieldProcessor (Xapian::QueryParser &parser_, notmuch_database_t *notmuch_)\r
++      : parser(parser_), notmuch(notmuch_) { };\r
++\r
++    Xapian::Query operator()(const std::string & str);\r
++};\r
++#endif\r
++#endif /* NOTMUCH_SUBJECT_FP_H */\r
+-- \r
+2.8.1\r
+\r