From f69215d41ff89998cac1ac0d7d13116d5aeb5276 Mon Sep 17 00:00:00 2001 From: Carl Worth Date: Tue, 13 Oct 2009 15:56:16 -0700 Subject: [PATCH] notmuch-index-message: Add code to actually create a Xapian index Most of this code is fairly clean and works well. One part is fairly painful---namely extracting the body of an email message from libgmime. Currently, I'm just extracting the offset to the end of the headers, and then separately opening the message. Surely there's a better way. Anyway, with that the results are looking very similar to sup-sync now, (as verified by xapian-dump). The only substantial difference I'm seeing now is that sup does not seem to index quoted portions of messages nor signatures. I'm not actually sure whether I want to follow sup's lead in that or not. --- notmuch-index-message.cc | 204 ++++++++++++++++++++++++++++----------- 1 file changed, 150 insertions(+), 54 deletions(-) diff --git a/notmuch-index-message.cc b/notmuch-index-message.cc index cf99e6e4..39be64ec 100644 --- a/notmuch-index-message.cc +++ b/notmuch-index-message.cc @@ -23,8 +23,14 @@ #include #include +#include + #include +#include + +using namespace std; + #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0])) /* These prefix values are specifically chosen to be compatible @@ -61,6 +67,15 @@ prefix_t BOOLEAN_PREFIX[] = { { "ref", "R" } }; +/* Similarly, these value numbers are also chosen to be sup + * compatible. */ + +typedef enum { + NOTMUCH_VALUE_MESSAGE_ID = 0, + NOTMUCH_VALUE_THREAD = 1, + NOTMUCH_VALUE_DATE = 2 +} notmuch_value_t; + static const char * find_prefix (const char *name) { @@ -80,48 +95,73 @@ find_prefix (const char *name) int TERM_COMBINED = 0; static void -print_term (const char *prefix_name, const char *value) +add_term (Xapian::Document doc, + const char *prefix_name, + const char *value) { const char *prefix; + char *term; if (value == NULL) return; prefix = find_prefix (prefix_name); - if (TERM_COMBINED) - printf ("\"%s%s\", ", prefix, value); - else - printf ("[\"%s\", \"%s\"], ", value, prefix); + term = g_strdup_printf ("%s%s", prefix, value); + + doc.add_term (term); + + g_free (term); +} + +static void +gen_terms (Xapian::TermGenerator term_gen, + const char *prefix_name, + const char *text) +{ + const char *prefix; + + if (text == NULL) + return; + + prefix = find_prefix (prefix_name); + + term_gen.index_text (text, 1, prefix); } static void -add_address_name (InternetAddress *address, const char *prefix_name) +gen_terms_address_name (Xapian::TermGenerator term_gen, + InternetAddress *address, + const char *prefix_name) { const char *name; name = internet_address_get_name (address); if (name) - print_term (prefix_name, name); + gen_terms (term_gen, prefix_name, name); } static void -add_address_names (InternetAddressList *addresses, const char *address_type) +gen_terms_address_names (Xapian::TermGenerator term_gen, + InternetAddressList *addresses, + const char *address_type) { int i; InternetAddress *address; for (i = 0; i < internet_address_list_length (addresses); i++) { address = internet_address_list_get_address (addresses, i); - add_address_name (address, address_type); - add_address_name (address, "name"); - add_address_name (address, "body"); + gen_terms_address_name (term_gen, address, address_type); + gen_terms_address_name (term_gen, address, "name"); + gen_terms_address_name (term_gen, address, "body"); } } static void -add_address_addr (InternetAddress *address, const char *prefix_name) +add_term_address_addr (Xapian::Document doc, + InternetAddress *address, + const char *prefix_name) { InternetAddressMailbox *mailbox = INTERNET_ADDRESS_MAILBOX (address); const char *addr; @@ -129,19 +169,21 @@ add_address_addr (InternetAddress *address, const char *prefix_name) addr = internet_address_mailbox_get_addr (mailbox); if (addr) - print_term (prefix_name, addr); + add_term (doc, prefix_name, addr); } static void -add_address_addrs (InternetAddressList *addresses, const char *address_type) +add_terms_address_addrs (Xapian::Document doc, + InternetAddressList *addresses, + const char *address_type) { int i; InternetAddress *address; for (i = 0; i < internet_address_list_length (addresses); i++) { address = internet_address_list_get_address (addresses, i); - add_address_addr (address, address_type); - add_address_addr (address, "email"); + add_term_address_addr (doc, address, address_type); + add_term_address_addr (doc, address, "email"); } } @@ -152,23 +194,29 @@ main (int argc, char **argv) GMimeParser *parser; GMimeMessage *message; InternetAddressList *addresses; + GIOChannel *channel; + GIOStatus gio_status; + GError *error = NULL; - const char *filename; + const char *database_path, *filename; FILE *file; + gint64 body_offset; + char *body_str; const char *value, *from; time_t time; struct tm gm_time_tm; - char time_str[16]; /* YYYYMMDDHHMMSS + 1 for Y100k compatibility ;-) */ + char date_str[16]; /* YYYYMMDDHHMMSS + 1 for Y100k compatibility ;-) */ - if (argc < 2) { - fprintf (stderr, "Usage: %s \n", + if (argc < 3) { + fprintf (stderr, "Usage: %s \n", argv[0]); exit (1); } - filename = argv[1]; + database_path = argv[1]; + filename = argv[2]; file = fopen (filename, "r"); if (! file) { @@ -184,56 +232,104 @@ main (int argc, char **argv) message = g_mime_parser_construct_message (parser); - printf ("text is:\n["); - from = g_mime_message_get_sender (message); - addresses = internet_address_list_parse_string (from); + try { + Xapian::WritableDatabase db; + Xapian::TermGenerator term_gen; + Xapian::Document doc; - add_address_names (addresses, "from_name"); + doc = Xapian::Document (); - add_address_names (g_mime_message_get_all_recipients (message), - "to_name"); + doc.set_data (filename); - value = g_mime_message_get_subject (message); - print_term ("subject", value); - print_term ("body", value); + db = Xapian::WritableDatabase (database_path, + Xapian::DB_CREATE_OR_OPEN); - printf ("]\nterms is:\n["); + term_gen = Xapian::TermGenerator (); + term_gen.set_stemmer (Xapian::Stem ("english")); - TERM_COMBINED = 1; + term_gen.set_document (doc); - from = g_mime_message_get_sender (message); - addresses = internet_address_list_parse_string (from); + from = g_mime_message_get_sender (message); + addresses = internet_address_list_parse_string (from); - add_address_addrs (addresses, "from_email"); + gen_terms_address_names (term_gen, addresses, "from_name"); - add_address_addrs (g_mime_message_get_all_recipients (message), - "to_email"); + addresses = g_mime_message_get_all_recipients (message); + gen_terms_address_names (term_gen, addresses, "to_name"); - g_mime_message_get_date (message, &time, NULL); + value = g_mime_message_get_subject (message); + gen_terms (term_gen, "subject", value); + gen_terms (term_gen, "body", value); - gmtime_r (&time, &gm_time_tm); + body_offset = g_mime_parser_get_headers_end (parser); + channel = g_io_channel_new_file (filename, "r", &error); + if (channel == NULL) { + fprintf (stderr, "Error: %s\n", error->message); + exit (1); + } - if (strftime (time_str, sizeof (time_str), - "%Y%m%d%H%M%S", &gm_time_tm) == 0) { - fprintf (stderr, "Internal error formatting time\n"); - exit (1); - } + gio_status = g_io_channel_seek_position (channel, body_offset, + G_SEEK_SET, &error); + if (gio_status != G_IO_STATUS_NORMAL) { + fprintf (stderr, "Error: %s\n", error->message); + exit (1); + } + + gio_status = g_io_channel_read_to_end (channel, &body_str, + NULL, &error); + if (gio_status != G_IO_STATUS_NORMAL) { + fprintf (stderr, "Error: %s\n", error->message); + exit (1); + } + + gen_terms (term_gen, "body", body_str); + + g_free (body_str); + g_io_channel_close (channel); + + from = g_mime_message_get_sender (message); + addresses = internet_address_list_parse_string (from); + + add_terms_address_addrs (doc, addresses, "from_email"); + + add_terms_address_addrs (doc, + g_mime_message_get_all_recipients (message), + "to_email"); - print_term ("date", time_str); + g_mime_message_get_date (message, &time, NULL); - print_term ("label", "inbox"); - print_term ("label", "unread"); - print_term ("type", "mail"); + gmtime_r (&time, &gm_time_tm); - value = g_mime_message_get_message_id (message); - print_term ("msgid", value); + if (strftime (date_str, sizeof (date_str), + "%Y%m%d%H%M%S", &gm_time_tm) == 0) { + fprintf (stderr, "Internal error formatting time\n"); + exit (1); + } - print_term ("source_id", "1"); + add_term (doc, "date", date_str); - value = g_mime_message_get_message_id (message); - print_term ("thread", value); + add_term (doc, "label", "inbox"); + add_term (doc, "label", "unread"); + add_term (doc, "type", "mail"); - printf ("]\n"); + value = g_mime_message_get_message_id (message); + add_term (doc, "msgid", value); + + add_term (doc, "source_id", "1"); + + add_term (doc, "thread", value); + + doc.add_value (NOTMUCH_VALUE_MESSAGE_ID, value); + doc.add_value (NOTMUCH_VALUE_THREAD, value); + + doc.add_value (NOTMUCH_VALUE_DATE, Xapian::sortable_serialise (time)); + + db.add_document (doc); + + } catch (const Xapian::Error &error) { + cerr << "A Xapian exception occurred: " << error.get_msg () << endl; + exit (1); + } g_object_unref (message); g_object_unref (parser); -- 2.26.2