From a18b9b63e9df0e43307fd03c4779e702bf70e909 Mon Sep 17 00:00:00 2001 From: Michal Sojka Date: Mon, 27 Oct 2014 15:50:54 +0100 Subject: [PATCH] [PATCH v4 5/6] cli: search: Add configurable way to filter out duplicate addresses --- 66/2bae04c774f059bd142c6c31658092b809518a | 545 ++++++++++++++++++++++ 1 file changed, 545 insertions(+) create mode 100644 66/2bae04c774f059bd142c6c31658092b809518a diff --git a/66/2bae04c774f059bd142c6c31658092b809518a b/66/2bae04c774f059bd142c6c31658092b809518a new file mode 100644 index 000000000..7b425a65b --- /dev/null +++ b/66/2bae04c774f059bd142c6c31658092b809518a @@ -0,0 +1,545 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id 0FC08429E25 + for ; Mon, 27 Oct 2014 07:51:27 -0700 (PDT) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Spam-Flag: NO +X-Spam-Score: -2.3 +X-Spam-Level: +X-Spam-Status: No, score=-2.3 tagged_above=-999 required=5 + tests=[RCVD_IN_DNSWL_MED=-2.3] autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id 4iANC6I8aJqI for ; + Mon, 27 Oct 2014 07:51:19 -0700 (PDT) +Received: from max.feld.cvut.cz (max.feld.cvut.cz [147.32.192.36]) + by olra.theworths.org (Postfix) with ESMTP id 8DB89431FDB + for ; Mon, 27 Oct 2014 07:51:14 -0700 (PDT) +Received: from localhost (unknown [192.168.200.7]) + by max.feld.cvut.cz (Postfix) with ESMTP id 0683A5CCF9D + for ; Mon, 27 Oct 2014 15:51:12 +0100 (CET) +X-Virus-Scanned: IMAP STYX AMAVIS +Received: from max.feld.cvut.cz ([192.168.200.1]) + by localhost (styx.feld.cvut.cz [192.168.200.7]) (amavisd-new, + port 10044) with ESMTP id 6W_eysLIWzpe for ; + Mon, 27 Oct 2014 15:51:08 +0100 (CET) +Received: from imap.feld.cvut.cz (imap.feld.cvut.cz [147.32.192.34]) + by max.feld.cvut.cz (Postfix) with ESMTP id 9A4A35CCF92 + for ; Mon, 27 Oct 2014 15:51:07 +0100 (CET) +Received: from wsh by steelpick.2x.cz with local (Exim 4.84) + (envelope-from ) + id 1XildT-0000os-JS; Mon, 27 Oct 2014 15:51:07 +0100 +From: Michal Sojka +To: notmuch@notmuchmail.org +Subject: [PATCH v4 5/6] cli: search: Add configurable way to filter out + duplicate addresses +Date: Mon, 27 Oct 2014 15:50:54 +0100 +Message-Id: <1414421455-3037-6-git-send-email-sojkam1@fel.cvut.cz> +X-Mailer: git-send-email 2.1.1 +In-Reply-To: <1414421455-3037-1-git-send-email-sojkam1@fel.cvut.cz> +References: <1414421455-3037-1-git-send-email-sojkam1@fel.cvut.cz> +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Mon, 27 Oct 2014 14:51:27 -0000 + +This adds an algorithm to filter out duplicate addresses from address +outputs (sender, receivers). The algorithm can be configured with +--filter-by command line option. + +The code here is an extended version of a patch from Jani Nikula. +--- + completion/notmuch-completion.bash | 6 ++- + completion/notmuch-completion.zsh | 3 +- + doc/man1/notmuch-search.rst | 38 +++++++++++++++ + notmuch-search.c | 98 +++++++++++++++++++++++++++++++++++--- + test/T090-search-output.sh | 87 +++++++++++++++++++++++++++++++++ + test/T095-search-filter-by.sh | 64 +++++++++++++++++++++++++ + 6 files changed, 288 insertions(+), 8 deletions(-) + create mode 100755 test/T095-search-filter-by.sh + +diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash +index cfbd389..6b6d43a 100644 +--- a/completion/notmuch-completion.bash ++++ b/completion/notmuch-completion.bash +@@ -305,12 +305,16 @@ _notmuch_search() + COMPREPLY=( $( compgen -W "true false flag all" -- "${cur}" ) ) + return + ;; ++ --filter-by) ++ COMPREPLY=( $( compgen -W "nameaddr name addr addrfold nameaddrfold" -- "${cur}" ) ) ++ return ++ ;; + esac + + ! $split && + case "${cur}" in + -*) +- local options="--format= --output= --sort= --offset= --limit= --exclude= --duplicate=" ++ local options="--format= --output= --sort= --offset= --limit= --exclude= --duplicate= --filter-by=" + compopt -o nospace + COMPREPLY=( $(compgen -W "$options" -- ${cur}) ) + ;; +diff --git a/completion/notmuch-completion.zsh b/completion/notmuch-completion.zsh +index 3e52a00..3e535df 100644 +--- a/completion/notmuch-completion.zsh ++++ b/completion/notmuch-completion.zsh +@@ -53,7 +53,8 @@ _notmuch_search() + '--max-threads=[display only the first x threads from the search results]:number of threads to show: ' \ + '--first=[omit the first x threads from the search results]:number of threads to omit: ' \ + '--sort=[sort results]:sorting:((newest-first\:"reverse chronological order" oldest-first\:"chronological order"))' \ +- '--output=[select what to output]:output:((summary threads messages files tags sender recipients))' ++ '--output=[select what to output]:output:((summary threads messages files tags sender recipients))' \ ++ '--filter-by=[filter out duplicate addresses]:filter-by:((nameaddr\:"both name and address part" name\:"name part" addr\:"address part" addrfold\:"case-insensitive address part" nameaddrfold\:"name and case-insensitive address part"))' + } + + _notmuch() +diff --git a/doc/man1/notmuch-search.rst b/doc/man1/notmuch-search.rst +index b6607c9..84af2da 100644 +--- a/doc/man1/notmuch-search.rst ++++ b/doc/man1/notmuch-search.rst +@@ -85,6 +85,9 @@ Supported options for **search** include + (--format=text0), as a JSON array (--format=json), or as + an S-Expression list (--format=sexp). + ++ Duplicate addresses are filtered out. Filtering can be ++ configured with the --filter-by option. ++ + Note: Searching for **sender** should be much faster than + searching for **recipients**, because sender addresses are + cached directly in the database whereas other addresses +@@ -151,6 +154,41 @@ Supported options for **search** include + prefix. The prefix matches messages based on filenames. This + option filters filenames of the matching messages. + ++ ``--filter-by=``\ (**nameaddr**\ \|\ **name** \|\ **addr**\ \|\ **addrfold**\ \|\ **nameaddrfold**\) ++ ++ Can be used with ``--output=sender`` or ++ ``--output=recipients`` to filter out duplicate addresses. The ++ filtering algorithm receives a sequence of email addresses and ++ outputs the same sequence without the addresses that are ++ considered a duplicate of a previously output address. What is ++ considered a duplicate depends on how the two addresses are ++ compared and this can be controlled with the follwing flags: ++ ++ **nameaddr** means that both name and address parts are ++ compared in case-sensitive manner. Therefore, all same looking ++ addresses strings are considered duplicate. This is the ++ default. ++ ++ **name** means that only the name part is compared (in ++ case-sensitive manner). For example, the addresses "John Doe ++ " and "John Doe " will be ++ considered duplicate. ++ ++ **addr** means that only the address part is compared (in ++ case-sensitive manner). For example, the addresses "John Doe ++ " and "Dr. John Doe " will ++ be considered duplicate. ++ ++ **addrfold** is like **addr**, but comparison is done in ++ canse-insensitive manner. For example, the addresses "John Doe ++ " and "Dr. John Doe " will ++ be considered duplicate. ++ ++ **nameaddrfold** is like **nameaddr**, but address comparison ++ is done in canse-insensitive manner. For example, the ++ addresses "John Doe " and "John Doe ++ " will be considered duplicate. ++ + EXIT STATUS + =========== + +diff --git a/notmuch-search.c b/notmuch-search.c +index ce3bfb2..47aa979 100644 +--- a/notmuch-search.c ++++ b/notmuch-search.c +@@ -34,6 +34,14 @@ typedef enum { + + #define OUTPUT_ADDRESS_FLAGS (OUTPUT_SENDER | OUTPUT_RECIPIENTS) + ++typedef enum { ++ FILTER_BY_NAMEADDR = 0, ++ FILTER_BY_NAME, ++ FILTER_BY_ADDR, ++ FILTER_BY_ADDRFOLD, ++ FILTER_BY_NAMEADDRFOLD, ++} filter_by_t; ++ + typedef struct { + sprinter_t *format; + notmuch_query_t *query; +@@ -42,6 +50,7 @@ typedef struct { + int offset; + int limit; + int dupe; ++ filter_by_t filter_by; + } search_options_t; + + typedef struct { +@@ -229,6 +238,52 @@ do_search_threads (search_options_t *opt) + return 0; + } + ++/* Returns TRUE iff name and/or addr is considered duplicite. */ ++static notmuch_bool_t ++check_duplicite (const search_options_t *opt, GHashTable *addrs, const char *name, const char *addr) ++{ ++ notmuch_bool_t duplicite; ++ char *key; ++ ++ if (opt->filter_by == FILTER_BY_ADDRFOLD || ++ opt->filter_by == FILTER_BY_NAMEADDRFOLD) { ++ gchar *folded = g_utf8_casefold (addr, -1); ++ addr = talloc_strdup (opt->format, folded); ++ g_free (folded); ++ } ++ switch (opt->filter_by) { ++ case FILTER_BY_NAMEADDR: ++ case FILTER_BY_NAMEADDRFOLD: ++ key = talloc_asprintf (opt->format, "%s <%s>", name, addr); ++ break; ++ case FILTER_BY_NAME: ++ key = talloc_strdup (opt->format, name); /* !name results in !key */ ++ break; ++ case FILTER_BY_ADDR: ++ case FILTER_BY_ADDRFOLD: ++ key = talloc_strdup (opt->format, addr); ++ break; ++ default: ++ INTERNAL_ERROR("invalid --filter-by flags"); ++ } ++ ++ if (opt->filter_by == FILTER_BY_ADDRFOLD || ++ opt->filter_by == FILTER_BY_NAMEADDRFOLD) ++ talloc_free ((char*)addr); ++ ++ if (! key) ++ return FALSE; ++ ++ duplicite = g_hash_table_lookup_extended (addrs, key, NULL, NULL); ++ ++ if (! duplicite) ++ g_hash_table_insert (addrs, key, NULL); ++ else ++ talloc_free (key); ++ ++ return duplicite; ++} ++ + static void + print_mailbox (const search_options_t *opt, const mailbox_t *mailbox) + { +@@ -263,7 +318,8 @@ print_mailbox (const search_options_t *opt, const mailbox_t *mailbox) + } + + static void +-process_address_list (const search_options_t *opt, InternetAddressList *list) ++process_address_list (const search_options_t *opt, GHashTable *addrs, ++ InternetAddressList *list) + { + InternetAddress *address; + int i; +@@ -279,7 +335,7 @@ process_address_list (const search_options_t *opt, InternetAddressList *list) + if (group_list == NULL) + continue; + +- process_address_list (opt, group_list); ++ process_address_list (opt, addrs, group_list); + } else { + InternetAddressMailbox *mailbox = INTERNET_ADDRESS_MAILBOX (address); + mailbox_t mbx = { +@@ -287,13 +343,16 @@ process_address_list (const search_options_t *opt, InternetAddressList *list) + .addr = internet_address_mailbox_get_addr (mailbox), + }; + ++ if (check_duplicite (opt, addrs, mbx.name, mbx.addr)) ++ continue; ++ + print_mailbox (opt, &mbx); + } + } + } + + static void +-process_address_header (const search_options_t *opt, const char *value) ++process_address_header (const search_options_t *opt, GHashTable *addrs, const char *value) + { + InternetAddressList *list; + +@@ -304,7 +363,13 @@ process_address_header (const search_options_t *opt, const char *value) + if (list == NULL) + return; + +- process_address_list (opt, list); ++ process_address_list (opt, addrs, list); ++} ++ ++static void ++_my_talloc_free_for_g_hash (void *ptr) ++{ ++ talloc_free (ptr); + } + + static int +@@ -314,8 +379,13 @@ do_search_messages (search_options_t *opt) + notmuch_messages_t *messages; + notmuch_filenames_t *filenames; + sprinter_t *format = opt->format; ++ GHashTable *addresses = NULL; + int i; + ++ if (opt->output & OUTPUT_ADDRESS_FLAGS) ++ addresses = g_hash_table_new_full (g_str_hash, g_str_equal, ++ _my_talloc_free_for_g_hash, NULL); ++ + if (opt->offset < 0) { + opt->offset += notmuch_query_count_messages (opt->query); + if (opt->offset < 0) +@@ -363,7 +433,7 @@ do_search_messages (search_options_t *opt) + const char *addrs; + + addrs = notmuch_message_get_header (message, "from"); +- process_address_header (opt, addrs); ++ process_address_header (opt, addresses, addrs); + } + + if (opt->output & OUTPUT_RECIPIENTS) { +@@ -373,7 +443,7 @@ do_search_messages (search_options_t *opt) + + for (j = 0; j < ARRAY_SIZE (hdrs); j++) { + addrs = notmuch_message_get_header (message, hdrs[j]); +- process_address_header (opt, addrs); ++ process_address_header (opt, addresses, addrs); + } + } + } +@@ -381,6 +451,9 @@ do_search_messages (search_options_t *opt) + notmuch_message_destroy (message); + } + ++ if (addresses) ++ g_hash_table_unref (addresses); ++ + notmuch_messages_destroy (messages); + + format->end (format); +@@ -447,6 +520,7 @@ notmuch_search_command (notmuch_config_t *config, int argc, char *argv[]) + .offset = 0, + .limit = -1, /* unlimited */ + .dupe = -1, ++ .filter_by = FILTER_BY_NAMEADDR, + }; + char *query_str; + int opt_index, ret; +@@ -490,6 +564,13 @@ notmuch_search_command (notmuch_config_t *config, int argc, char *argv[]) + { NOTMUCH_OPT_INT, &opt.offset, "offset", 'O', 0 }, + { NOTMUCH_OPT_INT, &opt.limit, "limit", 'L', 0 }, + { NOTMUCH_OPT_INT, &opt.dupe, "duplicate", 'D', 0 }, ++ { NOTMUCH_OPT_KEYWORD, &opt.filter_by, "filter-by", 'b', ++ (notmuch_keyword_t []){ { "nameaddr", FILTER_BY_NAMEADDR }, ++ { "name", FILTER_BY_NAME }, ++ { "addr", FILTER_BY_ADDR }, ++ { "addrfold", FILTER_BY_ADDRFOLD }, ++ { "nameaddrfold", FILTER_BY_NAMEADDRFOLD }, ++ { 0, 0 } } }, + { 0, 0, 0, 0, 0 } + }; + +@@ -500,6 +581,11 @@ notmuch_search_command (notmuch_config_t *config, int argc, char *argv[]) + if (! opt.output) + opt.output = OUTPUT_SUMMARY; + ++ if (opt.filter_by && !(opt.output & OUTPUT_ADDRESS_FLAGS)) { ++ fprintf (stderr, "Error: --filter-by can only be used with address output.\n"); ++ return EXIT_FAILURE; ++ } ++ + switch (format_sel) { + case NOTMUCH_FORMAT_TEXT: + opt.format = sprinter_text_create (config, stdout); +diff --git a/test/T090-search-output.sh b/test/T090-search-output.sh +index 947d572..841a721 100755 +--- a/test/T090-search-output.sh ++++ b/test/T090-search-output.sh +@@ -387,6 +387,93 @@ cat <EXPECTED + EOF + test_expect_equal_file OUTPUT EXPECTED + ++test_begin_subtest "--output=sender" ++notmuch search --output=sender '*' >OUTPUT ++cat <EXPECTED ++François Boulogne ++Olivier Berger ++Chris Wilson ++Carl Worth ++Alexander Botero-Lowry ++Keith Packard ++Jjgod Jiang ++Rolland Santimano ++Jan Janak ++Stewart Smith ++Lars Kellogg-Stedman ++Alex Botero-Lowry ++Ingmar Vanhassel ++Aron Griffis ++Adrian Perez de Castro ++Israel Herraiz ++Mikhail Gusarov ++EOF ++test_expect_equal_file OUTPUT EXPECTED ++ ++test_begin_subtest "--output=sender --format=json" ++notmuch search --output=sender --format=json '*' >OUTPUT ++cat <EXPECTED ++[{"name": "François Boulogne", "address": "boulogne.f@gmail.com"}, ++{"name": "Olivier Berger", "address": "olivier.berger@it-sudparis.eu"}, ++{"name": "Chris Wilson", "address": "chris@chris-wilson.co.uk"}, ++{"name": "Carl Worth", "address": "cworth@cworth.org"}, ++{"name": "Alexander Botero-Lowry", "address": "alex.boterolowry@gmail.com"}, ++{"name": "Keith Packard", "address": "keithp@keithp.com"}, ++{"name": "Jjgod Jiang", "address": "gzjjgod@gmail.com"}, ++{"name": "Rolland Santimano", "address": "rollandsantimano@yahoo.com"}, ++{"name": "Jan Janak", "address": "jan@ryngle.com"}, ++{"name": "Stewart Smith", "address": "stewart@flamingspork.com"}, ++{"name": "Lars Kellogg-Stedman", "address": "lars@seas.harvard.edu"}, ++{"name": "Alex Botero-Lowry", "address": "alex.boterolowry@gmail.com"}, ++{"name": "Ingmar Vanhassel", "address": "ingmar@exherbo.org"}, ++{"name": "Aron Griffis", "address": "agriffis@n01se.net"}, ++{"name": "Adrian Perez de Castro", "address": "aperez@igalia.com"}, ++{"name": "Israel Herraiz", "address": "isra@herraiz.org"}, ++{"name": "Mikhail Gusarov", "address": "dottedmag@dottedmag.net"}] ++EOF ++test_expect_equal_file OUTPUT EXPECTED ++ ++test_begin_subtest "--output=recipients" ++notmuch search --output=recipients '*' >OUTPUT ++cat <EXPECTED ++Allan McRae ++Discussion about the Arch User Repository (AUR) ++olivier.berger@it-sudparis.eu ++notmuch@notmuchmail.org ++notmuch ++Keith Packard ++Mikhail Gusarov ++EOF ++test_expect_equal_file OUTPUT EXPECTED ++ ++test_begin_subtest "--output=sender --output=recipients" ++notmuch search --output=sender --output=recipients '*' >OUTPUT ++cat <EXPECTED ++François Boulogne ++Allan McRae ++Discussion about the Arch User Repository (AUR) ++Olivier Berger ++olivier.berger@it-sudparis.eu ++Chris Wilson ++notmuch@notmuchmail.org ++Carl Worth ++Alexander Botero-Lowry ++Keith Packard ++Jjgod Jiang ++Rolland Santimano ++Jan Janak ++Stewart Smith ++Lars Kellogg-Stedman ++notmuch ++Alex Botero-Lowry ++Ingmar Vanhassel ++Aron Griffis ++Adrian Perez de Castro ++Israel Herraiz ++Mikhail Gusarov ++EOF ++test_expect_equal_file OUTPUT EXPECTED ++ + test_begin_subtest "sanitize output for quoted-printable line-breaks in author and subject" + add_message "[subject]='two =?ISO-8859-1?Q?line=0A_subject?= + headers'" +diff --git a/test/T095-search-filter-by.sh b/test/T095-search-filter-by.sh +new file mode 100755 +index 0000000..97d9a9b +--- /dev/null ++++ b/test/T095-search-filter-by.sh +@@ -0,0 +1,64 @@ ++#!/usr/bin/env bash ++test_description='duplicite address filtering in "notmuch search --output=recipients"' ++. ./test-lib.sh ++ ++add_message '[to]="Real Name , Real Name "' ++add_message '[to]="Nickname "' '[cc]="Real Name "' ++add_message '[to]="Nickname "' '[bcc]="Real Name "' ++ ++test_begin_subtest "--output=recipients" ++notmuch search --output=recipients "*" >OUTPUT ++cat <EXPECTED ++Real Name ++Real Name ++Nickname ++Real Name ++EOF ++test_expect_equal_file OUTPUT EXPECTED ++ ++test_begin_subtest "--output=recipients --filter-by=nameaddr" ++notmuch search --output=recipients --filter-by=nameaddr "*" >OUTPUT ++# The same as above ++cat <EXPECTED ++Real Name ++Real Name ++Nickname ++Real Name ++EOF ++test_expect_equal_file OUTPUT EXPECTED ++ ++test_begin_subtest "--output=recipients --filter-by=name" ++notmuch search --output=recipients --filter-by=name "*" >OUTPUT ++cat <EXPECTED ++Real Name ++Nickname ++EOF ++test_expect_equal_file OUTPUT EXPECTED ++ ++test_begin_subtest "--output=recipients --filter-by=addr" ++notmuch search --output=recipients --filter-by=addr "*" >OUTPUT ++cat <EXPECTED ++Real Name ++Real Name ++Real Name ++EOF ++test_expect_equal_file OUTPUT EXPECTED ++ ++test_begin_subtest "--output=recipients --filter-by=addrfold" ++notmuch search --output=recipients --filter-by=addrfold "*" >OUTPUT ++cat <EXPECTED ++Real Name ++Real Name ++EOF ++test_expect_equal_file OUTPUT EXPECTED ++ ++test_begin_subtest "--output=recipients --filter-by=nameaddrfold" ++notmuch search --output=recipients --filter-by=nameaddrfold "*" >OUTPUT ++cat <EXPECTED ++Real Name ++Real Name ++Nickname ++EOF ++test_expect_equal_file OUTPUT EXPECTED ++ ++test_done +-- +2.1.1 + -- 2.26.2