Sender: Michal Nazarewicz <mpn@google.com>
From: Michal Nazarewicz <mina86@mina86.com>
To: Dmitry Kurochkin <dmitry.kurochkin@gmail.com>, notmuch@notmuchmail.org
Subject: Re: [PATCH] Add notmuch-remove-duplicates.py script to contrib.
In-Reply-To: <1346784785-19746-1-git-send-email-dmitry.kurochkin@gmail.com>
Organization: http://mina86.com/
References: <1346784785-19746-1-git-send-email-dmitry.kurochkin@gmail.com>
User-Agent: Notmuch/0.14+2~g416b120 (http://notmuchmail.org) Emacs/24.2.50.1
	(x86_64-unknown-linux-gnu)
Face: iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAJFBMVEWbfGlUPDDHgE57V0jUupKjgIObY0PLrom9mH4dFRK4gmjPs41MxjOgAAACQElEQVQ4jW3TMWvbQBQHcBk1xE6WyALX1069oZBMlq+ouUwpEQQ6uRjttkWP4CmBgGM0BQLBdPFZYPsyFUo6uEtKDQ7oy/U96XR2Ux8ehH/89Z6enqxBcS7Lg81jmSuujrfCZcLI/TYYvbGj+jbgFpHJ/bqQAUISj8iLyu4LuFHJTosxsucO4jSDNE0Hq3hwK/ceQ5sx97b8LcUDsILfk+ovHkOIsMbBfg43VuQ5Ln9YAGCkUdKJoXR9EclFBhixy3EGVz1K6eEkhxCAkeMMnqoAhAKwhoUJkDrCqvbecaYINlFKSRS1i12VKH1XpUd4qxL876EkMcDvHj3s5RBajHHMlA5iK32e0C7VgG0RlzFPvoYHZLRmAC0BmNcBruhkE0KsMsbEc62ZwUJDxWUdMsMhVqovoT96i/DnX/ASvz/6hbCabELLk/6FF/8PNpPCGqcZTGFcBhhAaZZDbQPaAB3+KrWWy2XgbYDNIinkdWAFcCpraDE/knwe5DBqGmgzESl1p2E4MWAz0VUPgYYzmfWb9yS4vCvgsxJriNTHoIBz5YteBvg+VGISQWUqhMiByPIPpygeDBE6elD973xWwKkEiHZAHKjhuPsFnBuArrzxtakRcISv+XMIPl4aGBUJm8Emk7qBYU8IlgNEIpiJhk/No24jHwkKTFHDWfPniR4iw5vJaw2nzSjfq2zffcE/GDjRC2dn0J0XwPAbDL84TvaFCJEU4Oml9pRyEUhR3Cl2t01AoEjRbs0sYugp14/4X5n4pU4EHHnMAAAAAElFTkSuQmCC
Date: Tue, 04 Sep 2012 21:43:53 +0200
Message-ID: <xa1tligpk1za.fsf@mina86.com>
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="=-=-="
Precedence: list

--=-=-=
Content-Type: text/plain; charset=utf-8
Content-Transfer-Encoding: quoted-printable

On Tue, Sep 04 2012, Dmitry Kurochkin wrote:
> The script removes duplicate message files.  It takes no options.
>
> Files are assumed duplicates if their content is the same except for
> ignored headers.  Currently, the only ignored header is Received:.
> ---
>  contrib/notmuch-remove-duplicates.py |   95 ++++++++++++++++++++++++++++=
++++++
>  1 file changed, 95 insertions(+)
>  create mode 100755 contrib/notmuch-remove-duplicates.py
>
> diff --git a/contrib/notmuch-remove-duplicates.py b/contrib/notmuch-remov=
e-duplicates.py
> new file mode 100755
> index 0000000..dbe2e25
> --- /dev/null
> +++ b/contrib/notmuch-remove-duplicates.py
> @@ -0,0 +1,95 @@
> +#!/usr/bin/env python
> +
> +import sys
> +
> +IGNORED_HEADERS =3D [ "Received:" ]
> +
> +if len(sys.argv) !=3D 1:
> +    print "Usage: %s" % sys.argv[0]
> +    print
> +    print "The script removes duplicate message files.  Takes no options=
."
> +    print "Requires notmuch python module."
> +    print
> +    print "Files are assumed duplicates if their content is the same"
> +    print "except for the following headers: %s." % ", ".join(IGNORED_HE=
ADERS)
> +    exit(1)

It's much better put inside a main() function, which is than called only
if the script is run directly.

> +
> +import notmuch
> +import os
> +import time
> +
> +class MailComparator:
> +    """Checks if mail files are duplicates."""
> +    def __init__(self, filename):
> +        self.filename =3D filename
> +        self.mail =3D self.readFile(self.filename)
> +
> +    def isDuplicate(self, filename):
> +        return self.mail =3D=3D self.readFile(filename)
> +
> +    @staticmethod
> +    def readFile(filename):
> +        with open(filename) as f:
> +            data =3D ""
> +            while True:
> +                line =3D f.readline()
> +                for header in IGNORED_HEADERS:
> +                    if line.startswith(header):

Case of headers should be ignored, but this does not ignore it.

> +                        # skip header continuation lines
> +                        while True:
> +                            line =3D f.readline()
> +                            if len(line) =3D=3D 0 or line[0] not in [" "=
, "\t"]:
> +                                break
> +                        break

This will ignore line just after the ignored header.

> +                else:
> +                    data +=3D line
> +                    if line =3D=3D "\n":
> +                        break
> +            data +=3D f.read()
> +            return data
> +
> +db =3D notmuch.Database()
> +query =3D db.create_query('*')
> +print "Number of messages: %s" % query.count_messages()
> +
> +files_count =3D 0
> +for root, dirs, files in os.walk(db.get_path()):
> +    if not root.startswith(os.path.join(db.get_path(), ".notmuch/")):
> +        files_count +=3D len(files)
> +print "Number of files: %s" % files_count
> +print "Estimated number of duplicates: %s" % (files_count - query.count_=
messages())
> +
> +msgs =3D query.search_messages()
> +msg_count =3D 0
> +suspected_duplicates_count =3D 0
> +duplicates_count =3D 0
> +timestamp =3D time.time()
> +for msg in msgs:
> +    msg_count +=3D 1
> +    if len(msg.get_filenames()) > 1:
> +        filenames =3D msg.get_filenames()
> +        comparator =3D MailComparator(filenames.next())
> +        for filename in filenames:

Strictly speaking, you need to compare each file to each file, and not
just every file to the first file.

> +            if os.path.realpath(comparator.filename) =3D=3D os.path.real=
path(filename):
> +                print "Message '%s' has filenames pointing to the
> same file: '%s' '%s'" % (msg.get_message_id(), comparator.filename,
> filename)

So why aren't those removed?

> +            elif comparator.isDuplicate(filename):
> +                os.remove(filename)
> +                duplicates_count +=3D 1
> +            else:
> +                #print "Potential duplicates: %s" % msg.get_message_id()
> +                suspected_duplicates_count +=3D 1
> +
> +    new_timestamp =3D time.time()
> +    if new_timestamp - timestamp > 1:
> +        timestamp =3D new_timestamp
> +        sys.stdout.write("\rProcessed %s messages, removed %s duplicates=
..." % (msg_count, duplicates_count))
> +        sys.stdout.flush()
> +
> +print "\rFinished. Processed %s messages, removed %s duplicates." % (msg=
_count, duplicates_count)
> +if duplicates_count > 0:
> +    print "You might want to run 'notmuch new' now."
> +
> +if suspected_duplicates_count > 0:
> +    print
> +    print "Found %s messages with duplicate IDs but different content." =
% suspected_duplicates_count
> +    print "Perhaps we should ignore more headers."

Please consider the following instead (not tested):


#!/usr/bin/env python

import collections
import notmuch
import os
import re
import sys
import time


IGNORED_HEADERS =3D [ 'Received' ]


isIgnoredHeadersLine =3D re.compile(
    r'^(?:%s)\s*:' % '|'.join(IGNORED_HEADERS),
    re.IGNORECASE).search

doesStartWithWS =3D re.compile(r'^\s').search


def usage(argv0):
    print """Usage: %s [<query-string>]

The script removes duplicate message files.  Takes no options."
Requires notmuch python module."

Files are assumed duplicates if their content is the same"
except for the following headers: %s.""" % (argv0, ', '.join(IGNORED_HEADER=
S))


def readMailFile(filename):
    with open(filename) as fd:
        data =3D []
        skip_header =3D False
        for line in fd:
            if doesStartWithWS(line):
                if not skip_header:
                    data.append(line)
            elif isIgnoredHeadersLine(line):
                skip_header =3D True
            else:
                data.append(line)
                if line =3D=3D '\n':
                    break
        data.append(fd.read())
        return ''.join(data)


def dedupMessage(msg):
    filenames =3D msg.get_filenames()
    if len(filenames) <=3D 1:
        return (0, 0)

    realpaths =3D collections.defaultdict(list)
    contents =3D collections.defaultdict(list)
    for filename in filenames:
        real =3D os.path.realpath(filename)
        lst =3D realpaths[real]
        lst.append(filename)
        if len(lst) =3D=3D 1:
            contents[readMailFile(real)].append(real)

    duplicates =3D 0

    for filenames in contents.itervalues():
        if len(filenames) > 1:
            print 'Files with the same content:'
            print ' ', filenames.pop()
            duplicates +=3D len(filenames)
            for filename in filenames:
                del realpaths[filename]
            #     os.remane(filename)

    for real, filenames in realpaths.iteritems():
        if len(filenames) > 1:
            print 'Files pointing to the same message:'
            print ' ', filenames.pop()
            duplicates +=3D len(filenames)
            # for filename in filenames:
            #     os.remane(filename)

    return (duplicates, len(realpaths) - 1)


def dedupQuery(query):
    print 'Number of messages: %s' % query.count_messages()
    msg_count =3D 0
    suspected_count =3D 0
    duplicates_count =3D 0
    timestamp =3D time.time()
    msgs =3D query.search_messages()
    for msg in msgs:
        msg_count +=3D 1
        d, s =3D dedupMessage(msg)
        duplicates_count +=3D d
        suspected_count +=3D d

        new_timestamp =3D time.time()
        if new_timestamp - timestamp > 1:
            timestamp =3D new_timestamp
            sys.stdout.write('\rProcessed %s messages, removed %s duplicate=
s...'
                             % (msg_count, duplicates_count))
            sys.stdout.flush()

    print '\rFinished. Processed %s messages, removed %s duplicates.' % (
        msg_count, duplicates_count)
    if duplicates_count > 0:
        print 'You might want to run "notmuch new" now.'

    if suspected_duplicates_count > 0:
        print """
Found %d messages with duplicate IDs but different content.
Perhaps we should ignore more headers.""" % suspected_count


def main(argv):
    if len(argv) =3D=3D 1:
        query =3D '*'
    elif len(argv) =3D=3D 2:
        query =3D argv[1]
    else:
        usage(argv[0])
        return 1

    db =3D notmuch.Database()
    query =3D db.create_query(query)
    dedupQuery(db, query)
    return 0


if __name__ =3D=3D '__main__':
    sys.exit(main(sys.argv))



--=20
Best regards,                                         _     _
.o. | Liege of Serenely Enlightened Majesty of      o' \,=3D./ `o
..o | Computer Science,  Micha=C5=82 =E2=80=9Cmina86=E2=80=9D Nazarewicz   =
 (o o)
ooo +----<email/xmpp: mpn@google.com>--------------ooO--(_)--Ooo--
--=-=-=
Content-Type: multipart/signed; boundary="==-=-=";
	micalg=pgp-sha1; protocol="application/pgp-signature"

--==-=-=
Content-Type: text/plain


--==-=-=
Content-Type: application/pgp-signature

-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.10 (GNU/Linux)

iQIcBAEBAgAGBQJQRln5AAoJECBgQBJQdR/0EQQP/17AJmk0zYfSsuIz4I7H3Ykm
9YMt/yK1hn/u6yDtylSbnjTgWT0t6OWbOplIzmW9q6iqwhMcdqP20HXMEkqSbNLa
7WJ+4VXKRLb6PC3bQmM8eqYNtflgEAEWyAuZSQv9f893e6vH/e+7yoFtxaUypcUW
Wf8qm3T1ljle+2S1xGbteoVDUSHY5epesXlWR6hVA9Qclc/4xpVLNapx3EKRkxBh
vOpe+u5ATa04DYvIOoGVl723PBIHpm25cGen5lc8vOjXKwqhG0G7di5E29BhAyVT
yZorKrfsBRTTIlYEErakrzGhiMP3zRnCQmFWvIj/ASbiOUnX8ktFMjfqe+DNW3zq
T/2jpdzhBdVyioLhBIsMGLdsW6yIk3LURcw4uTijEG2ITj9kdQspydGFahTJk1Ly
cIls19AMCK7xfGBt8o3xYMX6v/bOxpz/Hot0e+SdHQtiByIUKfJMF7gMo6YyxRfh
cq1mgoLm+L4/zdrf4IMZDUpoMM8q4yr3eJibINlLxAmRnD3CpnVE2wf6mExLnYxy
PTIQ9p3pRHsxbRuHvYylJfNNlGpjsRFSgKeRF50iFY+TnzUh+40Tp18BTbL9Dd7R
UGuIMScxZ6qKt5MQhfBw1F+JpaaIsLTMSh1MjdzCvNVVRvkx7MQuUxiEcOj7wyEf
0zip9GVi2hAsumsgL0Wz
=mXjE
-----END PGP SIGNATURE-----
--==-=-=--

--=-=-=--