48/0c467ec54a5db31a1f5cf7ba80c3bdbb93745f

   1 Return-Path: <dmitry.kurochkin@gmail.com>\r
   2 X-Original-To: notmuch@notmuchmail.org\r
   3 Delivered-To: notmuch@notmuchmail.org\r
   4 Received: from localhost (localhost [127.0.0.1])\r
   5         by olra.theworths.org (Postfix) with ESMTP id 33790431FB6\r
   6         for <notmuch@notmuchmail.org>; Tue,  4 Sep 2012 11:53:12 -0700 (PDT)\r
   7 X-Virus-Scanned: Debian amavisd-new at olra.theworths.org\r
   8 X-Spam-Flag: NO\r
   9 X-Spam-Score: -0.799\r
  10 X-Spam-Level: \r
  11 X-Spam-Status: No, score=-0.799 tagged_above=-999 required=5\r
  12         tests=[DKIM_SIGNED=0.1, DKIM_VALID=-0.1, DKIM_VALID_AU=-0.1,\r
  13         FREEMAIL_FROM=0.001, RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled\r
  14 Received: from olra.theworths.org ([127.0.0.1])\r
  15         by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024)\r
  16         with ESMTP id A2Pzhqsv8xA6 for <notmuch@notmuchmail.org>;\r
  17         Tue,  4 Sep 2012 11:53:11 -0700 (PDT)\r
  18 Received: from mail-ee0-f53.google.com (mail-ee0-f53.google.com\r
  19  [74.125.83.53])        (using TLSv1 with cipher RC4-SHA (128/128 bits))        (No client\r
  20  certificate requested) by olra.theworths.org (Postfix) with ESMTPS id\r
  21  4B2D0431FAF    for <notmuch@notmuchmail.org>; Tue,  4 Sep 2012 11:53:11 -0700\r
  22  (PDT)\r
  23 Received: by eekb47 with SMTP id b47so2947704eek.26\r
  24         for <notmuch@notmuchmail.org>; Tue, 04 Sep 2012 11:53:08 -0700 (PDT)\r
  25 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20120113;\r
  26         h=from:to:subject:date:message-id:x-mailer;\r
  27         bh=dozgH3mWp36ddi1pgxlbLiGB1CaZxhWjD8Nva5iDvAc=;\r
  28         b=mA8A+Gnzud/GFb9xDcnbG27IpaeEWC4DvYRj9DEIZbpASSVp4Medlim0oZbjXHM37f\r
  29         nmFHfPmz26nHqklfz1rR8OyxM9ssCZGaCxKAqaQdWYuKZZfFwo3TKIWDOUWzeFxI2u2N\r
  30         gEH5pIC2G1LyihObX9omjUr+E/AtsWcgW+Pvk44Mv1eM6AykJtCrxql0JhOGzV4LrJq0\r
  31         X2Ik17BOtWq6AXtsDGNB9ZEbdT7m2Ft+3JmIiBimQ5VsYhi3GFARroq8a4LtOBWFAzK8\r
  32         z1qF7b7N+sPeVsue23xzVnQfm/hptZVFhbT2o4JNf3Y3vCgLazvLPl9ZU9lZ1FMdeOtK\r
  33         y+Fg==\r
  34 Received: by 10.14.204.72 with SMTP id g48mr27528941eeo.45.1346784788676;\r
  35         Tue, 04 Sep 2012 11:53:08 -0700 (PDT)\r
  36 Received: from localhost ([2001:470:1f0b:14dd:224:d7ff:fee2:c588])\r
  37         by mx.google.com with ESMTPS id e7sm47648765eep.2.2012.09.04.11.53.07\r
  38         (version=TLSv1/SSLv3 cipher=OTHER);\r
  39         Tue, 04 Sep 2012 11:53:07 -0700 (PDT)\r
  40 From: Dmitry Kurochkin <dmitry.kurochkin@gmail.com>\r
  41 To: notmuch@notmuchmail.org\r
  42 Subject: [PATCH] Add notmuch-remove-duplicates.py script to contrib.\r
  43 Date: Tue,  4 Sep 2012 22:53:05 +0400\r
  44 Message-Id: <1346784785-19746-1-git-send-email-dmitry.kurochkin@gmail.com>\r
  45 X-Mailer: git-send-email 1.7.10.4\r
  46 X-BeenThere: notmuch@notmuchmail.org\r
  47 X-Mailman-Version: 2.1.13\r
  48 Precedence: list\r
  49 List-Id: "Use and development of the notmuch mail system."\r
  50         <notmuch.notmuchmail.org>\r
  51 List-Unsubscribe: <http://notmuchmail.org/mailman/options/notmuch>,\r
  52         <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>\r
  53 List-Archive: <http://notmuchmail.org/pipermail/notmuch>\r
  54 List-Post: <mailto:notmuch@notmuchmail.org>\r
  55 List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>\r
  56 List-Subscribe: <http://notmuchmail.org/mailman/listinfo/notmuch>,\r
  57         <mailto:notmuch-request@notmuchmail.org?subject=subscribe>\r
  58 X-List-Received-Date: Tue, 04 Sep 2012 18:53:12 -0000\r
  59 \r
  60 The script removes duplicate message files.  It takes no options.\r
  61 \r
  62 Files are assumed duplicates if their content is the same except for\r
  63 ignored headers.  Currently, the only ignored header is Received:.\r
  64 ---\r
  65  contrib/notmuch-remove-duplicates.py |   95 ++++++++++++++++++++++++++++++++++\r
  66  1 file changed, 95 insertions(+)\r
  67  create mode 100755 contrib/notmuch-remove-duplicates.py\r
  68 \r
  69 diff --git a/contrib/notmuch-remove-duplicates.py b/contrib/notmuch-remove-duplicates.py\r
  70 new file mode 100755\r
  71 index 0000000..dbe2e25\r
  72 --- /dev/null\r
  73 +++ b/contrib/notmuch-remove-duplicates.py\r
  74 @@ -0,0 +1,95 @@\r
  75 +#!/usr/bin/env python\r
  76 +\r
  77 +import sys\r
  78 +\r
  79 +IGNORED_HEADERS = [ "Received:" ]\r
  80 +\r
  81 +if len(sys.argv) != 1:\r
  82 +    print "Usage: %s" % sys.argv[0]\r
  83 +    print\r
  84 +    print "The script removes duplicate message files.  Takes no options."\r
  85 +    print "Requires notmuch python module."\r
  86 +    print\r
  87 +    print "Files are assumed duplicates if their content is the same"\r
  88 +    print "except for the following headers: %s." % ", ".join(IGNORED_HEADERS)\r
  89 +    exit(1)\r
  90 +\r
  91 +import notmuch\r
  92 +import os\r
  93 +import time\r
  94 +\r
  95 +class MailComparator:\r
  96 +    """Checks if mail files are duplicates."""\r
  97 +    def __init__(self, filename):\r
  98 +        self.filename = filename\r
  99 +        self.mail = self.readFile(self.filename)\r
 100 +\r
 101 +    def isDuplicate(self, filename):\r
 102 +        return self.mail == self.readFile(filename)\r
 103 +\r
 104 +    @staticmethod\r
 105 +    def readFile(filename):\r
 106 +        with open(filename) as f:\r
 107 +            data = ""\r
 108 +            while True:\r
 109 +                line = f.readline()\r
 110 +                for header in IGNORED_HEADERS:\r
 111 +                    if line.startswith(header):\r
 112 +                        # skip header continuation lines\r
 113 +                        while True:\r
 114 +                            line = f.readline()\r
 115 +                            if len(line) == 0 or line[0] not in [" ", "\t"]:\r
 116 +                                break\r
 117 +                        break\r
 118 +                else:\r
 119 +                    data += line\r
 120 +                    if line == "\n":\r
 121 +                        break\r
 122 +            data += f.read()\r
 123 +            return data\r
 124 +\r
 125 +db = notmuch.Database()\r
 126 +query = db.create_query('*')\r
 127 +print "Number of messages: %s" % query.count_messages()\r
 128 +\r
 129 +files_count = 0\r
 130 +for root, dirs, files in os.walk(db.get_path()):\r
 131 +    if not root.startswith(os.path.join(db.get_path(), ".notmuch/")):\r
 132 +        files_count += len(files)\r
 133 +print "Number of files: %s" % files_count\r
 134 +print "Estimated number of duplicates: %s" % (files_count - query.count_messages())\r
 135 +\r
 136 +msgs = query.search_messages()\r
 137 +msg_count = 0\r
 138 +suspected_duplicates_count = 0\r
 139 +duplicates_count = 0\r
 140 +timestamp = time.time()\r
 141 +for msg in msgs:\r
 142 +    msg_count += 1\r
 143 +    if len(msg.get_filenames()) > 1:\r
 144 +        filenames = msg.get_filenames()\r
 145 +        comparator = MailComparator(filenames.next())\r
 146 +        for filename in filenames:\r
 147 +            if os.path.realpath(comparator.filename) == os.path.realpath(filename):\r
 148 +                print "Message '%s' has filenames pointing to the same file: '%s' '%s'" % (msg.get_message_id(), comparator.filename, filename)\r
 149 +            elif comparator.isDuplicate(filename):\r
 150 +                os.remove(filename)\r
 151 +                duplicates_count += 1\r
 152 +            else:\r
 153 +                #print "Potential duplicates: %s" % msg.get_message_id()\r
 154 +                suspected_duplicates_count += 1\r
 155 +\r
 156 +    new_timestamp = time.time()\r
 157 +    if new_timestamp - timestamp > 1:\r
 158 +        timestamp = new_timestamp\r
 159 +        sys.stdout.write("\rProcessed %s messages, removed %s duplicates..." % (msg_count, duplicates_count))\r
 160 +        sys.stdout.flush()\r
 161 +\r
 162 +print "\rFinished. Processed %s messages, removed %s duplicates." % (msg_count, duplicates_count)\r
 163 +if duplicates_count > 0:\r
 164 +    print "You might want to run 'notmuch new' now."\r
 165 +\r
 166 +if suspected_duplicates_count > 0:\r
 167 +    print\r
 168 +    print "Found %s messages with duplicate IDs but different content." % suspected_duplicates_count\r
 169 +    print "Perhaps we should ignore more headers."\r
 170 -- \r
 171 1.7.10.4\r
 172 \r