From: Dmitry Kurochkin Date: Tue, 4 Sep 2012 18:53:05 +0000 (+0400) Subject: [PATCH] Add notmuch-remove-duplicates.py script to contrib. X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=f1366b22238162b1521a2f05559937c8bb337984;p=notmuch-archives.git [PATCH] Add notmuch-remove-duplicates.py script to contrib. --- diff --git a/48/0c467ec54a5db31a1f5cf7ba80c3bdbb93745f b/48/0c467ec54a5db31a1f5cf7ba80c3bdbb93745f new file mode 100644 index 000000000..02a87e926 --- /dev/null +++ b/48/0c467ec54a5db31a1f5cf7ba80c3bdbb93745f @@ -0,0 +1,172 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id 33790431FB6 + for ; Tue, 4 Sep 2012 11:53:12 -0700 (PDT) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Spam-Flag: NO +X-Spam-Score: -0.799 +X-Spam-Level: +X-Spam-Status: No, score=-0.799 tagged_above=-999 required=5 + tests=[DKIM_SIGNED=0.1, DKIM_VALID=-0.1, DKIM_VALID_AU=-0.1, + FREEMAIL_FROM=0.001, RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id A2Pzhqsv8xA6 for ; + Tue, 4 Sep 2012 11:53:11 -0700 (PDT) +Received: from mail-ee0-f53.google.com (mail-ee0-f53.google.com + [74.125.83.53]) (using TLSv1 with cipher RC4-SHA (128/128 bits)) (No client + certificate requested) by olra.theworths.org (Postfix) with ESMTPS id + 4B2D0431FAF for ; Tue, 4 Sep 2012 11:53:11 -0700 + (PDT) +Received: by eekb47 with SMTP id b47so2947704eek.26 + for ; Tue, 04 Sep 2012 11:53:08 -0700 (PDT) +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20120113; + h=from:to:subject:date:message-id:x-mailer; + bh=dozgH3mWp36ddi1pgxlbLiGB1CaZxhWjD8Nva5iDvAc=; + b=mA8A+Gnzud/GFb9xDcnbG27IpaeEWC4DvYRj9DEIZbpASSVp4Medlim0oZbjXHM37f + nmFHfPmz26nHqklfz1rR8OyxM9ssCZGaCxKAqaQdWYuKZZfFwo3TKIWDOUWzeFxI2u2N + gEH5pIC2G1LyihObX9omjUr+E/AtsWcgW+Pvk44Mv1eM6AykJtCrxql0JhOGzV4LrJq0 + X2Ik17BOtWq6AXtsDGNB9ZEbdT7m2Ft+3JmIiBimQ5VsYhi3GFARroq8a4LtOBWFAzK8 + z1qF7b7N+sPeVsue23xzVnQfm/hptZVFhbT2o4JNf3Y3vCgLazvLPl9ZU9lZ1FMdeOtK + y+Fg== +Received: by 10.14.204.72 with SMTP id g48mr27528941eeo.45.1346784788676; + Tue, 04 Sep 2012 11:53:08 -0700 (PDT) +Received: from localhost ([2001:470:1f0b:14dd:224:d7ff:fee2:c588]) + by mx.google.com with ESMTPS id e7sm47648765eep.2.2012.09.04.11.53.07 + (version=TLSv1/SSLv3 cipher=OTHER); + Tue, 04 Sep 2012 11:53:07 -0700 (PDT) +From: Dmitry Kurochkin +To: notmuch@notmuchmail.org +Subject: [PATCH] Add notmuch-remove-duplicates.py script to contrib. +Date: Tue, 4 Sep 2012 22:53:05 +0400 +Message-Id: <1346784785-19746-1-git-send-email-dmitry.kurochkin@gmail.com> +X-Mailer: git-send-email 1.7.10.4 +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Tue, 04 Sep 2012 18:53:12 -0000 + +The script removes duplicate message files. It takes no options. + +Files are assumed duplicates if their content is the same except for +ignored headers. Currently, the only ignored header is Received:. +--- + contrib/notmuch-remove-duplicates.py | 95 ++++++++++++++++++++++++++++++++++ + 1 file changed, 95 insertions(+) + create mode 100755 contrib/notmuch-remove-duplicates.py + +diff --git a/contrib/notmuch-remove-duplicates.py b/contrib/notmuch-remove-duplicates.py +new file mode 100755 +index 0000000..dbe2e25 +--- /dev/null ++++ b/contrib/notmuch-remove-duplicates.py +@@ -0,0 +1,95 @@ ++#!/usr/bin/env python ++ ++import sys ++ ++IGNORED_HEADERS = [ "Received:" ] ++ ++if len(sys.argv) != 1: ++ print "Usage: %s" % sys.argv[0] ++ print ++ print "The script removes duplicate message files. Takes no options." ++ print "Requires notmuch python module." ++ print ++ print "Files are assumed duplicates if their content is the same" ++ print "except for the following headers: %s." % ", ".join(IGNORED_HEADERS) ++ exit(1) ++ ++import notmuch ++import os ++import time ++ ++class MailComparator: ++ """Checks if mail files are duplicates.""" ++ def __init__(self, filename): ++ self.filename = filename ++ self.mail = self.readFile(self.filename) ++ ++ def isDuplicate(self, filename): ++ return self.mail == self.readFile(filename) ++ ++ @staticmethod ++ def readFile(filename): ++ with open(filename) as f: ++ data = "" ++ while True: ++ line = f.readline() ++ for header in IGNORED_HEADERS: ++ if line.startswith(header): ++ # skip header continuation lines ++ while True: ++ line = f.readline() ++ if len(line) == 0 or line[0] not in [" ", "\t"]: ++ break ++ break ++ else: ++ data += line ++ if line == "\n": ++ break ++ data += f.read() ++ return data ++ ++db = notmuch.Database() ++query = db.create_query('*') ++print "Number of messages: %s" % query.count_messages() ++ ++files_count = 0 ++for root, dirs, files in os.walk(db.get_path()): ++ if not root.startswith(os.path.join(db.get_path(), ".notmuch/")): ++ files_count += len(files) ++print "Number of files: %s" % files_count ++print "Estimated number of duplicates: %s" % (files_count - query.count_messages()) ++ ++msgs = query.search_messages() ++msg_count = 0 ++suspected_duplicates_count = 0 ++duplicates_count = 0 ++timestamp = time.time() ++for msg in msgs: ++ msg_count += 1 ++ if len(msg.get_filenames()) > 1: ++ filenames = msg.get_filenames() ++ comparator = MailComparator(filenames.next()) ++ for filename in filenames: ++ if os.path.realpath(comparator.filename) == os.path.realpath(filename): ++ print "Message '%s' has filenames pointing to the same file: '%s' '%s'" % (msg.get_message_id(), comparator.filename, filename) ++ elif comparator.isDuplicate(filename): ++ os.remove(filename) ++ duplicates_count += 1 ++ else: ++ #print "Potential duplicates: %s" % msg.get_message_id() ++ suspected_duplicates_count += 1 ++ ++ new_timestamp = time.time() ++ if new_timestamp - timestamp > 1: ++ timestamp = new_timestamp ++ sys.stdout.write("\rProcessed %s messages, removed %s duplicates..." % (msg_count, duplicates_count)) ++ sys.stdout.flush() ++ ++print "\rFinished. Processed %s messages, removed %s duplicates." % (msg_count, duplicates_count) ++if duplicates_count > 0: ++ print "You might want to run 'notmuch new' now." ++ ++if suspected_duplicates_count > 0: ++ print ++ print "Found %s messages with duplicate IDs but different content." % suspected_duplicates_count ++ print "Perhaps we should ignore more headers." +-- +1.7.10.4 +