Re: Added messages / total files count difference.

diff --git a/bf/fa0ec37e3aa2f3fff6218b8d241fa736f2ec29 b/bf/fa0ec37e3aa2f3fff6218b8d241fa736f2ec29
new file mode 100644 (file)
index 0000000..9c9a739
--- /dev/null
+++ b/bf/fa0ec37e3aa2f3fff6218b8d241fa736f2ec29
@@ -0,0 +1,150 @@
+Return-Path: <tomi.ollila@nixu.com>\r
+X-Original-To: notmuch@notmuchmail.org\r
+Delivered-To: notmuch@notmuchmail.org\r
+Received: from localhost (localhost [127.0.0.1])\r
+       by olra.theworths.org (Postfix) with ESMTP id 3B715431FD0\r
+       for <notmuch@notmuchmail.org>; Wed, 10 Aug 2011 01:43:20 -0700 (PDT)\r
+X-Virus-Scanned: Debian amavisd-new at olra.theworths.org\r
+X-Spam-Flag: NO\r
+X-Spam-Score: 0\r
+X-Spam-Level: \r
+X-Spam-Status: No, score=0 tagged_above=-999 required=5 tests=[none]\r
+       autolearn=disabled\r
+Received: from olra.theworths.org ([127.0.0.1])\r
+       by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024)\r
+       with ESMTP id E6L1ep4Nqywb for <notmuch@notmuchmail.org>;\r
+       Wed, 10 Aug 2011 01:43:18 -0700 (PDT)\r
+Received: from taco2.nixu.fi (taco2.nixu.fi [194.197.118.31])\r
+       (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits))\r
+       (No client certificate requested)\r
+       by olra.theworths.org (Postfix) with ESMTPS id 9ABDC431FB6\r
+       for <notmuch@notmuchmail.org>; Wed, 10 Aug 2011 01:43:18 -0700 (PDT)\r
+Received: from taco2.nixu.fi (localhost [127.0.0.1])\r
+       by taco2.nixu.fi (8.14.3/8.14.3/Debian-5+lenny1) with ESMTP id\r
+       p7A8fsim014713\r
+       (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=NOT)\r
+       for <notmuch@notmuchmail.org>; Wed, 10 Aug 2011 11:41:54 +0300\r
+Received: (from too@localhost)\r
+       by taco2.nixu.fi (8.14.3/8.14.3/Submit) id p7A8fswU014712;\r
+       Wed, 10 Aug 2011 11:41:54 +0300\r
+X-Authentication-Warning: taco2.nixu.fi: too set sender to\r
+       tomi.ollila@nixu.com using -f\r
+From: Tomi Ollila <tomi.ollila@nixu.com>\r
+To: notmuch@notmuchmail.org\r
+Subject: Re: Added messages / total files count difference.\r
+References: <yf639han8zz.fsf@taco2.nixu.fi>\r
+X-Face: HhBM'cA~<r"^Xv\KRN0P{vn'Y"Kd;zg_y3S[4)KSN~s?O\"QPoL\r
+       $[Xv_BD:i/F$WiEWax}R(MPS`^UaptOGD`*/=@\1lKoVa9tnrg0TW?"r7aRtgk[F\r
+       !)g;OY^,BjTbr)Np:%c_o'jj,Z\r
+Date: Wed, 10 Aug 2011 11:41:54 +0300\r
+In-Reply-To: <yf639han8zz.fsf@taco2.nixu.fi> (Tomi Ollila's message of "Tue,\r
+       09 Aug 2011 14:02:08 +0300")\r
+Message-ID: <yf6hb5plktp.fsf@taco2.nixu.fi>\r
+User-Agent: Gnus/5.110014 (No Gnus v0.14) Emacs/22.2 (gnu/linux)\r
+MIME-Version: 1.0\r
+Content-Type: text/plain\r
+X-BeenThere: notmuch@notmuchmail.org\r
+X-Mailman-Version: 2.1.13\r
+Precedence: list\r
+List-Id: "Use and development of the notmuch mail system."\r
+       <notmuch.notmuchmail.org>\r
+List-Unsubscribe: <http://notmuchmail.org/mailman/options/notmuch>,\r
+       <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>\r
+List-Archive: <http://notmuchmail.org/pipermail/notmuch>\r
+List-Post: <mailto:notmuch@notmuchmail.org>\r
+List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>\r
+List-Subscribe: <http://notmuchmail.org/mailman/listinfo/notmuch>,\r
+       <mailto:notmuch-request@notmuchmail.org?subject=subscribe>\r
+X-List-Received-Date: Wed, 10 Aug 2011 08:43:20 -0000\r
+\r
+On Tue 09 Aug 2011 14:02, Tomi Ollila <tomi.ollila@nixu.com> writes:\r
+\r
+> Hi\r
+>\r
+> I get this output:\r
+>\r
+> $ notmuch new --verbose\r
+> Found 15559 total files (that's not much mail).\r
+> Processed 15559 total files in 5m 53s (43 files/sec.).\r
+> Added 15546 new messages to the database.\r
+>\r
+> $ find * -type f | wc\r
+>   15559   15559  529027\r
+>\r
+> How can I determine which 13 files were dropped. All of those\r
+> 15559 files should be mails. I tried to check through mail files that\r
+> have no 'Subject:' header but those were (at least one) indexed. Could\r
+> it be about duplicate Message-ID: or something ?\r
+>\r
+> $ notmuch --version\r
+> notmuch 0.7-7-g68e8560\r
+\r
+It is about duplicate Message-ID:s\r
+\r
+It would be nice that 'notmuch new' printes information about this\r
+if this were to happen (as I recall it does when new file found\r
+is not (considered as) a mail file).\r
+\r
+The steps I took to figure this out (not all iterations with & without\r
+'wc':s shown) at the end of this email.\r
+\r
+>\r
+> Tomi\r
+\r
+Tomi\r
+\r
+--8<----8<----8<----8<----8<----8<----8<----8<----8<----8<--\r
+\r
+$ find ~/mail/mails/* -type f | sort >! filenames-fs\r
+$ wc filenames-fs \r
+ 15559  15559 855766 filenames-fs\r
+\r
+$ cd /path/to/notmuch-git/bindings/python\r
+$ cat > foo.py\r
+import notmuch\r
+db = notmuch.Database()\r
+msgs = notmuch.Query(db,'').search_messages()\r
+\r
+for f in msgs:\r
+    print f.get_filename()\r
+\r
+$ PYTHONPATH=/path/to/python-json:`pwd` python foo.py | sort > filenames-db\r
+$ wc filenames-db\r
+ 15546  15546 855037 filenames-db\r
+\r
+$ diff filenames-db filenames-fs | grep mails | wc\r
+     13      26     755\r
+\r
+$ cd ~/mail\r
+$ cat >midcheck.pl\r
+use strict;\r
+use warnings;\r
+\r
+my %msgids;\r
+\r
+foreach (<mails/*/*>) {\r
+    my $fn = $_;\r
+    my $mid;\r
+    open I, '<', $fn or die $!;\r
+    while (<I>) {\r
+        $mid = $1, next if /^Message-ID:\s*(.*)/i;\r
+        last if /^$/;\r
+    }\r
+    close I;\r
+    unless ($mid) {\r
+        print "$fn: no Message-ID (in same line with header tag?)\n";\r
+        next;\r
+    }\r
+    my $fn0 = $msgids{$mid};\r
+    if (defined $fn0) {\r
+        print "Files '$fn0' and '$fn' have same msg id: $mid\n";\r
+    }\r
+    else {\r
+        $msgids{$mid} = $fn;\r
+    }\r
+}\r
+\r
+$ perl midcheck.pl | wc\r
+     13     117    2098\r
+$ perl midcheck.pl | grep \^Files | wc\r
+     13     117    2098\r