From a0c85e9db6c388487c50233be5345f11973e9b82 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Mon, 26 Nov 2012 01:16:01 +1900 Subject: [PATCH] [PATCH 3/3] lib: Reject multi-message mboxes and deprecate single-message mbox --- aa/3f707aafe93a3cb6ce54e5024f46ed02283be7 | 184 ++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 aa/3f707aafe93a3cb6ce54e5024f46ed02283be7 diff --git a/aa/3f707aafe93a3cb6ce54e5024f46ed02283be7 b/aa/3f707aafe93a3cb6ce54e5024f46ed02283be7 new file mode 100644 index 000000000..beb7517b1 --- /dev/null +++ b/aa/3f707aafe93a3cb6ce54e5024f46ed02283be7 @@ -0,0 +1,184 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id 1A1D2431FAF + for ; Sat, 24 Nov 2012 22:17:45 -0800 (PST) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Spam-Flag: NO +X-Spam-Score: -0.7 +X-Spam-Level: +X-Spam-Status: No, score=-0.7 tagged_above=-999 required=5 + tests=[RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id ZtnqlEfSeZH5 for ; + Sat, 24 Nov 2012 22:17:44 -0800 (PST) +Received: from dmz-mailsec-scanner-3.mit.edu (DMZ-MAILSEC-SCANNER-3.MIT.EDU + [18.9.25.14]) + by olra.theworths.org (Postfix) with ESMTP id 152CB431FAE + for ; Sat, 24 Nov 2012 22:17:43 -0800 (PST) +X-AuditID: 1209190e-b7f756d000000904-78-50b1b8079998 +Received: from mailhub-auth-1.mit.edu ( [18.9.21.35]) + by dmz-mailsec-scanner-3.mit.edu (Symantec Messaging Gateway) with SMTP + id 77.1B.02308.708B1B05; Sun, 25 Nov 2012 01:17:43 -0500 (EST) +Received: from outgoing.mit.edu (OUTGOING-AUTH.MIT.EDU [18.7.22.103]) + by mailhub-auth-1.mit.edu (8.13.8/8.9.2) with ESMTP id qAP6G5jY005391; + Sun, 25 Nov 2012 01:16:05 -0500 +Received: from drake.dyndns.org + (209-6-116-242.c3-0.arl-ubr1.sbo-arl.ma.cable.rcn.com + [209.6.116.242]) (authenticated bits=0) + (User authenticated as amdragon@ATHENA.MIT.EDU) + by outgoing.mit.edu (8.13.6/8.12.4) with ESMTP id qAP6G3I1006919 + (version=TLSv1/SSLv3 cipher=AES256-SHA bits=256 verify=NOT); + Sun, 25 Nov 2012 01:16:04 -0500 (EST) +Received: from amthrax by drake.dyndns.org with local (Exim 4.77) + (envelope-from ) + id 1TcVVb-0008HQ-AG; Sun, 25 Nov 2012 01:16:03 -0500 +From: Austin Clements +To: notmuch@notmuchmail.org +Subject: [PATCH 3/3] lib: Reject multi-message mboxes and deprecate + single-message mbox +Date: Sun, 25 Nov 2012 01:16:01 -0500 +Message-Id: <1353824161-31717-3-git-send-email-amdragon@mit.edu> +X-Mailer: git-send-email 1.7.10.4 +In-Reply-To: <1353824161-31717-1-git-send-email-amdragon@mit.edu> +References: <1353824161-31717-1-git-send-email-amdragon@mit.edu> +X-Brightmail-Tracker: + H4sIAAAAAAAAA+NgFtrHIsWRmVeSWpSXmKPExsUixCmqrMu+Y2OAwfr1yhY3WrsZLa7fnMns + wOTxbNUtZo8th94zBzBFcdmkpOZklqUW6dslcGV0tjWxFqyWr3i7+xRjA+MEyS5GTg4JAROJ + e41T2SBsMYkL99YD2VwcQgL7GCUubO2CcjYwSky+OZ8RwnnEJDHp7UsWCGcuo8TsW7MZQfrZ + BDQktu1fDmaLCEhL7Lw7mxXEZhawlLj6tZ0JxBYWCJfo+twDto9FQFXiQOMGsHpeAQeJz6/+ + skDcoSjR/WwCWA2ngKPEwr4DYLYQUM35ycdZJjDyL2BkWMUom5JbpZubmJlTnJqsW5ycmJeX + WqRrrJebWaKXmlK6iREcTJJ8Oxi/HlQ6xCjAwajEw3sjcWOAEGtiWXFl7iFGSQ4mJVHeqZuB + QnxJ+SmVGYnFGfFFpTmpxYcYJTiYlUR4D6wFyvGmJFZWpRblw6SkOViUxHmvpNz0FxJITyxJ + zU5NLUgtgsnKcHAoSfBu3wrUKFiUmp5akZaZU4KQZuLgBBnOAzR80TaQ4cUFibnFmekQ+VOM + ilLivBNBEgIgiYzSPLheWLS/YhQHekWYNwGkigeYKOC6XwENZgIa/HT2OpDBJYkIKakGRlHW + d9YM1+NaTj6OOfjshGXnk1/Bh7VLdyUsyv7zraonbsctrvzuV1/v7JrCt3yCenzVcb5Gu4NT + 3zYWpN7SZD/OyLs/c5nZ9Xfca2aocRZ+9TQuTpwWfMu54vINLctFD5ZLPhOIXyGvfuWQg7rL + grDL2ZfX6a1pera/96L+UrGqQ07tHpHM65RYijMSDbWYi4oTAREvBKTRAgAA +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Sun, 25 Nov 2012 06:17:45 -0000 + +Previously, we would treat multi-message mboxes as one giant email, +which, besides the obvious incorrect indexing, often led to +out-of-memory errors for archival mboxes. Now we explicitly reject +multi-message mboxes. For historical reasons, we retain support for +single-message mboxes, but official deprecate this behavior. +--- + lib/database.cc | 4 +++- + lib/index.cc | 28 ++++++++++++++++++++++++++++ + test/new | 8 +++++--- + 3 files changed, 36 insertions(+), 4 deletions(-) + +diff --git a/lib/database.cc b/lib/database.cc +index 4df3217..91d4329 100644 +--- a/lib/database.cc ++++ b/lib/database.cc +@@ -1821,7 +1821,9 @@ notmuch_database_add_message (notmuch_database_t *notmuch, + date = notmuch_message_file_get_header (message_file, "date"); + _notmuch_message_set_header_values (message, date, from, subject); + +- _notmuch_message_index_file (message, filename); ++ ret = _notmuch_message_index_file (message, filename); ++ if (ret) ++ goto DONE; + } else { + ret = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID; + } +diff --git a/lib/index.cc b/lib/index.cc +index e377732..da0e6ce 100644 +--- a/lib/index.cc ++++ b/lib/index.cc +@@ -435,6 +435,9 @@ _notmuch_message_index_file (notmuch_message_t *message, + const char *from, *subject; + notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS; + static int initialized = 0; ++ char from_buf[5]; ++ bool is_mbox = false; ++ static bool mbox_warning = false; + + if (! initialized) { + g_mime_init (0); +@@ -448,13 +451,38 @@ _notmuch_message_index_file (notmuch_message_t *message, + goto DONE; + } + ++ /* Is this mbox? */ ++ if (fread (from_buf, sizeof (from_buf), 1, file) == 1 && ++ strncmp (from_buf, "From ", 5) == 0) ++ is_mbox = true; ++ rewind (file); ++ + /* Evil GMime steals my FILE* here so I won't fclose it. */ + stream = g_mime_stream_file_new (file); + + parser = g_mime_parser_new_with_stream (stream); ++ g_mime_parser_set_scan_from (parser, is_mbox); + + mime_message = g_mime_parser_construct_message (parser); + ++ if (is_mbox) { ++ if (!g_mime_parser_eos (parser)) { ++ /* This is a multi-message mbox. */ ++ ret = NOTMUCH_STATUS_FILE_NOT_EMAIL; ++ goto DONE; ++ } ++ /* For historical reasons, we support single-message mboxes, ++ * but this behavior is likely to change in the future, so ++ * warn. */ ++ if (!mbox_warning) { ++ mbox_warning = true; ++ fprintf (stderr, "\ ++Warning: %s is an mbox containing a single message,\n\ ++likely caused by misconfigured mail delivery. Support for single-message\n\ ++mboxes is deprecated and may be removed in the future.\n", filename); ++ } ++ } ++ + from = g_mime_message_get_sender (mime_message); + addresses = internet_address_list_parse_string (from); + +diff --git a/test/new b/test/new +index 29f9aff..f562cec 100755 +--- a/test/new ++++ b/test/new +@@ -163,7 +163,7 @@ rm -rf "${MAIL_DIR}"/two + output=$(NOTMUCH_NEW) + test_expect_equal "$output" "No new mail. Removed 3 messages." + +-test_begin_subtest "Support single-message mbox" ++test_begin_subtest "Support single-message mbox (deprecated)" + cat > "${MAIL_DIR}"/mbox_file1 < +@@ -174,11 +174,13 @@ Body. + EOF + output=$(NOTMUCH_NEW 2>&1) + test_expect_equal "$output" \ +-"Added 1 new message to the database." ++"Warning: ${MAIL_DIR}/mbox_file1 is an mbox containing a single message, ++likely caused by misconfigured mail delivery. Support for single-message ++mboxes is deprecated and may be removed in the future. ++Added 1 new message to the database." + + # This test requires that notmuch new has been run at least once. + test_begin_subtest "Skip and report non-mail files" +-test_subtest_known_broken + generate_message + mkdir -p "${MAIL_DIR}"/.git && touch "${MAIL_DIR}"/.git/config + touch "${MAIL_DIR}"/ignored_file +-- +1.7.10.4 + -- 2.26.2