From: Carl Worth <cworth@cworth.org>
To: Michal Sojka <sojkam1@fel.cvut.cz>, notmuch@notmuchmail.org
Subject: Re: [PATCH v4 0/4] Maildir synchronization
In-Reply-To: <87iq066cbd.fsf@steelpick.2x.cz>
References: <87tyk3vpxd.fsf@wsheee.2x.cz>
	<1288560558-18915-1-git-send-email-sojkam1@fel.cvut.cz>
	<877hgsrjau.fsf@yoom.home.cworth.org> <87d3qhq527.fsf@resox.2x.cz>
	<874obrmww9.fsf@yoom.home.cworth.org>
	<87iq066cbd.fsf@steelpick.2x.cz>
User-Agent: Notmuch/0.4 (http://notmuchmail.org) Emacs/23.2.1
	(i486-pc-linux-gnu)
Date: Tue, 09 Nov 2010 12:38:49 -0800
Message-ID: <87oc9ykzae.fsf@yoom.home.cworth.org>
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="=-=-="
Precedence: list

--=-=-=
Content-Type: multipart/signed; boundary="==-=-=";
	micalg=pgp-sha1; protocol="application/pgp-signature"

--==-=-=
Content-Transfer-Encoding: quoted-printable

On Tue, 09 Nov 2010 11:06:30 +0100, Michal Sojka <sojkam1@fel.cvut.cz> wrot=
e:
> This sounds good. Still it will be neccessary to synchronize with all
> files, not only the first one.

OK. I'll add that to the list of things I'll fix up.

> > I'd like to get things merged today, so I plan to take your patches and
> > then add new commits on top to implement the functions I described
> > above.
>=20
> Great! Seems that it is still not finished and I do not want to do
> duplicite work, so let me know what is the current status and whether
> you need some help from me (e.g. tests for multiple messages with same
> ID). I should be on IRC today.

Yesterday ended up being very busy, but I'm working on this stuff
now. My plan is to merge it and then release notmuch 0.5. (And yes, if I
had a more strict release manager then 0.5 would have been released
yesterday as-is---but it turns out my release manager had an equally
busy day yesterday).

If you wanted to write some more tests, then that would be very useful.

I've been changing the way the tests are written, so you might want to
look at the attached file to see what I'm doing and match it. Notable
differences:

	* Using test_begin_subtest and test_expect_equal rather than the
          list-of-commands to test_expect_success.

	* Not emitting separate line-item results for things that are
          already tested in other scripts, (like "add message" and
          "search for message", etc.)

The updating I've done here only goes as far as just before "Add a
message to new/ without info". And it looks like one change I made
inadvertently broke a later test, so it's expected that "Check that
removing info did not change tags" currently fails. [And this failure
shows what I don't like about test_expect_success---if we were using
test_expect_equal it would be trivial to see what problem I made here.]

Finally, when going through these tests I saw:

	"Removing of unread tag should fail without cur/"

And that's behavior I do not want. Adding and removing tags should be
reliable whether or not the maildir synchronization can succeed. In this
specific case, the right answer is probably to say that a directory
without "new" and "cur" is not a maildir so no synchronization should be
done.

Notmuch does need to be able to support things like mh format still. Do
the current patches break that by doing maildir-style renaming in
non-maildir directories? If so, we'll need to fix that as well. And that
might require an "is_maildir" term to be stored for directory documents
in the database.

Again, that's something I can help with.

Thanks again,

=2DCarl


--==-=-=
Content-Type: application/pgp-signature

-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.10 (GNU/Linux)

iD8DBQFM2bFZ6JDdNq8qSWgRAoc0AKCSOhpUcAj9zi97HqEv11RvrXTyrACfffC7
5wx1KHaiPBWaTrD8v9LglZ8=
=t4Ei
-----END PGP SIGNATURE-----
--==-=-=--

--=-=-=
Content-Disposition: inline; filename=maildir-sync

#!/bin/bash

test_description="maildir synchronization"

. ./test-lib.sh

# Much easier to examine differences if the "notmuch show
# --format=json" output includes some newlines. Also, need to avoid
# including the local value of MAIL_DIR in the result.
filter_show_json() {
    sed -e 's/, /,\n/g'  | sed -e "s|${MAIL_DIR}/|MAIL_DIR/|"
    echo
}

cat >> "$NOTMUCH_CONFIG" <<EOF
[maildir]
synchronize_flags=true
EOF

test_begin_subtest "Adding 'S' flag to existing file removes 'unread' tag"
add_message [subject]='"test message"' [date]='"Sat, 01 Jan 2000 12:00:00 -0000"' [filename]='"msg-001:2,"'
NOTMUCH_NEW > /dev/null
output=$(notmuch search tag:inbox | notmuch_search_sanitize)
output+="
"
mv "${gen_msg_filename}" "${gen_msg_filename}S"
increment_mtime "$(dirname "${gen_msg_filename}")"
output+=$(NOTMUCH_NEW)
output+="
"
output+=$(notmuch search tag:inbox | notmuch_search_sanitize)
test_expect_equal "$output" "thread:XXX   2000-01-01 [1/1] Notmuch Test Suite; test message (inbox unread)
No new mail. Detected 1 file rename.
thread:XXX   2000-01-01 [1/1] Notmuch Test Suite; test message (inbox)"

test_begin_subtest "Adding message with 'S' flag prevents 'unread' tag"
add_message [subject]='"test message 2"' [date]='"Sat, 01 Jan 2000 12:00:00 -0000"' [filename]='"msg-002:2,S"' &&
output=$(notmuch search tag:inbox | notmuch_search_sanitize)
test_expect_equal "$output" "thread:XXX   2000-01-01 [1/1] Notmuch Test Suite; test message (inbox)
thread:XXX   2000-01-01 [1/1] Notmuch Test Suite; test message 2 (inbox)"

test_begin_subtest "Adding 'replied' flag adds 'R' tag to filename"
notmuch tag +replied tag:inbox
output=$(ls -1 "${MAIL_DIR}")
test_expect_equal "$output" "msg-001:2,RS
msg-002:2,RS"

test_begin_subtest "notmuch show works with renamed file (without notmuch new)"
output=$(notmuch show --format=json id:msg-001@notmuch-test-suite | filter_show_json)
test_expect_equal "$output" '[[[{"id": "msg-001@notmuch-test-suite",
"match": true,
"filename": "MAIL_DIR/msg-001:2,RS",
"timestamp": 946728000,
"date_relative": "2000-01-01",
"tags": ["inbox","replied"],
"headers": {"Subject": "test message",
"From": "Notmuch Test Suite <test_suite@notmuchmail.org>",
"To": "Notmuch Test Suite <test_suite@notmuchmail.org>",
"Cc": "",
"Bcc": "",
"Date": "Sat,
01 Jan 2000 12:00:00 -0000"},
"body": [{"id": 1,
"content-type": "text/plain",
"content": "This is just a test message (#1)\n"}]},
[]]]]'

test_expect_success 'notmuch reply works with renamed file (without notmuch new)' 'notmuch reply id:msg-001@notmuch-test-suite'

test_begin_subtest "notmuch new detects no file rename after tag->flag synchronization"
increment_mtime "$(dirname ${gen_msg_filename})"
output=$(NOTMUCH_NEW)
test_expect_equal "$output" "No new mail."

test_expect_success "Add a message to new/ without info" '
generate_message [subject]="\"test message 3\"" [date]="\"Sat, 01 Jan 2000 12:00:00 -0000\"" [dir]=new &&
NOTMUCH_NEW > actual &&
test_cmp - actual <<EOF
Added 1 new message to the database.
EOF
'
test_expect_success "Check that the message has inbox and unread tags" '
notmuch search tag:inbox and tag:unread | notmuch_search_sanitize > actual &&
test_cmp - actual <<EOF
thread:XXX   2000-01-01 [1/1] Notmuch Test Suite; test message 3 (inbox unread)
EOF
'
test_expect_success "Tag the message with 'tmp' tag" '
notmuch tag +tmp tag:inbox and tag:unread'

test_expect_success "Check that the message was not moved from new/ to cur/" '
echo filename:$gen_msg_filename > expected &&
notmuch show id:$gen_msg_id|grep -o "filename:.*$" > actual &&
test_cmp expected actual &&
test -f "$gen_msg_filename"
'
test_expect_success "Check that the message was not renamed" '
ls "${MAIL_DIR}/new" > actual &&
test_cmp - actual <<EOF
msg-003
EOF
'
test_expect_success 'Removing of unread tag should fail without cur/' '
test_must_fail notmuch tag -unread tag:inbox and tag:unread
'
test_expect_success "Check that the tags were not changed" '
notmuch search tag:inbox and tag:unread | notmuch_search_sanitize > actual &&
test_cmp - actual <<EOF
thread:XXX   2000-01-01 [1/1] Notmuch Test Suite; test message 3 (inbox tmp unread)
EOF
'

# notmuch new is not necessary here, but we run it in order to check
# for 'no rename' later (*).
test_expect_success 'Create cur/ and let notmuch know about it' '
mkdir "$MAIL_DIR/cur" &&
notmuch new
'
test_expect_success 'Removing of unread tag should pass with cur/' '
notmuch tag -unread tag:inbox and tag:unread
'
test_expect_success 'Check that the message was moved to cur/' '\
ls "$MAIL_DIR/cur" > actual &&
test_cmp - actual <<EOF
msg-003:2,S
EOF
'
test_expect_success 'No rename should be detected by notmuch new' '
increment_mtime "$MAIL_DIR/cur" &&
notmuch new > actual &&
test_cmp - actual <<EOF
No new mail.
EOF
'
# (*) If notmuch new was not run we've got "Processed 1 file in almost
# no time" here. The reason is that removing unread tag in a previous
# test created directory document in the database but this document
# was not linked as subdirectory of $MAIL_DIR. Therefore notmuch new
# could not reach the cur/ directory and its files in it during
# recurive traversal.
test_expect_success 'Remove info from file name' '
mv "$MAIL_DIR/cur/msg-003:2,S" "$MAIL_DIR/cur/msg-003" &&
increment_mtime "$MAIL_DIR/cur" &&
NOTMUCH_NEW > actual
test_cmp - actual <<EOF
No new mail. Detected 1 file rename.
EOF
'
test_expect_success "Check that removing info did not change tags" '
notmuch search tag:inbox | notmuch_search_sanitize > actual &&
test_cmp - actual <<EOF
thread:XXX   2000-01-01 [1/1] Notmuch Test Suite; test message 3 (inbox tmp)
EOF
'
test_expect_success "Add a message to fakenew/ without info" '
generate_message [subject]="\"test message 4\"" [date]="\"Sat, 01 Jan 2000 12:00:00 -0000\"" [dir]=fakenew &&
NOTMUCH_NEW > actual &&
test_cmp - actual <<EOF
Added 1 new message to the database.
EOF
'
test_expect_success "Check that the message has inbox and unread tags" '
notmuch search tag:inbox and tag:unread | notmuch_search_sanitize > actual &&
test_cmp - actual <<EOF
thread:XXX   2000-01-01 [1/1] Notmuch Test Suite; test message 4 (inbox unread)
EOF
'
test_expect_success 'Removing of unread tag should leave the message in fakenew/' '
notmuch tag -unread tag:inbox and tag:unread &&
ls "$MAIL_DIR/fakenew" > actual &&
test_cmp - actual <<EOF
msg-004:2,S
EOF
'

test_expect_success 'Make maildir flags out of sync with the database' '
ls $MAIL_DIR > expected &&
mv $MAIL_DIR/msg-001:2,RS $MAIL_DIR/msg-001:2, &&
mv $MAIL_DIR/msg-002:2,RS $MAIL_DIR/msg-002:2, &&
increment_mtime $MAIL_DIR
'

test_expect_success 'Test whether dump/new/restore synchronizes the maildir flags with the database' '
notmuch dump dump.txt &&
notmuch new &&
notmuch restore dump.txt &&
ls $MAIL_DIR > actual &&
test_cmp expected actual
'

test_done

--=-=-=


-- 
carl.d.worth@intel.com

--=-=-=--