X-Git-Url: http://git.tremily.us/?p=pygrader.git;a=blobdiff_plain;f=pygrader%2Fmailpipe.py;h=48c724f1287afce2d7e5de22a2a3561fb507b8c3;hp=8d556fb09daf4cbbe8162e22f3c632bcf4f94a6e;hb=09bf6f517462446242dbc289c510a93efaaf6e4d;hpb=522783a9af5ade8cd7c0a6ce4435abe589eb5440 diff --git a/pygrader/mailpipe.py b/pygrader/mailpipe.py index 8d556fb..48c724f 100644 --- a/pygrader/mailpipe.py +++ b/pygrader/mailpipe.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 W. Trevor King +# Copyright (C) 2012 W. Trevor King # # This file is part of pygrader. # @@ -14,36 +14,111 @@ # You should have received a copy of the GNU General Public License along with # pygrader. If not, see . +"Incoming email processing." + from __future__ import absolute_import from email import message_from_file as _message_from_file from email.header import decode_header as _decode_header -from email.utils import formatdate as _formatdate -import hashlib as _hashlib -import locale as _locale +from email.mime.text import MIMEText as _MIMEText import mailbox as _mailbox -import os as _os -import os.path as _os_path +import re as _re import sys as _sys -import time as _time -from pgp_mime import verify as _verify +import pgp_mime as _pgp_mime from lxml import etree as _etree from . import LOG as _LOG -from .color import standard_colors as _standard_colors -from .color import color_string as _color_string +from .email import construct_email as _construct_email from .email import construct_response as _construct_response -from .extract_mime import extract_mime as _extract_mime from .extract_mime import message_time as _message_time from .model.person import Person as _Person -from .storage import assignment_path as _assignment_path -from .storage import set_late as _set_late + +from .handler import InsecureMessage as _InsecureMessage +from .handler import InvalidAssignmentSubject as _InvalidAssignmentSubject +from .handler import InvalidMessage as _InvalidMessage +from .handler import InvalidStudentSubject as _InvalidStudentSubject +from .handler import InvalidSubjectMessage as _InvalidSubjectMessage +from .handler import PermissionViolationMessage as _PermissionViolationMessage +from .handler import Response as _Response +from .handler import UnsignedMessage as _UnsignedMessage +from .handler.get import run as _handle_get +from .handler.grade import run as _handle_grade +from .handler.grade import MissingGradeMessage as _MissingGradeMessage +from .handler.submission import run as _handle_submission +from .handler.submission import InvalidSubmission as _InvalidSubmission + + +_TAG_REGEXP = _re.compile('^.*\[([^]]*)\].*$') + + +class NoReturnPath (_InvalidMessage): + def __init__(self, address, **kwargs): + if 'error' not in kwargs: + kwargs['error'] = 'no Return-Path' + super(NoReturnPath, self).__init__(**kwargs) + + +class UnregisteredAddress (_InvalidMessage): + def __init__(self, address, **kwargs): + if 'error' not in kwargs: + kwargs['error'] = 'unregistered address {}'.format(address) + super(UnregisteredAddress, self).__init__(**kwargs) + self.address = address + + +class AmbiguousAddress (_InvalidMessage): + def __init__(self, address, people, **kwargs): + if 'error' not in kwargs: + kwargs['error'] = 'ambiguous address {}'.format(address) + super(AmbiguousAddress, self).__init__(**kwargs) + self.address = address + self.people = people + + +class WrongSignatureMessage (_InsecureMessage): + def __init__(self, pgp_key=None, fingerprints=None, decrypted=None, + **kwargs): + if 'error' not in kwargs: + kwargs['error'] = 'not signed by the expected key' + super(WrongSignatureMessage, self).__init__(**kwargs) + self.pgp_key = pgp_key + self.fingerprints = fingerprints + self.decrypted = decrypted + +class UnverifiedSignatureMessage (_InsecureMessage): + def __init__(self, signature=None, decrypted=None, **kwargs): + if 'error' not in kwargs: + kwargs['error'] = 'unverified signature' + super(UnverifiedSignatureMessage, self).__init__(**kwargs) + self.signature = signature + self.decrypted = decrypted + + +class SubjectlessMessage (_InvalidSubjectMessage): + def __init__(self, **kwargs): + if 'error' not in kwargs: + kwargs['error'] = 'no subject' + super(SubjectlessMessage, self).__init__(**kwargs) + + +class InvalidHandlerMessage (_InvalidSubjectMessage): + def __init__(self, target=None, handlers=None, **kwargs): + if 'error' not in kwargs: + kwargs['error'] = 'no handler for {!r}'.format(target) + super(InvalidHandlerMessage, self).__init__(**kwargs) + self.target = target + self.handlers = handlers def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, - output=None, max_late=0, respond=None, use_color=None, - dry_run=False, **kwargs): + output=None, continue_after_invalid_message=False, max_late=0, + trust_email_infrastructure=False, + handlers={ + 'get': _handle_get, + 'grade': _handle_grade, + 'submit': _handle_submission, + }, respond=None, dry_run=False, **kwargs): """Run from procmail to sort incomming submissions For example, you can setup your ``.procmailrc`` like this:: @@ -59,50 +134,153 @@ def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, # Grab all incoming homeworks emails. This rule eats matching emails # (i.e. no further procmail processing). :0 - * ^Subject:.*\[phys160-sub] + * ^Subject:.*\[phys160:submit] | "$PYGRADE_MAILPIPE" mailpipe If you don't want procmail to eat the message, you can use the ``c`` flag (carbon copy) by starting your rule off with ``:0 c``. - >>> from asyncore import loop >>> from io import StringIO >>> from pgp_mime.email import encodedMIMEText - >>> from pygrader.test.course import StubCourse - >>> from pygrader.test.client import MessageSender - >>> from pygrader.test.server import SMTPServer - - Messages with unrecognized ``Return-Path``\s are silently dropped: + >>> from .handler import InvalidMessage, Response + >>> from .test.course import StubCourse >>> course = StubCourse() - >>> def process(peer, mailfrom, rcpttos, data): + >>> def respond(message): + ... print('respond with:\\n{}'.format(message.as_string())) + >>> def process(message): ... mailpipe( ... basedir=course.basedir, course=course.course, - ... stream=StringIO(data), output=course.mailbox) + ... stream=StringIO(message.as_string()), + ... output=course.mailbox, + ... continue_after_invalid_message=True, + ... respond=respond) >>> message = encodedMIMEText('The answer is 42.') >>> message['Message-ID'] = '<123.456@home.net>' - >>> message['Return-Path'] = '' >>> message['Received'] = ( ... 'from smtp.home.net (smtp.home.net [123.456.123.456]) ' ... 'by smtp.mail.uu.edu (Postfix) with ESMTP id 5BA225C83EF ' ... 'for ; Sun, 09 Oct 2011 11:50:46 -0400 (EDT)') >>> message['From'] = 'Billy B ' >>> message['To'] = 'phys101 ' - >>> message['Subject'] = 'assignment 1 submission' - >>> messages = [message] - >>> ms = MessageSender(address=('localhost', 1025), messages=messages) - >>> loop() - >>> course.print_tree() # doctest: +REPORT_UDIFF + >>> message['Subject'] = '[submit] assignment 1' + + Messages with unrecognized ``Return-Path``\s are silently dropped: + + >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS + >>> course.print_tree() # doctest: +REPORT_UDIFF, +ELLIPSIS course.conf + mail + mail/cur + mail/new + mail/tmp + + Response to a message from an unregistered person: + + >>> message['Return-Path'] = '' + >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS + respond with: + Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...==" + MIME-Version: 1.0 + Content-Disposition: inline + Date: ... + From: Robot101 + Reply-to: Robot101 + To: "invalid.return.path@home.net" + Subject: unregistered address invalid.return.path@home.net + + --===============...== + Content-Type: multipart/mixed; boundary="===============...==" + MIME-Version: 1.0 + + --===============...== + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: inline + + invalid.return.path@home.net, + + Your email address is not registered with pygrader for + Physics 101. If you feel it should be, contact your professor + or TA. + + Yours, + phys-101 robot + + --===============...== + Content-Type: message/rfc822 + MIME-Version: 1.0 + + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: inline + Message-ID: <123.456@home.net> + Received: from smtp.home.net (smtp.home.net [123.456.123.456]) by smtp.mail.uu.edu (Postfix) with ESMTP id 5BA225C83EF for ; Sun, 09 Oct 2011 11:50:46 -0400 (EDT) + From: Billy B + To: phys101 + Subject: [submit] assignment 1 + Return-Path: + + The answer is 42. + --===============...==-- + --===============...== + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Description: OpenPGP digital signature + Content-Type: application/pgp-signature; name="signature.asc"; charset="us-ascii" + + -----BEGIN PGP SIGNATURE----- + Version: GnuPG v2.0.19 (GNU/Linux) + + ... + -----END PGP SIGNATURE----- + + --===============...==-- If we add a valid ``Return-Path``, we get the expected delivery: - >>> server = SMTPServer( - ... ('localhost', 1025), None, process=process, count=1) >>> del message['Return-Path'] >>> message['Return-Path'] = '' - >>> ms = MessageSender(address=('localhost', 1025), messages=messages) - >>> loop() + >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS + respond with: + Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...==" + MIME-Version: 1.0 + Content-Disposition: inline + Date: ... + From: Robot101 + Reply-to: Robot101 + To: Bilbo Baggins + Subject: Received Assignment 1 submission + + --===============...== + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Disposition: inline + Content-Transfer-Encoding: 7bit + + Billy, + + We received your submission for Assignment 1 on Sun, 09 Oct 2011 15:50:46 -0000. + + Yours, + phys-101 robot + + --===============...== + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Description: OpenPGP digital signature + Content-Type: application/pgp-signature; name="signature.asc"; charset="us-ascii" + + -----BEGIN PGP SIGNATURE----- + Version: GnuPG v2.0.19 (GNU/Linux) + + ... + -----END PGP SIGNATURE----- + + --===============...==-- + >>> course.print_tree() # doctest: +REPORT_UDIFF, +ELLIPSIS Bilbo_Baggins Bilbo_Baggins/Assignment_1 @@ -120,8 +298,6 @@ def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, The last ``Received`` is used to timestamp the message: - >>> server = SMTPServer( - ... ('localhost', 1025), None, process=process, count=1) >>> del message['Message-ID'] >>> message['Message-ID'] = '' >>> del message['Received'] @@ -133,9 +309,44 @@ def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, ... 'from smtp.home.net (smtp.home.net [123.456.123.456]) ' ... 'by smtp.mail.uu.edu (Postfix) with ESMTP id 5BA225C83EF ' ... 'for ; Mon, 09 Oct 2011 11:50:46 -0400 (EDT)') - >>> messages = [message] - >>> ms = MessageSender(address=('localhost', 1025), messages=messages) - >>> loop() + >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS + respond with: + Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...==" + MIME-Version: 1.0 + Content-Disposition: inline + Date: ... + From: Robot101 + Reply-to: Robot101 + To: Bilbo Baggins + Subject: Received Assignment 1 submission + + --===============...== + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Disposition: inline + Content-Transfer-Encoding: 7bit + + Billy, + + We received your submission for Assignment 1 on Mon, 10 Oct 2011 16:50:46 -0000. + + Yours, + phys-101 robot + + --===============...== + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Description: OpenPGP digital signature + Content-Type: application/pgp-signature; name="signature.asc"; charset="us-ascii" + + -----BEGIN PGP SIGNATURE----- + Version: GnuPG v2.0.19 (GNU/Linux) + + ... + -----END PGP SIGNATURE----- + + --===============...==-- + >>> course.print_tree() # doctest: +REPORT_UDIFF, +ELLIPSIS Bilbo_Baggins Bilbo_Baggins/Assignment_1 @@ -157,25 +368,15 @@ def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, You can send receipts to the acknowledge incoming messages, which includes warnings about dropped messages (except for messages without ``Return-Path`` and messages where the ``Return-Path`` - email belongs to multiple ``People``. Both of these cases should - only come from problems with pygrader configuration). + email belongs to multiple ``People``. The former should only + occur with malicious emails, and the latter with improper pygrader + configurations). Response to a successful submission: - >>> def respond(message): - ... print('respond with:\\n{}'.format(message.as_string())) - >>> def process(peer, mailfrom, rcpttos, data): - ... mailpipe( - ... basedir=course.basedir, course=course.course, - ... stream=StringIO(data), output=course.mailbox, - ... respond=respond) - >>> server = SMTPServer( - ... ('localhost', 1025), None, process=process, count=1) >>> del message['Message-ID'] >>> message['Message-ID'] = '' - >>> messages = [message] - >>> ms = MessageSender(address=('localhost', 1025), messages=messages) - >>> loop() # doctest: +REPORT_UDIFF, +ELLIPSIS + >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS respond with: Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...==" MIME-Version: 1.0 @@ -184,17 +385,13 @@ def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, From: Robot101 Reply-to: Robot101 To: Bilbo Baggins - Subject: received Assignment 1 submission - - --===============...== - Content-Type: multipart/mixed; boundary="===============...==" - MIME-Version: 1.0 + Subject: Received Assignment 1 submission --===============...== Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 - Content-Transfer-Encoding: 7bit Content-Disposition: inline + Content-Transfer-Encoding: 7bit Billy, @@ -202,24 +399,7 @@ def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, Yours, phys-101 robot - --===============...== - Content-Type: message/rfc822 - MIME-Version: 1.0 - Content-Type: text/plain; charset="us-ascii" - MIME-Version: 1.0 - Content-Transfer-Encoding: 7bit - Content-Disposition: inline - From: Billy B - To: phys101 - Subject: assignment 1 submission - Return-Path: - Received: from smtp.mail.uu.edu (localhost.localdomain [127.0.0.1]) by smtp.mail.uu.edu (Postfix) with SMTP id 68CB45C8453 for ; Mon, 10 Oct 2011 12:50:46 -0400 (EDT) - Received: from smtp.home.net (smtp.home.net [123.456.123.456]) by smtp.mail.uu.edu (Postfix) with ESMTP id 5BA225C83EF for ; Mon, 09 Oct 2011 11:50:46 -0400 (EDT) - Message-ID: - - The answer is 42. - --===============...==-- --===============...== MIME-Version: 1.0 Content-Transfer-Encoding: 7bit @@ -227,7 +407,7 @@ def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, Content-Type: application/pgp-signature; name="signature.asc"; charset="us-ascii" -----BEGIN PGP SIGNATURE----- - Version: GnuPG v2.0.17 (GNU/Linux) + Version: GnuPG v2.0.19 (GNU/Linux) ... -----END PGP SIGNATURE----- @@ -236,13 +416,9 @@ def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, Response to a submission on an unsubmittable assignment: - >>> server = SMTPServer( - ... ('localhost', 1025), None, process=process, count=1) >>> del message['Subject'] - >>> message['Subject'] = 'attendance 1 submission' - >>> messages = [message] - >>> ms = MessageSender(address=('localhost', 1025), messages=messages) - >>> loop() # doctest: +REPORT_UDIFF, +ELLIPSIS + >>> message['Subject'] = '[submit] attendance 1' + >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS respond with: Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...==" MIME-Version: 1.0 @@ -251,7 +427,7 @@ def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, From: Robot101 Reply-to: Robot101 To: Bilbo Baggins - Subject: received invalid Attendance 1 submission + Subject: Received invalid Attendance 1 submission --===============...== Content-Type: multipart/mixed; boundary="===============...==" @@ -265,11 +441,12 @@ def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, Billy, - We received your submission for Attendance 1, but you are not allowed - to submit that assignment via email. + We received your submission for Attendance 1, but you are not + allowed to submit that assignment via email. Yours, phys-101 robot + --===============...== Content-Type: message/rfc822 MIME-Version: 1.0 @@ -284,7 +461,7 @@ def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, Received: from smtp.mail.uu.edu (localhost.localdomain [127.0.0.1]) by smtp.mail.uu.edu (Postfix) with SMTP id 68CB45C8453 for ; Mon, 10 Oct 2011 12:50:46 -0400 (EDT) Received: from smtp.home.net (smtp.home.net [123.456.123.456]) by smtp.mail.uu.edu (Postfix) with ESMTP id 5BA225C83EF for ; Mon, 09 Oct 2011 11:50:46 -0400 (EDT) Message-ID: - Subject: attendance 1 submission + Subject: [submit] attendance 1 The answer is 42. --===============...==-- @@ -295,7 +472,7 @@ def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, Content-Type: application/pgp-signature; name="signature.asc"; charset="us-ascii" -----BEGIN PGP SIGNATURE----- - Version: GnuPG v2.0.17 (GNU/Linux) + Version: GnuPG v2.0.19 (GNU/Linux) ... -----END PGP SIGNATURE----- @@ -304,13 +481,9 @@ def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, Response to a bad subject: - >>> server = SMTPServer( - ... ('localhost', 1025), None, process=process, count=1) >>> del message['Subject'] >>> message['Subject'] = 'need help for the first homework' - >>> messages = [message] - >>> ms = MessageSender(address=('localhost', 1025), messages=messages) - >>> loop() # doctest: +REPORT_UDIFF, +ELLIPSIS + >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS respond with: Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...==" MIME-Version: 1.0 @@ -319,7 +492,7 @@ def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, From: Robot101 Reply-to: Robot101 To: Bilbo Baggins - Subject: received 'need help for the first homework' + Subject: no tag in 'need help for the first homework' --===============...== Content-Type: multipart/mixed; boundary="===============...==" @@ -333,16 +506,12 @@ def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, Billy, - We got an email from you with the following subject: - 'need help for the first homework' - which does not match any submittable assignment name for - Physics 101. - Remember to use the full name for the assignment in the - subject. For example: - Assignment 1 submission + We received an email message from you with an invalid + subject. Yours, phys-101 robot + --===============...== Content-Type: message/rfc822 MIME-Version: 1.0 @@ -368,7 +537,7 @@ def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, Content-Type: application/pgp-signature; name="signature.asc"; charset="us-ascii" -----BEGIN PGP SIGNATURE----- - Version: GnuPG v2.0.17 (GNU/Linux) + Version: GnuPG v2.0.19 (GNU/Linux) ... -----END PGP SIGNATURE----- @@ -377,12 +546,8 @@ def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, Response to a missing subject: - >>> server = SMTPServer( - ... ('localhost', 1025), None, process=process, count=1) >>> del message['Subject'] - >>> messages = [message] - >>> ms = MessageSender(address=('localhost', 1025), messages=messages) - >>> loop() # doctest: +REPORT_UDIFF, +ELLIPSIS + >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS respond with: Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...==" MIME-Version: 1.0 @@ -409,6 +574,7 @@ def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, Yours, phys-101 robot + --===============...== Content-Type: message/rfc822 MIME-Version: 1.0 @@ -433,7 +599,7 @@ def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, Content-Type: application/pgp-signature; name="signature.asc"; charset="us-ascii" -----BEGIN PGP SIGNATURE----- - Version: GnuPG v2.0.17 (GNU/Linux) + Version: GnuPG v2.0.19 (GNU/Linux) ... -----END PGP SIGNATURE----- @@ -444,12 +610,8 @@ def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, >>> student = course.course.person(email='bb@greyhavens.net') >>> student.pgp_key = '4332B6E3' - >>> server = SMTPServer( - ... ('localhost', 1025), None, process=process, count=1) >>> del message['Subject'] - >>> messages = [message] - >>> ms = MessageSender(address=('localhost', 1025), messages=messages) - >>> loop() # doctest: +REPORT_UDIFF, +ELLIPSIS + >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS respond with: Content-Type: multipart/encrypted; protocol="application/pgp-encrypted"; micalg="pgp-sha1"; boundary="===============...==" MIME-Version: 1.0 @@ -474,100 +636,85 @@ def mailpipe(basedir, course, stream=None, mailbox=None, input_=None, Content-Type: application/octet-stream; name="encrypted.asc"; charset="us-ascii" -----BEGIN PGP MESSAGE----- - Version: GnuPG v2.0.17 (GNU/Linux) + Version: GnuPG v2.0.19 (GNU/Linux) ... -----END PGP MESSAGE----- --===============...==-- - Response to a message from an unregistered person: - - >>> server = SMTPServer( - ... ('localhost', 1025), None, process=process, count=1) - >>> del message['Return-Path'] - >>> message['Return-Path'] = '' - >>> messages = [message] - >>> ms = MessageSender(address=('localhost', 1025), messages=messages) - >>> loop() # doctest: +REPORT_UDIFF, +ELLIPSIS - respond with: - Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...==" - MIME-Version: 1.0 - Content-Disposition: inline - Date: ... - From: Robot101 - Reply-to: Robot101 - To: "invalid.return.path@home.net" - Subject: unregistered address invalid.return.path@home.net - - --===============...== - Content-Type: multipart/mixed; boundary="===============...==" - MIME-Version: 1.0 - - --===============...== - Content-Type: text/plain; charset="us-ascii" - MIME-Version: 1.0 - Content-Transfer-Encoding: 7bit - Content-Disposition: inline - - invalid.return.path@home.net, - - Your email address is not registered with pygrader for - Physics 101. If you feel it should be, contact your professor - or TA. - - Yours, - phys-101 robot - --===============...== - Content-Type: message/rfc822 - MIME-Version: 1.0 - - Content-Type: text/plain; charset="us-ascii" - MIME-Version: 1.0 - Content-Transfer-Encoding: 7bit - Content-Disposition: inline - From: Billy B - To: phys101 - Received: from smtp.mail.uu.edu (localhost.localdomain [127.0.0.1]) by smtp.mail.uu.edu (Postfix) with SMTP id 68CB45C8453 for ; Mon, 10 Oct 2011 12:50:46 -0400 (EDT) - Received: from smtp.home.net (smtp.home.net [123.456.123.456]) by smtp.mail.uu.edu (Postfix) with ESMTP id 5BA225C83EF for ; Mon, 09 Oct 2011 11:50:46 -0400 (EDT) - Message-ID: - Return-Path: - - The answer is 42. - --===============...==-- - --===============...== - MIME-Version: 1.0 - Content-Transfer-Encoding: 7bit - Content-Description: OpenPGP digital signature - Content-Type: application/pgp-signature; name="signature.asc"; charset="us-ascii" - - -----BEGIN PGP SIGNATURE----- - Version: GnuPG v2.0.17 (GNU/Linux) - - ... - -----END PGP SIGNATURE----- - - --===============...==-- - >>> course.cleanup() """ if stream is None: stream = _sys.stdin - for msg,person,assignment,time in _load_messages( + for original,message,person,subject,target in _load_messages( course=course, stream=stream, mailbox=mailbox, input_=input_, - output=output, respond=respond, use_color=use_color, dry_run=dry_run): - assignment_path = _assignment_path(basedir, assignment, person) - _save_local_message_copy( - msg=msg, person=person, assignment_path=assignment_path, - use_color=use_color, dry_run=dry_run) - _extract_mime(message=msg, output=assignment_path, dry_run=dry_run) - _check_late( - basedir=basedir, assignment=assignment, person=person, time=time, - max_late=max_late, use_color=use_color, dry_run=dry_run) + output=output, dry_run=dry_run, + continue_after_invalid_message=continue_after_invalid_message, + trust_email_infrastructure=trust_email_infrastructure, + respond=respond): + try: + handler = _get_handler(handlers=handlers, target=target) + _LOG.debug('handling {}'.format(target)) + handler( + basedir=basedir, course=course, message=message, + person=person, subject=subject, + max_late=max_late, + trust_email_infrastructure=trust_email_infrastructure, + dry_run=dry_run) + except _InvalidMessage as error: + error.course = course + error.message = original + for attribute,value in [('person', person), + ('subject', subject), + ('target', target)]: + if (value is not None and + getattr(error, attribute, None) is None): + setattr(error, attribute, value) + _LOG.warn('invalid message {}'.format(error.message_id())) + if not continue_after_invalid_message: + raise + _LOG.warn('{}'.format(error)) + if respond: + response = _get_error_response(error) + respond(response) + except _Response as response: + if respond: + msg = response.message + if not response.complete: + author = course.robot + target = person + msg = response.message + if isinstance(response.message, _MIMEText): + # Manipulate body (based on pgp_mime.append_text) + original_encoding = msg.get_charset().input_charset + original_payload = str( + msg.get_payload(decode=True), original_encoding) + new_payload = ( + '{},\n\n' + '{}\n\n' + 'Yours,\n' + '{}\n').format( + target.alias(), original_payload, author.alias()) + new_encoding = _pgp_mime.guess_encoding(new_payload) + if msg.get('content-transfer-encoding', None): + # clear CTE so set_payload will set it properly + del msg['content-transfer-encoding'] + msg.set_payload(new_payload, new_encoding) + subject = msg['Subject'] + assert subject is not None, msg + del msg['Subject'] + msg = _construct_email( + author=author, targets=[person], subject=subject, + message=msg) + respond(msg) def _load_messages(course, stream, mailbox=None, input_=None, output=None, - respond=None, use_color=None, dry_run=False): + continue_after_invalid_message=False, + trust_email_infrastructure=False, respond=None, + dry_run=False): if mailbox is None: + _LOG.debug('loading message from {}'.format(stream)) mbox = None messages = [(None,_message_from_file(stream))] if output is not None: @@ -579,227 +726,158 @@ def _load_messages(course, stream, mailbox=None, input_=None, output=None, ombox = _mailbox.mbox(output, factory=None, create=True) elif mailbox == 'maildir': mbox = _mailbox.Maildir(input_, factory=None, create=False) - messages = mbox.items() + messages = [] + for key,msg in mbox.items(): + subpath = mbox._lookup(key) + if subpath.endswith('.gitignore'): + _LOG.debug('skipping non-message {}'.format(subpath)) + continue + messages.append((key, msg)) if output is not None: ombox = _mailbox.Maildir(output, factory=None, create=True) else: raise ValueError(mailbox) + messages.sort(key=_get_message_time) for key,msg in messages: - ret = _parse_message( - course=course, msg=msg, respond=respond, use_color=use_color) - if ret: - if output is not None and dry_run is False: - # move message from input mailbox to output mailbox - ombox.add(msg) - if mbox is not None: - del mbox[key] - yield ret - -def _parse_message(course, msg, respond=None, use_color=None): - highlight,lowlight,good,bad = _standard_colors(use_color=use_color) - mid = msg['Message-ID'] - sender = msg['Return-Path'] # RFC 822 + try: + ret = _parse_message( + course=course, message=msg, + trust_email_infrastructure=trust_email_infrastructure) + except _InvalidMessage as error: + error.message = msg + _LOG.warn('invalid message {}'.format(error.message_id())) + if not continue_after_invalid_message: + raise + _LOG.warn('{}'.format(error)) + if respond: + response = _get_error_response(error) + if response is not None: + respond(response) + continue + if output is not None and dry_run is False: + # move message from input mailbox to output mailbox + ombox.add(msg) + if mbox is not None: + del mbox[key] + yield ret + +def _parse_message(course, message, trust_email_infrastructure=False): + """Parse an incoming email and respond if neccessary. + + Return ``(msg, person, assignment, time)`` on successful parsing. + Return ``None`` on failure. + """ + original = message + person = subject = target = None + try: + person = _get_message_person(course=course, message=message) + if person.pgp_key: + _LOG.debug('verify message is from {}'.format(person)) + try: + message = _get_verified_message(message, person.pgp_key) + except _UnsignedMessage as error: + if trust_email_infrastructure: + _LOG.warn('{}'.format(error)) + else: + raise + subject = _get_message_subject(message=message) + target = _get_message_target(subject=subject) + except _InvalidMessage as error: + error.course = course + error.message = original + for attribute,value in [('person', person), + ('subject', subject), + ('target', target)]: + if (value is not None and + getattr(error, attribute, None) is None): + setattr(error, attribute, value) + raise + return (original, message, person, subject, target) + +def _get_message_person(course, message): + sender = message['Return-Path'] # RFC 822 if sender is None: - _LOG.debug(_color_string( - string='no Return-Path in {}'.format(mid), color=lowlight)) - return None + raise NoReturnPath(message) sender = sender[1:-1] # strip wrapping '<' and '>' - people = list(course.find_people(email=sender)) if len(people) == 0: - _LOG.warn(_color_string( - string='no person found to match {}'.format(sender), - color=bad)) - if respond: - person = _Person(name=sender, emails=[sender]) - response_subject = 'unregistered address {}'.format(sender) - response_text = ( - '{},\n\n' - 'Your email address is not registered with pygrader for\n' - '{}. If you feel it should be, contact your professor\n' - 'or TA.\n\n' - 'Yours,\n{}').format( - sender, course.name, course.robot.alias()) - response = _construct_response( - author=course.robot, targets=[person], - subject=response_subject, text=response_text, original=msg) - respond(response) - return None + raise UnregisteredAddress(message=message, address=sender) if len(people) > 1: - _LOG.warn(_color_string( - string='multiple people match {} ({})'.format( - sender, ', '.join(str(p) for p in people)), - color=bad)) - return None - person = people[0] - - if person.pgp_key: - original = msg - msg = _get_verified_message(msg, person.pgp_key, use_color=use_color) - if msg is None: - if respond: - response_subject = 'unsigned message {}'.format(mid) - response_text = ( - '{},\n\n' - 'We received an email message from you without a valid\n' - 'PGP signature.\n\n' - 'Yours,\n{}').format( - person.alias(), course.robot.alias()) - response = _construct_response( - author=course.robot, targets=[person], - subject=response_subject, text=response_text, - original=original) - respond(response) - return None - - if msg['Subject'] is None: - _LOG.warn(_color_string( - string='no subject in {}'.format(mid), color=bad)) - if respond: - response_subject = 'no subject in {}'.format(mid) - response_text = ( - '{},\n\n' - 'We received an email message from you without a subject.\n\n' - 'Yours,\n{}').format( - person.alias(), course.robot.alias()) - response = _construct_response( - author=course.robot, targets=[person], - subject=response_subject, text=response_text, original=msg) - respond(response) - return None - parts = _decode_header(msg['Subject']) - if len(parts) != 1: - _LOG.warn(_color_string( - string='multi-part header {}'.format(parts), color=bad)) - return None - subject,encoding = parts[0] - if encoding is None: - encoding = 'ascii' + raise AmbiguousAddress(message=message, address=sender, people=people) + return people[0] + +def _get_message_subject(message): + """ + >>> from email.header import Header + >>> from pgp_mime.email import encodedMIMEText + >>> message = encodedMIMEText('The answer is 42.') + >>> message['Message-ID'] = 'msg-id' + >>> _get_message_subject(message=message) + Traceback (most recent call last): + ... + pygrader.mailpipe.SubjectlessMessage: no subject + >>> del message['Subject'] + >>> subject = Header('unicode part', 'utf-8') + >>> subject.append('-ascii part', 'ascii') + >>> message['Subject'] = subject.encode() + >>> _get_message_subject(message=message) + 'unicode part-ascii part' + >>> del message['Subject'] + >>> message['Subject'] = 'clean subject' + >>> _get_message_subject(message=message) + 'clean subject' + """ + if message['Subject'] is None: + raise SubjectlessMessage(subject=None, message=message) + + parts = _decode_header(message['Subject']) + part_strings = [] + for string,encoding in parts: + if encoding is None: + encoding = 'ascii' + if not isinstance(string, str): + string = str(string, encoding) + part_strings.append(string) + subject = ''.join(part_strings) _LOG.debug('decoded header {} -> {}'.format(parts[0], subject)) + return subject.lower().replace('#', '') - subject = subject.lower().replace('#', '') - for assignment in course.assignments: - if _match_assignment(assignment, subject): - break - if not _match_assignment(assignment, subject): - _LOG.warn(_color_string( - string='no assignment found in {}'.format(repr(subject)), - color=bad)) - if respond: - response_subject = "received '{}'".format(subject) - submittable_assignments = [ - a for a in course.assignments if a.submittable] - if not submittable_assignments: - hint = ( - 'In fact, there are no submittable assignments for\n' - 'this course!\n') - else: - hint = ( - 'Remember to use the full name for the assignment in the\n' - 'subject. For example:\n' - ' {} submission\n\n').format( - submittable_assignments[0].name) - response_text = ( - '{},\n\n' - 'We got an email from you with the following subject:\n' - ' {}\n' - 'which does not match any submittable assignment name for\n' - '{}.\n' - '{}' - 'Yours,\n{}').format( - person.alias(), repr(subject), course.name, hint, - course.robot.alias()) - response = _construct_response( - author=course.robot, targets=[person], - subject=response_subject, text=response_text, original=msg) - respond(response) - return None - - if not assignment.submittable: - response_subject = 'received invalid {} submission'.format( - assignment.name) - response_text = ( - '{},\n\n' - 'We received your submission for {}, but you are not allowed\n' - 'to submit that assignment via email.\n\n' - 'Yours,\n{}').format( - person.alias(), assignment.name, course.robot.alias()) - response = _construct_response( - author=course.robot, targets=[person], - subject=response_subject, text=response_text, original=msg) - respond(response) - - time = _message_time(message=msg, use_color=use_color) - - if respond: - response_subject = 'received {} submission'.format(assignment.name) - if time: - time_str = 'on {}'.format(_formatdate(time)) - else: - time_str = 'at an unknown time' - response_text = ( - '{},\n\n' - 'We received your submission for {} {}.\n\n' - 'Yours,\n{}').format( - person.alias(), assignment.name, time_str, course.robot.alias()) - response = _construct_response( - author=course.robot, targets=[person], - subject=response_subject, text=response_text, original=msg) - respond(response) - return (msg, person, assignment, time) - -def _match_assignment(assignment, subject): - return assignment.name.lower() in subject - -def _save_local_message_copy(msg, person, assignment_path, use_color=None, - dry_run=False): - highlight,lowlight,good,bad = _standard_colors(use_color=use_color) - try: - _os.makedirs(assignment_path) - except OSError: - pass - mpath = _os_path.join(assignment_path, 'mail') +def _get_message_target(subject): + """ + >>> _get_message_target(subject='no tag') + Traceback (most recent call last): + ... + pygrader.handler.InvalidSubjectMessage: no tag in 'no tag' + >>> _get_message_target(subject='[] empty tag') + Traceback (most recent call last): + ... + pygrader.handler.InvalidSubjectMessage: empty tag in '[] empty tag' + >>> _get_message_target(subject='[abc] empty tag') + 'abc' + >>> _get_message_target(subject='[phys160:abc] empty tag') + 'abc' + """ + match = _TAG_REGEXP.match(subject) + if match is None: + raise _InvalidSubjectMessage( + subject=subject, error='no tag in {!r}'.format(subject)) + tag = match.group(1) + if tag == '': + raise _InvalidSubjectMessage( + subject=subject, error='empty tag in {!r}'.format(subject)) + target = tag.rsplit(':', 1)[-1] + _LOG.debug('extracted target {} -> {}'.format(subject, target)) + return target + +def _get_handler(handlers, target): try: - mbox = _mailbox.Maildir(mpath, factory=None, create=not dry_run) - except _mailbox.NoSuchMailboxError as e: - _LOG.debug(_color_string( - string='could not open mailbox at {}'.format(mpath), - color=bad)) - mbox = None - new_msg = True - else: - new_msg = True - for other_msg in mbox: - if other_msg['Message-ID'] == msg['Message-ID']: - new_msg = False - break - if new_msg: - _LOG.debug(_color_string( - string='saving email from {} to {}'.format( - person, assignment_path), color=good)) - if mbox is not None and not dry_run: - mdmsg = _mailbox.MaildirMessage(msg) - mdmsg.add_flag('S') - mbox.add(mdmsg) - mbox.close() - else: - _LOG.debug(_color_string( - string='already found {} in {}'.format( - msg['Message-ID'], mpath), color=good)) - -def _check_late(basedir, assignment, person, time, max_late=0, use_color=None, - dry_run=False): - highlight,lowlight,good,bad = _standard_colors(use_color=use_color) - if time > assignment.due + max_late: - dt = time - assignment.due - _LOG.warn(_color_string( - string='{} {} late by {} seconds ({} hours)'.format( - person.name, assignment.name, dt, dt/3600.), - color=bad)) - if not dry_run: - _set_late(basedir=basedir, assignment=assignment, person=person) - -def _get_verified_message(message, pgp_key, use_color=None): + handler = handlers[target] + except KeyError as error: + raise InvalidHandlerMessage( + target=target, handlers=handlers) from error + return handler + +def _get_verified_message(message, pgp_key): """ >>> from pgp_mime import sign, encodedMIMEText @@ -821,6 +899,10 @@ def _get_verified_message(message, pgp_key, use_color=None): We check that the message is signed, and that it is signed by the appropriate key. + >>> signed.authenticated + Traceback (most recent call last): + ... + AttributeError: 'MIMEMultipart' object has no attribute 'authenticated' >>> our_message = _get_verified_message(signed, pgp_key='4332B6E3') >>> print(our_message.as_string()) # doctest: +REPORT_UDIFF Content-Type: text/plain; charset="us-ascii" @@ -832,47 +914,47 @@ def _get_verified_message(message, pgp_key, use_color=None): Received: from smtp.home.net ... 1.23 joules + >>> our_message.authenticated + True - If it is signed, but not by the right key, we get ``None``. + If it is signed, but not by the right key, we get an error. >>> print(_get_verified_message(signed, pgp_key='01234567')) - None + Traceback (most recent call last): + ... + pygrader.mailpipe.WrongSignatureMessage: not signed by the expected key - If it is not signed at all, we get ``None``. + If it is not signed at all, we get another error. >>> print(_get_verified_message(message, pgp_key='4332B6E3')) - None + Traceback (most recent call last): + ... + pygrader.handler.UnsignedMessage: unsigned message """ - highlight,lowlight,good,bad = _standard_colors(use_color=use_color) mid = message['message-id'] try: - decrypted,verified,result = _verify(message=message) - except (ValueError, AssertionError): - _LOG.warn(_color_string( - string='could not verify {} (not signed?)'.format(mid), - color=bad)) - return None - _LOG.info(_color_string(str(result, 'utf-8'), color=lowlight)) + decrypted,verified,result = _pgp_mime.verify(message=message) + except (ValueError, AssertionError) as error: + raise _UnsignedMessage(message=message) from error + _LOG.debug(str(result, 'utf-8')) tree = _etree.fromstring(result.replace(b'\x00', b'')) match = None + fingerprints = [] for signature in tree.findall('.//signature'): for fingerprint in signature.iterchildren('fpr'): - if fingerprint.text.endswith(pgp_key): - match = signature - break - if match is None: - _LOG.warn(_color_string( - string='{} is not signed by the expected key'.format(mid), - color=bad)) - return None + fingerprints.append(fingerprint) + matches = [f for f in fingerprints if f.text.endswith(pgp_key)] + if len(matches) == 0: + raise WrongSignatureMessage( + message=message, pgp_key=pgp_key, fingerprints=fingerprints, + decrypted=decrypted) + match = matches[0] if not verified: sumhex = list(signature.iterchildren('summary'))[0].get('value') summary = int(sumhex, 16) if summary != 0: - _LOG.warn(_color_string( - string='{} has an unverified signature'.format(mid), - color=bad)) - return None + raise UnverifiedSignatureMessage( + message=message, signature=signature, decrypted=decrypted) # otherwise, we may have an untrusted key. We'll count that # as verified here, because the caller is explicity looking # for signatures by this fingerprint. @@ -882,4 +964,118 @@ def _get_verified_message(message, pgp_key, use_color=None): 'content-disposition', ]: decrypted[k] = v + decrypted.authenticated = True return decrypted + +def _get_error_response(error): + author = error.course.robot + target = getattr(error, 'person', None) + subject = str(error) + if isinstance(error, _InvalidSubmission): + subject = 'Received invalid {} submission'.format( + error.assignment.name) + text = ( + 'We received your submission for {}, but you are not\n' + 'allowed to submit that assignment via email.' + ).format(error.assignment.name) + elif isinstance(error, _MissingGradeMessage): + subject = 'No grade in {!r}'.format(error.subject) + text = ( + 'Your grade submission did not include a text/plain\n' + 'part containing the new grade and comment.' + ) + elif isinstance(error, InvalidHandlerMessage): + targets = sorted(error.handlers.keys()) + if not targets: + hint = ( + 'In fact, there are no available handlers for this\n' + 'course!') + else: + hint = ( + 'Perhaps you meant to use one of the following:\n' + ' {}').format('\n '.join(targets)) + text = ( + 'We got an email from you with the following subject:\n' + ' {!r}\n' + 'which does not match any submittable handler name for\n' + '{}.\n' + '{}').format(error.subject, error.course.name, hint) + elif isinstance(error, SubjectlessMessage): + subject = 'no subject in {}'.format(error.message['Message-ID']) + text = 'We received an email message from you without a subject.' + elif isinstance(error, AmbiguousAddress): + text = ( + 'Multiple people match {} ({})'.format( + error.address, ', '.join(p.name for p in error.people))) + elif isinstance(error, UnregisteredAddress): + target = _Person(name=error.address, emails=[error.address]) + text = ( + 'Your email address is not registered with pygrader for\n' + '{}. If you feel it should be, contact your professor\n' + 'or TA.').format(error.course.name) + elif isinstance(error, NoReturnPath): + return + elif isinstance(error, _InvalidAssignmentSubject): + if error.assignments: + hint = ( + 'but it matches several assignments:\n' + ' * {}').format('\n * '.join( + a.name for a in error.assignments)) + else: + # prefer a submittable example assignment + assignments = [ + a for a in error.course.assignments if a.submittable] + assignments += course.assignments # but fall back to any one + hint = ( + 'Remember to use the full name for the assignment in the\n' + 'subject. For example:\n' + ' {} submission').format(assignments[0].name) + text = ( + 'We got an email from you with the following subject:\n' + ' {!r}\n{}').format(error.subject, hint) + elif isinstance(error, _InvalidStudentSubject): + text = ( + 'We got an email from you with the following subject:\n' + ' {!r}\n' + 'but it matches several students:\n' + ' * {}').format( + error.subject, '\n * '.join(s.name for s in error.students)) + elif isinstance(error, _InvalidSubjectMessage): + text = ( + 'We received an email message from you with an invalid\n' + 'subject.') + elif isinstance(error, _UnsignedMessage): + subject = 'unsigned message {}'.format(error.message['Message-ID']) + text = ( + 'We received an email message from you without a valid\n' + 'PGP signature.' + ) + elif isinstance(error, _PermissionViolationMessage): + text = ( + 'We got an email from you with the following subject:\n' + ' {!r}\n' + "but you can't do that unless you belong to one of the\n" + 'following groups:\n' + ' * {}').format( + error.subject, '\n * '.join(error.allowed_groups)) + elif isinstance(error, _InvalidMessage): + text = subject + else: + raise NotImplementedError((type(error), error)) + if target is None: + raise NotImplementedError((type(error), error)) + return _construct_response( + author=author, + targets=[target], + subject=subject, + text=( + '{},\n\n' + '{}\n\n' + 'Yours,\n' + '{}\n'.format(target.alias(), text, author.alias())), + original=error.message) + +def _get_message_time(key_message): + "Key function for sorting mailbox (key,message) tuples." + key,message = key_message + return _message_time(message)