-# Copyright (C) 2012 W. Trevor King <wking@drexel.edu>
+# Copyright (C) 2012 W. Trevor King <wking@tremily.us>
#
# This file is part of pygrader.
#
# You should have received a copy of the GNU General Public License along with
# pygrader. If not, see <http://www.gnu.org/licenses/>.
+"Incoming email processing."
+
from __future__ import absolute_import
from email import message_from_file as _message_from_file
from email.header import decode_header as _decode_header
-from email.utils import formatdate as _formatdate
-import hashlib as _hashlib
-import locale as _locale
+from email.mime.text import MIMEText as _MIMEText
import mailbox as _mailbox
-import os as _os
-import os.path as _os_path
+import re as _re
import sys as _sys
-import time as _time
-from pgp_mime import verify as _verify
+import pgp_mime as _pgp_mime
from lxml import etree as _etree
from . import LOG as _LOG
-from .color import standard_colors as _standard_colors
-from .color import color_string as _color_string
+from .email import construct_email as _construct_email
from .email import construct_response as _construct_response
-from .extract_mime import extract_mime as _extract_mime
from .extract_mime import message_time as _message_time
from .model.person import Person as _Person
-from .storage import assignment_path as _assignment_path
-from .storage import set_late as _set_late
+
+from .handler import InsecureMessage as _InsecureMessage
+from .handler import InvalidAssignmentSubject as _InvalidAssignmentSubject
+from .handler import InvalidMessage as _InvalidMessage
+from .handler import InvalidStudentSubject as _InvalidStudentSubject
+from .handler import InvalidSubjectMessage as _InvalidSubjectMessage
+from .handler import PermissionViolationMessage as _PermissionViolationMessage
+from .handler import Response as _Response
+from .handler import UnsignedMessage as _UnsignedMessage
+from .handler.get import run as _handle_get
+from .handler.grade import run as _handle_grade
+from .handler.grade import MissingGradeMessage as _MissingGradeMessage
+from .handler.submission import run as _handle_submission
+from .handler.submission import InvalidSubmission as _InvalidSubmission
+
+
+_TAG_REGEXP = _re.compile('^.*\[([^]]*)\].*$')
+
+
+class NoReturnPath (_InvalidMessage):
+ def __init__(self, address, **kwargs):
+ if 'error' not in kwargs:
+ kwargs['error'] = 'no Return-Path'
+ super(NoReturnPath, self).__init__(**kwargs)
+
+
+class UnregisteredAddress (_InvalidMessage):
+ def __init__(self, address, **kwargs):
+ if 'error' not in kwargs:
+ kwargs['error'] = 'unregistered address {}'.format(address)
+ super(UnregisteredAddress, self).__init__(**kwargs)
+ self.address = address
+
+
+class AmbiguousAddress (_InvalidMessage):
+ def __init__(self, address, people, **kwargs):
+ if 'error' not in kwargs:
+ kwargs['error'] = 'ambiguous address {}'.format(address)
+ super(AmbiguousAddress, self).__init__(**kwargs)
+ self.address = address
+ self.people = people
+
+
+class WrongSignatureMessage (_InsecureMessage):
+ def __init__(self, pgp_key=None, fingerprints=None, decrypted=None,
+ **kwargs):
+ if 'error' not in kwargs:
+ kwargs['error'] = 'not signed by the expected key'
+ super(WrongSignatureMessage, self).__init__(**kwargs)
+ self.pgp_key = pgp_key
+ self.fingerprints = fingerprints
+ self.decrypted = decrypted
+
+class UnverifiedSignatureMessage (_InsecureMessage):
+ def __init__(self, signature=None, decrypted=None, **kwargs):
+ if 'error' not in kwargs:
+ kwargs['error'] = 'unverified signature'
+ super(UnverifiedSignatureMessage, self).__init__(**kwargs)
+ self.signature = signature
+ self.decrypted = decrypted
+
+
+class SubjectlessMessage (_InvalidSubjectMessage):
+ def __init__(self, **kwargs):
+ if 'error' not in kwargs:
+ kwargs['error'] = 'no subject'
+ super(SubjectlessMessage, self).__init__(**kwargs)
+
+
+class InvalidHandlerMessage (_InvalidSubjectMessage):
+ def __init__(self, target=None, handlers=None, **kwargs):
+ if 'error' not in kwargs:
+ kwargs['error'] = 'no handler for {!r}'.format(target)
+ super(InvalidHandlerMessage, self).__init__(**kwargs)
+ self.target = target
+ self.handlers = handlers
def mailpipe(basedir, course, stream=None, mailbox=None, input_=None,
- output=None, max_late=0, respond=None, use_color=None,
- dry_run=False, **kwargs):
+ output=None, continue_after_invalid_message=False, max_late=0,
+ trust_email_infrastructure=False,
+ handlers={
+ 'get': _handle_get,
+ 'grade': _handle_grade,
+ 'submit': _handle_submission,
+ }, respond=None, dry_run=False, **kwargs):
"""Run from procmail to sort incomming submissions
For example, you can setup your ``.procmailrc`` like this::
# Grab all incoming homeworks emails. This rule eats matching emails
# (i.e. no further procmail processing).
:0
- * ^Subject:.*\[phys160-sub]
+ * ^Subject:.*\[phys160:submit]
| "$PYGRADE_MAILPIPE" mailpipe
If you don't want procmail to eat the message, you can use the
``c`` flag (carbon copy) by starting your rule off with ``:0 c``.
- >>> from asyncore import loop
>>> from io import StringIO
>>> from pgp_mime.email import encodedMIMEText
- >>> from pygrader.test.course import StubCourse
- >>> from pygrader.test.client import MessageSender
- >>> from pygrader.test.server import SMTPServer
-
- Messages with unrecognized ``Return-Path``\s are silently dropped:
+ >>> from .handler import InvalidMessage, Response
+ >>> from .test.course import StubCourse
>>> course = StubCourse()
- >>> def process(peer, mailfrom, rcpttos, data):
+ >>> def respond(message):
+ ... print('respond with:\\n{}'.format(message.as_string()))
+ >>> def process(message):
... mailpipe(
... basedir=course.basedir, course=course.course,
- ... stream=StringIO(data), output=course.mailbox)
+ ... stream=StringIO(message.as_string()),
+ ... output=course.mailbox,
+ ... continue_after_invalid_message=True,
+ ... respond=respond)
>>> message = encodedMIMEText('The answer is 42.')
>>> message['Message-ID'] = '<123.456@home.net>'
- >>> message['Return-Path'] = '<invalid.return.path@home.net>'
>>> message['Received'] = (
... 'from smtp.home.net (smtp.home.net [123.456.123.456]) '
... 'by smtp.mail.uu.edu (Postfix) with ESMTP id 5BA225C83EF '
... 'for <wking@tremily.us>; Sun, 09 Oct 2011 11:50:46 -0400 (EDT)')
>>> message['From'] = 'Billy B <bb@greyhavens.net>'
>>> message['To'] = 'phys101 <phys101@tower.edu>'
- >>> message['Subject'] = 'assignment 1 submission'
- >>> messages = [message]
- >>> ms = MessageSender(address=('localhost', 1025), messages=messages)
- >>> loop()
- >>> course.print_tree() # doctest: +REPORT_UDIFF
+ >>> message['Subject'] = '[submit] assignment 1'
+
+ Messages with unrecognized ``Return-Path``\s are silently dropped:
+
+ >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS
+ >>> course.print_tree() # doctest: +REPORT_UDIFF, +ELLIPSIS
course.conf
+ mail
+ mail/cur
+ mail/new
+ mail/tmp
+
+ Response to a message from an unregistered person:
+
+ >>> message['Return-Path'] = '<invalid.return.path@home.net>'
+ >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS
+ respond with:
+ Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...=="
+ MIME-Version: 1.0
+ Content-Disposition: inline
+ Date: ...
+ From: Robot101 <phys101@tower.edu>
+ Reply-to: Robot101 <phys101@tower.edu>
+ To: "invalid.return.path@home.net" <invalid.return.path@home.net>
+ Subject: unregistered address invalid.return.path@home.net
+ <BLANKLINE>
+ --===============...==
+ Content-Type: multipart/mixed; boundary="===============...=="
+ MIME-Version: 1.0
+ <BLANKLINE>
+ --===============...==
+ Content-Type: text/plain; charset="us-ascii"
+ MIME-Version: 1.0
+ Content-Transfer-Encoding: 7bit
+ Content-Disposition: inline
+ <BLANKLINE>
+ invalid.return.path@home.net,
+ <BLANKLINE>
+ Your email address is not registered with pygrader for
+ Physics 101. If you feel it should be, contact your professor
+ or TA.
+ <BLANKLINE>
+ Yours,
+ phys-101 robot
+ <BLANKLINE>
+ --===============...==
+ Content-Type: message/rfc822
+ MIME-Version: 1.0
+ <BLANKLINE>
+ Content-Type: text/plain; charset="us-ascii"
+ MIME-Version: 1.0
+ Content-Transfer-Encoding: 7bit
+ Content-Disposition: inline
+ Message-ID: <123.456@home.net>
+ Received: from smtp.home.net (smtp.home.net [123.456.123.456]) by smtp.mail.uu.edu (Postfix) with ESMTP id 5BA225C83EF for <wking@tremily.us>; Sun, 09 Oct 2011 11:50:46 -0400 (EDT)
+ From: Billy B <bb@greyhavens.net>
+ To: phys101 <phys101@tower.edu>
+ Subject: [submit] assignment 1
+ Return-Path: <invalid.return.path@home.net>
+ <BLANKLINE>
+ The answer is 42.
+ --===============...==--
+ --===============...==
+ MIME-Version: 1.0
+ Content-Transfer-Encoding: 7bit
+ Content-Description: OpenPGP digital signature
+ Content-Type: application/pgp-signature; name="signature.asc"; charset="us-ascii"
+ <BLANKLINE>
+ -----BEGIN PGP SIGNATURE-----
+ Version: GnuPG v2.0.19 (GNU/Linux)
+ <BLANKLINE>
+ ...
+ -----END PGP SIGNATURE-----
+ <BLANKLINE>
+ --===============...==--
If we add a valid ``Return-Path``, we get the expected delivery:
- >>> server = SMTPServer(
- ... ('localhost', 1025), None, process=process, count=1)
>>> del message['Return-Path']
>>> message['Return-Path'] = '<bb@greyhavens.net>'
- >>> ms = MessageSender(address=('localhost', 1025), messages=messages)
- >>> loop()
+ >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS
+ respond with:
+ Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...=="
+ MIME-Version: 1.0
+ Content-Disposition: inline
+ Date: ...
+ From: Robot101 <phys101@tower.edu>
+ Reply-to: Robot101 <phys101@tower.edu>
+ To: Bilbo Baggins <bb@shire.org>
+ Subject: Received Assignment 1 submission
+ <BLANKLINE>
+ --===============...==
+ Content-Type: text/plain; charset="us-ascii"
+ MIME-Version: 1.0
+ Content-Disposition: inline
+ Content-Transfer-Encoding: 7bit
+ <BLANKLINE>
+ Billy,
+ <BLANKLINE>
+ We received your submission for Assignment 1 on Sun, 09 Oct 2011 15:50:46 -0000.
+ <BLANKLINE>
+ Yours,
+ phys-101 robot
+ <BLANKLINE>
+ --===============...==
+ MIME-Version: 1.0
+ Content-Transfer-Encoding: 7bit
+ Content-Description: OpenPGP digital signature
+ Content-Type: application/pgp-signature; name="signature.asc"; charset="us-ascii"
+ <BLANKLINE>
+ -----BEGIN PGP SIGNATURE-----
+ Version: GnuPG v2.0.19 (GNU/Linux)
+ <BLANKLINE>
+ ...
+ -----END PGP SIGNATURE-----
+ <BLANKLINE>
+ --===============...==--
+
>>> course.print_tree() # doctest: +REPORT_UDIFF, +ELLIPSIS
Bilbo_Baggins
Bilbo_Baggins/Assignment_1
The last ``Received`` is used to timestamp the message:
- >>> server = SMTPServer(
- ... ('localhost', 1025), None, process=process, count=1)
>>> del message['Message-ID']
>>> message['Message-ID'] = '<abc.def@home.net>'
>>> del message['Received']
... 'from smtp.home.net (smtp.home.net [123.456.123.456]) '
... 'by smtp.mail.uu.edu (Postfix) with ESMTP id 5BA225C83EF '
... 'for <wking@tremily.us>; Mon, 09 Oct 2011 11:50:46 -0400 (EDT)')
- >>> messages = [message]
- >>> ms = MessageSender(address=('localhost', 1025), messages=messages)
- >>> loop()
+ >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS
+ respond with:
+ Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...=="
+ MIME-Version: 1.0
+ Content-Disposition: inline
+ Date: ...
+ From: Robot101 <phys101@tower.edu>
+ Reply-to: Robot101 <phys101@tower.edu>
+ To: Bilbo Baggins <bb@shire.org>
+ Subject: Received Assignment 1 submission
+ <BLANKLINE>
+ --===============...==
+ Content-Type: text/plain; charset="us-ascii"
+ MIME-Version: 1.0
+ Content-Disposition: inline
+ Content-Transfer-Encoding: 7bit
+ <BLANKLINE>
+ Billy,
+ <BLANKLINE>
+ We received your submission for Assignment 1 on Mon, 10 Oct 2011 16:50:46 -0000.
+ <BLANKLINE>
+ Yours,
+ phys-101 robot
+ <BLANKLINE>
+ --===============...==
+ MIME-Version: 1.0
+ Content-Transfer-Encoding: 7bit
+ Content-Description: OpenPGP digital signature
+ Content-Type: application/pgp-signature; name="signature.asc"; charset="us-ascii"
+ <BLANKLINE>
+ -----BEGIN PGP SIGNATURE-----
+ Version: GnuPG v2.0.19 (GNU/Linux)
+ <BLANKLINE>
+ ...
+ -----END PGP SIGNATURE-----
+ <BLANKLINE>
+ --===============...==--
+
>>> course.print_tree() # doctest: +REPORT_UDIFF, +ELLIPSIS
Bilbo_Baggins
Bilbo_Baggins/Assignment_1
You can send receipts to the acknowledge incoming messages, which
includes warnings about dropped messages (except for messages
without ``Return-Path`` and messages where the ``Return-Path``
- email belongs to multiple ``People``. Both of these cases should
- only come from problems with pygrader configuration).
+ email belongs to multiple ``People``. The former should only
+ occur with malicious emails, and the latter with improper pygrader
+ configurations).
Response to a successful submission:
- >>> def respond(message):
- ... print('respond with:\\n{}'.format(message.as_string()))
- >>> def process(peer, mailfrom, rcpttos, data):
- ... mailpipe(
- ... basedir=course.basedir, course=course.course,
- ... stream=StringIO(data), output=course.mailbox,
- ... respond=respond)
- >>> server = SMTPServer(
- ... ('localhost', 1025), None, process=process, count=1)
>>> del message['Message-ID']
>>> message['Message-ID'] = '<hgi.jlk@home.net>'
- >>> messages = [message]
- >>> ms = MessageSender(address=('localhost', 1025), messages=messages)
- >>> loop() # doctest: +REPORT_UDIFF, +ELLIPSIS
+ >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS
respond with:
Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...=="
MIME-Version: 1.0
From: Robot101 <phys101@tower.edu>
Reply-to: Robot101 <phys101@tower.edu>
To: Bilbo Baggins <bb@shire.org>
- Subject: received Assignment 1 submission
- <BLANKLINE>
- --===============...==
- Content-Type: multipart/mixed; boundary="===============...=="
- MIME-Version: 1.0
+ Subject: Received Assignment 1 submission
<BLANKLINE>
--===============...==
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
- Content-Transfer-Encoding: 7bit
Content-Disposition: inline
+ Content-Transfer-Encoding: 7bit
<BLANKLINE>
Billy,
<BLANKLINE>
<BLANKLINE>
Yours,
phys-101 robot
- --===============...==
- Content-Type: message/rfc822
- MIME-Version: 1.0
- <BLANKLINE>
- Content-Type: text/plain; charset="us-ascii"
- MIME-Version: 1.0
- Content-Transfer-Encoding: 7bit
- Content-Disposition: inline
- From: Billy B <bb@greyhavens.net>
- To: phys101 <phys101@tower.edu>
- Subject: assignment 1 submission
- Return-Path: <bb@greyhavens.net>
- Received: from smtp.mail.uu.edu (localhost.localdomain [127.0.0.1]) by smtp.mail.uu.edu (Postfix) with SMTP id 68CB45C8453 for <wking@tremily.us>; Mon, 10 Oct 2011 12:50:46 -0400 (EDT)
- Received: from smtp.home.net (smtp.home.net [123.456.123.456]) by smtp.mail.uu.edu (Postfix) with ESMTP id 5BA225C83EF for <wking@tremily.us>; Mon, 09 Oct 2011 11:50:46 -0400 (EDT)
- Message-ID: <hgi.jlk@home.net>
<BLANKLINE>
- The answer is 42.
- --===============...==--
--===============...==
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Response to a submission on an unsubmittable assignment:
- >>> server = SMTPServer(
- ... ('localhost', 1025), None, process=process, count=1)
>>> del message['Subject']
- >>> message['Subject'] = 'attendance 1 submission'
- >>> messages = [message]
- >>> ms = MessageSender(address=('localhost', 1025), messages=messages)
- >>> loop() # doctest: +REPORT_UDIFF, +ELLIPSIS
+ >>> message['Subject'] = '[submit] attendance 1'
+ >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS
respond with:
Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...=="
MIME-Version: 1.0
From: Robot101 <phys101@tower.edu>
Reply-to: Robot101 <phys101@tower.edu>
To: Bilbo Baggins <bb@shire.org>
- Subject: received invalid Attendance 1 submission
+ Subject: Received invalid Attendance 1 submission
<BLANKLINE>
--===============...==
Content-Type: multipart/mixed; boundary="===============...=="
<BLANKLINE>
Billy,
<BLANKLINE>
- We received your submission for Attendance 1, but you are not allowed
- to submit that assignment via email.
+ We received your submission for Attendance 1, but you are not
+ allowed to submit that assignment via email.
<BLANKLINE>
Yours,
phys-101 robot
+ <BLANKLINE>
--===============...==
Content-Type: message/rfc822
MIME-Version: 1.0
Received: from smtp.mail.uu.edu (localhost.localdomain [127.0.0.1]) by smtp.mail.uu.edu (Postfix) with SMTP id 68CB45C8453 for <wking@tremily.us>; Mon, 10 Oct 2011 12:50:46 -0400 (EDT)
Received: from smtp.home.net (smtp.home.net [123.456.123.456]) by smtp.mail.uu.edu (Postfix) with ESMTP id 5BA225C83EF for <wking@tremily.us>; Mon, 09 Oct 2011 11:50:46 -0400 (EDT)
Message-ID: <hgi.jlk@home.net>
- Subject: attendance 1 submission
+ Subject: [submit] attendance 1
<BLANKLINE>
The answer is 42.
--===============...==--
Response to a bad subject:
- >>> server = SMTPServer(
- ... ('localhost', 1025), None, process=process, count=1)
>>> del message['Subject']
>>> message['Subject'] = 'need help for the first homework'
- >>> messages = [message]
- >>> ms = MessageSender(address=('localhost', 1025), messages=messages)
- >>> loop() # doctest: +REPORT_UDIFF, +ELLIPSIS
+ >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS
respond with:
Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...=="
MIME-Version: 1.0
From: Robot101 <phys101@tower.edu>
Reply-to: Robot101 <phys101@tower.edu>
To: Bilbo Baggins <bb@shire.org>
- Subject: received 'need help for the first homework'
+ Subject: no tag in 'need help for the first homework'
<BLANKLINE>
--===============...==
Content-Type: multipart/mixed; boundary="===============...=="
<BLANKLINE>
Billy,
<BLANKLINE>
- We got an email from you with the following subject:
- 'need help for the first homework'
- which does not match any submittable assignment name for
- Physics 101.
- Remember to use the full name for the assignment in the
- subject. For example:
- Assignment 1 submission
+ We received an email message from you with an invalid
+ subject.
<BLANKLINE>
Yours,
phys-101 robot
+ <BLANKLINE>
--===============...==
Content-Type: message/rfc822
MIME-Version: 1.0
Response to a missing subject:
- >>> server = SMTPServer(
- ... ('localhost', 1025), None, process=process, count=1)
>>> del message['Subject']
- >>> messages = [message]
- >>> ms = MessageSender(address=('localhost', 1025), messages=messages)
- >>> loop() # doctest: +REPORT_UDIFF, +ELLIPSIS
+ >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS
respond with:
Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...=="
MIME-Version: 1.0
<BLANKLINE>
Yours,
phys-101 robot
+ <BLANKLINE>
--===============...==
Content-Type: message/rfc822
MIME-Version: 1.0
>>> student = course.course.person(email='bb@greyhavens.net')
>>> student.pgp_key = '4332B6E3'
- >>> server = SMTPServer(
- ... ('localhost', 1025), None, process=process, count=1)
>>> del message['Subject']
- >>> messages = [message]
- >>> ms = MessageSender(address=('localhost', 1025), messages=messages)
- >>> loop() # doctest: +REPORT_UDIFF, +ELLIPSIS
+ >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS
respond with:
Content-Type: multipart/encrypted; protocol="application/pgp-encrypted"; micalg="pgp-sha1"; boundary="===============...=="
MIME-Version: 1.0
<BLANKLINE>
--===============...==--
- Response to a message from an unregistered person:
-
- >>> server = SMTPServer(
- ... ('localhost', 1025), None, process=process, count=1)
- >>> del message['Return-Path']
- >>> message['Return-Path'] = '<invalid.return.path@home.net>'
- >>> messages = [message]
- >>> ms = MessageSender(address=('localhost', 1025), messages=messages)
- >>> loop() # doctest: +REPORT_UDIFF, +ELLIPSIS
- respond with:
- Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...=="
- MIME-Version: 1.0
- Content-Disposition: inline
- Date: ...
- From: Robot101 <phys101@tower.edu>
- Reply-to: Robot101 <phys101@tower.edu>
- To: "invalid.return.path@home.net" <invalid.return.path@home.net>
- Subject: unregistered address invalid.return.path@home.net
- <BLANKLINE>
- --===============...==
- Content-Type: multipart/mixed; boundary="===============...=="
- MIME-Version: 1.0
- <BLANKLINE>
- --===============...==
- Content-Type: text/plain; charset="us-ascii"
- MIME-Version: 1.0
- Content-Transfer-Encoding: 7bit
- Content-Disposition: inline
- <BLANKLINE>
- invalid.return.path@home.net,
- <BLANKLINE>
- Your email address is not registered with pygrader for
- Physics 101. If you feel it should be, contact your professor
- or TA.
- <BLANKLINE>
- Yours,
- phys-101 robot
- --===============...==
- Content-Type: message/rfc822
- MIME-Version: 1.0
- <BLANKLINE>
- Content-Type: text/plain; charset="us-ascii"
- MIME-Version: 1.0
- Content-Transfer-Encoding: 7bit
- Content-Disposition: inline
- From: Billy B <bb@greyhavens.net>
- To: phys101 <phys101@tower.edu>
- Received: from smtp.mail.uu.edu (localhost.localdomain [127.0.0.1]) by smtp.mail.uu.edu (Postfix) with SMTP id 68CB45C8453 for <wking@tremily.us>; Mon, 10 Oct 2011 12:50:46 -0400 (EDT)
- Received: from smtp.home.net (smtp.home.net [123.456.123.456]) by smtp.mail.uu.edu (Postfix) with ESMTP id 5BA225C83EF for <wking@tremily.us>; Mon, 09 Oct 2011 11:50:46 -0400 (EDT)
- Message-ID: <hgi.jlk@home.net>
- Return-Path: <invalid.return.path@home.net>
- <BLANKLINE>
- The answer is 42.
- --===============...==--
- --===============...==
- MIME-Version: 1.0
- Content-Transfer-Encoding: 7bit
- Content-Description: OpenPGP digital signature
- Content-Type: application/pgp-signature; name="signature.asc"; charset="us-ascii"
- <BLANKLINE>
- -----BEGIN PGP SIGNATURE-----
- Version: GnuPG v2.0.19 (GNU/Linux)
- <BLANKLINE>
- ...
- -----END PGP SIGNATURE-----
- <BLANKLINE>
- --===============...==--
-
>>> course.cleanup()
"""
if stream is None:
stream = _sys.stdin
- for msg,person,assignment,time in _load_messages(
+ for original,message,person,subject,target in _load_messages(
course=course, stream=stream, mailbox=mailbox, input_=input_,
- output=output, respond=respond, use_color=use_color, dry_run=dry_run):
- assignment_path = _assignment_path(basedir, assignment, person)
- _save_local_message_copy(
- msg=msg, person=person, assignment_path=assignment_path,
- use_color=use_color, dry_run=dry_run)
- _extract_mime(message=msg, output=assignment_path, dry_run=dry_run)
- _check_late(
- basedir=basedir, assignment=assignment, person=person, time=time,
- max_late=max_late, use_color=use_color, dry_run=dry_run)
+ output=output, dry_run=dry_run,
+ continue_after_invalid_message=continue_after_invalid_message,
+ trust_email_infrastructure=trust_email_infrastructure,
+ respond=respond):
+ try:
+ handler = _get_handler(handlers=handlers, target=target)
+ _LOG.debug('handling {}'.format(target))
+ handler(
+ basedir=basedir, course=course, message=message,
+ person=person, subject=subject,
+ max_late=max_late,
+ trust_email_infrastructure=trust_email_infrastructure,
+ dry_run=dry_run)
+ except _InvalidMessage as error:
+ error.course = course
+ error.message = original
+ for attribute,value in [('person', person),
+ ('subject', subject),
+ ('target', target)]:
+ if (value is not None and
+ getattr(error, attribute, None) is None):
+ setattr(error, attribute, value)
+ _LOG.warn('invalid message {}'.format(error.message_id()))
+ if not continue_after_invalid_message:
+ raise
+ _LOG.warn('{}'.format(error))
+ if respond:
+ response = _get_error_response(error)
+ respond(response)
+ except _Response as response:
+ if respond:
+ msg = response.message
+ if not response.complete:
+ author = course.robot
+ target = person
+ msg = response.message
+ if isinstance(response.message, _MIMEText):
+ # Manipulate body (based on pgp_mime.append_text)
+ original_encoding = msg.get_charset().input_charset
+ original_payload = str(
+ msg.get_payload(decode=True), original_encoding)
+ new_payload = (
+ '{},\n\n'
+ '{}\n\n'
+ 'Yours,\n'
+ '{}\n').format(
+ target.alias(), original_payload, author.alias())
+ new_encoding = _pgp_mime.guess_encoding(new_payload)
+ if msg.get('content-transfer-encoding', None):
+ # clear CTE so set_payload will set it properly
+ del msg['content-transfer-encoding']
+ msg.set_payload(new_payload, new_encoding)
+ subject = msg['Subject']
+ assert subject is not None, msg
+ del msg['Subject']
+ msg = _construct_email(
+ author=author, targets=[person], subject=subject,
+ message=msg)
+ respond(msg)
def _load_messages(course, stream, mailbox=None, input_=None, output=None,
- respond=None, use_color=None, dry_run=False):
+ continue_after_invalid_message=False,
+ trust_email_infrastructure=False, respond=None,
+ dry_run=False):
if mailbox is None:
+ _LOG.debug('loading message from {}'.format(stream))
mbox = None
messages = [(None,_message_from_file(stream))]
if output is not None:
ombox = _mailbox.mbox(output, factory=None, create=True)
elif mailbox == 'maildir':
mbox = _mailbox.Maildir(input_, factory=None, create=False)
- messages = mbox.items()
+ messages = []
+ for key,msg in mbox.items():
+ subpath = mbox._lookup(key)
+ if subpath.endswith('.gitignore'):
+ _LOG.debug('skipping non-message {}'.format(subpath))
+ continue
+ messages.append((key, msg))
if output is not None:
ombox = _mailbox.Maildir(output, factory=None, create=True)
else:
raise ValueError(mailbox)
+ messages.sort(key=_get_message_time)
for key,msg in messages:
- ret = _parse_message(
- course=course, msg=msg, respond=respond, use_color=use_color)
- if ret:
- if output is not None and dry_run is False:
- # move message from input mailbox to output mailbox
- ombox.add(msg)
- if mbox is not None:
- del mbox[key]
- yield ret
-
-def _parse_message(course, msg, respond=None, use_color=None):
- highlight,lowlight,good,bad = _standard_colors(use_color=use_color)
- mid = msg['Message-ID']
- sender = msg['Return-Path'] # RFC 822
+ try:
+ ret = _parse_message(
+ course=course, message=msg,
+ trust_email_infrastructure=trust_email_infrastructure)
+ except _InvalidMessage as error:
+ error.message = msg
+ _LOG.warn('invalid message {}'.format(error.message_id()))
+ if not continue_after_invalid_message:
+ raise
+ _LOG.warn('{}'.format(error))
+ if respond:
+ response = _get_error_response(error)
+ if response is not None:
+ respond(response)
+ continue
+ if output is not None and dry_run is False:
+ # move message from input mailbox to output mailbox
+ ombox.add(msg)
+ if mbox is not None:
+ del mbox[key]
+ yield ret
+
+def _parse_message(course, message, trust_email_infrastructure=False):
+ """Parse an incoming email and respond if neccessary.
+
+ Return ``(msg, person, assignment, time)`` on successful parsing.
+ Return ``None`` on failure.
+ """
+ original = message
+ person = subject = target = None
+ try:
+ person = _get_message_person(course=course, message=message)
+ if person.pgp_key:
+ _LOG.debug('verify message is from {}'.format(person))
+ try:
+ message = _get_verified_message(message, person.pgp_key)
+ except _UnsignedMessage as error:
+ if trust_email_infrastructure:
+ _LOG.warn('{}'.format(error))
+ else:
+ raise
+ subject = _get_message_subject(message=message)
+ target = _get_message_target(subject=subject)
+ except _InvalidMessage as error:
+ error.course = course
+ error.message = original
+ for attribute,value in [('person', person),
+ ('subject', subject),
+ ('target', target)]:
+ if (value is not None and
+ getattr(error, attribute, None) is None):
+ setattr(error, attribute, value)
+ raise
+ return (original, message, person, subject, target)
+
+def _get_message_person(course, message):
+ sender = message['Return-Path'] # RFC 822
if sender is None:
- _LOG.debug(_color_string(
- string='no Return-Path in {}'.format(mid), color=lowlight))
- return None
+ raise NoReturnPath(message)
sender = sender[1:-1] # strip wrapping '<' and '>'
-
people = list(course.find_people(email=sender))
if len(people) == 0:
- _LOG.warn(_color_string(
- string='no person found to match {}'.format(sender),
- color=bad))
- if respond:
- person = _Person(name=sender, emails=[sender])
- response_subject = 'unregistered address {}'.format(sender)
- response_text = (
- '{},\n\n'
- 'Your email address is not registered with pygrader for\n'
- '{}. If you feel it should be, contact your professor\n'
- 'or TA.\n\n'
- 'Yours,\n{}').format(
- sender, course.name, course.robot.alias())
- response = _construct_response(
- author=course.robot, targets=[person],
- subject=response_subject, text=response_text, original=msg)
- respond(response)
- return None
+ raise UnregisteredAddress(message=message, address=sender)
if len(people) > 1:
- _LOG.warn(_color_string(
- string='multiple people match {} ({})'.format(
- sender, ', '.join(str(p) for p in people)),
- color=bad))
- return None
- person = people[0]
-
- if person.pgp_key:
- original = msg
- msg = _get_verified_message(msg, person.pgp_key, use_color=use_color)
- if msg is None:
- if respond:
- response_subject = 'unsigned message {}'.format(mid)
- response_text = (
- '{},\n\n'
- 'We received an email message from you without a valid\n'
- 'PGP signature.\n\n'
- 'Yours,\n{}').format(
- person.alias(), course.robot.alias())
- response = _construct_response(
- author=course.robot, targets=[person],
- subject=response_subject, text=response_text,
- original=original)
- respond(response)
- return None
-
- if msg['Subject'] is None:
- _LOG.warn(_color_string(
- string='no subject in {}'.format(mid), color=bad))
- if respond:
- response_subject = 'no subject in {}'.format(mid)
- response_text = (
- '{},\n\n'
- 'We received an email message from you without a subject.\n\n'
- 'Yours,\n{}').format(
- person.alias(), course.robot.alias())
- response = _construct_response(
- author=course.robot, targets=[person],
- subject=response_subject, text=response_text, original=msg)
- respond(response)
- return None
- parts = _decode_header(msg['Subject'])
- if len(parts) != 1:
- _LOG.warn(_color_string(
- string='multi-part header {}'.format(parts), color=bad))
- return None
- subject,encoding = parts[0]
- if encoding is None:
- encoding = 'ascii'
+ raise AmbiguousAddress(message=message, address=sender, people=people)
+ return people[0]
+
+def _get_message_subject(message):
+ """
+ >>> from email.header import Header
+ >>> from pgp_mime.email import encodedMIMEText
+ >>> message = encodedMIMEText('The answer is 42.')
+ >>> message['Message-ID'] = 'msg-id'
+ >>> _get_message_subject(message=message)
+ Traceback (most recent call last):
+ ...
+ pygrader.mailpipe.SubjectlessMessage: no subject
+ >>> del message['Subject']
+ >>> subject = Header('unicode part', 'utf-8')
+ >>> subject.append('-ascii part', 'ascii')
+ >>> message['Subject'] = subject.encode()
+ >>> _get_message_subject(message=message)
+ 'unicode part-ascii part'
+ >>> del message['Subject']
+ >>> message['Subject'] = 'clean subject'
+ >>> _get_message_subject(message=message)
+ 'clean subject'
+ """
+ if message['Subject'] is None:
+ raise SubjectlessMessage(subject=None, message=message)
+
+ parts = _decode_header(message['Subject'])
+ part_strings = []
+ for string,encoding in parts:
+ if encoding is None:
+ encoding = 'ascii'
+ if not isinstance(string, str):
+ string = str(string, encoding)
+ part_strings.append(string)
+ subject = ''.join(part_strings)
_LOG.debug('decoded header {} -> {}'.format(parts[0], subject))
+ return subject.lower().replace('#', '')
- subject = subject.lower().replace('#', '')
- for assignment in course.assignments:
- if _match_assignment(assignment, subject):
- break
- if not _match_assignment(assignment, subject):
- _LOG.warn(_color_string(
- string='no assignment found in {}'.format(repr(subject)),
- color=bad))
- if respond:
- response_subject = "received '{}'".format(subject)
- submittable_assignments = [
- a for a in course.assignments if a.submittable]
- if not submittable_assignments:
- hint = (
- 'In fact, there are no submittable assignments for\n'
- 'this course!\n')
- else:
- hint = (
- 'Remember to use the full name for the assignment in the\n'
- 'subject. For example:\n'
- ' {} submission\n\n').format(
- submittable_assignments[0].name)
- response_text = (
- '{},\n\n'
- 'We got an email from you with the following subject:\n'
- ' {}\n'
- 'which does not match any submittable assignment name for\n'
- '{}.\n'
- '{}'
- 'Yours,\n{}').format(
- person.alias(), repr(subject), course.name, hint,
- course.robot.alias())
- response = _construct_response(
- author=course.robot, targets=[person],
- subject=response_subject, text=response_text, original=msg)
- respond(response)
- return None
-
- if not assignment.submittable:
- response_subject = 'received invalid {} submission'.format(
- assignment.name)
- response_text = (
- '{},\n\n'
- 'We received your submission for {}, but you are not allowed\n'
- 'to submit that assignment via email.\n\n'
- 'Yours,\n{}').format(
- person.alias(), assignment.name, course.robot.alias())
- response = _construct_response(
- author=course.robot, targets=[person],
- subject=response_subject, text=response_text, original=msg)
- respond(response)
-
- time = _message_time(message=msg, use_color=use_color)
-
- if respond:
- response_subject = 'received {} submission'.format(assignment.name)
- if time:
- time_str = 'on {}'.format(_formatdate(time))
- else:
- time_str = 'at an unknown time'
- response_text = (
- '{},\n\n'
- 'We received your submission for {} {}.\n\n'
- 'Yours,\n{}').format(
- person.alias(), assignment.name, time_str, course.robot.alias())
- response = _construct_response(
- author=course.robot, targets=[person],
- subject=response_subject, text=response_text, original=msg)
- respond(response)
- return (msg, person, assignment, time)
-
-def _match_assignment(assignment, subject):
- return assignment.name.lower() in subject
-
-def _save_local_message_copy(msg, person, assignment_path, use_color=None,
- dry_run=False):
- highlight,lowlight,good,bad = _standard_colors(use_color=use_color)
- try:
- _os.makedirs(assignment_path)
- except OSError:
- pass
- mpath = _os_path.join(assignment_path, 'mail')
+def _get_message_target(subject):
+ """
+ >>> _get_message_target(subject='no tag')
+ Traceback (most recent call last):
+ ...
+ pygrader.handler.InvalidSubjectMessage: no tag in 'no tag'
+ >>> _get_message_target(subject='[] empty tag')
+ Traceback (most recent call last):
+ ...
+ pygrader.handler.InvalidSubjectMessage: empty tag in '[] empty tag'
+ >>> _get_message_target(subject='[abc] empty tag')
+ 'abc'
+ >>> _get_message_target(subject='[phys160:abc] empty tag')
+ 'abc'
+ """
+ match = _TAG_REGEXP.match(subject)
+ if match is None:
+ raise _InvalidSubjectMessage(
+ subject=subject, error='no tag in {!r}'.format(subject))
+ tag = match.group(1)
+ if tag == '':
+ raise _InvalidSubjectMessage(
+ subject=subject, error='empty tag in {!r}'.format(subject))
+ target = tag.rsplit(':', 1)[-1]
+ _LOG.debug('extracted target {} -> {}'.format(subject, target))
+ return target
+
+def _get_handler(handlers, target):
try:
- mbox = _mailbox.Maildir(mpath, factory=None, create=not dry_run)
- except _mailbox.NoSuchMailboxError as e:
- _LOG.debug(_color_string(
- string='could not open mailbox at {}'.format(mpath),
- color=bad))
- mbox = None
- new_msg = True
- else:
- new_msg = True
- for other_msg in mbox:
- if other_msg['Message-ID'] == msg['Message-ID']:
- new_msg = False
- break
- if new_msg:
- _LOG.debug(_color_string(
- string='saving email from {} to {}'.format(
- person, assignment_path), color=good))
- if mbox is not None and not dry_run:
- mdmsg = _mailbox.MaildirMessage(msg)
- mdmsg.add_flag('S')
- mbox.add(mdmsg)
- mbox.close()
- else:
- _LOG.debug(_color_string(
- string='already found {} in {}'.format(
- msg['Message-ID'], mpath), color=good))
-
-def _check_late(basedir, assignment, person, time, max_late=0, use_color=None,
- dry_run=False):
- highlight,lowlight,good,bad = _standard_colors(use_color=use_color)
- if time > assignment.due + max_late:
- dt = time - assignment.due
- _LOG.warn(_color_string(
- string='{} {} late by {} seconds ({} hours)'.format(
- person.name, assignment.name, dt, dt/3600.),
- color=bad))
- if not dry_run:
- _set_late(basedir=basedir, assignment=assignment, person=person)
-
-def _get_verified_message(message, pgp_key, use_color=None):
+ handler = handlers[target]
+ except KeyError as error:
+ raise InvalidHandlerMessage(
+ target=target, handlers=handlers) from error
+ return handler
+
+def _get_verified_message(message, pgp_key):
"""
>>> from pgp_mime import sign, encodedMIMEText
We check that the message is signed, and that it is signed by the
appropriate key.
+ >>> signed.authenticated
+ Traceback (most recent call last):
+ ...
+ AttributeError: 'MIMEMultipart' object has no attribute 'authenticated'
>>> our_message = _get_verified_message(signed, pgp_key='4332B6E3')
>>> print(our_message.as_string()) # doctest: +REPORT_UDIFF
Content-Type: text/plain; charset="us-ascii"
Received: from smtp.home.net ...
<BLANKLINE>
1.23 joules
+ >>> our_message.authenticated
+ True
- If it is signed, but not by the right key, we get ``None``.
+ If it is signed, but not by the right key, we get an error.
>>> print(_get_verified_message(signed, pgp_key='01234567'))
- None
+ Traceback (most recent call last):
+ ...
+ pygrader.mailpipe.WrongSignatureMessage: not signed by the expected key
- If it is not signed at all, we get ``None``.
+ If it is not signed at all, we get another error.
>>> print(_get_verified_message(message, pgp_key='4332B6E3'))
- None
+ Traceback (most recent call last):
+ ...
+ pygrader.handler.UnsignedMessage: unsigned message
"""
- highlight,lowlight,good,bad = _standard_colors(use_color=use_color)
mid = message['message-id']
try:
- decrypted,verified,result = _verify(message=message)
- except (ValueError, AssertionError):
- _LOG.warn(_color_string(
- string='could not verify {} (not signed?)'.format(mid),
- color=bad))
- return None
- _LOG.info(_color_string(str(result, 'utf-8'), color=lowlight))
+ decrypted,verified,result = _pgp_mime.verify(message=message)
+ except (ValueError, AssertionError) as error:
+ raise _UnsignedMessage(message=message) from error
+ _LOG.debug(str(result, 'utf-8'))
tree = _etree.fromstring(result.replace(b'\x00', b''))
match = None
+ fingerprints = []
for signature in tree.findall('.//signature'):
for fingerprint in signature.iterchildren('fpr'):
- if fingerprint.text.endswith(pgp_key):
- match = signature
- break
- if match is None:
- _LOG.warn(_color_string(
- string='{} is not signed by the expected key'.format(mid),
- color=bad))
- return None
+ fingerprints.append(fingerprint)
+ matches = [f for f in fingerprints if f.text.endswith(pgp_key)]
+ if len(matches) == 0:
+ raise WrongSignatureMessage(
+ message=message, pgp_key=pgp_key, fingerprints=fingerprints,
+ decrypted=decrypted)
+ match = matches[0]
if not verified:
sumhex = list(signature.iterchildren('summary'))[0].get('value')
summary = int(sumhex, 16)
if summary != 0:
- _LOG.warn(_color_string(
- string='{} has an unverified signature'.format(mid),
- color=bad))
- return None
+ raise UnverifiedSignatureMessage(
+ message=message, signature=signature, decrypted=decrypted)
# otherwise, we may have an untrusted key. We'll count that
# as verified here, because the caller is explicity looking
# for signatures by this fingerprint.
'content-disposition',
]:
decrypted[k] = v
+ decrypted.authenticated = True
return decrypted
+
+def _get_error_response(error):
+ author = error.course.robot
+ target = getattr(error, 'person', None)
+ subject = str(error)
+ if isinstance(error, _InvalidSubmission):
+ subject = 'Received invalid {} submission'.format(
+ error.assignment.name)
+ text = (
+ 'We received your submission for {}, but you are not\n'
+ 'allowed to submit that assignment via email.'
+ ).format(error.assignment.name)
+ elif isinstance(error, _MissingGradeMessage):
+ subject = 'No grade in {!r}'.format(error.subject)
+ text = (
+ 'Your grade submission did not include a text/plain\n'
+ 'part containing the new grade and comment.'
+ )
+ elif isinstance(error, InvalidHandlerMessage):
+ targets = sorted(error.handlers.keys())
+ if not targets:
+ hint = (
+ 'In fact, there are no available handlers for this\n'
+ 'course!')
+ else:
+ hint = (
+ 'Perhaps you meant to use one of the following:\n'
+ ' {}').format('\n '.join(targets))
+ text = (
+ 'We got an email from you with the following subject:\n'
+ ' {!r}\n'
+ 'which does not match any submittable handler name for\n'
+ '{}.\n'
+ '{}').format(error.subject, error.course.name, hint)
+ elif isinstance(error, SubjectlessMessage):
+ subject = 'no subject in {}'.format(error.message['Message-ID'])
+ text = 'We received an email message from you without a subject.'
+ elif isinstance(error, AmbiguousAddress):
+ text = (
+ 'Multiple people match {} ({})'.format(
+ error.address, ', '.join(p.name for p in error.people)))
+ elif isinstance(error, UnregisteredAddress):
+ target = _Person(name=error.address, emails=[error.address])
+ text = (
+ 'Your email address is not registered with pygrader for\n'
+ '{}. If you feel it should be, contact your professor\n'
+ 'or TA.').format(error.course.name)
+ elif isinstance(error, NoReturnPath):
+ return
+ elif isinstance(error, _InvalidAssignmentSubject):
+ if error.assignments:
+ hint = (
+ 'but it matches several assignments:\n'
+ ' * {}').format('\n * '.join(
+ a.name for a in error.assignments))
+ else:
+ # prefer a submittable example assignment
+ assignments = [
+ a for a in error.course.assignments if a.submittable]
+ assignments += course.assignments # but fall back to any one
+ hint = (
+ 'Remember to use the full name for the assignment in the\n'
+ 'subject. For example:\n'
+ ' {} submission').format(assignments[0].name)
+ text = (
+ 'We got an email from you with the following subject:\n'
+ ' {!r}\n{}').format(error.subject, hint)
+ elif isinstance(error, _InvalidStudentSubject):
+ text = (
+ 'We got an email from you with the following subject:\n'
+ ' {!r}\n'
+ 'but it matches several students:\n'
+ ' * {}').format(
+ error.subject, '\n * '.join(s.name for s in error.students))
+ elif isinstance(error, _InvalidSubjectMessage):
+ text = (
+ 'We received an email message from you with an invalid\n'
+ 'subject.')
+ elif isinstance(error, _UnsignedMessage):
+ subject = 'unsigned message {}'.format(error.message['Message-ID'])
+ text = (
+ 'We received an email message from you without a valid\n'
+ 'PGP signature.'
+ )
+ elif isinstance(error, _PermissionViolationMessage):
+ text = (
+ 'We got an email from you with the following subject:\n'
+ ' {!r}\n'
+ "but you can't do that unless you belong to one of the\n"
+ 'following groups:\n'
+ ' * {}').format(
+ error.subject, '\n * '.join(error.allowed_groups))
+ elif isinstance(error, _InvalidMessage):
+ text = subject
+ else:
+ raise NotImplementedError((type(error), error))
+ if target is None:
+ raise NotImplementedError((type(error), error))
+ return _construct_response(
+ author=author,
+ targets=[target],
+ subject=subject,
+ text=(
+ '{},\n\n'
+ '{}\n\n'
+ 'Yours,\n'
+ '{}\n'.format(target.alias(), text, author.alias())),
+ original=error.message)
+
+def _get_message_time(key_message):
+ "Key function for sorting mailbox (key,message) tuples."
+ key,message = key_message
+ return _message_time(message)
projects / pygrader.git / blobdiff