1 # Copyright (C) 2012 W. Trevor King <wking@tremily.us>
3 # This file is part of pygrader.
5 # pygrader is free software: you can redistribute it and/or modify it under the
6 # terms of the GNU General Public License as published by the Free Software
7 # Foundation, either version 3 of the License, or (at your option) any later
10 # pygrader is distributed in the hope that it will be useful, but WITHOUT ANY
11 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
12 # A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 # You should have received a copy of the GNU General Public License along with
15 # pygrader. If not, see <http://www.gnu.org/licenses/>.
17 "Incoming email processing."
19 from __future__ import absolute_import
21 from email import message_from_file as _message_from_file
22 from email.header import decode_header as _decode_header
23 from email.mime.text import MIMEText as _MIMEText
24 from email.utils import parseaddr as _parseaddr
25 import mailbox as _mailbox
29 import pgp_mime as _pgp_mime
30 import pgp_mime.key as _pgp_mime_key
32 from . import LOG as _LOG
33 from .email import construct_email as _construct_email
34 from .email import construct_response as _construct_response
35 from .extract_mime import message_time as _message_time
36 from .model.person import Person as _Person
38 from .handler import InsecureMessage as _InsecureMessage
39 from .handler import InvalidAssignmentSubject as _InvalidAssignmentSubject
40 from .handler import InvalidMessage as _InvalidMessage
41 from .handler import InvalidStudentSubject as _InvalidStudentSubject
42 from .handler import InvalidSubjectMessage as _InvalidSubjectMessage
43 from .handler import PermissionViolationMessage as _PermissionViolationMessage
44 from .handler import Response as _Response
45 from .handler import UnsignedMessage as _UnsignedMessage
46 from .handler.get import run as _handle_get
47 from .handler.grade import run as _handle_grade
48 from .handler.grade import MissingGradeMessage as _MissingGradeMessage
49 from .handler.submission import run as _handle_submission
50 from .handler.submission import InvalidSubmission as _InvalidSubmission
53 _TAG_REGEXP = _re.compile('^.*\[([^]]*)\].*$')
56 class NoReturnPath (_InvalidMessage):
57 def __init__(self, address, **kwargs):
58 if 'error' not in kwargs:
59 kwargs['error'] = 'no Return-Path'
60 super(NoReturnPath, self).__init__(**kwargs)
63 class UnregisteredAddress (_InvalidMessage):
64 def __init__(self, address, **kwargs):
65 if 'error' not in kwargs:
66 kwargs['error'] = 'unregistered address {}'.format(address)
67 super(UnregisteredAddress, self).__init__(**kwargs)
68 self.address = address
71 class AmbiguousAddress (_InvalidMessage):
72 def __init__(self, address, people, **kwargs):
73 if 'error' not in kwargs:
74 kwargs['error'] = 'ambiguous address {}'.format(address)
75 super(AmbiguousAddress, self).__init__(**kwargs)
76 self.address = address
80 class WrongSignatureMessage (_InsecureMessage):
81 def __init__(self, pgp_key=None, signatures=None, fingerprints=None,
82 decrypted=None, **kwargs):
83 if 'error' not in kwargs:
84 kwargs['error'] = 'not signed by the expected key'
85 super(WrongSignatureMessage, self).__init__(**kwargs)
86 self.pgp_key = pgp_key
87 self.signatures = signatures
88 self.fingerprints = fingerprints
89 self.decrypted = decrypted
92 class UnverifiedSignatureMessage (_InsecureMessage):
93 def __init__(self, signature=None, decrypted=None, **kwargs):
94 if 'error' not in kwargs:
95 kwargs['error'] = 'unverified signature'
96 super(UnverifiedSignatureMessage, self).__init__(**kwargs)
97 self.signature = signature
98 self.decrypted = decrypted
101 class SubjectlessMessage (_InvalidSubjectMessage):
102 def __init__(self, **kwargs):
103 if 'error' not in kwargs:
104 kwargs['error'] = 'no subject'
105 super(SubjectlessMessage, self).__init__(**kwargs)
108 class InvalidHandlerMessage (_InvalidSubjectMessage):
109 def __init__(self, target=None, handlers=None, **kwargs):
110 if 'error' not in kwargs:
111 kwargs['error'] = 'no handler for {!r}'.format(target)
112 super(InvalidHandlerMessage, self).__init__(**kwargs)
114 self.handlers = handlers
117 def mailpipe(basedir, course, stream=None, mailbox=None, input_=None,
118 output=None, continue_after_invalid_message=False, max_late=0,
119 trust_email_infrastructure=False,
122 'grade': _handle_grade,
123 'submit': _handle_submission,
124 }, respond=None, dry_run=False, **kwargs):
125 """Run from procmail to sort incomming submissions
127 For example, you can setup your ``.procmailrc`` like this::
132 DEFAULT=$MAILDIR/mbox
133 LOGFILE=$MAILDIR/procmail.log
135 PYGRADE_MAILPIPE="pg.py -d $HOME/grades/phys160"
137 # Grab all incoming homeworks emails. This rule eats matching emails
138 # (i.e. no further procmail processing).
140 * ^Subject:.*\[phys160:submit]
141 | "$PYGRADE_MAILPIPE" mailpipe
143 If you don't want procmail to eat the message, you can use the
144 ``c`` flag (carbon copy) by starting your rule off with ``:0 c``.
146 >>> from io import StringIO
147 >>> from pgp_mime.email import encodedMIMEText
148 >>> from .handler import InvalidMessage, Response
149 >>> from .test.course import StubCourse
151 >>> course = StubCourse()
152 >>> def respond(message):
153 ... print('respond with:\\n{}'.format(message.as_string()))
154 >>> def process(message):
156 ... basedir=course.basedir, course=course.course,
157 ... stream=StringIO(message.as_string()),
158 ... output=course.mailbox,
159 ... continue_after_invalid_message=True,
161 >>> message = encodedMIMEText('The answer is 42.')
162 >>> message['Message-ID'] = '<123.456@home.net>'
163 >>> message['Received'] = (
164 ... 'from smtp.home.net (smtp.home.net [123.456.123.456]) '
165 ... 'by smtp.mail.uu.edu (Postfix) with ESMTP id 5BA225C83EF '
166 ... 'for <wking@tremily.us>; Sun, 09 Oct 2011 11:50:46 -0400 (EDT)')
167 >>> message['From'] = 'Billy B <bb@greyhavens.net>'
168 >>> message['To'] = 'phys101 <phys101@tower.edu>'
169 >>> message['Subject'] = '[submit] assignment 1'
171 Messages with unrecognized ``Return-Path``\s are silently dropped:
173 >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS
174 >>> course.print_tree() # doctest: +REPORT_UDIFF, +ELLIPSIS
181 Response to a message from an unregistered person:
183 >>> message['Return-Path'] = '<invalid.return.path@home.net>'
184 >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS
186 Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...=="
188 Content-Disposition: inline
190 From: Robot101 <phys101@tower.edu>
191 Reply-to: Robot101 <phys101@tower.edu>
192 To: "invalid.return.path@home.net" <invalid.return.path@home.net>
193 Subject: unregistered address invalid.return.path@home.net
195 --===============...==
196 Content-Type: multipart/mixed; boundary="===============...=="
199 --===============...==
200 Content-Type: text/plain; charset="us-ascii"
202 Content-Transfer-Encoding: 7bit
203 Content-Disposition: inline
205 invalid.return.path@home.net,
207 Your email address is not registered with pygrader for
208 Physics 101. If you feel it should be, contact your professor
214 --===============...==
215 Content-Type: message/rfc822
218 Content-Type: text/plain; charset="us-ascii"
220 Content-Transfer-Encoding: 7bit
221 Content-Disposition: inline
222 Message-ID: <123.456@home.net>
223 Received: from smtp.home.net (smtp.home.net [123.456.123.456]) by smtp.mail.uu.edu (Postfix) with ESMTP id 5BA225C83EF for <wking@tremily.us>; Sun, 09 Oct 2011 11:50:46 -0400 (EDT)
224 From: Billy B <bb@greyhavens.net>
225 To: phys101 <phys101@tower.edu>
226 Subject: [submit] assignment 1
227 Return-Path: <invalid.return.path@home.net>
230 --===============...==--
231 --===============...==
233 Content-Transfer-Encoding: 7bit
234 Content-Description: OpenPGP digital signature
235 Content-Type: application/pgp-signature; name="signature.asc"; charset="us-ascii"
237 -----BEGIN PGP SIGNATURE-----
238 Version: GnuPG v2.0.19 (GNU/Linux)
241 -----END PGP SIGNATURE-----
243 --===============...==--
245 If we add a valid ``Return-Path``, we get the expected delivery:
247 >>> del message['Return-Path']
248 >>> message['Return-Path'] = '<bb@greyhavens.net>'
249 >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS
251 Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...=="
253 Content-Disposition: inline
255 From: Robot101 <phys101@tower.edu>
256 Reply-to: Robot101 <phys101@tower.edu>
257 To: Bilbo Baggins <bb@shire.org>
258 Subject: Received Assignment 1 submission
260 --===============...==
261 Content-Type: text/plain; charset="us-ascii"
263 Content-Disposition: inline
264 Content-Transfer-Encoding: 7bit
268 We received your submission for Assignment 1 on Sun, 09 Oct 2011 15:50:46 -0000.
273 --===============...==
275 Content-Transfer-Encoding: 7bit
276 Content-Description: OpenPGP digital signature
277 Content-Type: application/pgp-signature; name="signature.asc"; charset="us-ascii"
279 -----BEGIN PGP SIGNATURE-----
280 Version: GnuPG v2.0.19 (GNU/Linux)
283 -----END PGP SIGNATURE-----
285 --===============...==--
287 >>> course.print_tree() # doctest: +REPORT_UDIFF, +ELLIPSIS
289 Bilbo_Baggins/Assignment_1
290 Bilbo_Baggins/Assignment_1/mail
291 Bilbo_Baggins/Assignment_1/mail/cur
292 Bilbo_Baggins/Assignment_1/mail/new
293 Bilbo_Baggins/Assignment_1/mail/new/...:2,S
294 Bilbo_Baggins/Assignment_1/mail/tmp
302 The last ``Received`` is used to timestamp the message:
304 >>> del message['Message-ID']
305 >>> message['Message-ID'] = '<abc.def@home.net>'
306 >>> del message['Received']
307 >>> message['Received'] = (
308 ... 'from smtp.mail.uu.edu (localhost.localdomain [127.0.0.1]) '
309 ... 'by smtp.mail.uu.edu (Postfix) with SMTP id 68CB45C8453 '
310 ... 'for <wking@tremily.us>; Mon, 10 Oct 2011 12:50:46 -0400 (EDT)')
311 >>> message['Received'] = (
312 ... 'from smtp.home.net (smtp.home.net [123.456.123.456]) '
313 ... 'by smtp.mail.uu.edu (Postfix) with ESMTP id 5BA225C83EF '
314 ... 'for <wking@tremily.us>; Mon, 09 Oct 2011 11:50:46 -0400 (EDT)')
315 >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS
317 Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...=="
319 Content-Disposition: inline
321 From: Robot101 <phys101@tower.edu>
322 Reply-to: Robot101 <phys101@tower.edu>
323 To: Bilbo Baggins <bb@shire.org>
324 Subject: Received Assignment 1 submission
326 --===============...==
327 Content-Type: text/plain; charset="us-ascii"
329 Content-Disposition: inline
330 Content-Transfer-Encoding: 7bit
334 We received your submission for Assignment 1 on Mon, 10 Oct 2011 16:50:46 -0000.
339 --===============...==
341 Content-Transfer-Encoding: 7bit
342 Content-Description: OpenPGP digital signature
343 Content-Type: application/pgp-signature; name="signature.asc"; charset="us-ascii"
345 -----BEGIN PGP SIGNATURE-----
346 Version: GnuPG v2.0.19 (GNU/Linux)
349 -----END PGP SIGNATURE-----
351 --===============...==--
353 >>> course.print_tree() # doctest: +REPORT_UDIFF, +ELLIPSIS
355 Bilbo_Baggins/Assignment_1
356 Bilbo_Baggins/Assignment_1/late
357 Bilbo_Baggins/Assignment_1/mail
358 Bilbo_Baggins/Assignment_1/mail/cur
359 Bilbo_Baggins/Assignment_1/mail/new
360 Bilbo_Baggins/Assignment_1/mail/new/...:2,S
361 Bilbo_Baggins/Assignment_1/mail/new/...:2,S
362 Bilbo_Baggins/Assignment_1/mail/tmp
371 You can send receipts to the acknowledge incoming messages, which
372 includes warnings about dropped messages (except for messages
373 without ``Return-Path`` and messages where the ``Return-Path``
374 email belongs to multiple ``People``. The former should only
375 occur with malicious emails, and the latter with improper pygrader
378 Response to a successful submission:
380 >>> del message['Message-ID']
381 >>> message['Message-ID'] = '<hgi.jlk@home.net>'
382 >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS
384 Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...=="
386 Content-Disposition: inline
388 From: Robot101 <phys101@tower.edu>
389 Reply-to: Robot101 <phys101@tower.edu>
390 To: Bilbo Baggins <bb@shire.org>
391 Subject: Received Assignment 1 submission
393 --===============...==
394 Content-Type: text/plain; charset="us-ascii"
396 Content-Disposition: inline
397 Content-Transfer-Encoding: 7bit
401 We received your submission for Assignment 1 on Mon, 10 Oct 2011 16:50:46 -0000.
406 --===============...==
408 Content-Transfer-Encoding: 7bit
409 Content-Description: OpenPGP digital signature
410 Content-Type: application/pgp-signature; name="signature.asc"; charset="us-ascii"
412 -----BEGIN PGP SIGNATURE-----
413 Version: GnuPG v2.0.19 (GNU/Linux)
416 -----END PGP SIGNATURE-----
418 --===============...==--
420 Response to a submission on an unsubmittable assignment:
422 >>> del message['Subject']
423 >>> message['Subject'] = '[submit] attendance 1'
424 >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS
426 Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...=="
428 Content-Disposition: inline
430 From: Robot101 <phys101@tower.edu>
431 Reply-to: Robot101 <phys101@tower.edu>
432 To: Bilbo Baggins <bb@shire.org>
433 Subject: Received invalid Attendance 1 submission
435 --===============...==
436 Content-Type: multipart/mixed; boundary="===============...=="
439 --===============...==
440 Content-Type: text/plain; charset="us-ascii"
442 Content-Transfer-Encoding: 7bit
443 Content-Disposition: inline
447 We received your submission for Attendance 1, but you are not
448 allowed to submit that assignment via email.
453 --===============...==
454 Content-Type: message/rfc822
457 Content-Type: text/plain; charset="us-ascii"
459 Content-Transfer-Encoding: 7bit
460 Content-Disposition: inline
461 From: Billy B <bb@greyhavens.net>
462 To: phys101 <phys101@tower.edu>
463 Return-Path: <bb@greyhavens.net>
464 Received: from smtp.mail.uu.edu (localhost.localdomain [127.0.0.1]) by smtp.mail.uu.edu (Postfix) with SMTP id 68CB45C8453 for <wking@tremily.us>; Mon, 10 Oct 2011 12:50:46 -0400 (EDT)
465 Received: from smtp.home.net (smtp.home.net [123.456.123.456]) by smtp.mail.uu.edu (Postfix) with ESMTP id 5BA225C83EF for <wking@tremily.us>; Mon, 09 Oct 2011 11:50:46 -0400 (EDT)
466 Message-ID: <hgi.jlk@home.net>
467 Subject: [submit] attendance 1
470 --===============...==--
471 --===============...==
473 Content-Transfer-Encoding: 7bit
474 Content-Description: OpenPGP digital signature
475 Content-Type: application/pgp-signature; name="signature.asc"; charset="us-ascii"
477 -----BEGIN PGP SIGNATURE-----
478 Version: GnuPG v2.0.19 (GNU/Linux)
481 -----END PGP SIGNATURE-----
483 --===============...==--
485 Response to a bad subject:
487 >>> del message['Subject']
488 >>> message['Subject'] = 'need help for the first homework'
489 >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS
491 Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...=="
493 Content-Disposition: inline
495 From: Robot101 <phys101@tower.edu>
496 Reply-to: Robot101 <phys101@tower.edu>
497 To: Bilbo Baggins <bb@shire.org>
498 Subject: no tag in 'need help for the first homework'
500 --===============...==
501 Content-Type: multipart/mixed; boundary="===============...=="
504 --===============...==
505 Content-Type: text/plain; charset="us-ascii"
507 Content-Transfer-Encoding: 7bit
508 Content-Disposition: inline
512 We received an email message from you with an invalid
518 --===============...==
519 Content-Type: message/rfc822
522 Content-Type: text/plain; charset="us-ascii"
524 Content-Transfer-Encoding: 7bit
525 Content-Disposition: inline
526 From: Billy B <bb@greyhavens.net>
527 To: phys101 <phys101@tower.edu>
528 Return-Path: <bb@greyhavens.net>
529 Received: from smtp.mail.uu.edu (localhost.localdomain [127.0.0.1]) by smtp.mail.uu.edu (Postfix) with SMTP id 68CB45C8453 for <wking@tremily.us>; Mon, 10 Oct 2011 12:50:46 -0400 (EDT)
530 Received: from smtp.home.net (smtp.home.net [123.456.123.456]) by smtp.mail.uu.edu (Postfix) with ESMTP id 5BA225C83EF for <wking@tremily.us>; Mon, 09 Oct 2011 11:50:46 -0400 (EDT)
531 Message-ID: <hgi.jlk@home.net>
532 Subject: need help for the first homework
535 --===============...==--
536 --===============...==
538 Content-Transfer-Encoding: 7bit
539 Content-Description: OpenPGP digital signature
540 Content-Type: application/pgp-signature; name="signature.asc"; charset="us-ascii"
542 -----BEGIN PGP SIGNATURE-----
543 Version: GnuPG v2.0.19 (GNU/Linux)
546 -----END PGP SIGNATURE-----
548 --===============...==--
550 Response to a missing subject:
552 >>> del message['Subject']
553 >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS
555 Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg="pgp-sha1"; boundary="===============...=="
557 Content-Disposition: inline
559 From: Robot101 <phys101@tower.edu>
560 Reply-to: Robot101 <phys101@tower.edu>
561 To: Bilbo Baggins <bb@shire.org>
562 Subject: no subject in <hgi.jlk@home.net>
564 --===============...==
565 Content-Type: multipart/mixed; boundary="===============...=="
568 --===============...==
569 Content-Type: text/plain; charset="us-ascii"
571 Content-Transfer-Encoding: 7bit
572 Content-Disposition: inline
576 We received an email message from you without a subject.
581 --===============...==
582 Content-Type: message/rfc822
585 Content-Type: text/plain; charset="us-ascii"
587 Content-Transfer-Encoding: 7bit
588 Content-Disposition: inline
589 From: Billy B <bb@greyhavens.net>
590 To: phys101 <phys101@tower.edu>
591 Return-Path: <bb@greyhavens.net>
592 Received: from smtp.mail.uu.edu (localhost.localdomain [127.0.0.1]) by smtp.mail.uu.edu (Postfix) with SMTP id 68CB45C8453 for <wking@tremily.us>; Mon, 10 Oct 2011 12:50:46 -0400 (EDT)
593 Received: from smtp.home.net (smtp.home.net [123.456.123.456]) by smtp.mail.uu.edu (Postfix) with ESMTP id 5BA225C83EF for <wking@tremily.us>; Mon, 09 Oct 2011 11:50:46 -0400 (EDT)
594 Message-ID: <hgi.jlk@home.net>
597 --===============...==--
598 --===============...==
600 Content-Transfer-Encoding: 7bit
601 Content-Description: OpenPGP digital signature
602 Content-Type: application/pgp-signature; name="signature.asc"; charset="us-ascii"
604 -----BEGIN PGP SIGNATURE-----
605 Version: GnuPG v2.0.19 (GNU/Linux)
608 -----END PGP SIGNATURE-----
610 --===============...==--
612 Response to an insecure message from a person with a PGP key:
614 >>> student = course.course.person(email='bb@greyhavens.net')
615 >>> student.pgp_key = '4332B6E3'
616 >>> del message['Subject']
617 >>> process(message) # doctest: +REPORT_UDIFF, +ELLIPSIS
619 Content-Type: multipart/encrypted; protocol="application/pgp-encrypted"; micalg="pgp-sha1"; boundary="===============...=="
621 Content-Disposition: inline
623 From: Robot101 <phys101@tower.edu>
624 Reply-to: Robot101 <phys101@tower.edu>
625 To: Bilbo Baggins <bb@shire.org>
626 Subject: unsigned message <hgi.jlk@home.net>
628 --===============...==
630 Content-Transfer-Encoding: 7bit
631 Content-Type: application/pgp-encrypted; charset="us-ascii"
635 --===============...==
637 Content-Transfer-Encoding: 7bit
638 Content-Description: OpenPGP encrypted message
639 Content-Type: application/octet-stream; name="encrypted.asc"; charset="us-ascii"
641 -----BEGIN PGP MESSAGE-----
642 Version: GnuPG v2.0.19 (GNU/Linux)
645 -----END PGP MESSAGE-----
647 --===============...==--
653 for original,message,person,subject,target in _load_messages(
654 course=course, stream=stream, mailbox=mailbox, input_=input_,
655 output=output, dry_run=dry_run,
656 continue_after_invalid_message=continue_after_invalid_message,
657 trust_email_infrastructure=trust_email_infrastructure,
660 handler = _get_handler(handlers=handlers, target=target)
661 _LOG.debug('handling {}'.format(target))
663 basedir=basedir, course=course, message=message,
664 person=person, subject=subject,
666 trust_email_infrastructure=trust_email_infrastructure,
668 except _InvalidMessage as error:
669 error.course = course
670 error.message = original
671 for attribute,value in [('person', person),
672 ('subject', subject),
674 if (value is not None and
675 getattr(error, attribute, None) is None):
676 setattr(error, attribute, value)
677 _LOG.warn('invalid message {}'.format(error.message_id()))
678 if not continue_after_invalid_message:
680 _LOG.warn('{}'.format(error))
682 response = _get_error_response(error)
684 except _Response as response:
686 msg = response.message
687 if not response.complete:
688 author = course.robot
690 msg = response.message
691 if isinstance(response.message, _MIMEText):
692 # Manipulate body (based on pgp_mime.append_text)
693 original_encoding = msg.get_charset().input_charset
694 original_payload = str(
695 msg.get_payload(decode=True), original_encoding)
701 target.alias(), original_payload, author.alias())
702 new_encoding = _pgp_mime.guess_encoding(new_payload)
703 if msg.get('content-transfer-encoding', None):
704 # clear CTE so set_payload will set it properly
705 del msg['content-transfer-encoding']
706 msg.set_payload(new_payload, new_encoding)
707 subject = msg['Subject']
708 assert subject is not None, msg
710 msg = _construct_email(
711 author=author, targets=[person], subject=subject,
715 def _load_messages(course, stream, mailbox=None, input_=None, output=None,
716 continue_after_invalid_message=False,
717 trust_email_infrastructure=False, respond=None,
720 _LOG.debug('loading message from {}'.format(stream))
722 messages = [(None,_message_from_file(stream))]
723 if output is not None:
724 ombox = _mailbox.Maildir(output, factory=None, create=True)
725 elif mailbox == 'mbox':
726 mbox = _mailbox.mbox(input_, factory=None, create=False)
727 messages = mbox.items()
728 if output is not None:
729 ombox = _mailbox.mbox(output, factory=None, create=True)
730 elif mailbox == 'maildir':
731 mbox = _mailbox.Maildir(input_, factory=None, create=False)
733 for key,msg in mbox.items():
734 subpath = mbox._lookup(key)
735 if subpath.endswith('.gitignore'):
736 _LOG.debug('skipping non-message {}'.format(subpath))
738 messages.append((key, msg))
739 if output is not None:
740 ombox = _mailbox.Maildir(output, factory=None, create=True)
742 raise ValueError(mailbox)
743 messages.sort(key=_get_message_time)
744 for key,msg in messages:
746 ret = _parse_message(
747 course=course, message=msg,
748 trust_email_infrastructure=trust_email_infrastructure)
749 except _InvalidMessage as error:
751 _LOG.warn('invalid message {}'.format(error.message_id()))
752 if not continue_after_invalid_message:
754 _LOG.warn('{}'.format(error))
756 response = _get_error_response(error)
757 if response is not None:
760 if output is not None and dry_run is False:
761 # move message from input mailbox to output mailbox
767 def _parse_message(course, message, trust_email_infrastructure=False):
768 """Parse an incoming email and respond if neccessary.
770 Return ``(msg, person, assignment, time)`` on successful parsing.
771 Return ``None`` on failure.
774 person = subject = target = None
776 person = _get_message_person(course=course, message=message)
778 _LOG.debug('verify message is from {}'.format(person))
780 message = _get_verified_message(message, person.pgp_key)
781 except _UnsignedMessage as error:
782 if trust_email_infrastructure:
783 _LOG.warn('{}'.format(error))
786 subject = _get_message_subject(message=message)
787 target = _get_message_target(subject=subject)
788 except _InvalidMessage as error:
789 error.course = course
790 error.message = original
791 for attribute,value in [('person', person),
792 ('subject', subject),
794 if (value is not None and
795 getattr(error, attribute, None) is None):
796 setattr(error, attribute, value)
798 return (original, message, person, subject, target)
800 def _get_message_person(course, message, trust_admin_from=True):
801 """Get the `Person` that sent the message.
803 We use 'Return-Path' (envelope from) instead of the message's From
804 header, because it's more consistent and harder to fake. However,
805 there may be times when you *want* to send a message in somebody
808 For example, if a student submitted an assignment from an
809 unexpected address, you might add that address to their entry in
810 your course config, and then bounce the message back into
811 pygrader. In this case, the From header will still be the
812 student, but the 'Return-Path' will be you. With
813 `trust_admin_from` (on by default), messages who's 'Return-Path'
814 matches a professor or TA will have their 'From' line used to find
815 the final person responsible for the message.
817 >>> from pygrader.model.course import Course
818 >>> from pygrader.model.person import Person
819 >>> from pgp_mime import encodedMIMEText
821 >>> course = Course(people=[
823 ... name='Gandalf', emails=['g@grey.edu'], groups=['professors']),
824 ... Person(name='Bilbo', emails=['bb@shire.org']),
826 >>> message = encodedMIMEText('testing')
827 >>> message['Return-Path'] = '<g@grey.edu>'
828 >>> message['From'] = 'Bill <bb@shire.org>'
829 >>> message['Message-ID'] = '<123.456@home.net>'
831 >>> person = _get_message_person(course=course, message=message)
835 >>> person = _get_message_person(
836 ... course=course, message=message, trust_admin_from=False)
840 sender = message['return-path'] # RFC 822
842 raise NoReturnPath(message)
843 sender = sender[1:-1] # strip wrapping '<' and '>'
844 people = list(course.find_people(email=sender))
846 raise UnregisteredAddress(message=message, address=sender)
848 raise AmbiguousAddress(message=message, address=sender, people=people)
850 if trust_admin_from and person.is_admin():
851 mid = message['message-id']
852 from_headers = message.get_all('from')
853 if len(from_headers) == 0:
854 _LOG.debug("no 'From' headers in {}".format(mid))
855 elif len(from_headers) > 1:
856 _LOG.debug("multiple 'From' headers in {}".format(mid))
858 name,address = _parseaddr(from_headers[0])
859 people = list(course.find_people(email=address))
861 _LOG.debug("'From' address {} is unregistered".format(address))
863 _LOG.debug("'From' address {} is ambiguous".format(address))
864 _LOG.debug('message from {} treated as being from {}'.format(
867 _LOG.debug('message from {}'.format(person))
870 def _get_message_subject(message):
872 >>> from email.header import Header
873 >>> from pgp_mime.email import encodedMIMEText
874 >>> message = encodedMIMEText('The answer is 42.')
875 >>> message['Message-ID'] = 'msg-id'
876 >>> _get_message_subject(message=message)
877 Traceback (most recent call last):
879 pygrader.mailpipe.SubjectlessMessage: no subject
880 >>> del message['Subject']
881 >>> subject = Header('unicode part', 'utf-8')
882 >>> subject.append('-ascii part', 'ascii')
883 >>> message['Subject'] = subject.encode()
884 >>> _get_message_subject(message=message)
885 'unicode part-ascii part'
886 >>> del message['Subject']
887 >>> message['Subject'] = 'clean subject'
888 >>> _get_message_subject(message=message)
891 if message['Subject'] is None:
892 raise SubjectlessMessage(subject=None, message=message)
894 parts = _decode_header(message['Subject'])
896 for string,encoding in parts:
899 if not isinstance(string, str):
900 string = str(string, encoding)
901 part_strings.append(string)
902 subject = ''.join(part_strings)
903 _LOG.debug('decoded header {} -> {}'.format(parts[0], subject))
904 return subject.lower().replace('#', '')
906 def _get_message_target(subject):
908 >>> _get_message_target(subject='no tag')
909 Traceback (most recent call last):
911 pygrader.handler.InvalidSubjectMessage: no tag in 'no tag'
912 >>> _get_message_target(subject='[] empty tag')
913 Traceback (most recent call last):
915 pygrader.handler.InvalidSubjectMessage: empty tag in '[] empty tag'
916 >>> _get_message_target(subject='[abc] empty tag')
918 >>> _get_message_target(subject='[phys160:abc] empty tag')
921 match = _TAG_REGEXP.match(subject)
923 raise _InvalidSubjectMessage(
924 subject=subject, error='no tag in {!r}'.format(subject))
927 raise _InvalidSubjectMessage(
928 subject=subject, error='empty tag in {!r}'.format(subject))
929 target = tag.rsplit(':', 1)[-1]
930 _LOG.debug('extracted target {} -> {}'.format(subject, target))
933 def _get_handler(handlers, target):
935 handler = handlers[target]
936 except KeyError as error:
937 raise InvalidHandlerMessage(
938 target=target, handlers=handlers) from error
941 def _get_verified_message(message, pgp_key):
944 >>> from pgp_mime import sign, encodedMIMEText
946 The student composes a message...
948 >>> message = encodedMIMEText('1.23 joules')
950 ... and signs it (with the pgp-mime test key).
952 >>> signed = sign(message, signers=['pgp-mime-test'])
954 As it is being delivered, the message picks up extra headers.
956 >>> signed['Message-ID'] = '<01234567@home.net>'
957 >>> signed['Received'] = 'from smtp.mail.uu.edu ...'
958 >>> signed['Received'] = 'from smtp.home.net ...'
960 We check that the message is signed, and that it is signed by the
963 >>> signed.authenticated
964 Traceback (most recent call last):
966 AttributeError: 'MIMEMultipart' object has no attribute 'authenticated'
967 >>> our_message = _get_verified_message(signed, pgp_key='4332B6E3')
968 >>> print(our_message.as_string()) # doctest: +REPORT_UDIFF
969 Content-Type: text/plain; charset="us-ascii"
971 Content-Transfer-Encoding: 7bit
972 Content-Disposition: inline
973 Message-ID: <01234567@home.net>
974 Received: from smtp.mail.uu.edu ...
975 Received: from smtp.home.net ...
978 >>> our_message.authenticated
981 If it is signed, but not by the right key, we get an error.
983 >>> print(_get_verified_message(signed, pgp_key='01234567'))
984 Traceback (most recent call last):
986 pygrader.mailpipe.WrongSignatureMessage: not signed by the expected key
988 If it is not signed at all, we get another error.
990 >>> print(_get_verified_message(message, pgp_key='4332B6E3'))
991 Traceback (most recent call last):
993 pygrader.handler.UnsignedMessage: unsigned message
995 mid = message['message-id']
997 decrypted,verified,signatures = _pgp_mime.verify(message=message)
998 except (ValueError, AssertionError) as error:
999 raise _UnsignedMessage(message=message) from error
1000 for signature in signatures:
1001 _LOG.debug(signature.dumps())
1003 fingerprints = dict((s.fingerprint, s) for s in signatures)
1004 for s in signatures:
1005 for key in _pgp_mime_key.lookup_keys([s.fingerprint]):
1006 if key.subkeys[0].fingerprint != s.fingerprint:
1007 # the signature was made with a subkey. Add the primary.
1008 fingerprints[key.subkeys[0].fingerprint] = s
1009 if pgp_key.startswith('0x'):
1010 key_tail = pgp_key[len('0x'):]
1013 matches = [fingerprints[f] for f in fingerprints.keys()
1014 if f.endswith(key_tail)]
1015 if len(matches) == 0:
1016 raise WrongSignatureMessage(
1017 message=message, pgp_key=pgp_key, signatures=signatures,
1018 fingerprints=fingerprints, decrypted=decrypted)
1019 signature = matches[0]
1021 problems = [k for k,v in signature.summary.items() if v]
1022 for good in ['green', 'valid']:
1023 if good in problems:
1024 problems.remove(good)
1026 raise UnverifiedSignatureMessage(
1027 message=message, signature=signature, decrypted=decrypted)
1028 # otherwise, we may have an untrusted key. We'll count that
1029 # as verified here, because the caller is explicity looking
1030 # for signatures by this fingerprint.
1031 for k,v in message.items(): # copy over useful headers
1032 if k.lower() not in ['content-type',
1034 'content-disposition',
1037 decrypted.authenticated = True
1040 def _get_error_response(error):
1041 author = error.course.robot
1042 target = getattr(error, 'person', None)
1043 subject = str(error)
1044 if isinstance(error, _InvalidSubmission):
1045 subject = 'Received invalid {} submission'.format(
1046 error.assignment.name)
1048 'We received your submission for {}, but you are not\n'
1049 'allowed to submit that assignment via email.'
1050 ).format(error.assignment.name)
1051 elif isinstance(error, _MissingGradeMessage):
1052 subject = 'No grade in {!r}'.format(error.subject)
1054 'Your grade submission did not include a text/plain\n'
1055 'part containing the new grade and comment.'
1057 elif isinstance(error, InvalidHandlerMessage):
1058 targets = sorted(error.handlers.keys())
1061 'In fact, there are no available handlers for this\n'
1065 'Perhaps you meant to use one of the following:\n'
1066 ' {}').format('\n '.join(targets))
1068 'We received an email from you with the following subject:\n'
1070 'which does not match any submittable handler name for\n'
1072 '{}').format(error.subject, error.course.name, hint)
1073 elif isinstance(error, SubjectlessMessage):
1074 subject = 'no subject in {}'.format(error.message['Message-ID'])
1075 text = 'We received an email message from you without a subject.'
1076 elif isinstance(error, AmbiguousAddress):
1078 'Multiple people match {} ({})'.format(
1079 error.address, ', '.join(p.name for p in error.people)))
1080 elif isinstance(error, UnregisteredAddress):
1081 target = _Person(name=error.address, emails=[error.address])
1083 'Your email address is not registered with pygrader for\n'
1084 '{}. If you feel it should be, contact your professor\n'
1085 'or TA.').format(error.course.name)
1086 elif isinstance(error, NoReturnPath):
1088 elif isinstance(error, _InvalidAssignmentSubject):
1089 if error.assignments:
1091 'but it matches several assignments:\n'
1092 ' * {}').format('\n * '.join(
1093 a.name for a in error.assignments))
1095 # prefer a submittable example assignment
1097 a for a in error.course.assignments if a.submittable]
1098 assignments += error.course.assignments # but fall back to any one
1100 'Remember to use the full name for the assignment in the\n'
1101 'subject. For example:\n'
1102 ' {} submission').format(assignments[0].name)
1104 'We received an email from you with the following subject:\n'
1105 ' {!r}\n{}').format(error.subject, hint)
1106 elif isinstance(error, _InvalidStudentSubject):
1108 'We received an email from you with the following subject:\n'
1110 'but it matches several students:\n'
1112 error.subject, '\n * '.join(s.name for s in error.students))
1113 elif isinstance(error, _InvalidSubjectMessage):
1115 'We received an email message from you with an invalid\n'
1117 elif isinstance(error, _UnsignedMessage):
1118 subject = 'unsigned message {}'.format(error.message['Message-ID'])
1120 'We received an email message from you without a PGP\n'
1123 elif isinstance(error, WrongSignatureMessage):
1125 'We received an email message from you without a valid',
1126 'PGP signature. We were expecting a signature by',
1127 '{}, but got signatures by:'.format(error.person.pgp_key),
1129 lines.extend([' {}'.format(s.fingerprint) for s in error.signatures])
1130 text = '\n'.join(lines)
1131 elif isinstance(error, UnverifiedSignatureMessage):
1133 'We received an email message from you with an unverified\n'
1136 'If this is the key you intended to use, contact your\n'
1138 ).format(error.signature.dumps(prefix=' '))
1139 elif isinstance(error, _PermissionViolationMessage):
1141 'We received an email from you with the following subject:\n'
1143 "but you can't do that unless you belong to one of the\n"
1144 'following groups:\n'
1146 error.subject, '\n * '.join(error.allowed_groups))
1147 elif isinstance(error, _InvalidMessage):
1149 'We received an email from you with the following subject:\n'
1151 'but the message was invalid:\n'
1152 ' {}').format(error.subject, error)
1154 raise NotImplementedError((type(error), error))
1156 raise NotImplementedError((type(error), error))
1157 return _construct_response(
1165 '{}\n'.format(target.alias(), text, author.alias())),
1166 original=error.message)
1168 def _get_message_time(key_message):
1169 "Key function for sorting mailbox (key,message) tuples."
1170 key,message = key_message
1171 return _message_time(message)