1 # Copyright (C) 2012 W. Trevor King <wking@drexel.edu>
3 # This file is part of pygrader.
5 # pygrader is free software: you can redistribute it and/or modify it under the
6 # terms of the GNU General Public License as published by the Free Software
7 # Foundation, either version 3 of the License, or (at your option) any later
10 # pygrader is distributed in the hope that it will be useful, but WITHOUT ANY
11 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
12 # A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 # You should have received a copy of the GNU General Public License along with
15 # pygrader. If not, see <http://www.gnu.org/licenses/>.
17 from __future__ import absolute_import
19 from email import message_from_file as _message_from_file
20 from email.header import decode_header as _decode_header
21 import hashlib as _hashlib
22 import locale as _locale
23 import mailbox as _mailbox
25 import os.path as _os_path
29 from pgp_mime import verify as _verify
30 from lxml import etree as _etree
32 from . import LOG as _LOG
33 from .color import standard_colors as _standard_colors
34 from .color import color_string as _color_string
35 from .extract_mime import extract_mime as _extract_mime
36 from .extract_mime import message_time as _message_time
37 from .storage import assignment_path as _assignment_path
38 from .storage import set_late as _set_late
41 def mailpipe(basedir, course, stream=None, mailbox=None, input_=None,
42 output=None, max_late=0, use_color=None, dry_run=False, **kwargs):
43 """Run from procmail to sort incomming submissions
45 For example, you can setup your ``.procmailrc`` like this::
51 LOGFILE=$MAILDIR/procmail.log
53 PYGRADE_MAILPIPE="pg.py -d $HOME/grades/phys160"
55 # Grab all incoming homeworks emails. This rule eats matching emails
56 # (i.e. no further procmail processing).
58 * ^Subject:.*\[phys160-sub]
59 | "$PYGRADE_MAILPIPE" mailpipe
61 If you don't want procmail to eat the message, you can use the
62 ``c`` flag (carbon copy) by starting your rule off with ``:0 c``.
64 >>> from asyncore import loop
65 >>> from io import StringIO
66 >>> from pgp_mime.email import encodedMIMEText
67 >>> from pygrader.test.course import StubCourse
68 >>> from pygrader.test.client import MessageSender
69 >>> from pygrader.test.server import SMTPServer
71 Messages with unrecognized ``Return-Path``\s are silently dropped:
73 >>> course = StubCourse()
74 >>> def process(peer, mailfrom, rcpttos, data):
76 ... basedir=course.basedir, course=course.course,
77 ... stream=StringIO(data), output=course.mailbox)
78 >>> message = encodedMIMEText('The answer is 42.')
79 >>> message['Message-ID'] = '<123.456@home.net>'
80 >>> message['Return-Path'] = '<invalid.return.path@home.net>'
81 >>> message['Received'] = (
82 ... 'from smtp.home.net (smtp.home.net [123.456.123.456]) '
83 ... 'by smtp.mail.uu.edu (Postfix) with ESMTP id 5BA225C83EF '
84 ... 'for <wking@tremily.us>; Mon, 09 Oct 2011 11:50:46 -0400 (EDT)')
85 >>> message['From'] = 'Billy B <bb@greyhavens.net>'
86 >>> message['To'] = 'S <eye@tower.edu>'
87 >>> message['Subject'] = 'assignment 1 submission'
88 >>> messages = [message]
89 >>> ms = MessageSender(address=('localhost', 1025), messages=messages)
91 >>> course.print_tree() # doctest: +REPORT_UDIFF
94 If we add a valid ``Return-Path``, we get the expected delivery:
96 >>> server = SMTPServer(
97 ... ('localhost', 1025), None, process=process, count=1)
98 >>> del message['Return-Path']
99 >>> message['Return-Path'] = '<bb@greyhavens.net>'
100 >>> ms = MessageSender(address=('localhost', 1025), messages=messages)
102 >>> course.print_tree() # doctest: +REPORT_UDIFF, +ELLIPSIS
104 Bilbo_Baggins/Assignment_1
105 Bilbo_Baggins/Assignment_1/mail
106 Bilbo_Baggins/Assignment_1/mail/cur
107 Bilbo_Baggins/Assignment_1/mail/new
108 Bilbo_Baggins/Assignment_1/mail/new/...:2,S
109 Bilbo_Baggins/Assignment_1/mail/tmp
117 The last ``Received`` is used to timestamp the message:
119 >>> server = SMTPServer(
120 ... ('localhost', 1025), None, process=process, count=1)
121 >>> del message['Message-ID']
122 >>> message['Message-ID'] = '<abc.def@home.net>'
123 >>> del message['Received']
124 >>> message['Received'] = (
125 ... 'from smtp.mail.uu.edu (localhost.localdomain [127.0.0.1]) '
126 ... 'by smtp.mail.uu.edu (Postfix) with SMTP id 68CB45C8453 '
127 ... 'for <wking@tremily.us>; Mon, 10 Oct 2011 12:50:46 -0400 (EDT)')
128 >>> message['Received'] = (
129 ... 'from smtp.home.net (smtp.home.net [123.456.123.456]) '
130 ... 'by smtp.mail.uu.edu (Postfix) with ESMTP id 5BA225C83EF '
131 ... 'for <wking@tremily.us>; Mon, 09 Oct 2011 11:50:46 -0400 (EDT)')
132 >>> messages = [message]
133 >>> ms = MessageSender(address=('localhost', 1025), messages=messages)
135 >>> course.print_tree() # doctest: +REPORT_UDIFF, +ELLIPSIS
137 Bilbo_Baggins/Assignment_1
138 Bilbo_Baggins/Assignment_1/late
139 Bilbo_Baggins/Assignment_1/mail
140 Bilbo_Baggins/Assignment_1/mail/cur
141 Bilbo_Baggins/Assignment_1/mail/new
142 Bilbo_Baggins/Assignment_1/mail/new/...:2,S
143 Bilbo_Baggins/Assignment_1/mail/new/...:2,S
144 Bilbo_Baggins/Assignment_1/mail/tmp
157 for msg,person,assignment,time in _load_messages(
158 course=course, stream=stream, mailbox=mailbox, input_=input_,
159 output=output, use_color=use_color, dry_run=dry_run):
160 assignment_path = _assignment_path(basedir, assignment, person)
161 _save_local_message_copy(
162 msg=msg, person=person, assignment_path=assignment_path,
163 use_color=use_color, dry_run=dry_run)
164 _extract_mime(message=msg, output=assignment_path, dry_run=dry_run)
166 basedir=basedir, assignment=assignment, person=person, time=time,
167 max_late=max_late, use_color=use_color, dry_run=dry_run)
169 def _load_messages(course, stream, mailbox=None, input_=None, output=None,
170 use_color=None, dry_run=False):
173 messages = [(None,_message_from_file(stream))]
174 if output is not None:
175 ombox = _mailbox.Maildir(output, factory=None, create=True)
176 elif mailbox == 'mbox':
177 mbox = _mailbox.mbox(input_, factory=None, create=False)
178 messages = mbox.items()
179 if output is not None:
180 ombox = _mailbox.mbox(output, factory=None, create=True)
181 elif mailbox == 'maildir':
182 mbox = _mailbox.Maildir(input_, factory=None, create=False)
183 messages = mbox.items()
184 if output is not None:
185 ombox = _mailbox.Maildir(output, factory=None, create=True)
187 raise ValueError(mailbox)
188 for key,msg in messages:
189 ret = _parse_message(
190 course=course, msg=msg, use_color=use_color)
192 if output is not None and dry_run is False:
193 # move message from input mailbox to output mailbox
199 def _parse_message(course, msg, use_color=None):
200 highlight,lowlight,good,bad = _standard_colors(use_color=use_color)
201 mid = msg['Message-ID']
202 sender = msg['Return-Path'] # RFC 822
204 _LOG.debug(_color_string(
205 string='no Return-Path in {}'.format(mid), color=lowlight))
207 sender = sender[1:-1] # strip wrapping '<' and '>'
209 people = list(course.find_people(email=sender))
211 _LOG.warn(_color_string(
212 string='no person found to match {}'.format(sender),
216 _LOG.warn(_color_string(
217 string='multiple people match {} ({})'.format(
218 sender, ', '.join(str(p) for p in people)),
224 msg = _get_verified_message(msg, person.pgp_key, use_color=use_color)
228 if msg['Subject'] is None:
229 _LOG.warn(_color_string(
230 string='no subject in {}'.format(mid), color=bad))
232 parts = _decode_header(msg['Subject'])
234 _LOG.warn(_color_string(
235 string='multi-part header {}'.format(parts), color=bad))
237 subject,encoding = parts[0]
240 _LOG.debug('decoded header {} -> {}'.format(parts[0], subject))
241 subject = subject.lower().replace('#', '')
242 for assignment in course.assignments:
243 if _match_assignment(assignment, subject):
245 if not _match_assignment(assignment, subject):
246 _LOG.warn(_color_string(
247 string='no assignment found in {}'.format(repr(subject)),
251 time = _message_time(message=msg, use_color=use_color)
252 return (msg, person, assignment, time)
254 def _match_assignment(assignment, subject):
255 return assignment.name.lower() in subject
257 def _save_local_message_copy(msg, person, assignment_path, use_color=None,
259 highlight,lowlight,good,bad = _standard_colors(use_color=use_color)
261 _os.makedirs(assignment_path)
264 mpath = _os_path.join(assignment_path, 'mail')
266 mbox = _mailbox.Maildir(mpath, factory=None, create=not dry_run)
267 except _mailbox.NoSuchMailboxError as e:
268 _LOG.debug(_color_string(
269 string='could not open mailbox at {}'.format(mpath),
275 for other_msg in mbox:
276 if other_msg['Message-ID'] == msg['Message-ID']:
280 _LOG.debug(_color_string(
281 string='saving email from {} to {}'.format(
282 person, assignment_path), color=good))
283 if mbox is not None and not dry_run:
284 mdmsg = _mailbox.MaildirMessage(msg)
289 _LOG.debug(_color_string(
290 string='already found {} in {}'.format(
291 msg['Message-ID'], mpath), color=good))
293 def _check_late(basedir, assignment, person, time, max_late=0, use_color=None,
295 highlight,lowlight,good,bad = _standard_colors(use_color=use_color)
296 if time > assignment.due + max_late:
297 dt = time - assignment.due
298 _LOG.warn(_color_string(
299 string='{} {} late by {} seconds ({} hours)'.format(
300 person.name, assignment.name, dt, dt/3600.),
303 _set_late(basedir=basedir, assignment=assignment, person=person)
305 def _get_verified_message(message, pgp_key, use_color=None):
308 >>> from copy import deepcopy
309 >>> from pgp_mime import sign, encodedMIMEText
311 The student composes a message...
313 >>> message = encodedMIMEText('1.23 joules')
315 ... and signs it (with the pgp-mime test key).
317 >>> signed = sign(message, signers=['pgp-mime-test'])
319 As it is being delivered, the message picks up extra headers.
321 >>> signed['Message-ID'] = '<01234567@home.net>'
322 >>> signed['Received'] = 'from smtp.mail.uu.edu ...'
323 >>> signed['Received'] = 'from smtp.home.net ...'
325 We check that the message is signed, and that it is signed by the
328 >>> our_message = _get_verified_message(
329 ... deepcopy(signed), pgp_key='4332B6E3')
330 >>> print(our_message.as_string()) # doctest: +REPORT_UDIFF
331 Content-Type: text/plain; charset="us-ascii"
333 Content-Transfer-Encoding: 7bit
334 Content-Disposition: inline
335 Message-ID: <01234567@home.net>
336 Received: from smtp.mail.uu.edu ...
337 Received: from smtp.home.net ...
341 If it is signed, but not by the right key, we get ``None``.
343 >>> print(_get_verified_message(
344 ... deepcopy(signed), pgp_key='01234567'))
347 If it is not signed at all, we get ``None``.
349 >>> print(_get_verified_message(
350 ... deepcopy(message), pgp_key='4332B6E3'))
353 highlight,lowlight,good,bad = _standard_colors(use_color=use_color)
354 mid = message['message-id']
356 decrypted,verified,result = _verify(message=message)
357 except (ValueError, AssertionError):
358 _LOG.warn(_color_string(
359 string='could not verify {} (not signed?)'.format(mid),
362 _LOG.info(_color_string(str(result, 'utf-8'), color=lowlight))
363 tree = _etree.fromstring(result.replace(b'\x00', b''))
365 for signature in tree.findall('.//signature'):
366 for fingerprint in signature.iterchildren('fpr'):
367 if fingerprint.text.endswith(pgp_key):
371 _LOG.warn(_color_string(
372 string='{} is not signed by the expected key'.format(mid),
376 sumhex = list(signature.iterchildren('summary'))[0].get('value')
377 summary = int(sumhex, 16)
379 _LOG.warn(_color_string(
380 string='{} has an unverified signature'.format(mid),
383 # otherwise, we may have an untrusted key. We'll count that
384 # as verified here, because the caller is explicity looking
385 # for signatures by this fingerprint.
386 for k,v in message.items(): # copy over useful headers
387 if k.lower() not in ['content-type',
389 'content-disposition',