2 # Copyright (C) 2009-2010 W. Trevor King <wking@drexel.edu>
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 2 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
14 # You should have received a copy of the GNU General Public License along
15 # with this program; if not, write to the Free Software Foundation, Inc.,
16 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
18 Convert an mbox into xml suitable for input into be.
19 $ be-mbox-to-xml file.mbox | be import-xml -c <ID> -
20 mbox is a flat-file format, consisting of a series of messages.
21 Messages begin with a a From_ line, followed by RFC 822 email,
22 followed by a blank line.
27 from libbe.encoding import get_encoding, set_IO_stream_encodings
28 from libbe.utility import time_to_str
29 from mailbox import mbox, Message # the mailbox people really want an on-disk copy
30 from time import asctime, gmtime, mktime
32 from xml.sax.saxutils import escape
34 DEFAULT_ENCODING = get_encoding()
35 set_IO_stream_encodings(DEFAULT_ENCODING)
39 def normalize_email_address(address):
41 Standardize whitespace, etc.
43 addr = email.utils.formataddr(email.utils.parseaddr(address))
48 def normalize_RFC_2822_date(date):
50 Some email clients write non-RFC 2822-compliant date tags like:
51 Fri, 18 Sep 2009 08:49:02 -0400 (EDT)
52 with the non-standard (EDT) timezone name. This funtion attempts
53 to deal with such inconsistencies.
55 time_tuple = email.utils.parsedate(date)
56 assert time_tuple != None, \
57 'unparsable date: "%s"' % date
58 return time_to_str(mktime(time_tuple))
60 def comment_message_to_xml(message, fields=None):
64 new_fields[u'alt-id'] = message[u'message-id']
65 new_fields[u'in-reply-to'] = message[u'in-reply-to']
66 new_fields[u'author'] = normalize_email_address(message[u'from'])
67 new_fields[u'date'] = message[u'date']
68 if new_fields[u'date'] != None:
69 new_fields[u'date'] = normalize_RFC_2822_date(new_fields[u'date'])
70 new_fields[u'content-type'] = message.get_content_type()
71 for k,v in new_fields.items():
72 if v != None and type(v) != types.UnicodeType:
73 fields[k] = unicode(v, encoding=DEFAULT_ENCODING)
74 elif v == None and k in fields:
75 new_fields[k] = fields[k]
76 for k,v in fields.items():
77 if k not in new_fields:
78 new_fields.k = fields[k]
81 if fields[u'in-reply-to'] == None:
82 if message[u'references'] != None:
83 refs = message[u'references'].split()
84 for ref in refs: # search for a known reference id.
86 fields[u'in-reply-to'] = ref
88 if fields[u'in-reply-to'] == None and len(refs) > 0:
89 fields[u'in-reply-to'] = refs[0] # default to the first
90 else: # check for mutliple in-reply-to references.
91 refs = fields[u'in-reply-to'].split()
93 for ref in refs: # search for a known reference id.
95 fields[u'in-reply-to'] = ref
98 if found_ref == False and len(refs) > 0:
99 fields[u'in-reply-to'] = refs[0] # default to the first
101 if fields[u'alt-id'] != None:
102 KNOWN_IDS.append(fields[u'alt-id'])
104 if message.is_multipart():
106 alt_id = fields[u'alt-id']
107 from_str = fields[u'author']
108 date = fields[u'date']
109 for m in message.walk():
112 fields[u'author'] = from_str
113 fields[u'date'] = date
114 if len(ret) > 0: # we've added one part already
115 fields.pop(u'alt-id') # don't pass alt-id to other parts
116 fields[u'in-reply-to'] = alt_id # others respond to first
117 ret.append(comment_message_to_xml(m, fields))
118 return u'\n'.join(ret)
120 charset = message.get_content_charset(DEFAULT_ENCODING).lower()
121 #assert charset == DEFAULT_ENCODING.lower(), \
122 # u"Unknown charset: %s" % charset
124 if message[u'content-transfer-encoding'] == None:
125 encoding = DEFAULT_ENCODING
127 encoding = message[u'content-transfer-encoding'].lower()
128 body = message.get_payload(decode=True) # attempt to decode
129 assert body != None, "Unable to decode?"
130 if fields[u'content-type'].startswith(u"text/"):
131 body = unicode(body, encoding=charset).rstrip(u'\n')
133 body = base64.encode(body)
134 fields[u'body'] = body
135 lines = [u"<comment>"]
136 for tag,body in fields.items():
139 lines.append(u" <%s>%s</%s>" % (tag, ebody, tag))
140 lines.append(u"</comment>")
141 return u'\n'.join(lines)
143 def main(mbox_filename):
144 mb = mbox(mbox_filename)
145 print u'<?xml version="1.0" encoding="%s" ?>' % DEFAULT_ENCODING
148 print comment_message_to_xml(message)
152 if __name__ == "__main__":