From 0abb6957899e0ba5842816757e126905535e02c3 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Tue, 4 Feb 2014 02:59:28 +1600 Subject: [PATCH] [PATCH 10/17] nmbug-status: Add Page and HtmlPage for modular rendering --- 50/44c5d089e8676a7077854d7fcb1e03bb60d1f0 | 444 ++++++++++++++++++++++ 1 file changed, 444 insertions(+) create mode 100644 50/44c5d089e8676a7077854d7fcb1e03bb60d1f0 diff --git a/50/44c5d089e8676a7077854d7fcb1e03bb60d1f0 b/50/44c5d089e8676a7077854d7fcb1e03bb60d1f0 new file mode 100644 index 000000000..4e2bb4f0e --- /dev/null +++ b/50/44c5d089e8676a7077854d7fcb1e03bb60d1f0 @@ -0,0 +1,444 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id 59509429E35 + for ; Mon, 3 Feb 2014 03:10:17 -0800 (PST) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Amavis-Alert: BAD HEADER SECTION, Duplicate header field: "References" +X-Spam-Flag: NO +X-Spam-Score: 0 +X-Spam-Level: +X-Spam-Status: No, score=0 tagged_above=-999 required=5 + tests=[DKIM_SIGNED=0.1, DKIM_VALID=-0.1, RCVD_IN_DNSWL_NONE=-0.0001] + autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id YYMknRkc3ykA for ; + Mon, 3 Feb 2014 03:10:09 -0800 (PST) +Received: from qmta07.westchester.pa.mail.comcast.net + (qmta07.westchester.pa.mail.comcast.net [76.96.62.64]) + by olra.theworths.org (Postfix) with ESMTP id 4C3DA431E64 + for ; Mon, 3 Feb 2014 03:10:09 -0800 (PST) +Received: from omta14.westchester.pa.mail.comcast.net ([76.96.62.60]) + by qmta07.westchester.pa.mail.comcast.net with comcast + id Mn3z1n0011HzFnQ57nA9a7; Mon, 03 Feb 2014 11:10:09 +0000 +Received: from odin.tremily.us ([24.18.63.50]) + by omta14.westchester.pa.mail.comcast.net with comcast + id Mn881n00d152l3L3an89be; Mon, 03 Feb 2014 11:08:09 +0000 +Received: from mjolnir.tremily.us (unknown [192.168.0.140]) + by odin.tremily.us (Postfix) with ESMTPS id 1BEBFFB4D54; + Mon, 3 Feb 2014 03:00:42 -0800 (PST) +Received: (nullmailer pid 700 invoked by uid 1000); + Mon, 03 Feb 2014 10:59:41 -0000 +From: "W. Trevor King" +To: notmuch@notmuchmail.org +Subject: [PATCH 10/17] nmbug-status: Add Page and HtmlPage for modular + rendering +Date: Mon, 3 Feb 2014 02:59:28 -0800 +Message-Id: + +X-Mailer: git-send-email 1.8.5.2.8.g0f6c0d1 +In-Reply-To: +References: +In-Reply-To: +References: +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=comcast.net; + s=q20121106; t=1391425809; + bh=lkyEHMaK0pWitOsDQufbYK9aEYIMwylLXL0ZMnpMSsI=; + h=Received:Received:Received:Received:From:To:Subject:Date: + Message-Id; + b=CBHmANbcwj9AXJZ+aTm56FEbaHgI4uy1b/t0CW0woHPrTjwY5hHcVdNgXal1eYZhp + dwcMKNFFJdONf9nWz3bJ2S0jyllcpuoUCAjXpwJwHom1RDGIRP/8iNA/3qmkG7hcIQ + c4Dk/9ajCSe/lcUk7FBlD6FWirkEAlgx9TWt7STMWMw+fFlFafseRwgWcVXhD6pnim + pU2uJMV6XLcxHM8yLL5x1EeHEuBx5OH2NFETvPBs6PjeI6HvuJmHSV6zWG7f4uwCw+ + QVTV7mkW8mDWHlp9wf1MWuRps4CgGk3S904j0Iz0L+22Hj4mBLEP3FSVP4Oe9I6D/T + PVYCM2kgIrp3w== +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Mon, 03 Feb 2014 11:10:17 -0000 + +I was having trouble understanding the logic of the longish print_view +function, so I refactored the output generation into modular bits. +The basic text rendering is handled by Page, which has enough hooks +that HtmlPage can borrow the logic and slot-in HTML generators. + +By modularizing the logic it should also be easier to build other +renderers if folks want to customize the layout for other projects. + +Timezones +========= + +This commit has not effect on the output, except that some dates have +been converted from the sender's timezone to UTC due to: + + - val = m.get_header(header) + - ... + - if header == 'date': + - val = str.join(' ', val.split(None)[1:4]) + - val = str(datetime.datetime.strptime(val, '%d %b %Y').date()) + ... + + value = str(datetime.datetime.utcfromtimestamp( + + message.get_date()).date()) + +I also tweaked the HTML header date to be utcnow instead of the local +now() to make all times independent of the generator's local time. +This matches Gmane, which converts all Date headers to UTC (although +they use a 'GMT' suffix). Notmuch uses +g_mime_utils_header_decode_date to calculate the UTC timestamps, but +uses a NULL tz_offset which drops the information we'd need to get +back to the sender's local time [1]. With the generator's local time +arbitrarily different from the sender's and viewer's local time, +sticking with UTC seems the best bet. + +[1]: https://developer.gnome.org/gmime/stable/gmime-gmime-utils.html#g-mime-utils-header-decode-date +--- + devel/nmbug/nmbug-status | 292 +++++++++++++++++++++++++++-------------------- + 1 file changed, 171 insertions(+), 121 deletions(-) + +diff --git a/devel/nmbug/nmbug-status b/devel/nmbug/nmbug-status +index 22b6b10..7778029 100755 +--- a/devel/nmbug/nmbug-status ++++ b/devel/nmbug/nmbug-status +@@ -5,10 +5,13 @@ + # dependencies + # - python 2.6 for json + # - argparse; either python 2.7, or install separately ++# - collections; python 2.7 + + from __future__ import print_function ++from __future__ import unicode_literals + + import codecs ++import collections + import datetime + import email.utils + import locale +@@ -24,6 +27,7 @@ import subprocess + + + _ENCODING = locale.getpreferredencoding() or sys.getdefaultencoding() ++_PAGES = {} + + + def read_config(path=None, encoding=None): +@@ -50,104 +54,175 @@ def read_config(path=None, encoding=None): + return json.load(fp) + + +-class Thread: +- def __init__(self, last, lines): +- self.last = last +- self.lines = lines +- +- def join_utf8_with_newlines(self): +- return '\n'.join( (line.encode('utf-8') for line in self.lines) ) +- +- +-def output_with_separator(threadlist, sep): +- outputs = (thread.join_utf8_with_newlines() for thread in threadlist) +- print(sep.join(outputs)) +- +- +-def print_view(database, title, query, comment, +- headers=('date', 'from', 'subject')): +- +- query_string = ' and '.join(query) +- q_new = notmuch.Query(database, query_string) +- q_new.set_sort(notmuch.Query.SORT.OLDEST_FIRST) +- +- last_thread_id = '' +- threads = {} +- threadlist = [] +- out = {} +- last = None +- lines = None +- +- if output_format == 'html': +- print('

%s

' % (title, title)) +- print(comment) +- print('The view is generated from the following query:') +- print('
') +- print(query_string) +- print('
') +- print('\n') +- +- for m in q_new.search_messages(): +- +- thread_id = m.get_thread_id() +- +- if thread_id != last_thread_id: +- if threads.has_key(thread_id): +- last = threads[thread_id].last +- lines = threads[thread_id].lines ++class Thread (list): ++ def __init__(self): ++ self.running_data = {} ++ ++ ++class Page (object): ++ def __init__(self, header=None, footer=None): ++ self.header = header ++ self.footer = footer ++ ++ def write(self, database, views, stream=None): ++ if not stream: ++ try: # Python 3 ++ byte_stream = sys.stdout.buffer ++ except AttributeError: # Python 2 ++ byte_stream = sys.stdout ++ stream = codecs.getwriter(encoding='UTF-8')(stream=byte_stream) ++ self._write_header(views=views, stream=stream) ++ for view in views: ++ self._write_view(database=database, view=view, stream=stream) ++ self._write_footer(views=views, stream=stream) ++ ++ def _write_header(self, views, stream): ++ if self.header: ++ stream.write(self.header) ++ ++ def _write_footer(self, views, stream): ++ if self.footer: ++ stream.write(self.footer) ++ ++ def _write_view(self, database, view, stream): ++ if 'query-string' not in view: ++ query = view['query'] ++ view['query-string'] = ' and '.join(query) ++ q = notmuch.Query(database, view['query-string']) ++ q.set_sort(notmuch.Query.SORT.OLDEST_FIRST) ++ threads = self._get_threads(messages=q.search_messages()) ++ self._write_view_header(view=view, stream=stream) ++ self._write_threads(threads=threads, stream=stream) ++ ++ def _get_threads(self, messages): ++ threads = collections.OrderedDict() ++ for message in messages: ++ thread_id = message.get_thread_id() ++ if thread_id in threads: ++ thread = threads[thread_id] + else: +- last = {} +- lines = [] +- thread = Thread(last, lines) ++ thread = Thread() + threads[thread_id] = thread +- for h in headers: +- last[h] = '' +- threadlist.append(thread) +- last_thread_id = thread_id +- ++ thread.running_data, display_data = self._message_display_data( ++ running_data=thread.running_data, message=message) ++ thread.append(display_data) ++ return list(threads.values()) ++ ++ def _write_view_header(self, view, stream): ++ pass ++ ++ def _write_threads(self, threads, stream): ++ for thread in threads: ++ for message_display_data in thread: ++ stream.write( ++ ('{date:10.10s} {from:20.20s} {subject:40.40s}\n' ++ '{message-id-term:>72}\n' ++ ).format(**message_display_data)) ++ if thread != threads[-1]: ++ stream.write('\n') ++ ++ def _message_display_data(self, running_data, message): ++ headers = ('thread-id', 'message-id', 'date', 'from', 'subject') ++ data = {} + for header in headers: +- val = m.get_header(header) +- +- if header == 'date': +- val = str.join(' ', val.split(None)[1:4]) +- val = str(datetime.datetime.strptime(val, '%d %b %Y').date()) +- elif header == 'from': +- (val, addr) = email.utils.parseaddr(val) +- if val == '': +- val = addr.split('@')[0] +- +- if header != 'subject' and last[header] == val: +- out[header] = '' ++ if header == 'thread-id': ++ value = message.get_thread_id() ++ elif header == 'message-id': ++ value = message.get_message_id() ++ data['message-id-term'] = 'id:"{}"'.format(value) ++ elif header == 'date': ++ value = str(datetime.datetime.utcfromtimestamp( ++ message.get_date()).date()) + else: +- out[header] = val +- last[header] = val +- +- mid = m.get_message_id() +- out['id'] = 'id:"%s"' % mid +- +- if output_format == 'html': +- +- out['subject'] = '%s' % ( +- quote(mid), out['subject']) +- +- lines.append(' ') +- lines.append(' ') +- else: +- lines.append('%(date)-10.10s %(from)-20.20s %(subject)-40.40s\n%(id)72s' % out) +- +- if output_format == 'html': +- output_with_separator(threadlist, +- '\n\n') +- print('
%s' % out['date']) +- lines.append('%s' % out['id']) +- lines.append('
%s' % out['from']) +- lines.append('%s' % out['subject']) +- lines.append('

') +- else: +- output_with_separator(threadlist, '\n\n') +- ++ value = message.get_header(header) ++ if header == 'from': ++ (value, addr) = email.utils.parseaddr(value) ++ if not value: ++ value = addr.split('@')[0] ++ data[header] = value ++ next_running_data = data.copy() ++ for header, value in data.items(): ++ if header in ['message-id', 'subject']: ++ continue ++ if value == running_data.get(header, None): ++ data[header] = '' ++ return (next_running_data, data) ++ ++ ++class HtmlPage (Page): ++ def _write_header(self, views, stream): ++ super(HtmlPage, self)._write_header(views=views, stream=stream) ++ stream.write('
    \n') ++ for view in views: ++ stream.write( ++ '
  • {title}
    • \n'.format(**view)) ++ stream.write('
\n') ++ ++ def _write_view_header(self, view, stream): ++ stream.write('

{title}

\n'.format(**view)) ++ if 'comment' in view: ++ stream.write(view['comment']) ++ stream.write('\n') ++ for line in [ ++ 'The view is generated from the following query:', ++ '
', ++ view['query-string'], ++ '
', ++ ]: ++ stream.write(line) ++ stream.write('\n') ++ ++ def _write_threads(self, threads, stream): ++ if not threads: ++ return ++ stream.write('\n') ++ for thread in threads: ++ for message_display_data in thread: ++ stream.write(( ++ '\n' ++ '\n' ++ ).format(**message_display_data)) ++ if thread != threads[-1]: ++ stream.write('\n') ++ stream.write('
{date}\n' ++ '{message-id-term}\n' ++ '
{from}\n' ++ '{subject}\n' ++ '

\n') ++ ++ def _message_display_data(self, *args, **kwargs): ++ running_data, display_data = super( ++ HtmlPage, self)._message_display_data( ++ *args, **kwargs) ++ if 'subject' in display_data and 'message-id' in display_data: ++ d = { ++ 'message-id': quote(display_data['message-id']), ++ 'subject': display_data['subject'], ++ } ++ display_data['subject'] = ( ++ '{subject}' ++ ).format(**d) ++ return (running_data, display_data) ++ ++ ++_PAGES['text'] = Page() ++_PAGES['html'] = HtmlPage( ++ header=''' ++ ++ ++ ++ ++Notmuch Patches ++ ++ ++

Notmuch Patches

++Generated: {date}
++For more infomation see nmbug ++

Views

++'''.format(date=datetime.datetime.utcnow().date()), ++ footer='\n\n', ++ ) + +-# parse command line arguments + + parser = argparse.ArgumentParser() + parser.add_argument('--text', help='output plain text format', +@@ -177,34 +252,9 @@ else: + import notmuch + + if args.text: +- output_format = 'text' ++ page = _PAGES['text'] + else: +- output_format = 'html' +- +-# main program ++ page = _PAGES['html'] + + db = notmuch.Database(mode=notmuch.Database.MODE.READ_ONLY) +- +-if output_format == 'html': +- print(''' +- +- +- +- +-Notmuch Patches +- +- +-

Notmuch Patches

+-Generated: {date}
+-For more infomation see nmbug +-

Views

+-
    '''.format(date=datetime.datetime.utcnow().date())) +- for view in config['views']: +- print('
  • %(title)s
    • ' % view) +- print('
') +- +-for view in config['views']: +- print_view(database=db, **view) +- +-if output_format == 'html': +- print('\n') ++page.write(database=db, views=config['views']) +-- +1.8.5.2.8.g0f6c0d1 + -- 2.26.2