From: W. Trevor King Date: Wed, 19 May 2010 06:31:00 +0000 (-0400) Subject: Add unicode-capable IO to hooke.ui.commandline. X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=3ce07bb5d6effc9a788f5a1ef5e4eff8eb4a7174;p=hooke.git Add unicode-capable IO to hooke.ui.commandline. The previous implementation produced: $ python bin/hooke -c 'flat_filter_playlist' ... hooke> Traceback (most recent call last): ... UnicodeEncodeError: 'ascii' codec can't encode character u'\xec'... The new encoding detection functions in hooke.util.encoding came from my libbe.util.encoding module for Bugs Everywhere. From my original BE commit (Tue 2008-11-25 15:47:19 -0500): get_encoding() is from Trac http://trac.edgewall.org/browser/trunk/trac/util/datefmt.py format_datetime() Trac has a BSD license http://trac.edgewall.org/wiki/TracLicense I don't know if such a small snippet requires us to "reproduce the above copyright" or where we need to reproduce it if it is needed. The stdout/stdin replacement code follows http://wiki.python.org/moin/ShellRedirectionFails References: http://wiki.python.org/moin/Unicode http://www.amk.ca/python/howto/unicode http://www.python.org/dev/peps/pep-0100/ --- diff --git a/hooke/plugin/convfilt.py b/hooke/plugin/convfilt.py index e7debb6..b4af4f6 100644 --- a/hooke/plugin/convfilt.py +++ b/hooke/plugin/convfilt.py @@ -189,7 +189,7 @@ class ConvolutionFilterCommand (FilterCommand): .. [#brucale2009] M. Brucale, M. Sandal, S. Di Maio, A. Rampioni, I. Tessari, L. Tosatto, M. Bisaglia, L. Bubacco, B. Samorì. "Pathogenic mutations shift the equilibria of - :math:`\alpha`-Synuclein single molecules towards structured + α-Synuclein single molecules towards structured conformers." Chembiochem., 2009. doi: `10.1002/cbic.200800581 `_ diff --git a/hooke/plugin/flatfilt.py b/hooke/plugin/flatfilt.py index 8d62608..ea1ddf9 100644 --- a/hooke/plugin/flatfilt.py +++ b/hooke/plugin/flatfilt.py @@ -168,7 +168,7 @@ class FlatFilterCommand (FilterCommand): .. [#sandal2008] M. Sandal, F. Valle, I. Tessari, S. Mammi, E. Bergantino, F. Musiani, M. Brucale, L. Bubacco, B. Samorì. - "Conformational equilibria in monomeric :math:`\alpha`-Synuclein at the + "Conformational equilibria in monomeric α-Synuclein at the single molecule level." PLOS Biology, 2009. doi: `10.1371/journal.pbio.0060006 `_ diff --git a/hooke/ui/commandline.py b/hooke/ui/commandline.py index 8a49718..e8db0af 100644 --- a/hooke/ui/commandline.py +++ b/hooke/ui/commandline.py @@ -20,6 +20,7 @@ line. """ +import codecs import cmd import optparse import readline # including readline makes cmd.Cmd.cmdloop() smarter @@ -28,6 +29,7 @@ import shlex from ..command import CommandExit, Exit, Command, Argument, StoreValue from ..interaction import Request, BooleanRequest, ReloadUserInterfaceConfig from ..ui import UserInterface, CommandMessage +from ..util.encoding import get_input_encoding, get_output_encoding # Define a few helper classes. @@ -364,16 +366,20 @@ class CommandLine (UserInterface): def __init__(self): super(CommandLine, self).__init__(name='command line') - def run(self, commands, ui_to_command_queue, command_to_ui_queue): + def _cmd(self, commands, ui_to_command_queue, command_to_ui_queue): cmd = HookeCmd(self, commands, inqueue=ui_to_command_queue, outqueue=command_to_ui_queue) + cmd.stdin = codecs.getreader(get_input_encoding())(cmd.stdin) + cmd.stdout = codecs.getwriter(get_output_encoding())(cmd.stdout) + return cmd + + def run(self, commands, ui_to_command_queue, command_to_ui_queue): + cmd = self._cmd(commands, ui_to_command_queue, command_to_ui_queue) cmd.cmdloop(self._splash_text()) def run_lines(self, commands, ui_to_command_queue, command_to_ui_queue, lines): - cmd = HookeCmd(self, commands, - inqueue=ui_to_command_queue, - outqueue=command_to_ui_queue) + cmd = self._cmd(commands, ui_to_command_queue, command_to_ui_queue) for line in lines: cmd.onecmd(line) diff --git a/hooke/util/encoding.py b/hooke/util/encoding.py new file mode 100644 index 0000000..e517b9e --- /dev/null +++ b/hooke/util/encoding.py @@ -0,0 +1,32 @@ +# Copyright + +"""Define useful encoding-extraction functions. +""" + +import locale +import sys + + +def get_encoding(): + """Guess a useful input/output/filesystem encoding. + + Maybe we need seperate encodings for input/output and filesystem? + Hmm. + """ + encoding = locale.getpreferredencoding() or sys.getdefaultencoding() + if sys.platform != 'win32' or sys.version_info[:2] > (2, 3): + encoding = locale.getlocale(locale.LC_TIME)[1] or encoding + # Python 2.3 on windows doesn't know about 'XYZ' alias for 'cpXYZ' + return encoding + +def get_input_encoding(): + "Guess the input encoding." + return get_encoding() + +def get_output_encoding(): + "Guess the output encoding." + return get_encoding() + +def get_filesystem_encoding(): + "Guess the filesystem encoding." + return get_encoding() diff --git a/test/unicode_output.py b/test/unicode_output.py new file mode 100644 index 0000000..0e37e97 --- /dev/null +++ b/test/unicode_output.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2010 W. Trevor King +# +# This file is part of Hooke. +# +# Hooke is free software: you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation, either +# version 3 of the License, or (at your option) any later version. +# +# Hooke is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with Hooke. If not, see +# . + +u""" +>>> from hooke.hooke import Hooke, HookeRunner +>>> h = Hooke() +>>> r = HookeRunner() +>>> h = r.run_lines(h, ['help flat_filter_playlist']) # doctest: +REPORT_UDIFF +ELLIPSIS +Command: flat_filter_playlist... + F. Musiani, M. Brucale, L. Bubacco, B. Samorì. + "Conformational equilibria in monomeric α-Synuclein at the +... +""" + +import sys +reload(sys) +sys.setdefaultencoding('utf-8') + +# setdefaultencoding to work around doctest/unicode limitations: +# Doctest: test.unicode_output +# ... /usr/lib/python2.6/doctest.py:1475: UnicodeWarning: Unicode +# equal comparison failed to convert both arguments to Unicode - +# interpreting them as being unequal +# +# see +# http://stackoverflow.com/questions/1733414/how-do-i-include-unicode-strings-in-python-doctests +# for details. +# +# Unfortunately, the setdefaultencoding also makes the +# codecs.getwriter() code that this test is supposed to check a no-op. +# Ah well, guess this will have to wait for Python 3.