Add unicode-capable IO to hooke.ui.commandline.

The previous implementation produced:
  $ python bin/hooke -c 'flat_filter_playlist'
  ...
  hooke> Traceback (most recent call last):
  ...
  UnicodeEncodeError: 'ascii' codec can't encode character u'\xec'...

The new encoding detection functions in hooke.util.encoding came from
my libbe.util.encoding module for Bugs Everywhere.  From my original
BE commit (Tue 2008-11-25 15:47:19 -0500):

  get_encoding() is from Trac
    http://trac.edgewall.org/browser/trunk/trac/util/datefmt.py
    format_datetime()
  Trac has a BSD license
    http://trac.edgewall.org/wiki/TracLicense
  I don't know if such a small snippet requires us to "reproduce the
  above copyright" or where we need to reproduce it if it is needed.

  The stdout/stdin replacement code follows
    http://wiki.python.org/moin/ShellRedirectionFails

  References:
    http://wiki.python.org/moin/Unicode
    http://www.amk.ca/python/howto/unicode
    http://www.python.org/dev/peps/pep-0100/

diff --git a/hooke/plugin/convfilt.py b/hooke/plugin/convfilt.py
index e7debb6acbb43c772d563c13a61c6b0fbd78947d..b4af4f63003c482708c512872d00804998f7a272 100644 (file)
--- a/hooke/plugin/convfilt.py
+++ b/hooke/plugin/convfilt.py
@@ -189,7 +189,7 @@ class ConvolutionFilterCommand (FilterCommand):
     .. [#brucale2009] M. Brucale, M. Sandal, S. Di Maio, A. Rampioni,
       I. Tessari, L. Tosatto, M. Bisaglia, L. Bubacco, B. Samorì.
       "Pathogenic mutations shift the equilibria of
-      :math:`\alpha`-Synuclein single molecules towards structured
+      α-Synuclein single molecules towards structured
       conformers."
       Chembiochem., 2009.
       doi: `10.1002/cbic.200800581 <http://dx.doi.org/10.1002/cbic.200800581>`_

diff --git a/hooke/plugin/flatfilt.py b/hooke/plugin/flatfilt.py
index 8d62608398ec47f0d5119c1cd3c7c2dd78305f8f..ea1ddf9fe923749a872203eb4ca7a26540290a81 100644 (file)
--- a/hooke/plugin/flatfilt.py
+++ b/hooke/plugin/flatfilt.py
@@ -168,7 +168,7 @@ class FlatFilterCommand (FilterCommand):
 
     .. [#sandal2008] M. Sandal, F. Valle, I. Tessari, S. Mammi, E. Bergantino,
       F. Musiani, M. Brucale, L. Bubacco, B. Samorì.
-      "Conformational equilibria in monomeric :math:`\alpha`-Synuclein at the
+      "Conformational equilibria in monomeric α-Synuclein at the
       single molecule level."
       PLOS Biology, 2009.
       doi: `10.1371/journal.pbio.0060006 <http://dx.doi.org/10.1371/journal.pbio.0060006>`_

diff --git a/hooke/ui/commandline.py b/hooke/ui/commandline.py
index 8a497181b8e0808d70775dfb6a3979b3f70aa044..e8db0af9a0924f9bbe9cc0c2d94898bfb4d37548 100644 (file)
--- a/hooke/ui/commandline.py
+++ b/hooke/ui/commandline.py
@@ -20,6 +20,7 @@
 line.
 """
 
+import codecs
 import cmd
 import optparse
 import readline # including readline makes cmd.Cmd.cmdloop() smarter
@@ -28,6 +29,7 @@ import shlex
 from ..command import CommandExit, Exit, Command, Argument, StoreValue
 from ..interaction import Request, BooleanRequest, ReloadUserInterfaceConfig
 from ..ui import UserInterface, CommandMessage
+from ..util.encoding import get_input_encoding, get_output_encoding
 
 
 # Define a few helper classes.
@@ -364,16 +366,20 @@ class CommandLine (UserInterface):
     def __init__(self):
         super(CommandLine, self).__init__(name='command line')
 
-    def run(self, commands, ui_to_command_queue, command_to_ui_queue):
+    def _cmd(self, commands, ui_to_command_queue, command_to_ui_queue):
         cmd = HookeCmd(self, commands,
                        inqueue=ui_to_command_queue,
                        outqueue=command_to_ui_queue)
+        cmd.stdin = codecs.getreader(get_input_encoding())(cmd.stdin)
+        cmd.stdout = codecs.getwriter(get_output_encoding())(cmd.stdout)
+        return cmd
+
+    def run(self, commands, ui_to_command_queue, command_to_ui_queue):
+        cmd = self._cmd(commands, ui_to_command_queue, command_to_ui_queue)
         cmd.cmdloop(self._splash_text())
 
     def run_lines(self, commands, ui_to_command_queue, command_to_ui_queue,
                   lines):
-        cmd = HookeCmd(self, commands,
-                       inqueue=ui_to_command_queue,
-                       outqueue=command_to_ui_queue)
+        cmd = self._cmd(commands, ui_to_command_queue, command_to_ui_queue)
         for line in lines:
             cmd.onecmd(line)

diff --git a/hooke/util/encoding.py b/hooke/util/encoding.py
new file mode 100644 (file)
index 0000000..e517b9e
--- /dev/null
+++ b/hooke/util/encoding.py
@@ -0,0 +1,32 @@
+# Copyright
+
+"""Define useful encoding-extraction functions.
+"""
+
+import locale
+import sys
+
+
+def get_encoding():
+    """Guess a useful input/output/filesystem encoding.
+
+    Maybe we need seperate encodings for input/output and filesystem?
+    Hmm.
+    """
+    encoding = locale.getpreferredencoding() or sys.getdefaultencoding()
+    if sys.platform != 'win32' or sys.version_info[:2] > (2, 3):
+        encoding = locale.getlocale(locale.LC_TIME)[1] or encoding
+        # Python 2.3 on windows doesn't know about 'XYZ' alias for 'cpXYZ'
+    return encoding
+
+def get_input_encoding():
+    "Guess the input encoding."
+    return get_encoding()
+
+def get_output_encoding():
+    "Guess the output encoding."
+    return get_encoding()
+
+def get_filesystem_encoding():
+    "Guess the filesystem encoding."
+    return get_encoding()

diff --git a/test/unicode_output.py b/test/unicode_output.py
new file mode 100644 (file)
index 0000000..0e37e97
--- /dev/null
+++ b/test/unicode_output.py
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2010 W. Trevor King <wking@drexel.edu>
+#
+# This file is part of Hooke.
+#
+# Hooke is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+#
+# Hooke is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with Hooke.  If not, see
+# <http://www.gnu.org/licenses/>.
+
+u"""
+>>> from hooke.hooke import Hooke, HookeRunner
+>>> h = Hooke()
+>>> r = HookeRunner()
+>>> h = r.run_lines(h, ['help flat_filter_playlist']) # doctest: +REPORT_UDIFF +ELLIPSIS
+Command: flat_filter_playlist...
+      F. Musiani, M. Brucale, L. Bubacco, B. Samorì.
+      "Conformational equilibria in monomeric α-Synuclein at the
+...
+"""
+
+import sys
+reload(sys)
+sys.setdefaultencoding('utf-8')
+
+# setdefaultencoding to work around doctest/unicode limitations:
+#   Doctest: test.unicode_output
+#   ... /usr/lib/python2.6/doctest.py:1475: UnicodeWarning: Unicode
+#   equal comparison failed to convert both arguments to Unicode -
+#   interpreting them as being unequal
+#
+# see
+#   http://stackoverflow.com/questions/1733414/how-do-i-include-unicode-strings-in-python-doctests
+# for details.
+#
+# Unfortunately, the setdefaultencoding also makes the
+# codecs.getwriter() code that this test is supposed to check a no-op.
+# Ah well, guess this will have to wait for Python 3.