Fix command-line encoding processing.

String command-line options are converted to unicode using the input
encoding.

We use the fact that Python sets up the original sys.stdout to
determine the terminal encoding.

This should fix Anders Sneckenborg's issues with Swedish characters:

  C:\temp\slask4>be new "Svenska tecken åäö"
  Created bug with ID 6be/5c3

  C:\temp\slask4>
  C:\temp\slask4>
  C:\temp\slask4>be list
  ERROR:
  'ascii' codec can't decode byte 0xe5 in position 15: ordinal not in
  range(128)
  You should set a locale that supports unicode, e.g.
    export LANG=en_US.utf8
  See http://docs.python.org/library/locale.html for details

diff --git a/libbe/ui/command_line.py b/libbe/ui/command_line.py
index 5f42147b7c2ea90972dc65c40dbc34eebc178fb2..52daa4bde2e52a6c1aab25bb5b2cf44ef8168587 100644 (file)
--- a/libbe/ui/command_line.py
+++ b/libbe/ui/command_line.py
@@ -29,6 +29,8 @@ import libbe.command
 import libbe.command.util
 import libbe.version
 import libbe.ui.util.pager
+import libbe.util.encoding
+
 
 if libbe.TESTING == True:
     import doctest
@@ -86,11 +88,11 @@ class CmdOptionParser(optparse.OptionParser):
             if '_' in name: # reconstruct original option name
                 options[name.replace('_', '-')] = options.pop(name)
         for name,value in options.items():
+            argument = None
+            option = self._option_by_name[name]
+            if option.arg != None:
+                argument = option.arg
             if value == '--complete':
-                argument = None
-                option = self._option_by_name[name]
-                if option.arg != None:
-                    argument = option.arg
                 fragment = None
                 indices = [i for i,arg in enumerate(args)
                            if arg == '--complete']
@@ -108,22 +110,28 @@ class CmdOptionParser(optparse.OptionParser):
                 if i+1 < len(args):
                     fragment = args[i+1]
                 self.complete(argument, fragment)
+            elif argument is not None:
+                value = self.process_raw_argument(argument=argument, value=value)
+                options[name] = value
         for i,arg in enumerate(parsed_args):
+            if i > 0 and self.command.name == 'be':
+                break # let this pass through for the command parser to handle
+            elif i < len(self.command.args):
+                argument = self.command.args[i]
+            elif len(self.command.args) == 0:
+                break # command doesn't take arguments
+            else:
+                argument = self.command.args[-1]
+                if argument.repeatable == False:
+                    raise libbe.command.UserError('Too many arguments')
             if arg == '--complete':
-                if i > 0 and self.command.name == 'be':
-                    break # let this pass through for the command parser to handle
-                elif i < len(self.command.args):
-                    argument = self.command.args[i]
-                elif len(self.command.args) == 0:
-                    break # command doesn't take arguments
-                else:
-                    argument = self.command.args[-1]
-                    if argument.repeatable == False:
-                        raise libbe.command.UserError('Too many arguments')
                 fragment = None
                 if i < len(parsed_args) - 1:
                     fragment = parsed_args[i+1]
                 self.complete(argument, fragment)
+            else:
+                value = self.process_raw_argument(argument=argument, value=arg)
+                parsed_args[i] = value
         if len(parsed_args) > len(self.command.args) \
                 and self.command.args[-1].repeatable == False:
             raise libbe.command.UserError('Too many arguments')
@@ -154,6 +162,16 @@ class CmdOptionParser(optparse.OptionParser):
             print >> self.command.stdout, '\n'.join(comps)
         raise CallbackExit
 
+    def process_raw_argument(self, argument, value):
+        if value == argument.default:
+            return value
+        if argument.type == 'string':
+            if not hasattr(self, 'argv_encoding'):
+                self.argv_encoding = libbe.util.encoding.get_argv_encoding()
+            return unicode(value, self.argv_encoding)
+        return value
+
+
 class BE (libbe.command.Command):
     """Class for parsing the command line arguments for `be`.
     This class does not contain a useful _run() method.  Call this

diff --git a/libbe/util/encoding.py b/libbe/util/encoding.py
index 5950bb9a2558797b72478f08fd211a5dcbe17972..c759529fdf6ce8d99783853278a44b5e64775744 100644 (file)
--- a/libbe/util/encoding.py
+++ b/libbe/util/encoding.py
@@ -49,11 +49,14 @@ def get_input_encoding():
     return get_encoding()
 
 def get_output_encoding():
-    return get_encoding()
+    return sys.__stdout__.encoding
 
 def get_filesystem_encoding():
     return get_encoding()
 
+def get_argv_encoding():
+    return get_encoding()
+
 def known_encoding(encoding):
     """
     >>> known_encoding("highly-unlikely-encoding")