byte encode docstrings correctly

diff --git a/Cython/Compiler/PyrexTypes.py b/Cython/Compiler/PyrexTypes.py
index e44de1900813d0ffe7a7a748b8a82964e58db4fd..31181064d8508aa2d009d1b82b191494ccdcb07f 100644 (file)
--- a/Cython/Compiler/PyrexTypes.py
+++ b/Cython/Compiler/PyrexTypes.py
@@ -2,6 +2,7 @@
 #   Pyrex - Types
 #
 
+from Cython import Utils
 import Naming
 
 class BaseType:
@@ -922,23 +923,6 @@ class CEnumType(CType):
             return self.base_declaration_code(public_decl(base, dll_linkage), entity_code)
 
 
-def _escape_byte_string(s):
-    s = s.replace('\0', r'\x00')
-    try:
-        s.decode("ASCII")
-        return s
-    except UnicodeDecodeError:
-        pass
-    l = []
-    append = l.append
-    for c in s:
-        o = ord(c)
-        if o >= 128:
-            append('\\x%X' % o)
-        else:
-            append(c)
-    return ''.join(l)
-
 class CStringType:
     #  Mixin class for C string types.
 
@@ -951,7 +935,7 @@ class CStringType:
 
     def literal_code(self, value):
         assert isinstance(value, str)
-        return '"%s"' % _escape_byte_string(value)
+        return '"%s"' % Utils.escape_byte_string(value)
 
 
 class CUTF8StringType:
@@ -965,7 +949,7 @@ class CUTF8StringType:
 
     def literal_code(self, value):
         assert isinstance(value, str)
-        return '"%s"' % _escape_byte_string(value)
+        return '"%s"' % Utils.escape_byte_string(value)
 
 
 class CCharArrayType(CStringType, CArrayType):

diff --git a/Cython/Compiler/TypeSlots.py b/Cython/Compiler/TypeSlots.py
index d05edb6182ada53f032909f5368872ca0bfc00dc..9f06a445532e9f1a3536716cf9d97b13f6141055 100644 (file)
--- a/Cython/Compiler/TypeSlots.py
+++ b/Cython/Compiler/TypeSlots.py
@@ -3,6 +3,7 @@
 #           and associated know-how.
 #
 
+from Cython import Utils
 import Naming
 import PyrexTypes
 import sys
@@ -298,7 +299,11 @@ class DocStringSlot(SlotDescriptor):
     
     def slot_code(self, scope):
         if scope.doc is not None:
-            return '"%s"' % scope.doc
+            if scope.doc.is_unicode:
+                doc = scope.doc.utf8encode()
+            else:
+                doc = scope.doc.byteencode()
+            return '"%s"' % Utils.escape_byte_string(doc)
         else:
             return "0"
 

diff --git a/Cython/Utils.py b/Cython/Utils.py
index 20125e24372712f062056f1e7a045a303ce41de4..de22a6a9da14e3d12ea2e5221c064cd155896898 100644 (file)
--- a/Cython/Utils.py
+++ b/Cython/Utils.py
@@ -91,3 +91,20 @@ class EncodedString(unicode):
 #    def __eq__(self, other):
 #        return unicode.__eq__(self, other) and \
 #            getattr(other, 'encoding', '') == self.encoding
+
+def escape_byte_string(s):
+    s = s.replace('\0', r'\x00')
+    try:
+        s.decode("ASCII")
+        return s
+    except UnicodeDecodeError:
+        pass
+    l = []
+    append = l.append
+    for c in s:
+        o = ord(c)
+        if o >= 128:
+            append('\\x%X' % o)
+        else:
+            append(c)
+    return ''.join(l)