From 9eefda4b4e50e0c00416813078dba370b880b867 Mon Sep 17 00:00:00 2001
From: Stefan Behnel <scoder@users.berlios.de>
Date: Sat, 17 May 2008 06:37:25 +0200
Subject: [PATCH] enable ur'abc' literals and b'abc'/br'abc' literals

---
 Cython/Compiler/Lexicon.py  |  6 +++---
 Cython/Compiler/Parsing.py  | 20 ++++++++++++--------
 Cython/Compiler/Scanning.py |  4 +++-
 tests/run/append.pyx        |  5 +++--
 4 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/Cython/Compiler/Lexicon.py b/Cython/Compiler/Lexicon.py
index cd9cfdb9..0f8bc898 100644
--- a/Cython/Compiler/Lexicon.py
+++ b/Cython/Compiler/Lexicon.py
@@ -5,7 +5,8 @@
 #   to be rebuilt next time pyrexc is run.
 #
 
-string_prefixes = "cCrRuU"
+raw_prefixes = "rR"
+string_prefixes = "cCuUbB"
 
 def make_lexicon():
     from Cython.Plex import \
@@ -55,9 +56,8 @@ def make_lexicon():
         + Rep(non_dq | (Str('"') + non_dq) | (Str('""') + non_dq)) 
         + Str('"""')
     )
-    stringlit = Opt(Any(string_prefixes)) + (sq_string | dq_string | tsq_string| tdq_string)
     
-    beginstring = Opt(Any(string_prefixes)) + (Str("'") | Str('"') | Str("'''") | Str('"""'))
+    beginstring = Opt(Any(string_prefixes)) + Opt(Any(raw_prefixes)) + (Str("'") | Str('"') | Str("'''") | Str('"""'))
     two_oct = octdigit + octdigit
     three_oct = octdigit + octdigit + octdigit
     two_hex = hexdigit + hexdigit
diff --git a/Cython/Compiler/Parsing.py b/Cython/Compiler/Parsing.py
index 69d45896..c52b19b9 100644
--- a/Cython/Compiler/Parsing.py
+++ b/Cython/Compiler/Parsing.py
@@ -512,7 +512,7 @@ def p_name(s, name):
 
 def p_cat_string_literal(s):
     # A sequence of one or more adjacent string literals.
-    # Returns (kind, value) where kind in ('', 'c', 'r', 'u')
+    # Returns (kind, value) where kind in ('b', 'c', 'u')
     kind, value = p_string_literal(s)
     if kind != 'c':
         strings = [value]
@@ -537,19 +537,23 @@ def p_opt_string_literal(s):
 
 def p_string_literal(s):
     # A single string or char literal.
-    # Returns (kind, value) where kind in ('', 'c', 'r', 'u')
+    # Returns (kind, value) where kind in ('b', 'c', 'u')
     # s.sy == 'BEGIN_STRING'
     pos = s.position()
-    #is_raw = s.systring[:1].lower() == "r"
+    is_raw = 0
     kind = s.systring[:1].lower()
-    if kind not in "cru":
+    if kind == 'r':
+        kind = ''
+        is_raw = 1
+    elif kind in 'ub':
+        is_raw = s.systring[1:2].lower() == 'r'
+    elif kind != 'c':
         kind = ''
     if Future.unicode_literals in s.context.future_directives:
         if kind == '':
             kind = 'u'
-        elif kind == 'u':
-            s.error("string literal must not start with 'u' when importing __future__.unicode_literals")
-            return ('u', '')
+    elif kind == '':
+        kind = 'b'
     chars = []
     while 1:
         s.next()
@@ -562,7 +566,7 @@ def p_string_literal(s):
             chars.append(systr)
         elif sy == 'ESCAPE':
             systr = s.systring
-            if kind == 'r':
+            if is_raw:
                 if systr == '\\\n':
                     chars.append(r'\\\n')
                 elif systr == r'\"':
diff --git a/Cython/Compiler/Scanning.py b/Cython/Compiler/Scanning.py
index 67016d99..bb338a5e 100644
--- a/Cython/Compiler/Scanning.py
+++ b/Cython/Compiler/Scanning.py
@@ -15,7 +15,7 @@ from Cython import Plex, Utils
 from Cython.Plex import Scanner
 from Cython.Plex.Errors import UnrecognizedInput
 from Errors import CompileError, error
-from Lexicon import string_prefixes, make_lexicon
+from Lexicon import string_prefixes, raw_prefixes, make_lexicon
 
 plex_version = getattr(Plex, '_version', None)
 #print "Plex version:", plex_version ###
@@ -262,6 +262,8 @@ class PyrexScanner(Scanner):
     def begin_string_action(self, text):
         if text[:1] in string_prefixes:
             text = text[1:]
+        if text[:1] in raw_prefixes:
+            text = text[1:]
         self.begin(self.string_states[text])
         self.produce('BEGIN_STRING')
     
diff --git a/tests/run/append.pyx b/tests/run/append.pyx
index d8d72ce1..29575aa3 100644
--- a/tests/run/append.pyx
+++ b/tests/run/append.pyx
@@ -25,11 +25,12 @@ class A:
     def append(self, x):
         print u"appending"
         return x
-        
+
 class B(list):
     def append(self, *args):
+        append = super(B, self).append
         for arg in args:
-            list.append(self, arg)
+            append(arg)
 
 def test_append(L):
     print L.append(1)
-- 
2.26.2