enable ur'abc' literals and b'abc'/br'abc' literals

Cython/Compiler/Lexicon.py

@@ -5,7 +5,8 @@
 #   to be rebuilt next time pyrexc is run.
 #
 
-string_prefixes = "cCrRuU"
+raw_prefixes = "rR"
+string_prefixes = "cCuUbB"
 
 def make_lexicon():
     from Cython.Plex import \
@@ -55,9 +56,8 @@ def make_lexicon():
         + Rep(non_dq | (Str('"') + non_dq) | (Str('""') + non_dq)) 
         + Str('"""')
     )
-    stringlit = Opt(Any(string_prefixes)) + (sq_string | dq_string | tsq_string| tdq_string)
     
-    beginstring = Opt(Any(string_prefixes)) + (Str("'") | Str('"') | Str("'''") | Str('"""'))
+    beginstring = Opt(Any(string_prefixes)) + Opt(Any(raw_prefixes)) + (Str("'") | Str('"') | Str("'''") | Str('"""'))
     two_oct = octdigit + octdigit
     three_oct = octdigit + octdigit + octdigit
     two_hex = hexdigit + hexdigit

Cython/Compiler/Parsing.py

@@ -512,7 +512,7 @@ def p_name(s, name):
 
 def p_cat_string_literal(s):
     # A sequence of one or more adjacent string literals.
-    # Returns (kind, value) where kind in ('', 'c', 'r', 'u')
+    # Returns (kind, value) where kind in ('b', 'c', 'u')
     kind, value = p_string_literal(s)
     if kind != 'c':
         strings = [value]
@@ -537,19 +537,23 @@ def p_opt_string_literal(s):
 
 def p_string_literal(s):
     # A single string or char literal.
-    # Returns (kind, value) where kind in ('', 'c', 'r', 'u')
+    # Returns (kind, value) where kind in ('b', 'c', 'u')
     # s.sy == 'BEGIN_STRING'
     pos = s.position()
-    #is_raw = s.systring[:1].lower() == "r"
+    is_raw = 0
     kind = s.systring[:1].lower()
-    if kind not in "cru":
+    if kind == 'r':
+        kind = ''
+        is_raw = 1
+    elif kind in 'ub':
+        is_raw = s.systring[1:2].lower() == 'r'
+    elif kind != 'c':
         kind = ''
     if Future.unicode_literals in s.context.future_directives:
         if kind == '':
             kind = 'u'
-        elif kind == 'u':
-            s.error("string literal must not start with 'u' when importing __future__.unicode_literals")
-            return ('u', '')
+    elif kind == '':
+        kind = 'b'
     chars = []
     while 1:
         s.next()
@@ -562,7 +566,7 @@ def p_string_literal(s):
             chars.append(systr)
         elif sy == 'ESCAPE':
             systr = s.systring
-            if kind == 'r':
+            if is_raw:
                 if systr == '\\\n':
                     chars.append(r'\\\n')
                 elif systr == r'\"':

Cython/Compiler/Scanning.py

@@ -15,7 +15,7 @@ from Cython import Plex, Utils
 from Cython.Plex import Scanner
 from Cython.Plex.Errors import UnrecognizedInput
 from Errors import CompileError, error
-from Lexicon import string_prefixes, make_lexicon
+from Lexicon import string_prefixes, raw_prefixes, make_lexicon
 
 plex_version = getattr(Plex, '_version', None)
 #print "Plex version:", plex_version ###
@@ -262,6 +262,8 @@ class PyrexScanner(Scanner):
     def begin_string_action(self, text):
         if text[:1] in string_prefixes:
             text = text[1:]
+        if text[:1] in raw_prefixes:
+            text = text[1:]
         self.begin(self.string_states[text])
         self.produce('BEGIN_STRING')
     

tests/run/append.pyx

@@ -25,11 +25,12 @@ class A:
     def append(self, x):
         print u"appending"
         return x
-        
+
 class B(list):
     def append(self, *args):
+        append = super(B, self).append
         for arg in args:
-            list.append(self, arg)
+            append(arg)
 
 def test_append(L):
     print L.append(1)