Commit 9eefda4b authored by Stefan Behnel's avatar Stefan Behnel

enable ur'abc' literals and b'abc'/br'abc' literals

parent 0e875775
...@@ -5,7 +5,8 @@ ...@@ -5,7 +5,8 @@
# to be rebuilt next time pyrexc is run. # to be rebuilt next time pyrexc is run.
# #
string_prefixes = "cCrRuU" raw_prefixes = "rR"
string_prefixes = "cCuUbB"
def make_lexicon(): def make_lexicon():
from Cython.Plex import \ from Cython.Plex import \
...@@ -55,9 +56,8 @@ def make_lexicon(): ...@@ -55,9 +56,8 @@ def make_lexicon():
+ Rep(non_dq | (Str('"') + non_dq) | (Str('""') + non_dq)) + Rep(non_dq | (Str('"') + non_dq) | (Str('""') + non_dq))
+ Str('"""') + Str('"""')
) )
stringlit = Opt(Any(string_prefixes)) + (sq_string | dq_string | tsq_string| tdq_string)
beginstring = Opt(Any(string_prefixes)) + (Str("'") | Str('"') | Str("'''") | Str('"""')) beginstring = Opt(Any(string_prefixes)) + Opt(Any(raw_prefixes)) + (Str("'") | Str('"') | Str("'''") | Str('"""'))
two_oct = octdigit + octdigit two_oct = octdigit + octdigit
three_oct = octdigit + octdigit + octdigit three_oct = octdigit + octdigit + octdigit
two_hex = hexdigit + hexdigit two_hex = hexdigit + hexdigit
......
...@@ -512,7 +512,7 @@ def p_name(s, name): ...@@ -512,7 +512,7 @@ def p_name(s, name):
def p_cat_string_literal(s): def p_cat_string_literal(s):
# A sequence of one or more adjacent string literals. # A sequence of one or more adjacent string literals.
# Returns (kind, value) where kind in ('', 'c', 'r', 'u') # Returns (kind, value) where kind in ('b', 'c', 'u')
kind, value = p_string_literal(s) kind, value = p_string_literal(s)
if kind != 'c': if kind != 'c':
strings = [value] strings = [value]
...@@ -537,19 +537,23 @@ def p_opt_string_literal(s): ...@@ -537,19 +537,23 @@ def p_opt_string_literal(s):
def p_string_literal(s): def p_string_literal(s):
# A single string or char literal. # A single string or char literal.
# Returns (kind, value) where kind in ('', 'c', 'r', 'u') # Returns (kind, value) where kind in ('b', 'c', 'u')
# s.sy == 'BEGIN_STRING' # s.sy == 'BEGIN_STRING'
pos = s.position() pos = s.position()
#is_raw = s.systring[:1].lower() == "r" is_raw = 0
kind = s.systring[:1].lower() kind = s.systring[:1].lower()
if kind not in "cru": if kind == 'r':
kind = ''
is_raw = 1
elif kind in 'ub':
is_raw = s.systring[1:2].lower() == 'r'
elif kind != 'c':
kind = '' kind = ''
if Future.unicode_literals in s.context.future_directives: if Future.unicode_literals in s.context.future_directives:
if kind == '': if kind == '':
kind = 'u' kind = 'u'
elif kind == 'u': elif kind == '':
s.error("string literal must not start with 'u' when importing __future__.unicode_literals") kind = 'b'
return ('u', '')
chars = [] chars = []
while 1: while 1:
s.next() s.next()
...@@ -562,7 +566,7 @@ def p_string_literal(s): ...@@ -562,7 +566,7 @@ def p_string_literal(s):
chars.append(systr) chars.append(systr)
elif sy == 'ESCAPE': elif sy == 'ESCAPE':
systr = s.systring systr = s.systring
if kind == 'r': if is_raw:
if systr == '\\\n': if systr == '\\\n':
chars.append(r'\\\n') chars.append(r'\\\n')
elif systr == r'\"': elif systr == r'\"':
......
...@@ -15,7 +15,7 @@ from Cython import Plex, Utils ...@@ -15,7 +15,7 @@ from Cython import Plex, Utils
from Cython.Plex import Scanner from Cython.Plex import Scanner
from Cython.Plex.Errors import UnrecognizedInput from Cython.Plex.Errors import UnrecognizedInput
from Errors import CompileError, error from Errors import CompileError, error
from Lexicon import string_prefixes, make_lexicon from Lexicon import string_prefixes, raw_prefixes, make_lexicon
plex_version = getattr(Plex, '_version', None) plex_version = getattr(Plex, '_version', None)
#print "Plex version:", plex_version ### #print "Plex version:", plex_version ###
...@@ -262,6 +262,8 @@ class PyrexScanner(Scanner): ...@@ -262,6 +262,8 @@ class PyrexScanner(Scanner):
def begin_string_action(self, text): def begin_string_action(self, text):
if text[:1] in string_prefixes: if text[:1] in string_prefixes:
text = text[1:] text = text[1:]
if text[:1] in raw_prefixes:
text = text[1:]
self.begin(self.string_states[text]) self.begin(self.string_states[text])
self.produce('BEGIN_STRING') self.produce('BEGIN_STRING')
......
...@@ -28,8 +28,9 @@ class A: ...@@ -28,8 +28,9 @@ class A:
class B(list): class B(list):
def append(self, *args): def append(self, *args):
append = super(B, self).append
for arg in args: for arg in args:
list.append(self, arg) append(arg)
def test_append(L): def test_append(L):
print L.append(1) print L.append(1)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment