Commit 075e2829 authored by Stefan Behnel's avatar Stefan Behnel

move Cython language keywords out of the way when parsing .py files

parent 7072be52
...@@ -14,6 +14,7 @@ cdef class PyrexScanner(Scanner): ...@@ -14,6 +14,7 @@ cdef class PyrexScanner(Scanner):
cdef public bint compile_time_expr cdef public bint compile_time_expr
cdef public bint parse_comments cdef public bint parse_comments
cdef public source_encoding cdef public source_encoding
cdef set keywords
cdef public list indentation_stack cdef public list indentation_stack
cdef public indentation_char cdef public indentation_char
cdef public int bracket_nesting_level cdef public int bracket_nesting_level
......
...@@ -37,13 +37,17 @@ def get_lexicon(): ...@@ -37,13 +37,17 @@ def get_lexicon():
#------------------------------------------------------------------ #------------------------------------------------------------------
reserved_words = [ py_reserved_words = [
"global", "include", "ctypedef", "cdef", "def", "class", "global", "def", "class", "print", "del", "pass", "break",
"print", "del", "pass", "break", "continue", "return", "continue", "return", "raise", "import", "exec", "try",
"raise", "import", "exec", "try", "except", "finally", "except", "finally", "while", "if", "elif", "else", "for",
"while", "if", "elif", "else", "for", "in", "assert", "in", "assert", "and", "or", "not", "is", "in", "lambda",
"and", "or", "not", "is", "in", "lambda", "from", "yield", "from", "yield", "with", "nonlocal",
"cimport", "by", "with", "cpdef", "DEF", "IF", "ELIF", "ELSE" ]
pyx_reserved_words = py_reserved_words + [
"include", "ctypedef", "cdef", "cpdef",
"cimport", "by", "DEF", "IF", "ELIF", "ELSE"
] ]
class Method(object): class Method(object):
...@@ -57,17 +61,6 @@ class Method(object): ...@@ -57,17 +61,6 @@ class Method(object):
#------------------------------------------------------------------ #------------------------------------------------------------------
def build_resword_dict():
d = {}
for word in reserved_words:
d[word] = 1
return d
cython.declare(resword_dict=dict)
resword_dict = build_resword_dict()
#------------------------------------------------------------------
class CompileTimeScope(object): class CompileTimeScope(object):
def __init__(self, outer = None): def __init__(self, outer = None):
...@@ -123,11 +116,23 @@ class SourceDescriptor(object): ...@@ -123,11 +116,23 @@ class SourceDescriptor(object):
""" """
A SourceDescriptor should be considered immutable. A SourceDescriptor should be considered immutable.
""" """
_file_type = 'pyx'
_escaped_description = None _escaped_description = None
_cmp_name = '' _cmp_name = ''
def __str__(self): def __str__(self):
assert False # To catch all places where a descriptor is used directly as a filename assert False # To catch all places where a descriptor is used directly as a filename
def set_file_type_from_name(self, filename):
name, ext = os.path.splitext(filename)
self._file_type = ext in ('.pyx', '.pxd', '.py') and ext[1:] or 'pyx'
def is_cython_file(self):
return self._file_type in ('pyx', 'pxd')
def is_python_file(self):
return self._file_type == 'py'
def get_escaped_description(self): def get_escaped_description(self):
if self._escaped_description is None: if self._escaped_description is None:
self._escaped_description = \ self._escaped_description = \
...@@ -165,6 +170,7 @@ class FileSourceDescriptor(SourceDescriptor): ...@@ -165,6 +170,7 @@ class FileSourceDescriptor(SourceDescriptor):
""" """
def __init__(self, filename): def __init__(self, filename):
self.filename = filename self.filename = filename
self.set_file_type_from_name(filename)
self._cmp_name = filename self._cmp_name = filename
def get_lines(self, encoding=None, error_handling=None): def get_lines(self, encoding=None, error_handling=None):
...@@ -196,6 +202,7 @@ class StringSourceDescriptor(SourceDescriptor): ...@@ -196,6 +202,7 @@ class StringSourceDescriptor(SourceDescriptor):
""" """
def __init__(self, name, code): def __init__(self, name, code):
self.name = name self.name = name
self.set_file_type_from_name(name)
self.codelines = [x + "\n" for x in code.split("\n")] self.codelines = [x + "\n" for x in code.split("\n")]
self._cmp_name = name self._cmp_name = name
...@@ -247,6 +254,10 @@ class PyrexScanner(Scanner): ...@@ -247,6 +254,10 @@ class PyrexScanner(Scanner):
self.compile_time_expr = 0 self.compile_time_expr = 0
self.parse_comments = parse_comments self.parse_comments = parse_comments
self.source_encoding = source_encoding self.source_encoding = source_encoding
if filename.is_python_file():
self.keywords = cython.set(py_reserved_words)
else:
self.keywords = cython.set(pyx_reserved_words)
self.trace = trace_scanner self.trace = trace_scanner
self.indentation_stack = [0] self.indentation_stack = [0]
self.indentation_char = None self.indentation_char = None
...@@ -346,7 +357,7 @@ class PyrexScanner(Scanner): ...@@ -346,7 +357,7 @@ class PyrexScanner(Scanner):
except UnrecognizedInput: except UnrecognizedInput:
self.error("Unrecognized character") self.error("Unrecognized character")
if sy == IDENT: if sy == IDENT:
if systring in resword_dict: if systring in self.keywords:
if systring == 'print' and \ if systring == 'print' and \
print_function in self.context.future_directives: print_function in self.context.future_directives:
systring = EncodedString(systring) systring = EncodedString(systring)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment