Commit cf8dc445 authored by Stefan Behnel's avatar Stefan Behnel

merge

parents 35b11d4c 99327e65
...@@ -717,9 +717,11 @@ class CVarDefNode(StatNode): ...@@ -717,9 +717,11 @@ class CVarDefNode(StatNode):
# in_pxd boolean # in_pxd boolean
# api boolean # api boolean
# need_properties [entry] # need_properties [entry]
# pxd_locals [CVarDefNode] (used for functions declared in pxd)
child_attrs = ["base_type", "declarators"] child_attrs = ["base_type", "declarators"]
need_properties = () need_properties = ()
pxd_locals = []
def analyse_declarations(self, env, dest_scope = None): def analyse_declarations(self, env, dest_scope = None):
if not dest_scope: if not dest_scope:
...@@ -755,6 +757,7 @@ class CVarDefNode(StatNode): ...@@ -755,6 +757,7 @@ class CVarDefNode(StatNode):
entry = dest_scope.declare_cfunction(name, type, declarator.pos, entry = dest_scope.declare_cfunction(name, type, declarator.pos,
cname = cname, visibility = self.visibility, in_pxd = self.in_pxd, cname = cname, visibility = self.visibility, in_pxd = self.in_pxd,
api = self.api) api = self.api)
entry.pxd_locals = self.pxd_locals
else: else:
if self.in_pxd and self.visibility != 'extern': if self.in_pxd and self.visibility != 'extern':
error(self.pos, error(self.pos,
...@@ -891,10 +894,12 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -891,10 +894,12 @@ class FuncDefNode(StatNode, BlockNode):
# #filename string C name of filename string const # #filename string C name of filename string const
# entry Symtab.Entry # entry Symtab.Entry
# needs_closure boolean Whether or not this function has inner functions/classes/yield # needs_closure boolean Whether or not this function has inner functions/classes/yield
# pxd_locals [CVarDefNode] locals defined in the pxd
py_func = None py_func = None
assmt = None assmt = None
needs_closure = False needs_closure = False
pxd_locals = []
def analyse_default_values(self, env): def analyse_default_values(self, env):
genv = env.global_scope() genv = env.global_scope()
...@@ -1498,7 +1503,8 @@ class DefNode(FuncDefNode): ...@@ -1498,7 +1503,8 @@ class DefNode(FuncDefNode):
with_gil = cfunc.type.with_gil, with_gil = cfunc.type.with_gil,
nogil = cfunc.type.nogil, nogil = cfunc.type.nogil,
visibility = 'private', visibility = 'private',
api = False) api = False,
pxd_locals = cfunc.pxd_locals)
def analyse_declarations(self, env): def analyse_declarations(self, env):
if 'locals' in env.directives: if 'locals' in env.directives:
...@@ -2317,10 +2323,11 @@ class PyClassDefNode(ClassDefNode): ...@@ -2317,10 +2323,11 @@ class PyClassDefNode(ClassDefNode):
elif len(bases) == 1: elif len(bases) == 1:
base = bases[0] base = bases[0]
path = [] path = []
while isinstance(base, ExprNodes.AttributeNode): from ExprNodes import AttributeNode, NameNode
while isinstance(base, AttributeNode):
path.insert(0, base.attribute) path.insert(0, base.attribute)
base = base.obj base = base.obj
if isinstance(base, ExprNodes.NameNode): if isinstance(base, NameNode):
path.insert(0, base.name) path.insert(0, base.name)
base_class_name = path[-1] base_class_name = path[-1]
if len(path) > 1: if len(path) > 1:
...@@ -4439,6 +4446,9 @@ class FromImportStatNode(StatNode): ...@@ -4439,6 +4446,9 @@ class FromImportStatNode(StatNode):
env.use_utility_code(ExprNodes.type_test_utility_code) env.use_utility_code(ExprNodes.type_test_utility_code)
break break
else: else:
entry = env.lookup(target.name)
if entry.is_type and entry.type.name == name and entry.type.module_name == self.module.module_name.value:
continue # already cimported
self.interned_items.append( self.interned_items.append(
(env.intern_identifier(name), target)) (env.intern_identifier(name), target))
target.analyse_target_expression(env, None) target.analyse_target_expression(env, None)
......
...@@ -235,6 +235,22 @@ class PxdPostParse(CythonTransform): ...@@ -235,6 +235,22 @@ class PxdPostParse(CythonTransform):
and node.name in ('__getbuffer__', '__releasebuffer__')): and node.name in ('__getbuffer__', '__releasebuffer__')):
ok = True ok = True
if isinstance(node, CFuncDefNode):
ok = True
for stat in node.body.stats:
if not isinstance(stat, CVarDefNode):
self.context.error("C function definition not allowed here")
ok = False
break
node = CVarDefNode(node.pos,
visibility = node.visibility,
base_type = node.base_type,
declarators = [node.declarator],
in_pxd = True,
api = node.api,
overridable = node.overridable,
pxd_locals = node.body.stats)
if not ok: if not ok:
self.context.nonfatal_error(PostParseError(node.pos, self.context.nonfatal_error(PostParseError(node.pos,
self.ERR_FUNCDEF_NOT_ALLOWED)) self.ERR_FUNCDEF_NOT_ALLOWED))
...@@ -547,6 +563,8 @@ property NAME: ...@@ -547,6 +563,8 @@ property NAME:
lenv.declare_var(var, type, type_node.pos) lenv.declare_var(var, type, type_node.pos)
else: else:
error(type_node.pos, "Not a type") error(type_node.pos, "Not a type")
for stat in node.pxd_locals:
stat.analyse_declarations(lenv)
node.body.analyse_declarations(lenv) node.body.analyse_declarations(lenv)
self.env_stack.append(lenv) self.env_stack.append(lenv)
self.visitchildren(node) self.visitchildren(node)
......
...@@ -2145,7 +2145,7 @@ def p_c_func_or_var_declaration(s, pos, ctx): ...@@ -2145,7 +2145,7 @@ def p_c_func_or_var_declaration(s, pos, ctx):
assignable = 1, nonempty = 1) assignable = 1, nonempty = 1)
declarator.overridable = ctx.overridable declarator.overridable = ctx.overridable
if s.sy == ':': if s.sy == ':':
if ctx.level not in ('module', 'c_class'): if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd'):
s.error("C function definition not allowed here") s.error("C function definition not allowed here")
doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1) doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1)
result = Nodes.CFuncDefNode(pos, result = Nodes.CFuncDefNode(pos,
......
from Cython.Plex.Scanners cimport Scanner
cdef class PyrexScanner(Scanner):
cdef public context
cdef public list included_files
cdef public compile_time_env
cdef public bint compile_time_eval
cdef public bint compile_time_expr
cdef public bint parse_comments
cdef public source_encoding
cdef public list indentation_stack
cdef public indentation_char
cdef public int bracket_nesting_level
cdef public sy
cdef public systring
cdef long current_level(self)
cpdef begin(self, state)
cpdef next(self)
cpdef bint expect(self, what, message = *) except -2
cpdef indentation_action(self, text):
cdef:
long current_level
long new_level
...@@ -12,14 +12,17 @@ import sys ...@@ -12,14 +12,17 @@ import sys
from time import time from time import time
from Cython import Plex, Utils from Cython import Plex, Utils
from Cython.Plex import Scanner from Cython.Plex.Scanners import Scanner
from Cython.Plex.Errors import UnrecognizedInput from Cython.Plex.Errors import UnrecognizedInput
from Errors import CompileError, error from Errors import CompileError, error
from Lexicon import string_prefixes, raw_prefixes, make_lexicon from Lexicon import string_prefixes, raw_prefixes, make_lexicon
from StringEncoding import EncodedString from StringEncoding import EncodedString
plex_version = getattr(Plex, '_version', None) try:
plex_version = Plex._version
except AttributeError:
plex_version = None
#print "Plex version:", plex_version ### #print "Plex version:", plex_version ###
debug_scanner = 0 debug_scanner = 0
...@@ -91,7 +94,7 @@ def try_to_unpickle_lexicon(): ...@@ -91,7 +94,7 @@ def try_to_unpickle_lexicon():
source_file = os.path.join(dir, "Lexicon.py") source_file = os.path.join(dir, "Lexicon.py")
lexicon_hash = hash_source_file(source_file) lexicon_hash = hash_source_file(source_file)
lexicon_pickle = os.path.join(dir, "Lexicon.pickle") lexicon_pickle = os.path.join(dir, "Lexicon.pickle")
f = open_pickled_lexicon(expected_hash = lexicon_hash) f = open_pickled_lexicon(lexicon_hash)
if f: if f:
if notify_lexicon_unpickling: if notify_lexicon_unpickling:
t0 = time() t0 = time()
...@@ -165,6 +168,8 @@ def build_resword_dict(): ...@@ -165,6 +168,8 @@ def build_resword_dict():
d[word] = 1 d[word] = 1
return d return d
resword_dict = build_resword_dict()
#------------------------------------------------------------------ #------------------------------------------------------------------
class CompileTimeScope(object): class CompileTimeScope(object):
...@@ -286,7 +291,6 @@ class PyrexScanner(Scanner): ...@@ -286,7 +291,6 @@ class PyrexScanner(Scanner):
# compile_time_env dict Environment for conditional compilation # compile_time_env dict Environment for conditional compilation
# compile_time_eval boolean In a true conditional compilation context # compile_time_eval boolean In a true conditional compilation context
# compile_time_expr boolean In a compile-time expression context # compile_time_expr boolean In a compile-time expression context
resword_dict = build_resword_dict()
def __init__(self, file, filename, parent_scanner = None, def __init__(self, file, filename, parent_scanner = None,
scope = None, context = None, source_encoding=None, parse_comments=True, initial_pos=None): scope = None, context = None, source_encoding=None, parse_comments=True, initial_pos=None):
...@@ -404,14 +408,14 @@ class PyrexScanner(Scanner): ...@@ -404,14 +408,14 @@ class PyrexScanner(Scanner):
except UnrecognizedInput: except UnrecognizedInput:
self.error("Unrecognized character") self.error("Unrecognized character")
if sy == 'IDENT': if sy == 'IDENT':
if systring in self.resword_dict: if systring in resword_dict:
sy = systring sy = systring
else: else:
systring = EncodedString(systring) systring = EncodedString(systring)
systring.encoding = self.source_encoding systring.encoding = self.source_encoding
self.sy = sy self.sy = sy
self.systring = systring self.systring = systring
if debug_scanner: if False: # debug_scanner:
_, line, col = self.position() _, line, col = self.position()
if not self.systring or self.sy == self.systring: if not self.systring or self.sy == self.systring:
t = self.sy t = self.sy
......
...@@ -821,6 +821,8 @@ class ModuleScope(Scope): ...@@ -821,6 +821,8 @@ class ModuleScope(Scope):
self.cached_builtins = [] self.cached_builtins = []
self.undeclared_cached_builtins = [] self.undeclared_cached_builtins = []
self.namespace_cname = self.module_cname self.namespace_cname = self.module_cname
for name in ['__builtins__', '__name__', '__file__', '__doc__']:
self.declare_var(EncodedString(name), py_object_type, None)
def qualifying_scope(self): def qualifying_scope(self):
return self.parent_module return self.parent_module
......
import cython
cdef class Scanner:
cdef public lexicon
cdef public stream
cdef public name
cdef public buffer
cdef public long buf_start_pos
cdef public long next_pos
cdef public long cur_pos
cdef public long cur_line
cdef public long cur_line_start
cdef public long start_pos
cdef public long start_line
cdef public long start_col
cdef public text
cdef public initial_state # int?
cdef public state_name
cdef public list queue
cdef public bint trace
cdef public cur_char
cdef public input_state
cdef public level # int?
cpdef next_char(self):
cdef:
long input_state
cpdef run_machine_inlined(self):
cdef:
long cur_pos
long cur_line
long cur_line_start
long input_state
long next_pos
long buf_start_pos
long buf_len
long buf_index
bint trace
long discard
cpdef begin(self, state)
cpdef produce(self, value, text = *)
...@@ -10,6 +10,8 @@ ...@@ -10,6 +10,8 @@
import Errors import Errors
from Regexps import BOL, EOL, EOF from Regexps import BOL, EOL, EOF
import cython
class Scanner: class Scanner:
""" """
A Scanner is used to read tokens from a stream of characters A Scanner is used to read tokens from a stream of characters
...@@ -42,23 +44,23 @@ class Scanner: ...@@ -42,23 +44,23 @@ class Scanner:
""" """
lexicon = None # Lexicon # lexicon = None # Lexicon
stream = None # file-like object # stream = None # file-like object
name = '' # name = ''
buffer = '' # buffer = ''
buf_start_pos = 0 # position in input of start of buffer # buf_start_pos = 0 # position in input of start of buffer
next_pos = 0 # position in input of next char to read # next_pos = 0 # position in input of next char to read
cur_pos = 0 # position in input of current char # cur_pos = 0 # position in input of current char
cur_line = 1 # line number of current char # cur_line = 1 # line number of current char
cur_line_start = 0 # position in input of start of current line # cur_line_start = 0 # position in input of start of current line
start_pos = 0 # position in input of start of token # start_pos = 0 # position in input of start of token
start_line = 0 # line number of start of token # start_line = 0 # line number of start of token
start_col = 0 # position in line of start of token # start_col = 0 # position in line of start of token
text = None # text of last token read # text = None # text of last token read
initial_state = None # Node # initial_state = None # Node
state_name = '' # Name of initial state # state_name = '' # Name of initial state
queue = None # list of tokens to be returned # queue = None # list of tokens to be returned
trace = 0 # trace = 0
def __init__(self, lexicon, stream, name = '', initial_pos = None): def __init__(self, lexicon, stream, name = '', initial_pos = None):
""" """
...@@ -73,6 +75,17 @@ class Scanner: ...@@ -73,6 +75,17 @@ class Scanner:
|name| is optional, and may be the name of the file being |name| is optional, and may be the name of the file being
scanned or any other identifying string. scanned or any other identifying string.
""" """
self.buffer = ''
self.buf_start_pos = 0
self.next_pos = 0
self.cur_pos = 0
self.cur_line = 1
self.start_pos = 0
self.start_line = 0
self.start_col = 0
self.text = None
self.state_name = None
self.lexicon = lexicon self.lexicon = lexicon
self.stream = stream self.stream = stream
self.name = name self.name = name
...@@ -374,6 +387,3 @@ class Scanner: ...@@ -374,6 +387,3 @@ class Scanner:
Override this method if you want something to be done at Override this method if you want something to be done at
end of file. end of file.
""" """
# For backward compatibility:
setattr(Scanner, "yield", Scanner.produce)
...@@ -31,7 +31,7 @@ except ValueError: ...@@ -31,7 +31,7 @@ except ValueError:
try: try:
from Cython.Compiler.Main import compile from Cython.Compiler.Main import compile
source_root = os.path.dirname(__file__) source_root = os.path.dirname(__file__)
compiled_modules = ["Cython.Plex.Scanners"] compiled_modules = ["Cython.Plex.Scanners", "Cython.Compiler.Scanning"]
extensions = [] extensions = []
for module in compiled_modules: for module in compiled_modules:
source_file = os.path.join(source_root, *module.split('.')) source_file = os.path.join(source_root, *module.split('.'))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment