ParseTreeTransforms.py 36.8 KB
Newer Older
1
from Cython.Compiler.Visitor import VisitorTransform, CythonTransform, TreeVisitor
2
from Cython.Compiler.ModuleNode import ModuleNode
3
from Cython.Compiler.Nodes import *
4
from Cython.Compiler.ExprNodes import *
5
from Cython.Compiler.UtilNodes import *
6
from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
7
from Cython.Compiler.StringEncoding import EncodedString
8
from Cython.Compiler.Errors import error, CompileError
Stefan Behnel's avatar
Stefan Behnel committed
9 10 11 12
try:
    set
except NameError:
    from sets import Set as set
13
import copy
14

15 16 17 18 19 20 21 22 23

class NameNodeCollector(TreeVisitor):
    """Collect all NameNodes of a (sub-)tree in the ``name_nodes``
    attribute.
    """
    def __init__(self):
        super(NameNodeCollector, self).__init__()
        self.name_nodes = []

24
    visit_Node = TreeVisitor.visitchildren
25 26 27 28 29

    def visit_NameNode(self, node):
        self.name_nodes.append(node)


30
class SkipDeclarations(object):
31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56
    """
    Variable and function declarations can often have a deep tree structure, 
    and yet most transformations don't need to descend to this depth. 
    
    Declaration nodes are removed after AnalyseDeclarationsTransform, so there 
    is no need to use this for transformations after that point. 
    """
    def visit_CTypeDefNode(self, node):
        return node
    
    def visit_CVarDefNode(self, node):
        return node
    
    def visit_CDeclaratorNode(self, node):
        return node
    
    def visit_CBaseTypeNode(self, node):
        return node
    
    def visit_CEnumDefNode(self, node):
        return node

    def visit_CStructOrUnionDefNode(self, node):
        return node


57
class NormalizeTree(CythonTransform):
58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
    """
    This transform fixes up a few things after parsing
    in order to make the parse tree more suitable for
    transforms.

    a) After parsing, blocks with only one statement will
    be represented by that statement, not by a StatListNode.
    When doing transforms this is annoying and inconsistent,
    as one cannot in general remove a statement in a consistent
    way and so on. This transform wraps any single statements
    in a StatListNode containing a single statement.

    b) The PassStatNode is a noop and serves no purpose beyond
    plugging such one-statement blocks; i.e., once parsed a
`    "pass" can just as well be represented using an empty
    StatListNode. This means less special cases to worry about
    in subsequent transforms (one always checks to see if a
    StatListNode has no children to see if the block is empty).
    """

78 79
    def __init__(self, context):
        super(NormalizeTree, self).__init__(context)
80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114
        self.is_in_statlist = False
        self.is_in_expr = False

    def visit_ExprNode(self, node):
        stacktmp = self.is_in_expr
        self.is_in_expr = True
        self.visitchildren(node)
        self.is_in_expr = stacktmp
        return node

    def visit_StatNode(self, node, is_listcontainer=False):
        stacktmp = self.is_in_statlist
        self.is_in_statlist = is_listcontainer
        self.visitchildren(node)
        self.is_in_statlist = stacktmp
        if not self.is_in_statlist and not self.is_in_expr:
            return StatListNode(pos=node.pos, stats=[node])
        else:
            return node

    def visit_StatListNode(self, node):
        self.is_in_statlist = True
        self.visitchildren(node)
        self.is_in_statlist = False
        return node

    def visit_ParallelAssignmentNode(self, node):
        return self.visit_StatNode(node, True)
    
    def visit_CEnumDefNode(self, node):
        return self.visit_StatNode(node, True)

    def visit_CStructOrUnionDefNode(self, node):
        return self.visit_StatNode(node, True)

115 116 117 118 119 120 121
    # Eliminate PassStatNode
    def visit_PassStatNode(self, node):
        if not self.is_in_statlist:
            return StatListNode(pos=node.pos, stats=[])
        else:
            return []

122 123 124
    def visit_CDeclaratorNode(self, node):
        return node    

125

126 127 128
class PostParseError(CompileError): pass

# error strings checked by unit tests, so define them
129
ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
130
ERR_BUF_LOCALONLY = 'Buffer types only allowed as function local variables'
131 132
ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
133 134 135 136 137 138 139 140
class PostParse(CythonTransform):
    """
    Basic interpretation of the parse tree, as well as validity
    checking that can be done on a very basic level on the parse
    tree (while still not being a problem with the basic syntax,
    as such).

    Specifically:
141
    - Default values to cdef assignments are turned into single
142 143
    assignments following the declaration (everywhere but in class
    bodies, where they raise a compile error)
144 145 146 147 148 149 150 151 152 153 154 155 156
    
    - Interpret some node structures into Python runtime values.
    Some nodes take compile-time arguments (currently:
    CBufferAccessTypeNode[args] and __cythonbufferdefaults__ = {args}),
    which should be interpreted. This happens in a general way
    and other steps should be taken to ensure validity.

    Type arguments cannot be interpreted in this way.

    - For __cythonbufferdefaults__ the arguments are checked for
    validity.

    CBufferAccessTypeNode has its options interpreted:
157 158 159
    Any first positional argument goes into the "dtype" attribute,
    any "ndim" keyword argument goes into the "ndim" attribute and
    so on. Also it is checked that the option combination is valid.
160 161
    - __cythonbufferdefaults__ attributes are parsed and put into the
    type information.
162 163 164 165 166 167

    Note: Currently Parsing.py does a lot of interpretation and
    reorganization that can be refactored into this transform
    if a more pure Abstract Syntax Tree is wanted.
    """

168
    # Track our context.
169 170
    scope_type = None # can be either of 'module', 'function', 'class'

171 172 173 174 175 176
    def __init__(self, context):
        super(PostParse, self).__init__(context)
        self.specialattribute_handlers = {
            '__cythonbufferdefaults__' : self.handle_bufferdefaults
        }

177 178
    def visit_ModuleNode(self, node):
        self.scope_type = 'module'
179
        self.scope_node = node
180 181
        self.visitchildren(node)
        return node
182 183 184 185 186

    def visit_scope(self, node, scope_type):
        prev = self.scope_type, self.scope_node
        self.scope_type = scope_type
        self.scope_node = node
187
        self.visitchildren(node)
188
        self.scope_type, self.scope_node = prev
189
        return node
190 191 192
    
    def visit_ClassDefNode(self, node):
        return self.visit_scope(node, 'class')
193 194

    def visit_FuncDefNode(self, node):
195 196 197 198
        return self.visit_scope(node, 'function')

    def visit_CStructOrUnionDefNode(self, node):
        return self.visit_scope(node, 'struct')
199 200

    # cdef variables
201 202 203
    def handle_bufferdefaults(self, decl):
        if not isinstance(decl.default, DictNode):
            raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
204 205
        self.scope_node.buffer_defaults_node = decl.default
        self.scope_node.buffer_defaults_pos = decl.pos
206

207 208
    def visit_CVarDefNode(self, node):
        # This assumes only plain names and pointers are assignable on
209 210 211
        # declaration. Also, it makes use of the fact that a cdef decl
        # must appear before the first use, so we don't have to deal with
        # "i = 3; cdef int i = i" and can simply move the nodes around.
212 213
        try:
            self.visitchildren(node)
214 215 216 217 218 219 220 221
            stats = [node]
            newdecls = []
            for decl in node.declarators:
                declbase = decl
                while isinstance(declbase, CPtrDeclaratorNode):
                    declbase = declbase.base
                if isinstance(declbase, CNameDeclaratorNode):
                    if declbase.default is not None:
222 223
                        if self.scope_type in ('class', 'struct'):
                            if isinstance(self.scope_node, CClassDefNode):
224 225 226 227 228 229 230
                                handler = self.specialattribute_handlers.get(decl.name)
                                if handler:
                                    if decl is not declbase:
                                        raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
                                    handler(decl)
                                    continue # Remove declaration
                            raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
231
                        first_assignment = self.scope_type != 'module'
232 233
                        stats.append(SingleAssignmentNode(node.pos,
                            lhs=NameNode(node.pos, name=declbase.name),
234
                            rhs=declbase.default, first=first_assignment))
235 236 237 238
                        declbase.default = None
                newdecls.append(decl)
            node.declarators = newdecls
            return stats
239 240 241 242 243
        except PostParseError, e:
            # An error in a cdef clause is ok, simply remove the declaration
            # and try to move on to report more errors
            self.context.nonfatal_error(e)
            return None
244

245
    def visit_CBufferAccessTypeNode(self, node):
246 247
        if not self.scope_type == 'function':
            raise PostParseError(node.pos, ERR_BUF_LOCALONLY)
248 249
        return node

250
class PxdPostParse(CythonTransform, SkipDeclarations):
251 252 253
    """
    Basic interpretation/validity checking that should only be
    done on pxd trees.
254 255 256 257 258 259

    A lot of this checking currently happens in the parser; but
    what is listed below happens here.

    - "def" functions are let through only if they fill the
    getbuffer/releasebuffer slots
260 261 262
    
    - cdef functions are let through only if they are on the
    top level and are declared "inline"
263
    """
264 265
    ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
    ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
266 267 268 269 270 271 272 273 274 275 276 277 278 279 280

    def __call__(self, node):
        self.scope_type = 'pxd'
        return super(PxdPostParse, self).__call__(node)

    def visit_CClassDefNode(self, node):
        old = self.scope_type
        self.scope_type = 'cclass'
        self.visitchildren(node)
        self.scope_type = old
        return node

    def visit_FuncDefNode(self, node):
        # FuncDefNode always come with an implementation (without
        # an imp they are CVarDefNodes..)
281
        err = self.ERR_INLINE_ONLY
282 283 284

        if (isinstance(node, DefNode) and self.scope_type == 'cclass'
            and node.name in ('__getbuffer__', '__releasebuffer__')):
285
            err = None # allow these slots
286 287
            
        if isinstance(node, CFuncDefNode):
288 289 290 291 292 293 294 295 296
            if u'inline' in node.modifiers and self.scope_type == 'pxd':
                node.inline_in_pxd = True
                if node.visibility != 'private':
                    err = self.ERR_NOGO_WITH_INLINE % node.visibility
                elif node.api:
                    err = self.ERR_NOGO_WITH_INLINE % 'api'
                else:
                    err = None # allow inline function
            else:
297 298
                err = self.ERR_INLINE_ONLY

299 300
        if err:
            self.context.nonfatal_error(PostParseError(node.pos, err))
301 302 303
            return None
        else:
            return node
304 305
    
class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
306 307 308 309 310 311 312
    """
    After parsing, options can be stored in a number of places:
    - #cython-comments at the top of the file (stored in ModuleNode)
    - Command-line arguments overriding these
    - @cython.optionname decorators
    - with cython.optionname: statements

313 314 315 316 317 318 319 320 321 322 323 324 325
    This transform is responsible for interpreting these various sources
    and store the option in two ways:
    - Set the directives attribute of the ModuleNode for global directives.
    - Use a CompilerDirectivesNode to override directives for a subtree.

    (The first one is primarily to not have to modify with the tree
    structure, so that ModuleNode stay on top.)

    The directives are stored in dictionaries from name to value in effect.
    Each such dictionary is always filled in for all possible directives,
    using default values where no value is given by the user.

    The available directives are controlled in Options.py.
326 327 328

    Note that we have to run this prior to analysis, and so some minor
    duplication of functionality has to occur: We manually track cimports
329
    and which names the "cython" module may have been imported to.
330
    """
331
    special_methods = set(['declare', 'union', 'struct', 'typedef', 'sizeof', 'cast', 'address', 'pointer', 'compiled', 'NULL'])
332 333

    def __init__(self, context, compilation_option_overrides):
334
        super(InterpretCompilerDirectives, self).__init__(context)
335 336 337
        self.compilation_option_overrides = {}
        for key, value in compilation_option_overrides.iteritems():
            self.compilation_option_overrides[unicode(key)] = value
338
        self.cython_module_names = set()
339
        self.option_names = {}
340

341
    # Set up processing and handle the cython: comments.
342 343
    def visit_ModuleNode(self, node):
        options = copy.copy(Options.option_defaults)
344 345 346 347 348
        for key, value in self.compilation_option_overrides.iteritems():
            if key in node.option_comments and node.option_comments[key] != value:
                warning(node.pos, "Compiler directive differs between environment and file header; this will change "
                        "in Cython 0.12. See http://article.gmane.org/gmane.comp.python.cython.devel/5233", 2)
                break
349 350 351
        options.update(node.option_comments)
        options.update(self.compilation_option_overrides)
        self.options = options
352
        node.directives = options
353
        self.visitchildren(node)
354
        node.cython_module_names = self.cython_module_names
355 356
        return node

357 358 359 360 361 362 363 364
    # Track cimports of the cython module.
    def visit_CImportStatNode(self, node):
        if node.module_name == u"cython":
            if node.as_name:
                modname = node.as_name
            else:
                modname = u"cython"
            self.cython_module_names.add(modname)
365 366 367 368 369 370
        return node
    
    def visit_FromCImportStatNode(self, node):
        if node.module_name == u"cython":
            newimp = []
            for pos, name, as_name, kind in node.imported_names:
371 372 373
                if (name in Options.option_types or 
                        name in self.special_methods or
                        PyrexTypes.parse_basic_type(name)):
Robert Bradshaw's avatar
Robert Bradshaw committed
374 375
                    if as_name is None:
                        as_name = name
376 377 378 379 380 381
                    self.option_names[as_name] = name
                    if kind is not None:
                        self.context.nonfatal_error(PostParseError(pos,
                            "Compiler option imports must be plain imports"))
                else:
                    newimp.append((pos, name, as_name, kind))
Robert Bradshaw's avatar
Robert Bradshaw committed
382 383 384
            if not newimp:
                return None
            node.imported_names = newimp
385
        return node
386
        
Robert Bradshaw's avatar
Robert Bradshaw committed
387 388 389
    def visit_FromImportStatNode(self, node):
        if node.module.module_name.value == u"cython":
            newimp = []
390 391 392 393 394
            for name, name_node in node.items:
                if (name in Options.option_types or 
                        name in self.special_methods or
                        PyrexTypes.parse_basic_type(name)):
                    self.option_names[name_node.name] = name
Robert Bradshaw's avatar
Robert Bradshaw committed
395
                else:
396
                    newimp.append((name, name_node))
Robert Bradshaw's avatar
Robert Bradshaw committed
397 398 399 400 401
            if not newimp:
                return None
            node.items = newimp
        return node

402 403 404
    def visit_SingleAssignmentNode(self, node):
        if (isinstance(node.rhs, ImportNode) and
                node.rhs.module_name.value == u'cython'):
405 406 407 408
            node = CImportStatNode(node.pos, 
                                   module_name = u'cython',
                                   as_name = node.lhs.name)
            self.visit_CImportStatNode(node)
409 410
        else:
            self.visitchildren(node)
411 412 413 414 415
        return node
            
    def visit_NameNode(self, node):
        if node.name in self.cython_module_names:
            node.is_cython_module = True
Robert Bradshaw's avatar
Robert Bradshaw committed
416 417
        else:
            node.cython_attribute = self.option_names.get(node.name)
418
        return node
419

420 421 422 423
    def try_to_parse_option(self, node):
        # If node is the contents of an option (in a with statement or
        # decorator), returns (optionname, value).
        # Otherwise, returns None
424
        optname = None
425
        if isinstance(node, CallNode):
Robert Bradshaw's avatar
Robert Bradshaw committed
426
            self.visit(node.function)
427
            optname = node.function.as_cython_attribute()
428 429

        if optname:
430 431
            optiontype = Options.option_types.get(optname)
            if optiontype:
432
                args, kwds = node.explicit_args_kwds()
433
                if optiontype is bool:
434
                    if kwds is not None or len(args) != 1 or not isinstance(args[0], BoolNode):
435 436 437
                        raise PostParseError(dec.function.pos,
                            'The %s option takes one compile-time boolean argument' % optname)
                    return (optname, args[0].value)
438 439 440 441 442
                elif optiontype is str:
                    if kwds is not None or len(args) != 1 or not isinstance(args[0], StringNode):
                        raise PostParseError(dec.function.pos,
                            'The %s option takes one compile-time string argument' % optname)
                    return (optname, args[0].value)
443 444 445 446 447
                elif optiontype is dict:
                    if len(args) != 0:
                        raise PostParseError(dec.function.pos,
                            'The %s option takes no prepositional arguments' % optname)
                    return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs])
448 449
                else:
                    assert False
450 451

        return None
452

453
    def visit_with_options(self, body, options):
454 455 456 457
        oldoptions = self.options
        newoptions = copy.copy(oldoptions)
        newoptions.update(options)
        self.options = newoptions
458 459 460
        assert isinstance(body, StatListNode), body
        retbody = self.visit_Node(body)
        directive = CompilerDirectivesNode(pos=retbody.pos, body=retbody,
461
                                           directives=newoptions)
462
        self.options = oldoptions
463
        return directive
464 465
 
    # Handle decorators
466
    def visit_FuncDefNode(self, node):
467
        options = []
468 469 470 471 472 473 474
        
        if node.decorators:
            # Split the decorators into two lists -- real decorators and options
            realdecs = []
            for dec in node.decorators:
                option = self.try_to_parse_option(dec.decorator)
                if option is not None:
475
                    options.append(option)
476 477
                else:
                    realdecs.append(dec)
478 479 480 481
            if realdecs and isinstance(node, CFuncDefNode):
                raise PostParseError(realdecs[0].pos, "Cdef functions cannot take arbitrary decorators.")
            else:
                node.decorators = realdecs
482
        
483 484 485 486 487 488
        if options:
            optdict = {}
            options.reverse() # Decorators coming first take precedence
            for option in options:
                name, value = option
                optdict[name] = value
489 490
            body = StatListNode(node.pos, stats=[node])
            return self.visit_with_options(body, optdict)
491 492
        else:
            return self.visit_Node(node)
493 494 495 496 497 498 499 500 501 502
    
    def visit_CVarDefNode(self, node):
        if node.decorators:
            for dec in node.decorators:
                option = self.try_to_parse_option(dec.decorator)
                if option is not None and option[0] == u'locals':
                    node.directive_locals = option[1]
                else:
                    raise PostParseError(dec.pos, "Cdef functions can only take cython.locals() decorator.")
        return node
503
                                   
504 505 506 507 508 509 510
    # Handle with statements
    def visit_WithStatNode(self, node):
        option = self.try_to_parse_option(node.manager)
        if option is not None:
            if node.target is not None:
                raise PostParseError(node.pos, "Compiler option with statements cannot contain 'as'")
            name, value = option
511
            return self.visit_with_options(node.body, {name:value})
512 513 514
        else:
            return self.visit_Node(node)

515
class WithTransform(CythonTransform, SkipDeclarations):
516

517 518 519
    # EXCINFO is manually set to a variable that contains
    # the exc_info() tuple that can be generated by the enclosing except
    # statement.
520 521 522 523 524 525 526
    template_without_target = TreeFragment(u"""
        MGR = EXPR
        EXIT = MGR.__exit__
        MGR.__enter__()
        EXC = True
        try:
            try:
527
                EXCINFO = None
528 529 530
                BODY
            except:
                EXC = False
531
                if not EXIT(*EXCINFO):
532 533 534 535
                    raise
        finally:
            if EXC:
                EXIT(None, None, None)
536
    """, temps=[u'MGR', u'EXC', u"EXIT"],
537
    pipeline=[NormalizeTree(None)])
538 539 540 541 542 543 544 545

    template_with_target = TreeFragment(u"""
        MGR = EXPR
        EXIT = MGR.__exit__
        VALUE = MGR.__enter__()
        EXC = True
        try:
            try:
546
                EXCINFO = None
547 548 549 550
                TARGET = VALUE
                BODY
            except:
                EXC = False
551
                if not EXIT(*EXCINFO):
552 553 554 555
                    raise
        finally:
            if EXC:
                EXIT(None, None, None)
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
556 557
            MGR = EXIT = VALUE = EXC = None
            
558
    """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE"],
559
    pipeline=[NormalizeTree(None)])
560 561

    def visit_WithStatNode(self, node):
562 563 564 565
        # TODO: Cleanup badly needed
        TemplateTransform.temp_name_counter += 1
        handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter
        
566
        self.visitchildren(node, ['body'])
567
        excinfo_temp = NameNode(node.pos, name=handle)#TempHandle(Builtin.tuple_type)
568 569 570 571
        if node.target is not None:
            result = self.template_with_target.substitute({
                u'EXPR' : node.manager,
                u'BODY' : node.body,
572
                u'TARGET' : node.target,
573
                u'EXCINFO' : excinfo_temp
574
                }, pos=node.pos)
575 576 577 578
        else:
            result = self.template_without_target.substitute({
                u'EXPR' : node.manager,
                u'BODY' : node.body,
579
                u'EXCINFO' : excinfo_temp
580
                }, pos=node.pos)
581 582

        # Set except excinfo target to EXCINFO
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
583
        try_except = result.stats[-1].body.stats[-1]
584 585
        try_except.except_clauses[0].excinfo_target = NameNode(node.pos, name=handle)
#            excinfo_temp.ref(node.pos))
586

587 588
#        result.stats[-1].body.stats[-1] = TempsBlockNode(
#            node.pos, temps=[excinfo_temp], body=try_except)
589 590

        return result
591 592 593 594 595
        
    def visit_ExprNode(self, node):
        # With statements are never inside expressions.
        return node
        
596

597
class DecoratorTransform(CythonTransform, SkipDeclarations):
598

599
    def visit_DefNode(self, func_node):
600
        self.visitchildren(func_node)
601 602
        if not func_node.decorators:
            return func_node
603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630
        return self._handle_decorators(
            func_node, func_node.name)

    def _visit_CClassDefNode(self, class_node):
        # This doesn't currently work, so it's disabled (also in the
        # parser).
        #
        # Problem: assignments to cdef class names do not work.  They
        # would require an additional check anyway, as the extension
        # type must not change its C type, so decorators cannot
        # replace an extension type, just alter it and return it.

        self.visitchildren(class_node)
        if not class_node.decorators:
            return class_node
        return self._handle_decorators(
            class_node, class_node.class_name)

    def visit_ClassDefNode(self, class_node):
        self.visitchildren(class_node)
        if not class_node.decorators:
            return class_node
        return self._handle_decorators(
            class_node, class_node.name)

    def _handle_decorators(self, node, name):
        decorator_result = NameNode(node.pos, name = name)
        for decorator in node.decorators[::-1]:
631 632 633 634 635
            decorator_result = SimpleCallNode(
                decorator.pos,
                function = decorator.decorator,
                args = [decorator_result])

636
        name_node = NameNode(node.pos, name = name)
637
        reassignment = SingleAssignmentNode(
638 639
            node.pos,
            lhs = name_node,
640
            rhs = decorator_result)
641
        return [node, reassignment]
642

643

644
class AnalyseDeclarationsTransform(CythonTransform):
645

646 647 648 649 650 651 652 653
    basic_property = TreeFragment(u"""
property NAME:
    def __get__(self):
        return ATTR
    def __set__(self, value):
        ATTR = value
    """, level='c_class')

654 655
    def __call__(self, root):
        self.env_stack = [root.scope]
656
        # needed to determine if a cdef var is declared after it's used.
657
        self.seen_vars_stack = []
658
        return super(AnalyseDeclarationsTransform, self).__call__(root)        
659
    
660
    def visit_NameNode(self, node):
661
        self.seen_vars_stack[-1].add(node.name)
662 663
        return node

664
    def visit_ModuleNode(self, node):
665
        self.seen_vars_stack.append(set())
666 667
        node.analyse_declarations(self.env_stack[-1])
        self.visitchildren(node)
668
        self.seen_vars_stack.pop()
669
        return node
670
        
671
    def visit_FuncDefNode(self, node):
672
        self.seen_vars_stack.append(set())
673 674 675
        lenv = node.create_local_scope(self.env_stack[-1])
        node.body.analyse_control_flow(lenv) # this will be totally refactored
        node.declare_arguments(lenv)
676 677 678 679 680 681 682
        for var, type_node in node.directive_locals.items():
            if not lenv.lookup_here(var):   # don't redeclare args
                type = type_node.analyse_as_type(lenv)
                if type:
                    lenv.declare_var(var, type, type_node.pos)
                else:
                    error(type_node.pos, "Not a type")
683 684 685 686
        node.body.analyse_declarations(lenv)
        self.env_stack.append(lenv)
        self.visitchildren(node)
        self.env_stack.pop()
687
        self.seen_vars_stack.pop()
688
        return node
689
    
690 691 692 693
    # Some nodes are no longer needed after declaration
    # analysis and can be dropped. The analysis was performed
    # on these nodes in a seperate recursive process from the
    # enclosing function or module, so we can simply drop them.
694
    def visit_CDeclaratorNode(self, node):
695 696
        # necessary to ensure that all CNameDeclaratorNodes are visited.
        self.visitchildren(node)
697 698 699 700 701 702 703 704 705 706 707 708 709 710
        return node
    
    def visit_CTypeDefNode(self, node):
        return node

    def visit_CBaseTypeNode(self, node):
        return None
    
    def visit_CEnumDefNode(self, node):
        return None

    def visit_CStructOrUnionDefNode(self, node):
        return None

711
    def visit_CNameDeclaratorNode(self, node):
712 713 714
        if node.name in self.seen_vars_stack[-1]:
            entry = self.env_stack[-1].lookup(node.name)
            if entry is None or entry.visibility != 'extern':
715
                warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2)
716 717 718
        self.visitchildren(node)
        return node

719
    def visit_CVarDefNode(self, node):
720 721 722 723

        # to ensure all CNameDeclaratorNodes are visited.
        self.visitchildren(node)

724 725 726 727 728 729
        if node.need_properties:
            # cdef public attributes may need type testing on 
            # assignment, so we create a property accesss
            # mechanism for them. 
            stats = []
            for entry in node.need_properties:
730
                property = self.create_Property(entry)
731 732 733 734 735 736
                property.analyse_declarations(node.dest_scope)
                self.visit(property)
                stats.append(property)
            return StatListNode(pos=node.pos, stats=stats)
        else:
            return None
737
            
738 739
    def create_Property(self, entry):
        template = self.basic_property
740
        property = template.substitute({
741 742 743 744 745 746
                u"ATTR": AttributeNode(pos=entry.pos,
                                       obj=NameNode(pos=entry.pos, name="self"), 
                                       attribute=entry.name),
            }, pos=entry.pos).stats[0]
        property.name = entry.name
        return property
747

748
class AnalyseExpressionsTransform(CythonTransform):
Robert Bradshaw's avatar
Robert Bradshaw committed
749

750 751 752 753 754 755 756 757 758
    def visit_ModuleNode(self, node):
        node.body.analyse_expressions(node.scope)
        self.visitchildren(node)
        return node
        
    def visit_FuncDefNode(self, node):
        node.body.analyse_expressions(node.local_scope)
        self.visitchildren(node)
        return node
759 760 761 762 763 764 765 766 767
        
class AlignFunctionDefinitions(CythonTransform):
    """
    This class takes the signatures from a .pxd file and applies them to 
    the def methods in a .py file. 
    """
    
    def visit_ModuleNode(self, node):
        self.scope = node.scope
768
        self.directives = node.directives
769 770 771 772 773 774 775 776 777 778 779 780
        self.visitchildren(node)
        return node
    
    def visit_PyClassDefNode(self, node):
        pxd_def = self.scope.lookup(node.name)
        if pxd_def:
            if pxd_def.is_cclass:
                return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
            else:
                error(node.pos, "'%s' redeclared" % node.name)
                error(pxd_def.pos, "previous declaration here")
                return None
781 782
        else:
            return node
783 784 785 786 787 788 789 790 791 792 793 794 795 796 797
        
    def visit_CClassDefNode(self, node, pxd_def=None):
        if pxd_def is None:
            pxd_def = self.scope.lookup(node.class_name)
        if pxd_def:
            outer_scope = self.scope
            self.scope = pxd_def.type.scope
        self.visitchildren(node)
        if pxd_def:
            self.scope = outer_scope
        return node
        
    def visit_DefNode(self, node):
        pxd_def = self.scope.lookup(node.name)
        if pxd_def:
798 799 800
            if self.scope.is_c_class_scope and len(pxd_def.type.args) > 0:
                # The self parameter type needs adjusting.
                pxd_def.type.args[0].type = self.scope.parent_type
801 802 803 804 805 806
            if pxd_def.is_cfunction:
                node = node.as_cfunction(pxd_def)
            else:
                error(node.pos, "'%s' redeclared" % node.name)
                error(pxd_def.pos, "previous declaration here")
                return None
807 808
        elif self.scope.is_module_scope and self.directives['auto_cpdef']:
            node = node.as_cfunction(scope=self.scope)
809 810 811 812
        # Enable this when internal def functions are allowed. 
        # self.visitchildren(node)
        return node
        
813

814
class MarkClosureVisitor(CythonTransform):
Robert Bradshaw's avatar
Robert Bradshaw committed
815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832
    
    needs_closure = False
    
    def visit_FuncDefNode(self, node):
        self.needs_closure = False
        self.visitchildren(node)
        node.needs_closure = self.needs_closure
        self.needs_closure = True
        return node
        
    def visit_ClassDefNode(self, node):
        self.visitchildren(node)
        self.needs_closure = True
        return node
        
    def visit_YieldNode(self, node):
        self.needs_closure = True
        
833
class CreateClosureClasses(CythonTransform):
834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858
    # Output closure classes in module scope for all functions
    # that need it. 
    
    def visit_ModuleNode(self, node):
        self.module_scope = node.scope
        self.visitchildren(node)
        return node

    def create_class_from_scope(self, node, target_module_scope):
        as_name = temp_name_handle("closure")
        func_scope = node.local_scope

        entry = target_module_scope.declare_c_class(name = as_name,
            pos = node.pos, defining = True, implementing = True)
        class_scope = entry.type.scope
        for entry in func_scope.entries.values():
            class_scope.declare_var(pos=node.pos,
                                    name=entry.name,
                                    cname=entry.cname,
                                    type=entry.type,
                                    is_cdef=True)
            
    def visit_FuncDefNode(self, node):
        self.create_class_from_scope(node, self.module_scope)
        return node
859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895


class GilCheck(VisitorTransform):
    """
    Call `node.gil_check(env)` on each node to make sure we hold the
    GIL when we need it.  Raise an error when on Python operations
    inside a `nogil` environment.
    """
    def __call__(self, root):
        self.env_stack = [root.scope]
        return super(GilCheck, self).__call__(root)

    def visit_FuncDefNode(self, node):
        self.env_stack.append(node.local_scope)
        if node.gil_check is not None:
            node.gil_check(self.env_stack[-1])
        self.visitchildren(node)
        self.env_stack.pop()
        return node

    def visit_GILStatNode(self, node):
        # FIXME: should we do some kind of GIL checking here, too?
        # if node.gil_check is not None:
        #     node.gil_check(self.env_stack[-1])
        env = self.env_stack[-1]
        was_nogil = env.nogil
        env.nogil = node.state == 'nogil'
        self.visitchildren(node)
        env.nogil = was_nogil
        return node

    def visit_Node(self, node):
        if self.env_stack and node.gil_check is not None:
            node.gil_check(self.env_stack[-1])
        self.visitchildren(node)
        return node

896

Robert Bradshaw's avatar
Robert Bradshaw committed
897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913
class EnvTransform(CythonTransform):
    """
    This transformation keeps a stack of the environments. 
    """
    def __call__(self, root):
        self.env_stack = [root.scope]
        return super(EnvTransform, self).__call__(root)        
    
    def visit_FuncDefNode(self, node):
        self.env_stack.append(node.local_scope)
        self.visitchildren(node)
        self.env_stack.pop()
        return node


class TransformBuiltinMethods(EnvTransform):

914 915 916 917 918 919
    def visit_SingleAssignmentNode(self, node):
        if node.declaration_only:
            return None
        else:
            self.visitchildren(node)
            return node
920 921
    
    def visit_AttributeNode(self, node):
922 923 924 925 926 927 928
        return self.visit_cython_attribute(node)

    def visit_NameNode(self, node):
        return self.visit_cython_attribute(node)
        
    def visit_cython_attribute(self, node):
        attribute = node.as_cython_attribute()
929 930 931
        if attribute:
            if attribute == u'compiled':
                node = BoolNode(node.pos, value=True)
932 933 934
            elif attribute == u'NULL':
                node = NullNode(node.pos)
            elif not PyrexTypes.parse_basic_type(attribute):
Robert Bradshaw's avatar
Robert Bradshaw committed
935
                error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
936 937 938 939
        return node

    def visit_SimpleCallNode(self, node):

Robert Bradshaw's avatar
Robert Bradshaw committed
940
        # locals builtin
Robert Bradshaw's avatar
Robert Bradshaw committed
941 942 943 944 945 946 947 948
        if isinstance(node.function, ExprNodes.NameNode):
            if node.function.name == 'locals':
                pos = node.pos
                lenv = self.env_stack[-1]
                items = [ExprNodes.DictItemNode(pos, 
                                                key=ExprNodes.IdentifierStringNode(pos, value=var),
                                                value=ExprNodes.NameNode(pos, name=var)) for var in lenv.entries]
                return ExprNodes.DictNode(pos, key_value_pairs=items)
949 950

        # cython.foo
951
        function = node.function.as_cython_attribute()
952 953 954
        if function:
            if function == u'cast':
                if len(node.args) != 2:
955
                    error(node.function.pos, u"cast takes exactly two arguments")
956 957 958 959 960 961 962 963 964 965 966 967 968 969 970
                else:
                    type = node.args[0].analyse_as_type(self.env_stack[-1])
                    if type:
                        node = TypecastNode(node.function.pos, type=type, operand=node.args[1])
                    else:
                        error(node.args[0].pos, "Not a type")
            elif function == u'sizeof':
                if len(node.args) != 1:
                    error(node.function.pos, u"sizeof takes exactly one argument" % function)
                else:
                    type = node.args[0].analyse_as_type(self.env_stack[-1])
                    if type:
                        node = SizeofTypeNode(node.function.pos, arg_type=type)
                    else:
                        node = SizeofVarNode(node.function.pos, operand=node.args[0])
971 972 973 974 975
            elif function == 'address':
                if len(node.args) != 1:
                    error(node.function.pos, u"sizeof takes exactly one argument" % function)
                else:
                    node = AmpersandNode(node.function.pos, operand=node.args[0])
976 977 978 979 980 981 982 983 984 985 986 987
            elif function == 'cmod':
                if len(node.args) != 2:
                    error(node.function.pos, u"cmod takes exactly one argument" % function)
                else:
                    node = binop_node(node.function.pos, '%', node.args[0], node.args[1])
                    node.cdivision = True
            elif function == 'cdiv':
                if len(node.args) != 2:
                    error(node.function.pos, u"cmod takes exactly one argument" % function)
                else:
                    node = binop_node(node.function.pos, '/', node.args[0], node.args[1])
                    node.cdivision = True
988 989 990 991
            else:
                error(node.function.pos, u"'%s' not a valid cython language construct" % function)
        
        self.visitchildren(node)
Robert Bradshaw's avatar
Robert Bradshaw committed
992
        return node