Parsing.py 109 KB
Newer Older
1
# cython: auto_cpdef=True, infer_types=True, language_level=3, py2_import=True
William Stein's avatar
William Stein committed
2
#
3
#   Parser
William Stein's avatar
William Stein committed
4 5
#

6 7
from __future__ import absolute_import

8 9
# This should be done automatically
import cython
10
cython.declare(Nodes=object, ExprNodes=object, EncodedString=object,
11 12
               BytesLiteral=object, StringEncoding=object,
               FileSourceDescriptor=object, lookup_unicodechar=object,
13
               Future=object, Options=object, error=object, warning=object,
14
               Builtin=object, ModuleNode=object, Utils=object,
15 16
               re=object, _unicode=object, _bytes=object,
               partial=object, reduce=object)
17

18
import re
19
from unicodedata import lookup as lookup_unicodechar
20
from functools import partial, reduce
Lisandro Dalcin's avatar
Lisandro Dalcin committed
21

22 23 24 25 26 27 28 29 30 31 32 33
from .Scanning import PyrexScanner, FileSourceDescriptor
from . import Nodes
from . import ExprNodes
from . import Builtin
from . import StringEncoding
from .StringEncoding import EncodedString, BytesLiteral, _unicode, _bytes
from .ModuleNode import ModuleNode
from .Errors import error, warning
from .. import Utils
from . import Future
from . import Options

William Stein's avatar
William Stein committed
34

35 36 37 38 39 40 41 42 43
class Ctx(object):
    #  Parsing context
    level = 'other'
    visibility = 'private'
    cdef_flag = 0
    typedef_flag = 0
    api = 0
    overridable = 0
    nogil = 0
44
    namespace = None
Danilo Freitas's avatar
Danilo Freitas committed
45
    templates = None
46
    allow_struct_enum_decorator = False
47 48 49 50 51 52 53 54 55 56 57

    def __init__(self, **kwds):
        self.__dict__.update(kwds)

    def __call__(self, **kwds):
        ctx = Ctx()
        d = ctx.__dict__
        d.update(self.__dict__)
        d.update(kwds)
        return ctx

58
def p_ident(s, message="Expected an identifier"):
William Stein's avatar
William Stein committed
59 60 61 62 63 64 65 66 67 68 69 70
    if s.sy == 'IDENT':
        name = s.systring
        s.next()
        return name
    else:
        s.error(message)

def p_ident_list(s):
    names = []
    while s.sy == 'IDENT':
        names.append(s.systring)
        s.next()
Stefan Behnel's avatar
Stefan Behnel committed
71
        if s.sy != ',':
William Stein's avatar
William Stein committed
72 73 74 75 76 77 78 79 80 81
            break
        s.next()
    return names

#------------------------------------------
#
#   Expressions
#
#------------------------------------------

82 83 84 85 86 87
def p_binop_operator(s):
    pos = s.position()
    op = s.sy
    s.next()
    return op, pos

William Stein's avatar
William Stein committed
88 89 90
def p_binop_expr(s, ops, p_sub_expr):
    n1 = p_sub_expr(s)
    while s.sy in ops:
91
        op, pos = p_binop_operator(s)
William Stein's avatar
William Stein committed
92 93
        n2 = p_sub_expr(s)
        n1 = ExprNodes.binop_node(pos, op, n1, n2)
94 95 96 97 98
        if op == '/':
            if Future.division in s.context.future_directives:
                n1.truedivision = True
            else:
                n1.truedivision = None # unknown
William Stein's avatar
William Stein committed
99 100
    return n1

Stefan Behnel's avatar
Stefan Behnel committed
101 102 103 104 105 106 107 108 109 110
#lambdef: 'lambda' [varargslist] ':' test

def p_lambdef(s, allow_conditional=True):
    # s.sy == 'lambda'
    pos = s.position()
    s.next()
    if s.sy == ':':
        args = []
        star_arg = starstar_arg = None
    else:
111 112
        args, star_arg, starstar_arg = p_varargslist(
            s, terminator=':', annotated=False)
Stefan Behnel's avatar
Stefan Behnel committed
113 114
    s.expect(':')
    if allow_conditional:
115
        expr = p_test(s)
Stefan Behnel's avatar
Stefan Behnel committed
116 117 118 119 120 121 122 123 124 125 126 127
    else:
        expr = p_test_nocond(s)
    return ExprNodes.LambdaNode(
        pos, args = args,
        star_arg = star_arg, starstar_arg = starstar_arg,
        result_expr = expr)

#lambdef_nocond: 'lambda' [varargslist] ':' test_nocond

def p_lambdef_nocond(s):
    return p_lambdef(s, allow_conditional=False)

128
#test: or_test ['if' or_test 'else' test] | lambdef
William Stein's avatar
William Stein committed
129

Robert Bradshaw's avatar
Robert Bradshaw committed
130
def p_test(s):
131 132
    if s.sy == 'lambda':
        return p_lambdef(s)
Robert Bradshaw's avatar
Robert Bradshaw committed
133 134 135 136 137
    pos = s.position()
    expr = p_or_test(s)
    if s.sy == 'if':
        s.next()
        test = p_or_test(s)
Stefan Behnel's avatar
Stefan Behnel committed
138 139 140
        s.expect('else')
        other = p_test(s)
        return ExprNodes.CondExprNode(pos, test=test, true_val=expr, false_val=other)
Robert Bradshaw's avatar
Robert Bradshaw committed
141 142 143
    else:
        return expr

Stefan Behnel's avatar
Stefan Behnel committed
144 145 146 147 148 149 150
#test_nocond: or_test | lambdef_nocond

def p_test_nocond(s):
    if s.sy == 'lambda':
        return p_lambdef_nocond(s)
    else:
        return p_or_test(s)
Robert Bradshaw's avatar
Robert Bradshaw committed
151 152 153 154

#or_test: and_test ('or' and_test)*

def p_or_test(s):
William Stein's avatar
William Stein committed
155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186
    return p_rassoc_binop_expr(s, ('or',), p_and_test)

def p_rassoc_binop_expr(s, ops, p_subexpr):
    n1 = p_subexpr(s)
    if s.sy in ops:
        pos = s.position()
        op = s.sy
        s.next()
        n2 = p_rassoc_binop_expr(s, ops, p_subexpr)
        n1 = ExprNodes.binop_node(pos, op, n1, n2)
    return n1

#and_test: not_test ('and' not_test)*

def p_and_test(s):
    #return p_binop_expr(s, ('and',), p_not_test)
    return p_rassoc_binop_expr(s, ('and',), p_not_test)

#not_test: 'not' not_test | comparison

def p_not_test(s):
    if s.sy == 'not':
        pos = s.position()
        s.next()
        return ExprNodes.NotNode(pos, operand = p_not_test(s))
    else:
        return p_comparison(s)

#comparison: expr (comp_op expr)*
#comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'

def p_comparison(s):
187
    n1 = p_starred_expr(s)
William Stein's avatar
William Stein committed
188 189 190
    if s.sy in comparison_ops:
        pos = s.position()
        op = p_cmp_op(s)
191
        n2 = p_starred_expr(s)
192
        n1 = ExprNodes.PrimaryCmpNode(pos,
William Stein's avatar
William Stein committed
193 194 195 196 197
            operator = op, operand1 = n1, operand2 = n2)
        if s.sy in comparison_ops:
            n1.cascade = p_cascaded_cmp(s)
    return n1

198 199 200 201 202 203
def p_test_or_starred_expr(s):
    if s.sy == '*':
        return p_starred_expr(s)
    else:
        return p_test(s)

204
def p_starred_expr(s):
205
    pos = s.position()
206 207 208 209 210 211
    if s.sy == '*':
        starred = True
        s.next()
    else:
        starred = False
    expr = p_bit_expr(s)
212
    if starred:
213
        expr = ExprNodes.StarredUnpackingNode(pos, expr)
214 215
    return expr

William Stein's avatar
William Stein committed
216 217 218
def p_cascaded_cmp(s):
    pos = s.position()
    op = p_cmp_op(s)
219
    n2 = p_starred_expr(s)
220
    result = ExprNodes.CascadedCmpNode(pos,
William Stein's avatar
William Stein committed
221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243
        operator = op, operand2 = n2)
    if s.sy in comparison_ops:
        result.cascade = p_cascaded_cmp(s)
    return result

def p_cmp_op(s):
    if s.sy == 'not':
        s.next()
        s.expect('in')
        op = 'not_in'
    elif s.sy == 'is':
        s.next()
        if s.sy == 'not':
            s.next()
            op = 'is_not'
        else:
            op = 'is'
    else:
        op = s.sy
        s.next()
    if op == '<>':
        op = '!='
    return op
244

245
comparison_ops = cython.declare(set, set([
246
    '<', '>', '==', '>=', '<=', '<>', '!=',
William Stein's avatar
William Stein committed
247
    'in', 'is', 'not'
248
]))
William Stein's avatar
William Stein committed
249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274

#expr: xor_expr ('|' xor_expr)*

def p_bit_expr(s):
    return p_binop_expr(s, ('|',), p_xor_expr)

#xor_expr: and_expr ('^' and_expr)*

def p_xor_expr(s):
    return p_binop_expr(s, ('^',), p_and_expr)

#and_expr: shift_expr ('&' shift_expr)*

def p_and_expr(s):
    return p_binop_expr(s, ('&',), p_shift_expr)

#shift_expr: arith_expr (('<<'|'>>') arith_expr)*

def p_shift_expr(s):
    return p_binop_expr(s, ('<<', '>>'), p_arith_expr)

#arith_expr: term (('+'|'-') term)*

def p_arith_expr(s):
    return p_binop_expr(s, ('+', '-'), p_term)

275
#term: factor (('*'|'@'|'/'|'%'|'//') factor)*
William Stein's avatar
William Stein committed
276 277

def p_term(s):
278
    return p_binop_expr(s, ('*', '@', '/', '%', '//'), p_factor)
William Stein's avatar
William Stein committed
279 280 281 282

#factor: ('+'|'-'|'~'|'&'|typecast|sizeof) factor | power

def p_factor(s):
283 284 285 286
    # little indirection for C-ification purposes
    return _p_factor(s)

def _p_factor(s):
William Stein's avatar
William Stein committed
287 288 289 290 291 292
    sy = s.sy
    if sy in ('+', '-', '~'):
        op = s.sy
        pos = s.position()
        s.next()
        return ExprNodes.unop_node(pos, op, p_factor(s))
293 294 295 296 297 298 299 300 301 302 303
    elif not s.in_python_file:
        if sy == '&':
            pos = s.position()
            s.next()
            arg = p_factor(s)
            return ExprNodes.AmpersandNode(pos, operand = arg)
        elif sy == "<":
            return p_typecast(s)
        elif sy == 'IDENT' and s.systring == "sizeof":
            return p_sizeof(s)
    return p_power(s)
William Stein's avatar
William Stein committed
304 305 306 307 308 309

def p_typecast(s):
    # s.sy == "<"
    pos = s.position()
    s.next()
    base_type = p_c_base_type(s)
310
    is_memslice = isinstance(base_type, Nodes.MemoryViewSliceTypeNode)
Robert Bradshaw's avatar
Robert Bradshaw committed
311 312 313 314
    is_template = isinstance(base_type, Nodes.TemplatedTypeNode)
    is_const = isinstance(base_type, Nodes.CConstTypeNode)
    if (not is_memslice and not is_template and not is_const
        and base_type.name is None):
315
        s.error("Unknown type")
William Stein's avatar
William Stein committed
316
    declarator = p_c_declarator(s, empty = 1)
317 318 319 320 321
    if s.sy == '?':
        s.next()
        typecheck = 1
    else:
        typecheck = 0
William Stein's avatar
William Stein committed
322 323
    s.expect(">")
    operand = p_factor(s)
324 325 326 327
    if is_memslice:
        return ExprNodes.CythonArrayNode(pos, base_type_node=base_type,
                                         operand=operand)

328 329
    return ExprNodes.TypecastNode(pos,
        base_type = base_type,
William Stein's avatar
William Stein committed
330
        declarator = declarator,
331 332
        operand = operand,
        typecheck = typecheck)
William Stein's avatar
William Stein committed
333 334 335 336 337 338

def p_sizeof(s):
    # s.sy == ident "sizeof"
    pos = s.position()
    s.next()
    s.expect('(')
339
    # Here we decide if we are looking at an expression or type
340 341
    # If it is actually a type, but parsable as an expression,
    # we treat it as an expression here.
342
    if looking_at_expr(s):
343
        operand = p_test(s)
344 345
        node = ExprNodes.SizeofVarNode(pos, operand = operand)
    else:
William Stein's avatar
William Stein committed
346 347
        base_type = p_c_base_type(s)
        declarator = p_c_declarator(s, empty = 1)
348
        node = ExprNodes.SizeofTypeNode(pos,
William Stein's avatar
William Stein committed
349 350 351 352
            base_type = base_type, declarator = declarator)
    s.expect(')')
    return node

353 354 355 356
def p_yield_expression(s):
    # s.sy == "yield"
    pos = s.position()
    s.next()
357 358 359 360
    is_yield_from = False
    if s.sy == 'from':
        is_yield_from = True
        s.next()
361
    if s.sy != ')' and s.sy not in statement_terminators:
362
        arg = p_testlist(s)
363
    else:
364
        if is_yield_from:
365 366
            s.error("'yield from' requires a source argument",
                    pos=pos, fatal=False)
367
        arg = None
368 369 370 371
    if is_yield_from:
        return ExprNodes.YieldFromExprNode(pos, arg=arg)
    else:
        return ExprNodes.YieldExprNode(pos, arg=arg)
372 373 374 375 376

def p_yield_statement(s):
    # s.sy == "yield"
    yield_expr = p_yield_expression(s)
    return Nodes.ExprStatNode(yield_expr.pos, expr=yield_expr)
377

William Stein's avatar
William Stein committed
378 379 380
#power: atom trailer* ('**' factor)*

def p_power(s):
381
    if s.systring == 'new' and s.peek()[0] == 'IDENT':
Danilo Freitas's avatar
Danilo Freitas committed
382
        return p_new_expr(s)
William Stein's avatar
William Stein committed
383 384 385 386 387 388 389 390 391 392
    n1 = p_atom(s)
    while s.sy in ('(', '[', '.'):
        n1 = p_trailer(s, n1)
    if s.sy == '**':
        pos = s.position()
        s.next()
        n2 = p_factor(s)
        n1 = ExprNodes.binop_node(pos, '**', n1, n2)
    return n1

Danilo Freitas's avatar
Danilo Freitas committed
393
def p_new_expr(s):
Danilo Freitas's avatar
Danilo Freitas committed
394
    # s.systring == 'new'.
Danilo Freitas's avatar
Danilo Freitas committed
395 396
    pos = s.position()
    s.next()
Robert Bradshaw's avatar
Robert Bradshaw committed
397 398
    cppclass = p_c_base_type(s)
    return p_call(s, ExprNodes.NewExprNode(pos, cppclass = cppclass))
Danilo Freitas's avatar
Danilo Freitas committed
399

William Stein's avatar
William Stein committed
400 401 402 403 404 405 406 407 408 409
#trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME

def p_trailer(s, node1):
    pos = s.position()
    if s.sy == '(':
        return p_call(s, node1)
    elif s.sy == '[':
        return p_index(s, node1)
    else: # s.sy == '.'
        s.next()
410
        name = p_ident(s)
411
        return ExprNodes.AttributeNode(pos,
412
            obj=node1, attribute=name)
William Stein's avatar
William Stein committed
413

414

William Stein's avatar
William Stein committed
415 416 417
# arglist:  argument (',' argument)* [',']
# argument: [test '='] test       # Really [keyword '='] test

418 419 420 421 422 423 424
# since PEP 448:
# argument: ( test [comp_for] |
#             test '=' test |
#             '**' expr |
#             star_expr )

def p_call_parse_args(s, allow_genexp=True):
William Stein's avatar
William Stein committed
425 426 427 428 429
    # s.sy == '('
    pos = s.position()
    s.next()
    positional_args = []
    keyword_args = []
430 431 432
    starstar_seen = False
    last_was_tuple_unpack = False
    while s.sy != ')':
433
        if s.sy == '*':
434 435 436 437 438 439
            if starstar_seen:
                s.error("Non-keyword arg following keyword arg", pos=s.position())
            s.next()
            positional_args.append(p_test(s))
            last_was_tuple_unpack = True
        elif s.sy == '**':
William Stein's avatar
William Stein committed
440
            s.next()
441 442
            keyword_args.append(p_test(s))
            starstar_seen = True
William Stein's avatar
William Stein committed
443
        else:
444
            arg = p_test(s)
445 446 447 448
            if s.sy == '=':
                s.next()
                if not arg.is_name:
                    s.error("Expected an identifier before '='",
449
                            pos=arg.pos)
450
                encoded_name = s.context.intern_ustring(arg.name)
451 452
                keyword = ExprNodes.IdentifierStringNode(
                    arg.pos, value=encoded_name)
453
                arg = p_test(s)
454 455 456
                keyword_args.append((keyword, arg))
            else:
                if keyword_args:
457 458 459 460 461 462
                    s.error("Non-keyword arg following keyword arg", pos=arg.pos)
                if positional_args and not last_was_tuple_unpack:
                    positional_args[-1].append(arg)
                else:
                    positional_args.append([arg])
                last_was_tuple_unpack = False
Stefan Behnel's avatar
Stefan Behnel committed
463
        if s.sy != ',':
William Stein's avatar
William Stein committed
464 465
            break
        s.next()
466

467
    if s.sy == 'for':
468 469
        if not keyword_args and not last_was_tuple_unpack:
            if len(positional_args) == 1 and len(positional_args[0]) == 1:
470
                positional_args = [[p_genexp(s, positional_args[0][0])]]
William Stein's avatar
William Stein committed
471
    s.expect(')')
472
    return positional_args or [[]], keyword_args
473

474 475

def p_call_build_packed_args(pos, positional_args, keyword_args):
476
    keyword_dict = None
477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511

    subtuples = [
        ExprNodes.TupleNode(pos, args=arg) if isinstance(arg, list) else ExprNodes.AsTupleNode(pos, arg=arg)
        for arg in positional_args
    ]
    # TODO: implement a faster way to join tuples than creating each one and adding them
    arg_tuple = reduce(partial(ExprNodes.binop_node, pos, '+'), subtuples)

    if keyword_args:
        kwargs = []
        dict_items = []
        for item in keyword_args:
            if isinstance(item, tuple):
                key, value = item
                dict_items.append(ExprNodes.DictItemNode(pos=key.pos, key=key, value=value))
            elif item.is_dict_literal:
                # unpack "**{a:b}" directly
                dict_items.extend(item.key_value_pairs)
            else:
                if dict_items:
                    kwargs.append(ExprNodes.DictNode(
                        dict_items[0].pos, key_value_pairs=dict_items, reject_duplicates=True))
                    dict_items = []
                kwargs.append(item)

        if dict_items:
            kwargs.append(ExprNodes.DictNode(
                dict_items[0].pos, key_value_pairs=dict_items, reject_duplicates=True))

        if kwargs:
            if len(kwargs) == 1 and kwargs[0].is_dict_literal:
                # only simple keyword arguments found -> one dict
                keyword_dict = kwargs[0]
            else:
                # at least one **kwargs
512
                keyword_dict = ExprNodes.MergedDictNode(pos, keyword_args=kwargs)
513

514 515
    return arg_tuple, keyword_dict

516

517 518 519
def p_call(s, function):
    # s.sy == '('
    pos = s.position()
520
    positional_args, keyword_args = p_call_parse_args(s)
521

522 523
    if not keyword_args and len(positional_args) == 1 and isinstance(positional_args[0], list):
        return ExprNodes.SimpleCallNode(pos, function=function, args=positional_args[0])
William Stein's avatar
William Stein committed
524
    else:
525 526 527 528
        arg_tuple, keyword_dict = p_call_build_packed_args(pos, positional_args, keyword_args)
        return ExprNodes.GeneralCallNode(
            pos, function=function, positional_args=arg_tuple, keyword_args=keyword_dict)

William Stein's avatar
William Stein committed
529 530 531 532 533 534 535 536 537

#lambdef: 'lambda' [varargslist] ':' test

#subscriptlist: subscript (',' subscript)* [',']

def p_index(s, base):
    # s.sy == '['
    pos = s.position()
    s.next()
538 539
    subscripts, is_single_value = p_subscript_list(s)
    if is_single_value and len(subscripts[0]) == 2:
William Stein's avatar
William Stein committed
540
        start, stop = subscripts[0]
541
        result = ExprNodes.SliceIndexNode(pos,
William Stein's avatar
William Stein committed
542 543 544
            base = base, start = start, stop = stop)
    else:
        indexes = make_slice_nodes(pos, subscripts)
545
        if is_single_value:
William Stein's avatar
William Stein committed
546 547 548 549 550 551 552 553 554
            index = indexes[0]
        else:
            index = ExprNodes.TupleNode(pos, args = indexes)
        result = ExprNodes.IndexNode(pos,
            base = base, index = index)
    s.expect(']')
    return result

def p_subscript_list(s):
555
    is_single_value = True
William Stein's avatar
William Stein committed
556 557
    items = [p_subscript(s)]
    while s.sy == ',':
558
        is_single_value = False
William Stein's avatar
William Stein committed
559 560 561 562
        s.next()
        if s.sy == ']':
            break
        items.append(p_subscript(s))
563
    return items, is_single_value
William Stein's avatar
William Stein committed
564 565 566 567 568 569 570 571

#subscript: '.' '.' '.' | test | [test] ':' [test] [':' [test]]

def p_subscript(s):
    # Parse a subscript and return a list of
    # 1, 2 or 3 ExprNodes, depending on how
    # many slice elements were encountered.
    pos = s.position()
572 573 574 575 576 577 578 579 580 581
    start = p_slice_element(s, (':',))
    if s.sy != ':':
        return [start]
    s.next()
    stop = p_slice_element(s, (':', ',', ']'))
    if s.sy != ':':
        return [start, stop]
    s.next()
    step = p_slice_element(s, (':', ',', ']'))
    return [start, stop, step]
William Stein's avatar
William Stein committed
582 583 584 585 586

def p_slice_element(s, follow_set):
    # Simple expression which may be missing iff
    # it is followed by something in follow_set.
    if s.sy not in follow_set:
587
        return p_test(s)
William Stein's avatar
William Stein committed
588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618
    else:
        return None

def expect_ellipsis(s):
    s.expect('.')
    s.expect('.')
    s.expect('.')

def make_slice_nodes(pos, subscripts):
    # Convert a list of subscripts as returned
    # by p_subscript_list into a list of ExprNodes,
    # creating SliceNodes for elements with 2 or
    # more components.
    result = []
    for subscript in subscripts:
        if len(subscript) == 1:
            result.append(subscript[0])
        else:
            result.append(make_slice_node(pos, *subscript))
    return result

def make_slice_node(pos, start, stop = None, step = None):
    if not start:
        start = ExprNodes.NoneNode(pos)
    if not stop:
        stop = ExprNodes.NoneNode(pos)
    if not step:
        step = ExprNodes.NoneNode(pos)
    return ExprNodes.SliceNode(pos,
        start = start, stop = stop, step = step)

619
#atom: '(' [yield_expr|testlist_comp] ')' | '[' [listmaker] ']' | '{' [dict_or_set_maker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+
William Stein's avatar
William Stein committed
620 621 622 623 624 625 626 627

def p_atom(s):
    pos = s.position()
    sy = s.sy
    if sy == '(':
        s.next()
        if s.sy == ')':
            result = ExprNodes.TupleNode(pos, args = [])
628 629
        elif s.sy == 'yield':
            result = p_yield_expression(s)
William Stein's avatar
William Stein committed
630
        else:
631
            result = p_testlist_comp(s)
William Stein's avatar
William Stein committed
632 633 634 635 636
        s.expect(')')
        return result
    elif sy == '[':
        return p_list_maker(s)
    elif sy == '{':
637
        return p_dict_or_set_maker(s)
William Stein's avatar
William Stein committed
638 639
    elif sy == '`':
        return p_backquote_expr(s)
640 641 642
    elif sy == '.':
        expect_ellipsis(s)
        return ExprNodes.EllipsisNode(pos)
William Stein's avatar
William Stein committed
643
    elif sy == 'INT':
644
        return p_int_literal(s)
William Stein's avatar
William Stein committed
645 646 647 648 649 650 651 652
    elif sy == 'FLOAT':
        value = s.systring
        s.next()
        return ExprNodes.FloatNode(pos, value = value)
    elif sy == 'IMAG':
        value = s.systring[:-1]
        s.next()
        return ExprNodes.ImagNode(pos, value = value)
653
    elif sy == 'BEGIN_STRING':
654
        kind, bytes_value, unicode_value = p_cat_string_literal(s)
William Stein's avatar
William Stein committed
655
        if kind == 'c':
656
            return ExprNodes.CharNode(pos, value = bytes_value)
657
        elif kind == 'u':
658
            return ExprNodes.UnicodeNode(pos, value = unicode_value, bytes_value = bytes_value)
659
        elif kind == 'b':
660
            return ExprNodes.BytesNode(pos, value = bytes_value)
William Stein's avatar
William Stein committed
661
        else:
662
            return ExprNodes.StringNode(pos, value = bytes_value, unicode_value = unicode_value)
William Stein's avatar
William Stein committed
663
    elif sy == 'IDENT':
664
        name = s.systring
William Stein's avatar
William Stein committed
665 666 667
        s.next()
        if name == "None":
            return ExprNodes.NoneNode(pos)
668
        elif name == "True":
669
            return ExprNodes.BoolNode(pos, value=True)
670
        elif name == "False":
671
            return ExprNodes.BoolNode(pos, value=False)
672
        elif name == "NULL" and not s.in_python_file:
673
            return ExprNodes.NullNode(pos)
William Stein's avatar
William Stein committed
674
        else:
675
            return p_name(s, name)
William Stein's avatar
William Stein committed
676 677 678
    else:
        s.error("Expected an identifier or literal")

679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709
def p_int_literal(s):
    pos = s.position()
    value = s.systring
    s.next()
    unsigned = ""
    longness = ""
    while value[-1] in u"UuLl":
        if value[-1] in u"Ll":
            longness += "L"
        else:
            unsigned += "U"
        value = value[:-1]
    # '3L' is ambiguous in Py2 but not in Py3.  '3U' and '3LL' are
    # illegal in Py2 Python files.  All suffixes are illegal in Py3
    # Python files.
    is_c_literal = None
    if unsigned:
        is_c_literal = True
    elif longness:
        if longness == 'LL' or s.context.language_level >= 3:
            is_c_literal = True
    if s.in_python_file:
        if is_c_literal:
            error(pos, "illegal integer literal syntax in Python source file")
        is_c_literal = False
    return ExprNodes.IntNode(pos,
                             is_c_literal = is_c_literal,
                             value = value,
                             unsigned = unsigned,
                             longness = longness)

710

711 712
def p_name(s, name):
    pos = s.position()
713 714
    if not s.compile_time_expr and name in s.compile_time_env:
        value = s.compile_time_env.lookup_here(name)
715 716 717 718 719 720 721 722
        node = wrap_compile_time_constant(pos, value)
        if node is not None:
            return node
    return ExprNodes.NameNode(pos, name=name)


def wrap_compile_time_constant(pos, value):
    rep = repr(value)
723 724
    if value is None:
        return ExprNodes.NoneNode(pos)
725 726
    elif value is Ellipsis:
        return ExprNodes.EllipsisNode(pos)
727
    elif isinstance(value, bool):
728 729 730 731 732 733 734 735
        return ExprNodes.BoolNode(pos, value=value)
    elif isinstance(value, int):
        return ExprNodes.IntNode(pos, value=rep)
    elif isinstance(value, long):
        return ExprNodes.IntNode(pos, value=rep, longness="L")
    elif isinstance(value, float):
        return ExprNodes.FloatNode(pos, value=rep)
    elif isinstance(value, _unicode):
736
        return ExprNodes.UnicodeNode(pos, value=EncodedString(value))
737
    elif isinstance(value, _bytes):
738
        return ExprNodes.BytesNode(pos, value=BytesLiteral(value))
739 740 741 742 743
    elif isinstance(value, tuple):
        args = [wrap_compile_time_constant(pos, arg)
                for arg in value]
        if None not in args:
            return ExprNodes.TupleNode(pos, args=args)
744
        else:
745 746
            # error already reported
            return None
747 748
    error(pos, "Invalid type for compile-time constant: %r (type %s)"
               % (value, value.__class__.__name__))
749 750
    return None

751

William Stein's avatar
William Stein committed
752 753
def p_cat_string_literal(s):
    # A sequence of one or more adjacent string literals.
754 755 756 757 758 759 760 761 762 763 764 765 766 767 768
    # Returns (kind, bytes_value, unicode_value)
    # where kind in ('b', 'c', 'u', '')
    kind, bytes_value, unicode_value = p_string_literal(s)
    if kind == 'c' or s.sy != 'BEGIN_STRING':
        return kind, bytes_value, unicode_value
    bstrings, ustrings = [bytes_value], [unicode_value]
    bytes_value = unicode_value = None
    while s.sy == 'BEGIN_STRING':
        pos = s.position()
        next_kind, next_bytes_value, next_unicode_value = p_string_literal(s)
        if next_kind == 'c':
            error(pos, "Cannot concatenate char literal with another string or char literal")
        elif next_kind != kind:
            error(pos, "Cannot mix string literals of different types, expected %s'', got %s''" %
                  (kind, next_kind))
769
        else:
770 771 772
            bstrings.append(next_bytes_value)
            ustrings.append(next_unicode_value)
    # join and rewrap the partial literals
773
    if kind in ('b', 'c', '') or kind == 'u' and None not in bstrings:
774
        # Py3 enforced unicode literals are parsed as bytes/unicode combination
775
        bytes_value = BytesLiteral( StringEncoding.join_bytes(bstrings) )
776 777 778 779 780 781
        bytes_value.encoding = s.source_encoding
    if kind in ('u', ''):
        unicode_value = EncodedString( u''.join([ u for u in ustrings if u is not None ]) )
    return kind, bytes_value, unicode_value

def p_opt_string_literal(s, required_type='u'):
782
    if s.sy == 'BEGIN_STRING':
783 784 785 786 787 788 789
        kind, bytes_value, unicode_value = p_string_literal(s, required_type)
        if required_type == 'u':
            return unicode_value
        elif required_type == 'b':
            return bytes_value
        else:
            s.error("internal parser configuration error")
William Stein's avatar
William Stein committed
790 791 792
    else:
        return None

793 794 795 796 797 798
def check_for_non_ascii_characters(string):
    for c in string:
        if c >= u'\x80':
            return True
    return False

799
def p_string_literal(s, kind_override=None):
800 801 802 803 804 805 806
    # A single string or char literal.  Returns (kind, bvalue, uvalue)
    # where kind in ('b', 'c', 'u', '').  The 'bvalue' is the source
    # code byte sequence of the string literal, 'uvalue' is the
    # decoded Unicode string.  Either of the two may be None depending
    # on the 'kind' of string, only unprefixed strings have both
    # representations.

William Stein's avatar
William Stein committed
807 808
    # s.sy == 'BEGIN_STRING'
    pos = s.position()
809
    is_raw = False
810
    is_python3_source = s.context.language_level >= 3
Stefan Behnel's avatar
Stefan Behnel committed
811
    has_non_ascii_literal_characters = False
William Stein's avatar
William Stein committed
812
    kind = s.systring[:1].lower()
813
    if kind == 'r':
814 815 816 817 818 819
        # Py3 allows both 'br' and 'rb' as prefix
        if s.systring[1:2].lower() == 'b':
            kind = 'b'
        else:
            kind = ''
        is_raw = True
820 821 822
    elif kind in 'ub':
        is_raw = s.systring[1:2].lower() == 'r'
    elif kind != 'c':
William Stein's avatar
William Stein committed
823
        kind = ''
824 825 826 827 828 829 830 831 832 833 834 835
    if kind == '' and kind_override is None and Future.unicode_literals in s.context.future_directives:
        chars = StringEncoding.StrLiteralBuilder(s.source_encoding)
        kind = 'u'
    else:
        if kind_override is not None and kind_override in 'ub':
            kind = kind_override
        if kind == 'u':
            chars = StringEncoding.UnicodeLiteralBuilder()
        elif kind == '':
            chars = StringEncoding.StrLiteralBuilder(s.source_encoding)
        else:
            chars = StringEncoding.BytesLiteralBuilder(s.source_encoding)
836

William Stein's avatar
William Stein committed
837 838 839
    while 1:
        s.next()
        sy = s.sy
840
        systr = s.systring
William Stein's avatar
William Stein committed
841 842
        #print "p_string_literal: sy =", sy, repr(s.systring) ###
        if sy == 'CHARS':
843
            chars.append(systr)
Stefan Behnel's avatar
Stefan Behnel committed
844 845
            if is_python3_source and not has_non_ascii_literal_characters and check_for_non_ascii_characters(systr):
                has_non_ascii_literal_characters = True
William Stein's avatar
William Stein committed
846
        elif sy == 'ESCAPE':
847
            if is_raw:
848
                chars.append(systr)
Stefan Behnel's avatar
Stefan Behnel committed
849
                if is_python3_source and not has_non_ascii_literal_characters \
850
                       and check_for_non_ascii_characters(systr):
Stefan Behnel's avatar
Stefan Behnel committed
851
                    has_non_ascii_literal_characters = True
William Stein's avatar
William Stein committed
852 853
            else:
                c = systr[1]
854 855 856
                if c in u"01234567":
                    chars.append_charval( int(systr[1:], 8) )
                elif c in u"'\"\\":
857
                    chars.append(c)
858 859 860 861
                elif c in u"abfnrtv":
                    chars.append(
                        StringEncoding.char_from_escape_sequence(systr))
                elif c == u'\n':
William Stein's avatar
William Stein committed
862
                    pass
863
                elif c == u'x':   # \xXX
864 865 866
                    if len(systr) == 4:
                        chars.append_charval( int(systr[2:], 16) )
                    else:
867 868
                        s.error("Invalid hex escape '%s'" % systr,
                                fatal=False)
869 870 871 872
                elif c in u'NUu' and kind in ('u', ''):   # \uxxxx, \Uxxxxxxxx, \N{...}
                    chrval = -1
                    if c == u'N':
                        try:
873
                            chrval = ord(lookup_unicodechar(systr[3:-1]))
874
                        except KeyError:
875 876
                            s.error("Unknown Unicode character name %s" %
                                    repr(systr[3:-1]).lstrip('u'))
877 878 879
                    elif len(systr) in (6,10):
                        chrval = int(systr[2:], 16)
                        if chrval > 1114111: # sys.maxunicode:
880
                            s.error("Invalid unicode escape '%s'" % systr)
881
                            chrval = -1
882
                    else:
883 884
                        s.error("Invalid unicode escape '%s'" % systr,
                                fatal=False)
885 886
                    if chrval >= 0:
                        chars.append_uescape(chrval, systr)
William Stein's avatar
William Stein committed
887
                else:
888
                    chars.append(u'\\' + systr[1:])
Stefan Behnel's avatar
Stefan Behnel committed
889
                    if is_python3_source and not has_non_ascii_literal_characters \
890
                           and check_for_non_ascii_characters(systr):
Stefan Behnel's avatar
Stefan Behnel committed
891
                        has_non_ascii_literal_characters = True
William Stein's avatar
William Stein committed
892
        elif sy == 'NEWLINE':
893
            chars.append(u'\n')
William Stein's avatar
William Stein committed
894 895 896
        elif sy == 'END_STRING':
            break
        elif sy == 'EOF':
897
            s.error("Unclosed string literal", pos=pos)
William Stein's avatar
William Stein committed
898
        else:
899
            s.error("Unexpected token %r:%r in string literal" %
William Stein's avatar
William Stein committed
900
                    (sy, s.systring))
901

902
    if kind == 'c':
903 904 905 906
        unicode_value = None
        bytes_value = chars.getchar()
        if len(bytes_value) != 1:
            error(pos, u"invalid character literal: %r" % bytes_value)
907
    else:
908
        bytes_value, unicode_value = chars.getstrings()
Stefan Behnel's avatar
Stefan Behnel committed
909
        if is_python3_source and has_non_ascii_literal_characters:
910 911
            # Python 3 forbids literal non-ASCII characters in byte strings
            if kind != 'u':
912 913
                s.error("bytes can only contain ASCII literal characters.",
                        pos=pos, fatal=False)
914
            bytes_value = None
William Stein's avatar
William Stein committed
915
    s.next()
916
    return (kind, bytes_value, unicode_value)
William Stein's avatar
William Stein committed
917

918 919 920 921

# since PEP 448:
# list_display  ::=     "[" [listmaker] "]"
# listmaker     ::=     (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
Stefan Behnel's avatar
Stefan Behnel committed
922
# comp_iter     ::=     comp_for | comp_if
923 924
# comp_for      ::=     "for" expression_list "in" testlist [comp_iter]
# comp_if       ::=     "if" test [comp_iter]
925

William Stein's avatar
William Stein committed
926 927 928 929
def p_list_maker(s):
    # s.sy == '['
    pos = s.position()
    s.next()
Robert Bradshaw's avatar
Robert Bradshaw committed
930 931
    if s.sy == ']':
        s.expect(']')
932 933 934
        return ExprNodes.ListNode(pos, args=[])

    expr = p_test_or_starred_expr(s)
Robert Bradshaw's avatar
Robert Bradshaw committed
935
    if s.sy == 'for':
936 937
        if expr.is_starred:
            s.error("iterable unpacking cannot be used in comprehension")
938
        append = ExprNodes.ComprehensionAppendNode(pos, expr=expr)
939
        loop = p_comp_for(s, append)
Robert Bradshaw's avatar
Robert Bradshaw committed
940
        s.expect(']')
941
        return ExprNodes.ComprehensionNode(
942
            pos, loop=loop, append=append, type=Builtin.list_type,
943
            # list comprehensions leak their loop variable in Py2
944 945 946 947 948 949
            has_local_scope=s.context.language_level >= 3)

    # (merged) list literal
    if s.sy == ',':
        s.next()
        exprs = p_test_or_starred_expr_list(s, expr)
Robert Bradshaw's avatar
Robert Bradshaw committed
950
    else:
951 952 953 954
        exprs = [expr]
    s.expect(']')
    return ExprNodes.ListNode(pos, args=exprs)

955

Stefan Behnel's avatar
Stefan Behnel committed
956
def p_comp_iter(s, body):
Robert Bradshaw's avatar
Robert Bradshaw committed
957
    if s.sy == 'for':
958
        return p_comp_for(s, body)
Robert Bradshaw's avatar
Robert Bradshaw committed
959
    elif s.sy == 'if':
Stefan Behnel's avatar
Stefan Behnel committed
960
        return p_comp_if(s, body)
Robert Bradshaw's avatar
Robert Bradshaw committed
961
    else:
962 963
        # insert the 'append' operation into the loop
        return body
William Stein's avatar
William Stein committed
964

965
def p_comp_for(s, body):
Robert Bradshaw's avatar
Robert Bradshaw committed
966 967 968
    # s.sy == 'for'
    pos = s.position()
    s.next()
969
    kw = p_for_bounds(s, allow_testlist=False)
970
    kw.update(else_clause = None, body = p_comp_iter(s, body))
Robert Bradshaw's avatar
Robert Bradshaw committed
971
    return Nodes.ForStatNode(pos, **kw)
972

Stefan Behnel's avatar
Stefan Behnel committed
973
def p_comp_if(s, body):
Robert Bradshaw's avatar
Robert Bradshaw committed
974 975 976
    # s.sy == 'if'
    pos = s.position()
    s.next()
Stefan Behnel's avatar
Stefan Behnel committed
977
    test = p_test_nocond(s)
978
    return Nodes.IfStatNode(pos,
979
        if_clauses = [Nodes.IfClauseNode(pos, condition = test,
Stefan Behnel's avatar
Stefan Behnel committed
980
                                         body = p_comp_iter(s, body))],
Robert Bradshaw's avatar
Robert Bradshaw committed
981
        else_clause = None )
982

983 984 985 986 987 988

# since PEP 448:
#dictorsetmaker: ( ((test ':' test | '**' expr)
#                   (comp_for | (',' (test ':' test | '**' expr))* [','])) |
#                  ((test | star_expr)
#                   (comp_for | (',' (test | star_expr))* [','])) )
William Stein's avatar
William Stein committed
989

990
def p_dict_or_set_maker(s):
William Stein's avatar
William Stein committed
991 992 993
    # s.sy == '{'
    pos = s.position()
    s.next()
994
    if s.sy == '}':
William Stein's avatar
William Stein committed
995
        s.next()
996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009
        return ExprNodes.DictNode(pos, key_value_pairs=[])

    parts = []
    target_type = 0
    last_was_simple_item = False
    while True:
        if s.sy in ('*', '**'):
            # merged set/dict literal
            if target_type == 0:
                target_type = 1 if s.sy == '*' else 2  # 'stars'
            elif target_type != len(s.sy):
                s.error("unexpected %sitem found in %s literal" % (
                    s.sy, 'set' if target_type == 1 else 'dict'))
            s.next()
1010 1011 1012
            if s.sy == '*':
                s.error("expected expression, found '*'")
            item = p_starred_expr(s)
1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031
            parts.append(item)
            last_was_simple_item = False
        else:
            item = p_test(s)
            if target_type == 0:
                target_type = 2 if s.sy == ':' else 1  # dict vs. set
            if target_type == 2:
                # dict literal
                s.expect(':')
                key = item
                value = p_test(s)
                item = ExprNodes.DictItemNode(key.pos, key=key, value=value)
            if last_was_simple_item:
                parts[-1].append(item)
            else:
                parts.append([item])
                last_was_simple_item = True

        if s.sy == ',':
1032
            s.next()
1033 1034
            if s.sy == '}':
                break
1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049
        else:
            break

    if s.sy == 'for':
        # dict/set comprehension
        if len(parts) == 1 and isinstance(parts[0], list) and len(parts[0]) == 1:
            item = parts[0][0]
            if target_type == 2:
                assert isinstance(item, ExprNodes.DictItemNode), type(item)
                comprehension_type = Builtin.dict_type
                append = ExprNodes.DictComprehensionAppendNode(
                    item.pos, key_expr=item.key, value_expr=item.value)
            else:
                comprehension_type = Builtin.set_type
                append = ExprNodes.ComprehensionAppendNode(item.pos, expr=item)
1050
            loop = p_comp_for(s, append)
1051
            s.expect('}')
1052
            return ExprNodes.ComprehensionNode(pos, loop=loop, append=append, type=comprehension_type)
1053
        else:
1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078
            # syntax error, try to find a good error message
            if len(parts) == 1 and not isinstance(parts[0], list):
                s.error("iterable unpacking cannot be used in comprehension")
            else:
                # e.g. "{1,2,3 for ..."
                s.expect('}')
            return ExprNodes.DictNode(pos, key_value_pairs=[])

    s.expect('}')
    if target_type == 1:
        # (merged) set literal
        items = []
        set_items = []
        for part in parts:
            if isinstance(part, list):
                set_items.extend(part)
            else:
                if set_items:
                    items.append(ExprNodes.SetNode(set_items[0].pos, args=set_items))
                    set_items = []
                items.append(part)
        if set_items:
            items.append(ExprNodes.SetNode(set_items[0].pos, args=set_items))
        if len(items) == 1 and items[0].is_set_literal:
            return items[0]
1079
        return ExprNodes.MergedSequenceNode(pos, args=items, type=Builtin.set_type)
1080
    else:
1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097
        # (merged) dict literal
        items = []
        dict_items = []
        for part in parts:
            if isinstance(part, list):
                dict_items.extend(part)
            else:
                if dict_items:
                    items.append(ExprNodes.DictNode(dict_items[0].pos, key_value_pairs=dict_items))
                    dict_items = []
                items.append(part)
        if dict_items:
            items.append(ExprNodes.DictNode(dict_items[0].pos, key_value_pairs=dict_items))
        if len(items) == 1 and items[0].is_dict_literal:
            return items[0]
        return ExprNodes.MergedDictNode(pos, keyword_args=items, reject_duplicates=False)

William Stein's avatar
William Stein committed
1098

1099
# NOTE: no longer in Py3 :)
William Stein's avatar
William Stein committed
1100 1101 1102 1103
def p_backquote_expr(s):
    # s.sy == '`'
    pos = s.position()
    s.next()
1104 1105 1106 1107
    args = [p_test(s)]
    while s.sy == ',':
        s.next()
        args.append(p_test(s))
William Stein's avatar
William Stein committed
1108
    s.expect('`')
1109 1110 1111 1112
    if len(args) == 1:
        arg = args[0]
    else:
        arg = ExprNodes.TupleNode(pos, args = args)
William Stein's avatar
William Stein committed
1113 1114
    return ExprNodes.BackquoteNode(pos, arg = arg)

1115 1116
def p_simple_expr_list(s, expr=None):
    exprs = expr is not None and [expr] or []
William Stein's avatar
William Stein committed
1117
    while s.sy not in expr_terminators:
1118
        exprs.append( p_test(s) )
Stefan Behnel's avatar
Stefan Behnel committed
1119
        if s.sy != ',':
William Stein's avatar
William Stein committed
1120 1121 1122 1123
            break
        s.next()
    return exprs

1124

1125 1126 1127
def p_test_or_starred_expr_list(s, expr=None):
    exprs = expr is not None and [expr] or []
    while s.sy not in expr_terminators:
1128
        exprs.append(p_test_or_starred_expr(s))
1129 1130 1131 1132
        if s.sy != ',':
            break
        s.next()
    return exprs
1133

1134 1135 1136 1137

#testlist: test (',' test)* [',']

def p_testlist(s):
William Stein's avatar
William Stein committed
1138
    pos = s.position()
1139
    expr = p_test(s)
William Stein's avatar
William Stein committed
1140 1141
    if s.sy == ',':
        s.next()
1142
        exprs = p_simple_expr_list(s, expr)
William Stein's avatar
William Stein committed
1143 1144 1145 1146
        return ExprNodes.TupleNode(pos, args = exprs)
    else:
        return expr

1147
# testlist_star_expr: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
Robert Bradshaw's avatar
Robert Bradshaw committed
1148

1149
def p_testlist_star_expr(s):
Robert Bradshaw's avatar
Robert Bradshaw committed
1150
    pos = s.position()
1151
    expr = p_test_or_starred_expr(s)
Robert Bradshaw's avatar
Robert Bradshaw committed
1152
    if s.sy == ',':
1153
        s.next()
1154 1155
        exprs = p_test_or_starred_expr_list(s, expr)
        return ExprNodes.TupleNode(pos, args = exprs)
Robert Bradshaw's avatar
Robert Bradshaw committed
1156 1157 1158
    else:
        return expr

1159 1160 1161
# testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )

def p_testlist_comp(s):
Robert Bradshaw's avatar
Robert Bradshaw committed
1162
    pos = s.position()
1163
    expr = p_test_or_starred_expr(s)
Robert Bradshaw's avatar
Robert Bradshaw committed
1164
    if s.sy == ',':
1165
        s.next()
1166
        exprs = p_test_or_starred_expr_list(s, expr)
Robert Bradshaw's avatar
Robert Bradshaw committed
1167
        return ExprNodes.TupleNode(pos, args = exprs)
1168 1169
    elif s.sy == 'for':
        return p_genexp(s, expr)
Robert Bradshaw's avatar
Robert Bradshaw committed
1170 1171
    else:
        return expr
1172 1173 1174

def p_genexp(s, expr):
    # s.sy == 'for'
1175 1176
    loop = p_comp_for(s, Nodes.ExprStatNode(
        expr.pos, expr = ExprNodes.YieldExprNode(expr.pos, arg=expr)))
1177 1178
    return ExprNodes.GeneratorExpressionNode(expr.pos, loop=loop)

1179 1180
expr_terminators = cython.declare(set, set([
    ')', ']', '}', ':', '=', 'NEWLINE']))
William Stein's avatar
William Stein committed
1181

1182

William Stein's avatar
William Stein committed
1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195
#-------------------------------------------------------
#
#   Statements
#
#-------------------------------------------------------

def p_global_statement(s):
    # assume s.sy == 'global'
    pos = s.position()
    s.next()
    names = p_ident_list(s)
    return Nodes.GlobalNode(pos, names = names)

1196

1197 1198 1199 1200 1201 1202
def p_nonlocal_statement(s):
    pos = s.position()
    s.next()
    names = p_ident_list(s)
    return Nodes.NonlocalNode(pos, names = names)

1203

William Stein's avatar
William Stein committed
1204
def p_expression_or_assignment(s):
1205
    expr_list = [p_testlist_star_expr(s)]
1206 1207 1208 1209 1210 1211
    if s.sy == '=' and expr_list[0].is_starred:
        # This is a common enough error to make when learning Cython to let
        # it fail as early as possible and give a very clear error message.
        s.error("a starred assignment target must be in a list or tuple"
                " - maybe you meant to use an index assignment: var[0] = ...",
                pos=expr_list[0].pos)
William Stein's avatar
William Stein committed
1212 1213
    while s.sy == '=':
        s.next()
1214 1215 1216 1217 1218
        if s.sy == 'yield':
            expr = p_yield_expression(s)
        else:
            expr = p_testlist_star_expr(s)
        expr_list.append(expr)
William Stein's avatar
William Stein committed
1219
    if len(expr_list) == 1:
1220
        if re.match(r"([-+*/%^&|]|<<|>>|\*\*|//|@)=", s.sy):
1221
            lhs = expr_list[0]
1222 1223 1224 1225 1226 1227
            if isinstance(lhs, ExprNodes.SliceIndexNode):
                # implementation requires IndexNode
                lhs = ExprNodes.IndexNode(
                    lhs.pos,
                    base=lhs.base,
                    index=make_slice_node(lhs.pos, lhs.start, lhs.stop))
1228
            elif not isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode, ExprNodes.NameNode)):
1229
                error(lhs.pos, "Illegal operand for inplace operation.")
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
1230
            operator = s.sy[:-1]
1231
            s.next()
1232 1233 1234 1235
            if s.sy == 'yield':
                rhs = p_yield_expression(s)
            else:
                rhs = p_testlist(s)
1236
            return Nodes.InPlaceAssignmentNode(lhs.pos, operator=operator, lhs=lhs, rhs=rhs)
1237
        expr = expr_list[0]
1238
        return Nodes.ExprStatNode(expr.pos, expr=expr)
1239

1240 1241
    rhs = expr_list[-1]
    if len(expr_list) == 2:
1242
        return Nodes.SingleAssignmentNode(rhs.pos, lhs=expr_list[0], rhs=rhs)
William Stein's avatar
William Stein committed
1243
    else:
1244 1245
        return Nodes.CascadedAssignmentNode(rhs.pos, lhs_list=expr_list[:-1], rhs=rhs)

William Stein's avatar
William Stein committed
1246 1247 1248 1249

def p_print_statement(s):
    # s.sy == 'print'
    pos = s.position()
1250
    ends_with_comma = 0
William Stein's avatar
William Stein committed
1251 1252
    s.next()
    if s.sy == '>>':
1253
        s.next()
1254
        stream = p_test(s)
1255 1256 1257 1258 1259
        if s.sy == ',':
            s.next()
            ends_with_comma = s.sy in ('NEWLINE', 'EOF')
    else:
        stream = None
William Stein's avatar
William Stein committed
1260 1261
    args = []
    if s.sy not in ('NEWLINE', 'EOF'):
1262
        args.append(p_test(s))
William Stein's avatar
William Stein committed
1263 1264 1265
        while s.sy == ',':
            s.next()
            if s.sy in ('NEWLINE', 'EOF'):
1266
                ends_with_comma = 1
William Stein's avatar
William Stein committed
1267
                break
1268
            args.append(p_test(s))
1269
    arg_tuple = ExprNodes.TupleNode(pos, args=args)
1270
    return Nodes.PrintStatNode(pos,
1271 1272 1273
        arg_tuple=arg_tuple, stream=stream,
        append_newline=not ends_with_comma)

William Stein's avatar
William Stein committed
1274

1275 1276 1277 1278
def p_exec_statement(s):
    # s.sy == 'exec'
    pos = s.position()
    s.next()
1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290
    code = p_bit_expr(s)
    if isinstance(code, ExprNodes.TupleNode):
        # Py3 compatibility syntax
        tuple_variant = True
        args = code.args
        if len(args) not in (2, 3):
            s.error("expected tuple of length 2 or 3, got length %d" % len(args),
                    pos=pos, fatal=False)
            args = [code]
    else:
        tuple_variant = False
        args = [code]
1291
    if s.sy == 'in':
1292 1293 1294
        if tuple_variant:
            s.error("tuple variant of exec does not support additional 'in' arguments",
                    fatal=False)
1295
        s.next()
1296
        args.append(p_test(s))
1297 1298
        if s.sy == ',':
            s.next()
1299
            args.append(p_test(s))
1300
    return Nodes.ExecStatNode(pos, args=args)
1301

William Stein's avatar
William Stein committed
1302 1303 1304 1305
def p_del_statement(s):
    # s.sy == 'del'
    pos = s.position()
    s.next()
1306
    # FIXME: 'exprlist' in Python
William Stein's avatar
William Stein committed
1307 1308 1309 1310 1311 1312 1313
    args = p_simple_expr_list(s)
    return Nodes.DelStatNode(pos, args = args)

def p_pass_statement(s, with_newline = 0):
    pos = s.position()
    s.expect('pass')
    if with_newline:
1314
        s.expect_newline("Expected a newline", ignore_semicolon=True)
William Stein's avatar
William Stein committed
1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333
    return Nodes.PassStatNode(pos)

def p_break_statement(s):
    # s.sy == 'break'
    pos = s.position()
    s.next()
    return Nodes.BreakStatNode(pos)

def p_continue_statement(s):
    # s.sy == 'continue'
    pos = s.position()
    s.next()
    return Nodes.ContinueStatNode(pos)

def p_return_statement(s):
    # s.sy == 'return'
    pos = s.position()
    s.next()
    if s.sy not in statement_terminators:
1334
        value = p_testlist(s)
William Stein's avatar
William Stein committed
1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345
    else:
        value = None
    return Nodes.ReturnStatNode(pos, value = value)

def p_raise_statement(s):
    # s.sy == 'raise'
    pos = s.position()
    s.next()
    exc_type = None
    exc_value = None
    exc_tb = None
Haoyu Bai's avatar
Haoyu Bai committed
1346
    cause = None
William Stein's avatar
William Stein committed
1347
    if s.sy not in statement_terminators:
1348
        exc_type = p_test(s)
William Stein's avatar
William Stein committed
1349 1350
        if s.sy == ',':
            s.next()
1351
            exc_value = p_test(s)
William Stein's avatar
William Stein committed
1352 1353
            if s.sy == ',':
                s.next()
1354
                exc_tb = p_test(s)
Haoyu Bai's avatar
Haoyu Bai committed
1355 1356 1357
        elif s.sy == 'from':
            s.next()
            cause = p_test(s)
1358
    if exc_type or exc_value or exc_tb:
1359
        return Nodes.RaiseStatNode(pos,
1360 1361
            exc_type = exc_type,
            exc_value = exc_value,
Haoyu Bai's avatar
Haoyu Bai committed
1362 1363
            exc_tb = exc_tb,
            cause = cause)
1364 1365
    else:
        return Nodes.ReraiseStatNode(pos)
William Stein's avatar
William Stein committed
1366

1367

William Stein's avatar
William Stein committed
1368 1369 1370 1371 1372
def p_import_statement(s):
    # s.sy in ('import', 'cimport')
    pos = s.position()
    kind = s.sy
    s.next()
1373
    items = [p_dotted_name(s, as_allowed=1)]
William Stein's avatar
William Stein committed
1374 1375
    while s.sy == ',':
        s.next()
1376
        items.append(p_dotted_name(s, as_allowed=1))
William Stein's avatar
William Stein committed
1377
    stats = []
1378
    is_absolute = Future.absolute_import in s.context.future_directives
William Stein's avatar
William Stein committed
1379 1380
    for pos, target_name, dotted_name, as_name in items:
        if kind == 'cimport':
1381 1382 1383 1384 1385
            stat = Nodes.CImportStatNode(
                pos,
                module_name=dotted_name,
                as_name=as_name,
                is_absolute=is_absolute)
William Stein's avatar
William Stein committed
1386
        else:
1387
            if as_name and "." in dotted_name:
1388
                name_list = ExprNodes.ListNode(pos, args=[
1389
                    ExprNodes.IdentifierStringNode(pos, value=s.context.intern_ustring("*"))])
1390 1391
            else:
                name_list = None
1392 1393 1394 1395 1396 1397 1398 1399
            stat = Nodes.SingleAssignmentNode(
                pos,
                lhs=ExprNodes.NameNode(pos, name=as_name or target_name),
                rhs=ExprNodes.ImportNode(
                    pos,
                    module_name=ExprNodes.IdentifierStringNode(pos, value=dotted_name),
                    level=0 if is_absolute else None,
                    name_list=name_list))
William Stein's avatar
William Stein committed
1400
        stats.append(stat)
1401 1402
    return Nodes.StatListNode(pos, stats=stats)

William Stein's avatar
William Stein committed
1403

Stefan Behnel's avatar
Stefan Behnel committed
1404
def p_from_import_statement(s, first_statement = 0):
William Stein's avatar
William Stein committed
1405 1406 1407
    # s.sy == 'from'
    pos = s.position()
    s.next()
Haoyu Bai's avatar
Haoyu Bai committed
1408 1409 1410 1411 1412 1413 1414
    if s.sy == '.':
        # count relative import level
        level = 0
        while s.sy == '.':
            level += 1
            s.next()
    else:
1415
        level = None
1416
    if level is not None and s.sy in ('import', 'cimport'):
Haoyu Bai's avatar
Haoyu Bai committed
1417
        # we are dealing with "from .. import foo, bar"
1418
        dotted_name_pos, dotted_name = s.position(), s.context.intern_ustring('')
William Stein's avatar
William Stein committed
1419
    else:
1420 1421 1422 1423
        if level is None and Future.absolute_import in s.context.future_directives:
            level = 0
        (dotted_name_pos, _, dotted_name, _) = p_dotted_name(s, as_allowed=False)
    if s.sy not in ('import', 'cimport'):
William Stein's avatar
William Stein committed
1424
        s.error("Expected 'import' or 'cimport'")
1425 1426
    kind = s.sy
    s.next()
Haoyu Bai's avatar
Haoyu Bai committed
1427

1428
    is_cimport = kind == 'cimport'
1429
    is_parenthesized = False
William Stein's avatar
William Stein committed
1430
    if s.sy == '*':
1431
        imported_names = [(s.position(), s.context.intern_ustring("*"), None, None)]
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
1432 1433
        s.next()
    else:
1434 1435 1436
        if s.sy == '(':
            is_parenthesized = True
            s.next()
1437
        imported_names = [p_imported_name(s, is_cimport)]
William Stein's avatar
William Stein committed
1438 1439
    while s.sy == ',':
        s.next()
1440 1441
        if is_parenthesized and s.sy == ')':
            break
1442
        imported_names.append(p_imported_name(s, is_cimport))
1443 1444
    if is_parenthesized:
        s.expect(')')
Stefan Behnel's avatar
Stefan Behnel committed
1445 1446 1447
    if dotted_name == '__future__':
        if not first_statement:
            s.error("from __future__ imports must occur at the beginning of the file")
1448
        elif level:
Haoyu Bai's avatar
Haoyu Bai committed
1449
            s.error("invalid syntax")
Stefan Behnel's avatar
Stefan Behnel committed
1450
        else:
1451
            for (name_pos, name, as_name, kind) in imported_names:
1452 1453 1454
                if name == "braces":
                    s.error("not a chance", name_pos)
                    break
Stefan Behnel's avatar
Stefan Behnel committed
1455 1456 1457
                try:
                    directive = getattr(Future, name)
                except AttributeError:
1458
                    s.error("future feature %s is not defined" % name, name_pos)
Stefan Behnel's avatar
Stefan Behnel committed
1459 1460 1461 1462
                    break
                s.context.future_directives.add(directive)
        return Nodes.PassStatNode(pos)
    elif kind == 'cimport':
1463 1464 1465 1466
        return Nodes.FromCImportStatNode(
            pos, module_name=dotted_name,
            relative_level=level,
            imported_names=imported_names)
William Stein's avatar
William Stein committed
1467 1468 1469
    else:
        imported_name_strings = []
        items = []
1470
        for (name_pos, name, as_name, kind) in imported_names:
William Stein's avatar
William Stein committed
1471
            imported_name_strings.append(
1472
                ExprNodes.IdentifierStringNode(name_pos, value=name))
William Stein's avatar
William Stein committed
1473
            items.append(
1474
                (name, ExprNodes.NameNode(name_pos, name=as_name or name)))
William Stein's avatar
William Stein committed
1475
        import_list = ExprNodes.ListNode(
1476
            imported_names[0][0], args=imported_name_strings)
William Stein's avatar
William Stein committed
1477 1478
        return Nodes.FromImportStatNode(pos,
            module = ExprNodes.ImportNode(dotted_name_pos,
1479
                module_name = ExprNodes.IdentifierStringNode(pos, value = dotted_name),
Haoyu Bai's avatar
Haoyu Bai committed
1480
                level = level,
William Stein's avatar
William Stein committed
1481 1482 1483
                name_list = import_list),
            items = items)

1484 1485

imported_name_kinds = cython.declare(set, set(['class', 'struct', 'union']))
1486 1487

def p_imported_name(s, is_cimport):
William Stein's avatar
William Stein committed
1488
    pos = s.position()
1489 1490 1491 1492
    kind = None
    if is_cimport and s.systring in imported_name_kinds:
        kind = s.systring
        s.next()
William Stein's avatar
William Stein committed
1493 1494
    name = p_ident(s)
    as_name = p_as_name(s)
1495
    return (pos, name, as_name, kind)
William Stein's avatar
William Stein committed
1496

1497

William Stein's avatar
William Stein committed
1498 1499 1500 1501 1502 1503 1504 1505 1506 1507
def p_dotted_name(s, as_allowed):
    pos = s.position()
    target_name = p_ident(s)
    as_name = None
    names = [target_name]
    while s.sy == '.':
        s.next()
        names.append(p_ident(s))
    if as_allowed:
        as_name = p_as_name(s)
1508 1509
    return (pos, target_name, s.context.intern_ustring(u'.'.join(names)), as_name)

William Stein's avatar
William Stein committed
1510 1511 1512 1513 1514 1515 1516 1517

def p_as_name(s):
    if s.sy == 'IDENT' and s.systring == 'as':
        s.next()
        return p_ident(s)
    else:
        return None

1518

William Stein's avatar
William Stein committed
1519 1520 1521 1522
def p_assert_statement(s):
    # s.sy == 'assert'
    pos = s.position()
    s.next()
1523
    cond = p_test(s)
William Stein's avatar
William Stein committed
1524 1525
    if s.sy == ',':
        s.next()
1526
        value = p_test(s)
William Stein's avatar
William Stein committed
1527 1528 1529 1530
    else:
        value = None
    return Nodes.AssertStatNode(pos, cond = cond, value = value)

1531

1532
statement_terminators = cython.declare(set, set([';', 'NEWLINE', 'EOF']))
William Stein's avatar
William Stein committed
1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547

def p_if_statement(s):
    # s.sy == 'if'
    pos = s.position()
    s.next()
    if_clauses = [p_if_clause(s)]
    while s.sy == 'elif':
        s.next()
        if_clauses.append(p_if_clause(s))
    else_clause = p_else_clause(s)
    return Nodes.IfStatNode(pos,
        if_clauses = if_clauses, else_clause = else_clause)

def p_if_clause(s):
    pos = s.position()
1548
    test = p_test(s)
William Stein's avatar
William Stein committed
1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563
    body = p_suite(s)
    return Nodes.IfClauseNode(pos,
        condition = test, body = body)

def p_else_clause(s):
    if s.sy == 'else':
        s.next()
        return p_suite(s)
    else:
        return None

def p_while_statement(s):
    # s.sy == 'while'
    pos = s.position()
    s.next()
1564
    test = p_test(s)
William Stein's avatar
William Stein committed
1565 1566
    body = p_suite(s)
    else_clause = p_else_clause(s)
1567 1568
    return Nodes.WhileStatNode(pos,
        condition = test, body = body,
William Stein's avatar
William Stein committed
1569 1570 1571 1572 1573 1574
        else_clause = else_clause)

def p_for_statement(s):
    # s.sy == 'for'
    pos = s.position()
    s.next()
1575
    kw = p_for_bounds(s, allow_testlist=True)
1576 1577
    body = p_suite(s)
    else_clause = p_else_clause(s)
Robert Bradshaw's avatar
Robert Bradshaw committed
1578
    kw.update(body = body, else_clause = else_clause)
Robert Bradshaw's avatar
Robert Bradshaw committed
1579
    return Nodes.ForStatNode(pos, **kw)
1580

1581
def p_for_bounds(s, allow_testlist=True):
William Stein's avatar
William Stein committed
1582 1583 1584
    target = p_for_target(s)
    if s.sy == 'in':
        s.next()
1585
        iterator = p_for_iterator(s, allow_testlist)
1586
        return dict( target = target, iterator = iterator )
1587
    elif not s.in_python_file:
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
1588 1589 1590 1591 1592 1593
        if s.sy == 'from':
            s.next()
            bound1 = p_bit_expr(s)
        else:
            # Support shorter "for a <= x < b" syntax
            bound1, target = target, None
William Stein's avatar
William Stein committed
1594 1595 1596 1597 1598 1599
        rel1 = p_for_from_relation(s)
        name2_pos = s.position()
        name2 = p_ident(s)
        rel2_pos = s.position()
        rel2 = p_for_from_relation(s)
        bound2 = p_bit_expr(s)
1600
        step = p_for_from_step(s)
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
1601 1602 1603 1604
        if target is None:
            target = ExprNodes.NameNode(name2_pos, name = name2)
        else:
            if not target.is_name:
1605
                error(target.pos,
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
1606 1607 1608 1609
                    "Target of for-from statement must be a variable name")
            elif name2 != target.name:
                error(name2_pos,
                    "Variable name in for-from range does not match target")
Stefan Behnel's avatar
Stefan Behnel committed
1610
        if rel1[0] != rel2[0]:
William Stein's avatar
William Stein committed
1611 1612
            error(rel2_pos,
                "Relation directions in for-from do not match")
1613 1614 1615
        return dict(target = target,
                    bound1 = bound1,
                    relation1 = rel1,
1616 1617 1618 1619
                    relation2 = rel2,
                    bound2 = bound2,
                    step = step,
                    )
1620 1621 1622
    else:
        s.expect('in')
        return {}
William Stein's avatar
William Stein committed
1623 1624 1625 1626 1627 1628 1629 1630

def p_for_from_relation(s):
    if s.sy in inequality_relations:
        op = s.sy
        s.next()
        return op
    else:
        s.error("Expected one of '<', '<=', '>' '>='")
1631

1632
def p_for_from_step(s):
1633
    if s.sy == 'IDENT' and s.systring == 'by':
1634 1635 1636 1637 1638
        s.next()
        step = p_bit_expr(s)
        return step
    else:
        return None
William Stein's avatar
William Stein committed
1639

1640
inequality_relations = cython.declare(set, set(['<', '<=', '>', '>=']))
William Stein's avatar
William Stein committed
1641

1642
def p_target(s, terminator):
William Stein's avatar
William Stein committed
1643
    pos = s.position()
1644
    expr = p_starred_expr(s)
William Stein's avatar
William Stein committed
1645 1646 1647
    if s.sy == ',':
        s.next()
        exprs = [expr]
1648
        while s.sy != terminator:
1649
            exprs.append(p_starred_expr(s))
Stefan Behnel's avatar
Stefan Behnel committed
1650
            if s.sy != ',':
William Stein's avatar
William Stein committed
1651 1652 1653 1654 1655 1656
                break
            s.next()
        return ExprNodes.TupleNode(pos, args = exprs)
    else:
        return expr

1657 1658 1659
def p_for_target(s):
    return p_target(s, 'in')

1660
def p_for_iterator(s, allow_testlist=True):
William Stein's avatar
William Stein committed
1661
    pos = s.position()
1662 1663 1664 1665
    if allow_testlist:
        expr = p_testlist(s)
    else:
        expr = p_or_test(s)
William Stein's avatar
William Stein committed
1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680
    return ExprNodes.IteratorNode(pos, sequence = expr)

def p_try_statement(s):
    # s.sy == 'try'
    pos = s.position()
    s.next()
    body = p_suite(s)
    except_clauses = []
    else_clause = None
    if s.sy in ('except', 'else'):
        while s.sy == 'except':
            except_clauses.append(p_except_clause(s))
        if s.sy == 'else':
            s.next()
            else_clause = p_suite(s)
1681
        body = Nodes.TryExceptStatNode(pos,
William Stein's avatar
William Stein committed
1682 1683
            body = body, except_clauses = except_clauses,
            else_clause = else_clause)
1684 1685 1686 1687
        if s.sy != 'finally':
            return body
        # try-except-finally is equivalent to nested try-except/try-finally
    if s.sy == 'finally':
William Stein's avatar
William Stein committed
1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700
        s.next()
        finally_clause = p_suite(s)
        return Nodes.TryFinallyStatNode(pos,
            body = body, finally_clause = finally_clause)
    else:
        s.error("Expected 'except' or 'finally'")

def p_except_clause(s):
    # s.sy == 'except'
    pos = s.position()
    s.next()
    exc_type = None
    exc_value = None
1701
    is_except_as = False
Stefan Behnel's avatar
Stefan Behnel committed
1702
    if s.sy != ':':
1703
        exc_type = p_test(s)
1704 1705 1706 1707 1708
        # normalise into list of single exception tests
        if isinstance(exc_type, ExprNodes.TupleNode):
            exc_type = exc_type.args
        else:
            exc_type = [exc_type]
1709 1710
        if s.sy == ',' or (s.sy == 'IDENT' and s.systring == 'as'
                           and s.context.language_level == 2):
William Stein's avatar
William Stein committed
1711
            s.next()
1712
            exc_value = p_test(s)
1713
        elif s.sy == 'IDENT' and s.systring == 'as':
1714
            # Py3 syntax requires a name here
1715
            s.next()
1716 1717 1718
            pos2 = s.position()
            name = p_ident(s)
            exc_value = ExprNodes.NameNode(pos2, name = name)
1719
            is_except_as = True
William Stein's avatar
William Stein committed
1720 1721
    body = p_suite(s)
    return Nodes.ExceptClauseNode(pos,
1722 1723
        pattern = exc_type, target = exc_value,
        body = body, is_except_as=is_except_as)
William Stein's avatar
William Stein committed
1724

1725
def p_include_statement(s, ctx):
William Stein's avatar
William Stein committed
1726 1727
    pos = s.position()
    s.next() # 'include'
1728
    unicode_include_file_name = p_string_literal(s, 'u')[2]
William Stein's avatar
William Stein committed
1729
    s.expect_newline("Syntax error in include statement")
1730
    if s.compile_time_eval:
1731
        include_file_name = unicode_include_file_name
1732 1733
        include_file_path = s.context.find_include_file(include_file_name, pos)
        if include_file_path:
1734
            s.included_files.append(include_file_name)
1735
            f = Utils.open_source_file(include_file_path, mode="rU")
1736
            source_desc = FileSourceDescriptor(include_file_path)
1737
            s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding, parse_comments=s.parse_comments)
1738
            try:
1739
                tree = p_statement_list(s2, ctx)
1740 1741 1742 1743 1744
            finally:
                f.close()
            return tree
        else:
            return None
William Stein's avatar
William Stein committed
1745
    else:
1746 1747 1748 1749
        return Nodes.PassStatNode(pos)

def p_with_statement(s):
    s.next() # 'with'
1750
    if s.systring == 'template' and not s.in_python_file:
1751 1752 1753 1754 1755 1756 1757
        node = p_with_template(s)
    else:
        node = p_with_items(s)
    return node

def p_with_items(s):
    pos = s.position()
1758
    if not s.in_python_file and s.sy == 'IDENT' and s.systring in ('nogil', 'gil'):
1759 1760
        state = s.systring
        s.next()
1761
        if s.sy == ',':
Danilo Freitas's avatar
Danilo Freitas committed
1762
            s.next()
1763
            body = p_with_items(s)
Danilo Freitas's avatar
Danilo Freitas committed
1764
        else:
1765 1766
            body = p_suite(s)
        return Nodes.GILStatNode(pos, state = state, body = body)
1767
    else:
1768
        manager = p_test(s)
1769 1770 1771
        target = None
        if s.sy == 'IDENT' and s.systring == 'as':
            s.next()
1772 1773 1774 1775 1776 1777
            target = p_starred_expr(s)
        if s.sy == ',':
            s.next()
            body = p_with_items(s)
        else:
            body = p_suite(s)
1778
    return Nodes.WithStatNode(pos, manager = manager,
Robert Bradshaw's avatar
Robert Bradshaw committed
1779
                              target = target, body = body)
1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804

def p_with_template(s):
    pos = s.position()
    templates = []
    s.next()
    s.expect('[')
    templates.append(s.systring)
    s.next()
    while s.systring == ',':
        s.next()
        templates.append(s.systring)
        s.next()
    s.expect(']')
    if s.sy == ':':
        s.next()
        s.expect_newline("Syntax error in template function declaration")
        s.expect_indent()
        body_ctx = Ctx()
        body_ctx.templates = templates
        func_or_var = p_c_func_or_var_declaration(s, pos, body_ctx)
        s.expect_dedent()
        return func_or_var
    else:
        error(pos, "Syntax error in template function declaration")

Stefan Behnel's avatar
Stefan Behnel committed
1805
def p_simple_statement(s, first_statement = 0):
William Stein's avatar
William Stein committed
1806 1807 1808
    #print "p_simple_statement:", s.sy, s.systring ###
    if s.sy == 'global':
        node = p_global_statement(s)
1809 1810
    elif s.sy == 'nonlocal':
        node = p_nonlocal_statement(s)
William Stein's avatar
William Stein committed
1811 1812
    elif s.sy == 'print':
        node = p_print_statement(s)
1813 1814
    elif s.sy == 'exec':
        node = p_exec_statement(s)
William Stein's avatar
William Stein committed
1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827
    elif s.sy == 'del':
        node = p_del_statement(s)
    elif s.sy == 'break':
        node = p_break_statement(s)
    elif s.sy == 'continue':
        node = p_continue_statement(s)
    elif s.sy == 'return':
        node = p_return_statement(s)
    elif s.sy == 'raise':
        node = p_raise_statement(s)
    elif s.sy in ('import', 'cimport'):
        node = p_import_statement(s)
    elif s.sy == 'from':
Stefan Behnel's avatar
Stefan Behnel committed
1828
        node = p_from_import_statement(s, first_statement = first_statement)
1829
    elif s.sy == 'yield':
1830
        node = p_yield_statement(s)
William Stein's avatar
William Stein committed
1831 1832 1833 1834 1835 1836 1837 1838
    elif s.sy == 'assert':
        node = p_assert_statement(s)
    elif s.sy == 'pass':
        node = p_pass_statement(s)
    else:
        node = p_expression_or_assignment(s)
    return node

1839
def p_simple_statement_list(s, ctx, first_statement = 0):
William Stein's avatar
William Stein committed
1840 1841
    # Parse a series of simple statements on one line
    # separated by semicolons.
Stefan Behnel's avatar
Stefan Behnel committed
1842
    stat = p_simple_statement(s, first_statement = first_statement)
1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863
    pos = stat.pos
    stats = []
    if not isinstance(stat, Nodes.PassStatNode):
        stats.append(stat)
    while s.sy == ';':
        #print "p_simple_statement_list: maybe more to follow" ###
        s.next()
        if s.sy in ('NEWLINE', 'EOF'):
            break
        stat = p_simple_statement(s, first_statement = first_statement)
        if isinstance(stat, Nodes.PassStatNode):
            continue
        stats.append(stat)
        first_statement = False

    if not stats:
        stat = Nodes.PassStatNode(pos)
    elif len(stats) == 1:
        stat = stats[0]
    else:
        stat = Nodes.StatListNode(pos, stats = stats)
William Stein's avatar
William Stein committed
1864 1865 1866
    s.expect_newline("Syntax error in simple statement list")
    return stat

1867 1868 1869
def p_compile_time_expr(s):
    old = s.compile_time_expr
    s.compile_time_expr = 1
1870
    expr = p_testlist(s)
1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883
    s.compile_time_expr = old
    return expr

def p_DEF_statement(s):
    pos = s.position()
    denv = s.compile_time_env
    s.next() # 'DEF'
    name = p_ident(s)
    s.expect('=')
    expr = p_compile_time_expr(s)
    value = expr.compile_time_value(denv)
    #print "p_DEF_statement: %s = %r" % (name, value) ###
    denv.declare(name, value)
1884
    s.expect_newline("Expected a newline", ignore_semicolon=True)
1885 1886
    return Nodes.PassStatNode(pos)

1887
def p_IF_statement(s, ctx):
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
1888
    pos = s.position()
1889 1890 1891 1892 1893 1894 1895 1896
    saved_eval = s.compile_time_eval
    current_eval = saved_eval
    denv = s.compile_time_env
    result = None
    while 1:
        s.next() # 'IF' or 'ELIF'
        expr = p_compile_time_expr(s)
        s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv))
1897
        body = p_suite(s, ctx)
1898 1899 1900
        if s.compile_time_eval:
            result = body
            current_eval = 0
Stefan Behnel's avatar
Stefan Behnel committed
1901
        if s.sy != 'ELIF':
1902 1903 1904 1905
            break
    if s.sy == 'ELSE':
        s.next()
        s.compile_time_eval = current_eval
1906
        body = p_suite(s, ctx)
1907 1908 1909
        if current_eval:
            result = body
    if not result:
Stefan Behnel's avatar
Stefan Behnel committed
1910
        result = Nodes.PassStatNode(pos)
1911 1912 1913
    s.compile_time_eval = saved_eval
    return result

1914 1915
def p_statement(s, ctx, first_statement = 0):
    cdef_flag = ctx.cdef_flag
Robert Bradshaw's avatar
Robert Bradshaw committed
1916
    decorators = None
William Stein's avatar
William Stein committed
1917
    if s.sy == 'ctypedef':
1918
        if ctx.level not in ('module', 'module_pxd'):
William Stein's avatar
William Stein committed
1919
            s.error("ctypedef statement not allowed here")
1920 1921
        #if ctx.api:
        #    error(s.position(), "'api' not allowed with 'ctypedef'")
1922
        return p_ctypedef_statement(s, ctx)
1923 1924 1925
    elif s.sy == 'DEF':
        return p_DEF_statement(s)
    elif s.sy == 'IF':
1926
        return p_IF_statement(s, ctx)
1927
    elif s.sy == '@':
Haoyu Bai's avatar
Haoyu Bai committed
1928
        if ctx.level not in ('module', 'class', 'c_class', 'function', 'property', 'module_pxd', 'c_class_pxd', 'other'):
1929 1930 1931
            s.error('decorator not allowed here')
        s.level = ctx.level
        decorators = p_decorators(s)
1932 1933
        bad_toks =  'def', 'cdef', 'cpdef', 'class'
        if not ctx.allow_struct_enum_decorator and s.sy not in bad_toks:
1934
            s.error("Decorators can only be followed by functions or classes")
1935 1936 1937
    elif s.sy == 'pass' and cdef_flag:
        # empty cdef block
        return p_pass_statement(s, with_newline = 1)
1938 1939 1940 1941 1942

    overridable = 0
    if s.sy == 'cdef':
        cdef_flag = 1
        s.next()
1943
    elif s.sy == 'cpdef':
1944 1945 1946 1947 1948 1949 1950 1951 1952
        cdef_flag = 1
        overridable = 1
        s.next()
    if cdef_flag:
        if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):
            s.error('cdef statement not allowed here')
        s.level = ctx.level
        node = p_cdef_statement(s, ctx(overridable = overridable))
        if decorators is not None:
1953 1954 1955 1956
            tup = Nodes.CFuncDefNode, Nodes.CVarDefNode, Nodes.CClassDefNode
            if ctx.allow_struct_enum_decorator:
                tup += Nodes.CStructOrUnionDefNode, Nodes.CEnumDefNode
            if not isinstance(node, tup):
1957
                s.error("Decorators can only be followed by functions or classes")
1958 1959
            node.decorators = decorators
        return node
William Stein's avatar
William Stein committed
1960
    else:
1961
        if ctx.api:
1962
            s.error("'api' not allowed with this statement", fatal=False)
1963
        elif s.sy == 'def':
1964 1965 1966
            # def statements aren't allowed in pxd files, except
            # as part of a cdef class
            if ('pxd' in ctx.level) and (ctx.level != 'c_class_pxd'):
1967
                s.error('def statement not allowed here')
1968
            s.level = ctx.level
1969 1970
            return p_def_statement(s, decorators)
        elif s.sy == 'class':
1971
            if ctx.level not in ('module', 'function', 'class', 'other'):
1972
                s.error("class definition not allowed here")
1973
            return p_class_statement(s, decorators)
1974 1975 1976 1977 1978 1979 1980
        elif s.sy == 'include':
            if ctx.level not in ('module', 'module_pxd'):
                s.error("include statement not allowed here")
            return p_include_statement(s, ctx)
        elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':
            return p_property_decl(s)
        elif s.sy == 'pass' and ctx.level != 'property':
1981
            return p_pass_statement(s, with_newline=True)
William Stein's avatar
William Stein committed
1982
        else:
1983
            if ctx.level in ('c_class_pxd', 'property'):
1984 1985 1986
                node = p_ignorable_statement(s)
                if node is not None:
                    return node
1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997
                s.error("Executable statement not allowed here")
            if s.sy == 'if':
                return p_if_statement(s)
            elif s.sy == 'while':
                return p_while_statement(s)
            elif s.sy == 'for':
                return p_for_statement(s)
            elif s.sy == 'try':
                return p_try_statement(s)
            elif s.sy == 'with':
                return p_with_statement(s)
1998
            else:
1999 2000
                return p_simple_statement_list(
                    s, ctx, first_statement = first_statement)
William Stein's avatar
William Stein committed
2001

2002
def p_statement_list(s, ctx, first_statement = 0):
William Stein's avatar
William Stein committed
2003 2004 2005 2006
    # Parse a series of statements separated by newlines.
    pos = s.position()
    stats = []
    while s.sy not in ('DEDENT', 'EOF'):
2007 2008 2009 2010 2011 2012 2013 2014
        stat = p_statement(s, ctx, first_statement = first_statement)
        if isinstance(stat, Nodes.PassStatNode):
            continue
        stats.append(stat)
        first_statement = False
    if not stats:
        return Nodes.PassStatNode(pos)
    elif len(stats) == 1:
2015 2016 2017
        return stats[0]
    else:
        return Nodes.StatListNode(pos, stats = stats)
William Stein's avatar
William Stein committed
2018

2019 2020 2021 2022 2023 2024

def p_suite(s, ctx=Ctx()):
    return p_suite_with_docstring(s, ctx, with_doc_only=False)[1]


def p_suite_with_docstring(s, ctx, with_doc_only=False):
William Stein's avatar
William Stein committed
2025 2026 2027 2028 2029
    s.expect(':')
    doc = None
    if s.sy == 'NEWLINE':
        s.next()
        s.expect_indent()
2030
        if with_doc_only:
2031
            doc = p_doc_string(s)
2032
        body = p_statement_list(s, ctx)
William Stein's avatar
William Stein committed
2033 2034
        s.expect_dedent()
    else:
2035
        if ctx.api:
2036
            s.error("'api' not allowed with this statement", fatal=False)
2037 2038
        if ctx.level in ('module', 'class', 'function', 'other'):
            body = p_simple_statement_list(s, ctx)
William Stein's avatar
William Stein committed
2039 2040
        else:
            body = p_pass_statement(s)
2041
            s.expect_newline("Syntax error in declarations", ignore_semicolon=True)
2042 2043 2044 2045
    if not with_doc_only:
        doc, body = _extract_docstring(body)
    return doc, body

William Stein's avatar
William Stein committed
2046

2047
def p_positional_and_keyword_args(s, end_sy_set, templates = None):
2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060
    """
    Parses positional and keyword arguments. end_sy_set
    should contain any s.sy that terminate the argument list.
    Argument expansion (* and **) are not allowed.

    Returns: (positional_args, keyword_args)
    """
    positional_args = []
    keyword_args = []
    pos_idx = 0

    while s.sy not in end_sy_set:
        if s.sy == '*' or s.sy == '**':
2061
            s.error('Argument expansion not allowed here.', fatal=False)
2062 2063

        parsed_type = False
2064
        if s.sy == 'IDENT' and s.peek()[0] == '=':
2065
            ident = s.systring
2066
            s.next() # s.sy is '='
2067
            s.next()
2068
            if looking_at_expr(s):
2069
                arg = p_test(s)
2070 2071
            else:
                base_type = p_c_base_type(s, templates = templates)
2072
                declarator = p_c_declarator(s, empty = 1)
2073
                arg = Nodes.CComplexBaseTypeNode(base_type.pos,
2074 2075
                    base_type = base_type, declarator = declarator)
                parsed_type = True
2076
            keyword_node = ExprNodes.IdentifierStringNode(arg.pos, value=ident)
2077 2078
            keyword_args.append((keyword_node, arg))
            was_keyword = True
2079

2080
        else:
2081
            if looking_at_expr(s):
2082
                arg = p_test(s)
2083 2084
            else:
                base_type = p_c_base_type(s, templates = templates)
2085
                declarator = p_c_declarator(s, empty = 1)
2086
                arg = Nodes.CComplexBaseTypeNode(base_type.pos,
2087
                    base_type = base_type, declarator = declarator)
2088 2089 2090 2091 2092
                parsed_type = True
            positional_args.append(arg)
            pos_idx += 1
            if len(keyword_args) > 0:
                s.error("Non-keyword arg following keyword arg",
2093
                        pos=arg.pos)
2094 2095 2096 2097

        if s.sy != ',':
            if s.sy not in end_sy_set:
                if parsed_type:
2098
                    s.error("Unmatched %s" % " or ".join(end_sy_set))
2099 2100 2101 2102
            break
        s.next()
    return positional_args, keyword_args

Danilo Freitas's avatar
Danilo Freitas committed
2103
def p_c_base_type(s, self_flag = 0, nonempty = 0, templates = None):
William Stein's avatar
William Stein committed
2104 2105 2106
    # If self_flag is true, this is the base type for the
    # self argument of a C method of an extension type.
    if s.sy == '(':
2107
        return p_c_complex_base_type(s, templates = templates)
William Stein's avatar
William Stein committed
2108
    else:
Danilo Freitas's avatar
Danilo Freitas committed
2109
        return p_c_simple_base_type(s, self_flag, nonempty = nonempty, templates = templates)
William Stein's avatar
William Stein committed
2110

2111 2112 2113 2114 2115 2116 2117 2118
def p_calling_convention(s):
    if s.sy == 'IDENT' and s.systring in calling_convention_words:
        result = s.systring
        s.next()
        return result
    else:
        return ""

Stefan Behnel's avatar
Stefan Behnel committed
2119

2120 2121
calling_convention_words = cython.declare(
    set, set(["__stdcall", "__cdecl", "__fastcall"]))
2122

Stefan Behnel's avatar
Stefan Behnel committed
2123

2124
def p_c_complex_base_type(s, templates = None):
William Stein's avatar
William Stein committed
2125 2126 2127
    # s.sy == '('
    pos = s.position()
    s.next()
Stefan Behnel's avatar
Stefan Behnel committed
2128 2129 2130 2131
    base_type = p_c_base_type(s, templates=templates)
    declarator = p_c_declarator(s, empty=True)
    type_node = Nodes.CComplexBaseTypeNode(
        pos, base_type=base_type, declarator=declarator)
2132 2133 2134 2135 2136 2137
    if s.sy == ',':
        components = [type_node]
        while s.sy == ',':
            s.next()
            if s.sy == ')':
                break
Stefan Behnel's avatar
Stefan Behnel committed
2138 2139 2140 2141
            base_type = p_c_base_type(s, templates=templates)
            declarator = p_c_declarator(s, empty=True)
            components.append(Nodes.CComplexBaseTypeNode(
                pos, base_type=base_type, declarator=declarator))
2142 2143 2144
        type_node = Nodes.CTupleBaseTypeNode(pos, components = components)

    s.expect(')')
2145 2146 2147 2148 2149 2150
    if s.sy == '[':
        if is_memoryviewslice_access(s):
            type_node = p_memoryviewslice_access(s, type_node)
        else:
            type_node = p_buffer_or_template(s, type_node, templates)
    return type_node
2151

William Stein's avatar
William Stein committed
2152

Danilo Freitas's avatar
Danilo Freitas committed
2153
def p_c_simple_base_type(s, self_flag, nonempty, templates = None):
2154
    #print "p_c_simple_base_type: self_flag =", self_flag, nonempty
William Stein's avatar
William Stein committed
2155 2156 2157
    is_basic = 0
    signed = 1
    longness = 0
2158
    complex = 0
William Stein's avatar
William Stein committed
2159
    module_path = []
2160
    pos = s.position()
2161 2162
    if not s.sy == 'IDENT':
        error(pos, "Expected an identifier, found '%s'" % s.sy)
Robert Bradshaw's avatar
Robert Bradshaw committed
2163 2164 2165 2166 2167
    if s.systring == 'const':
        s.next()
        base_type = p_c_base_type(s,
            self_flag = self_flag, nonempty = nonempty, templates = templates)
        return Nodes.CConstTypeNode(pos, base_type = base_type)
William Stein's avatar
William Stein committed
2168 2169 2170
    if looking_at_base_type(s):
        #print "p_c_simple_base_type: looking_at_base_type at", s.position()
        is_basic = 1
2171 2172
        if s.sy == 'IDENT' and s.systring in special_basic_c_types:
            signed, longness = special_basic_c_types[s.systring]
William Stein's avatar
William Stein committed
2173 2174 2175
            name = s.systring
            s.next()
        else:
2176 2177 2178 2179 2180
            signed, longness = p_sign_and_longness(s)
            if s.sy == 'IDENT' and s.systring in basic_c_type_names:
                name = s.systring
                s.next()
            else:
Stefan Behnel's avatar
Stefan Behnel committed
2181
                name = 'int'  # long [int], short [int], long [int] complex, etc.
2182 2183 2184
        if s.sy == 'IDENT' and s.systring == 'complex':
            complex = 1
            s.next()
2185 2186
    elif looking_at_dotted_name(s):
        #print "p_c_simple_base_type: looking_at_type_name at", s.position()
2187
        name = s.systring
2188 2189 2190 2191 2192
        s.next()
        while s.sy == '.':
            module_path.append(name)
            s.next()
            name = p_ident(s)
2193
    else:
2194 2195 2196
        name = s.systring
        s.next()
        if nonempty and s.sy != 'IDENT':
2197
            # Make sure this is not a declaration of a variable or function.
2198 2199
            if s.sy == '(':
                s.next()
2200 2201
                if (s.sy == '*' or s.sy == '**' or s.sy == '&'
                        or (s.sy == 'IDENT' and s.systring in calling_convention_words)):
2202 2203 2204 2205 2206
                    s.put_back('(', '(')
                else:
                    s.put_back('(', '(')
                    s.put_back('IDENT', name)
                    name = None
Robert Bradshaw's avatar
Robert Bradshaw committed
2207
            elif s.sy not in ('*', '**', '[', '&'):
2208 2209
                s.put_back('IDENT', name)
                name = None
Danilo Freitas's avatar
Danilo Freitas committed
2210

2211
    type_node = Nodes.CSimpleBaseTypeNode(pos,
William Stein's avatar
William Stein committed
2212 2213
        name = name, module_path = module_path,
        is_basic_c_type = is_basic, signed = signed,
2214
        complex = complex, longness = longness,
Danilo Freitas's avatar
Danilo Freitas committed
2215
        is_self_arg = self_flag, templates = templates)
William Stein's avatar
William Stein committed
2216

2217
    #    declarations here.
2218
    if s.sy == '[':
2219
        if is_memoryviewslice_access(s):
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
2220
            type_node = p_memoryviewslice_access(s, type_node)
2221 2222
        else:
            type_node = p_buffer_or_template(s, type_node, templates)
2223

Robert Bradshaw's avatar
Robert Bradshaw committed
2224 2225 2226 2227
    if s.sy == '.':
        s.next()
        name = p_ident(s)
        type_node = Nodes.CNestedBaseTypeNode(pos, base_type = type_node, name = name)
2228

Robert Bradshaw's avatar
Robert Bradshaw committed
2229
    return type_node
2230

2231
def p_buffer_or_template(s, base_type_node, templates):
2232 2233 2234
    # s.sy == '['
    pos = s.position()
    s.next()
2235 2236
    # Note that buffer_positional_options_count=1, so the only positional argument is dtype.
    # For templated types, all parameters are types.
2237
    positional_args, keyword_args = (
2238
        p_positional_and_keyword_args(s, (']',), templates)
2239 2240
    )
    s.expect(']')
2241

2242 2243
    if s.sy == '[':
        base_type_node = p_buffer_or_template(s, base_type_node, templates)
2244 2245 2246 2247 2248 2249

    keyword_dict = ExprNodes.DictNode(pos,
        key_value_pairs = [
            ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)
            for key, value in keyword_args
        ])
2250
    result = Nodes.TemplatedTypeNode(pos,
2251 2252
        positional_args = positional_args,
        keyword_args = keyword_dict,
2253
        base_type_node = base_type_node)
2254
    return result
2255

2256 2257 2258 2259 2260 2261
def p_bracketed_base_type(s, base_type_node, nonempty, empty):
    # s.sy == '['
    if empty and not nonempty:
        # sizeof-like thing.  Only anonymous C arrays allowed (int[SIZE]).
        return base_type_node
    elif not empty and nonempty:
2262 2263 2264
        # declaration of either memoryview slice or buffer.
        if is_memoryviewslice_access(s):
            return p_memoryviewslice_access(s, base_type_node)
2265
        else:
Mark Florisson's avatar
Mark Florisson committed
2266 2267
            return p_buffer_or_template(s, base_type_node, None)
            # return p_buffer_access(s, base_type_node)
2268
    elif not empty and not nonempty:
2269 2270 2271 2272 2273
        # only anonymous C arrays and memoryview slice arrays here.  We
        # disallow buffer declarations for now, due to ambiguity with anonymous
        # C arrays.
        if is_memoryviewslice_access(s):
            return p_memoryviewslice_access(s, base_type_node)
2274 2275 2276
        else:
            return base_type_node

2277
def is_memoryviewslice_access(s):
2278
    # s.sy == '['
2279
    # a memoryview slice declaration is distinguishable from a buffer access
2280
    # declaration by the first entry in the bracketed list.  The buffer will
2281
    # not have an unnested colon in the first entry; the memoryview slice will.
2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292
    saved = [(s.sy, s.systring)]
    s.next()
    retval = False
    if s.systring == ':':
        retval = True
    elif s.sy == 'INT':
        saved.append((s.sy, s.systring))
        s.next()
        if s.sy == ':':
            retval = True

2293
    for sv in saved[::-1]:
2294 2295 2296 2297
        s.put_back(*sv)

    return retval

2298
def p_memoryviewslice_access(s, base_type_node):
2299 2300 2301
    # s.sy == '['
    pos = s.position()
    s.next()
2302
    subscripts, _ = p_subscript_list(s)
2303 2304 2305 2306 2307 2308
    # make sure each entry in subscripts is a slice
    for subscript in subscripts:
        if len(subscript) < 2:
            s.error("An axis specification in memoryview declaration does not have a ':'.")
    s.expect(']')
    indexes = make_slice_nodes(pos, subscripts)
2309
    result = Nodes.MemoryViewSliceTypeNode(pos,
2310 2311 2312
            base_type_node = base_type_node,
            axes = indexes)
    return result
2313

2314
def looking_at_name(s):
2315 2316
    return s.sy == 'IDENT' and not s.systring in calling_convention_words

2317
def looking_at_expr(s):
2318
    if s.systring in base_type_start_words:
2319
        return False
2320 2321 2322 2323 2324
    elif s.sy == 'IDENT':
        is_type = False
        name = s.systring
        dotted_path = []
        s.next()
2325

2326 2327 2328 2329
        while s.sy == '.':
            s.next()
            dotted_path.append(s.systring)
            s.expect('IDENT')
2330

2331
        saved = s.sy, s.systring
2332 2333 2334
        if s.sy == 'IDENT':
            is_type = True
        elif s.sy == '*' or s.sy == '**':
2335
            s.next()
2336
            is_type = s.sy in (')', ']')
2337 2338 2339 2340 2341 2342 2343 2344 2345
            s.put_back(*saved)
        elif s.sy == '(':
            s.next()
            is_type = s.sy == '*'
            s.put_back(*saved)
        elif s.sy == '[':
            s.next()
            is_type = s.sy == ']'
            s.put_back(*saved)
2346

2347 2348 2349 2350
        dotted_path.reverse()
        for p in dotted_path:
            s.put_back('IDENT', p)
            s.put_back('.', '.')
2351

2352
        s.put_back('IDENT', name)
2353
        return not is_type and saved[0]
2354
    else:
2355
        return True
William Stein's avatar
William Stein committed
2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369

def looking_at_base_type(s):
    #print "looking_at_base_type?", s.sy, s.systring, s.position()
    return s.sy == 'IDENT' and s.systring in base_type_start_words

def looking_at_dotted_name(s):
    if s.sy == 'IDENT':
        name = s.systring
        s.next()
        result = s.sy == '.'
        s.put_back('IDENT', name)
        return result
    else:
        return 0
2370

2371 2372 2373 2374 2375 2376
def looking_at_call(s):
    "See if we're looking at a.b.c("
    # Don't mess up the original position, so save and restore it.
    # Unfortunately there's no good way to handle this, as a subsequent call
    # to next() will not advance the position until it reads a new token.
    position = s.start_line, s.start_col
Mark Florisson's avatar
Mark Florisson committed
2377
    result = looking_at_expr(s) == u'('
2378 2379 2380 2381
    if not result:
        s.start_line, s.start_col = position
    return result

2382 2383
basic_c_type_names = cython.declare(
    set, set(["void", "char", "int", "float", "double", "bint"]))
2384

2385
special_basic_c_types = cython.declare(dict, {
2386
    # name : (signed, longness)
2387
    "Py_UNICODE" : (0, 0),
Stefan Behnel's avatar
Stefan Behnel committed
2388
    "Py_UCS4"    : (0, 0),
Stefan Behnel's avatar
Stefan Behnel committed
2389
    "Py_hash_t"  : (2, 0),
2390
    "Py_ssize_t" : (2, 0),
2391
    "ssize_t"    : (2, 0),
2392
    "size_t"     : (0, 0),
2393
    "ptrdiff_t"  : (2, 0),
2394
})
William Stein's avatar
William Stein committed
2395

2396 2397
sign_and_longness_words = cython.declare(
    set, set(["short", "long", "signed", "unsigned"]))
William Stein's avatar
William Stein committed
2398

2399 2400 2401 2402 2403
base_type_start_words = cython.declare(
    set,
    basic_c_type_names
    | sign_and_longness_words
    | set(special_basic_c_types))
William Stein's avatar
William Stein committed
2404

2405 2406
struct_enum_union = cython.declare(
    set, set(["struct", "union", "enum", "packed"]))
Mark Florisson's avatar
Mark Florisson committed
2407

William Stein's avatar
William Stein committed
2408 2409 2410 2411 2412 2413
def p_sign_and_longness(s):
    signed = 1
    longness = 0
    while s.sy == 'IDENT' and s.systring in sign_and_longness_words:
        if s.systring == 'unsigned':
            signed = 0
2414 2415
        elif s.systring == 'signed':
            signed = 2
William Stein's avatar
William Stein committed
2416 2417 2418 2419 2420 2421 2422 2423
        elif s.systring == 'short':
            longness = -1
        elif s.systring == 'long':
            longness += 1
        s.next()
    return signed, longness

def p_opt_cname(s):
2424 2425 2426
    literal = p_opt_string_literal(s, 'u')
    if literal is not None:
        cname = EncodedString(literal)
Stefan Behnel's avatar
Stefan Behnel committed
2427
        cname.encoding = s.source_encoding
William Stein's avatar
William Stein committed
2428 2429 2430 2431
    else:
        cname = None
    return cname

2432 2433 2434
def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0,
                   assignable = 0, nonempty = 0,
                   calling_convention_allowed = 0):
2435 2436
    # If empty is true, the declarator must be empty. If nonempty is true,
    # the declarator must be nonempty. Otherwise we don't care.
William Stein's avatar
William Stein committed
2437 2438 2439
    # If cmethod_flag is true, then if this declarator declares
    # a function, it's a C method of an extension type.
    pos = s.position()
2440 2441
    if s.sy == '(':
        s.next()
2442
        if s.sy == ')' or looking_at_name(s):
2443
            base = Nodes.CNameDeclaratorNode(pos, name=s.context.intern_ustring(u""), cname=None)
2444
            result = p_c_func_declarator(s, pos, ctx, base, cmethod_flag)
2445
        else:
2446 2447 2448 2449
            result = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
                                    cmethod_flag = cmethod_flag,
                                    nonempty = nonempty,
                                    calling_convention_allowed = 1)
2450 2451
            s.expect(')')
    else:
2452 2453
        result = p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
                                       assignable, nonempty)
Stefan Behnel's avatar
Stefan Behnel committed
2454
    if not calling_convention_allowed and result.calling_convention and s.sy != '(':
2455 2456 2457 2458 2459 2460 2461 2462
        error(s.position(), "%s on something that is not a function"
            % result.calling_convention)
    while s.sy in ('[', '('):
        pos = s.position()
        if s.sy == '[':
            result = p_c_array_declarator(s, result)
        else: # sy == '('
            s.next()
2463
            result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag)
2464 2465 2466 2467 2468 2469
        cmethod_flag = 0
    return result

def p_c_array_declarator(s, base):
    pos = s.position()
    s.next() # '['
Stefan Behnel's avatar
Stefan Behnel committed
2470
    if s.sy != ']':
2471
        dim = p_testlist(s)
2472 2473 2474 2475 2476
    else:
        dim = None
    s.expect(']')
    return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim)

root's avatar
root committed
2477
def p_c_func_declarator(s, pos, ctx, base, cmethod_flag):
2478
    #  Opening paren has already been skipped
2479 2480
    args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag,
                        nonempty_declarators = 0)
2481 2482 2483 2484 2485
    ellipsis = p_optional_ellipsis(s)
    s.expect(')')
    nogil = p_nogil(s)
    exc_val, exc_check = p_exception_value_clause(s)
    with_gil = p_with_gil(s)
2486
    return Nodes.CFuncDeclaratorNode(pos,
2487 2488
        base = base, args = args, has_varargs = ellipsis,
        exception_value = exc_val, exception_check = exc_check,
2489
        nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil)
2490

2491
supported_overloaded_operators = cython.declare(set, set([
2492
    '+', '-', '*', '/', '%',
Robert Bradshaw's avatar
Robert Bradshaw committed
2493
    '++', '--', '~', '|', '&', '^', '<<', '>>', ',',
Robert Bradshaw's avatar
Robert Bradshaw committed
2494
    '==', '!=', '>=', '>', '<=', '<',
2495
    '[]', '()', '!',
2496
]))
2497

2498 2499
def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
                          assignable, nonempty):
2500 2501
    pos = s.position()
    calling_convention = p_calling_convention(s)
William Stein's avatar
William Stein committed
2502 2503
    if s.sy == '*':
        s.next()
2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516
        if s.systring == 'const':
            const_pos = s.position()
            s.next()
            const_base = p_c_declarator(s, ctx, empty = empty,
                                       is_type = is_type,
                                       cmethod_flag = cmethod_flag,
                                       assignable = assignable,
                                       nonempty = nonempty)
            base = Nodes.CConstDeclaratorNode(const_pos, base = const_base)
        else:
            base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
                                  cmethod_flag = cmethod_flag,
                                  assignable = assignable, nonempty = nonempty)
2517
        result = Nodes.CPtrDeclaratorNode(pos,
William Stein's avatar
William Stein committed
2518 2519 2520
            base = base)
    elif s.sy == '**': # scanner returns this as a single token
        s.next()
2521 2522 2523
        base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
                              cmethod_flag = cmethod_flag,
                              assignable = assignable, nonempty = nonempty)
William Stein's avatar
William Stein committed
2524 2525 2526
        result = Nodes.CPtrDeclaratorNode(pos,
            base = Nodes.CPtrDeclaratorNode(pos,
                base = base))
Robert Bradshaw's avatar
merge  
Robert Bradshaw committed
2527
    elif s.sy == '&':
2528 2529 2530 2531 2532
        s.next()
        base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
                              cmethod_flag = cmethod_flag,
                              assignable = assignable, nonempty = nonempty)
        result = Nodes.CReferenceDeclaratorNode(pos, base = base)
William Stein's avatar
William Stein committed
2533
    else:
2534 2535
        rhs = None
        if s.sy == 'IDENT':
2536
            name = s.systring
2537 2538
            if empty:
                error(s.position(), "Declarator should be empty")
William Stein's avatar
William Stein committed
2539
            s.next()
2540
            cname = p_opt_cname(s)
2541
            if name != 'operator' and s.sy == '=' and assignable:
2542
                s.next()
2543
                rhs = p_test(s)
William Stein's avatar
William Stein committed
2544
        else:
2545 2546 2547 2548
            if nonempty:
                error(s.position(), "Empty declarator")
            name = ""
            cname = None
2549
        if cname is None and ctx.namespace is not None and nonempty:
2550
            cname = ctx.namespace + "::" + name
2551
        if name == 'operator' and ctx.visibility == 'extern' and nonempty:
2552
            op = s.sy
2553
            if [1 for c in op if c in '+-*/<=>!%&|([^~,']:
2554
                s.next()
2555 2556 2557 2558 2559 2560 2561
                # Handle diphthong operators.
                if op == '(':
                    s.expect(')')
                    op = '()'
                elif op == '[':
                    s.expect(']')
                    op = '[]'
Stefan Behnel's avatar
Stefan Behnel committed
2562 2563
                elif op in ('-', '+', '|', '&') and s.sy == op:
                    op *= 2       # ++, --, ...
2564
                    s.next()
Stefan Behnel's avatar
Stefan Behnel committed
2565 2566
                elif s.sy == '=':
                    op += s.sy    # +=, -=, ...
2567 2568
                    s.next()
                if op not in supported_overloaded_operators:
2569 2570 2571
                    s.error("Overloading operator '%s' not yet supported." % op,
                            fatal=False)
                name += op
2572
        result = Nodes.CNameDeclaratorNode(pos,
Robert Bradshaw's avatar
Robert Bradshaw committed
2573
            name = name, cname = cname, default = rhs)
2574
    result.calling_convention = calling_convention
William Stein's avatar
William Stein committed
2575 2576
    return result

2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591
def p_nogil(s):
    if s.sy == 'IDENT' and s.systring == 'nogil':
        s.next()
        return 1
    else:
        return 0

def p_with_gil(s):
    if s.sy == 'with':
        s.next()
        s.expect_keyword('gil')
        return 1
    else:
        return 0

William Stein's avatar
William Stein committed
2592 2593 2594 2595 2596 2597 2598 2599
def p_exception_value_clause(s):
    exc_val = None
    exc_check = 0
    if s.sy == 'except':
        s.next()
        if s.sy == '*':
            exc_check = 1
            s.next()
Felix Wu's avatar
Felix Wu committed
2600 2601 2602
        elif s.sy == '+':
            exc_check = '+'
            s.next()
Robert Bradshaw's avatar
Robert Bradshaw committed
2603 2604 2605 2606
            if s.sy == 'IDENT':
                name = s.systring
                s.next()
                exc_val = p_name(s, name)
William Stein's avatar
William Stein committed
2607 2608 2609 2610
        else:
            if s.sy == '?':
                exc_check = 1
                s.next()
2611
            exc_val = p_test(s)
William Stein's avatar
William Stein committed
2612 2613
    return exc_val, exc_check

2614
c_arg_list_terminators = cython.declare(set, set(['*', '**', '.', ')']))
William Stein's avatar
William Stein committed
2615

2616
def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0,
2617
                 nonempty_declarators = 0, kw_only = 0, annotated = 1):
2618 2619
    #  Comma-separated list of C argument declarations, possibly empty.
    #  May have a trailing comma.
William Stein's avatar
William Stein committed
2620
    args = []
2621 2622
    is_self_arg = cmethod_flag
    while s.sy not in c_arg_list_terminators:
2623
        args.append(p_c_arg_decl(s, ctx, in_pyfunc, is_self_arg,
2624 2625
            nonempty = nonempty_declarators, kw_only = kw_only,
            annotated = annotated))
2626 2627 2628 2629
        if s.sy != ',':
            break
        s.next()
        is_self_arg = 0
William Stein's avatar
William Stein committed
2630 2631 2632 2633 2634 2635 2636 2637 2638
    return args

def p_optional_ellipsis(s):
    if s.sy == '.':
        expect_ellipsis(s)
        return 1
    else:
        return 0

2639 2640
def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0,
                 kw_only = 0, annotated = 1):
William Stein's avatar
William Stein committed
2641
    pos = s.position()
2642
    not_none = or_none = 0
William Stein's avatar
William Stein committed
2643
    default = None
2644
    annotation = None
2645 2646 2647 2648 2649 2650 2651 2652 2653
    if s.in_python_file:
        # empty type declaration
        base_type = Nodes.CSimpleBaseTypeNode(pos,
            name = None, module_path = [],
            is_basic_c_type = 0, signed = 0,
            complex = 0, longness = 0,
            is_self_arg = cmethod_flag, templates = None)
    else:
        base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty)
2654
    declarator = p_c_declarator(s, ctx, nonempty = nonempty)
2655 2656
    if s.sy in ('not', 'or') and not s.in_python_file:
        kind = s.sy
William Stein's avatar
William Stein committed
2657 2658 2659 2660 2661 2662
        s.next()
        if s.sy == 'IDENT' and s.systring == 'None':
            s.next()
        else:
            s.error("Expected 'None'")
        if not in_pyfunc:
2663 2664 2665
            error(pos, "'%s None' only allowed in Python functions" % kind)
        or_none = kind == 'or'
        not_none = kind == 'not'
2666 2667
    if annotated and s.sy == ':':
        s.next()
2668
        annotation = p_test(s)
William Stein's avatar
William Stein committed
2669 2670
    if s.sy == '=':
        s.next()
Stefan Behnel's avatar
Stefan Behnel committed
2671
        if 'pxd' in ctx.level:
2672 2673
            if s.sy not in ['*', '?']:
                error(pos, "default values cannot be specified in pxd files, use ? or *")
Robert Bradshaw's avatar
Robert Bradshaw committed
2674
            default = ExprNodes.BoolNode(1)
2675 2676
            s.next()
        else:
2677
            default = p_test(s)
William Stein's avatar
William Stein committed
2678 2679 2680 2681
    return Nodes.CArgDeclNode(pos,
        base_type = base_type,
        declarator = declarator,
        not_none = not_none,
2682
        or_none = or_none,
2683
        default = default,
2684
        annotation = annotation,
2685
        kw_only = kw_only)
William Stein's avatar
William Stein committed
2686

2687 2688 2689 2690 2691 2692 2693
def p_api(s):
    if s.sy == 'IDENT' and s.systring == 'api':
        s.next()
        return 1
    else:
        return 0

2694
def p_cdef_statement(s, ctx):
William Stein's avatar
William Stein committed
2695
    pos = s.position()
2696 2697 2698 2699 2700 2701 2702
    ctx.visibility = p_visibility(s, ctx.visibility)
    ctx.api = ctx.api or p_api(s)
    if ctx.api:
        if ctx.visibility not in ('private', 'public'):
            error(pos, "Cannot combine 'api' with '%s'" % ctx.visibility)
    if (ctx.visibility == 'extern') and s.sy == 'from':
        return p_cdef_extern_block(s, pos, ctx)
Robert Bradshaw's avatar
Robert Bradshaw committed
2703 2704
    elif s.sy == 'import':
        s.next()
2705
        return p_cdef_extern_block(s, pos, ctx)
2706
    elif p_nogil(s):
2707
        ctx.nogil = 1
2708 2709 2710 2711 2712 2713
        if ctx.overridable:
            error(pos, "cdef blocks cannot be declared cpdef")
        return p_cdef_block(s, ctx)
    elif s.sy == ':':
        if ctx.overridable:
            error(pos, "cdef blocks cannot be declared cpdef")
2714
        return p_cdef_block(s, ctx)
William Stein's avatar
William Stein committed
2715
    elif s.sy == 'class':
2716
        if ctx.level not in ('module', 'module_pxd'):
William Stein's avatar
William Stein committed
2717
            error(pos, "Extension type definition not allowed here")
2718 2719
        if ctx.overridable:
            error(pos, "Extension types cannot be declared cpdef")
2720
        return p_c_class_definition(s, pos, ctx)
Robert Bradshaw's avatar
Robert Bradshaw committed
2721 2722
    elif s.sy == 'IDENT' and s.systring == 'cppclass':
        return p_cpp_class_definition(s, pos, ctx)
Mark Florisson's avatar
Mark Florisson committed
2723
    elif s.sy == 'IDENT' and s.systring in struct_enum_union:
2724
        if ctx.level not in ('module', 'module_pxd'):
William Stein's avatar
William Stein committed
2725
            error(pos, "C struct/union/enum definition not allowed here")
2726
        if ctx.overridable:
2727 2728
            if s.systring != 'enum':
                error(pos, "C struct/union cannot be declared cpdef")
Mark Florisson's avatar
Mark Florisson committed
2729 2730 2731
        return p_struct_enum(s, pos, ctx)
    elif s.sy == 'IDENT' and s.systring == 'fused':
        return p_fused_definition(s, pos, ctx)
William Stein's avatar
William Stein committed
2732
    else:
2733
        return p_c_func_or_var_declaration(s, pos, ctx)
2734

2735 2736
def p_cdef_block(s, ctx):
    return p_suite(s, ctx(cdef_flag = 1))
William Stein's avatar
William Stein committed
2737

2738
def p_cdef_extern_block(s, pos, ctx):
2739 2740
    if ctx.overridable:
        error(pos, "cdef extern blocks cannot be declared cpdef")
William Stein's avatar
William Stein committed
2741 2742 2743 2744 2745
    include_file = None
    s.expect('from')
    if s.sy == '*':
        s.next()
    else:
2746
        include_file = p_string_literal(s, 'u')[2]
2747
    ctx = ctx(cdef_flag = 1, visibility = 'extern')
2748 2749
    if s.systring == "namespace":
        s.next()
2750
        ctx.namespace = p_string_literal(s, 'u')[2]
2751 2752 2753
    if p_nogil(s):
        ctx.nogil = 1
    body = p_suite(s, ctx)
William Stein's avatar
William Stein committed
2754 2755
    return Nodes.CDefExternNode(pos,
        include_file = include_file,
2756
        body = body,
Robert Bradshaw's avatar
Robert Bradshaw committed
2757
        namespace = ctx.namespace)
William Stein's avatar
William Stein committed
2758

2759
def p_c_enum_definition(s, pos, ctx):
William Stein's avatar
William Stein committed
2760 2761 2762 2763 2764 2765
    # s.sy == ident 'enum'
    s.next()
    if s.sy == 'IDENT':
        name = s.systring
        s.next()
        cname = p_opt_cname(s)
2766 2767
        if cname is None and ctx.namespace is not None:
            cname = ctx.namespace + "::" + name
William Stein's avatar
William Stein committed
2768 2769 2770 2771 2772 2773
    else:
        name = None
        cname = None
    items = None
    s.expect(':')
    items = []
Stefan Behnel's avatar
Stefan Behnel committed
2774
    if s.sy != 'NEWLINE':
2775
        p_c_enum_line(s, ctx, items)
William Stein's avatar
William Stein committed
2776 2777 2778 2779
    else:
        s.next() # 'NEWLINE'
        s.expect_indent()
        while s.sy not in ('DEDENT', 'EOF'):
2780
            p_c_enum_line(s, ctx, items)
William Stein's avatar
William Stein committed
2781
        s.expect_dedent()
2782 2783 2784
    return Nodes.CEnumDefNode(
        pos, name = name, cname = cname, items = items,
        typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,
2785
        create_wrapper = ctx.overridable,
2786
        api = ctx.api, in_pxd = ctx.level == 'module_pxd')
William Stein's avatar
William Stein committed
2787

2788
def p_c_enum_line(s, ctx, items):
Stefan Behnel's avatar
Stefan Behnel committed
2789
    if s.sy != 'pass':
2790
        p_c_enum_item(s, ctx, items)
William Stein's avatar
William Stein committed
2791 2792 2793 2794
        while s.sy == ',':
            s.next()
            if s.sy in ('NEWLINE', 'EOF'):
                break
2795
            p_c_enum_item(s, ctx, items)
William Stein's avatar
William Stein committed
2796 2797 2798 2799
    else:
        s.next()
    s.expect_newline("Syntax error in enum item list")

2800
def p_c_enum_item(s, ctx, items):
William Stein's avatar
William Stein committed
2801 2802 2803
    pos = s.position()
    name = p_ident(s)
    cname = p_opt_cname(s)
2804 2805
    if cname is None and ctx.namespace is not None:
        cname = ctx.namespace + "::" + name
William Stein's avatar
William Stein committed
2806 2807 2808
    value = None
    if s.sy == '=':
        s.next()
2809
        value = p_test(s)
2810
    items.append(Nodes.CEnumDefItemNode(pos,
William Stein's avatar
William Stein committed
2811 2812
        name = name, cname = cname, value = value))

2813
def p_c_struct_or_union_definition(s, pos, ctx):
2814 2815 2816 2817
    packed = False
    if s.systring == 'packed':
        packed = True
        s.next()
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
2818
        if s.sy != 'IDENT' or s.systring != 'struct':
2819
            s.expected('struct')
William Stein's avatar
William Stein committed
2820 2821 2822 2823 2824
    # s.sy == ident 'struct' or 'union'
    kind = s.systring
    s.next()
    name = p_ident(s)
    cname = p_opt_cname(s)
2825 2826
    if cname is None and ctx.namespace is not None:
        cname = ctx.namespace + "::" + name
William Stein's avatar
William Stein committed
2827 2828 2829 2830 2831 2832
    attributes = None
    if s.sy == ':':
        s.next()
        s.expect('NEWLINE')
        s.expect_indent()
        attributes = []
2833
        body_ctx = Ctx()
Stefan Behnel's avatar
Stefan Behnel committed
2834 2835
        while s.sy != 'DEDENT':
            if s.sy != 'pass':
William Stein's avatar
William Stein committed
2836
                attributes.append(
2837
                    p_c_func_or_var_declaration(s, s.position(), body_ctx))
William Stein's avatar
William Stein committed
2838 2839 2840 2841 2842 2843
            else:
                s.next()
                s.expect_newline("Expected a newline")
        s.expect_dedent()
    else:
        s.expect_newline("Syntax error in struct or union definition")
Robert Bradshaw's avatar
Robert Bradshaw committed
2844
    return Nodes.CStructOrUnionDefNode(pos,
William Stein's avatar
William Stein committed
2845
        name = name, cname = cname, kind = kind, attributes = attributes,
2846
        typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,
2847
        api = ctx.api, in_pxd = ctx.level == 'module_pxd', packed = packed)
William Stein's avatar
William Stein committed
2848

Mark Florisson's avatar
Mark Florisson committed
2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888
def p_fused_definition(s, pos, ctx):
    """
    c(type)def fused my_fused_type:
        ...
    """
    # s.systring == 'fused'

    if ctx.level not in ('module', 'module_pxd'):
        error(pos, "Fused type definition not allowed here")

    s.next()
    name = p_ident(s)

    s.expect(":")
    s.expect_newline()
    s.expect_indent()

    types = []
    while s.sy != 'DEDENT':
        if s.sy != 'pass':
            #types.append(p_c_declarator(s))
            types.append(p_c_base_type(s)) #, nonempty=1))
        else:
            s.next()

        s.expect_newline()

    s.expect_dedent()

    if not types:
        error(pos, "Need at least one type")

    return Nodes.FusedTypeNode(pos, name=name, types=types)

def p_struct_enum(s, pos, ctx):
    if s.systring == 'enum':
        return p_c_enum_definition(s, pos, ctx)
    else:
        return p_c_struct_or_union_definition(s, pos, ctx)

William Stein's avatar
William Stein committed
2889 2890 2891 2892 2893
def p_visibility(s, prev_visibility):
    pos = s.position()
    visibility = prev_visibility
    if s.sy == 'IDENT' and s.systring in ('extern', 'public', 'readonly'):
        visibility = s.systring
Stefan Behnel's avatar
Stefan Behnel committed
2894
        if prev_visibility != 'private' and visibility != prev_visibility:
William Stein's avatar
William Stein committed
2895
            s.error("Conflicting visibility options '%s' and '%s'"
2896
                % (prev_visibility, visibility), fatal=False)
William Stein's avatar
William Stein committed
2897 2898
        s.next()
    return visibility
2899

2900
def p_c_modifiers(s):
2901
    if s.sy == 'IDENT' and s.systring in ('inline',):
2902
        modifier = s.systring
2903
        s.next()
2904 2905
        return [modifier] + p_c_modifiers(s)
    return []
William Stein's avatar
William Stein committed
2906

2907 2908
def p_c_func_or_var_declaration(s, pos, ctx):
    cmethod_flag = ctx.level in ('c_class', 'c_class_pxd')
2909
    modifiers = p_c_modifiers(s)
Danilo Freitas's avatar
Danilo Freitas committed
2910
    base_type = p_c_base_type(s, nonempty = 1, templates = ctx.templates)
2911 2912 2913
    declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,
                                assignable = 1, nonempty = 1)
    declarator.overridable = ctx.overridable
Robert Bradshaw's avatar
Robert Bradshaw committed
2914 2915 2916 2917 2918
    if s.sy == 'IDENT' and s.systring == 'const' and ctx.level == 'cpp_class':
        s.next()
        is_const_method = 1
    else:
        is_const_method = 0
William Stein's avatar
William Stein committed
2919
    if s.sy == ':':
2920
        if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd', 'cpp_class') and not ctx.templates:
William Stein's avatar
William Stein committed
2921
            s.error("C function definition not allowed here")
2922
        doc, suite = p_suite_with_docstring(s, Ctx(level='function'))
William Stein's avatar
William Stein committed
2923
        result = Nodes.CFuncDefNode(pos,
2924
            visibility = ctx.visibility,
William Stein's avatar
William Stein committed
2925
            base_type = base_type,
2926
            declarator = declarator,
2927
            body = suite,
2928
            doc = doc,
2929
            modifiers = modifiers,
2930
            api = ctx.api,
Robert Bradshaw's avatar
Robert Bradshaw committed
2931 2932
            overridable = ctx.overridable,
            is_const_method = is_const_method)
William Stein's avatar
William Stein committed
2933
    else:
Stefan Behnel's avatar
Stefan Behnel committed
2934
        #if api:
2935
        #    s.error("'api' not allowed with variable declaration")
2936 2937
        if is_const_method:
            declarator.is_const_method = is_const_method
William Stein's avatar
William Stein committed
2938 2939 2940 2941 2942
        declarators = [declarator]
        while s.sy == ',':
            s.next()
            if s.sy == 'NEWLINE':
                break
2943 2944
            declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,
                                        assignable = 1, nonempty = 1)
William Stein's avatar
William Stein committed
2945
            declarators.append(declarator)
2946
        doc_line = s.start_line + 1
2947
        s.expect_newline("Syntax error in C variable declaration", ignore_semicolon=True)
Stefan Behnel's avatar
Stefan Behnel committed
2948
        if ctx.level in ('c_class', 'c_class_pxd') and s.start_line == doc_line:
2949 2950 2951
            doc = p_doc_string(s)
        else:
            doc = None
2952
        result = Nodes.CVarDefNode(pos,
2953 2954
            visibility = ctx.visibility,
            base_type = base_type,
2955
            declarators = declarators,
2956
            in_pxd = ctx.level in ('module_pxd', 'c_class_pxd'),
2957
            doc = doc,
2958
            api = ctx.api,
2959
            modifiers = modifiers,
2960
            overridable = ctx.overridable)
William Stein's avatar
William Stein committed
2961 2962
    return result

2963
def p_ctypedef_statement(s, ctx):
William Stein's avatar
William Stein committed
2964 2965 2966
    # s.sy == 'ctypedef'
    pos = s.position()
    s.next()
2967
    visibility = p_visibility(s, ctx.visibility)
2968
    api = p_api(s)
2969
    ctx = ctx(typedef_flag = 1, visibility = visibility)
2970 2971
    if api:
        ctx.api = 1
William Stein's avatar
William Stein committed
2972
    if s.sy == 'class':
2973
        return p_c_class_definition(s, pos, ctx)
Mark Florisson's avatar
Mark Florisson committed
2974 2975 2976 2977
    elif s.sy == 'IDENT' and s.systring in struct_enum_union:
        return p_struct_enum(s, pos, ctx)
    elif s.sy == 'IDENT' and s.systring == 'fused':
        return p_fused_definition(s, pos, ctx)
William Stein's avatar
William Stein committed
2978
    else:
2979
        base_type = p_c_base_type(s, nonempty = 1)
2980
        declarator = p_c_declarator(s, ctx, is_type = 1, nonempty = 1)
2981
        s.expect_newline("Syntax error in ctypedef statement", ignore_semicolon=True)
2982 2983
        return Nodes.CTypeDefNode(
            pos, base_type = base_type,
Robert Bradshaw's avatar
Robert Bradshaw committed
2984
            declarator = declarator,
2985
            visibility = visibility, api = api,
2986
            in_pxd = ctx.level == 'module_pxd')
William Stein's avatar
William Stein committed
2987

2988 2989
def p_decorators(s):
    decorators = []
2990
    while s.sy == '@':
2991 2992
        pos = s.position()
        s.next()
2993 2994
        decstring = p_dotted_name(s, as_allowed=0)[2]
        names = decstring.split('.')
2995
        decorator = ExprNodes.NameNode(pos, name=s.context.intern_ustring(names[0]))
2996
        for name in names[1:]:
2997 2998
            decorator = ExprNodes.AttributeNode(
                pos, attribute=s.context.intern_ustring(name), obj=decorator)
2999 3000 3001 3002 3003 3004 3005
        if s.sy == '(':
            decorator = p_call(s, decorator)
        decorators.append(Nodes.DecoratorNode(pos, decorator=decorator))
        s.expect_newline("Expected a newline after decorator")
    return decorators

def p_def_statement(s, decorators=None):
William Stein's avatar
William Stein committed
3006 3007 3008
    # s.sy == 'def'
    pos = s.position()
    s.next()
3009
    name = p_ident(s)
3010
    s.expect('(')
Stefan Behnel's avatar
Stefan Behnel committed
3011 3012 3013
    args, star_arg, starstar_arg = p_varargslist(s, terminator=')')
    s.expect(')')
    if p_nogil(s):
3014
        error(pos, "Python function cannot be declared nogil")
3015 3016 3017
    return_type_annotation = None
    if s.sy == '->':
        s.next()
3018
        return_type_annotation = p_test(s)
3019
    doc, body = p_suite_with_docstring(s, Ctx(level='function'))
3020
    return Nodes.DefNode(pos, name = name, args = args,
Stefan Behnel's avatar
Stefan Behnel committed
3021
        star_arg = star_arg, starstar_arg = starstar_arg,
3022 3023
        doc = doc, body = body, decorators = decorators,
        return_type_annotation = return_type_annotation)
Stefan Behnel's avatar
Stefan Behnel committed
3024

3025 3026 3027
def p_varargslist(s, terminator=')', annotated=1):
    args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1,
                        annotated = annotated)
William Stein's avatar
William Stein committed
3028 3029 3030 3031
    star_arg = None
    starstar_arg = None
    if s.sy == '*':
        s.next()
3032
        if s.sy == 'IDENT':
3033
            star_arg = p_py_arg_decl(s, annotated=annotated)
William Stein's avatar
William Stein committed
3034 3035
        if s.sy == ',':
            s.next()
3036
            args.extend(p_c_arg_list(s, in_pyfunc = 1,
3037
                nonempty_declarators = 1, kw_only = 1, annotated = annotated))
Stefan Behnel's avatar
Stefan Behnel committed
3038
        elif s.sy != terminator:
3039 3040
            s.error("Syntax error in Python function argument list")
    if s.sy == '**':
William Stein's avatar
William Stein committed
3041
        s.next()
3042
        starstar_arg = p_py_arg_decl(s, annotated=annotated)
Stefan Behnel's avatar
Stefan Behnel committed
3043
    return (args, star_arg, starstar_arg)
William Stein's avatar
William Stein committed
3044

3045
def p_py_arg_decl(s, annotated = 1):
William Stein's avatar
William Stein committed
3046 3047
    pos = s.position()
    name = p_ident(s)
3048
    annotation = None
3049
    if annotated and s.sy == ':':
3050
        s.next()
3051
        annotation = p_test(s)
3052
    return Nodes.PyArgDeclNode(pos, name = name, annotation = annotation)
William Stein's avatar
William Stein committed
3053

3054

3055
def p_class_statement(s, decorators):
William Stein's avatar
William Stein committed
3056 3057 3058
    # s.sy == 'class'
    pos = s.position()
    s.next()
3059 3060
    class_name = EncodedString(p_ident(s))
    class_name.encoding = s.source_encoding  # FIXME: why is this needed?
3061 3062
    arg_tuple = None
    keyword_dict = None
William Stein's avatar
William Stein committed
3063
    if s.sy == '(':
3064 3065
        positional_args, keyword_args = p_call_parse_args(s, allow_genexp=False)
        arg_tuple, keyword_dict = p_call_build_packed_args(pos, positional_args, keyword_args)
3066 3067
    if arg_tuple is None:
        # XXX: empty arg_tuple
3068 3069
        arg_tuple = ExprNodes.TupleNode(pos, args=[])
    doc, body = p_suite_with_docstring(s, Ctx(level='class'))
3070 3071 3072 3073 3074 3075
    return Nodes.PyClassDefNode(
        pos, name=class_name,
        bases=arg_tuple,
        keyword_args=keyword_dict,
        doc=doc, body=body, decorators=decorators,
        force_py3_semantics=s.context.language_level >= 3)
William Stein's avatar
William Stein committed
3076

3077

3078
def p_c_class_definition(s, pos,  ctx):
William Stein's avatar
William Stein committed
3079 3080 3081 3082 3083 3084 3085 3086
    # s.sy == 'class'
    s.next()
    module_path = []
    class_name = p_ident(s)
    while s.sy == '.':
        s.next()
        module_path.append(class_name)
        class_name = p_ident(s)
3087
    if module_path and ctx.visibility != 'extern':
William Stein's avatar
William Stein committed
3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104
        error(pos, "Qualified class name only allowed for 'extern' C class")
    if module_path and s.sy == 'IDENT' and s.systring == 'as':
        s.next()
        as_name = p_ident(s)
    else:
        as_name = class_name
    objstruct_name = None
    typeobj_name = None
    base_class_module = None
    base_class_name = None
    if s.sy == '(':
        s.next()
        base_class_path = [p_ident(s)]
        while s.sy == '.':
            s.next()
            base_class_path.append(p_ident(s))
        if s.sy == ',':
3105
            s.error("C class may only have one base class", fatal=False)
William Stein's avatar
William Stein committed
3106 3107 3108 3109
        s.expect(')')
        base_class_module = ".".join(base_class_path[:-1])
        base_class_name = base_class_path[-1]
    if s.sy == '[':
3110 3111
        if ctx.visibility not in ('public', 'extern') and not ctx.api:
            error(s.position(), "Name options only allowed for 'public', 'api', or 'extern' C class")
William Stein's avatar
William Stein committed
3112 3113
        objstruct_name, typeobj_name = p_c_class_options(s)
    if s.sy == ':':
3114
        if ctx.level == 'module_pxd':
William Stein's avatar
William Stein committed
3115 3116 3117
            body_level = 'c_class_pxd'
        else:
            body_level = 'c_class'
3118
        doc, body = p_suite_with_docstring(s, Ctx(level=body_level))
William Stein's avatar
William Stein committed
3119 3120 3121 3122
    else:
        s.expect_newline("Syntax error in C class definition")
        doc = None
        body = None
3123
    if ctx.visibility == 'extern':
William Stein's avatar
William Stein committed
3124 3125 3126 3127
        if not module_path:
            error(pos, "Module name required for 'extern' C class")
        if typeobj_name:
            error(pos, "Type object name specification not allowed for 'extern' C class")
3128
    elif ctx.visibility == 'public':
William Stein's avatar
William Stein committed
3129 3130 3131 3132
        if not objstruct_name:
            error(pos, "Object struct name specification required for 'public' C class")
        if not typeobj_name:
            error(pos, "Type object name specification required for 'public' C class")
3133 3134
    elif ctx.visibility == 'private':
        if ctx.api:
3135 3136 3137 3138
            if not objstruct_name:
                error(pos, "Object struct name specification required for 'api' C class")
            if not typeobj_name:
                error(pos, "Type object name specification required for 'api' C class")
3139
    else:
3140
        error(pos, "Invalid class visibility '%s'" % ctx.visibility)
William Stein's avatar
William Stein committed
3141
    return Nodes.CClassDefNode(pos,
3142 3143 3144
        visibility = ctx.visibility,
        typedef_flag = ctx.typedef_flag,
        api = ctx.api,
William Stein's avatar
William Stein committed
3145 3146 3147 3148 3149 3150 3151
        module_name = ".".join(module_path),
        class_name = class_name,
        as_name = as_name,
        base_class_module = base_class_module,
        base_class_name = base_class_name,
        objstruct_name = objstruct_name,
        typeobj_name = typeobj_name,
3152
        in_pxd = ctx.level == 'module_pxd',
William Stein's avatar
William Stein committed
3153 3154 3155 3156 3157 3158 3159 3160
        doc = doc,
        body = body)

def p_c_class_options(s):
    objstruct_name = None
    typeobj_name = None
    s.expect('[')
    while 1:
Stefan Behnel's avatar
Stefan Behnel committed
3161
        if s.sy != 'IDENT':
William Stein's avatar
William Stein committed
3162 3163 3164 3165 3166 3167 3168
            break
        if s.systring == 'object':
            s.next()
            objstruct_name = p_ident(s)
        elif s.systring == 'type':
            s.next()
            typeobj_name = p_ident(s)
Stefan Behnel's avatar
Stefan Behnel committed
3169
        if s.sy != ',':
William Stein's avatar
William Stein committed
3170 3171 3172 3173 3174
            break
        s.next()
    s.expect(']', "Expected 'object' or 'type'")
    return objstruct_name, typeobj_name

3175

William Stein's avatar
William Stein committed
3176 3177
def p_property_decl(s):
    pos = s.position()
3178
    s.next()  # 'property'
William Stein's avatar
William Stein committed
3179
    name = p_ident(s)
3180 3181 3182 3183
    doc, body = p_suite_with_docstring(
        s, Ctx(level='property'), with_doc_only=True)
    return Nodes.PropertyNode(pos, name=name, doc=doc, body=body)

William Stein's avatar
William Stein committed
3184

3185 3186 3187 3188 3189 3190 3191
def p_ignorable_statement(s):
    """
    Parses any kind of ignorable statement that is allowed in .pxd files.
    """
    if s.sy == 'BEGIN_STRING':
        pos = s.position()
        string_node = p_atom(s)
3192
        s.expect_newline("Syntax error in string", ignore_semicolon=True)
3193 3194 3195 3196
        return Nodes.ExprStatNode(pos, expr=string_node)
    return None


3197 3198 3199 3200
def p_doc_string(s):
    if s.sy == 'BEGIN_STRING':
        pos = s.position()
        kind, bytes_result, unicode_result = p_cat_string_literal(s)
3201
        s.expect_newline("Syntax error in doc string", ignore_semicolon=True)
3202 3203 3204 3205 3206 3207 3208
        if kind in ('u', ''):
            return unicode_result
        warning(pos, "Python 3 requires docstrings to be unicode strings")
        return bytes_result
    else:
        return None

3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244

def _extract_docstring(node):
    """
    Extract a docstring from a statement or from the first statement
    in a list.  Remove the statement if found.  Return a tuple
    (plain-docstring or None, node).
    """
    doc_node = None
    if node is None:
        pass
    elif isinstance(node, Nodes.ExprStatNode):
        if node.expr.is_string_literal:
            doc_node = node.expr
            node = Nodes.StatListNode(node.pos, stats=[])
    elif isinstance(node, Nodes.StatListNode) and node.stats:
        stats = node.stats
        if isinstance(stats[0], Nodes.ExprStatNode):
            if stats[0].expr.is_string_literal:
                doc_node = stats[0].expr
                del stats[0]

    if doc_node is None:
        doc = None
    elif isinstance(doc_node, ExprNodes.BytesNode):
        warning(node.pos,
                "Python 3 requires docstrings to be unicode strings")
        doc = doc_node.value
    elif isinstance(doc_node, ExprNodes.StringNode):
        doc = doc_node.unicode_value
        if doc is None:
            doc = doc_node.value
    else:
        doc = doc_node.value
    return doc, node


3245 3246
def p_code(s, level=None, ctx=Ctx):
    body = p_statement_list(s, ctx(level = level), first_statement = 1)
3247 3248 3249 3250
    if s.sy != 'EOF':
        s.error("Syntax error in statement [%s,%s]" % (
            repr(s.sy), repr(s.systring)))
    return body
William Stein's avatar
William Stein committed
3251

3252 3253
_match_compiler_directive_comment = cython.declare(object, re.compile(
    r"^#\s*cython\s*:\s*((\w|[.])+\s*=.*)$").match)
3254 3255

def p_compiler_directive_comments(s):
3256
    result = {}
3257
    while s.sy == 'commentline':
3258
        m = _match_compiler_directive_comment(s.systring)
3259
        if m:
3260
            directives = m.group(1).strip()
3261
            try:
3262 3263
                result.update(Options.parse_directive_list(
                    directives, ignore_unknown=True))
3264
            except ValueError, e:
3265
                s.error(e.args[0], fatal=False)
3266 3267 3268
        s.next()
    return result

3269
def p_module(s, pxd, full_module_name, ctx=Ctx):
William Stein's avatar
William Stein committed
3270
    pos = s.position()
3271

3272
    directive_comments = p_compiler_directive_comments(s)
3273 3274
    s.parse_comments = False

3275
    if 'language_level' in directive_comments:
Stefan Behnel's avatar
Stefan Behnel committed
3276
        s.context.set_language_level(directive_comments['language_level'])
3277

3278
    doc = p_doc_string(s)
William Stein's avatar
William Stein committed
3279 3280 3281 3282
    if pxd:
        level = 'module_pxd'
    else:
        level = 'module'
3283

3284
    body = p_statement_list(s, ctx(level=level), first_statement = 1)
Stefan Behnel's avatar
Stefan Behnel committed
3285
    if s.sy != 'EOF':
William Stein's avatar
William Stein committed
3286 3287
        s.error("Syntax error in statement [%s,%s]" % (
            repr(s.sy), repr(s.systring)))
3288 3289
    return ModuleNode(pos, doc = doc, body = body,
                      full_module_name = full_module_name,
3290
                      directive_comments = directive_comments)
William Stein's avatar
William Stein committed
3291

3292 3293 3294 3295 3296
def p_cpp_class_definition(s, pos,  ctx):
    # s.sy == 'cppclass'
    s.next()
    module_path = []
    class_name = p_ident(s)
3297 3298 3299
    cname = p_opt_cname(s)
    if cname is None and ctx.namespace is not None:
        cname = ctx.namespace + "::" + class_name
3300
    if s.sy == '.':
3301
        error(pos, "Qualified class name not allowed C++ class")
Danilo Freitas's avatar
Danilo Freitas committed
3302 3303
    if s.sy == '[':
        s.next()
3304
        templates = [p_ident(s)]
Danilo Freitas's avatar
Danilo Freitas committed
3305 3306
        while s.sy == ',':
            s.next()
3307
            templates.append(p_ident(s))
Danilo Freitas's avatar
Danilo Freitas committed
3308
        s.expect(']')
3309 3310
    else:
        templates = None
3311
    if s.sy == '(':
3312
        s.next()
3313
        base_classes = [p_c_base_type(s, templates = templates)]
3314
        while s.sy == ',':
3315
            s.next()
3316
            base_classes.append(p_c_base_type(s, templates = templates))
3317
        s.expect(')')
3318 3319
    else:
        base_classes = []
3320
    if s.sy == '[':
3321
        error(s.position(), "Name options not allowed for C++ class")
3322
    nogil = p_nogil(s)
3323
    if s.sy == ':':
3324 3325 3326 3327
        s.next()
        s.expect('NEWLINE')
        s.expect_indent()
        attributes = []
3328
        body_ctx = Ctx(visibility = ctx.visibility, level='cpp_class', nogil=nogil or ctx.nogil)
Danilo Freitas's avatar
Danilo Freitas committed
3329
        body_ctx.templates = templates
3330
        while s.sy != 'DEDENT':
3331 3332
            if s.sy != 'pass':
                attributes.append(p_cpp_class_attribute(s, body_ctx))
3333 3334 3335 3336
            else:
                s.next()
                s.expect_newline("Expected a newline")
        s.expect_dedent()
3337
    else:
3338
        attributes = None
3339 3340 3341
        s.expect_newline("Syntax error in C++ class definition")
    return Nodes.CppClassNode(pos,
        name = class_name,
3342
        cname = cname,
3343
        base_classes = base_classes,
3344 3345
        visibility = ctx.visibility,
        in_pxd = ctx.level == 'module_pxd',
Danilo Freitas's avatar
Danilo Freitas committed
3346 3347
        attributes = attributes,
        templates = templates)
3348

3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365
def p_cpp_class_attribute(s, ctx):
    decorators = None
    if s.sy == '@':
        decorators = p_decorators(s)
    if s.systring == 'cppclass':
        return p_cpp_class_definition(s, s.position(), ctx)
    else:
        node = p_c_func_or_var_declaration(s, s.position(), ctx)
        if decorators is not None:
            tup = Nodes.CFuncDefNode, Nodes.CVarDefNode, Nodes.CClassDefNode
            if ctx.allow_struct_enum_decorator:
                tup += Nodes.CStructOrUnionDefNode, Nodes.CEnumDefNode
            if not isinstance(node, tup):
                s.error("Decorators can only be followed by functions or classes")
            node.decorators = decorators
        return node

3366

William Stein's avatar
William Stein committed
3367 3368 3369 3370 3371 3372
#----------------------------------------------
#
#   Debugging
#
#----------------------------------------------

Stefan Behnel's avatar
Stefan Behnel committed
3373
def print_parse_tree(f, node, level, key = None):
William Stein's avatar
William Stein committed
3374 3375 3376 3377 3378 3379
    ind = "  " * level
    if node:
        f.write(ind)
        if key:
            f.write("%s: " % key)
        t = type(node)
Stefan Behnel's avatar
Stefan Behnel committed
3380
        if t is tuple:
William Stein's avatar
William Stein committed
3381 3382 3383 3384 3385
            f.write("(%s @ %s\n" % (node[0], node[1]))
            for i in xrange(2, len(node)):
                print_parse_tree(f, node[i], level+1)
            f.write("%s)\n" % ind)
            return
3386
        elif isinstance(node, Nodes.Node):
William Stein's avatar
William Stein committed
3387 3388 3389 3390 3391 3392
            try:
                tag = node.tag
            except AttributeError:
                tag = node.__class__.__name__
            f.write("%s @ %s\n" % (tag, node.pos))
            for name, value in node.__dict__.items():
Stefan Behnel's avatar
Stefan Behnel committed
3393
                if name != 'tag' and name != 'pos':
William Stein's avatar
William Stein committed
3394 3395
                    print_parse_tree(f, value, level+1, name)
            return
Stefan Behnel's avatar
Stefan Behnel committed
3396
        elif t is list:
William Stein's avatar
William Stein committed
3397 3398 3399 3400 3401 3402
            f.write("[\n")
            for i in xrange(len(node)):
                print_parse_tree(f, node[i], level+1)
            f.write("%s]\n" % ind)
            return
    f.write("%s%s\n" % (ind, node))