Parsing.py 91.2 KB
Newer Older
1
# cython: auto_cpdef=True, infer_types=True
William Stein's avatar
William Stein committed
2 3 4 5
#
#   Pyrex Parser
#

6 7 8 9
# This should be done automatically
import cython
cython.declare(Nodes=object, ExprNodes=object, EncodedString=object)

10 11 12
import os
import re
import sys
Lisandro Dalcin's avatar
Lisandro Dalcin committed
13

14
from Cython.Compiler.Scanning import PyrexScanner, FileSourceDescriptor
William Stein's avatar
William Stein committed
15 16
import Nodes
import ExprNodes
17
import StringEncoding
18
from StringEncoding import EncodedString, BytesLiteral, _unicode, _bytes
19
from ModuleNode import ModuleNode
20
from Errors import error, warning, InternalError
21
from Cython import Utils
Stefan Behnel's avatar
Stefan Behnel committed
22
import Future
23
import Options
William Stein's avatar
William Stein committed
24

25 26 27 28 29 30 31 32 33
class Ctx(object):
    #  Parsing context
    level = 'other'
    visibility = 'private'
    cdef_flag = 0
    typedef_flag = 0
    api = 0
    overridable = 0
    nogil = 0
34
    namespace = None
Danilo Freitas's avatar
Danilo Freitas committed
35
    templates = None
36 37 38 39 40 41 42 43 44 45 46

    def __init__(self, **kwds):
        self.__dict__.update(kwds)

    def __call__(self, **kwds):
        ctx = Ctx()
        d = ctx.__dict__
        d.update(self.__dict__)
        d.update(kwds)
        return ctx

William Stein's avatar
William Stein committed
47 48 49 50 51 52 53 54 55 56 57 58 59
def p_ident(s, message = "Expected an identifier"):
    if s.sy == 'IDENT':
        name = s.systring
        s.next()
        return name
    else:
        s.error(message)

def p_ident_list(s):
    names = []
    while s.sy == 'IDENT':
        names.append(s.systring)
        s.next()
Stefan Behnel's avatar
Stefan Behnel committed
60
        if s.sy != ',':
William Stein's avatar
William Stein committed
61 62 63 64 65 66 67 68 69 70
            break
        s.next()
    return names

#------------------------------------------
#
#   Expressions
#
#------------------------------------------

71 72 73 74 75 76
def p_binop_operator(s):
    pos = s.position()
    op = s.sy
    s.next()
    return op, pos

William Stein's avatar
William Stein committed
77 78 79
def p_binop_expr(s, ops, p_sub_expr):
    n1 = p_sub_expr(s)
    while s.sy in ops:
80
        op, pos = p_binop_operator(s)
William Stein's avatar
William Stein committed
81 82
        n2 = p_sub_expr(s)
        n1 = ExprNodes.binop_node(pos, op, n1, n2)
83 84 85 86 87
        if op == '/':
            if Future.division in s.context.future_directives:
                n1.truedivision = True
            else:
                n1.truedivision = None # unknown
William Stein's avatar
William Stein committed
88 89
    return n1

Stefan Behnel's avatar
Stefan Behnel committed
90 91 92 93 94 95 96 97 98 99
#lambdef: 'lambda' [varargslist] ':' test

def p_lambdef(s, allow_conditional=True):
    # s.sy == 'lambda'
    pos = s.position()
    s.next()
    if s.sy == ':':
        args = []
        star_arg = starstar_arg = None
    else:
100 101
        args, star_arg, starstar_arg = p_varargslist(
            s, terminator=':', annotated=False)
Stefan Behnel's avatar
Stefan Behnel committed
102 103
    s.expect(':')
    if allow_conditional:
104
        expr = p_test(s)
Stefan Behnel's avatar
Stefan Behnel committed
105 106 107 108 109 110 111 112 113 114 115 116
    else:
        expr = p_test_nocond(s)
    return ExprNodes.LambdaNode(
        pos, args = args,
        star_arg = star_arg, starstar_arg = starstar_arg,
        result_expr = expr)

#lambdef_nocond: 'lambda' [varargslist] ':' test_nocond

def p_lambdef_nocond(s):
    return p_lambdef(s, allow_conditional=False)

117
#test: or_test ['if' or_test 'else' test] | lambdef
William Stein's avatar
William Stein committed
118

Robert Bradshaw's avatar
Robert Bradshaw committed
119
def p_test(s):
120 121
    if s.sy == 'lambda':
        return p_lambdef(s)
Robert Bradshaw's avatar
Robert Bradshaw committed
122 123 124 125 126
    pos = s.position()
    expr = p_or_test(s)
    if s.sy == 'if':
        s.next()
        test = p_or_test(s)
Stefan Behnel's avatar
Stefan Behnel committed
127 128 129
        s.expect('else')
        other = p_test(s)
        return ExprNodes.CondExprNode(pos, test=test, true_val=expr, false_val=other)
Robert Bradshaw's avatar
Robert Bradshaw committed
130 131 132
    else:
        return expr

Stefan Behnel's avatar
Stefan Behnel committed
133 134 135 136 137 138 139
#test_nocond: or_test | lambdef_nocond

def p_test_nocond(s):
    if s.sy == 'lambda':
        return p_lambdef_nocond(s)
    else:
        return p_or_test(s)
Robert Bradshaw's avatar
Robert Bradshaw committed
140 141 142 143

#or_test: and_test ('or' and_test)*

def p_or_test(s):
William Stein's avatar
William Stein committed
144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175
    return p_rassoc_binop_expr(s, ('or',), p_and_test)

def p_rassoc_binop_expr(s, ops, p_subexpr):
    n1 = p_subexpr(s)
    if s.sy in ops:
        pos = s.position()
        op = s.sy
        s.next()
        n2 = p_rassoc_binop_expr(s, ops, p_subexpr)
        n1 = ExprNodes.binop_node(pos, op, n1, n2)
    return n1

#and_test: not_test ('and' not_test)*

def p_and_test(s):
    #return p_binop_expr(s, ('and',), p_not_test)
    return p_rassoc_binop_expr(s, ('and',), p_not_test)

#not_test: 'not' not_test | comparison

def p_not_test(s):
    if s.sy == 'not':
        pos = s.position()
        s.next()
        return ExprNodes.NotNode(pos, operand = p_not_test(s))
    else:
        return p_comparison(s)

#comparison: expr (comp_op expr)*
#comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'

def p_comparison(s):
176
    n1 = p_starred_expr(s)
William Stein's avatar
William Stein committed
177 178 179
    if s.sy in comparison_ops:
        pos = s.position()
        op = p_cmp_op(s)
180
        n2 = p_starred_expr(s)
William Stein's avatar
William Stein committed
181 182 183 184 185 186
        n1 = ExprNodes.PrimaryCmpNode(pos, 
            operator = op, operand1 = n1, operand2 = n2)
        if s.sy in comparison_ops:
            n1.cascade = p_cascaded_cmp(s)
    return n1

187 188 189 190 191 192
def p_test_or_starred_expr(s):
    if s.sy == '*':
        return p_starred_expr(s)
    else:
        return p_test(s)

193
def p_starred_expr(s):
194
    pos = s.position()
195 196 197 198 199 200
    if s.sy == '*':
        starred = True
        s.next()
    else:
        starred = False
    expr = p_bit_expr(s)
201 202
    if starred:
        expr = ExprNodes.StarredTargetNode(pos, expr)
203 204
    return expr

William Stein's avatar
William Stein committed
205 206 207
def p_cascaded_cmp(s):
    pos = s.position()
    op = p_cmp_op(s)
208
    n2 = p_starred_expr(s)
William Stein's avatar
William Stein committed
209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266
    result = ExprNodes.CascadedCmpNode(pos, 
        operator = op, operand2 = n2)
    if s.sy in comparison_ops:
        result.cascade = p_cascaded_cmp(s)
    return result

def p_cmp_op(s):
    if s.sy == 'not':
        s.next()
        s.expect('in')
        op = 'not_in'
    elif s.sy == 'is':
        s.next()
        if s.sy == 'not':
            s.next()
            op = 'is_not'
        else:
            op = 'is'
    else:
        op = s.sy
        s.next()
    if op == '<>':
        op = '!='
    return op
    
comparison_ops = (
    '<', '>', '==', '>=', '<=', '<>', '!=', 
    'in', 'is', 'not'
)

#expr: xor_expr ('|' xor_expr)*

def p_bit_expr(s):
    return p_binop_expr(s, ('|',), p_xor_expr)

#xor_expr: and_expr ('^' and_expr)*

def p_xor_expr(s):
    return p_binop_expr(s, ('^',), p_and_expr)

#and_expr: shift_expr ('&' shift_expr)*

def p_and_expr(s):
    return p_binop_expr(s, ('&',), p_shift_expr)

#shift_expr: arith_expr (('<<'|'>>') arith_expr)*

def p_shift_expr(s):
    return p_binop_expr(s, ('<<', '>>'), p_arith_expr)

#arith_expr: term (('+'|'-') term)*

def p_arith_expr(s):
    return p_binop_expr(s, ('+', '-'), p_term)

#term: factor (('*'|'/'|'%') factor)*

def p_term(s):
Robert Bradshaw's avatar
Robert Bradshaw committed
267
    return p_binop_expr(s, ('*', '/', '%', '//'), p_factor)
William Stein's avatar
William Stein committed
268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294

#factor: ('+'|'-'|'~'|'&'|typecast|sizeof) factor | power

def p_factor(s):
    sy = s.sy
    if sy in ('+', '-', '~'):
        op = s.sy
        pos = s.position()
        s.next()
        return ExprNodes.unop_node(pos, op, p_factor(s))
    elif sy == '&':
        pos = s.position()
        s.next()
        arg = p_factor(s)
        return ExprNodes.AmpersandNode(pos, operand = arg)
    elif sy == "<":
        return p_typecast(s)
    elif sy == 'IDENT' and s.systring == "sizeof":
        return p_sizeof(s)
    else:
        return p_power(s)

def p_typecast(s):
    # s.sy == "<"
    pos = s.position()
    s.next()
    base_type = p_c_base_type(s)
295 296
    if base_type.name is None:
        s.error("Unknown type")
William Stein's avatar
William Stein committed
297
    declarator = p_c_declarator(s, empty = 1)
298 299 300 301 302
    if s.sy == '?':
        s.next()
        typecheck = 1
    else:
        typecheck = 0
William Stein's avatar
William Stein committed
303 304 305 306 307
    s.expect(">")
    operand = p_factor(s)
    return ExprNodes.TypecastNode(pos, 
        base_type = base_type, 
        declarator = declarator,
308 309
        operand = operand,
        typecheck = typecheck)
William Stein's avatar
William Stein committed
310 311 312 313 314 315

def p_sizeof(s):
    # s.sy == ident "sizeof"
    pos = s.position()
    s.next()
    s.expect('(')
316 317 318
    # Here we decide if we are looking at an expression or type
    # If it is actually a type, but parsable as an expression, 
    # we treat it as an expression here. 
319
    if looking_at_expr(s):
320
        operand = p_test(s)
321 322
        node = ExprNodes.SizeofVarNode(pos, operand = operand)
    else:
William Stein's avatar
William Stein committed
323 324 325 326 327 328 329
        base_type = p_c_base_type(s)
        declarator = p_c_declarator(s, empty = 1)
        node = ExprNodes.SizeofTypeNode(pos, 
            base_type = base_type, declarator = declarator)
    s.expect(')')
    return node

330 331 332 333
def p_yield_expression(s):
    # s.sy == "yield"
    pos = s.position()
    s.next()
334
    if s.sy != ')' and s.sy not in statement_terminators:
335
        arg = p_testlist(s)
336 337 338 339 340 341 342 343
    else:
        arg = None
    return ExprNodes.YieldExprNode(pos, arg=arg)

def p_yield_statement(s):
    # s.sy == "yield"
    yield_expr = p_yield_expression(s)
    return Nodes.ExprStatNode(yield_expr.pos, expr=yield_expr)
344

William Stein's avatar
William Stein committed
345 346 347
#power: atom trailer* ('**' factor)*

def p_power(s):
348
    if s.systring == 'new' and s.peek()[0] == 'IDENT':
Danilo Freitas's avatar
Danilo Freitas committed
349
        return p_new_expr(s)
William Stein's avatar
William Stein committed
350 351 352 353 354 355 356 357 358 359
    n1 = p_atom(s)
    while s.sy in ('(', '[', '.'):
        n1 = p_trailer(s, n1)
    if s.sy == '**':
        pos = s.position()
        s.next()
        n2 = p_factor(s)
        n1 = ExprNodes.binop_node(pos, '**', n1, n2)
    return n1

Danilo Freitas's avatar
Danilo Freitas committed
360
def p_new_expr(s):
Danilo Freitas's avatar
Danilo Freitas committed
361
    # s.systring == 'new'.
Danilo Freitas's avatar
Danilo Freitas committed
362 363
    pos = s.position()
    s.next()
Robert Bradshaw's avatar
Robert Bradshaw committed
364 365
    cppclass = p_c_base_type(s)
    return p_call(s, ExprNodes.NewExprNode(pos, cppclass = cppclass))
Danilo Freitas's avatar
Danilo Freitas committed
366

William Stein's avatar
William Stein committed
367 368 369 370 371 372 373 374 375 376
#trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME

def p_trailer(s, node1):
    pos = s.position()
    if s.sy == '(':
        return p_call(s, node1)
    elif s.sy == '[':
        return p_index(s, node1)
    else: # s.sy == '.'
        s.next()
377
        name = EncodedString( p_ident(s) )
William Stein's avatar
William Stein committed
378 379 380 381 382 383 384 385 386 387 388 389 390 391
        return ExprNodes.AttributeNode(pos, 
            obj = node1, attribute = name)

# arglist:  argument (',' argument)* [',']
# argument: [test '='] test       # Really [keyword '='] test

def p_call(s, function):
    # s.sy == '('
    pos = s.position()
    s.next()
    positional_args = []
    keyword_args = []
    star_arg = None
    starstar_arg = None
392 393 394 395 396
    while s.sy not in ('**', ')'):
        if s.sy == '*':
            if star_arg:
                s.error("only one star-arg parameter allowed",
                    pos = s.position())
William Stein's avatar
William Stein committed
397
            s.next()
398
            star_arg = p_test(s)
William Stein's avatar
William Stein committed
399
        else:
400
            arg = p_test(s)
401 402 403 404 405 406
            if s.sy == '=':
                s.next()
                if not arg.is_name:
                    s.error("Expected an identifier before '='",
                        pos = arg.pos)
                encoded_name = EncodedString(arg.name)
407
                keyword = ExprNodes.IdentifierStringNode(arg.pos, value = encoded_name)
408
                arg = p_test(s)
409 410 411 412 413 414 415 416 417
                keyword_args.append((keyword, arg))
            else:
                if keyword_args:
                    s.error("Non-keyword arg following keyword arg",
                        pos = arg.pos)
                if star_arg:
                    s.error("Non-keyword arg following star-arg",
                        pos = arg.pos)
                positional_args.append(arg)
Stefan Behnel's avatar
Stefan Behnel committed
418
        if s.sy != ',':
William Stein's avatar
William Stein committed
419 420
            break
        s.next()
421

422 423 424 425
    if s.sy == 'for':
        if len(positional_args) == 1 and not star_arg:
            positional_args = [ p_genexp(s, positional_args[0]) ]
    elif s.sy == '**':
William Stein's avatar
William Stein committed
426
        s.next()
427
        starstar_arg = p_test(s)
William Stein's avatar
William Stein committed
428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449
        if s.sy == ',':
            s.next()
    s.expect(')')
    if not (keyword_args or star_arg or starstar_arg):
        return ExprNodes.SimpleCallNode(pos,
            function = function,
            args = positional_args)
    else:
        arg_tuple = None
        keyword_dict = None
        if positional_args or not star_arg:
            arg_tuple = ExprNodes.TupleNode(pos, 
                args = positional_args)
        if star_arg:
            star_arg_tuple = ExprNodes.AsTupleNode(pos, arg = star_arg)
            if arg_tuple:
                arg_tuple = ExprNodes.binop_node(pos, 
                    operator = '+', operand1 = arg_tuple,
                    operand2 = star_arg_tuple)
            else:
                arg_tuple = star_arg_tuple
        if keyword_args:
450 451
            keyword_args = [ExprNodes.DictItemNode(pos=key.pos, key=key, value=value) 
                              for key, value in keyword_args]
William Stein's avatar
William Stein committed
452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499
            keyword_dict = ExprNodes.DictNode(pos,
                key_value_pairs = keyword_args)
        return ExprNodes.GeneralCallNode(pos, 
            function = function,
            positional_args = arg_tuple,
            keyword_args = keyword_dict,
            starstar_arg = starstar_arg)

#lambdef: 'lambda' [varargslist] ':' test

#subscriptlist: subscript (',' subscript)* [',']

def p_index(s, base):
    # s.sy == '['
    pos = s.position()
    s.next()
    subscripts = p_subscript_list(s)
    if len(subscripts) == 1 and len(subscripts[0]) == 2:
        start, stop = subscripts[0]
        result = ExprNodes.SliceIndexNode(pos, 
            base = base, start = start, stop = stop)
    else:
        indexes = make_slice_nodes(pos, subscripts)
        if len(indexes) == 1:
            index = indexes[0]
        else:
            index = ExprNodes.TupleNode(pos, args = indexes)
        result = ExprNodes.IndexNode(pos,
            base = base, index = index)
    s.expect(']')
    return result

def p_subscript_list(s):
    items = [p_subscript(s)]
    while s.sy == ',':
        s.next()
        if s.sy == ']':
            break
        items.append(p_subscript(s))
    return items

#subscript: '.' '.' '.' | test | [test] ':' [test] [':' [test]]

def p_subscript(s):
    # Parse a subscript and return a list of
    # 1, 2 or 3 ExprNodes, depending on how
    # many slice elements were encountered.
    pos = s.position()
500 501 502 503 504 505 506 507 508 509
    start = p_slice_element(s, (':',))
    if s.sy != ':':
        return [start]
    s.next()
    stop = p_slice_element(s, (':', ',', ']'))
    if s.sy != ':':
        return [start, stop]
    s.next()
    step = p_slice_element(s, (':', ',', ']'))
    return [start, stop, step]
William Stein's avatar
William Stein committed
510 511 512 513 514

def p_slice_element(s, follow_set):
    # Simple expression which may be missing iff
    # it is followed by something in follow_set.
    if s.sy not in follow_set:
515
        return p_test(s)
William Stein's avatar
William Stein committed
516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546
    else:
        return None

def expect_ellipsis(s):
    s.expect('.')
    s.expect('.')
    s.expect('.')

def make_slice_nodes(pos, subscripts):
    # Convert a list of subscripts as returned
    # by p_subscript_list into a list of ExprNodes,
    # creating SliceNodes for elements with 2 or
    # more components.
    result = []
    for subscript in subscripts:
        if len(subscript) == 1:
            result.append(subscript[0])
        else:
            result.append(make_slice_node(pos, *subscript))
    return result

def make_slice_node(pos, start, stop = None, step = None):
    if not start:
        start = ExprNodes.NoneNode(pos)
    if not stop:
        stop = ExprNodes.NoneNode(pos)
    if not step:
        step = ExprNodes.NoneNode(pos)
    return ExprNodes.SliceNode(pos,
        start = start, stop = stop, step = step)

547
#atom: '(' [yield_expr|testlist_comp] ')' | '[' [listmaker] ']' | '{' [dict_or_set_maker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+
William Stein's avatar
William Stein committed
548 549 550 551 552 553 554 555

def p_atom(s):
    pos = s.position()
    sy = s.sy
    if sy == '(':
        s.next()
        if s.sy == ')':
            result = ExprNodes.TupleNode(pos, args = [])
556 557
        elif s.sy == 'yield':
            result = p_yield_expression(s)
William Stein's avatar
William Stein committed
558
        else:
559
            result = p_testlist_comp(s)
William Stein's avatar
William Stein committed
560 561 562 563 564
        s.expect(')')
        return result
    elif sy == '[':
        return p_list_maker(s)
    elif sy == '{':
565
        return p_dict_or_set_maker(s)
William Stein's avatar
William Stein committed
566 567
    elif sy == '`':
        return p_backquote_expr(s)
568 569 570
    elif sy == '.':
        expect_ellipsis(s)
        return ExprNodes.EllipsisNode(pos)
William Stein's avatar
William Stein committed
571
    elif sy == 'INT':
572
        value = s.systring
William Stein's avatar
William Stein committed
573
        s.next()
574 575 576 577 578 579 580 581 582 583 584 585
        unsigned = ""
        longness = ""
        while value[-1] in "UuLl":
            if value[-1] in "Ll":
                longness += "L"
            else:
                unsigned += "U"
            value = value[:-1]
        return ExprNodes.IntNode(pos, 
                                 value = value,
                                 unsigned = unsigned,
                                 longness = longness)
William Stein's avatar
William Stein committed
586 587 588 589 590 591 592 593
    elif sy == 'FLOAT':
        value = s.systring
        s.next()
        return ExprNodes.FloatNode(pos, value = value)
    elif sy == 'IMAG':
        value = s.systring[:-1]
        s.next()
        return ExprNodes.ImagNode(pos, value = value)
594
    elif sy == 'BEGIN_STRING':
595
        kind, bytes_value, unicode_value = p_cat_string_literal(s)
William Stein's avatar
William Stein committed
596
        if kind == 'c':
597
            return ExprNodes.CharNode(pos, value = bytes_value)
598
        elif kind == 'u':
599
            return ExprNodes.UnicodeNode(pos, value = unicode_value, bytes_value = bytes_value)
600
        elif kind == 'b':
601
            return ExprNodes.BytesNode(pos, value = bytes_value)
William Stein's avatar
William Stein committed
602
        else:
603
            return ExprNodes.StringNode(pos, value = bytes_value, unicode_value = unicode_value)
William Stein's avatar
William Stein committed
604
    elif sy == 'IDENT':
605
        name = EncodedString( s.systring )
William Stein's avatar
William Stein committed
606 607 608
        s.next()
        if name == "None":
            return ExprNodes.NoneNode(pos)
609
        elif name == "True":
610
            return ExprNodes.BoolNode(pos, value=True)
611
        elif name == "False":
612
            return ExprNodes.BoolNode(pos, value=False)
613 614
        elif name == "NULL":
            return ExprNodes.NullNode(pos)
William Stein's avatar
William Stein committed
615
        else:
616
            return p_name(s, name)
William Stein's avatar
William Stein committed
617 618 619
    else:
        s.error("Expected an identifier or literal")

620 621
def p_name(s, name):
    pos = s.position()
622 623 624 625 626 627 628 629 630 631 632
    if not s.compile_time_expr and name in s.compile_time_env:
        value = s.compile_time_env.lookup_here(name)
        rep = repr(value)
        if isinstance(value, bool):
            return ExprNodes.BoolNode(pos, value = value)
        elif isinstance(value, int):
            return ExprNodes.IntNode(pos, value = rep)
        elif isinstance(value, long):
            return ExprNodes.IntNode(pos, value = rep, longness = "L")
        elif isinstance(value, float):
            return ExprNodes.FloatNode(pos, value = rep)
633 634 635 636
        elif isinstance(value, _unicode):
            return ExprNodes.UnicodeNode(pos, value = value)
        elif isinstance(value, _bytes):
            return ExprNodes.BytesNode(pos, value = value)
637
        else:
638 639
            error(pos, "Invalid type for compile-time constant: %s"
                % value.__class__.__name__)
640 641
    return ExprNodes.NameNode(pos, name = name)

William Stein's avatar
William Stein committed
642 643
def p_cat_string_literal(s):
    # A sequence of one or more adjacent string literals.
644 645 646 647 648 649 650 651 652 653 654 655 656 657 658
    # Returns (kind, bytes_value, unicode_value)
    # where kind in ('b', 'c', 'u', '')
    kind, bytes_value, unicode_value = p_string_literal(s)
    if kind == 'c' or s.sy != 'BEGIN_STRING':
        return kind, bytes_value, unicode_value
    bstrings, ustrings = [bytes_value], [unicode_value]
    bytes_value = unicode_value = None
    while s.sy == 'BEGIN_STRING':
        pos = s.position()
        next_kind, next_bytes_value, next_unicode_value = p_string_literal(s)
        if next_kind == 'c':
            error(pos, "Cannot concatenate char literal with another string or char literal")
        elif next_kind != kind:
            error(pos, "Cannot mix string literals of different types, expected %s'', got %s''" %
                  (kind, next_kind))
659
        else:
660 661 662
            bstrings.append(next_bytes_value)
            ustrings.append(next_unicode_value)
    # join and rewrap the partial literals
663
    if kind in ('b', 'c', '') or kind == 'u' and None not in bstrings:
664
        # Py3 enforced unicode literals are parsed as bytes/unicode combination
665
        bytes_value = BytesLiteral( StringEncoding.join_bytes(bstrings) )
666 667 668 669 670 671
        bytes_value.encoding = s.source_encoding
    if kind in ('u', ''):
        unicode_value = EncodedString( u''.join([ u for u in ustrings if u is not None ]) )
    return kind, bytes_value, unicode_value

def p_opt_string_literal(s, required_type='u'):
672
    if s.sy == 'BEGIN_STRING':
673 674 675 676 677 678 679
        kind, bytes_value, unicode_value = p_string_literal(s, required_type)
        if required_type == 'u':
            return unicode_value
        elif required_type == 'b':
            return bytes_value
        else:
            s.error("internal parser configuration error")
William Stein's avatar
William Stein committed
680 681 682
    else:
        return None

683 684 685 686 687 688
def check_for_non_ascii_characters(string):
    for c in string:
        if c >= u'\x80':
            return True
    return False

689
def p_string_literal(s, kind_override=None):
690 691 692 693 694 695 696
    # A single string or char literal.  Returns (kind, bvalue, uvalue)
    # where kind in ('b', 'c', 'u', '').  The 'bvalue' is the source
    # code byte sequence of the string literal, 'uvalue' is the
    # decoded Unicode string.  Either of the two may be None depending
    # on the 'kind' of string, only unprefixed strings have both
    # representations.

William Stein's avatar
William Stein committed
697 698
    # s.sy == 'BEGIN_STRING'
    pos = s.position()
699
    is_raw = 0
700
    is_python3_source = s.context.language_level >= 3
701
    has_non_ASCII_literal_characters = False
William Stein's avatar
William Stein committed
702
    kind = s.systring[:1].lower()
703 704 705 706 707 708
    if kind == 'r':
        kind = ''
        is_raw = 1
    elif kind in 'ub':
        is_raw = s.systring[1:2].lower() == 'r'
    elif kind != 'c':
William Stein's avatar
William Stein committed
709
        kind = ''
710 711 712 713 714 715 716 717 718 719 720 721
    if kind == '' and kind_override is None and Future.unicode_literals in s.context.future_directives:
        chars = StringEncoding.StrLiteralBuilder(s.source_encoding)
        kind = 'u'
    else:
        if kind_override is not None and kind_override in 'ub':
            kind = kind_override
        if kind == 'u':
            chars = StringEncoding.UnicodeLiteralBuilder()
        elif kind == '':
            chars = StringEncoding.StrLiteralBuilder(s.source_encoding)
        else:
            chars = StringEncoding.BytesLiteralBuilder(s.source_encoding)
William Stein's avatar
William Stein committed
722 723 724
    while 1:
        s.next()
        sy = s.sy
725
        systr = s.systring
William Stein's avatar
William Stein committed
726 727
        #print "p_string_literal: sy =", sy, repr(s.systring) ###
        if sy == 'CHARS':
728
            chars.append(systr)
729
            if is_python3_source and not has_non_ASCII_literal_characters and check_for_non_ascii_characters(systr):
730
                has_non_ASCII_literal_characters = True
William Stein's avatar
William Stein committed
731
        elif sy == 'ESCAPE':
732
            if is_raw:
733 734 735 736 737 738
                if systr == u'\\\n':
                    chars.append(u'\\\n')
                elif systr == u'\\\"':
                    chars.append(u'"')
                elif systr == u'\\\'':
                    chars.append(u"'")
William Stein's avatar
William Stein committed
739
                else:
740
                    chars.append(systr)
741 742
                    if is_python3_source and not has_non_ASCII_literal_characters \
                           and check_for_non_ascii_characters(systr):
743
                        has_non_ASCII_literal_characters = True
William Stein's avatar
William Stein committed
744 745
            else:
                c = systr[1]
746 747 748
                if c in u"01234567":
                    chars.append_charval( int(systr[1:], 8) )
                elif c in u"'\"\\":
749
                    chars.append(c)
750 751 752 753
                elif c in u"abfnrtv":
                    chars.append(
                        StringEncoding.char_from_escape_sequence(systr))
                elif c == u'\n':
William Stein's avatar
William Stein committed
754
                    pass
755 756 757 758
                elif c == u'x':
                    chars.append_charval( int(systr[2:], 16) )
                elif c in u'Uu':
                    if kind in ('u', ''):
759
                        chrval = int(systr[2:], 16)
760
                        if chrval > 1114111: # sys.maxunicode:
761 762 763
                            s.error("Invalid unicode escape '%s'" % systr,
                                    pos = pos)
                    else:
764
                        # unicode escapes in plain byte strings are not unescaped
765 766
                        chrval = None
                    chars.append_uescape(chrval, systr)
William Stein's avatar
William Stein committed
767
                else:
768
                    chars.append(u'\\' + systr[1:])
769 770
                    if is_python3_source and not has_non_ASCII_literal_characters \
                           and check_for_non_ascii_characters(systr):
771
                        has_non_ASCII_literal_characters = True
William Stein's avatar
William Stein committed
772
        elif sy == 'NEWLINE':
773
            chars.append(u'\n')
William Stein's avatar
William Stein committed
774 775 776 777 778 779 780 781
        elif sy == 'END_STRING':
            break
        elif sy == 'EOF':
            s.error("Unclosed string literal", pos = pos)
        else:
            s.error(
                "Unexpected token %r:%r in string literal" %
                    (sy, s.systring))
782
    if kind == 'c':
783 784 785 786
        unicode_value = None
        bytes_value = chars.getchar()
        if len(bytes_value) != 1:
            error(pos, u"invalid character literal: %r" % bytes_value)
787
    else:
788
        bytes_value, unicode_value = chars.getstrings()
789
        if is_python3_source and has_non_ASCII_literal_characters:
790 791 792 793
            # Python 3 forbids literal non-ASCII characters in byte strings
            if kind != 'u':
                s.error("bytes can only contain ASCII literal characters.", pos = pos)
            bytes_value = None
William Stein's avatar
William Stein committed
794
    s.next()
795
    return (kind, bytes_value, unicode_value)
William Stein's avatar
William Stein committed
796

Robert Bradshaw's avatar
Robert Bradshaw committed
797
# list_display      ::=      "[" [listmaker] "]"
Stefan Behnel's avatar
Stefan Behnel committed
798 799 800 801
# listmaker     ::=     expression ( comp_for | ( "," expression )* [","] )
# comp_iter     ::=     comp_for | comp_if
# comp_for     ::=     "for" expression_list "in" testlist [comp_iter]
# comp_if     ::=     "if" test [comp_iter]
Robert Bradshaw's avatar
Robert Bradshaw committed
802
        
William Stein's avatar
William Stein committed
803 804 805 806
def p_list_maker(s):
    # s.sy == '['
    pos = s.position()
    s.next()
Robert Bradshaw's avatar
Robert Bradshaw committed
807 808 809
    if s.sy == ']':
        s.expect(']')
        return ExprNodes.ListNode(pos, args = [])
810
    expr = p_test(s)
Robert Bradshaw's avatar
Robert Bradshaw committed
811
    if s.sy == 'for':
812 813 814
        target = ExprNodes.ListNode(pos, args = [])
        append = ExprNodes.ComprehensionAppendNode(
            pos, expr=expr, target=ExprNodes.CloneNode(target))
815
        loop = p_comp_for(s, append)
Robert Bradshaw's avatar
Robert Bradshaw committed
816
        s.expect(']')
817
        return ExprNodes.ComprehensionNode(
818 819
            pos, loop=loop, append=append, target=target,
            # list comprehensions leak their loop variable in Py2
Stefan Behnel's avatar
Stefan Behnel committed
820
            has_local_scope = s.context.language_level >= 3)
Robert Bradshaw's avatar
Robert Bradshaw committed
821 822 823
    else:
        if s.sy == ',':
            s.next()
824 825 826
            exprs = p_simple_expr_list(s, expr)
        else:
            exprs = [expr]
Robert Bradshaw's avatar
Robert Bradshaw committed
827 828 829
        s.expect(']')
        return ExprNodes.ListNode(pos, args = exprs)
        
Stefan Behnel's avatar
Stefan Behnel committed
830
def p_comp_iter(s, body):
Robert Bradshaw's avatar
Robert Bradshaw committed
831
    if s.sy == 'for':
Stefan Behnel's avatar
Stefan Behnel committed
832
        return p_comp_for(s, body)
Robert Bradshaw's avatar
Robert Bradshaw committed
833
    elif s.sy == 'if':
Stefan Behnel's avatar
Stefan Behnel committed
834
        return p_comp_if(s, body)
Robert Bradshaw's avatar
Robert Bradshaw committed
835
    else:
836 837
        # insert the 'append' operation into the loop
        return body
William Stein's avatar
William Stein committed
838

Stefan Behnel's avatar
Stefan Behnel committed
839
def p_comp_for(s, body):
Robert Bradshaw's avatar
Robert Bradshaw committed
840 841 842
    # s.sy == 'for'
    pos = s.position()
    s.next()
843
    kw = p_for_bounds(s, allow_testlist=False)
Robert Bradshaw's avatar
Robert Bradshaw committed
844
    kw['else_clause'] = None
Stefan Behnel's avatar
Stefan Behnel committed
845
    kw['body'] = p_comp_iter(s, body)
Robert Bradshaw's avatar
Robert Bradshaw committed
846 847
    return Nodes.ForStatNode(pos, **kw)
        
Stefan Behnel's avatar
Stefan Behnel committed
848
def p_comp_if(s, body):
Robert Bradshaw's avatar
Robert Bradshaw committed
849 850 851
    # s.sy == 'if'
    pos = s.position()
    s.next()
Stefan Behnel's avatar
Stefan Behnel committed
852
    test = p_test_nocond(s)
Robert Bradshaw's avatar
Robert Bradshaw committed
853
    return Nodes.IfStatNode(pos, 
854
        if_clauses = [Nodes.IfClauseNode(pos, condition = test,
Stefan Behnel's avatar
Stefan Behnel committed
855
                                         body = p_comp_iter(s, body))],
Robert Bradshaw's avatar
Robert Bradshaw committed
856
        else_clause = None )
857

William Stein's avatar
William Stein committed
858 859
#dictmaker: test ':' test (',' test ':' test)* [',']

860
def p_dict_or_set_maker(s):
William Stein's avatar
William Stein committed
861 862 863
    # s.sy == '{'
    pos = s.position()
    s.next()
864
    if s.sy == '}':
William Stein's avatar
William Stein committed
865
        s.next()
866
        return ExprNodes.DictNode(pos, key_value_pairs = [])
867
    item = p_test(s)
868 869 870 871 872
    if s.sy == ',' or s.sy == '}':
        # set literal
        values = [item]
        while s.sy == ',':
            s.next()
873 874
            if s.sy == '}':
                break
875
            values.append( p_test(s) )
876 877 878 879
        s.expect('}')
        return ExprNodes.SetNode(pos, args=values)
    elif s.sy == 'for':
        # set comprehension
880 881 882
        target = ExprNodes.SetNode(pos, args=[])
        append = ExprNodes.ComprehensionAppendNode(
            item.pos, expr=item, target=ExprNodes.CloneNode(target))
883
        loop = p_comp_for(s, append)
884
        s.expect('}')
885 886
        return ExprNodes.ComprehensionNode(
            pos, loop=loop, append=append, target=target)
887 888 889 890
    elif s.sy == ':':
        # dict literal or comprehension
        key = item
        s.next()
891
        value = p_test(s)
892 893
        if s.sy == 'for':
            # dict comprehension
894
            target = ExprNodes.DictNode(pos, key_value_pairs = [])
895
            append = ExprNodes.DictComprehensionAppendNode(
896 897
                item.pos, key_expr=key, value_expr=value,
                target=ExprNodes.CloneNode(target))
898
            loop = p_comp_for(s, append)
899 900 901
            s.expect('}')
            return ExprNodes.ComprehensionNode(
                pos, loop=loop, append=append, target=target)
902 903 904 905 906
        else:
            # dict literal
            items = [ExprNodes.DictItemNode(key.pos, key=key, value=value)]
            while s.sy == ',':
                s.next()
907 908
                if s.sy == '}':
                    break
909
                key = p_test(s)
910
                s.expect(':')
911
                value = p_test(s)
912 913 914 915 916 917 918 919
                items.append(
                    ExprNodes.DictItemNode(key.pos, key=key, value=value))
            s.expect('}')
            return ExprNodes.DictNode(pos, key_value_pairs=items)
    else:
        # raise an error
        s.expect('}')
    return ExprNodes.DictNode(pos, key_value_pairs = [])
William Stein's avatar
William Stein committed
920

921
# NOTE: no longer in Py3 :)
William Stein's avatar
William Stein committed
922 923 924 925
def p_backquote_expr(s):
    # s.sy == '`'
    pos = s.position()
    s.next()
926 927 928 929
    args = [p_test(s)]
    while s.sy == ',':
        s.next()
        args.append(p_test(s))
William Stein's avatar
William Stein committed
930
    s.expect('`')
931 932 933 934
    if len(args) == 1:
        arg = args[0]
    else:
        arg = ExprNodes.TupleNode(pos, args = args)
William Stein's avatar
William Stein committed
935 936
    return ExprNodes.BackquoteNode(pos, arg = arg)

937 938
def p_simple_expr_list(s, expr=None):
    exprs = expr is not None and [expr] or []
William Stein's avatar
William Stein committed
939
    while s.sy not in expr_terminators:
940
        exprs.append( p_test(s) )
Stefan Behnel's avatar
Stefan Behnel committed
941
        if s.sy != ',':
William Stein's avatar
William Stein committed
942 943 944 945
            break
        s.next()
    return exprs

946 947 948 949 950 951 952 953 954 955 956 957 958
def p_test_or_starred_expr_list(s, expr=None):
    exprs = expr is not None and [expr] or []
    while s.sy not in expr_terminators:
        exprs.append( p_test_or_starred_expr(s) )
        if s.sy != ',':
            break
        s.next()
    return exprs
    

#testlist: test (',' test)* [',']

def p_testlist(s):
William Stein's avatar
William Stein committed
959
    pos = s.position()
960
    expr = p_test(s)
William Stein's avatar
William Stein committed
961 962
    if s.sy == ',':
        s.next()
963
        exprs = p_simple_expr_list(s, expr)
William Stein's avatar
William Stein committed
964 965 966 967
        return ExprNodes.TupleNode(pos, args = exprs)
    else:
        return expr

968
# testlist_star_expr: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
Robert Bradshaw's avatar
Robert Bradshaw committed
969

970
def p_testlist_star_expr(s):
Robert Bradshaw's avatar
Robert Bradshaw committed
971
    pos = s.position()
972
    expr = p_test_or_starred_expr(s)
Robert Bradshaw's avatar
Robert Bradshaw committed
973
    if s.sy == ',':
974
        s.next()
975 976
        exprs = p_test_or_starred_expr_list(s, expr)
        return ExprNodes.TupleNode(pos, args = exprs)
Robert Bradshaw's avatar
Robert Bradshaw committed
977 978 979
    else:
        return expr

980 981 982
# testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )

def p_testlist_comp(s):
Robert Bradshaw's avatar
Robert Bradshaw committed
983
    pos = s.position()
984
    expr = p_test_or_starred_expr(s)
Robert Bradshaw's avatar
Robert Bradshaw committed
985
    if s.sy == ',':
986
        s.next()
987
        exprs = p_test_or_starred_expr_list(s, expr)
Robert Bradshaw's avatar
Robert Bradshaw committed
988
        return ExprNodes.TupleNode(pos, args = exprs)
989 990
    elif s.sy == 'for':
        return p_genexp(s, expr)
Robert Bradshaw's avatar
Robert Bradshaw committed
991 992
    else:
        return expr
993 994 995

def p_genexp(s, expr):
    # s.sy == 'for'
996 997
    loop = p_comp_for(s, Nodes.ExprStatNode(
        expr.pos, expr = ExprNodes.YieldExprNode(expr.pos, arg=expr)))
998 999
    return ExprNodes.GeneratorExpressionNode(expr.pos, loop=loop)

William Stein's avatar
William Stein committed
1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015
expr_terminators = (')', ']', '}', ':', '=', 'NEWLINE')

#-------------------------------------------------------
#
#   Statements
#
#-------------------------------------------------------

def p_global_statement(s):
    # assume s.sy == 'global'
    pos = s.position()
    s.next()
    names = p_ident_list(s)
    return Nodes.GlobalNode(pos, names = names)

def p_expression_or_assignment(s):
1016
    expr_list = [p_testlist_star_expr(s)]
William Stein's avatar
William Stein committed
1017 1018
    while s.sy == '=':
        s.next()
1019 1020 1021 1022 1023
        if s.sy == 'yield':
            expr = p_yield_expression(s)
        else:
            expr = p_testlist_star_expr(s)
        expr_list.append(expr)
William Stein's avatar
William Stein committed
1024
    if len(expr_list) == 1:
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
1025
        if re.match(r"([+*/\%^\&|-]|<<|>>|\*\*|//)=", s.sy):
1026 1027 1028
            lhs = expr_list[0]
            if not isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode, ExprNodes.NameNode) ):
                error(lhs.pos, "Illegal operand for inplace operation.")
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
1029
            operator = s.sy[:-1]
1030
            s.next()
1031 1032 1033 1034
            if s.sy == 'yield':
                rhs = p_yield_expression(s)
            else:
                rhs = p_testlist(s)
1035
            return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs)
1036
        expr = expr_list[0]
1037
        if isinstance(expr, (ExprNodes.UnicodeNode, ExprNodes.StringNode, ExprNodes.BytesNode)):
1038
            return Nodes.PassStatNode(expr.pos)
1039 1040
        else:
            return Nodes.ExprStatNode(expr.pos, expr = expr)
1041

1042 1043
    rhs = expr_list[-1]
    if len(expr_list) == 2:
1044
        return Nodes.SingleAssignmentNode(rhs.pos, 
1045
            lhs = expr_list[0], rhs = rhs)
William Stein's avatar
William Stein committed
1046
    else:
1047
        return Nodes.CascadedAssignmentNode(rhs.pos,
1048
            lhs_list = expr_list[:-1], rhs = rhs)
William Stein's avatar
William Stein committed
1049 1050 1051 1052

def p_print_statement(s):
    # s.sy == 'print'
    pos = s.position()
1053
    ends_with_comma = 0
William Stein's avatar
William Stein committed
1054 1055
    s.next()
    if s.sy == '>>':
1056
        s.next()
1057
        stream = p_test(s)
1058 1059 1060 1061 1062
        if s.sy == ',':
            s.next()
            ends_with_comma = s.sy in ('NEWLINE', 'EOF')
    else:
        stream = None
William Stein's avatar
William Stein committed
1063 1064
    args = []
    if s.sy not in ('NEWLINE', 'EOF'):
1065
        args.append(p_test(s))
William Stein's avatar
William Stein committed
1066 1067 1068
        while s.sy == ',':
            s.next()
            if s.sy in ('NEWLINE', 'EOF'):
1069
                ends_with_comma = 1
William Stein's avatar
William Stein committed
1070
                break
1071
            args.append(p_test(s))
1072
    arg_tuple = ExprNodes.TupleNode(pos, args = args)
1073
    return Nodes.PrintStatNode(pos,
1074 1075
        arg_tuple = arg_tuple, stream = stream,
        append_newline = not ends_with_comma)
William Stein's avatar
William Stein committed
1076

1077 1078 1079 1080 1081 1082 1083
def p_exec_statement(s):
    # s.sy == 'exec'
    pos = s.position()
    s.next()
    args = [ p_bit_expr(s) ]
    if s.sy == 'in':
        s.next()
1084
        args.append(p_test(s))
1085 1086
        if s.sy == ',':
            s.next()
1087
            args.append(p_test(s))
1088 1089 1090 1091
    else:
        error(pos, "'exec' currently requires a target mapping (globals/locals)")
    return Nodes.ExecStatNode(pos, args = args)

William Stein's avatar
William Stein committed
1092 1093 1094 1095
def p_del_statement(s):
    # s.sy == 'del'
    pos = s.position()
    s.next()
1096
    # FIXME: 'exprlist' in Python
William Stein's avatar
William Stein committed
1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123
    args = p_simple_expr_list(s)
    return Nodes.DelStatNode(pos, args = args)

def p_pass_statement(s, with_newline = 0):
    pos = s.position()
    s.expect('pass')
    if with_newline:
        s.expect_newline("Expected a newline")
    return Nodes.PassStatNode(pos)

def p_break_statement(s):
    # s.sy == 'break'
    pos = s.position()
    s.next()
    return Nodes.BreakStatNode(pos)

def p_continue_statement(s):
    # s.sy == 'continue'
    pos = s.position()
    s.next()
    return Nodes.ContinueStatNode(pos)

def p_return_statement(s):
    # s.sy == 'return'
    pos = s.position()
    s.next()
    if s.sy not in statement_terminators:
1124
        value = p_testlist(s)
William Stein's avatar
William Stein committed
1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136
    else:
        value = None
    return Nodes.ReturnStatNode(pos, value = value)

def p_raise_statement(s):
    # s.sy == 'raise'
    pos = s.position()
    s.next()
    exc_type = None
    exc_value = None
    exc_tb = None
    if s.sy not in statement_terminators:
1137
        exc_type = p_test(s)
William Stein's avatar
William Stein committed
1138 1139
        if s.sy == ',':
            s.next()
1140
            exc_value = p_test(s)
William Stein's avatar
William Stein committed
1141 1142
            if s.sy == ',':
                s.next()
1143
                exc_tb = p_test(s)
1144 1145 1146 1147 1148 1149 1150
    if exc_type or exc_value or exc_tb:
        return Nodes.RaiseStatNode(pos, 
            exc_type = exc_type,
            exc_value = exc_value,
            exc_tb = exc_tb)
    else:
        return Nodes.ReraiseStatNode(pos)
William Stein's avatar
William Stein committed
1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162

def p_import_statement(s):
    # s.sy in ('import', 'cimport')
    pos = s.position()
    kind = s.sy
    s.next()
    items = [p_dotted_name(s, as_allowed = 1)]
    while s.sy == ',':
        s.next()
        items.append(p_dotted_name(s, as_allowed = 1))
    stats = []
    for pos, target_name, dotted_name, as_name in items:
1163
        dotted_name = EncodedString(dotted_name)
William Stein's avatar
William Stein committed
1164 1165 1166 1167 1168
        if kind == 'cimport':
            stat = Nodes.CImportStatNode(pos, 
                module_name = dotted_name,
                as_name = as_name)
        else:
1169 1170
            if as_name and "." in dotted_name:
                name_list = ExprNodes.ListNode(pos, args = [
1171
                        ExprNodes.IdentifierStringNode(pos, value = EncodedString("*"))])
1172 1173
            else:
                name_list = None
William Stein's avatar
William Stein committed
1174 1175 1176 1177
            stat = Nodes.SingleAssignmentNode(pos,
                lhs = ExprNodes.NameNode(pos, 
                    name = as_name or target_name),
                rhs = ExprNodes.ImportNode(pos, 
1178
                    module_name = ExprNodes.IdentifierStringNode(
1179
                        pos, value = dotted_name),
1180
                    name_list = name_list))
William Stein's avatar
William Stein committed
1181 1182 1183
        stats.append(stat)
    return Nodes.StatListNode(pos, stats = stats)

Stefan Behnel's avatar
Stefan Behnel committed
1184
def p_from_import_statement(s, first_statement = 0):
William Stein's avatar
William Stein committed
1185 1186 1187 1188 1189 1190 1191 1192 1193 1194
    # s.sy == 'from'
    pos = s.position()
    s.next()
    (dotted_name_pos, _, dotted_name, _) = \
        p_dotted_name(s, as_allowed = 0)
    if s.sy in ('import', 'cimport'):
        kind = s.sy
        s.next()
    else:
        s.error("Expected 'import' or 'cimport'")
1195
    is_cimport = kind == 'cimport'
1196
    is_parenthesized = False
William Stein's avatar
William Stein committed
1197
    if s.sy == '*':
1198
        imported_names = [(s.position(), "*", None, None)]
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
1199 1200
        s.next()
    else:
1201 1202 1203
        if s.sy == '(':
            is_parenthesized = True
            s.next()
1204
        imported_names = [p_imported_name(s, is_cimport)]
William Stein's avatar
William Stein committed
1205 1206
    while s.sy == ',':
        s.next()
1207 1208
        if is_parenthesized and s.sy == ')':
            break
1209
        imported_names.append(p_imported_name(s, is_cimport))
1210 1211
    if is_parenthesized:
        s.expect(')')
1212
    dotted_name = EncodedString(dotted_name)
Stefan Behnel's avatar
Stefan Behnel committed
1213 1214 1215 1216
    if dotted_name == '__future__':
        if not first_statement:
            s.error("from __future__ imports must occur at the beginning of the file")
        else:
1217
            for (name_pos, name, as_name, kind) in imported_names:
1218 1219 1220
                if name == "braces":
                    s.error("not a chance", name_pos)
                    break
Stefan Behnel's avatar
Stefan Behnel committed
1221 1222 1223
                try:
                    directive = getattr(Future, name)
                except AttributeError:
1224
                    s.error("future feature %s is not defined" % name, name_pos)
Stefan Behnel's avatar
Stefan Behnel committed
1225 1226 1227 1228
                    break
                s.context.future_directives.add(directive)
        return Nodes.PassStatNode(pos)
    elif kind == 'cimport':
William Stein's avatar
William Stein committed
1229 1230 1231 1232 1233 1234
        return Nodes.FromCImportStatNode(pos,
            module_name = dotted_name,
            imported_names = imported_names)
    else:
        imported_name_strings = []
        items = []
1235
        for (name_pos, name, as_name, kind) in imported_names:
1236
            encoded_name = EncodedString(name)
William Stein's avatar
William Stein committed
1237
            imported_name_strings.append(
1238
                ExprNodes.IdentifierStringNode(name_pos, value = encoded_name))
William Stein's avatar
William Stein committed
1239 1240 1241
            items.append(
                (name,
                 ExprNodes.NameNode(name_pos, 
Stefan Behnel's avatar
Stefan Behnel committed
1242
                                    name = as_name or name)))
William Stein's avatar
William Stein committed
1243 1244
        import_list = ExprNodes.ListNode(
            imported_names[0][0], args = imported_name_strings)
1245
        dotted_name = EncodedString(dotted_name)
William Stein's avatar
William Stein committed
1246 1247
        return Nodes.FromImportStatNode(pos,
            module = ExprNodes.ImportNode(dotted_name_pos,
1248
                module_name = ExprNodes.IdentifierStringNode(pos, value = dotted_name),
William Stein's avatar
William Stein committed
1249 1250 1251
                name_list = import_list),
            items = items)

1252 1253 1254
imported_name_kinds = ('class', 'struct', 'union')

def p_imported_name(s, is_cimport):
William Stein's avatar
William Stein committed
1255
    pos = s.position()
1256 1257 1258 1259
    kind = None
    if is_cimport and s.systring in imported_name_kinds:
        kind = s.systring
        s.next()
William Stein's avatar
William Stein committed
1260 1261
    name = p_ident(s)
    as_name = p_as_name(s)
1262
    return (pos, name, as_name, kind)
William Stein's avatar
William Stein committed
1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273

def p_dotted_name(s, as_allowed):
    pos = s.position()
    target_name = p_ident(s)
    as_name = None
    names = [target_name]
    while s.sy == '.':
        s.next()
        names.append(p_ident(s))
    if as_allowed:
        as_name = p_as_name(s)
Stefan Behnel's avatar
Stefan Behnel committed
1274
    return (pos, target_name, u'.'.join(names), as_name)
William Stein's avatar
William Stein committed
1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286

def p_as_name(s):
    if s.sy == 'IDENT' and s.systring == 'as':
        s.next()
        return p_ident(s)
    else:
        return None

def p_assert_statement(s):
    # s.sy == 'assert'
    pos = s.position()
    s.next()
1287
    cond = p_test(s)
William Stein's avatar
William Stein committed
1288 1289
    if s.sy == ',':
        s.next()
1290
        value = p_test(s)
William Stein's avatar
William Stein committed
1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310
    else:
        value = None
    return Nodes.AssertStatNode(pos, cond = cond, value = value)

statement_terminators = (';', 'NEWLINE', 'EOF')

def p_if_statement(s):
    # s.sy == 'if'
    pos = s.position()
    s.next()
    if_clauses = [p_if_clause(s)]
    while s.sy == 'elif':
        s.next()
        if_clauses.append(p_if_clause(s))
    else_clause = p_else_clause(s)
    return Nodes.IfStatNode(pos,
        if_clauses = if_clauses, else_clause = else_clause)

def p_if_clause(s):
    pos = s.position()
1311
    test = p_test(s)
William Stein's avatar
William Stein committed
1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326
    body = p_suite(s)
    return Nodes.IfClauseNode(pos,
        condition = test, body = body)

def p_else_clause(s):
    if s.sy == 'else':
        s.next()
        return p_suite(s)
    else:
        return None

def p_while_statement(s):
    # s.sy == 'while'
    pos = s.position()
    s.next()
1327
    test = p_test(s)
William Stein's avatar
William Stein committed
1328 1329 1330 1331 1332 1333 1334 1335 1336 1337
    body = p_suite(s)
    else_clause = p_else_clause(s)
    return Nodes.WhileStatNode(pos, 
        condition = test, body = body, 
        else_clause = else_clause)

def p_for_statement(s):
    # s.sy == 'for'
    pos = s.position()
    s.next()
1338
    kw = p_for_bounds(s, allow_testlist=True)
Robert Bradshaw's avatar
Robert Bradshaw committed
1339 1340 1341 1342
    kw['body'] = p_suite(s)
    kw['else_clause'] = p_else_clause(s)
    return Nodes.ForStatNode(pos, **kw)
            
1343
def p_for_bounds(s, allow_testlist=True):
William Stein's avatar
William Stein committed
1344 1345 1346
    target = p_for_target(s)
    if s.sy == 'in':
        s.next()
1347
        iterator = p_for_iterator(s, allow_testlist)
Robert Bradshaw's avatar
Robert Bradshaw committed
1348
        return { 'target': target, 'iterator': iterator }
1349
    elif not s.in_python_file:
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
1350 1351 1352 1353 1354 1355
        if s.sy == 'from':
            s.next()
            bound1 = p_bit_expr(s)
        else:
            # Support shorter "for a <= x < b" syntax
            bound1, target = target, None
William Stein's avatar
William Stein committed
1356 1357 1358 1359 1360 1361
        rel1 = p_for_from_relation(s)
        name2_pos = s.position()
        name2 = p_ident(s)
        rel2_pos = s.position()
        rel2 = p_for_from_relation(s)
        bound2 = p_bit_expr(s)
1362
        step = p_for_from_step(s)
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
1363 1364 1365 1366 1367 1368 1369 1370 1371
        if target is None:
            target = ExprNodes.NameNode(name2_pos, name = name2)
        else:
            if not target.is_name:
                error(target.pos, 
                    "Target of for-from statement must be a variable name")
            elif name2 != target.name:
                error(name2_pos,
                    "Variable name in for-from range does not match target")
Stefan Behnel's avatar
Stefan Behnel committed
1372
        if rel1[0] != rel2[0]:
William Stein's avatar
William Stein committed
1373 1374
            error(rel2_pos,
                "Relation directions in for-from do not match")
Robert Bradshaw's avatar
Robert Bradshaw committed
1375 1376 1377 1378
        return {'target': target, 
                'bound1': bound1, 
                'relation1': rel1, 
                'relation2': rel2,
1379 1380
                'bound2': bound2,
                'step': step }
1381 1382 1383
    else:
        s.expect('in')
        return {}
William Stein's avatar
William Stein committed
1384 1385 1386 1387 1388 1389 1390 1391

def p_for_from_relation(s):
    if s.sy in inequality_relations:
        op = s.sy
        s.next()
        return op
    else:
        s.error("Expected one of '<', '<=', '>' '>='")
1392

1393 1394 1395 1396 1397 1398 1399
def p_for_from_step(s):
    if s.sy == 'by':
        s.next()
        step = p_bit_expr(s)
        return step
    else:
        return None
William Stein's avatar
William Stein committed
1400 1401 1402

inequality_relations = ('<', '<=', '>', '>=')

1403
def p_target(s, terminator):
William Stein's avatar
William Stein committed
1404
    pos = s.position()
1405
    expr = p_starred_expr(s)
William Stein's avatar
William Stein committed
1406 1407 1408
    if s.sy == ',':
        s.next()
        exprs = [expr]
1409
        while s.sy != terminator:
1410
            exprs.append(p_starred_expr(s))
Stefan Behnel's avatar
Stefan Behnel committed
1411
            if s.sy != ',':
William Stein's avatar
William Stein committed
1412 1413 1414 1415 1416 1417
                break
            s.next()
        return ExprNodes.TupleNode(pos, args = exprs)
    else:
        return expr

1418 1419 1420
def p_for_target(s):
    return p_target(s, 'in')

1421
def p_for_iterator(s, allow_testlist=True):
William Stein's avatar
William Stein committed
1422
    pos = s.position()
1423 1424 1425 1426
    if allow_testlist:
        expr = p_testlist(s)
    else:
        expr = p_or_test(s)
William Stein's avatar
William Stein committed
1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441
    return ExprNodes.IteratorNode(pos, sequence = expr)

def p_try_statement(s):
    # s.sy == 'try'
    pos = s.position()
    s.next()
    body = p_suite(s)
    except_clauses = []
    else_clause = None
    if s.sy in ('except', 'else'):
        while s.sy == 'except':
            except_clauses.append(p_except_clause(s))
        if s.sy == 'else':
            s.next()
            else_clause = p_suite(s)
1442
        body = Nodes.TryExceptStatNode(pos,
William Stein's avatar
William Stein committed
1443 1444
            body = body, except_clauses = except_clauses,
            else_clause = else_clause)
1445 1446 1447 1448
        if s.sy != 'finally':
            return body
        # try-except-finally is equivalent to nested try-except/try-finally
    if s.sy == 'finally':
William Stein's avatar
William Stein committed
1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461
        s.next()
        finally_clause = p_suite(s)
        return Nodes.TryFinallyStatNode(pos,
            body = body, finally_clause = finally_clause)
    else:
        s.error("Expected 'except' or 'finally'")

def p_except_clause(s):
    # s.sy == 'except'
    pos = s.position()
    s.next()
    exc_type = None
    exc_value = None
Stefan Behnel's avatar
Stefan Behnel committed
1462
    if s.sy != ':':
1463
        exc_type = p_test(s)
1464 1465 1466 1467 1468
        # normalise into list of single exception tests
        if isinstance(exc_type, ExprNodes.TupleNode):
            exc_type = exc_type.args
        else:
            exc_type = [exc_type]
1469
        if s.sy == ',' or (s.sy == 'IDENT' and s.systring == 'as'):
William Stein's avatar
William Stein committed
1470
            s.next()
1471
            exc_value = p_test(s)
1472
        elif s.sy == 'IDENT' and s.systring == 'as':
1473
            # Py3 syntax requires a name here
1474
            s.next()
1475 1476 1477
            pos2 = s.position()
            name = p_ident(s)
            exc_value = ExprNodes.NameNode(pos2, name = name)
William Stein's avatar
William Stein committed
1478 1479 1480 1481
    body = p_suite(s)
    return Nodes.ExceptClauseNode(pos,
        pattern = exc_type, target = exc_value, body = body)

1482
def p_include_statement(s, ctx):
William Stein's avatar
William Stein committed
1483 1484
    pos = s.position()
    s.next() # 'include'
1485
    unicode_include_file_name = p_string_literal(s, 'u')[2]
William Stein's avatar
William Stein committed
1486
    s.expect_newline("Syntax error in include statement")
1487
    if s.compile_time_eval:
1488
        include_file_name = unicode_include_file_name
1489 1490
        include_file_path = s.context.find_include_file(include_file_name, pos)
        if include_file_path:
1491
            s.included_files.append(include_file_name)
1492
            f = Utils.open_source_file(include_file_path, mode="rU")
1493
            source_desc = FileSourceDescriptor(include_file_path)
1494
            s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding, parse_comments=s.parse_comments)
1495
            try:
1496
                tree = p_statement_list(s2, ctx)
1497 1498 1499 1500 1501
            finally:
                f.close()
            return tree
        else:
            return None
William Stein's avatar
William Stein committed
1502
    else:
1503 1504 1505 1506 1507
        return Nodes.PassStatNode(pos)

def p_with_statement(s):
    pos = s.position()
    s.next() # 'with'
Robert Bradshaw's avatar
Robert Bradshaw committed
1508
#    if s.sy == 'IDENT' and s.systring in ('gil', 'nogil'):
1509 1510 1511 1512 1513
    if s.sy == 'IDENT' and s.systring == 'nogil':
        state = s.systring
        s.next()
        body = p_suite(s)
        return Nodes.GILStatNode(pos, state = state, body = body)
Danilo Freitas's avatar
Danilo Freitas committed
1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536
    elif s.systring == 'template':
        templates = []
        s.next()
        s.expect('[')
        #s.next()
        templates.append(s.systring)
        s.next()
        while s.systring == ',':
            s.next()
            templates.append(s.systring)
            s.next()
        s.expect(']')
        if s.sy == ':':
            s.next()
            s.expect_newline("Syntax error in template function declaration")
            s.expect_indent()
            body_ctx = Ctx()
            body_ctx.templates = templates
            func_or_var = p_c_func_or_var_declaration(s, pos, body_ctx)
            s.expect_dedent()
            return func_or_var
        else:
            error(pos, "Syntax error in template function declaration")
1537
    else:
1538
        manager = p_test(s)
1539 1540 1541 1542 1543 1544 1545 1546
        target = None
        if s.sy == 'IDENT' and s.systring == 'as':
            s.next()
            allow_multi = (s.sy == '(')
            target = p_target(s, ':')
            if not allow_multi and isinstance(target, ExprNodes.TupleNode):
                s.error("Multiple with statement target values not allowed without paranthesis")
        body = p_suite(s)
1547
    return Nodes.WithStatNode(pos, manager = manager, 
Robert Bradshaw's avatar
Robert Bradshaw committed
1548
                              target = target, body = body)
William Stein's avatar
William Stein committed
1549
    
Stefan Behnel's avatar
Stefan Behnel committed
1550
def p_simple_statement(s, first_statement = 0):
William Stein's avatar
William Stein committed
1551 1552 1553 1554 1555
    #print "p_simple_statement:", s.sy, s.systring ###
    if s.sy == 'global':
        node = p_global_statement(s)
    elif s.sy == 'print':
        node = p_print_statement(s)
1556 1557
    elif s.sy == 'exec':
        node = p_exec_statement(s)
William Stein's avatar
William Stein committed
1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570
    elif s.sy == 'del':
        node = p_del_statement(s)
    elif s.sy == 'break':
        node = p_break_statement(s)
    elif s.sy == 'continue':
        node = p_continue_statement(s)
    elif s.sy == 'return':
        node = p_return_statement(s)
    elif s.sy == 'raise':
        node = p_raise_statement(s)
    elif s.sy in ('import', 'cimport'):
        node = p_import_statement(s)
    elif s.sy == 'from':
Stefan Behnel's avatar
Stefan Behnel committed
1571
        node = p_from_import_statement(s, first_statement = first_statement)
1572
    elif s.sy == 'yield':
1573
        node = p_yield_statement(s)
William Stein's avatar
William Stein committed
1574 1575 1576 1577 1578 1579 1580 1581
    elif s.sy == 'assert':
        node = p_assert_statement(s)
    elif s.sy == 'pass':
        node = p_pass_statement(s)
    else:
        node = p_expression_or_assignment(s)
    return node

1582
def p_simple_statement_list(s, ctx, first_statement = 0):
William Stein's avatar
William Stein committed
1583 1584
    # Parse a series of simple statements on one line
    # separated by semicolons.
Stefan Behnel's avatar
Stefan Behnel committed
1585
    stat = p_simple_statement(s, first_statement = first_statement)
William Stein's avatar
William Stein committed
1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597
    if s.sy == ';':
        stats = [stat]
        while s.sy == ';':
            #print "p_simple_statement_list: maybe more to follow" ###
            s.next()
            if s.sy in ('NEWLINE', 'EOF'):
                break
            stats.append(p_simple_statement(s))
        stat = Nodes.StatListNode(stats[0].pos, stats = stats)
    s.expect_newline("Syntax error in simple statement list")
    return stat

1598 1599 1600
def p_compile_time_expr(s):
    old = s.compile_time_expr
    s.compile_time_expr = 1
1601
    expr = p_testlist(s)
1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617
    s.compile_time_expr = old
    return expr

def p_DEF_statement(s):
    pos = s.position()
    denv = s.compile_time_env
    s.next() # 'DEF'
    name = p_ident(s)
    s.expect('=')
    expr = p_compile_time_expr(s)
    value = expr.compile_time_value(denv)
    #print "p_DEF_statement: %s = %r" % (name, value) ###
    denv.declare(name, value)
    s.expect_newline()
    return Nodes.PassStatNode(pos)

1618
def p_IF_statement(s, ctx):
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
1619
    pos = s.position()
1620 1621 1622 1623 1624 1625 1626 1627
    saved_eval = s.compile_time_eval
    current_eval = saved_eval
    denv = s.compile_time_env
    result = None
    while 1:
        s.next() # 'IF' or 'ELIF'
        expr = p_compile_time_expr(s)
        s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv))
1628
        body = p_suite(s, ctx)
1629 1630 1631
        if s.compile_time_eval:
            result = body
            current_eval = 0
Stefan Behnel's avatar
Stefan Behnel committed
1632
        if s.sy != 'ELIF':
1633 1634 1635 1636
            break
    if s.sy == 'ELSE':
        s.next()
        s.compile_time_eval = current_eval
1637
        body = p_suite(s, ctx)
1638 1639 1640
        if current_eval:
            result = body
    if not result:
Stefan Behnel's avatar
Stefan Behnel committed
1641
        result = Nodes.PassStatNode(pos)
1642 1643 1644
    s.compile_time_eval = saved_eval
    return result

1645 1646
def p_statement(s, ctx, first_statement = 0):
    cdef_flag = ctx.cdef_flag
Robert Bradshaw's avatar
Robert Bradshaw committed
1647
    decorators = None
William Stein's avatar
William Stein committed
1648
    if s.sy == 'ctypedef':
1649
        if ctx.level not in ('module', 'module_pxd'):
William Stein's avatar
William Stein committed
1650
            s.error("ctypedef statement not allowed here")
1651 1652
        #if ctx.api:
        #    error(s.position(), "'api' not allowed with 'ctypedef'")
1653
        return p_ctypedef_statement(s, ctx)
1654 1655 1656
    elif s.sy == 'DEF':
        return p_DEF_statement(s)
    elif s.sy == 'IF':
1657
        return p_IF_statement(s, ctx)
1658
    elif s.sy == 'DECORATOR':
1659
        if ctx.level not in ('module', 'class', 'c_class', 'function', 'property', 'module_pxd', 'c_class_pxd'):
1660
            print ctx.level
1661 1662 1663
            s.error('decorator not allowed here')
        s.level = ctx.level
        decorators = p_decorators(s)
1664 1665
        if s.sy not in ('def', 'cdef', 'cpdef', 'class'):
            s.error("Decorators can only be followed by functions or classes")
1666 1667 1668
    elif s.sy == 'pass' and cdef_flag:
        # empty cdef block
        return p_pass_statement(s, with_newline = 1)
1669 1670 1671 1672 1673

    overridable = 0
    if s.sy == 'cdef':
        cdef_flag = 1
        s.next()
1674
    elif s.sy == 'cpdef':
1675 1676 1677 1678 1679 1680 1681 1682 1683
        cdef_flag = 1
        overridable = 1
        s.next()
    if cdef_flag:
        if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):
            s.error('cdef statement not allowed here')
        s.level = ctx.level
        node = p_cdef_statement(s, ctx(overridable = overridable))
        if decorators is not None:
1684 1685
            if not isinstance(node, (Nodes.CFuncDefNode, Nodes.CVarDefNode, Nodes.CClassDefNode)):
                s.error("Decorators can only be followed by functions or classes")
1686 1687
            node.decorators = decorators
        return node
William Stein's avatar
William Stein committed
1688
    else:
1689
        if ctx.api:
1690
            s.error("'api' not allowed with this statement")
1691
        elif s.sy == 'def':
1692 1693 1694
            # def statements aren't allowed in pxd files, except
            # as part of a cdef class
            if ('pxd' in ctx.level) and (ctx.level != 'c_class_pxd'):
1695
                s.error('def statement not allowed here')
1696
            s.level = ctx.level
1697 1698 1699 1700
            return p_def_statement(s, decorators)
        elif s.sy == 'class':
            if ctx.level != 'module':
                s.error("class definition not allowed here")
1701
            return p_class_statement(s, decorators)
1702 1703 1704 1705 1706 1707 1708 1709
        elif s.sy == 'include':
            if ctx.level not in ('module', 'module_pxd'):
                s.error("include statement not allowed here")
            return p_include_statement(s, ctx)
        elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':
            return p_property_decl(s)
        elif s.sy == 'pass' and ctx.level != 'property':
            return p_pass_statement(s, with_newline = 1)
William Stein's avatar
William Stein committed
1710
        else:
1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722
            if ctx.level in ('c_class_pxd', 'property'):
                s.error("Executable statement not allowed here")
            if s.sy == 'if':
                return p_if_statement(s)
            elif s.sy == 'while':
                return p_while_statement(s)
            elif s.sy == 'for':
                return p_for_statement(s)
            elif s.sy == 'try':
                return p_try_statement(s)
            elif s.sy == 'with':
                return p_with_statement(s)
1723
            else:
1724 1725
                return p_simple_statement_list(
                    s, ctx, first_statement = first_statement)
William Stein's avatar
William Stein committed
1726

1727
def p_statement_list(s, ctx, first_statement = 0):
William Stein's avatar
William Stein committed
1728 1729 1730 1731
    # Parse a series of statements separated by newlines.
    pos = s.position()
    stats = []
    while s.sy not in ('DEDENT', 'EOF'):
1732
        stats.append(p_statement(s, ctx, first_statement = first_statement))
Stefan Behnel's avatar
Stefan Behnel committed
1733
        first_statement = 0
1734 1735 1736 1737
    if len(stats) == 1:
        return stats[0]
    else:
        return Nodes.StatListNode(pos, stats = stats)
William Stein's avatar
William Stein committed
1738

1739
def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):
William Stein's avatar
William Stein committed
1740 1741 1742 1743 1744 1745 1746
    pos = s.position()
    s.expect(':')
    doc = None
    stmts = []
    if s.sy == 'NEWLINE':
        s.next()
        s.expect_indent()
1747
        if with_doc or with_pseudo_doc:
William Stein's avatar
William Stein committed
1748
            doc = p_doc_string(s)
1749
        body = p_statement_list(s, ctx)
William Stein's avatar
William Stein committed
1750 1751
        s.expect_dedent()
    else:
1752
        if ctx.api:
1753
            s.error("'api' not allowed with this statement")
1754 1755
        if ctx.level in ('module', 'class', 'function', 'other'):
            body = p_simple_statement_list(s, ctx)
William Stein's avatar
William Stein committed
1756 1757 1758 1759 1760 1761 1762 1763
        else:
            body = p_pass_statement(s)
            s.expect_newline("Syntax error in declarations")
    if with_doc:
        return doc, body
    else:
        return body

1764
def p_positional_and_keyword_args(s, end_sy_set, templates = None):
1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780
    """
    Parses positional and keyword arguments. end_sy_set
    should contain any s.sy that terminate the argument list.
    Argument expansion (* and **) are not allowed.

    Returns: (positional_args, keyword_args)
    """
    positional_args = []
    keyword_args = []
    pos_idx = 0

    while s.sy not in end_sy_set:
        if s.sy == '*' or s.sy == '**':
            s.error('Argument expansion not allowed here.')

        parsed_type = False
1781
        if s.sy == 'IDENT' and s.peek()[0] == '=':
1782
            ident = s.systring
1783
            s.next() # s.sy is '='
1784
            s.next()
1785
            if looking_at_expr(s):
1786
                arg = p_test(s)
1787 1788
            else:
                base_type = p_c_base_type(s, templates = templates)
1789 1790 1791 1792 1793 1794 1795 1796
                declarator = p_c_declarator(s, empty = 1)
                arg = Nodes.CComplexBaseTypeNode(base_type.pos, 
                    base_type = base_type, declarator = declarator)
                parsed_type = True
            keyword_node = ExprNodes.IdentifierStringNode(
                arg.pos, value = EncodedString(ident))
            keyword_args.append((keyword_node, arg))
            was_keyword = True
1797
                
1798
        else:
1799
            if looking_at_expr(s):
1800
                arg = p_test(s)
1801 1802
            else:
                base_type = p_c_base_type(s, templates = templates)
1803 1804 1805
                declarator = p_c_declarator(s, empty = 1)
                arg = Nodes.CComplexBaseTypeNode(base_type.pos, 
                    base_type = base_type, declarator = declarator)
1806 1807 1808 1809 1810 1811 1812 1813 1814 1815
                parsed_type = True
            positional_args.append(arg)
            pos_idx += 1
            if len(keyword_args) > 0:
                s.error("Non-keyword arg following keyword arg",
                        pos = arg.pos)

        if s.sy != ',':
            if s.sy not in end_sy_set:
                if parsed_type:
1816
                    s.error("Unmatched %s" % " or ".join(end_sy_set))
1817 1818 1819 1820
            break
        s.next()
    return positional_args, keyword_args

Danilo Freitas's avatar
Danilo Freitas committed
1821
def p_c_base_type(s, self_flag = 0, nonempty = 0, templates = None):
William Stein's avatar
William Stein committed
1822 1823 1824 1825 1826
    # If self_flag is true, this is the base type for the
    # self argument of a C method of an extension type.
    if s.sy == '(':
        return p_c_complex_base_type(s)
    else:
Danilo Freitas's avatar
Danilo Freitas committed
1827
        return p_c_simple_base_type(s, self_flag, nonempty = nonempty, templates = templates)
William Stein's avatar
William Stein committed
1828

1829 1830 1831 1832 1833 1834 1835 1836
def p_calling_convention(s):
    if s.sy == 'IDENT' and s.systring in calling_convention_words:
        result = s.systring
        s.next()
        return result
    else:
        return ""

1837
calling_convention_words = ("__stdcall", "__cdecl", "__fastcall")
1838

William Stein's avatar
William Stein committed
1839 1840 1841 1842 1843 1844 1845 1846 1847 1848
def p_c_complex_base_type(s):
    # s.sy == '('
    pos = s.position()
    s.next()
    base_type = p_c_base_type(s)
    declarator = p_c_declarator(s, empty = 1)
    s.expect(')')
    return Nodes.CComplexBaseTypeNode(pos, 
        base_type = base_type, declarator = declarator)

Danilo Freitas's avatar
Danilo Freitas committed
1849
def p_c_simple_base_type(s, self_flag, nonempty, templates = None):
1850
    #print "p_c_simple_base_type: self_flag =", self_flag, nonempty
William Stein's avatar
William Stein committed
1851 1852 1853
    is_basic = 0
    signed = 1
    longness = 0
1854
    complex = 0
William Stein's avatar
William Stein committed
1855
    module_path = []
1856
    pos = s.position()
1857 1858
    if not s.sy == 'IDENT':
        error(pos, "Expected an identifier, found '%s'" % s.sy)
William Stein's avatar
William Stein committed
1859 1860 1861
    if looking_at_base_type(s):
        #print "p_c_simple_base_type: looking_at_base_type at", s.position()
        is_basic = 1
1862 1863
        if s.sy == 'IDENT' and s.systring in special_basic_c_types:
            signed, longness = special_basic_c_types[s.systring]
William Stein's avatar
William Stein committed
1864 1865 1866
            name = s.systring
            s.next()
        else:
1867 1868 1869 1870 1871 1872
            signed, longness = p_sign_and_longness(s)
            if s.sy == 'IDENT' and s.systring in basic_c_type_names:
                name = s.systring
                s.next()
            else:
                name = 'int'
1873 1874 1875
        if s.sy == 'IDENT' and s.systring == 'complex':
            complex = 1
            s.next()
1876 1877 1878 1879 1880 1881 1882 1883
    elif looking_at_dotted_name(s):
        #print "p_c_simple_base_type: looking_at_type_name at", s.position()
        name = s.systring
        s.next()
        while s.sy == '.':
            module_path.append(name)
            s.next()
            name = p_ident(s)
1884
    else:
1885 1886 1887
        name = s.systring
        s.next()
        if nonempty and s.sy != 'IDENT':
1888
            # Make sure this is not a declaration of a variable or function.  
1889 1890
            if s.sy == '(':
                s.next()
Robert Bradshaw's avatar
Robert Bradshaw committed
1891
                if s.sy == '*' or s.sy == '**' or s.sy == '&':
1892 1893 1894 1895 1896
                    s.put_back('(', '(')
                else:
                    s.put_back('(', '(')
                    s.put_back('IDENT', name)
                    name = None
Robert Bradshaw's avatar
Robert Bradshaw committed
1897
            elif s.sy not in ('*', '**', '[', '&'):
1898 1899
                s.put_back('IDENT', name)
                name = None
Danilo Freitas's avatar
Danilo Freitas committed
1900

1901
    type_node = Nodes.CSimpleBaseTypeNode(pos, 
William Stein's avatar
William Stein committed
1902 1903
        name = name, module_path = module_path,
        is_basic_c_type = is_basic, signed = signed,
1904
        complex = complex, longness = longness, 
Danilo Freitas's avatar
Danilo Freitas committed
1905
        is_self_arg = self_flag, templates = templates)
William Stein's avatar
William Stein committed
1906

1907
    if s.sy == '[':
Robert Bradshaw's avatar
Robert Bradshaw committed
1908 1909 1910 1911 1912 1913 1914 1915
        type_node = p_buffer_or_template(s, type_node, templates)
    
    if s.sy == '.':
        s.next()
        name = p_ident(s)
        type_node = Nodes.CNestedBaseTypeNode(pos, base_type = type_node, name = name)
    
    return type_node
1916

1917
def p_buffer_or_template(s, base_type_node, templates):
1918 1919 1920
    # s.sy == '['
    pos = s.position()
    s.next()
1921 1922
    # Note that buffer_positional_options_count=1, so the only positional argument is dtype. 
    # For templated types, all parameters are types. 
1923
    positional_args, keyword_args = (
1924
        p_positional_and_keyword_args(s, (']',), templates)
1925 1926 1927 1928 1929 1930 1931 1932
    )
    s.expect(']')

    keyword_dict = ExprNodes.DictNode(pos,
        key_value_pairs = [
            ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)
            for key, value in keyword_args
        ])
1933
    result = Nodes.TemplatedTypeNode(pos,
1934 1935
        positional_args = positional_args,
        keyword_args = keyword_dict,
1936
        base_type_node = base_type_node)
1937 1938 1939
    return result
    

1940
def looking_at_name(s):
1941 1942
    return s.sy == 'IDENT' and not s.systring in calling_convention_words

1943
def looking_at_expr(s):
1944
    if s.systring in base_type_start_words:
1945
        return False
1946 1947 1948 1949 1950 1951 1952 1953 1954 1955
    elif s.sy == 'IDENT':
        is_type = False
        name = s.systring
        dotted_path = []
        s.next()
        while s.sy == '.':
            s.next()
            dotted_path.append(s.systring)
            s.expect('IDENT')
        saved = s.sy, s.systring
1956 1957 1958
        if s.sy == 'IDENT':
            is_type = True
        elif s.sy == '*' or s.sy == '**':
1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974
            s.next()
            is_type = s.sy == ')'
            s.put_back(*saved)
        elif s.sy == '(':
            s.next()
            is_type = s.sy == '*'
            s.put_back(*saved)
        elif s.sy == '[':
            s.next()
            is_type = s.sy == ']'
            s.put_back(*saved)
        dotted_path.reverse()
        for p in dotted_path:
            s.put_back('IDENT', p)
            s.put_back('.', '.')
        s.put_back('IDENT', name)
1975
        return not is_type
1976
    else:
1977
        return True
William Stein's avatar
William Stein committed
1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991

def looking_at_base_type(s):
    #print "looking_at_base_type?", s.sy, s.systring, s.position()
    return s.sy == 'IDENT' and s.systring in base_type_start_words

def looking_at_dotted_name(s):
    if s.sy == 'IDENT':
        name = s.systring
        s.next()
        result = s.sy == '.'
        s.put_back('IDENT', name)
        return result
    else:
        return 0
1992

1993 1994 1995 1996
basic_c_type_names = ("void", "char", "int", "float", "double", "bint")

special_basic_c_types = {
    # name : (signed, longness)
1997
    "Py_UNICODE" : (0, 0),
1998
    "Py_ssize_t" : (2, 0),
1999
    "ssize_t"    : (2, 0),
2000 2001
    "size_t"     : (0, 0),
}
William Stein's avatar
William Stein committed
2002 2003 2004

sign_and_longness_words = ("short", "long", "signed", "unsigned")

2005
base_type_start_words = \
2006
    basic_c_type_names + sign_and_longness_words + tuple(special_basic_c_types)
William Stein's avatar
William Stein committed
2007 2008 2009 2010 2011 2012 2013

def p_sign_and_longness(s):
    signed = 1
    longness = 0
    while s.sy == 'IDENT' and s.systring in sign_and_longness_words:
        if s.systring == 'unsigned':
            signed = 0
2014 2015
        elif s.systring == 'signed':
            signed = 2
William Stein's avatar
William Stein committed
2016 2017 2018 2019 2020 2021 2022 2023
        elif s.systring == 'short':
            longness = -1
        elif s.systring == 'long':
            longness += 1
        s.next()
    return signed, longness

def p_opt_cname(s):
2024 2025 2026
    literal = p_opt_string_literal(s, 'u')
    if literal is not None:
        cname = EncodedString(literal)
Stefan Behnel's avatar
Stefan Behnel committed
2027
        cname.encoding = s.source_encoding
William Stein's avatar
William Stein committed
2028 2029 2030 2031
    else:
        cname = None
    return cname

2032 2033 2034
def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0,
                   assignable = 0, nonempty = 0,
                   calling_convention_allowed = 0):
2035 2036
    # If empty is true, the declarator must be empty. If nonempty is true,
    # the declarator must be nonempty. Otherwise we don't care.
William Stein's avatar
William Stein committed
2037 2038 2039
    # If cmethod_flag is true, then if this declarator declares
    # a function, it's a C method of an extension type.
    pos = s.position()
2040 2041
    if s.sy == '(':
        s.next()
2042
        if s.sy == ')' or looking_at_name(s):
2043
            base = Nodes.CNameDeclaratorNode(pos, name = EncodedString(u""), cname = None)
2044
            result = p_c_func_declarator(s, pos, ctx, base, cmethod_flag)
2045
        else:
2046 2047 2048 2049
            result = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
                                    cmethod_flag = cmethod_flag,
                                    nonempty = nonempty,
                                    calling_convention_allowed = 1)
2050 2051
            s.expect(')')
    else:
2052 2053
        result = p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
                                       assignable, nonempty)
Stefan Behnel's avatar
Stefan Behnel committed
2054
    if not calling_convention_allowed and result.calling_convention and s.sy != '(':
2055 2056 2057 2058 2059 2060 2061 2062
        error(s.position(), "%s on something that is not a function"
            % result.calling_convention)
    while s.sy in ('[', '('):
        pos = s.position()
        if s.sy == '[':
            result = p_c_array_declarator(s, result)
        else: # sy == '('
            s.next()
2063
            result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag)
2064 2065 2066 2067 2068 2069
        cmethod_flag = 0
    return result

def p_c_array_declarator(s, base):
    pos = s.position()
    s.next() # '['
Stefan Behnel's avatar
Stefan Behnel committed
2070
    if s.sy != ']':
2071
        dim = p_testlist(s)
2072 2073 2074 2075 2076
    else:
        dim = None
    s.expect(']')
    return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim)

2077
def p_c_func_declarator(s, pos, ctx, base, cmethod_flag):
2078
    #  Opening paren has already been skipped
2079 2080
    args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag,
                        nonempty_declarators = 0)
2081 2082 2083 2084 2085 2086 2087 2088
    ellipsis = p_optional_ellipsis(s)
    s.expect(')')
    nogil = p_nogil(s)
    exc_val, exc_check = p_exception_value_clause(s)
    with_gil = p_with_gil(s)
    return Nodes.CFuncDeclaratorNode(pos, 
        base = base, args = args, has_varargs = ellipsis,
        exception_value = exc_val, exception_check = exc_check,
2089
        nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil)
2090

Stefan Behnel's avatar
Stefan Behnel committed
2091
supported_overloaded_operators = cython.set([
Robert Bradshaw's avatar
Robert Bradshaw committed
2092
    '+', '-', '*', '/', '%', 
Robert Bradshaw's avatar
Robert Bradshaw committed
2093
    '++', '--', '~', '|', '&', '^', '<<', '>>', ',',
Robert Bradshaw's avatar
Robert Bradshaw committed
2094
    '==', '!=', '>=', '>', '<=', '<',
Robert Bradshaw's avatar
Robert Bradshaw committed
2095
    '[]', '()',
Robert Bradshaw's avatar
Robert Bradshaw committed
2096
])
2097

2098 2099
def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
                          assignable, nonempty):
2100 2101
    pos = s.position()
    calling_convention = p_calling_convention(s)
William Stein's avatar
William Stein committed
2102 2103
    if s.sy == '*':
        s.next()
2104 2105 2106
        base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
                              cmethod_flag = cmethod_flag,
                              assignable = assignable, nonempty = nonempty)
William Stein's avatar
William Stein committed
2107 2108 2109 2110
        result = Nodes.CPtrDeclaratorNode(pos, 
            base = base)
    elif s.sy == '**': # scanner returns this as a single token
        s.next()
2111 2112 2113
        base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
                              cmethod_flag = cmethod_flag,
                              assignable = assignable, nonempty = nonempty)
William Stein's avatar
William Stein committed
2114 2115 2116
        result = Nodes.CPtrDeclaratorNode(pos,
            base = Nodes.CPtrDeclaratorNode(pos,
                base = base))
Robert Bradshaw's avatar
merge  
Robert Bradshaw committed
2117
    elif s.sy == '&':
2118 2119 2120 2121 2122
        s.next()
        base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
                              cmethod_flag = cmethod_flag,
                              assignable = assignable, nonempty = nonempty)
        result = Nodes.CReferenceDeclaratorNode(pos, base = base)
William Stein's avatar
William Stein committed
2123
    else:
2124 2125
        rhs = None
        if s.sy == 'IDENT':
2126
            name = EncodedString(s.systring)
2127 2128
            if empty:
                error(s.position(), "Declarator should be empty")
William Stein's avatar
William Stein committed
2129
            s.next()
2130
            cname = p_opt_cname(s)
2131
            if name != 'operator' and s.sy == '=' and assignable:
2132
                s.next()
2133
                rhs = p_test(s)
William Stein's avatar
William Stein committed
2134
        else:
2135 2136 2137 2138
            if nonempty:
                error(s.position(), "Empty declarator")
            name = ""
            cname = None
2139
        if cname is None and ctx.namespace is not None and nonempty:
2140
            cname = ctx.namespace + "::" + name
2141
        if name == 'operator' and ctx.visibility == 'extern' and nonempty:
2142
            op = s.sy
Robert Bradshaw's avatar
Robert Bradshaw committed
2143
            if [c in '+-*/<=>!%&|([^~,' for c in op]:
2144
                s.next()
2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160
                # Handle diphthong operators.
                if op == '(':
                    s.expect(')')
                    op = '()'
                elif op == '[':
                    s.expect(']')
                    op = '[]'
                if op in ['-', '+', '|', '&'] and s.sy == op:
                    op = op*2
                    s.next()
                if s.sy == '=':
                    op += s.sy
                    s.next()
                if op not in supported_overloaded_operators:
                    s.error("Overloading operator '%s' not yet supported." % op)
                name = name+op
2161
        result = Nodes.CNameDeclaratorNode(pos,
Robert Bradshaw's avatar
Robert Bradshaw committed
2162
            name = name, cname = cname, default = rhs)
2163
    result.calling_convention = calling_convention
William Stein's avatar
William Stein committed
2164 2165
    return result

2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180
def p_nogil(s):
    if s.sy == 'IDENT' and s.systring == 'nogil':
        s.next()
        return 1
    else:
        return 0

def p_with_gil(s):
    if s.sy == 'with':
        s.next()
        s.expect_keyword('gil')
        return 1
    else:
        return 0

William Stein's avatar
William Stein committed
2181 2182 2183 2184 2185 2186 2187 2188
def p_exception_value_clause(s):
    exc_val = None
    exc_check = 0
    if s.sy == 'except':
        s.next()
        if s.sy == '*':
            exc_check = 1
            s.next()
Felix Wu's avatar
Felix Wu committed
2189 2190 2191
        elif s.sy == '+':
            exc_check = '+'
            s.next()
Robert Bradshaw's avatar
Robert Bradshaw committed
2192 2193 2194 2195
            if s.sy == 'IDENT':
                name = s.systring
                s.next()
                exc_val = p_name(s, name)
William Stein's avatar
William Stein committed
2196 2197 2198 2199
        else:
            if s.sy == '?':
                exc_check = 1
                s.next()
2200
            exc_val = p_test(s)
William Stein's avatar
William Stein committed
2201 2202 2203 2204
    return exc_val, exc_check

c_arg_list_terminators = ('*', '**', '.', ')')

2205
def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0,
2206
                 nonempty_declarators = 0, kw_only = 0, annotated = 1):
2207 2208
    #  Comma-separated list of C argument declarations, possibly empty.
    #  May have a trailing comma.
William Stein's avatar
William Stein committed
2209
    args = []
2210 2211
    is_self_arg = cmethod_flag
    while s.sy not in c_arg_list_terminators:
2212
        args.append(p_c_arg_decl(s, ctx, in_pyfunc, is_self_arg,
2213 2214
            nonempty = nonempty_declarators, kw_only = kw_only,
            annotated = annotated))
2215 2216 2217 2218
        if s.sy != ',':
            break
        s.next()
        is_self_arg = 0
William Stein's avatar
William Stein committed
2219 2220 2221 2222 2223 2224 2225 2226 2227
    return args

def p_optional_ellipsis(s):
    if s.sy == '.':
        expect_ellipsis(s)
        return 1
    else:
        return 0

2228 2229
def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0,
                 kw_only = 0, annotated = 1):
William Stein's avatar
William Stein committed
2230
    pos = s.position()
2231
    not_none = or_none = 0
William Stein's avatar
William Stein committed
2232
    default = None
2233
    annotation = None
2234 2235 2236 2237 2238 2239 2240 2241 2242
    if s.in_python_file:
        # empty type declaration
        base_type = Nodes.CSimpleBaseTypeNode(pos,
            name = None, module_path = [],
            is_basic_c_type = 0, signed = 0,
            complex = 0, longness = 0,
            is_self_arg = cmethod_flag, templates = None)
    else:
        base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty)
2243
    declarator = p_c_declarator(s, ctx, nonempty = nonempty)
2244 2245
    if s.sy in ('not', 'or') and not s.in_python_file:
        kind = s.sy
William Stein's avatar
William Stein committed
2246 2247 2248 2249 2250 2251
        s.next()
        if s.sy == 'IDENT' and s.systring == 'None':
            s.next()
        else:
            s.error("Expected 'None'")
        if not in_pyfunc:
2252 2253 2254
            error(pos, "'%s None' only allowed in Python functions" % kind)
        or_none = kind == 'or'
        not_none = kind == 'not'
2255 2256
    if annotated and s.sy == ':':
        s.next()
2257
        annotation = p_test(s)
William Stein's avatar
William Stein committed
2258 2259
    if s.sy == '=':
        s.next()
2260 2261 2262
        if 'pxd' in s.level:
            if s.sy not in ['*', '?']:
                error(pos, "default values cannot be specified in pxd files, use ? or *")
Robert Bradshaw's avatar
Robert Bradshaw committed
2263
            default = ExprNodes.BoolNode(1)
2264 2265
            s.next()
        else:
2266
            default = p_test(s)
William Stein's avatar
William Stein committed
2267 2268 2269 2270
    return Nodes.CArgDeclNode(pos,
        base_type = base_type,
        declarator = declarator,
        not_none = not_none,
2271
        or_none = or_none,
2272
        default = default,
2273
        annotation = annotation,
2274
        kw_only = kw_only)
William Stein's avatar
William Stein committed
2275

2276 2277 2278 2279 2280 2281 2282
def p_api(s):
    if s.sy == 'IDENT' and s.systring == 'api':
        s.next()
        return 1
    else:
        return 0

2283
def p_cdef_statement(s, ctx):
William Stein's avatar
William Stein committed
2284
    pos = s.position()
2285 2286 2287 2288 2289 2290 2291
    ctx.visibility = p_visibility(s, ctx.visibility)
    ctx.api = ctx.api or p_api(s)
    if ctx.api:
        if ctx.visibility not in ('private', 'public'):
            error(pos, "Cannot combine 'api' with '%s'" % ctx.visibility)
    if (ctx.visibility == 'extern') and s.sy == 'from':
        return p_cdef_extern_block(s, pos, ctx)
Robert Bradshaw's avatar
Robert Bradshaw committed
2292 2293
    elif s.sy == 'import':
        s.next()
2294
        return p_cdef_extern_block(s, pos, ctx)
2295
    elif p_nogil(s):
2296
        ctx.nogil = 1
2297 2298 2299 2300 2301 2302
        if ctx.overridable:
            error(pos, "cdef blocks cannot be declared cpdef")
        return p_cdef_block(s, ctx)
    elif s.sy == ':':
        if ctx.overridable:
            error(pos, "cdef blocks cannot be declared cpdef")
2303
        return p_cdef_block(s, ctx)
William Stein's avatar
William Stein committed
2304
    elif s.sy == 'class':
2305
        if ctx.level not in ('module', 'module_pxd'):
William Stein's avatar
William Stein committed
2306
            error(pos, "Extension type definition not allowed here")
2307 2308
        if ctx.overridable:
            error(pos, "Extension types cannot be declared cpdef")
2309
        return p_c_class_definition(s, pos, ctx)
Robert Bradshaw's avatar
Robert Bradshaw committed
2310 2311
    elif s.sy == 'IDENT' and s.systring == 'cppclass':
        if ctx.visibility != 'extern':
2312
            error(pos, "C++ classes need to be declared extern")
Robert Bradshaw's avatar
Robert Bradshaw committed
2313
        return p_cpp_class_definition(s, pos, ctx)
2314
    elif s.sy == 'IDENT' and s.systring in ("struct", "union", "enum", "packed"):
2315
        if ctx.level not in ('module', 'module_pxd'):
William Stein's avatar
William Stein committed
2316
            error(pos, "C struct/union/enum definition not allowed here")
2317 2318
        if ctx.overridable:
            error(pos, "C struct/union/enum cannot be declared cpdef")
William Stein's avatar
William Stein committed
2319
        if s.systring == "enum":
2320
            return p_c_enum_definition(s, pos, ctx)
William Stein's avatar
William Stein committed
2321
        else:
2322
            return p_c_struct_or_union_definition(s, pos, ctx)
William Stein's avatar
William Stein committed
2323
    else:
2324
        return p_c_func_or_var_declaration(s, pos, ctx)
2325

2326 2327
def p_cdef_block(s, ctx):
    return p_suite(s, ctx(cdef_flag = 1))
William Stein's avatar
William Stein committed
2328

2329
def p_cdef_extern_block(s, pos, ctx):
2330 2331
    if ctx.overridable:
        error(pos, "cdef extern blocks cannot be declared cpdef")
William Stein's avatar
William Stein committed
2332 2333 2334 2335 2336
    include_file = None
    s.expect('from')
    if s.sy == '*':
        s.next()
    else:
2337
        include_file = p_string_literal(s, 'u')[2]
2338
    ctx = ctx(cdef_flag = 1, visibility = 'extern')
2339 2340
    if s.systring == "namespace":
        s.next()
2341
        ctx.namespace = p_string_literal(s, 'u')[2]
2342 2343 2344
    if p_nogil(s):
        ctx.nogil = 1
    body = p_suite(s, ctx)
William Stein's avatar
William Stein committed
2345 2346
    return Nodes.CDefExternNode(pos,
        include_file = include_file,
2347
        body = body,
Robert Bradshaw's avatar
Robert Bradshaw committed
2348
        namespace = ctx.namespace)
William Stein's avatar
William Stein committed
2349

2350
def p_c_enum_definition(s, pos, ctx):
William Stein's avatar
William Stein committed
2351 2352 2353 2354 2355 2356
    # s.sy == ident 'enum'
    s.next()
    if s.sy == 'IDENT':
        name = s.systring
        s.next()
        cname = p_opt_cname(s)
2357 2358
        if cname is None and ctx.namespace is not None:
            cname = ctx.namespace + "::" + name
William Stein's avatar
William Stein committed
2359 2360 2361 2362 2363 2364
    else:
        name = None
        cname = None
    items = None
    s.expect(':')
    items = []
Stefan Behnel's avatar
Stefan Behnel committed
2365
    if s.sy != 'NEWLINE':
2366
        p_c_enum_line(s, ctx, items)
William Stein's avatar
William Stein committed
2367 2368 2369 2370
    else:
        s.next() # 'NEWLINE'
        s.expect_indent()
        while s.sy not in ('DEDENT', 'EOF'):
2371
            p_c_enum_line(s, ctx, items)
William Stein's avatar
William Stein committed
2372
        s.expect_dedent()
2373 2374 2375 2376
    return Nodes.CEnumDefNode(
        pos, name = name, cname = cname, items = items,
        typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,
        in_pxd = ctx.level == 'module_pxd')
William Stein's avatar
William Stein committed
2377

2378
def p_c_enum_line(s, ctx, items):
Stefan Behnel's avatar
Stefan Behnel committed
2379
    if s.sy != 'pass':
2380
        p_c_enum_item(s, ctx, items)
William Stein's avatar
William Stein committed
2381 2382 2383 2384
        while s.sy == ',':
            s.next()
            if s.sy in ('NEWLINE', 'EOF'):
                break
2385
            p_c_enum_item(s, ctx, items)
William Stein's avatar
William Stein committed
2386 2387 2388 2389
    else:
        s.next()
    s.expect_newline("Syntax error in enum item list")

2390
def p_c_enum_item(s, ctx, items):
William Stein's avatar
William Stein committed
2391 2392 2393
    pos = s.position()
    name = p_ident(s)
    cname = p_opt_cname(s)
2394 2395
    if cname is None and ctx.namespace is not None:
        cname = ctx.namespace + "::" + name
William Stein's avatar
William Stein committed
2396 2397 2398
    value = None
    if s.sy == '=':
        s.next()
2399
        value = p_test(s)
William Stein's avatar
William Stein committed
2400 2401 2402
    items.append(Nodes.CEnumDefItemNode(pos, 
        name = name, cname = cname, value = value))

2403
def p_c_struct_or_union_definition(s, pos, ctx):
2404 2405 2406 2407
    packed = False
    if s.systring == 'packed':
        packed = True
        s.next()
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
2408
        if s.sy != 'IDENT' or s.systring != 'struct':
2409
            s.expected('struct')
William Stein's avatar
William Stein committed
2410 2411 2412 2413 2414
    # s.sy == ident 'struct' or 'union'
    kind = s.systring
    s.next()
    name = p_ident(s)
    cname = p_opt_cname(s)
2415 2416
    if cname is None and ctx.namespace is not None:
        cname = ctx.namespace + "::" + name
William Stein's avatar
William Stein committed
2417 2418 2419 2420 2421 2422
    attributes = None
    if s.sy == ':':
        s.next()
        s.expect('NEWLINE')
        s.expect_indent()
        attributes = []
2423
        body_ctx = Ctx()
Stefan Behnel's avatar
Stefan Behnel committed
2424 2425
        while s.sy != 'DEDENT':
            if s.sy != 'pass':
William Stein's avatar
William Stein committed
2426
                attributes.append(
2427
                    p_c_func_or_var_declaration(s, s.position(), body_ctx))
William Stein's avatar
William Stein committed
2428 2429 2430 2431 2432 2433 2434 2435
            else:
                s.next()
                s.expect_newline("Expected a newline")
        s.expect_dedent()
    else:
        s.expect_newline("Syntax error in struct or union definition")
    return Nodes.CStructOrUnionDefNode(pos, 
        name = name, cname = cname, kind = kind, attributes = attributes,
2436
        typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,
2437
        in_pxd = ctx.level == 'module_pxd', packed = packed)
William Stein's avatar
William Stein committed
2438 2439 2440 2441 2442 2443

def p_visibility(s, prev_visibility):
    pos = s.position()
    visibility = prev_visibility
    if s.sy == 'IDENT' and s.systring in ('extern', 'public', 'readonly'):
        visibility = s.systring
Stefan Behnel's avatar
Stefan Behnel committed
2444
        if prev_visibility != 'private' and visibility != prev_visibility:
William Stein's avatar
William Stein committed
2445 2446 2447 2448
            s.error("Conflicting visibility options '%s' and '%s'"
                % (prev_visibility, visibility))
        s.next()
    return visibility
2449 2450
    
def p_c_modifiers(s):
2451
    if s.sy == 'IDENT' and s.systring in ('inline',):
2452
        modifier = s.systring
2453
        s.next()
2454 2455
        return [modifier] + p_c_modifiers(s)
    return []
William Stein's avatar
William Stein committed
2456

2457 2458
def p_c_func_or_var_declaration(s, pos, ctx):
    cmethod_flag = ctx.level in ('c_class', 'c_class_pxd')
2459
    modifiers = p_c_modifiers(s)
Danilo Freitas's avatar
Danilo Freitas committed
2460
    base_type = p_c_base_type(s, nonempty = 1, templates = ctx.templates)
2461 2462 2463
    declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,
                                assignable = 1, nonempty = 1)
    declarator.overridable = ctx.overridable
William Stein's avatar
William Stein committed
2464
    if s.sy == ':':
Danilo Freitas's avatar
Danilo Freitas committed
2465
        if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd') and not ctx.templates:
William Stein's avatar
William Stein committed
2466
            s.error("C function definition not allowed here")
2467
        doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1)
William Stein's avatar
William Stein committed
2468
        result = Nodes.CFuncDefNode(pos,
2469
            visibility = ctx.visibility,
William Stein's avatar
William Stein committed
2470 2471
            base_type = base_type,
            declarator = declarator, 
2472
            body = suite,
2473
            doc = doc,
2474
            modifiers = modifiers,
2475 2476
            api = ctx.api,
            overridable = ctx.overridable)
William Stein's avatar
William Stein committed
2477
    else:
Stefan Behnel's avatar
Stefan Behnel committed
2478
        #if api:
2479
        #    s.error("'api' not allowed with variable declaration")
William Stein's avatar
William Stein committed
2480 2481 2482 2483 2484
        declarators = [declarator]
        while s.sy == ',':
            s.next()
            if s.sy == 'NEWLINE':
                break
2485 2486
            declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,
                                        assignable = 1, nonempty = 1)
William Stein's avatar
William Stein committed
2487 2488 2489
            declarators.append(declarator)
        s.expect_newline("Syntax error in C variable declaration")
        result = Nodes.CVarDefNode(pos, 
2490 2491
            visibility = ctx.visibility,
            base_type = base_type,
2492
            declarators = declarators,
2493 2494 2495
            in_pxd = ctx.level == 'module_pxd',
            api = ctx.api,
            overridable = ctx.overridable)
William Stein's avatar
William Stein committed
2496 2497
    return result

2498
def p_ctypedef_statement(s, ctx):
William Stein's avatar
William Stein committed
2499 2500 2501
    # s.sy == 'ctypedef'
    pos = s.position()
    s.next()
2502
    visibility = p_visibility(s, ctx.visibility)
2503
    api = p_api(s)
2504
    ctx = ctx(typedef_flag = 1, visibility = visibility)
2505 2506
    if api:
        ctx.api = 1
William Stein's avatar
William Stein committed
2507
    if s.sy == 'class':
2508
        return p_c_class_definition(s, pos, ctx)
2509
    elif s.sy == 'IDENT' and s.systring in ('packed', 'struct', 'union', 'enum'):
William Stein's avatar
William Stein committed
2510
        if s.systring == 'enum':
2511
            return p_c_enum_definition(s, pos, ctx)
William Stein's avatar
William Stein committed
2512
        else:
2513
            return p_c_struct_or_union_definition(s, pos, ctx)
William Stein's avatar
William Stein committed
2514
    else:
2515
        base_type = p_c_base_type(s, nonempty = 1)
2516 2517
        if base_type.name is None:
            s.error("Syntax error in ctypedef statement")
2518
        declarator = p_c_declarator(s, ctx, is_type = 1, nonempty = 1)
William Stein's avatar
William Stein committed
2519
        s.expect_newline("Syntax error in ctypedef statement")
2520 2521 2522 2523
        return Nodes.CTypeDefNode(
            pos, base_type = base_type,
            declarator = declarator, visibility = visibility,
            in_pxd = ctx.level == 'module_pxd')
William Stein's avatar
William Stein committed
2524

2525 2526 2527 2528 2529
def p_decorators(s):
    decorators = []
    while s.sy == 'DECORATOR':
        pos = s.position()
        s.next()
2530 2531
        decstring = p_dotted_name(s, as_allowed=0)[2]
        names = decstring.split('.')
2532
        decorator = ExprNodes.NameNode(pos, name=EncodedString(names[0]))
2533 2534
        for name in names[1:]:
            decorator = ExprNodes.AttributeNode(pos,
2535
                                           attribute=EncodedString(name),
2536
                                           obj=decorator)
2537 2538 2539 2540 2541 2542 2543
        if s.sy == '(':
            decorator = p_call(s, decorator)
        decorators.append(Nodes.DecoratorNode(pos, decorator=decorator))
        s.expect_newline("Expected a newline after decorator")
    return decorators

def p_def_statement(s, decorators=None):
William Stein's avatar
William Stein committed
2544 2545 2546
    # s.sy == 'def'
    pos = s.position()
    s.next()
2547
    name = EncodedString( p_ident(s) )
William Stein's avatar
William Stein committed
2548
    s.expect('(');
Stefan Behnel's avatar
Stefan Behnel committed
2549 2550 2551
    args, star_arg, starstar_arg = p_varargslist(s, terminator=')')
    s.expect(')')
    if p_nogil(s):
2552
        error(pos, "Python function cannot be declared nogil")
2553 2554 2555
    return_type_annotation = None
    if s.sy == '->':
        s.next()
2556
        return_type_annotation = p_test(s)
Stefan Behnel's avatar
Stefan Behnel committed
2557 2558 2559
    doc, body = p_suite(s, Ctx(level = 'function'), with_doc = 1)
    return Nodes.DefNode(pos, name = name, args = args, 
        star_arg = star_arg, starstar_arg = starstar_arg,
2560 2561
        doc = doc, body = body, decorators = decorators,
        return_type_annotation = return_type_annotation)
Stefan Behnel's avatar
Stefan Behnel committed
2562

2563 2564 2565
def p_varargslist(s, terminator=')', annotated=1):
    args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1,
                        annotated = annotated)
William Stein's avatar
William Stein committed
2566 2567 2568 2569
    star_arg = None
    starstar_arg = None
    if s.sy == '*':
        s.next()
2570
        if s.sy == 'IDENT':
2571
            star_arg = p_py_arg_decl(s, annotated=annotated)
William Stein's avatar
William Stein committed
2572 2573
        if s.sy == ',':
            s.next()
2574
            args.extend(p_c_arg_list(s, in_pyfunc = 1,
2575
                nonempty_declarators = 1, kw_only = 1, annotated = annotated))
Stefan Behnel's avatar
Stefan Behnel committed
2576
        elif s.sy != terminator:
2577 2578
            s.error("Syntax error in Python function argument list")
    if s.sy == '**':
William Stein's avatar
William Stein committed
2579
        s.next()
2580
        starstar_arg = p_py_arg_decl(s, annotated=annotated)
Stefan Behnel's avatar
Stefan Behnel committed
2581
    return (args, star_arg, starstar_arg)
William Stein's avatar
William Stein committed
2582

2583
def p_py_arg_decl(s, annotated = 1):
William Stein's avatar
William Stein committed
2584 2585
    pos = s.position()
    name = p_ident(s)
2586
    annotation = None
2587
    if annotated and s.sy == ':':
2588
        s.next()
2589
        annotation = p_test(s)
2590
    return Nodes.PyArgDeclNode(pos, name = name, annotation = annotation)
William Stein's avatar
William Stein committed
2591

2592
def p_class_statement(s, decorators):
William Stein's avatar
William Stein committed
2593 2594 2595
    # s.sy == 'class'
    pos = s.position()
    s.next()
2596
    class_name = EncodedString( p_ident(s) )
2597
    class_name.encoding = s.source_encoding
William Stein's avatar
William Stein committed
2598 2599 2600 2601 2602 2603
    if s.sy == '(':
        s.next()
        base_list = p_simple_expr_list(s)
        s.expect(')')
    else:
        base_list = []
2604
    doc, body = p_suite(s, Ctx(level = 'class'), with_doc = 1)
William Stein's avatar
William Stein committed
2605 2606 2607
    return Nodes.PyClassDefNode(pos,
        name = class_name,
        bases = ExprNodes.TupleNode(pos, args = base_list),
2608
        doc = doc, body = body, decorators = decorators)
William Stein's avatar
William Stein committed
2609

2610
def p_c_class_definition(s, pos,  ctx):
William Stein's avatar
William Stein committed
2611 2612 2613 2614 2615 2616 2617 2618
    # s.sy == 'class'
    s.next()
    module_path = []
    class_name = p_ident(s)
    while s.sy == '.':
        s.next()
        module_path.append(class_name)
        class_name = p_ident(s)
2619
    if module_path and ctx.visibility != 'extern':
William Stein's avatar
William Stein committed
2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641
        error(pos, "Qualified class name only allowed for 'extern' C class")
    if module_path and s.sy == 'IDENT' and s.systring == 'as':
        s.next()
        as_name = p_ident(s)
    else:
        as_name = class_name
    objstruct_name = None
    typeobj_name = None
    base_class_module = None
    base_class_name = None
    if s.sy == '(':
        s.next()
        base_class_path = [p_ident(s)]
        while s.sy == '.':
            s.next()
            base_class_path.append(p_ident(s))
        if s.sy == ',':
            s.error("C class may only have one base class")
        s.expect(')')
        base_class_module = ".".join(base_class_path[:-1])
        base_class_name = base_class_path[-1]
    if s.sy == '[':
2642
        if ctx.visibility not in ('public', 'extern'):
William Stein's avatar
William Stein committed
2643 2644 2645
            error(s.position(), "Name options only allowed for 'public' or 'extern' C class")
        objstruct_name, typeobj_name = p_c_class_options(s)
    if s.sy == ':':
2646
        if ctx.level == 'module_pxd':
William Stein's avatar
William Stein committed
2647 2648 2649
            body_level = 'c_class_pxd'
        else:
            body_level = 'c_class'
2650
        doc, body = p_suite(s, Ctx(level = body_level), with_doc = 1)
William Stein's avatar
William Stein committed
2651 2652 2653 2654
    else:
        s.expect_newline("Syntax error in C class definition")
        doc = None
        body = None
2655
    if ctx.visibility == 'extern':
William Stein's avatar
William Stein committed
2656 2657 2658 2659
        if not module_path:
            error(pos, "Module name required for 'extern' C class")
        if typeobj_name:
            error(pos, "Type object name specification not allowed for 'extern' C class")
2660
    elif ctx.visibility == 'public':
William Stein's avatar
William Stein committed
2661 2662 2663 2664
        if not objstruct_name:
            error(pos, "Object struct name specification required for 'public' C class")
        if not typeobj_name:
            error(pos, "Type object name specification required for 'public' C class")
2665 2666
    elif ctx.visibility == 'private':
        if ctx.api:
Stefan Behnel's avatar
Stefan Behnel committed
2667
            error(pos, "Only 'public' C class can be declared 'api'")
2668
    else:
2669
        error(pos, "Invalid class visibility '%s'" % ctx.visibility)
William Stein's avatar
William Stein committed
2670
    return Nodes.CClassDefNode(pos,
2671 2672 2673
        visibility = ctx.visibility,
        typedef_flag = ctx.typedef_flag,
        api = ctx.api,
William Stein's avatar
William Stein committed
2674 2675 2676 2677 2678 2679 2680
        module_name = ".".join(module_path),
        class_name = class_name,
        as_name = as_name,
        base_class_module = base_class_module,
        base_class_name = base_class_name,
        objstruct_name = objstruct_name,
        typeobj_name = typeobj_name,
2681
        in_pxd = ctx.level == 'module_pxd',
William Stein's avatar
William Stein committed
2682 2683 2684 2685 2686 2687 2688 2689
        doc = doc,
        body = body)

def p_c_class_options(s):
    objstruct_name = None
    typeobj_name = None
    s.expect('[')
    while 1:
Stefan Behnel's avatar
Stefan Behnel committed
2690
        if s.sy != 'IDENT':
William Stein's avatar
William Stein committed
2691 2692 2693 2694 2695 2696 2697
            break
        if s.systring == 'object':
            s.next()
            objstruct_name = p_ident(s)
        elif s.systring == 'type':
            s.next()
            typeobj_name = p_ident(s)
Stefan Behnel's avatar
Stefan Behnel committed
2698
        if s.sy != ',':
William Stein's avatar
William Stein committed
2699 2700 2701 2702 2703 2704 2705 2706 2707
            break
        s.next()
    s.expect(']', "Expected 'object' or 'type'")
    return objstruct_name, typeobj_name

def p_property_decl(s):
    pos = s.position()
    s.next() # 'property'
    name = p_ident(s)
2708
    doc, body = p_suite(s, Ctx(level = 'property'), with_doc = 1)
William Stein's avatar
William Stein committed
2709 2710 2711
    return Nodes.PropertyNode(pos, name = name, doc = doc, body = body)

def p_doc_string(s):
2712
    if s.sy == 'BEGIN_STRING':
2713
        pos = s.position()
2714
        kind, bytes_result, unicode_result = p_cat_string_literal(s)
Stefan Behnel's avatar
Stefan Behnel committed
2715
        if s.sy != 'EOF':
William Stein's avatar
William Stein committed
2716
            s.expect_newline("Syntax error in doc string")
2717 2718 2719 2720
        if kind in ('u', ''):
            return unicode_result
        warning(pos, "Python 3 requires docstrings to be unicode strings")
        return bytes_result
William Stein's avatar
William Stein committed
2721 2722
    else:
        return None
2723

2724 2725 2726 2727 2728 2729
def p_code(s, level=None):
    body = p_statement_list(s, Ctx(level = level), first_statement = 1)
    if s.sy != 'EOF':
        s.error("Syntax error in statement [%s,%s]" % (
            repr(s.sy), repr(s.systring)))
    return body
William Stein's avatar
William Stein committed
2730

2731
COMPILER_DIRECTIVE_COMMENT_RE = re.compile(r"^#\s*cython:\s*((\w|[.])+\s*=.*)$")
2732 2733

def p_compiler_directive_comments(s):
2734
    result = {}
2735 2736 2737
    while s.sy == 'commentline':
        m = COMPILER_DIRECTIVE_COMMENT_RE.match(s.systring)
        if m:
2738
            directives = m.group(1).strip()
2739
            try:
2740 2741
                result.update( Options.parse_directive_list(
                    directives, ignore_unknown=True) )
2742
            except ValueError, e:
2743
                s.error(e.args[0], fatal=False)
2744 2745 2746
        s.next()
    return result

2747
def p_module(s, pxd, full_module_name):
William Stein's avatar
William Stein committed
2748
    pos = s.position()
2749

2750
    directive_comments = p_compiler_directive_comments(s)
2751 2752
    s.parse_comments = False

2753
    if 'language_level' in directive_comments:
Stefan Behnel's avatar
Stefan Behnel committed
2754
        s.context.set_language_level(directive_comments['language_level'])
2755

William Stein's avatar
William Stein committed
2756 2757 2758 2759 2760
    doc = p_doc_string(s)
    if pxd:
        level = 'module_pxd'
    else:
        level = 'module'
2761

2762
    body = p_statement_list(s, Ctx(level = level), first_statement = 1)
Stefan Behnel's avatar
Stefan Behnel committed
2763
    if s.sy != 'EOF':
William Stein's avatar
William Stein committed
2764 2765
        s.error("Syntax error in statement [%s,%s]" % (
            repr(s.sy), repr(s.systring)))
2766 2767
    return ModuleNode(pos, doc = doc, body = body,
                      full_module_name = full_module_name,
2768
                      directive_comments = directive_comments)
William Stein's avatar
William Stein committed
2769

2770 2771 2772 2773 2774
def p_cpp_class_definition(s, pos,  ctx):
    # s.sy == 'cppclass'
    s.next()
    module_path = []
    class_name = p_ident(s)
2775 2776 2777
    cname = p_opt_cname(s)
    if cname is None and ctx.namespace is not None:
        cname = ctx.namespace + "::" + class_name
2778
    if s.sy == '.':
2779
        error(pos, "Qualified class name not allowed C++ class")
Danilo Freitas's avatar
Danilo Freitas committed
2780 2781
    if s.sy == '[':
        s.next()
2782
        templates = [p_ident(s)]
Danilo Freitas's avatar
Danilo Freitas committed
2783 2784
        while s.sy == ',':
            s.next()
2785
            templates.append(p_ident(s))
Danilo Freitas's avatar
Danilo Freitas committed
2786
        s.expect(']')
2787 2788
    else:
        templates = None
2789
    if s.sy == '(':
2790 2791 2792
        s.next()
        base_classes = [p_dotted_name(s, False)[2]]
        while s.sy == ',':
2793
            s.next()
2794
            base_classes.append(p_dotted_name(s, False)[2])
2795
        s.expect(')')
2796 2797
    else:
        base_classes = []
2798
    if s.sy == '[':
2799
        error(s.position(), "Name options not allowed for C++ class")
2800
    if s.sy == ':':
2801 2802 2803 2804
        s.next()
        s.expect('NEWLINE')
        s.expect_indent()
        attributes = []
Robert Bradshaw's avatar
Robert Bradshaw committed
2805
        body_ctx = Ctx(visibility = ctx.visibility)
Danilo Freitas's avatar
Danilo Freitas committed
2806
        body_ctx.templates = templates
2807
        while s.sy != 'DEDENT':
Robert Bradshaw's avatar
Robert Bradshaw committed
2808 2809 2810 2811
            if s.systring == 'cppclass':
                attributes.append(
                    p_cpp_class_definition(s, s.position(), body_ctx))
            elif s.sy != 'pass':
2812 2813 2814 2815 2816 2817
                attributes.append(
                    p_c_func_or_var_declaration(s, s.position(), body_ctx))
            else:
                s.next()
                s.expect_newline("Expected a newline")
        s.expect_dedent()
2818
    else:
2819
        attributes = None
2820 2821 2822
        s.expect_newline("Syntax error in C++ class definition")
    return Nodes.CppClassNode(pos,
        name = class_name,
2823
        cname = cname,
2824
        base_classes = base_classes,
2825 2826
        visibility = ctx.visibility,
        in_pxd = ctx.level == 'module_pxd',
Danilo Freitas's avatar
Danilo Freitas committed
2827 2828
        attributes = attributes,
        templates = templates)
2829 2830 2831



William Stein's avatar
William Stein committed
2832 2833 2834 2835 2836 2837
#----------------------------------------------
#
#   Debugging
#
#----------------------------------------------

Stefan Behnel's avatar
Stefan Behnel committed
2838
def print_parse_tree(f, node, level, key = None):
Stefan Behnel's avatar
Stefan Behnel committed
2839
    from types import ListType, TupleType
2840
    from Nodes import Node
William Stein's avatar
William Stein committed
2841 2842 2843 2844 2845 2846
    ind = "  " * level
    if node:
        f.write(ind)
        if key:
            f.write("%s: " % key)
        t = type(node)
Stefan Behnel's avatar
Stefan Behnel committed
2847
        if t is tuple:
William Stein's avatar
William Stein committed
2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859
            f.write("(%s @ %s\n" % (node[0], node[1]))
            for i in xrange(2, len(node)):
                print_parse_tree(f, node[i], level+1)
            f.write("%s)\n" % ind)
            return
        elif isinstance(node, Node):
            try:
                tag = node.tag
            except AttributeError:
                tag = node.__class__.__name__
            f.write("%s @ %s\n" % (tag, node.pos))
            for name, value in node.__dict__.items():
Stefan Behnel's avatar
Stefan Behnel committed
2860
                if name != 'tag' and name != 'pos':
William Stein's avatar
William Stein committed
2861 2862
                    print_parse_tree(f, value, level+1, name)
            return
Stefan Behnel's avatar
Stefan Behnel committed
2863
        elif t is list:
William Stein's avatar
William Stein committed
2864 2865 2866 2867 2868 2869 2870
            f.write("[\n")
            for i in xrange(len(node)):
                print_parse_tree(f, node[i], level+1)
            f.write("%s]\n" % ind)
            return
    f.write("%s%s\n" % (ind, node))