Parsing.py 67.8 KB
Newer Older
William Stein's avatar
William Stein committed
1 2 3 4
#
#   Pyrex Parser
#

5
import os, re
William Stein's avatar
William Stein committed
6
from types import ListType, TupleType
7
from Scanning import PyrexScanner, FileSourceDescriptor
William Stein's avatar
William Stein committed
8 9
import Nodes
import ExprNodes
10
from ModuleNode import ModuleNode
William Stein's avatar
William Stein committed
11
from Errors import error, InternalError
12
from Cython import Utils
Stefan Behnel's avatar
Stefan Behnel committed
13
import Future
William Stein's avatar
William Stein committed
14

15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
class Ctx(object):
    #  Parsing context
    level = 'other'
    visibility = 'private'
    cdef_flag = 0
    typedef_flag = 0
    api = 0
    overridable = 0
    nogil = 0

    def __init__(self, **kwds):
        self.__dict__.update(kwds)

    def __call__(self, **kwds):
        ctx = Ctx()
        d = ctx.__dict__
        d.update(self.__dict__)
        d.update(kwds)
        return ctx

William Stein's avatar
William Stein committed
35 36 37 38 39 40 41 42 43 44 45 46 47
def p_ident(s, message = "Expected an identifier"):
    if s.sy == 'IDENT':
        name = s.systring
        s.next()
        return name
    else:
        s.error(message)

def p_ident_list(s):
    names = []
    while s.sy == 'IDENT':
        names.append(s.systring)
        s.next()
Stefan Behnel's avatar
Stefan Behnel committed
48
        if s.sy != ',':
William Stein's avatar
William Stein committed
49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68
            break
        s.next()
    return names

#------------------------------------------
#
#   Expressions
#
#------------------------------------------

def p_binop_expr(s, ops, p_sub_expr):
    n1 = p_sub_expr(s)
    while s.sy in ops:
        op = s.sy
        pos = s.position()
        s.next()
        n2 = p_sub_expr(s)
        n1 = ExprNodes.binop_node(pos, op, n1, n2)
    return n1

Robert Bradshaw's avatar
Robert Bradshaw committed
69
#expression: or_test [if or_test else test] | lambda_form
William Stein's avatar
William Stein committed
70 71

def p_simple_expr(s):
Robert Bradshaw's avatar
Robert Bradshaw committed
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
    pos = s.position()
    expr = p_or_test(s)
    if s.sy == 'if':
        s.next()
        test = p_or_test(s)
        if s.sy == 'else':
            s.next()
            other = p_test(s)
            return ExprNodes.CondExprNode(pos, test=test, true_val=expr, false_val=other)
        else:
            s.error("Expected 'else'")
    else:
        return expr
        
#test: or_test | lambda_form
        
def p_test(s):
    return p_or_test(s)

#or_test: and_test ('or' and_test)*

def p_or_test(s):
William Stein's avatar
William Stein committed
94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198
    return p_rassoc_binop_expr(s, ('or',), p_and_test)

def p_rassoc_binop_expr(s, ops, p_subexpr):
    n1 = p_subexpr(s)
    if s.sy in ops:
        pos = s.position()
        op = s.sy
        s.next()
        n2 = p_rassoc_binop_expr(s, ops, p_subexpr)
        n1 = ExprNodes.binop_node(pos, op, n1, n2)
    return n1

#and_test: not_test ('and' not_test)*

def p_and_test(s):
    #return p_binop_expr(s, ('and',), p_not_test)
    return p_rassoc_binop_expr(s, ('and',), p_not_test)

#not_test: 'not' not_test | comparison

def p_not_test(s):
    if s.sy == 'not':
        pos = s.position()
        s.next()
        return ExprNodes.NotNode(pos, operand = p_not_test(s))
    else:
        return p_comparison(s)

#comparison: expr (comp_op expr)*
#comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'

def p_comparison(s):
    n1 = p_bit_expr(s)
    if s.sy in comparison_ops:
        pos = s.position()
        op = p_cmp_op(s)
        n2 = p_bit_expr(s)
        n1 = ExprNodes.PrimaryCmpNode(pos, 
            operator = op, operand1 = n1, operand2 = n2)
        if s.sy in comparison_ops:
            n1.cascade = p_cascaded_cmp(s)
    return n1

def p_cascaded_cmp(s):
    pos = s.position()
    op = p_cmp_op(s)
    n2 = p_bit_expr(s)
    result = ExprNodes.CascadedCmpNode(pos, 
        operator = op, operand2 = n2)
    if s.sy in comparison_ops:
        result.cascade = p_cascaded_cmp(s)
    return result

def p_cmp_op(s):
    if s.sy == 'not':
        s.next()
        s.expect('in')
        op = 'not_in'
    elif s.sy == 'is':
        s.next()
        if s.sy == 'not':
            s.next()
            op = 'is_not'
        else:
            op = 'is'
    else:
        op = s.sy
        s.next()
    if op == '<>':
        op = '!='
    return op
    
comparison_ops = (
    '<', '>', '==', '>=', '<=', '<>', '!=', 
    'in', 'is', 'not'
)

#expr: xor_expr ('|' xor_expr)*

def p_bit_expr(s):
    return p_binop_expr(s, ('|',), p_xor_expr)

#xor_expr: and_expr ('^' and_expr)*

def p_xor_expr(s):
    return p_binop_expr(s, ('^',), p_and_expr)

#and_expr: shift_expr ('&' shift_expr)*

def p_and_expr(s):
    return p_binop_expr(s, ('&',), p_shift_expr)

#shift_expr: arith_expr (('<<'|'>>') arith_expr)*

def p_shift_expr(s):
    return p_binop_expr(s, ('<<', '>>'), p_arith_expr)

#arith_expr: term (('+'|'-') term)*

def p_arith_expr(s):
    return p_binop_expr(s, ('+', '-'), p_term)

#term: factor (('*'|'/'|'%') factor)*

def p_term(s):
Robert Bradshaw's avatar
Robert Bradshaw committed
199
    return p_binop_expr(s, ('*', '/', '%', '//'), p_factor)
William Stein's avatar
William Stein committed
200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227

#factor: ('+'|'-'|'~'|'&'|typecast|sizeof) factor | power

def p_factor(s):
    sy = s.sy
    if sy in ('+', '-', '~'):
        op = s.sy
        pos = s.position()
        s.next()
        return ExprNodes.unop_node(pos, op, p_factor(s))
    elif sy == '&':
        pos = s.position()
        s.next()
        arg = p_factor(s)
        return ExprNodes.AmpersandNode(pos, operand = arg)
    elif sy == "<":
        return p_typecast(s)
    elif sy == 'IDENT' and s.systring == "sizeof":
        return p_sizeof(s)
    else:
        return p_power(s)

def p_typecast(s):
    # s.sy == "<"
    pos = s.position()
    s.next()
    base_type = p_c_base_type(s)
    declarator = p_c_declarator(s, empty = 1)
228 229 230 231 232
    if s.sy == '?':
        s.next()
        typecheck = 1
    else:
        typecheck = 0
William Stein's avatar
William Stein committed
233 234 235 236 237
    s.expect(">")
    operand = p_factor(s)
    return ExprNodes.TypecastNode(pos, 
        base_type = base_type, 
        declarator = declarator,
238 239
        operand = operand,
        typecheck = typecheck)
William Stein's avatar
William Stein committed
240 241 242 243 244 245

def p_sizeof(s):
    # s.sy == ident "sizeof"
    pos = s.position()
    s.next()
    s.expect('(')
246
    if looking_at_type(s) or looking_at_dotted_name(s):
William Stein's avatar
William Stein committed
247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
        base_type = p_c_base_type(s)
        declarator = p_c_declarator(s, empty = 1)
        node = ExprNodes.SizeofTypeNode(pos, 
            base_type = base_type, declarator = declarator)
    else:
        operand = p_simple_expr(s)
        node = ExprNodes.SizeofVarNode(pos, operand = operand)
    s.expect(')')
    return node

#power: atom trailer* ('**' factor)*

def p_power(s):
    n1 = p_atom(s)
    while s.sy in ('(', '[', '.'):
        n1 = p_trailer(s, n1)
    if s.sy == '**':
        pos = s.position()
        s.next()
        n2 = p_factor(s)
        n1 = ExprNodes.binop_node(pos, '**', n1, n2)
    return n1

#trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME

def p_trailer(s, node1):
    pos = s.position()
    if s.sy == '(':
        return p_call(s, node1)
    elif s.sy == '[':
        return p_index(s, node1)
    else: # s.sy == '.'
        s.next()
280
        name = Utils.EncodedString( p_ident(s) )
William Stein's avatar
William Stein committed
281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301
        return ExprNodes.AttributeNode(pos, 
            obj = node1, attribute = name)

# arglist:  argument (',' argument)* [',']
# argument: [test '='] test       # Really [keyword '='] test

def p_call(s, function):
    # s.sy == '('
    pos = s.position()
    s.next()
    positional_args = []
    keyword_args = []
    star_arg = None
    starstar_arg = None
    while s.sy not in ('*', '**', ')'):
        arg = p_simple_expr(s)
        if s.sy == '=':
            s.next()
            if not arg.is_name:
                s.error("Expected an identifier before '='",
                    pos = arg.pos)
302
            encoded_name = Utils.EncodedString(arg.name)
303
            keyword = ExprNodes.IdentifierStringNode(arg.pos, 
304
                value = encoded_name)
William Stein's avatar
William Stein committed
305 306 307 308 309 310 311
            arg = p_simple_expr(s)
            keyword_args.append((keyword, arg))
        else:
            if keyword_args:
                s.error("Non-keyword arg following keyword arg",
                    pos = arg.pos)
            positional_args.append(arg)
Stefan Behnel's avatar
Stefan Behnel committed
312
        if s.sy != ',':
William Stein's avatar
William Stein committed
313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344
            break
        s.next()
    if s.sy == '*':
        s.next()
        star_arg = p_simple_expr(s)
        if s.sy == ',':
            s.next()
    if s.sy == '**':
        s.next()
        starstar_arg = p_simple_expr(s)
        if s.sy == ',':
            s.next()
    s.expect(')')
    if not (keyword_args or star_arg or starstar_arg):
        return ExprNodes.SimpleCallNode(pos,
            function = function,
            args = positional_args)
    else:
        arg_tuple = None
        keyword_dict = None
        if positional_args or not star_arg:
            arg_tuple = ExprNodes.TupleNode(pos, 
                args = positional_args)
        if star_arg:
            star_arg_tuple = ExprNodes.AsTupleNode(pos, arg = star_arg)
            if arg_tuple:
                arg_tuple = ExprNodes.binop_node(pos, 
                    operator = '+', operand1 = arg_tuple,
                    operand2 = star_arg_tuple)
            else:
                arg_tuple = star_arg_tuple
        if keyword_args:
345 346
            keyword_args = [ExprNodes.DictItemNode(pos=key.pos, key=key, value=value) 
                              for key, value in keyword_args]
William Stein's avatar
William Stein committed
347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399
            keyword_dict = ExprNodes.DictNode(pos,
                key_value_pairs = keyword_args)
        return ExprNodes.GeneralCallNode(pos, 
            function = function,
            positional_args = arg_tuple,
            keyword_args = keyword_dict,
            starstar_arg = starstar_arg)

#lambdef: 'lambda' [varargslist] ':' test

#subscriptlist: subscript (',' subscript)* [',']

def p_index(s, base):
    # s.sy == '['
    pos = s.position()
    s.next()
    subscripts = p_subscript_list(s)
    if len(subscripts) == 1 and len(subscripts[0]) == 2:
        start, stop = subscripts[0]
        result = ExprNodes.SliceIndexNode(pos, 
            base = base, start = start, stop = stop)
    else:
        indexes = make_slice_nodes(pos, subscripts)
        if len(indexes) == 1:
            index = indexes[0]
        else:
            index = ExprNodes.TupleNode(pos, args = indexes)
        result = ExprNodes.IndexNode(pos,
            base = base, index = index)
    s.expect(']')
    return result

def p_subscript_list(s):
    items = [p_subscript(s)]
    while s.sy == ',':
        s.next()
        if s.sy == ']':
            break
        items.append(p_subscript(s))
    return items

#subscript: '.' '.' '.' | test | [test] ':' [test] [':' [test]]

def p_subscript(s):
    # Parse a subscript and return a list of
    # 1, 2 or 3 ExprNodes, depending on how
    # many slice elements were encountered.
    pos = s.position()
    if s.sy == '.':
        expect_ellipsis(s)
        return [ExprNodes.EllipsisNode(pos)]
    else:
        start = p_slice_element(s, (':',))
Stefan Behnel's avatar
Stefan Behnel committed
400
        if s.sy != ':':
William Stein's avatar
William Stein committed
401 402 403
            return [start]
        s.next()
        stop = p_slice_element(s, (':', ',', ']'))
Stefan Behnel's avatar
Stefan Behnel committed
404
        if s.sy != ':':
William Stein's avatar
William Stein committed
405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465
            return [start, stop]
        s.next()
        step = p_slice_element(s, (':', ',', ']'))
        return [start, stop, step]

def p_slice_element(s, follow_set):
    # Simple expression which may be missing iff
    # it is followed by something in follow_set.
    if s.sy not in follow_set:
        return p_simple_expr(s)
    else:
        return None

def expect_ellipsis(s):
    s.expect('.')
    s.expect('.')
    s.expect('.')

def make_slice_nodes(pos, subscripts):
    # Convert a list of subscripts as returned
    # by p_subscript_list into a list of ExprNodes,
    # creating SliceNodes for elements with 2 or
    # more components.
    result = []
    for subscript in subscripts:
        if len(subscript) == 1:
            result.append(subscript[0])
        else:
            result.append(make_slice_node(pos, *subscript))
    return result

def make_slice_node(pos, start, stop = None, step = None):
    if not start:
        start = ExprNodes.NoneNode(pos)
    if not stop:
        stop = ExprNodes.NoneNode(pos)
    if not step:
        step = ExprNodes.NoneNode(pos)
    return ExprNodes.SliceNode(pos,
        start = start, stop = stop, step = step)

#atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+

def p_atom(s):
    pos = s.position()
    sy = s.sy
    if sy == '(':
        s.next()
        if s.sy == ')':
            result = ExprNodes.TupleNode(pos, args = [])
        else:
            result = p_expr(s)
        s.expect(')')
        return result
    elif sy == '[':
        return p_list_maker(s)
    elif sy == '{':
        return p_dict_maker(s)
    elif sy == '`':
        return p_backquote_expr(s)
    elif sy == 'INT':
466
        value = s.systring
William Stein's avatar
William Stein committed
467 468 469 470 471 472 473 474 475 476 477 478 479 480
        s.next()
        return ExprNodes.IntNode(pos, value = value)
    elif sy == 'LONG':
        value = s.systring
        s.next()
        return ExprNodes.LongNode(pos, value = value)
    elif sy == 'FLOAT':
        value = s.systring
        s.next()
        return ExprNodes.FloatNode(pos, value = value)
    elif sy == 'IMAG':
        value = s.systring[:-1]
        s.next()
        return ExprNodes.ImagNode(pos, value = value)
481
    elif sy == 'BEGIN_STRING':
William Stein's avatar
William Stein committed
482 483 484 485 486 487
        kind, value = p_cat_string_literal(s)
        if kind == 'c':
            return ExprNodes.CharNode(pos, value = value)
        else:
            return ExprNodes.StringNode(pos, value = value)
    elif sy == 'IDENT':
488
        name = Utils.EncodedString( s.systring )
William Stein's avatar
William Stein committed
489 490 491
        s.next()
        if name == "None":
            return ExprNodes.NoneNode(pos)
492
        elif name == "True":
493
            return ExprNodes.BoolNode(pos, value=True)
494
        elif name == "False":
495
            return ExprNodes.BoolNode(pos, value=False)
William Stein's avatar
William Stein committed
496
        else:
497
            return p_name(s, name)
William Stein's avatar
William Stein committed
498 499 500 501 502 503
    elif sy == 'NULL':
        s.next()
        return ExprNodes.NullNode(pos)
    else:
        s.error("Expected an identifier or literal")

504 505 506 507 508 509 510 511 512
def p_name(s, name):
    pos = s.position()
    if not s.compile_time_expr:
        try:
            value = s.compile_time_env.lookup_here(name)
        except KeyError:
            pass
        else:
            rep = repr(value)
513 514 515
            if isinstance(value, bool):
                return ExprNodes.BoolNode(pos, value = value)
            elif isinstance(value, int):
516 517 518 519 520
                return ExprNodes.IntNode(pos, value = rep)
            elif isinstance(value, long):
                return ExprNodes.LongNode(pos, value = rep)
            elif isinstance(value, float):
                return ExprNodes.FloatNode(pos, value = rep)
521
            elif isinstance(value, unicode):
522
                return ExprNodes.StringNode(pos, value = value)
523 524 525 526 527
            else:
                error(pos, "Invalid type for compile-time constant: %s"
                    % value.__class__.__name__)
    return ExprNodes.NameNode(pos, name = name)

William Stein's avatar
William Stein committed
528 529
def p_cat_string_literal(s):
    # A sequence of one or more adjacent string literals.
530
    # Returns (kind, value) where kind in ('b', 'c', 'u')
William Stein's avatar
William Stein committed
531
    kind, value = p_string_literal(s)
Stefan Behnel's avatar
Stefan Behnel committed
532
    if kind != 'c':
William Stein's avatar
William Stein committed
533
        strings = [value]
534
        while s.sy == 'BEGIN_STRING':
William Stein's avatar
William Stein committed
535 536
            next_kind, next_value = p_string_literal(s)
            if next_kind == 'c':
537 538
                error(s.position(),
                      "Cannot concatenate char literal with another string or char literal")
539 540
            elif next_kind == 'u':
                kind = 'u'
William Stein's avatar
William Stein committed
541
            strings.append(next_value)
542
        value = Utils.EncodedString( u''.join(strings) )
543 544
        if kind != 'u':
            value.encoding = s.source_encoding
William Stein's avatar
William Stein committed
545 546 547
    return kind, value

def p_opt_string_literal(s):
548
    if s.sy == 'BEGIN_STRING':
William Stein's avatar
William Stein committed
549 550 551 552 553 554
        return p_string_literal(s)
    else:
        return None

def p_string_literal(s):
    # A single string or char literal.
555
    # Returns (kind, value) where kind in ('b', 'c', 'u')
William Stein's avatar
William Stein committed
556 557
    # s.sy == 'BEGIN_STRING'
    pos = s.position()
558
    is_raw = 0
William Stein's avatar
William Stein committed
559
    kind = s.systring[:1].lower()
560 561 562 563 564 565
    if kind == 'r':
        kind = ''
        is_raw = 1
    elif kind in 'ub':
        is_raw = s.systring[1:2].lower() == 'r'
    elif kind != 'c':
William Stein's avatar
William Stein committed
566
        kind = ''
Stefan Behnel's avatar
Stefan Behnel committed
567 568 569
    if Future.unicode_literals in s.context.future_directives:
        if kind == '':
            kind = 'u'
570 571
    elif kind == '':
        kind = 'b'
William Stein's avatar
William Stein committed
572 573 574 575 576 577 578 579 580 581 582 583
    chars = []
    while 1:
        s.next()
        sy = s.sy
        #print "p_string_literal: sy =", sy, repr(s.systring) ###
        if sy == 'CHARS':
            systr = s.systring
            if len(systr) == 1 and systr in "'\"\n":
                chars.append('\\')
            chars.append(systr)
        elif sy == 'ESCAPE':
            systr = s.systring
584
            if is_raw:
William Stein's avatar
William Stein committed
585 586 587 588 589 590 591 592 593 594 595 596 597 598
                if systr == '\\\n':
                    chars.append(r'\\\n')
                elif systr == r'\"':
                    chars.append(r'\\\"')
                elif systr == r'\\':
                    chars.append(r'\\\\')
                else:
                    chars.append('\\' + systr)
            else:
                c = systr[1]
                if c in "'\"\\abfnrtv01234567":
                    chars.append(systr)
                elif c == '\n':
                    pass
599 600 601
                elif c in 'ux':
                    if kind == 'u':
                        try:
602 603
                            chars.append(
                                systr.encode("ASCII").decode('unicode_escape'))
604 605 606 607 608 609 610
                        except UnicodeDecodeError:
                            s.error("Invalid unicode escape '%s'" % systr,
                                    pos = pos)
                    elif c == 'x':
                        chars.append('\\x0' + systr[2:])
                    else:
                        chars.append(systr)
William Stein's avatar
William Stein committed
611
                else:
612
                    chars.append(r'\\' + systr[1:])
William Stein's avatar
William Stein committed
613 614 615 616 617 618 619 620 621 622 623
        elif sy == 'NEWLINE':
            chars.append(r'\n')
        elif sy == 'END_STRING':
            break
        elif sy == 'EOF':
            s.error("Unclosed string literal", pos = pos)
        else:
            s.error(
                "Unexpected token %r:%r in string literal" %
                    (sy, s.systring))
    s.next()
624
    value = Utils.EncodedString( u''.join(chars) )
625 626
    if kind != 'u':
        value.encoding = s.source_encoding
William Stein's avatar
William Stein committed
627 628 629
    #print "p_string_literal: value =", repr(value) ###
    return kind, value

Robert Bradshaw's avatar
Robert Bradshaw committed
630 631 632 633 634 635
# list_display  	::=  	"[" [listmaker] "]"
# listmaker 	::= 	expression ( list_for | ( "," expression )* [","] )
# list_iter 	::= 	list_for | list_if
# list_for 	::= 	"for" expression_list "in" testlist [list_iter]
# list_if 	::= 	"if" test [list_iter]
        
William Stein's avatar
William Stein committed
636 637 638 639
def p_list_maker(s):
    # s.sy == '['
    pos = s.position()
    s.next()
Robert Bradshaw's avatar
Robert Bradshaw committed
640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669
    if s.sy == ']':
        s.expect(']')
        return ExprNodes.ListNode(pos, args = [])
    expr = p_simple_expr(s)
    if s.sy == 'for':
        loop = p_list_for(s)
        s.expect(']')
        inner_loop = loop
        while not isinstance(inner_loop.body, Nodes.PassStatNode):
            inner_loop = inner_loop.body
            if isinstance(inner_loop, Nodes.IfStatNode):
                 inner_loop = inner_loop.if_clauses[0]
        append = ExprNodes.ListComprehensionAppendNode( pos, expr = expr )
        inner_loop.body = Nodes.ExprStatNode(pos, expr = append)
        return ExprNodes.ListComprehensionNode(pos, loop = loop, append = append)
    else:
        exprs = [expr]
        if s.sy == ',':
            s.next()
            exprs += p_simple_expr_list(s)
        s.expect(']')
        return ExprNodes.ListNode(pos, args = exprs)
        
def p_list_iter(s):
    if s.sy == 'for':
        return p_list_for(s)
    elif s.sy == 'if':
        return p_list_if(s)
    else:
        return Nodes.PassStatNode(s.position())
William Stein's avatar
William Stein committed
670

Robert Bradshaw's avatar
Robert Bradshaw committed
671 672 673 674 675 676 677 678 679 680 681 682 683
def p_list_for(s):
    # s.sy == 'for'
    pos = s.position()
    s.next()
    kw = p_for_bounds(s)
    kw['else_clause'] = None
    kw['body'] = p_list_iter(s)
    return Nodes.ForStatNode(pos, **kw)
        
def p_list_if(s):
    # s.sy == 'if'
    pos = s.position()
    s.next()
Robert Bradshaw's avatar
Robert Bradshaw committed
684
    test = p_test(s)
Robert Bradshaw's avatar
Robert Bradshaw committed
685 686 687 688
    return Nodes.IfStatNode(pos, 
        if_clauses = [Nodes.IfClauseNode(pos, condition = test, body = p_list_iter(s))],
        else_clause = None )
    
William Stein's avatar
William Stein committed
689 690 691 692 693 694 695
#dictmaker: test ':' test (',' test ':' test)* [',']

def p_dict_maker(s):
    # s.sy == '{'
    pos = s.position()
    s.next()
    items = []
Stefan Behnel's avatar
Stefan Behnel committed
696
    while s.sy != '}':
Robert Bradshaw's avatar
Robert Bradshaw committed
697
        items.append(p_dict_item(s))
Stefan Behnel's avatar
Stefan Behnel committed
698
        if s.sy != ',':
William Stein's avatar
William Stein committed
699 700 701 702
            break
        s.next()
    s.expect('}')
    return ExprNodes.DictNode(pos, key_value_pairs = items)
Robert Bradshaw's avatar
Robert Bradshaw committed
703 704 705 706 707 708
    
def p_dict_item(s):
    key = p_simple_expr(s)
    s.expect(':')
    value = p_simple_expr(s)
    return ExprNodes.DictItemNode(key.pos, key=key, value=value)
William Stein's avatar
William Stein committed
709 710 711 712 713 714 715 716 717 718 719 720 721

def p_backquote_expr(s):
    # s.sy == '`'
    pos = s.position()
    s.next()
    arg = p_expr(s)
    s.expect('`')
    return ExprNodes.BackquoteNode(pos, arg = arg)

def p_simple_expr_list(s):
    exprs = []
    while s.sy not in expr_terminators:
        exprs.append(p_simple_expr(s))
Stefan Behnel's avatar
Stefan Behnel committed
722
        if s.sy != ',':
William Stein's avatar
William Stein committed
723 724 725 726 727 728 729 730 731 732 733 734 735 736
            break
        s.next()
    return exprs

def p_expr(s):
    pos = s.position()
    expr = p_simple_expr(s)
    if s.sy == ',':
        s.next()
        exprs = [expr] + p_simple_expr_list(s)
        return ExprNodes.TupleNode(pos, args = exprs)
    else:
        return expr

Robert Bradshaw's avatar
Robert Bradshaw committed
737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752

#testlist: test (',' test)* [',']
# differs from p_expr only in the fact that it cannot contain conditional expressions

def p_testlist(s):
    pos = s.position()
    expr = p_test(s)
    if s.sy == ',':
        exprs = [expr]
        while s.sy == ',':
            s.next()
            exprs.append(p_test(s))
        return ExprNodes.TupleNode(pos, args = exprs)
    else:
        return expr
        
William Stein's avatar
William Stein committed
753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773
expr_terminators = (')', ']', '}', ':', '=', 'NEWLINE')

#-------------------------------------------------------
#
#   Statements
#
#-------------------------------------------------------

def p_global_statement(s):
    # assume s.sy == 'global'
    pos = s.position()
    s.next()
    names = p_ident_list(s)
    return Nodes.GlobalNode(pos, names = names)

def p_expression_or_assignment(s):
    expr_list = [p_expr(s)]
    while s.sy == '=':
        s.next()
        expr_list.append(p_expr(s))
    if len(expr_list) == 1:
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
774
        if re.match(r"([+*/\%^\&|-]|<<|>>|\*\*|//)=", s.sy):
775 776 777
            lhs = expr_list[0]
            if not isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode, ExprNodes.NameNode) ):
                error(lhs.pos, "Illegal operand for inplace operation.")
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
778
            operator = s.sy[:-1]
779 780 781
            s.next()
            rhs = p_expr(s)
            return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs)
782 783 784
        expr = expr_list[0]
        if isinstance(expr, ExprNodes.StringNode):
            return Nodes.PassStatNode(expr.pos)
785 786
        else:
            return Nodes.ExprStatNode(expr.pos, expr = expr)
William Stein's avatar
William Stein committed
787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833
    else:
        expr_list_list = []
        flatten_parallel_assignments(expr_list, expr_list_list)
        nodes = []
        for expr_list in expr_list_list:
            lhs_list = expr_list[:-1]
            rhs = expr_list[-1]
            if len(lhs_list) == 1:
                node = Nodes.SingleAssignmentNode(rhs.pos, 
                    lhs = lhs_list[0], rhs = rhs)
            else:
                node = Nodes.CascadedAssignmentNode(rhs.pos,
                    lhs_list = lhs_list, rhs = rhs)
            nodes.append(node)
        if len(nodes) == 1:
            return nodes[0]
        else:
            return Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)

def flatten_parallel_assignments(input, output):
    #  The input is a list of expression nodes, representing 
    #  the LHSs and RHS of one (possibly cascaded) assignment 
    #  statement. If they are all sequence constructors with 
    #  the same number of arguments, rearranges them into a
    #  list of equivalent assignments between the individual 
    #  elements. This transformation is applied recursively.
    size = find_parallel_assignment_size(input)
    if size >= 0:
        for i in range(size):
            new_exprs = [expr.args[i] for expr in input]
            flatten_parallel_assignments(new_exprs, output)
    else:
        output.append(input)

def find_parallel_assignment_size(input):
    #  The input is a list of expression nodes. If 
    #  they are all sequence constructors with the same number
    #  of arguments, return that number, else return -1.
    #  Produces an error message if they are all sequence
    #  constructors but not all the same size.
    for expr in input:
        if not expr.is_sequence_constructor:
            return -1
    rhs = input[-1]
    rhs_size = len(rhs.args)
    for lhs in input[:-1]:
        lhs_size = len(lhs.args)
Stefan Behnel's avatar
Stefan Behnel committed
834
        if lhs_size != rhs_size:
William Stein's avatar
William Stein committed
835 836 837 838 839 840 841 842 843 844 845 846
            error(lhs.pos, "Unpacking sequence of wrong size (expected %d, got %d)"
                % (lhs_size, rhs_size))
            return -1
    return rhs_size

def p_print_statement(s):
    # s.sy == 'print'
    pos = s.position()
    s.next()
    if s.sy == '>>':
        s.error("'print >>' not yet implemented")
    args = []
847
    ends_with_comma = 0
William Stein's avatar
William Stein committed
848 849 850 851 852
    if s.sy not in ('NEWLINE', 'EOF'):
        args.append(p_simple_expr(s))
        while s.sy == ',':
            s.next()
            if s.sy in ('NEWLINE', 'EOF'):
853
                ends_with_comma = 1
William Stein's avatar
William Stein committed
854 855
                break
            args.append(p_simple_expr(s))
856
    arg_tuple = ExprNodes.TupleNode(pos, args = args)
857 858
    return Nodes.PrintStatNode(pos,
        arg_tuple = arg_tuple, append_newline = not ends_with_comma)
William Stein's avatar
William Stein committed
859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910

def p_del_statement(s):
    # s.sy == 'del'
    pos = s.position()
    s.next()
    args = p_simple_expr_list(s)
    return Nodes.DelStatNode(pos, args = args)

def p_pass_statement(s, with_newline = 0):
    pos = s.position()
    s.expect('pass')
    if with_newline:
        s.expect_newline("Expected a newline")
    return Nodes.PassStatNode(pos)

def p_break_statement(s):
    # s.sy == 'break'
    pos = s.position()
    s.next()
    return Nodes.BreakStatNode(pos)

def p_continue_statement(s):
    # s.sy == 'continue'
    pos = s.position()
    s.next()
    return Nodes.ContinueStatNode(pos)

def p_return_statement(s):
    # s.sy == 'return'
    pos = s.position()
    s.next()
    if s.sy not in statement_terminators:
        value = p_expr(s)
    else:
        value = None
    return Nodes.ReturnStatNode(pos, value = value)

def p_raise_statement(s):
    # s.sy == 'raise'
    pos = s.position()
    s.next()
    exc_type = None
    exc_value = None
    exc_tb = None
    if s.sy not in statement_terminators:
        exc_type = p_simple_expr(s)
        if s.sy == ',':
            s.next()
            exc_value = p_simple_expr(s)
            if s.sy == ',':
                s.next()
                exc_tb = p_simple_expr(s)
911 912 913 914 915 916 917
    if exc_type or exc_value or exc_tb:
        return Nodes.RaiseStatNode(pos, 
            exc_type = exc_type,
            exc_value = exc_value,
            exc_tb = exc_tb)
    else:
        return Nodes.ReraiseStatNode(pos)
William Stein's avatar
William Stein committed
918 919 920 921 922 923 924 925 926 927 928 929

def p_import_statement(s):
    # s.sy in ('import', 'cimport')
    pos = s.position()
    kind = s.sy
    s.next()
    items = [p_dotted_name(s, as_allowed = 1)]
    while s.sy == ',':
        s.next()
        items.append(p_dotted_name(s, as_allowed = 1))
    stats = []
    for pos, target_name, dotted_name, as_name in items:
930
        dotted_name = Utils.EncodedString(dotted_name)
William Stein's avatar
William Stein committed
931 932 933 934 935
        if kind == 'cimport':
            stat = Nodes.CImportStatNode(pos, 
                module_name = dotted_name,
                as_name = as_name)
        else:
936 937
            if as_name and "." in dotted_name:
                name_list = ExprNodes.ListNode(pos, args = [
938
                    ExprNodes.StringNode(pos, value = Utils.EncodedString("*"))])
939 940
            else:
                name_list = None
William Stein's avatar
William Stein committed
941 942 943 944
            stat = Nodes.SingleAssignmentNode(pos,
                lhs = ExprNodes.NameNode(pos, 
                    name = as_name or target_name),
                rhs = ExprNodes.ImportNode(pos, 
945 946
                    module_name = ExprNodes.IdentifierStringNode(
                        pos, value = dotted_name),
947
                    name_list = name_list))
William Stein's avatar
William Stein committed
948 949 950
        stats.append(stat)
    return Nodes.StatListNode(pos, stats = stats)

Stefan Behnel's avatar
Stefan Behnel committed
951
def p_from_import_statement(s, first_statement = 0):
William Stein's avatar
William Stein committed
952 953 954 955 956 957 958 959 960 961
    # s.sy == 'from'
    pos = s.position()
    s.next()
    (dotted_name_pos, _, dotted_name, _) = \
        p_dotted_name(s, as_allowed = 0)
    if s.sy in ('import', 'cimport'):
        kind = s.sy
        s.next()
    else:
        s.error("Expected 'import' or 'cimport'")
962
    is_cimport = kind == 'cimport'
William Stein's avatar
William Stein committed
963
    if s.sy == '*':
964
        imported_names = [(s.position(), "*", None, None)]
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
965 966
        s.next()
    else:
967
        imported_names = [p_imported_name(s, is_cimport)]
William Stein's avatar
William Stein committed
968 969
    while s.sy == ',':
        s.next()
970
        imported_names.append(p_imported_name(s, is_cimport))
971
    dotted_name = Utils.EncodedString(dotted_name)
Stefan Behnel's avatar
Stefan Behnel committed
972 973 974 975
    if dotted_name == '__future__':
        if not first_statement:
            s.error("from __future__ imports must occur at the beginning of the file")
        else:
976
            for (name_pos, name, as_name, kind) in imported_names:
977 978 979
                if name == "braces":
                    s.error("not a chance", name_pos)
                    break
Stefan Behnel's avatar
Stefan Behnel committed
980 981 982
                try:
                    directive = getattr(Future, name)
                except AttributeError:
983
                    s.error("future feature %s is not defined" % name, name_pos)
Stefan Behnel's avatar
Stefan Behnel committed
984 985 986 987
                    break
                s.context.future_directives.add(directive)
        return Nodes.PassStatNode(pos)
    elif kind == 'cimport':
988
        for (name_pos, name, as_name, kind) in imported_names:
William Stein's avatar
William Stein committed
989 990 991 992 993 994 995 996
            local_name = as_name or name
            s.add_type_name(local_name)
        return Nodes.FromCImportStatNode(pos,
            module_name = dotted_name,
            imported_names = imported_names)
    else:
        imported_name_strings = []
        items = []
997
        for (name_pos, name, as_name, kind) in imported_names:
998
            encoded_name = Utils.EncodedString(name)
William Stein's avatar
William Stein committed
999
            imported_name_strings.append(
1000
                ExprNodes.IdentifierStringNode(name_pos, value = encoded_name))
William Stein's avatar
William Stein committed
1001 1002 1003
            items.append(
                (name,
                 ExprNodes.NameNode(name_pos, 
Stefan Behnel's avatar
Stefan Behnel committed
1004
                                    name = as_name or name)))
William Stein's avatar
William Stein committed
1005 1006
        import_list = ExprNodes.ListNode(
            imported_names[0][0], args = imported_name_strings)
1007
        dotted_name = Utils.EncodedString(dotted_name)
William Stein's avatar
William Stein committed
1008 1009
        return Nodes.FromImportStatNode(pos,
            module = ExprNodes.ImportNode(dotted_name_pos,
1010
                module_name = ExprNodes.IdentifierStringNode(pos, value = dotted_name),
William Stein's avatar
William Stein committed
1011 1012 1013
                name_list = import_list),
            items = items)

1014 1015 1016
imported_name_kinds = ('class', 'struct', 'union')

def p_imported_name(s, is_cimport):
William Stein's avatar
William Stein committed
1017
    pos = s.position()
1018 1019 1020 1021
    kind = None
    if is_cimport and s.systring in imported_name_kinds:
        kind = s.systring
        s.next()
William Stein's avatar
William Stein committed
1022 1023
    name = p_ident(s)
    as_name = p_as_name(s)
1024
    return (pos, name, as_name, kind)
William Stein's avatar
William Stein committed
1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035

def p_dotted_name(s, as_allowed):
    pos = s.position()
    target_name = p_ident(s)
    as_name = None
    names = [target_name]
    while s.sy == '.':
        s.next()
        names.append(p_ident(s))
    if as_allowed:
        as_name = p_as_name(s)
Stefan Behnel's avatar
Stefan Behnel committed
1036
    return (pos, target_name, u'.'.join(names), as_name)
William Stein's avatar
William Stein committed
1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099

def p_as_name(s):
    if s.sy == 'IDENT' and s.systring == 'as':
        s.next()
        return p_ident(s)
    else:
        return None

def p_assert_statement(s):
    # s.sy == 'assert'
    pos = s.position()
    s.next()
    cond = p_simple_expr(s)
    if s.sy == ',':
        s.next()
        value = p_simple_expr(s)
    else:
        value = None
    return Nodes.AssertStatNode(pos, cond = cond, value = value)

statement_terminators = (';', 'NEWLINE', 'EOF')

def p_if_statement(s):
    # s.sy == 'if'
    pos = s.position()
    s.next()
    if_clauses = [p_if_clause(s)]
    while s.sy == 'elif':
        s.next()
        if_clauses.append(p_if_clause(s))
    else_clause = p_else_clause(s)
    return Nodes.IfStatNode(pos,
        if_clauses = if_clauses, else_clause = else_clause)

def p_if_clause(s):
    pos = s.position()
    test = p_simple_expr(s)
    body = p_suite(s)
    return Nodes.IfClauseNode(pos,
        condition = test, body = body)

def p_else_clause(s):
    if s.sy == 'else':
        s.next()
        return p_suite(s)
    else:
        return None

def p_while_statement(s):
    # s.sy == 'while'
    pos = s.position()
    s.next()
    test = p_simple_expr(s)
    body = p_suite(s)
    else_clause = p_else_clause(s)
    return Nodes.WhileStatNode(pos, 
        condition = test, body = body, 
        else_clause = else_clause)

def p_for_statement(s):
    # s.sy == 'for'
    pos = s.position()
    s.next()
Robert Bradshaw's avatar
Robert Bradshaw committed
1100 1101 1102 1103 1104 1105
    kw = p_for_bounds(s)
    kw['body'] = p_suite(s)
    kw['else_clause'] = p_else_clause(s)
    return Nodes.ForStatNode(pos, **kw)
            
def p_for_bounds(s):
William Stein's avatar
William Stein committed
1106 1107 1108 1109
    target = p_for_target(s)
    if s.sy == 'in':
        s.next()
        iterator = p_for_iterator(s)
Robert Bradshaw's avatar
Robert Bradshaw committed
1110
        return { 'target': target, 'iterator': iterator }
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
1111 1112 1113 1114 1115 1116 1117
    else:
        if s.sy == 'from':
            s.next()
            bound1 = p_bit_expr(s)
        else:
            # Support shorter "for a <= x < b" syntax
            bound1, target = target, None
William Stein's avatar
William Stein committed
1118 1119 1120 1121 1122 1123
        rel1 = p_for_from_relation(s)
        name2_pos = s.position()
        name2 = p_ident(s)
        rel2_pos = s.position()
        rel2 = p_for_from_relation(s)
        bound2 = p_bit_expr(s)
1124
        step = p_for_from_step(s)
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
1125 1126 1127 1128 1129 1130 1131 1132 1133
        if target is None:
            target = ExprNodes.NameNode(name2_pos, name = name2)
        else:
            if not target.is_name:
                error(target.pos, 
                    "Target of for-from statement must be a variable name")
            elif name2 != target.name:
                error(name2_pos,
                    "Variable name in for-from range does not match target")
Stefan Behnel's avatar
Stefan Behnel committed
1134
        if rel1[0] != rel2[0]:
William Stein's avatar
William Stein committed
1135 1136
            error(rel2_pos,
                "Relation directions in for-from do not match")
Robert Bradshaw's avatar
Robert Bradshaw committed
1137 1138 1139 1140
        return {'target': target, 
                'bound1': bound1, 
                'relation1': rel1, 
                'relation2': rel2,
1141 1142
                'bound2': bound2,
                'step': step }
William Stein's avatar
William Stein committed
1143 1144 1145 1146 1147 1148 1149 1150

def p_for_from_relation(s):
    if s.sy in inequality_relations:
        op = s.sy
        s.next()
        return op
    else:
        s.error("Expected one of '<', '<=', '>' '>='")
1151

1152 1153 1154 1155 1156 1157 1158
def p_for_from_step(s):
    if s.sy == 'by':
        s.next()
        step = p_bit_expr(s)
        return step
    else:
        return None
William Stein's avatar
William Stein committed
1159 1160 1161 1162 1163 1164 1165 1166 1167

inequality_relations = ('<', '<=', '>', '>=')

def p_for_target(s):
    pos = s.position()
    expr = p_bit_expr(s)
    if s.sy == ',':
        s.next()
        exprs = [expr]
Stefan Behnel's avatar
Stefan Behnel committed
1168
        while s.sy != 'in':
William Stein's avatar
William Stein committed
1169
            exprs.append(p_bit_expr(s))
Stefan Behnel's avatar
Stefan Behnel committed
1170
            if s.sy != ',':
William Stein's avatar
William Stein committed
1171 1172 1173 1174 1175 1176 1177 1178
                break
            s.next()
        return ExprNodes.TupleNode(pos, args = exprs)
    else:
        return expr

def p_for_iterator(s):
    pos = s.position()
Robert Bradshaw's avatar
Robert Bradshaw committed
1179
    expr = p_testlist(s)
William Stein's avatar
William Stein committed
1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194
    return ExprNodes.IteratorNode(pos, sequence = expr)

def p_try_statement(s):
    # s.sy == 'try'
    pos = s.position()
    s.next()
    body = p_suite(s)
    except_clauses = []
    else_clause = None
    if s.sy in ('except', 'else'):
        while s.sy == 'except':
            except_clauses.append(p_except_clause(s))
        if s.sy == 'else':
            s.next()
            else_clause = p_suite(s)
1195
        body = Nodes.TryExceptStatNode(pos,
William Stein's avatar
William Stein committed
1196 1197
            body = body, except_clauses = except_clauses,
            else_clause = else_clause)
1198 1199 1200 1201
        if s.sy != 'finally':
            return body
        # try-except-finally is equivalent to nested try-except/try-finally
    if s.sy == 'finally':
William Stein's avatar
William Stein committed
1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214
        s.next()
        finally_clause = p_suite(s)
        return Nodes.TryFinallyStatNode(pos,
            body = body, finally_clause = finally_clause)
    else:
        s.error("Expected 'except' or 'finally'")

def p_except_clause(s):
    # s.sy == 'except'
    pos = s.position()
    s.next()
    exc_type = None
    exc_value = None
Stefan Behnel's avatar
Stefan Behnel committed
1215
    if s.sy != ':':
William Stein's avatar
William Stein committed
1216 1217 1218 1219 1220 1221 1222 1223
        exc_type = p_simple_expr(s)
        if s.sy == ',':
            s.next()
            exc_value = p_simple_expr(s)
    body = p_suite(s)
    return Nodes.ExceptClauseNode(pos,
        pattern = exc_type, target = exc_value, body = body)

1224
def p_include_statement(s, ctx):
William Stein's avatar
William Stein committed
1225 1226 1227 1228
    pos = s.position()
    s.next() # 'include'
    _, include_file_name = p_string_literal(s)
    s.expect_newline("Syntax error in include statement")
1229 1230 1231
    if s.compile_time_eval:
        include_file_path = s.context.find_include_file(include_file_name, pos)
        if include_file_path:
1232
            s.included_files.append(include_file_name)
1233
            f = Utils.open_source_file(include_file_path, mode="rU")
1234 1235
            source_desc = FileSourceDescriptor(include_file_path)
            s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding)
1236
            try:
1237
                tree = p_statement_list(s2, ctx)
1238 1239 1240 1241 1242
            finally:
                f.close()
            return tree
        else:
            return None
William Stein's avatar
William Stein committed
1243
    else:
1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255
        return Nodes.PassStatNode(pos)

def p_with_statement(s):
    pos = s.position()
    s.next() # 'with'
#	if s.sy == 'IDENT' and s.systring in ('gil', 'nogil'):
    if s.sy == 'IDENT' and s.systring == 'nogil':
        state = s.systring
        s.next()
        body = p_suite(s)
        return Nodes.GILStatNode(pos, state = state, body = body)
    else:
1256 1257
        s.error("Only 'with gil' and 'with nogil' implemented",
                pos = pos)
William Stein's avatar
William Stein committed
1258
    
Stefan Behnel's avatar
Stefan Behnel committed
1259
def p_simple_statement(s, first_statement = 0):
William Stein's avatar
William Stein committed
1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277
    #print "p_simple_statement:", s.sy, s.systring ###
    if s.sy == 'global':
        node = p_global_statement(s)
    elif s.sy == 'print':
        node = p_print_statement(s)
    elif s.sy == 'del':
        node = p_del_statement(s)
    elif s.sy == 'break':
        node = p_break_statement(s)
    elif s.sy == 'continue':
        node = p_continue_statement(s)
    elif s.sy == 'return':
        node = p_return_statement(s)
    elif s.sy == 'raise':
        node = p_raise_statement(s)
    elif s.sy in ('import', 'cimport'):
        node = p_import_statement(s)
    elif s.sy == 'from':
Stefan Behnel's avatar
Stefan Behnel committed
1278
        node = p_from_import_statement(s, first_statement = first_statement)
William Stein's avatar
William Stein committed
1279 1280 1281 1282 1283 1284 1285 1286
    elif s.sy == 'assert':
        node = p_assert_statement(s)
    elif s.sy == 'pass':
        node = p_pass_statement(s)
    else:
        node = p_expression_or_assignment(s)
    return node

1287
def p_simple_statement_list(s, ctx, first_statement = 0):
William Stein's avatar
William Stein committed
1288 1289
    # Parse a series of simple statements on one line
    # separated by semicolons.
Stefan Behnel's avatar
Stefan Behnel committed
1290
    stat = p_simple_statement(s, first_statement = first_statement)
William Stein's avatar
William Stein committed
1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302
    if s.sy == ';':
        stats = [stat]
        while s.sy == ';':
            #print "p_simple_statement_list: maybe more to follow" ###
            s.next()
            if s.sy in ('NEWLINE', 'EOF'):
                break
            stats.append(p_simple_statement(s))
        stat = Nodes.StatListNode(stats[0].pos, stats = stats)
    s.expect_newline("Syntax error in simple statement list")
    return stat

1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322
def p_compile_time_expr(s):
    old = s.compile_time_expr
    s.compile_time_expr = 1
    expr = p_expr(s)
    s.compile_time_expr = old
    return expr

def p_DEF_statement(s):
    pos = s.position()
    denv = s.compile_time_env
    s.next() # 'DEF'
    name = p_ident(s)
    s.expect('=')
    expr = p_compile_time_expr(s)
    value = expr.compile_time_value(denv)
    #print "p_DEF_statement: %s = %r" % (name, value) ###
    denv.declare(name, value)
    s.expect_newline()
    return Nodes.PassStatNode(pos)

1323
def p_IF_statement(s, ctx):
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
1324
    pos = s.position()
1325 1326 1327 1328 1329 1330 1331 1332
    saved_eval = s.compile_time_eval
    current_eval = saved_eval
    denv = s.compile_time_env
    result = None
    while 1:
        s.next() # 'IF' or 'ELIF'
        expr = p_compile_time_expr(s)
        s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv))
1333
        body = p_suite(s, ctx)
1334 1335 1336
        if s.compile_time_eval:
            result = body
            current_eval = 0
Stefan Behnel's avatar
Stefan Behnel committed
1337
        if s.sy != 'ELIF':
1338 1339 1340 1341
            break
    if s.sy == 'ELSE':
        s.next()
        s.compile_time_eval = current_eval
1342
        body = p_suite(s, ctx)
1343 1344 1345
        if current_eval:
            result = body
    if not result:
Stefan Behnel's avatar
Stefan Behnel committed
1346
        result = Nodes.PassStatNode(pos)
1347 1348 1349
    s.compile_time_eval = saved_eval
    return result

1350 1351
def p_statement(s, ctx, first_statement = 0):
    cdef_flag = ctx.cdef_flag
William Stein's avatar
William Stein committed
1352
    if s.sy == 'ctypedef':
1353
        if ctx.level not in ('module', 'module_pxd'):
William Stein's avatar
William Stein committed
1354
            s.error("ctypedef statement not allowed here")
1355
        if ctx.api:
1356
            error(s.position(), "'api' not allowed with 'ctypedef'")
1357
        return p_ctypedef_statement(s, ctx)
1358 1359 1360
    elif s.sy == 'DEF':
        return p_DEF_statement(s)
    elif s.sy == 'IF':
1361
        return p_IF_statement(s, ctx)
William Stein's avatar
William Stein committed
1362
    else:
1363 1364 1365 1366
        overridable = 0
        if s.sy == 'cdef':
            cdef_flag = 1
            s.next()
Robert Bradshaw's avatar
Robert Bradshaw committed
1367
        if s.sy == 'cpdef':
1368 1369 1370 1371
            cdef_flag = 1
            overridable = 1
            s.next()
        if cdef_flag:
1372
            if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):
1373
                s.error('cdef statement not allowed here')
1374 1375
            s.level = ctx.level
            return p_cdef_statement(s, ctx(overridable = overridable))
William Stein's avatar
William Stein committed
1376
        else:
1377
            if ctx.api:
1378 1379
                error(s.pos, "'api' not allowed with this statement")
            elif s.sy == 'def':
1380
                if ctx.level not in ('module', 'class', 'c_class', 'property'):
1381
                    s.error('def statement not allowed here')
1382
                s.level = ctx.level
1383 1384
                return p_def_statement(s)
            elif s.sy == 'class':
1385
                if ctx.level != 'module':
1386 1387 1388
                    s.error("class definition not allowed here")
                return p_class_statement(s)
            elif s.sy == 'include':
1389
                if ctx.level not in ('module', 'module_pxd'):
1390
                    s.error("include statement not allowed here")
1391 1392
                return p_include_statement(s, ctx)
            elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':
1393
                return p_property_decl(s)
1394
            elif s.sy == 'pass' and ctx.level != 'property':
1395 1396
                return p_pass_statement(s, with_newline = 1)
            else:
1397
                if ctx.level in ('c_class_pxd', 'property'):
1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409
                    s.error("Executable statement not allowed here")
                if s.sy == 'if':
                    return p_if_statement(s)
                elif s.sy == 'while':
                    return p_while_statement(s)
                elif s.sy == 'for':
                    return p_for_statement(s)
                elif s.sy == 'try':
                    return p_try_statement(s)
                elif s.sy == 'with':
                    return p_with_statement(s)
                else:
1410 1411
                    return p_simple_statement_list(
                        s, ctx, first_statement = first_statement)
William Stein's avatar
William Stein committed
1412

1413
def p_statement_list(s, ctx, first_statement = 0):
William Stein's avatar
William Stein committed
1414 1415 1416 1417
    # Parse a series of statements separated by newlines.
    pos = s.position()
    stats = []
    while s.sy not in ('DEDENT', 'EOF'):
1418
        stats.append(p_statement(s, ctx, first_statement = first_statement))
Stefan Behnel's avatar
Stefan Behnel committed
1419
        first_statement = 0
1420 1421 1422 1423
    if len(stats) == 1:
        return stats[0]
    else:
        return Nodes.StatListNode(pos, stats = stats)
William Stein's avatar
William Stein committed
1424

1425
def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):
William Stein's avatar
William Stein committed
1426 1427 1428 1429 1430 1431 1432
    pos = s.position()
    s.expect(':')
    doc = None
    stmts = []
    if s.sy == 'NEWLINE':
        s.next()
        s.expect_indent()
1433
        if with_doc or with_pseudo_doc:
William Stein's avatar
William Stein committed
1434
            doc = p_doc_string(s)
1435
        body = p_statement_list(s, ctx)
William Stein's avatar
William Stein committed
1436 1437
        s.expect_dedent()
    else:
1438
        if ctx.api:
1439
            error(s.pos, "'api' not allowed with this statement")
1440 1441
        if ctx.level in ('module', 'class', 'function', 'other'):
            body = p_simple_statement_list(s, ctx)
William Stein's avatar
William Stein committed
1442 1443 1444 1445 1446 1447 1448 1449
        else:
            body = p_pass_statement(s)
            s.expect_newline("Syntax error in declarations")
    if with_doc:
        return doc, body
    else:
        return body

1450
def p_c_base_type(s, self_flag = 0, nonempty = 0):
William Stein's avatar
William Stein committed
1451 1452 1453 1454 1455
    # If self_flag is true, this is the base type for the
    # self argument of a C method of an extension type.
    if s.sy == '(':
        return p_c_complex_base_type(s)
    else:
1456
        return p_c_simple_base_type(s, self_flag, nonempty = nonempty)
William Stein's avatar
William Stein committed
1457

1458 1459 1460 1461 1462 1463 1464 1465 1466 1467
def p_calling_convention(s):
    if s.sy == 'IDENT' and s.systring in calling_convention_words:
        result = s.systring
        s.next()
        return result
    else:
        return ""

calling_convention_words = ("__stdcall", "__cdecl")

William Stein's avatar
William Stein committed
1468 1469 1470 1471 1472 1473 1474 1475 1476 1477
def p_c_complex_base_type(s):
    # s.sy == '('
    pos = s.position()
    s.next()
    base_type = p_c_base_type(s)
    declarator = p_c_declarator(s, empty = 1)
    s.expect(')')
    return Nodes.CComplexBaseTypeNode(pos, 
        base_type = base_type, declarator = declarator)

1478
def p_c_simple_base_type(s, self_flag, nonempty):
William Stein's avatar
William Stein committed
1479 1480 1481 1482 1483
    #print "p_c_simple_base_type: self_flag =", self_flag
    is_basic = 0
    signed = 1
    longness = 0
    module_path = []
1484
    pos = s.position()
William Stein's avatar
William Stein committed
1485 1486 1487 1488 1489 1490 1491 1492 1493
    if looking_at_base_type(s):
        #print "p_c_simple_base_type: looking_at_base_type at", s.position()
        is_basic = 1
        signed, longness = p_sign_and_longness(s)
        if s.sy == 'IDENT' and s.systring in basic_c_type_names:
            name = s.systring
            s.next()
        else:
            name = 'int'
1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511
    elif s.looking_at_type_name():
        name = s.systring
        s.next()
        if nonempty and s.sy != 'IDENT':
            # Make sure this is not a declaration of a variable or 
            # function with the same name as a type.  
            if s.sy == '(':
                s.next()
                if s.sy == '*' or s.sy == '**':
                    s.put_back('(', '(')
                else:
                    s.put_back('(', '(')
                    s.put_back('IDENT', name)
                    name = None
            elif s.sy not in ('*', '**', '['):
                s.put_back('IDENT', name)
                name = None
    elif looking_at_dotted_name(s):
William Stein's avatar
William Stein committed
1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543
        #print "p_c_simple_base_type: looking_at_type_name at", s.position()
        name = s.systring
        s.next()
        while s.sy == '.':
            module_path.append(name)
            s.next()
            name = p_ident(s)
    else:
        #print "p_c_simple_base_type: not looking at type at", s.position()
        name = None
    return Nodes.CSimpleBaseTypeNode(pos, 
        name = name, module_path = module_path,
        is_basic_c_type = is_basic, signed = signed,
        longness = longness, is_self_arg = self_flag)

def looking_at_type(s):
    return looking_at_base_type(s) or s.looking_at_type_name()

def looking_at_base_type(s):
    #print "looking_at_base_type?", s.sy, s.systring, s.position()
    return s.sy == 'IDENT' and s.systring in base_type_start_words

def looking_at_dotted_name(s):
    if s.sy == 'IDENT':
        name = s.systring
        s.next()
        result = s.sy == '.'
        s.put_back('IDENT', name)
        return result
    else:
        return 0

1544
basic_c_type_names = ("void", "char", "int", "float", "double", "Py_ssize_t", "bint")
William Stein's avatar
William Stein committed
1545 1546 1547

sign_and_longness_words = ("short", "long", "signed", "unsigned")

1548 1549
base_type_start_words = \
    basic_c_type_names + sign_and_longness_words
William Stein's avatar
William Stein committed
1550 1551 1552 1553 1554 1555 1556

def p_sign_and_longness(s):
    signed = 1
    longness = 0
    while s.sy == 'IDENT' and s.systring in sign_and_longness_words:
        if s.systring == 'unsigned':
            signed = 0
1557 1558
        elif s.systring == 'signed':
            signed = 2
William Stein's avatar
William Stein committed
1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573
        elif s.systring == 'short':
            longness = -1
        elif s.systring == 'long':
            longness += 1
        s.next()
    return signed, longness

def p_opt_cname(s):
    literal = p_opt_string_literal(s)
    if literal:
        _, cname = literal
    else:
        cname = None
    return cname

1574 1575 1576
def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0,
                   assignable = 0, nonempty = 0,
                   calling_convention_allowed = 0):
1577 1578
    # If empty is true, the declarator must be empty. If nonempty is true,
    # the declarator must be nonempty. Otherwise we don't care.
William Stein's avatar
William Stein committed
1579 1580 1581
    # If cmethod_flag is true, then if this declarator declares
    # a function, it's a C method of an extension type.
    pos = s.position()
1582 1583 1584 1585
    if s.sy == '(':
        s.next()
        if s.sy == ')' or looking_at_type(s):
            base = Nodes.CNameDeclaratorNode(pos, name = "", cname = None)
1586
            result = p_c_func_declarator(s, pos, ctx, base, cmethod_flag)
1587
        else:
1588 1589 1590 1591
            result = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
                                    cmethod_flag = cmethod_flag,
                                    nonempty = nonempty,
                                    calling_convention_allowed = 1)
1592 1593
            s.expect(')')
    else:
1594 1595
        result = p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
                                       assignable, nonempty)
Stefan Behnel's avatar
Stefan Behnel committed
1596
    if not calling_convention_allowed and result.calling_convention and s.sy != '(':
1597 1598 1599 1600 1601 1602 1603 1604
        error(s.position(), "%s on something that is not a function"
            % result.calling_convention)
    while s.sy in ('[', '('):
        pos = s.position()
        if s.sy == '[':
            result = p_c_array_declarator(s, result)
        else: # sy == '('
            s.next()
1605
            result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag)
1606 1607 1608 1609 1610 1611
        cmethod_flag = 0
    return result

def p_c_array_declarator(s, base):
    pos = s.position()
    s.next() # '['
Stefan Behnel's avatar
Stefan Behnel committed
1612
    if s.sy != ']':
1613 1614 1615 1616 1617 1618
        dim = p_expr(s)
    else:
        dim = None
    s.expect(']')
    return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim)

1619
def p_c_func_declarator(s, pos, ctx, base, cmethod_flag):
1620
    #  Opening paren has already been skipped
1621 1622
    args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag,
                        nonempty_declarators = 0)
1623 1624 1625 1626 1627 1628 1629 1630
    ellipsis = p_optional_ellipsis(s)
    s.expect(')')
    nogil = p_nogil(s)
    exc_val, exc_check = p_exception_value_clause(s)
    with_gil = p_with_gil(s)
    return Nodes.CFuncDeclaratorNode(pos, 
        base = base, args = args, has_varargs = ellipsis,
        exception_value = exc_val, exception_check = exc_check,
1631
        nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil)
1632

1633 1634
def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
                          assignable, nonempty):
1635 1636
    pos = s.position()
    calling_convention = p_calling_convention(s)
William Stein's avatar
William Stein committed
1637 1638
    if s.sy == '*':
        s.next()
1639 1640 1641
        base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
                              cmethod_flag = cmethod_flag,
                              assignable = assignable, nonempty = nonempty)
William Stein's avatar
William Stein committed
1642 1643 1644 1645
        result = Nodes.CPtrDeclaratorNode(pos, 
            base = base)
    elif s.sy == '**': # scanner returns this as a single token
        s.next()
1646 1647 1648
        base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
                              cmethod_flag = cmethod_flag,
                              assignable = assignable, nonempty = nonempty)
William Stein's avatar
William Stein committed
1649 1650 1651 1652
        result = Nodes.CPtrDeclaratorNode(pos,
            base = Nodes.CPtrDeclaratorNode(pos,
                base = base))
    else:
1653 1654 1655 1656 1657 1658 1659
        rhs = None
        if s.sy == 'IDENT':
            name = s.systring
            if is_type:
                s.add_type_name(name)
            if empty:
                error(s.position(), "Declarator should be empty")
William Stein's avatar
William Stein committed
1660
            s.next()
1661
            cname = p_opt_cname(s)
1662 1663 1664
            if s.sy == '=' and assignable:
                s.next()
                rhs = p_simple_expr(s)
William Stein's avatar
William Stein committed
1665
        else:
1666 1667 1668 1669 1670 1671 1672
            if nonempty:
                error(s.position(), "Empty declarator")
            name = ""
            cname = None
        result = Nodes.CNameDeclaratorNode(pos,
            name = name, cname = cname, rhs = rhs)
    result.calling_convention = calling_convention
William Stein's avatar
William Stein committed
1673 1674
    return result

1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689
def p_nogil(s):
    if s.sy == 'IDENT' and s.systring == 'nogil':
        s.next()
        return 1
    else:
        return 0

def p_with_gil(s):
    if s.sy == 'with':
        s.next()
        s.expect_keyword('gil')
        return 1
    else:
        return 0

William Stein's avatar
William Stein committed
1690 1691 1692 1693 1694 1695 1696 1697
def p_exception_value_clause(s):
    exc_val = None
    exc_check = 0
    if s.sy == 'except':
        s.next()
        if s.sy == '*':
            exc_check = 1
            s.next()
Felix Wu's avatar
Felix Wu committed
1698 1699 1700
        elif s.sy == '+':
            exc_check = '+'
            s.next()
Robert Bradshaw's avatar
Robert Bradshaw committed
1701 1702 1703 1704
            if s.sy == 'IDENT':
                name = s.systring
                s.next()
                exc_val = p_name(s, name)
William Stein's avatar
William Stein committed
1705 1706 1707 1708
        else:
            if s.sy == '?':
                exc_check = 1
                s.next()
1709
            exc_val = p_simple_expr(s)
William Stein's avatar
William Stein committed
1710 1711 1712 1713
    return exc_val, exc_check

c_arg_list_terminators = ('*', '**', '.', ')')

1714 1715
def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0,
                 nonempty_declarators = 0, kw_only = 0):
1716 1717
    #  Comma-separated list of C argument declarations, possibly empty.
    #  May have a trailing comma.
William Stein's avatar
William Stein committed
1718
    args = []
1719 1720
    is_self_arg = cmethod_flag
    while s.sy not in c_arg_list_terminators:
1721
        args.append(p_c_arg_decl(s, ctx, in_pyfunc, is_self_arg,
1722 1723 1724 1725 1726
            nonempty = nonempty_declarators, kw_only = kw_only))
        if s.sy != ',':
            break
        s.next()
        is_self_arg = 0
William Stein's avatar
William Stein committed
1727 1728 1729 1730 1731 1732 1733 1734 1735
    return args

def p_optional_ellipsis(s):
    if s.sy == '.':
        expect_ellipsis(s)
        return 1
    else:
        return 0

1736
def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, kw_only = 0):
William Stein's avatar
William Stein committed
1737 1738 1739
    pos = s.position()
    not_none = 0
    default = None
1740
    base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty)
1741
    declarator = p_c_declarator(s, ctx, nonempty = nonempty)
William Stein's avatar
William Stein committed
1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752
    if s.sy == 'not':
        s.next()
        if s.sy == 'IDENT' and s.systring == 'None':
            s.next()
        else:
            s.error("Expected 'None'")
        if not in_pyfunc:
            error(pos, "'not None' only allowed in Python functions")
        not_none = 1
    if s.sy == '=':
        s.next()
1753 1754 1755 1756 1757 1758 1759
        if 'pxd' in s.level:
            if s.sy not in ['*', '?']:
                error(pos, "default values cannot be specified in pxd files, use ? or *")
            default = 1
            s.next()
        else:
            default = p_simple_expr(s)
William Stein's avatar
William Stein committed
1760 1761 1762 1763
    return Nodes.CArgDeclNode(pos,
        base_type = base_type,
        declarator = declarator,
        not_none = not_none,
1764 1765
        default = default,
        kw_only = kw_only)
William Stein's avatar
William Stein committed
1766

1767 1768 1769 1770 1771 1772 1773
def p_api(s):
    if s.sy == 'IDENT' and s.systring == 'api':
        s.next()
        return 1
    else:
        return 0

1774
def p_cdef_statement(s, ctx):
William Stein's avatar
William Stein committed
1775
    pos = s.position()
1776 1777 1778 1779 1780 1781 1782
    ctx.visibility = p_visibility(s, ctx.visibility)
    ctx.api = ctx.api or p_api(s)
    if ctx.api:
        if ctx.visibility not in ('private', 'public'):
            error(pos, "Cannot combine 'api' with '%s'" % ctx.visibility)
    if (ctx.visibility == 'extern') and s.sy == 'from':
        return p_cdef_extern_block(s, pos, ctx)
Robert Bradshaw's avatar
Robert Bradshaw committed
1783 1784
    elif s.sy == 'import':
        s.next()
1785 1786 1787 1788 1789
        return p_cdef_extern_block(s, pos, ctx)
    if p_nogil(s):
        ctx.nogil = 1
    if s.sy == ':':
        return p_cdef_block(s, ctx)
William Stein's avatar
William Stein committed
1790
    elif s.sy == 'class':
1791
        if ctx.level not in ('module', 'module_pxd'):
William Stein's avatar
William Stein committed
1792
            error(pos, "Extension type definition not allowed here")
1793
        #if ctx.api:
Stefan Behnel's avatar
Stefan Behnel committed
1794
        #    error(pos, "'api' not allowed with extension class")
1795
        return p_c_class_definition(s, pos, ctx)
William Stein's avatar
William Stein committed
1796
    elif s.sy == 'IDENT' and s.systring in struct_union_or_enum:
1797
        if ctx.level not in ('module', 'module_pxd'):
William Stein's avatar
William Stein committed
1798
            error(pos, "C struct/union/enum definition not allowed here")
1799
        #if ctx.visibility == 'public':
1800
        #    error(pos, "Public struct/union/enum definition not implemented")
1801
        #if ctx.api:
Stefan Behnel's avatar
Stefan Behnel committed
1802
        #    error(pos, "'api' not allowed with '%s'" % s.systring)
William Stein's avatar
William Stein committed
1803
        if s.systring == "enum":
1804
            return p_c_enum_definition(s, pos, ctx)
William Stein's avatar
William Stein committed
1805
        else:
1806
            return p_c_struct_or_union_definition(s, pos, ctx)
William Stein's avatar
William Stein committed
1807 1808 1809 1810 1811
    elif s.sy == 'pass':
        node = p_pass_statement(s)
        s.expect_newline('Expected a newline')
        return node
    else:
1812
        return p_c_func_or_var_declaration(s, pos, ctx)
1813

1814 1815
def p_cdef_block(s, ctx):
    return p_suite(s, ctx(cdef_flag = 1))
William Stein's avatar
William Stein committed
1816

1817
def p_cdef_extern_block(s, pos, ctx):
William Stein's avatar
William Stein committed
1818 1819 1820 1821 1822 1823
    include_file = None
    s.expect('from')
    if s.sy == '*':
        s.next()
    else:
        _, include_file = p_string_literal(s)
1824 1825 1826 1827
    ctx = ctx(cdef_flag = 1, visibility = 'extern')
    if p_nogil(s):
        ctx.nogil = 1
    body = p_suite(s, ctx)
William Stein's avatar
William Stein committed
1828 1829 1830 1831 1832 1833 1834 1835
    return Nodes.CDefExternNode(pos,
        include_file = include_file,
        body = body)

struct_union_or_enum = (
    "struct", "union", "enum"
)

1836
def p_c_enum_definition(s, pos, ctx):
William Stein's avatar
William Stein committed
1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849
    # s.sy == ident 'enum'
    s.next()
    if s.sy == 'IDENT':
        name = s.systring
        s.next()
        s.add_type_name(name)
        cname = p_opt_cname(s)
    else:
        name = None
        cname = None
    items = None
    s.expect(':')
    items = []
Stefan Behnel's avatar
Stefan Behnel committed
1850
    if s.sy != 'NEWLINE':
William Stein's avatar
William Stein committed
1851 1852 1853 1854 1855 1856 1857
        p_c_enum_line(s, items)
    else:
        s.next() # 'NEWLINE'
        s.expect_indent()
        while s.sy not in ('DEDENT', 'EOF'):
            p_c_enum_line(s, items)
        s.expect_dedent()
1858 1859 1860 1861
    return Nodes.CEnumDefNode(
        pos, name = name, cname = cname, items = items,
        typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,
        in_pxd = ctx.level == 'module_pxd')
William Stein's avatar
William Stein committed
1862 1863

def p_c_enum_line(s, items):
Stefan Behnel's avatar
Stefan Behnel committed
1864
    if s.sy != 'pass':
William Stein's avatar
William Stein committed
1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885
        p_c_enum_item(s, items)
        while s.sy == ',':
            s.next()
            if s.sy in ('NEWLINE', 'EOF'):
                break
            p_c_enum_item(s, items)
    else:
        s.next()
    s.expect_newline("Syntax error in enum item list")

def p_c_enum_item(s, items):
    pos = s.position()
    name = p_ident(s)
    cname = p_opt_cname(s)
    value = None
    if s.sy == '=':
        s.next()
        value = p_simple_expr(s)
    items.append(Nodes.CEnumDefItemNode(pos, 
        name = name, cname = cname, value = value))

1886
def p_c_struct_or_union_definition(s, pos, ctx):
William Stein's avatar
William Stein committed
1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898
    # s.sy == ident 'struct' or 'union'
    kind = s.systring
    s.next()
    name = p_ident(s)
    cname = p_opt_cname(s)
    s.add_type_name(name)
    attributes = None
    if s.sy == ':':
        s.next()
        s.expect('NEWLINE')
        s.expect_indent()
        attributes = []
1899
        body_ctx = Ctx()
Stefan Behnel's avatar
Stefan Behnel committed
1900 1901
        while s.sy != 'DEDENT':
            if s.sy != 'pass':
William Stein's avatar
William Stein committed
1902
                attributes.append(
1903
                    p_c_func_or_var_declaration(s, s.position(), body_ctx))
William Stein's avatar
William Stein committed
1904 1905 1906 1907 1908 1909 1910 1911
            else:
                s.next()
                s.expect_newline("Expected a newline")
        s.expect_dedent()
    else:
        s.expect_newline("Syntax error in struct or union definition")
    return Nodes.CStructOrUnionDefNode(pos, 
        name = name, cname = cname, kind = kind, attributes = attributes,
1912 1913
        typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,
        in_pxd = ctx.level == 'module_pxd')
William Stein's avatar
William Stein committed
1914 1915 1916 1917 1918 1919

def p_visibility(s, prev_visibility):
    pos = s.position()
    visibility = prev_visibility
    if s.sy == 'IDENT' and s.systring in ('extern', 'public', 'readonly'):
        visibility = s.systring
Stefan Behnel's avatar
Stefan Behnel committed
1920
        if prev_visibility != 'private' and visibility != prev_visibility:
William Stein's avatar
William Stein committed
1921 1922 1923 1924
            s.error("Conflicting visibility options '%s' and '%s'"
                % (prev_visibility, visibility))
        s.next()
    return visibility
1925 1926
    
def p_c_modifiers(s):
1927
    if s.sy == 'IDENT' and s.systring in ('inline',):
1928
        modifier = s.systring
1929
        s.next()
1930 1931
        return [modifier] + p_c_modifiers(s)
    return []
William Stein's avatar
William Stein committed
1932

1933 1934
def p_c_func_or_var_declaration(s, pos, ctx):
    cmethod_flag = ctx.level in ('c_class', 'c_class_pxd')
1935
    modifiers = p_c_modifiers(s)
1936
    base_type = p_c_base_type(s, nonempty = 1)
1937 1938 1939
    declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,
                                assignable = 1, nonempty = 1)
    declarator.overridable = ctx.overridable
William Stein's avatar
William Stein committed
1940
    if s.sy == ':':
1941
        if ctx.level not in ('module', 'c_class'):
William Stein's avatar
William Stein committed
1942
            s.error("C function definition not allowed here")
1943
        doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1)
William Stein's avatar
William Stein committed
1944
        result = Nodes.CFuncDefNode(pos,
1945
            visibility = ctx.visibility,
William Stein's avatar
William Stein committed
1946 1947
            base_type = base_type,
            declarator = declarator, 
1948
            body = suite,
1949
            doc = doc,
1950
            modifiers = modifiers,
1951 1952
            api = ctx.api,
            overridable = ctx.overridable)
William Stein's avatar
William Stein committed
1953
    else:
Stefan Behnel's avatar
Stefan Behnel committed
1954 1955
        #if api:
        #    error(s.pos, "'api' not allowed with variable declaration")
William Stein's avatar
William Stein committed
1956 1957 1958 1959 1960
        declarators = [declarator]
        while s.sy == ',':
            s.next()
            if s.sy == 'NEWLINE':
                break
1961 1962
            declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,
                                        assignable = 1, nonempty = 1)
William Stein's avatar
William Stein committed
1963 1964 1965
            declarators.append(declarator)
        s.expect_newline("Syntax error in C variable declaration")
        result = Nodes.CVarDefNode(pos, 
1966 1967
            visibility = ctx.visibility,
            base_type = base_type,
1968
            declarators = declarators,
1969 1970 1971
            in_pxd = ctx.level == 'module_pxd',
            api = ctx.api,
            overridable = ctx.overridable)
William Stein's avatar
William Stein committed
1972 1973
    return result

1974
def p_ctypedef_statement(s, ctx):
William Stein's avatar
William Stein committed
1975 1976 1977
    # s.sy == 'ctypedef'
    pos = s.position()
    s.next()
1978 1979
    visibility = p_visibility(s, ctx.visibility)
    ctx = ctx(typedef_flag = 1, visibility = visibility)
William Stein's avatar
William Stein committed
1980
    if s.sy == 'class':
1981
        return p_c_class_definition(s, pos, ctx)
William Stein's avatar
William Stein committed
1982 1983
    elif s.sy == 'IDENT' and s.systring in ('struct', 'union', 'enum'):
        if s.systring == 'enum':
1984
            return p_c_enum_definition(s, pos, ctx)
William Stein's avatar
William Stein committed
1985
        else:
1986
            return p_c_struct_or_union_definition(s, pos, ctx)
William Stein's avatar
William Stein committed
1987
    else:
1988
        base_type = p_c_base_type(s, nonempty = 1)
1989
        declarator = p_c_declarator(s, ctx, is_type = 1, nonempty = 1)
William Stein's avatar
William Stein committed
1990
        s.expect_newline("Syntax error in ctypedef statement")
1991 1992 1993 1994
        return Nodes.CTypeDefNode(
            pos, base_type = base_type,
            declarator = declarator, visibility = visibility,
            in_pxd = ctx.level == 'module_pxd')
William Stein's avatar
William Stein committed
1995 1996 1997 1998 1999 2000

def p_def_statement(s):
    # s.sy == 'def'
    pos = s.position()
    s.next()
    name = p_ident(s)
2001
    #args = []
William Stein's avatar
William Stein committed
2002
    s.expect('(');
2003
    args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1)
William Stein's avatar
William Stein committed
2004 2005 2006 2007
    star_arg = None
    starstar_arg = None
    if s.sy == '*':
        s.next()
2008 2009
        if s.sy == 'IDENT':
            star_arg = p_py_arg_decl(s)
William Stein's avatar
William Stein committed
2010 2011
        if s.sy == ',':
            s.next()
2012 2013 2014 2015 2016
            args.extend(p_c_arg_list(s, in_pyfunc = 1,
                nonempty_declarators = 1, kw_only = 1))
        elif s.sy != ')':
            s.error("Syntax error in Python function argument list")
    if s.sy == '**':
William Stein's avatar
William Stein committed
2017 2018 2019
        s.next()
        starstar_arg = p_py_arg_decl(s)
    s.expect(')')
2020 2021
    if p_nogil(s):
        error(s.pos, "Python function cannot be declared nogil")
2022
    doc, body = p_suite(s, Ctx(level = 'function'), with_doc = 1)
William Stein's avatar
William Stein committed
2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035
    return Nodes.DefNode(pos, name = name, args = args, 
        star_arg = star_arg, starstar_arg = starstar_arg,
        doc = doc, body = body)

def p_py_arg_decl(s):
    pos = s.position()
    name = p_ident(s)
    return Nodes.PyArgDeclNode(pos, name = name)

def p_class_statement(s):
    # s.sy == 'class'
    pos = s.position()
    s.next()
2036
    class_name = Utils.EncodedString( p_ident(s) )
2037
    class_name.encoding = s.source_encoding
William Stein's avatar
William Stein committed
2038 2039 2040 2041 2042 2043
    if s.sy == '(':
        s.next()
        base_list = p_simple_expr_list(s)
        s.expect(')')
    else:
        base_list = []
2044
    doc, body = p_suite(s, Ctx(level = 'class'), with_doc = 1)
William Stein's avatar
William Stein committed
2045 2046 2047 2048 2049
    return Nodes.PyClassDefNode(pos,
        name = class_name,
        bases = ExprNodes.TupleNode(pos, args = base_list),
        doc = doc, body = body)

2050
def p_c_class_definition(s, pos,  ctx):
William Stein's avatar
William Stein committed
2051 2052 2053 2054 2055 2056 2057 2058
    # s.sy == 'class'
    s.next()
    module_path = []
    class_name = p_ident(s)
    while s.sy == '.':
        s.next()
        module_path.append(class_name)
        class_name = p_ident(s)
2059
    if module_path and ctx.visibility != 'extern':
William Stein's avatar
William Stein committed
2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082
        error(pos, "Qualified class name only allowed for 'extern' C class")
    if module_path and s.sy == 'IDENT' and s.systring == 'as':
        s.next()
        as_name = p_ident(s)
    else:
        as_name = class_name
    s.add_type_name(as_name)
    objstruct_name = None
    typeobj_name = None
    base_class_module = None
    base_class_name = None
    if s.sy == '(':
        s.next()
        base_class_path = [p_ident(s)]
        while s.sy == '.':
            s.next()
            base_class_path.append(p_ident(s))
        if s.sy == ',':
            s.error("C class may only have one base class")
        s.expect(')')
        base_class_module = ".".join(base_class_path[:-1])
        base_class_name = base_class_path[-1]
    if s.sy == '[':
2083
        if ctx.visibility not in ('public', 'extern'):
William Stein's avatar
William Stein committed
2084 2085 2086
            error(s.position(), "Name options only allowed for 'public' or 'extern' C class")
        objstruct_name, typeobj_name = p_c_class_options(s)
    if s.sy == ':':
2087
        if ctx.level == 'module_pxd':
William Stein's avatar
William Stein committed
2088 2089 2090
            body_level = 'c_class_pxd'
        else:
            body_level = 'c_class'
2091
        doc, body = p_suite(s, Ctx(level = body_level), with_doc = 1)
William Stein's avatar
William Stein committed
2092 2093 2094 2095
    else:
        s.expect_newline("Syntax error in C class definition")
        doc = None
        body = None
2096
    if ctx.visibility == 'extern':
William Stein's avatar
William Stein committed
2097 2098 2099 2100
        if not module_path:
            error(pos, "Module name required for 'extern' C class")
        if typeobj_name:
            error(pos, "Type object name specification not allowed for 'extern' C class")
2101
    elif ctx.visibility == 'public':
William Stein's avatar
William Stein committed
2102 2103 2104 2105
        if not objstruct_name:
            error(pos, "Object struct name specification required for 'public' C class")
        if not typeobj_name:
            error(pos, "Type object name specification required for 'public' C class")
2106 2107
    elif ctx.visibility == 'private':
        if ctx.api:
Stefan Behnel's avatar
Stefan Behnel committed
2108
            error(pos, "Only 'public' C class can be declared 'api'")
2109
    else:
2110
        error(pos, "Invalid class visibility '%s'" % ctx.visibility)
William Stein's avatar
William Stein committed
2111
    return Nodes.CClassDefNode(pos,
2112 2113 2114
        visibility = ctx.visibility,
        typedef_flag = ctx.typedef_flag,
        api = ctx.api,
William Stein's avatar
William Stein committed
2115 2116 2117 2118 2119 2120 2121
        module_name = ".".join(module_path),
        class_name = class_name,
        as_name = as_name,
        base_class_module = base_class_module,
        base_class_name = base_class_name,
        objstruct_name = objstruct_name,
        typeobj_name = typeobj_name,
2122
        in_pxd = ctx.level == 'module_pxd',
William Stein's avatar
William Stein committed
2123 2124 2125 2126 2127 2128 2129 2130
        doc = doc,
        body = body)

def p_c_class_options(s):
    objstruct_name = None
    typeobj_name = None
    s.expect('[')
    while 1:
Stefan Behnel's avatar
Stefan Behnel committed
2131
        if s.sy != 'IDENT':
William Stein's avatar
William Stein committed
2132 2133 2134 2135 2136 2137 2138
            break
        if s.systring == 'object':
            s.next()
            objstruct_name = p_ident(s)
        elif s.systring == 'type':
            s.next()
            typeobj_name = p_ident(s)
Stefan Behnel's avatar
Stefan Behnel committed
2139
        if s.sy != ',':
William Stein's avatar
William Stein committed
2140 2141 2142 2143 2144 2145 2146 2147 2148
            break
        s.next()
    s.expect(']', "Expected 'object' or 'type'")
    return objstruct_name, typeobj_name

def p_property_decl(s):
    pos = s.position()
    s.next() # 'property'
    name = p_ident(s)
2149
    doc, body = p_suite(s, Ctx(level = 'property'), with_doc = 1)
William Stein's avatar
William Stein committed
2150 2151 2152
    return Nodes.PropertyNode(pos, name = name, doc = doc, body = body)

def p_doc_string(s):
2153
    if s.sy == 'BEGIN_STRING':
William Stein's avatar
William Stein committed
2154
        _, result = p_cat_string_literal(s)
Stefan Behnel's avatar
Stefan Behnel committed
2155
        if s.sy != 'EOF':
William Stein's avatar
William Stein committed
2156 2157 2158 2159 2160
            s.expect_newline("Syntax error in doc string")
        return result
    else:
        return None

2161
def p_module(s, pxd, full_module_name):
William Stein's avatar
William Stein committed
2162
    s.add_type_name("object")
2163
    s.add_type_name("Py_buffer")
William Stein's avatar
William Stein committed
2164 2165 2166 2167 2168 2169
    pos = s.position()
    doc = p_doc_string(s)
    if pxd:
        level = 'module_pxd'
    else:
        level = 'module'
2170
    body = p_statement_list(s, Ctx(level = level), first_statement = 1)
Stefan Behnel's avatar
Stefan Behnel committed
2171
    if s.sy != 'EOF':
William Stein's avatar
William Stein committed
2172 2173
        s.error("Syntax error in statement [%s,%s]" % (
            repr(s.sy), repr(s.systring)))
Robert Bradshaw's avatar
Robert Bradshaw committed
2174
    return ModuleNode(pos, doc = doc, body = body, full_module_name = full_module_name)
William Stein's avatar
William Stein committed
2175 2176 2177 2178 2179 2180 2181

#----------------------------------------------
#
#   Debugging
#
#----------------------------------------------

Stefan Behnel's avatar
Stefan Behnel committed
2182
def print_parse_tree(f, node, level, key = None):
2183
    from Nodes import Node
William Stein's avatar
William Stein committed
2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202
    ind = "  " * level
    if node:
        f.write(ind)
        if key:
            f.write("%s: " % key)
        t = type(node)
        if t == TupleType:
            f.write("(%s @ %s\n" % (node[0], node[1]))
            for i in xrange(2, len(node)):
                print_parse_tree(f, node[i], level+1)
            f.write("%s)\n" % ind)
            return
        elif isinstance(node, Node):
            try:
                tag = node.tag
            except AttributeError:
                tag = node.__class__.__name__
            f.write("%s @ %s\n" % (tag, node.pos))
            for name, value in node.__dict__.items():
Stefan Behnel's avatar
Stefan Behnel committed
2203
                if name != 'tag' and name != 'pos':
William Stein's avatar
William Stein committed
2204 2205 2206 2207 2208 2209 2210 2211 2212 2213
                    print_parse_tree(f, value, level+1, name)
            return
        elif t == ListType:
            f.write("[\n")
            for i in xrange(len(node)):
                print_parse_tree(f, node[i], level+1)
            f.write("%s]\n" % ind)
            return
    f.write("%s%s\n" % (ind, node))