Parsing.py 131 KB
Newer Older
1
# cython: auto_cpdef=True, infer_types=True, language_level=3, py2_import=True
William Stein's avatar
William Stein committed
2
#
3
#   Parser
William Stein's avatar
William Stein committed
4 5
#

6 7
from __future__ import absolute_import

8 9
# This should be done automatically
import cython
10
cython.declare(Nodes=object, ExprNodes=object, EncodedString=object,
11
               bytes_literal=object, StringEncoding=object,
12
               FileSourceDescriptor=object, lookup_unicodechar=object, unicode_category=object,
13
               Future=object, Options=object, error=object, warning=object,
14 15
               Builtin=object, ModuleNode=object, Utils=object, _unicode=object, _bytes=object,
               re=object, sys=object, _parse_escape_sequences=object, _parse_escape_sequences_raw=object,
16 17
               partial=object, reduce=object, _IS_PY3=cython.bint, _IS_2BYTE_UNICODE=cython.bint,
               _CDEF_MODIFIERS=tuple)
18

19
from io import StringIO
20
import re
21
import sys
22
from unicodedata import lookup as lookup_unicodechar, category as unicode_category
23
from functools import partial, reduce
Lisandro Dalcin's avatar
Lisandro Dalcin committed
24

25
from .Scanning import PyrexScanner, FileSourceDescriptor, StringSourceDescriptor
26 27 28 29
from . import Nodes
from . import ExprNodes
from . import Builtin
from . import StringEncoding
30
from .StringEncoding import EncodedString, bytes_literal, _unicode, _bytes
31 32 33 34 35 36
from .ModuleNode import ModuleNode
from .Errors import error, warning
from .. import Utils
from . import Future
from . import Options

37
_IS_PY3 = sys.version_info[0] >= 3
38
_IS_2BYTE_UNICODE = sys.maxunicode == 0xffff
39
_CDEF_MODIFIERS = ('inline', 'nogil', 'api')
40

William Stein's avatar
William Stein committed
41

42 43 44 45 46 47 48 49 50
class Ctx(object):
    #  Parsing context
    level = 'other'
    visibility = 'private'
    cdef_flag = 0
    typedef_flag = 0
    api = 0
    overridable = 0
    nogil = 0
51
    namespace = None
Danilo Freitas's avatar
Danilo Freitas committed
52
    templates = None
53
    allow_struct_enum_decorator = False
54
    modifiers = []
55 56 57 58 59 60 61 62 63 64 65

    def __init__(self, **kwds):
        self.__dict__.update(kwds)

    def __call__(self, **kwds):
        ctx = Ctx()
        d = ctx.__dict__
        d.update(self.__dict__)
        d.update(kwds)
        return ctx

66

67
def p_ident(s, message="Expected an identifier"):
William Stein's avatar
William Stein committed
68
    if s.sy == 'IDENT':
69
        name = s.context.intern_ustring(s.systring)
William Stein's avatar
William Stein committed
70 71 72 73 74 75 76 77
        s.next()
        return name
    else:
        s.error(message)

def p_ident_list(s):
    names = []
    while s.sy == 'IDENT':
78
        names.append(s.context.intern_ustring(s.systring))
William Stein's avatar
William Stein committed
79
        s.next()
Stefan Behnel's avatar
Stefan Behnel committed
80
        if s.sy != ',':
William Stein's avatar
William Stein committed
81 82 83 84 85 86 87 88 89 90
            break
        s.next()
    return names

#------------------------------------------
#
#   Expressions
#
#------------------------------------------

91 92 93 94 95 96
def p_binop_operator(s):
    pos = s.position()
    op = s.sy
    s.next()
    return op, pos

William Stein's avatar
William Stein committed
97 98 99
def p_binop_expr(s, ops, p_sub_expr):
    n1 = p_sub_expr(s)
    while s.sy in ops:
100
        op, pos = p_binop_operator(s)
William Stein's avatar
William Stein committed
101 102
        n2 = p_sub_expr(s)
        n1 = ExprNodes.binop_node(pos, op, n1, n2)
103 104 105 106
        if op == '/':
            if Future.division in s.context.future_directives:
                n1.truedivision = True
            else:
107
                n1.truedivision = None  # unknown
William Stein's avatar
William Stein committed
108 109
    return n1

Stefan Behnel's avatar
Stefan Behnel committed
110 111 112 113 114 115 116 117 118 119
#lambdef: 'lambda' [varargslist] ':' test

def p_lambdef(s, allow_conditional=True):
    # s.sy == 'lambda'
    pos = s.position()
    s.next()
    if s.sy == ':':
        args = []
        star_arg = starstar_arg = None
    else:
120 121
        args, star_arg, starstar_arg = p_varargslist(
            s, terminator=':', annotated=False)
Stefan Behnel's avatar
Stefan Behnel committed
122 123
    s.expect(':')
    if allow_conditional:
124
        expr = p_test(s)
Stefan Behnel's avatar
Stefan Behnel committed
125 126 127 128 129 130 131 132 133 134 135 136
    else:
        expr = p_test_nocond(s)
    return ExprNodes.LambdaNode(
        pos, args = args,
        star_arg = star_arg, starstar_arg = starstar_arg,
        result_expr = expr)

#lambdef_nocond: 'lambda' [varargslist] ':' test_nocond

def p_lambdef_nocond(s):
    return p_lambdef(s, allow_conditional=False)

137
#test: or_test ['if' or_test 'else' test] | lambdef
William Stein's avatar
William Stein committed
138

Robert Bradshaw's avatar
Robert Bradshaw committed
139
def p_test(s):
140 141
    if s.sy == 'lambda':
        return p_lambdef(s)
Robert Bradshaw's avatar
Robert Bradshaw committed
142 143 144 145 146
    pos = s.position()
    expr = p_or_test(s)
    if s.sy == 'if':
        s.next()
        test = p_or_test(s)
Stefan Behnel's avatar
Stefan Behnel committed
147 148 149
        s.expect('else')
        other = p_test(s)
        return ExprNodes.CondExprNode(pos, test=test, true_val=expr, false_val=other)
Robert Bradshaw's avatar
Robert Bradshaw committed
150 151 152
    else:
        return expr

Stefan Behnel's avatar
Stefan Behnel committed
153 154 155 156 157 158 159
#test_nocond: or_test | lambdef_nocond

def p_test_nocond(s):
    if s.sy == 'lambda':
        return p_lambdef_nocond(s)
    else:
        return p_or_test(s)
Robert Bradshaw's avatar
Robert Bradshaw committed
160 161 162 163

#or_test: and_test ('or' and_test)*

def p_or_test(s):
William Stein's avatar
William Stein committed
164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195
    return p_rassoc_binop_expr(s, ('or',), p_and_test)

def p_rassoc_binop_expr(s, ops, p_subexpr):
    n1 = p_subexpr(s)
    if s.sy in ops:
        pos = s.position()
        op = s.sy
        s.next()
        n2 = p_rassoc_binop_expr(s, ops, p_subexpr)
        n1 = ExprNodes.binop_node(pos, op, n1, n2)
    return n1

#and_test: not_test ('and' not_test)*

def p_and_test(s):
    #return p_binop_expr(s, ('and',), p_not_test)
    return p_rassoc_binop_expr(s, ('and',), p_not_test)

#not_test: 'not' not_test | comparison

def p_not_test(s):
    if s.sy == 'not':
        pos = s.position()
        s.next()
        return ExprNodes.NotNode(pos, operand = p_not_test(s))
    else:
        return p_comparison(s)

#comparison: expr (comp_op expr)*
#comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'

def p_comparison(s):
196
    n1 = p_starred_expr(s)
William Stein's avatar
William Stein committed
197 198 199
    if s.sy in comparison_ops:
        pos = s.position()
        op = p_cmp_op(s)
200
        n2 = p_starred_expr(s)
201
        n1 = ExprNodes.PrimaryCmpNode(pos,
William Stein's avatar
William Stein committed
202 203 204 205 206
            operator = op, operand1 = n1, operand2 = n2)
        if s.sy in comparison_ops:
            n1.cascade = p_cascaded_cmp(s)
    return n1

207 208 209 210 211 212
def p_test_or_starred_expr(s):
    if s.sy == '*':
        return p_starred_expr(s)
    else:
        return p_test(s)

213
def p_starred_expr(s):
214
    pos = s.position()
215 216 217 218 219 220
    if s.sy == '*':
        starred = True
        s.next()
    else:
        starred = False
    expr = p_bit_expr(s)
221
    if starred:
222
        expr = ExprNodes.StarredUnpackingNode(pos, expr)
223 224
    return expr

William Stein's avatar
William Stein committed
225 226 227
def p_cascaded_cmp(s):
    pos = s.position()
    op = p_cmp_op(s)
228
    n2 = p_starred_expr(s)
229
    result = ExprNodes.CascadedCmpNode(pos,
William Stein's avatar
William Stein committed
230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252
        operator = op, operand2 = n2)
    if s.sy in comparison_ops:
        result.cascade = p_cascaded_cmp(s)
    return result

def p_cmp_op(s):
    if s.sy == 'not':
        s.next()
        s.expect('in')
        op = 'not_in'
    elif s.sy == 'is':
        s.next()
        if s.sy == 'not':
            s.next()
            op = 'is_not'
        else:
            op = 'is'
    else:
        op = s.sy
        s.next()
    if op == '<>':
        op = '!='
    return op
253

254
comparison_ops = cython.declare(set, set([
255
    '<', '>', '==', '>=', '<=', '<>', '!=',
William Stein's avatar
William Stein committed
256
    'in', 'is', 'not'
257
]))
William Stein's avatar
William Stein committed
258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283

#expr: xor_expr ('|' xor_expr)*

def p_bit_expr(s):
    return p_binop_expr(s, ('|',), p_xor_expr)

#xor_expr: and_expr ('^' and_expr)*

def p_xor_expr(s):
    return p_binop_expr(s, ('^',), p_and_expr)

#and_expr: shift_expr ('&' shift_expr)*

def p_and_expr(s):
    return p_binop_expr(s, ('&',), p_shift_expr)

#shift_expr: arith_expr (('<<'|'>>') arith_expr)*

def p_shift_expr(s):
    return p_binop_expr(s, ('<<', '>>'), p_arith_expr)

#arith_expr: term (('+'|'-') term)*

def p_arith_expr(s):
    return p_binop_expr(s, ('+', '-'), p_term)

284
#term: factor (('*'|'@'|'/'|'%'|'//') factor)*
William Stein's avatar
William Stein committed
285 286

def p_term(s):
287
    return p_binop_expr(s, ('*', '@', '/', '%', '//'), p_factor)
William Stein's avatar
William Stein committed
288 289 290 291

#factor: ('+'|'-'|'~'|'&'|typecast|sizeof) factor | power

def p_factor(s):
292 293 294 295
    # little indirection for C-ification purposes
    return _p_factor(s)

def _p_factor(s):
William Stein's avatar
William Stein committed
296 297 298 299 300 301
    sy = s.sy
    if sy in ('+', '-', '~'):
        op = s.sy
        pos = s.position()
        s.next()
        return ExprNodes.unop_node(pos, op, p_factor(s))
302 303 304 305 306 307 308 309 310 311 312
    elif not s.in_python_file:
        if sy == '&':
            pos = s.position()
            s.next()
            arg = p_factor(s)
            return ExprNodes.AmpersandNode(pos, operand = arg)
        elif sy == "<":
            return p_typecast(s)
        elif sy == 'IDENT' and s.systring == "sizeof":
            return p_sizeof(s)
    return p_power(s)
William Stein's avatar
William Stein committed
313 314 315 316 317 318

def p_typecast(s):
    # s.sy == "<"
    pos = s.position()
    s.next()
    base_type = p_c_base_type(s)
319
    is_memslice = isinstance(base_type, Nodes.MemoryViewSliceTypeNode)
Robert Bradshaw's avatar
Robert Bradshaw committed
320
    is_template = isinstance(base_type, Nodes.TemplatedTypeNode)
321
    is_const_volatile = isinstance(base_type, Nodes.CConstOrVolatileTypeNode)
Stefan Behnel's avatar
Stefan Behnel committed
322
    if not is_memslice and not is_template and not is_const_volatile and base_type.name is None:
323
        s.error("Unknown type")
William Stein's avatar
William Stein committed
324
    declarator = p_c_declarator(s, empty = 1)
325 326 327 328 329
    if s.sy == '?':
        s.next()
        typecheck = 1
    else:
        typecheck = 0
William Stein's avatar
William Stein committed
330 331
    s.expect(">")
    operand = p_factor(s)
332
    if is_memslice:
Stefan Behnel's avatar
Stefan Behnel committed
333
        return ExprNodes.CythonArrayNode(pos, base_type_node=base_type, operand=operand)
334

335 336
    return ExprNodes.TypecastNode(pos,
        base_type = base_type,
William Stein's avatar
William Stein committed
337
        declarator = declarator,
338 339
        operand = operand,
        typecheck = typecheck)
William Stein's avatar
William Stein committed
340 341 342 343 344 345

def p_sizeof(s):
    # s.sy == ident "sizeof"
    pos = s.position()
    s.next()
    s.expect('(')
346
    # Here we decide if we are looking at an expression or type
347 348
    # If it is actually a type, but parsable as an expression,
    # we treat it as an expression here.
349
    if looking_at_expr(s):
350
        operand = p_test(s)
351 352
        node = ExprNodes.SizeofVarNode(pos, operand = operand)
    else:
William Stein's avatar
William Stein committed
353 354
        base_type = p_c_base_type(s)
        declarator = p_c_declarator(s, empty = 1)
355
        node = ExprNodes.SizeofTypeNode(pos,
William Stein's avatar
William Stein committed
356 357 358 359
            base_type = base_type, declarator = declarator)
    s.expect(')')
    return node

360

361 362 363 364
def p_yield_expression(s):
    # s.sy == "yield"
    pos = s.position()
    s.next()
365 366 367 368
    is_yield_from = False
    if s.sy == 'from':
        is_yield_from = True
        s.next()
369
    if s.sy != ')' and s.sy not in statement_terminators:
370 371
        # "yield from" does not support implicit tuples, but "yield" does ("yield 1,2")
        arg = p_test(s) if is_yield_from else p_testlist(s)
372
    else:
373
        if is_yield_from:
374 375
            s.error("'yield from' requires a source argument",
                    pos=pos, fatal=False)
376
        arg = None
377 378 379 380
    if is_yield_from:
        return ExprNodes.YieldFromExprNode(pos, arg=arg)
    else:
        return ExprNodes.YieldExprNode(pos, arg=arg)
381

382

383 384 385 386
def p_yield_statement(s):
    # s.sy == "yield"
    yield_expr = p_yield_expression(s)
    return Nodes.ExprStatNode(yield_expr.pos, expr=yield_expr)
387

388 389 390 391 392 393 394 395 396 397 398 399

def p_async_statement(s, ctx, decorators):
    # s.sy >> 'async' ...
    if s.sy == 'def':
        # 'async def' statements aren't allowed in pxd files
        if 'pxd' in ctx.level:
            s.error('def statement not allowed here')
        s.level = ctx.level
        return p_def_statement(s, decorators, is_async_def=True)
    elif decorators:
        s.error("Decorators can only be followed by functions or classes")
    elif s.sy == 'for':
400
        return p_for_statement(s, is_async=True)
401
    elif s.sy == 'with':
402 403
        s.next()
        return p_with_items(s, is_async=True)
404 405 406 407 408 409
    else:
        s.error("expected one of 'def', 'for', 'with' after 'async'")


#power: atom_expr ('**' factor)*
#atom_expr: ['await'] atom trailer*
William Stein's avatar
William Stein committed
410 411

def p_power(s):
412
    if s.systring == 'new' and s.peek()[0] == 'IDENT':
Danilo Freitas's avatar
Danilo Freitas committed
413
        return p_new_expr(s)
414 415 416 417
    await_pos = None
    if s.sy == 'await':
        await_pos = s.position()
        s.next()
William Stein's avatar
William Stein committed
418 419 420
    n1 = p_atom(s)
    while s.sy in ('(', '[', '.'):
        n1 = p_trailer(s, n1)
421 422
    if await_pos:
        n1 = ExprNodes.AwaitExprNode(await_pos, arg=n1)
William Stein's avatar
William Stein committed
423 424 425 426 427 428 429
    if s.sy == '**':
        pos = s.position()
        s.next()
        n2 = p_factor(s)
        n1 = ExprNodes.binop_node(pos, '**', n1, n2)
    return n1

430

Danilo Freitas's avatar
Danilo Freitas committed
431
def p_new_expr(s):
Danilo Freitas's avatar
Danilo Freitas committed
432
    # s.systring == 'new'.
Danilo Freitas's avatar
Danilo Freitas committed
433 434
    pos = s.position()
    s.next()
Robert Bradshaw's avatar
Robert Bradshaw committed
435 436
    cppclass = p_c_base_type(s)
    return p_call(s, ExprNodes.NewExprNode(pos, cppclass = cppclass))
Danilo Freitas's avatar
Danilo Freitas committed
437

William Stein's avatar
William Stein committed
438 439 440 441 442 443 444 445
#trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME

def p_trailer(s, node1):
    pos = s.position()
    if s.sy == '(':
        return p_call(s, node1)
    elif s.sy == '[':
        return p_index(s, node1)
446
    else:  # s.sy == '.'
William Stein's avatar
William Stein committed
447
        s.next()
448
        name = p_ident(s)
449
        return ExprNodes.AttributeNode(pos,
450
            obj=node1, attribute=name)
William Stein's avatar
William Stein committed
451

452

William Stein's avatar
William Stein committed
453 454 455
# arglist:  argument (',' argument)* [',']
# argument: [test '='] test       # Really [keyword '='] test

456 457 458 459 460 461 462
# since PEP 448:
# argument: ( test [comp_for] |
#             test '=' test |
#             '**' expr |
#             star_expr )

def p_call_parse_args(s, allow_genexp=True):
William Stein's avatar
William Stein committed
463 464 465 466 467
    # s.sy == '('
    pos = s.position()
    s.next()
    positional_args = []
    keyword_args = []
468 469 470
    starstar_seen = False
    last_was_tuple_unpack = False
    while s.sy != ')':
471
        if s.sy == '*':
472 473 474 475 476 477
            if starstar_seen:
                s.error("Non-keyword arg following keyword arg", pos=s.position())
            s.next()
            positional_args.append(p_test(s))
            last_was_tuple_unpack = True
        elif s.sy == '**':
William Stein's avatar
William Stein committed
478
            s.next()
479 480
            keyword_args.append(p_test(s))
            starstar_seen = True
William Stein's avatar
William Stein committed
481
        else:
482
            arg = p_test(s)
483 484 485 486
            if s.sy == '=':
                s.next()
                if not arg.is_name:
                    s.error("Expected an identifier before '='",
487
                            pos=arg.pos)
488
                encoded_name = s.context.intern_ustring(arg.name)
489 490
                keyword = ExprNodes.IdentifierStringNode(
                    arg.pos, value=encoded_name)
491
                arg = p_test(s)
492 493 494
                keyword_args.append((keyword, arg))
            else:
                if keyword_args:
495 496 497 498 499 500
                    s.error("Non-keyword arg following keyword arg", pos=arg.pos)
                if positional_args and not last_was_tuple_unpack:
                    positional_args[-1].append(arg)
                else:
                    positional_args.append([arg])
                last_was_tuple_unpack = False
Stefan Behnel's avatar
Stefan Behnel committed
501
        if s.sy != ',':
William Stein's avatar
William Stein committed
502 503
            break
        s.next()
504

505
    if s.sy in ('for', 'async'):
506 507
        if not keyword_args and not last_was_tuple_unpack:
            if len(positional_args) == 1 and len(positional_args[0]) == 1:
508
                positional_args = [[p_genexp(s, positional_args[0][0])]]
William Stein's avatar
William Stein committed
509
    s.expect(')')
510
    return positional_args or [[]], keyword_args
511

512 513

def p_call_build_packed_args(pos, positional_args, keyword_args):
514
    keyword_dict = None
515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549

    subtuples = [
        ExprNodes.TupleNode(pos, args=arg) if isinstance(arg, list) else ExprNodes.AsTupleNode(pos, arg=arg)
        for arg in positional_args
    ]
    # TODO: implement a faster way to join tuples than creating each one and adding them
    arg_tuple = reduce(partial(ExprNodes.binop_node, pos, '+'), subtuples)

    if keyword_args:
        kwargs = []
        dict_items = []
        for item in keyword_args:
            if isinstance(item, tuple):
                key, value = item
                dict_items.append(ExprNodes.DictItemNode(pos=key.pos, key=key, value=value))
            elif item.is_dict_literal:
                # unpack "**{a:b}" directly
                dict_items.extend(item.key_value_pairs)
            else:
                if dict_items:
                    kwargs.append(ExprNodes.DictNode(
                        dict_items[0].pos, key_value_pairs=dict_items, reject_duplicates=True))
                    dict_items = []
                kwargs.append(item)

        if dict_items:
            kwargs.append(ExprNodes.DictNode(
                dict_items[0].pos, key_value_pairs=dict_items, reject_duplicates=True))

        if kwargs:
            if len(kwargs) == 1 and kwargs[0].is_dict_literal:
                # only simple keyword arguments found -> one dict
                keyword_dict = kwargs[0]
            else:
                # at least one **kwargs
550
                keyword_dict = ExprNodes.MergedDictNode(pos, keyword_args=kwargs)
551

552 553
    return arg_tuple, keyword_dict

554

555 556 557
def p_call(s, function):
    # s.sy == '('
    pos = s.position()
558
    positional_args, keyword_args = p_call_parse_args(s)
559

560 561
    if not keyword_args and len(positional_args) == 1 and isinstance(positional_args[0], list):
        return ExprNodes.SimpleCallNode(pos, function=function, args=positional_args[0])
William Stein's avatar
William Stein committed
562
    else:
563 564 565 566
        arg_tuple, keyword_dict = p_call_build_packed_args(pos, positional_args, keyword_args)
        return ExprNodes.GeneralCallNode(
            pos, function=function, positional_args=arg_tuple, keyword_args=keyword_dict)

William Stein's avatar
William Stein committed
567 568 569 570 571 572 573 574 575

#lambdef: 'lambda' [varargslist] ':' test

#subscriptlist: subscript (',' subscript)* [',']

def p_index(s, base):
    # s.sy == '['
    pos = s.position()
    s.next()
576 577
    subscripts, is_single_value = p_subscript_list(s)
    if is_single_value and len(subscripts[0]) == 2:
William Stein's avatar
William Stein committed
578
        start, stop = subscripts[0]
579
        result = ExprNodes.SliceIndexNode(pos,
William Stein's avatar
William Stein committed
580 581 582
            base = base, start = start, stop = stop)
    else:
        indexes = make_slice_nodes(pos, subscripts)
583
        if is_single_value:
William Stein's avatar
William Stein committed
584 585 586 587 588 589 590 591 592
            index = indexes[0]
        else:
            index = ExprNodes.TupleNode(pos, args = indexes)
        result = ExprNodes.IndexNode(pos,
            base = base, index = index)
    s.expect(']')
    return result

def p_subscript_list(s):
593
    is_single_value = True
William Stein's avatar
William Stein committed
594 595
    items = [p_subscript(s)]
    while s.sy == ',':
596
        is_single_value = False
William Stein's avatar
William Stein committed
597 598 599 600
        s.next()
        if s.sy == ']':
            break
        items.append(p_subscript(s))
601
    return items, is_single_value
William Stein's avatar
William Stein committed
602 603 604 605 606 607 608 609

#subscript: '.' '.' '.' | test | [test] ':' [test] [':' [test]]

def p_subscript(s):
    # Parse a subscript and return a list of
    # 1, 2 or 3 ExprNodes, depending on how
    # many slice elements were encountered.
    pos = s.position()
610 611 612 613 614 615 616 617 618 619
    start = p_slice_element(s, (':',))
    if s.sy != ':':
        return [start]
    s.next()
    stop = p_slice_element(s, (':', ',', ']'))
    if s.sy != ':':
        return [start, stop]
    s.next()
    step = p_slice_element(s, (':', ',', ']'))
    return [start, stop, step]
William Stein's avatar
William Stein committed
620 621 622 623 624

def p_slice_element(s, follow_set):
    # Simple expression which may be missing iff
    # it is followed by something in follow_set.
    if s.sy not in follow_set:
625
        return p_test(s)
William Stein's avatar
William Stein committed
626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656
    else:
        return None

def expect_ellipsis(s):
    s.expect('.')
    s.expect('.')
    s.expect('.')

def make_slice_nodes(pos, subscripts):
    # Convert a list of subscripts as returned
    # by p_subscript_list into a list of ExprNodes,
    # creating SliceNodes for elements with 2 or
    # more components.
    result = []
    for subscript in subscripts:
        if len(subscript) == 1:
            result.append(subscript[0])
        else:
            result.append(make_slice_node(pos, *subscript))
    return result

def make_slice_node(pos, start, stop = None, step = None):
    if not start:
        start = ExprNodes.NoneNode(pos)
    if not stop:
        stop = ExprNodes.NoneNode(pos)
    if not step:
        step = ExprNodes.NoneNode(pos)
    return ExprNodes.SliceNode(pos,
        start = start, stop = stop, step = step)

657
#atom: '(' [yield_expr|testlist_comp] ')' | '[' [listmaker] ']' | '{' [dict_or_set_maker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+
William Stein's avatar
William Stein committed
658 659 660 661 662 663 664 665

def p_atom(s):
    pos = s.position()
    sy = s.sy
    if sy == '(':
        s.next()
        if s.sy == ')':
            result = ExprNodes.TupleNode(pos, args = [])
666 667
        elif s.sy == 'yield':
            result = p_yield_expression(s)
William Stein's avatar
William Stein committed
668
        else:
669
            result = p_testlist_comp(s)
William Stein's avatar
William Stein committed
670 671 672 673 674
        s.expect(')')
        return result
    elif sy == '[':
        return p_list_maker(s)
    elif sy == '{':
675
        return p_dict_or_set_maker(s)
William Stein's avatar
William Stein committed
676 677
    elif sy == '`':
        return p_backquote_expr(s)
678 679 680
    elif sy == '.':
        expect_ellipsis(s)
        return ExprNodes.EllipsisNode(pos)
William Stein's avatar
William Stein committed
681
    elif sy == 'INT':
682
        return p_int_literal(s)
William Stein's avatar
William Stein committed
683 684 685 686 687 688 689 690
    elif sy == 'FLOAT':
        value = s.systring
        s.next()
        return ExprNodes.FloatNode(pos, value = value)
    elif sy == 'IMAG':
        value = s.systring[:-1]
        s.next()
        return ExprNodes.ImagNode(pos, value = value)
691
    elif sy == 'BEGIN_STRING':
692
        kind, bytes_value, unicode_value = p_cat_string_literal(s)
William Stein's avatar
William Stein committed
693
        if kind == 'c':
694
            return ExprNodes.CharNode(pos, value = bytes_value)
695
        elif kind == 'u':
696
            return ExprNodes.UnicodeNode(pos, value = unicode_value, bytes_value = bytes_value)
697
        elif kind == 'b':
698
            return ExprNodes.BytesNode(pos, value = bytes_value)
699 700 701
        elif kind == 'f':
            return ExprNodes.JoinedStrNode(pos, values = unicode_value)
        elif kind == '':
702
            return ExprNodes.StringNode(pos, value = bytes_value, unicode_value = unicode_value)
703 704
        else:
            s.error("invalid string kind '%s'" % kind)
William Stein's avatar
William Stein committed
705
    elif sy == 'IDENT':
706
        name = s.systring
William Stein's avatar
William Stein committed
707
        if name == "None":
708
            result = ExprNodes.NoneNode(pos)
709
        elif name == "True":
710
            result = ExprNodes.BoolNode(pos, value=True)
711
        elif name == "False":
712
            result = ExprNodes.BoolNode(pos, value=False)
713
        elif name == "NULL" and not s.in_python_file:
714
            result = ExprNodes.NullNode(pos)
William Stein's avatar
William Stein committed
715
        else:
716 717 718
            result = p_name(s, name)
        s.next()
        return result
William Stein's avatar
William Stein committed
719 720 721
    else:
        s.error("Expected an identifier or literal")

722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752
def p_int_literal(s):
    pos = s.position()
    value = s.systring
    s.next()
    unsigned = ""
    longness = ""
    while value[-1] in u"UuLl":
        if value[-1] in u"Ll":
            longness += "L"
        else:
            unsigned += "U"
        value = value[:-1]
    # '3L' is ambiguous in Py2 but not in Py3.  '3U' and '3LL' are
    # illegal in Py2 Python files.  All suffixes are illegal in Py3
    # Python files.
    is_c_literal = None
    if unsigned:
        is_c_literal = True
    elif longness:
        if longness == 'LL' or s.context.language_level >= 3:
            is_c_literal = True
    if s.in_python_file:
        if is_c_literal:
            error(pos, "illegal integer literal syntax in Python source file")
        is_c_literal = False
    return ExprNodes.IntNode(pos,
                             is_c_literal = is_c_literal,
                             value = value,
                             unsigned = unsigned,
                             longness = longness)

753

754 755
def p_name(s, name):
    pos = s.position()
756 757
    if not s.compile_time_expr and name in s.compile_time_env:
        value = s.compile_time_env.lookup_here(name)
758 759 760 761 762 763 764 765
        node = wrap_compile_time_constant(pos, value)
        if node is not None:
            return node
    return ExprNodes.NameNode(pos, name=name)


def wrap_compile_time_constant(pos, value):
    rep = repr(value)
766 767
    if value is None:
        return ExprNodes.NoneNode(pos)
768 769
    elif value is Ellipsis:
        return ExprNodes.EllipsisNode(pos)
770
    elif isinstance(value, bool):
771 772
        return ExprNodes.BoolNode(pos, value=value)
    elif isinstance(value, int):
773
        return ExprNodes.IntNode(pos, value=rep, constant_result=value)
774
    elif isinstance(value, float):
775
        return ExprNodes.FloatNode(pos, value=rep, constant_result=value)
776 777 778 779 780 781 782 783 784
    elif isinstance(value, complex):
        node = ExprNodes.ImagNode(pos, value=repr(value.imag), constant_result=complex(0.0, value.imag))
        if value.real:
            # FIXME: should we care about -0.0 ?
            # probably not worth using the '-' operator for negative imag values
            node = ExprNodes.binop_node(
                pos, '+', ExprNodes.FloatNode(pos, value=repr(value.real), constant_result=value.real), node,
                constant_result=value)
        return node
785
    elif isinstance(value, _unicode):
786
        return ExprNodes.UnicodeNode(pos, value=EncodedString(value))
787
    elif isinstance(value, _bytes):
788 789
        bvalue = bytes_literal(value, 'ascii')  # actually: unknown encoding, but BytesLiteral requires one
        return ExprNodes.BytesNode(pos, value=bvalue, constant_result=value)
790 791 792 793 794
    elif isinstance(value, tuple):
        args = [wrap_compile_time_constant(pos, arg)
                for arg in value]
        if None not in args:
            return ExprNodes.TupleNode(pos, args=args)
795
        else:
796 797
            # error already reported
            return None
798
    elif not _IS_PY3 and isinstance(value, long):
799
        return ExprNodes.IntNode(pos, value=rep.rstrip('L'), constant_result=value)
800 801
    error(pos, "Invalid type for compile-time constant: %r (type %s)"
               % (value, value.__class__.__name__))
802 803
    return None

804

William Stein's avatar
William Stein committed
805 806
def p_cat_string_literal(s):
    # A sequence of one or more adjacent string literals.
807
    # Returns (kind, bytes_value, unicode_value)
808 809
    # where kind in ('b', 'c', 'u', 'f', '')
    pos = s.position()
810 811 812
    kind, bytes_value, unicode_value = p_string_literal(s)
    if kind == 'c' or s.sy != 'BEGIN_STRING':
        return kind, bytes_value, unicode_value
813
    bstrings, ustrings, positions = [bytes_value], [unicode_value], [pos]
814 815 816 817 818 819
    bytes_value = unicode_value = None
    while s.sy == 'BEGIN_STRING':
        pos = s.position()
        next_kind, next_bytes_value, next_unicode_value = p_string_literal(s)
        if next_kind == 'c':
            error(pos, "Cannot concatenate char literal with another string or char literal")
820
            continue
821
        elif next_kind != kind:
822
            # concatenating f strings and normal strings is allowed and leads to an f string
Stefan Behnel's avatar
Stefan Behnel committed
823
            if set([kind, next_kind]) in (set(['f', 'u']), set(['f', ''])):
824 825
                kind = 'f'
            else:
826 827
                error(pos, "Cannot mix string literals of different types, expected %s'', got %s''" % (
                    kind, next_kind))
828 829 830 831
                continue
        bstrings.append(next_bytes_value)
        ustrings.append(next_unicode_value)
        positions.append(pos)
832
    # join and rewrap the partial literals
833
    if kind in ('b', 'c', '') or kind == 'u' and None not in bstrings:
834
        # Py3 enforced unicode literals are parsed as bytes/unicode combination
835
        bytes_value = bytes_literal(StringEncoding.join_bytes(bstrings), s.source_encoding)
836
    if kind in ('u', ''):
Stefan Behnel's avatar
Stefan Behnel committed
837
        unicode_value = EncodedString(u''.join([u for u in ustrings if u is not None]))
838 839 840 841 842 843 844
    if kind == 'f':
        unicode_value = []
        for u, pos in zip(ustrings, positions):
            if isinstance(u, list):
                unicode_value += u
            else:
                # non-f-string concatenated into the f-string
Stefan Behnel's avatar
Stefan Behnel committed
845
                unicode_value.append(ExprNodes.UnicodeNode(pos, value=EncodedString(u)))
846 847
    return kind, bytes_value, unicode_value

Stefan Behnel's avatar
Stefan Behnel committed
848

849
def p_opt_string_literal(s, required_type='u'):
850
    if s.sy != 'BEGIN_STRING':
William Stein's avatar
William Stein committed
851
        return None
852 853 854 855 856 857 858 859 860 861
    pos = s.position()
    kind, bytes_value, unicode_value = p_string_literal(s, required_type)
    if required_type == 'u':
        if kind == 'f':
            s.error("f-string not allowed here", pos)
        return unicode_value
    elif required_type == 'b':
        return bytes_value
    else:
        s.error("internal parser configuration error")
William Stein's avatar
William Stein committed
862

Stefan Behnel's avatar
Stefan Behnel committed
863

864 865 866 867 868 869
def check_for_non_ascii_characters(string):
    for c in string:
        if c >= u'\x80':
            return True
    return False

Stefan Behnel's avatar
Stefan Behnel committed
870

871
def p_string_literal(s, kind_override=None):
872
    # A single string or char literal.  Returns (kind, bvalue, uvalue)
873
    # where kind in ('b', 'c', 'u', 'f', '').  The 'bvalue' is the source
874 875 876
    # code byte sequence of the string literal, 'uvalue' is the
    # decoded Unicode string.  Either of the two may be None depending
    # on the 'kind' of string, only unprefixed strings have both
877 878
    # representations. In f-strings, the uvalue is a list of the Unicode
    # strings and f-string expressions that make up the f-string.
879

William Stein's avatar
William Stein committed
880 881
    # s.sy == 'BEGIN_STRING'
    pos = s.position()
882
    is_python3_source = s.context.language_level >= 3
883
    has_non_ascii_literal_characters = False
884
    string_start_pos = (pos[0], pos[1], pos[2] + len(s.systring))
885
    kind_string = s.systring.rstrip('"\'').lower()
886 887 888 889 890 891 892 893 894
    if len(kind_string) > 1:
        if len(set(kind_string)) != len(kind_string):
            error(pos, 'Duplicate string prefix character')
        if 'b' in kind_string and 'u' in kind_string:
            error(pos, 'String prefixes b and u cannot be combined')
        if 'b' in kind_string and 'f' in kind_string:
            error(pos, 'String prefixes b and f cannot be combined')
        if 'u' in kind_string and 'f' in kind_string:
            error(pos, 'String prefixes u and f cannot be combined')
895 896 897 898 899 900 901

    is_raw = 'r' in kind_string

    if 'c' in kind_string:
        # this should never happen, since the lexer does not allow combining c
        # with other prefix characters
        if len(kind_string) != 1:
902
            error(pos, 'Invalid string prefix for character literal')
903 904
        kind = 'c'
    elif 'f' in kind_string:
905 906
        kind = 'f'     # u is ignored
        is_raw = True  # postpone the escape resolution
907 908 909 910 911
    elif 'b' in kind_string:
        kind = 'b'
    elif 'u' in kind_string:
        kind = 'u'
    else:
William Stein's avatar
William Stein committed
912
        kind = ''
913

914 915 916 917 918 919
    if kind == '' and kind_override is None and Future.unicode_literals in s.context.future_directives:
        chars = StringEncoding.StrLiteralBuilder(s.source_encoding)
        kind = 'u'
    else:
        if kind_override is not None and kind_override in 'ub':
            kind = kind_override
920
        if kind in ('u', 'f'):  # f-strings are scanned exactly like Unicode literals, but are parsed further later
921 922 923 924 925
            chars = StringEncoding.UnicodeLiteralBuilder()
        elif kind == '':
            chars = StringEncoding.StrLiteralBuilder(s.source_encoding)
        else:
            chars = StringEncoding.BytesLiteralBuilder(s.source_encoding)
926

William Stein's avatar
William Stein committed
927 928 929
    while 1:
        s.next()
        sy = s.sy
930
        systr = s.systring
931
        # print "p_string_literal: sy =", sy, repr(s.systring) ###
William Stein's avatar
William Stein committed
932
        if sy == 'CHARS':
933
            chars.append(systr)
934 935
            if is_python3_source and not has_non_ascii_literal_characters and check_for_non_ascii_characters(systr):
                has_non_ascii_literal_characters = True
William Stein's avatar
William Stein committed
936
        elif sy == 'ESCAPE':
937 938
            # in Py2, 'ur' raw unicode strings resolve unicode escapes but nothing else
            if is_raw and (is_python3_source or kind != 'u' or systr[1] not in u'Uu'):
939
                chars.append(systr)
940 941
                if is_python3_source and not has_non_ascii_literal_characters and check_for_non_ascii_characters(systr):
                    has_non_ascii_literal_characters = True
William Stein's avatar
William Stein committed
942
            else:
943
                _append_escape_sequence(kind, chars, systr, s)
William Stein's avatar
William Stein committed
944
        elif sy == 'NEWLINE':
945
            chars.append(u'\n')
William Stein's avatar
William Stein committed
946 947 948
        elif sy == 'END_STRING':
            break
        elif sy == 'EOF':
949
            s.error("Unclosed string literal", pos=pos)
William Stein's avatar
William Stein committed
950
        else:
951 952
            s.error("Unexpected token %r:%r in string literal" % (
                sy, s.systring))
953

954
    if kind == 'c':
955 956 957 958
        unicode_value = None
        bytes_value = chars.getchar()
        if len(bytes_value) != 1:
            error(pos, u"invalid character literal: %r" % bytes_value)
959
    else:
960
        bytes_value, unicode_value = chars.getstrings()
961 962
        if (has_non_ascii_literal_characters
                and is_python3_source and Future.unicode_literals in s.context.future_directives):
963
            # Python 3 forbids literal non-ASCII characters in byte strings
964
            if kind == 'b':
965
                s.error("bytes can only contain ASCII literal characters.", pos=pos)
966
            bytes_value = None
967
    if kind == 'f':
968
        unicode_value = p_f_string(s, unicode_value, string_start_pos, is_raw='r' in kind_string)
William Stein's avatar
William Stein committed
969
    s.next()
970
    return (kind, bytes_value, unicode_value)
William Stein's avatar
William Stein committed
971

972

973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990
def _append_escape_sequence(kind, builder, escape_sequence, s):
    c = escape_sequence[1]
    if c in u"01234567":
        builder.append_charval(int(escape_sequence[1:], 8))
    elif c in u"'\"\\":
        builder.append(c)
    elif c in u"abfnrtv":
        builder.append(StringEncoding.char_from_escape_sequence(escape_sequence))
    elif c == u'\n':
        pass  # line continuation
    elif c == u'x':  # \xXX
        if len(escape_sequence) == 4:
            builder.append_charval(int(escape_sequence[2:], 16))
        else:
            s.error("Invalid hex escape '%s'" % escape_sequence, fatal=False)
    elif c in u'NUu' and kind in ('u', 'f', ''):  # \uxxxx, \Uxxxxxxxx, \N{...}
        chrval = -1
        if c == u'N':
991
            uchar = None
992
            try:
993 994
                uchar = lookup_unicodechar(escape_sequence[3:-1])
                chrval = ord(uchar)
995 996 997
            except KeyError:
                s.error("Unknown Unicode character name %s" %
                        repr(escape_sequence[3:-1]).lstrip('u'), fatal=False)
998 999 1000 1001 1002 1003 1004 1005
            except TypeError:
                # 2-byte unicode build of CPython?
                if (uchar is not None and _IS_2BYTE_UNICODE and len(uchar) == 2 and
                        unicode_category(uchar[0]) == 'Cs' and unicode_category(uchar[1]) == 'Cs'):
                    # surrogate pair instead of single character
                    chrval = 0x10000 + (ord(uchar[0]) - 0xd800) >> 10 + (ord(uchar[1]) - 0xdc00)
                else:
                    raise
1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018
        elif len(escape_sequence) in (6, 10):
            chrval = int(escape_sequence[2:], 16)
            if chrval > 1114111:  # sys.maxunicode:
                s.error("Invalid unicode escape '%s'" % escape_sequence)
                chrval = -1
        else:
            s.error("Invalid unicode escape '%s'" % escape_sequence, fatal=False)
        if chrval >= 0:
            builder.append_uescape(chrval, escape_sequence)
    else:
        builder.append(escape_sequence)


1019
_parse_escape_sequences_raw, _parse_escape_sequences = [re.compile((
1020
    # escape sequences:
1021 1022 1023 1024 1025 1026 1027 1028 1029 1030
    br'(\\(?:' +
    (br'\\?' if is_raw else (
        br'[\\abfnrtv"\'{]|'
        br'[0-7]{2,3}|'
        br'N\{[^}]*\}|'
        br'x[0-9a-fA-F]{2}|'
        br'u[0-9a-fA-F]{4}|'
        br'U[0-9a-fA-F]{8}|'
        br'[NxuU]|'  # detect invalid escape sequences that do not match above
    )) +
1031 1032 1033 1034
    br')?|'
    # non-escape sequences:
    br'\{\{?|'
    br'\}\}?|'
1035 1036 1037
    br'[^\\{}]+)'
    ).decode('us-ascii')).match
    for is_raw in (True, False)]
1038 1039


1040 1041 1042 1043
def _f_string_error_pos(pos, string, i):
    return (pos[0], pos[1], pos[2] + i + 1)  # FIXME: handle newlines in string


1044
def p_f_string(s, unicode_value, pos, is_raw):
1045 1046 1047
    # Parses a PEP 498 f-string literal into a list of nodes. Nodes are either UnicodeNodes
    # or FormattedValueNodes.
    values = []
1048
    next_start = 0
1049
    size = len(unicode_value)
1050
    builder = StringEncoding.UnicodeLiteralBuilder()
1051
    _parse_seq = _parse_escape_sequences_raw if is_raw else _parse_escape_sequences
1052 1053 1054

    while next_start < size:
        end = next_start
1055
        match = _parse_seq(unicode_value, next_start)
1056
        if match is None:
1057
            error(_f_string_error_pos(pos, unicode_value, next_start), "Invalid escape sequence")
1058 1059 1060 1061 1062 1063 1064

        next_start = match.end()
        part = match.group()
        c = part[0]
        if c == '\\':
            if not is_raw and len(part) > 1:
                _append_escape_sequence('f', builder, part, s)
1065
            else:
1066 1067 1068 1069 1070 1071 1072 1073 1074 1075
                builder.append(part)
        elif c == '{':
            if part == '{{':
                builder.append('{')
            else:
                # start of an expression
                if builder.chars:
                    values.append(ExprNodes.UnicodeNode(pos, value=builder.getstring()))
                    builder = StringEncoding.UnicodeLiteralBuilder()
                next_start, expr_node = p_f_string_expr(s, unicode_value, pos, next_start, is_raw)
1076
                values.append(expr_node)
1077 1078 1079 1080
        elif c == '}':
            if part == '}}':
                builder.append('}')
            else:
1081 1082
                error(_f_string_error_pos(pos, unicode_value, end),
                      "f-string: single '}' is not allowed")
1083
        else:
1084
            builder.append(part)
1085

1086 1087
    if builder.chars:
        values.append(ExprNodes.UnicodeNode(pos, value=builder.getstring()))
1088 1089 1090
    return values


1091
def p_f_string_expr(s, unicode_value, pos, starting_index, is_raw):
1092 1093 1094 1095
    # Parses a {}-delimited expression inside an f-string. Returns a FormattedValueNode
    # and the index in the string that follows the expression.
    i = starting_index
    size = len(unicode_value)
1096
    conversion_char = terminal_char = format_spec = None
1097
    format_spec_str = None
1098
    NO_CHAR = 2**30
1099 1100

    nested_depth = 0
1101
    quote_char = NO_CHAR
1102
    in_triple_quotes = False
1103
    backslash_reported = False
1104 1105 1106

    while True:
        if i >= size:
1107
            break  # error will be reported below
1108 1109
        c = unicode_value[i]

1110
        if quote_char != NO_CHAR:
1111
            if c == '\\':
1112 1113 1114 1115 1116
                # avoid redundant error reports along '\' sequences
                if not backslash_reported:
                    error(_f_string_error_pos(pos, unicode_value, i),
                          "backslashes not allowed in f-strings")
                backslash_reported = True
1117 1118 1119 1120
            elif c == quote_char:
                if in_triple_quotes:
                    if i + 2 < size and unicode_value[i + 1] == c and unicode_value[i + 2] == c:
                        in_triple_quotes = False
1121
                        quote_char = NO_CHAR
1122 1123
                        i += 2
                else:
1124
                    quote_char = NO_CHAR
1125 1126 1127 1128 1129 1130
        elif c in '\'"':
            quote_char = c
            if i + 2 < size and unicode_value[i + 1] == c and unicode_value[i + 2] == c:
                in_triple_quotes = True
                i += 2
        elif c in '{[(':
1131 1132 1133 1134
            nested_depth += 1
        elif nested_depth != 0 and c in '}])':
            nested_depth -= 1
        elif c == '#':
1135 1136
            error(_f_string_error_pos(pos, unicode_value, i),
                  "format string cannot include #")
1137 1138 1139
        elif nested_depth == 0 and c in '!:}':
            # allow != as a special case
            if c == '!' and i + 1 < size and unicode_value[i + 1] == '=':
Jelle Zijlstra's avatar
Jelle Zijlstra committed
1140
                i += 1
1141 1142 1143 1144 1145 1146
                continue

            terminal_char = c
            break
        i += 1

1147 1148 1149 1150 1151
    # normalise line endings as the parser expects that
    expr_str = unicode_value[starting_index:i].replace('\r\n', '\n').replace('\r', '\n')
    expr_pos = (pos[0], pos[1], pos[2] + starting_index + 2)  # TODO: find exact code position (concat, multi-line, ...)

    if not expr_str.strip():
1152 1153
        error(_f_string_error_pos(pos, unicode_value, starting_index),
              "empty expression not allowed in f-string")
1154 1155 1156

    if terminal_char == '!':
        i += 1
1157
        if i + 2 > size:
1158
            pass  # error will be reported below
1159 1160 1161 1162
        else:
            conversion_char = unicode_value[i]
            i += 1
            terminal_char = unicode_value[i]
1163 1164

    if terminal_char == ':':
1165 1166
        in_triple_quotes = False
        in_string = False
1167 1168 1169 1170
        nested_depth = 0
        start_format_spec = i + 1
        while True:
            if i >= size:
1171
                break  # error will be reported below
1172
            c = unicode_value[i]
1173 1174 1175 1176
            if not in_triple_quotes and not in_string:
                if c == '{':
                    nested_depth += 1
                elif c == '}':
1177 1178 1179 1180 1181
                    if nested_depth > 0:
                        nested_depth -= 1
                    else:
                        terminal_char = c
                        break
1182 1183 1184 1185 1186 1187
            if c in '\'"':
                if not in_string and i + 2 < size and unicode_value[i + 1] == c and unicode_value[i + 2] == c:
                    in_triple_quotes = not in_triple_quotes
                    i += 2
                elif not in_triple_quotes:
                    in_string = not in_string
1188 1189 1190 1191 1192
            i += 1

        format_spec_str = unicode_value[start_format_spec:i]

    if terminal_char != '}':
1193 1194 1195
        error(_f_string_error_pos(pos, unicode_value, i),
              "missing '}' in format string expression" + (
                  ", found '%s'" % terminal_char if terminal_char else ""))
1196

1197 1198 1199
    # parse the expression as if it was surrounded by parentheses
    buf = StringIO('(%s)' % expr_str)
    scanner = PyrexScanner(buf, expr_pos[0], parent_scanner=s, source_encoding=s.source_encoding, initial_pos=expr_pos)
1200 1201 1202
    expr = p_testlist(scanner)  # TODO is testlist right here?

    # validate the conversion char
1203
    if conversion_char is not None and not ExprNodes.FormattedValueNode.find_conversion_func(conversion_char):
1204
        error(expr_pos, "invalid conversion character '%s'" % conversion_char)
1205 1206

    # the format spec is itself treated like an f-string
1207
    if format_spec_str:
1208
        format_spec = ExprNodes.JoinedStrNode(pos, values=p_f_string(s, format_spec_str, pos, is_raw))
1209 1210

    return i + 1, ExprNodes.FormattedValueNode(
1211
        pos, value=expr, conversion_char=conversion_char, format_spec=format_spec)
1212 1213


1214 1215 1216
# since PEP 448:
# list_display  ::=     "[" [listmaker] "]"
# listmaker     ::=     (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
Stefan Behnel's avatar
Stefan Behnel committed
1217
# comp_iter     ::=     comp_for | comp_if
1218
# comp_for      ::=     ["async"] "for" expression_list "in" testlist [comp_iter]
1219
# comp_if       ::=     "if" test [comp_iter]
1220

William Stein's avatar
William Stein committed
1221 1222 1223 1224
def p_list_maker(s):
    # s.sy == '['
    pos = s.position()
    s.next()
Robert Bradshaw's avatar
Robert Bradshaw committed
1225 1226
    if s.sy == ']':
        s.expect(']')
1227 1228 1229
        return ExprNodes.ListNode(pos, args=[])

    expr = p_test_or_starred_expr(s)
1230
    if s.sy in ('for', 'async'):
1231 1232
        if expr.is_starred:
            s.error("iterable unpacking cannot be used in comprehension")
1233
        append = ExprNodes.ComprehensionAppendNode(pos, expr=expr)
1234
        loop = p_comp_for(s, append)
Robert Bradshaw's avatar
Robert Bradshaw committed
1235
        s.expect(']')
1236
        return ExprNodes.ComprehensionNode(
1237
            pos, loop=loop, append=append, type=Builtin.list_type,
1238
            # list comprehensions leak their loop variable in Py2
1239 1240 1241 1242 1243 1244
            has_local_scope=s.context.language_level >= 3)

    # (merged) list literal
    if s.sy == ',':
        s.next()
        exprs = p_test_or_starred_expr_list(s, expr)
Robert Bradshaw's avatar
Robert Bradshaw committed
1245
    else:
1246 1247 1248 1249
        exprs = [expr]
    s.expect(']')
    return ExprNodes.ListNode(pos, args=exprs)

1250

Stefan Behnel's avatar
Stefan Behnel committed
1251
def p_comp_iter(s, body):
1252
    if s.sy in ('for', 'async'):
1253
        return p_comp_for(s, body)
Robert Bradshaw's avatar
Robert Bradshaw committed
1254
    elif s.sy == 'if':
Stefan Behnel's avatar
Stefan Behnel committed
1255
        return p_comp_if(s, body)
Robert Bradshaw's avatar
Robert Bradshaw committed
1256
    else:
1257 1258
        # insert the 'append' operation into the loop
        return body
William Stein's avatar
William Stein committed
1259

1260
def p_comp_for(s, body):
Robert Bradshaw's avatar
Robert Bradshaw committed
1261
    pos = s.position()
1262 1263 1264 1265 1266 1267 1268 1269 1270 1271
    # [async] for ...
    is_async = False
    if s.sy == 'async':
        is_async = True
        s.next()

    # s.sy == 'for'
    s.expect('for')
    kw = p_for_bounds(s, allow_testlist=False, is_async=is_async)
    kw.update(else_clause=None, body=p_comp_iter(s, body), is_async=is_async)
Robert Bradshaw's avatar
Robert Bradshaw committed
1272
    return Nodes.ForStatNode(pos, **kw)
1273

Stefan Behnel's avatar
Stefan Behnel committed
1274
def p_comp_if(s, body):
Robert Bradshaw's avatar
Robert Bradshaw committed
1275 1276 1277
    # s.sy == 'if'
    pos = s.position()
    s.next()
Stefan Behnel's avatar
Stefan Behnel committed
1278
    test = p_test_nocond(s)
1279
    return Nodes.IfStatNode(pos,
1280
        if_clauses = [Nodes.IfClauseNode(pos, condition = test,
Stefan Behnel's avatar
Stefan Behnel committed
1281
                                         body = p_comp_iter(s, body))],
Robert Bradshaw's avatar
Robert Bradshaw committed
1282
        else_clause = None )
1283

1284 1285 1286 1287 1288 1289

# since PEP 448:
#dictorsetmaker: ( ((test ':' test | '**' expr)
#                   (comp_for | (',' (test ':' test | '**' expr))* [','])) |
#                  ((test | star_expr)
#                   (comp_for | (',' (test | star_expr))* [','])) )
William Stein's avatar
William Stein committed
1290

1291
def p_dict_or_set_maker(s):
William Stein's avatar
William Stein committed
1292 1293 1294
    # s.sy == '{'
    pos = s.position()
    s.next()
1295
    if s.sy == '}':
William Stein's avatar
William Stein committed
1296
        s.next()
1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310
        return ExprNodes.DictNode(pos, key_value_pairs=[])

    parts = []
    target_type = 0
    last_was_simple_item = False
    while True:
        if s.sy in ('*', '**'):
            # merged set/dict literal
            if target_type == 0:
                target_type = 1 if s.sy == '*' else 2  # 'stars'
            elif target_type != len(s.sy):
                s.error("unexpected %sitem found in %s literal" % (
                    s.sy, 'set' if target_type == 1 else 'dict'))
            s.next()
1311 1312 1313
            if s.sy == '*':
                s.error("expected expression, found '*'")
            item = p_starred_expr(s)
1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332
            parts.append(item)
            last_was_simple_item = False
        else:
            item = p_test(s)
            if target_type == 0:
                target_type = 2 if s.sy == ':' else 1  # dict vs. set
            if target_type == 2:
                # dict literal
                s.expect(':')
                key = item
                value = p_test(s)
                item = ExprNodes.DictItemNode(key.pos, key=key, value=value)
            if last_was_simple_item:
                parts[-1].append(item)
            else:
                parts.append([item])
                last_was_simple_item = True

        if s.sy == ',':
1333
            s.next()
1334 1335
            if s.sy == '}':
                break
1336 1337 1338
        else:
            break

1339
    if s.sy in ('for', 'async'):
1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350
        # dict/set comprehension
        if len(parts) == 1 and isinstance(parts[0], list) and len(parts[0]) == 1:
            item = parts[0][0]
            if target_type == 2:
                assert isinstance(item, ExprNodes.DictItemNode), type(item)
                comprehension_type = Builtin.dict_type
                append = ExprNodes.DictComprehensionAppendNode(
                    item.pos, key_expr=item.key, value_expr=item.value)
            else:
                comprehension_type = Builtin.set_type
                append = ExprNodes.ComprehensionAppendNode(item.pos, expr=item)
1351
            loop = p_comp_for(s, append)
1352
            s.expect('}')
1353
            return ExprNodes.ComprehensionNode(pos, loop=loop, append=append, type=comprehension_type)
1354
        else:
1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379
            # syntax error, try to find a good error message
            if len(parts) == 1 and not isinstance(parts[0], list):
                s.error("iterable unpacking cannot be used in comprehension")
            else:
                # e.g. "{1,2,3 for ..."
                s.expect('}')
            return ExprNodes.DictNode(pos, key_value_pairs=[])

    s.expect('}')
    if target_type == 1:
        # (merged) set literal
        items = []
        set_items = []
        for part in parts:
            if isinstance(part, list):
                set_items.extend(part)
            else:
                if set_items:
                    items.append(ExprNodes.SetNode(set_items[0].pos, args=set_items))
                    set_items = []
                items.append(part)
        if set_items:
            items.append(ExprNodes.SetNode(set_items[0].pos, args=set_items))
        if len(items) == 1 and items[0].is_set_literal:
            return items[0]
1380
        return ExprNodes.MergedSequenceNode(pos, args=items, type=Builtin.set_type)
1381
    else:
1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398
        # (merged) dict literal
        items = []
        dict_items = []
        for part in parts:
            if isinstance(part, list):
                dict_items.extend(part)
            else:
                if dict_items:
                    items.append(ExprNodes.DictNode(dict_items[0].pos, key_value_pairs=dict_items))
                    dict_items = []
                items.append(part)
        if dict_items:
            items.append(ExprNodes.DictNode(dict_items[0].pos, key_value_pairs=dict_items))
        if len(items) == 1 and items[0].is_dict_literal:
            return items[0]
        return ExprNodes.MergedDictNode(pos, keyword_args=items, reject_duplicates=False)

William Stein's avatar
William Stein committed
1399

1400
# NOTE: no longer in Py3 :)
William Stein's avatar
William Stein committed
1401 1402 1403 1404
def p_backquote_expr(s):
    # s.sy == '`'
    pos = s.position()
    s.next()
1405 1406 1407 1408
    args = [p_test(s)]
    while s.sy == ',':
        s.next()
        args.append(p_test(s))
William Stein's avatar
William Stein committed
1409
    s.expect('`')
1410 1411 1412 1413
    if len(args) == 1:
        arg = args[0]
    else:
        arg = ExprNodes.TupleNode(pos, args = args)
William Stein's avatar
William Stein committed
1414 1415
    return ExprNodes.BackquoteNode(pos, arg = arg)

1416 1417
def p_simple_expr_list(s, expr=None):
    exprs = expr is not None and [expr] or []
William Stein's avatar
William Stein committed
1418
    while s.sy not in expr_terminators:
1419
        exprs.append( p_test(s) )
Stefan Behnel's avatar
Stefan Behnel committed
1420
        if s.sy != ',':
William Stein's avatar
William Stein committed
1421 1422 1423 1424
            break
        s.next()
    return exprs

1425

1426 1427 1428
def p_test_or_starred_expr_list(s, expr=None):
    exprs = expr is not None and [expr] or []
    while s.sy not in expr_terminators:
1429
        exprs.append(p_test_or_starred_expr(s))
1430 1431 1432 1433
        if s.sy != ',':
            break
        s.next()
    return exprs
1434

1435 1436 1437 1438

#testlist: test (',' test)* [',']

def p_testlist(s):
William Stein's avatar
William Stein committed
1439
    pos = s.position()
1440
    expr = p_test(s)
William Stein's avatar
William Stein committed
1441 1442
    if s.sy == ',':
        s.next()
1443
        exprs = p_simple_expr_list(s, expr)
William Stein's avatar
William Stein committed
1444 1445 1446 1447
        return ExprNodes.TupleNode(pos, args = exprs)
    else:
        return expr

1448
# testlist_star_expr: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
Robert Bradshaw's avatar
Robert Bradshaw committed
1449

1450
def p_testlist_star_expr(s):
Robert Bradshaw's avatar
Robert Bradshaw committed
1451
    pos = s.position()
1452
    expr = p_test_or_starred_expr(s)
Robert Bradshaw's avatar
Robert Bradshaw committed
1453
    if s.sy == ',':
1454
        s.next()
1455 1456
        exprs = p_test_or_starred_expr_list(s, expr)
        return ExprNodes.TupleNode(pos, args = exprs)
Robert Bradshaw's avatar
Robert Bradshaw committed
1457 1458 1459
    else:
        return expr

1460 1461 1462
# testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )

def p_testlist_comp(s):
Robert Bradshaw's avatar
Robert Bradshaw committed
1463
    pos = s.position()
1464
    expr = p_test_or_starred_expr(s)
Robert Bradshaw's avatar
Robert Bradshaw committed
1465
    if s.sy == ',':
1466
        s.next()
1467
        exprs = p_test_or_starred_expr_list(s, expr)
Robert Bradshaw's avatar
Robert Bradshaw committed
1468
        return ExprNodes.TupleNode(pos, args = exprs)
1469
    elif s.sy in ('for', 'async'):
1470
        return p_genexp(s, expr)
Robert Bradshaw's avatar
Robert Bradshaw committed
1471 1472
    else:
        return expr
1473 1474

def p_genexp(s, expr):
1475
    # s.sy == 'async' | 'for'
1476 1477
    loop = p_comp_for(s, Nodes.ExprStatNode(
        expr.pos, expr = ExprNodes.YieldExprNode(expr.pos, arg=expr)))
1478 1479
    return ExprNodes.GeneratorExpressionNode(expr.pos, loop=loop)

1480 1481
expr_terminators = cython.declare(set, set([
    ')', ']', '}', ':', '=', 'NEWLINE']))
William Stein's avatar
William Stein committed
1482

1483

William Stein's avatar
William Stein committed
1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496
#-------------------------------------------------------
#
#   Statements
#
#-------------------------------------------------------

def p_global_statement(s):
    # assume s.sy == 'global'
    pos = s.position()
    s.next()
    names = p_ident_list(s)
    return Nodes.GlobalNode(pos, names = names)

1497

1498 1499 1500 1501 1502 1503
def p_nonlocal_statement(s):
    pos = s.position()
    s.next()
    names = p_ident_list(s)
    return Nodes.NonlocalNode(pos, names = names)

1504

William Stein's avatar
William Stein committed
1505
def p_expression_or_assignment(s):
1506 1507 1508
    expr = p_testlist_star_expr(s)
    if s.sy == ':' and (expr.is_name or expr.is_subscript or expr.is_attribute):
        s.next()
1509
        expr.annotation = p_annotation(s)
1510
    if s.sy == '=' and expr.is_starred:
1511 1512 1513 1514
        # This is a common enough error to make when learning Cython to let
        # it fail as early as possible and give a very clear error message.
        s.error("a starred assignment target must be in a list or tuple"
                " - maybe you meant to use an index assignment: var[0] = ...",
1515 1516
                pos=expr.pos)
    expr_list = [expr]
William Stein's avatar
William Stein committed
1517 1518
    while s.sy == '=':
        s.next()
1519 1520 1521 1522 1523
        if s.sy == 'yield':
            expr = p_yield_expression(s)
        else:
            expr = p_testlist_star_expr(s)
        expr_list.append(expr)
William Stein's avatar
William Stein committed
1524
    if len(expr_list) == 1:
1525
        if re.match(r"([-+*/%^&|]|<<|>>|\*\*|//|@)=", s.sy):
1526
            lhs = expr_list[0]
1527 1528 1529 1530 1531 1532
            if isinstance(lhs, ExprNodes.SliceIndexNode):
                # implementation requires IndexNode
                lhs = ExprNodes.IndexNode(
                    lhs.pos,
                    base=lhs.base,
                    index=make_slice_node(lhs.pos, lhs.start, lhs.stop))
1533
            elif not isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode, ExprNodes.NameNode)):
1534
                error(lhs.pos, "Illegal operand for inplace operation.")
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
1535
            operator = s.sy[:-1]
1536
            s.next()
1537 1538 1539 1540
            if s.sy == 'yield':
                rhs = p_yield_expression(s)
            else:
                rhs = p_testlist(s)
1541
            return Nodes.InPlaceAssignmentNode(lhs.pos, operator=operator, lhs=lhs, rhs=rhs)
1542
        expr = expr_list[0]
1543
        return Nodes.ExprStatNode(expr.pos, expr=expr)
1544

1545 1546
    rhs = expr_list[-1]
    if len(expr_list) == 2:
1547
        return Nodes.SingleAssignmentNode(rhs.pos, lhs=expr_list[0], rhs=rhs)
William Stein's avatar
William Stein committed
1548
    else:
1549 1550
        return Nodes.CascadedAssignmentNode(rhs.pos, lhs_list=expr_list[:-1], rhs=rhs)

William Stein's avatar
William Stein committed
1551 1552 1553 1554

def p_print_statement(s):
    # s.sy == 'print'
    pos = s.position()
1555
    ends_with_comma = 0
William Stein's avatar
William Stein committed
1556 1557
    s.next()
    if s.sy == '>>':
1558
        s.next()
1559
        stream = p_test(s)
1560 1561 1562 1563 1564
        if s.sy == ',':
            s.next()
            ends_with_comma = s.sy in ('NEWLINE', 'EOF')
    else:
        stream = None
William Stein's avatar
William Stein committed
1565 1566
    args = []
    if s.sy not in ('NEWLINE', 'EOF'):
1567
        args.append(p_test(s))
William Stein's avatar
William Stein committed
1568 1569 1570
        while s.sy == ',':
            s.next()
            if s.sy in ('NEWLINE', 'EOF'):
1571
                ends_with_comma = 1
William Stein's avatar
William Stein committed
1572
                break
1573
            args.append(p_test(s))
1574
    arg_tuple = ExprNodes.TupleNode(pos, args=args)
1575
    return Nodes.PrintStatNode(pos,
1576 1577 1578
        arg_tuple=arg_tuple, stream=stream,
        append_newline=not ends_with_comma)

William Stein's avatar
William Stein committed
1579

1580 1581 1582 1583
def p_exec_statement(s):
    # s.sy == 'exec'
    pos = s.position()
    s.next()
1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595
    code = p_bit_expr(s)
    if isinstance(code, ExprNodes.TupleNode):
        # Py3 compatibility syntax
        tuple_variant = True
        args = code.args
        if len(args) not in (2, 3):
            s.error("expected tuple of length 2 or 3, got length %d" % len(args),
                    pos=pos, fatal=False)
            args = [code]
    else:
        tuple_variant = False
        args = [code]
1596
    if s.sy == 'in':
1597 1598 1599
        if tuple_variant:
            s.error("tuple variant of exec does not support additional 'in' arguments",
                    fatal=False)
1600
        s.next()
1601
        args.append(p_test(s))
1602 1603
        if s.sy == ',':
            s.next()
1604
            args.append(p_test(s))
1605
    return Nodes.ExecStatNode(pos, args=args)
1606

William Stein's avatar
William Stein committed
1607 1608 1609 1610
def p_del_statement(s):
    # s.sy == 'del'
    pos = s.position()
    s.next()
1611
    # FIXME: 'exprlist' in Python
William Stein's avatar
William Stein committed
1612 1613 1614 1615 1616 1617 1618
    args = p_simple_expr_list(s)
    return Nodes.DelStatNode(pos, args = args)

def p_pass_statement(s, with_newline = 0):
    pos = s.position()
    s.expect('pass')
    if with_newline:
1619
        s.expect_newline("Expected a newline", ignore_semicolon=True)
William Stein's avatar
William Stein committed
1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638
    return Nodes.PassStatNode(pos)

def p_break_statement(s):
    # s.sy == 'break'
    pos = s.position()
    s.next()
    return Nodes.BreakStatNode(pos)

def p_continue_statement(s):
    # s.sy == 'continue'
    pos = s.position()
    s.next()
    return Nodes.ContinueStatNode(pos)

def p_return_statement(s):
    # s.sy == 'return'
    pos = s.position()
    s.next()
    if s.sy not in statement_terminators:
1639
        value = p_testlist(s)
William Stein's avatar
William Stein committed
1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650
    else:
        value = None
    return Nodes.ReturnStatNode(pos, value = value)

def p_raise_statement(s):
    # s.sy == 'raise'
    pos = s.position()
    s.next()
    exc_type = None
    exc_value = None
    exc_tb = None
Haoyu Bai's avatar
Haoyu Bai committed
1651
    cause = None
William Stein's avatar
William Stein committed
1652
    if s.sy not in statement_terminators:
1653
        exc_type = p_test(s)
William Stein's avatar
William Stein committed
1654 1655
        if s.sy == ',':
            s.next()
1656
            exc_value = p_test(s)
William Stein's avatar
William Stein committed
1657 1658
            if s.sy == ',':
                s.next()
1659
                exc_tb = p_test(s)
Haoyu Bai's avatar
Haoyu Bai committed
1660 1661 1662
        elif s.sy == 'from':
            s.next()
            cause = p_test(s)
1663
    if exc_type or exc_value or exc_tb:
1664
        return Nodes.RaiseStatNode(pos,
1665 1666
            exc_type = exc_type,
            exc_value = exc_value,
Haoyu Bai's avatar
Haoyu Bai committed
1667 1668
            exc_tb = exc_tb,
            cause = cause)
1669 1670
    else:
        return Nodes.ReraiseStatNode(pos)
William Stein's avatar
William Stein committed
1671

1672

William Stein's avatar
William Stein committed
1673 1674 1675 1676 1677
def p_import_statement(s):
    # s.sy in ('import', 'cimport')
    pos = s.position()
    kind = s.sy
    s.next()
1678
    items = [p_dotted_name(s, as_allowed=1)]
William Stein's avatar
William Stein committed
1679 1680
    while s.sy == ',':
        s.next()
1681
        items.append(p_dotted_name(s, as_allowed=1))
William Stein's avatar
William Stein committed
1682
    stats = []
1683
    is_absolute = Future.absolute_import in s.context.future_directives
William Stein's avatar
William Stein committed
1684 1685
    for pos, target_name, dotted_name, as_name in items:
        if kind == 'cimport':
1686 1687 1688 1689 1690
            stat = Nodes.CImportStatNode(
                pos,
                module_name=dotted_name,
                as_name=as_name,
                is_absolute=is_absolute)
William Stein's avatar
William Stein committed
1691
        else:
1692 1693 1694 1695 1696 1697 1698
            stat = Nodes.SingleAssignmentNode(
                pos,
                lhs=ExprNodes.NameNode(pos, name=as_name or target_name),
                rhs=ExprNodes.ImportNode(
                    pos,
                    module_name=ExprNodes.IdentifierStringNode(pos, value=dotted_name),
                    level=0 if is_absolute else None,
1699 1700
                    get_top_level_module='.' in dotted_name and as_name is None,
                    name_list=None))
William Stein's avatar
William Stein committed
1701
        stats.append(stat)
1702 1703
    return Nodes.StatListNode(pos, stats=stats)

William Stein's avatar
William Stein committed
1704

Stefan Behnel's avatar
Stefan Behnel committed
1705
def p_from_import_statement(s, first_statement = 0):
William Stein's avatar
William Stein committed
1706 1707 1708
    # s.sy == 'from'
    pos = s.position()
    s.next()
Haoyu Bai's avatar
Haoyu Bai committed
1709 1710 1711 1712 1713 1714 1715
    if s.sy == '.':
        # count relative import level
        level = 0
        while s.sy == '.':
            level += 1
            s.next()
    else:
1716
        level = None
1717
    if level is not None and s.sy in ('import', 'cimport'):
Haoyu Bai's avatar
Haoyu Bai committed
1718
        # we are dealing with "from .. import foo, bar"
1719
        dotted_name_pos, dotted_name = s.position(), s.context.intern_ustring('')
William Stein's avatar
William Stein committed
1720
    else:
1721 1722 1723 1724
        if level is None and Future.absolute_import in s.context.future_directives:
            level = 0
        (dotted_name_pos, _, dotted_name, _) = p_dotted_name(s, as_allowed=False)
    if s.sy not in ('import', 'cimport'):
William Stein's avatar
William Stein committed
1725
        s.error("Expected 'import' or 'cimport'")
1726 1727
    kind = s.sy
    s.next()
Haoyu Bai's avatar
Haoyu Bai committed
1728

1729
    is_cimport = kind == 'cimport'
1730
    is_parenthesized = False
William Stein's avatar
William Stein committed
1731
    if s.sy == '*':
1732
        imported_names = [(s.position(), s.context.intern_ustring("*"), None, None)]
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
1733 1734
        s.next()
    else:
1735 1736 1737
        if s.sy == '(':
            is_parenthesized = True
            s.next()
1738
        imported_names = [p_imported_name(s, is_cimport)]
William Stein's avatar
William Stein committed
1739 1740
    while s.sy == ',':
        s.next()
1741 1742
        if is_parenthesized and s.sy == ')':
            break
1743
        imported_names.append(p_imported_name(s, is_cimport))
1744 1745
    if is_parenthesized:
        s.expect(')')
Stefan Behnel's avatar
Stefan Behnel committed
1746 1747 1748
    if dotted_name == '__future__':
        if not first_statement:
            s.error("from __future__ imports must occur at the beginning of the file")
1749
        elif level:
Haoyu Bai's avatar
Haoyu Bai committed
1750
            s.error("invalid syntax")
Stefan Behnel's avatar
Stefan Behnel committed
1751
        else:
1752
            for (name_pos, name, as_name, kind) in imported_names:
1753 1754 1755
                if name == "braces":
                    s.error("not a chance", name_pos)
                    break
Stefan Behnel's avatar
Stefan Behnel committed
1756 1757 1758
                try:
                    directive = getattr(Future, name)
                except AttributeError:
1759
                    s.error("future feature %s is not defined" % name, name_pos)
Stefan Behnel's avatar
Stefan Behnel committed
1760 1761 1762 1763
                    break
                s.context.future_directives.add(directive)
        return Nodes.PassStatNode(pos)
    elif kind == 'cimport':
1764 1765 1766 1767
        return Nodes.FromCImportStatNode(
            pos, module_name=dotted_name,
            relative_level=level,
            imported_names=imported_names)
William Stein's avatar
William Stein committed
1768 1769 1770
    else:
        imported_name_strings = []
        items = []
1771
        for (name_pos, name, as_name, kind) in imported_names:
William Stein's avatar
William Stein committed
1772
            imported_name_strings.append(
1773
                ExprNodes.IdentifierStringNode(name_pos, value=name))
William Stein's avatar
William Stein committed
1774
            items.append(
1775
                (name, ExprNodes.NameNode(name_pos, name=as_name or name)))
William Stein's avatar
William Stein committed
1776
        import_list = ExprNodes.ListNode(
1777
            imported_names[0][0], args=imported_name_strings)
William Stein's avatar
William Stein committed
1778 1779
        return Nodes.FromImportStatNode(pos,
            module = ExprNodes.ImportNode(dotted_name_pos,
1780
                module_name = ExprNodes.IdentifierStringNode(pos, value = dotted_name),
Haoyu Bai's avatar
Haoyu Bai committed
1781
                level = level,
William Stein's avatar
William Stein committed
1782 1783 1784
                name_list = import_list),
            items = items)

1785 1786

imported_name_kinds = cython.declare(set, set(['class', 'struct', 'union']))
1787 1788

def p_imported_name(s, is_cimport):
William Stein's avatar
William Stein committed
1789
    pos = s.position()
1790 1791 1792 1793
    kind = None
    if is_cimport and s.systring in imported_name_kinds:
        kind = s.systring
        s.next()
William Stein's avatar
William Stein committed
1794 1795
    name = p_ident(s)
    as_name = p_as_name(s)
1796
    return (pos, name, as_name, kind)
William Stein's avatar
William Stein committed
1797

1798

William Stein's avatar
William Stein committed
1799 1800 1801 1802 1803 1804 1805 1806 1807 1808
def p_dotted_name(s, as_allowed):
    pos = s.position()
    target_name = p_ident(s)
    as_name = None
    names = [target_name]
    while s.sy == '.':
        s.next()
        names.append(p_ident(s))
    if as_allowed:
        as_name = p_as_name(s)
1809 1810
    return (pos, target_name, s.context.intern_ustring(u'.'.join(names)), as_name)

William Stein's avatar
William Stein committed
1811 1812 1813 1814 1815 1816 1817 1818

def p_as_name(s):
    if s.sy == 'IDENT' and s.systring == 'as':
        s.next()
        return p_ident(s)
    else:
        return None

1819

William Stein's avatar
William Stein committed
1820 1821 1822 1823
def p_assert_statement(s):
    # s.sy == 'assert'
    pos = s.position()
    s.next()
1824
    cond = p_test(s)
William Stein's avatar
William Stein committed
1825 1826
    if s.sy == ',':
        s.next()
1827
        value = p_test(s)
William Stein's avatar
William Stein committed
1828 1829
    else:
        value = None
1830
    return Nodes.AssertStatNode(pos, condition=cond, value=value)
William Stein's avatar
William Stein committed
1831

1832

1833
statement_terminators = cython.declare(set, set([';', 'NEWLINE', 'EOF']))
William Stein's avatar
William Stein committed
1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848

def p_if_statement(s):
    # s.sy == 'if'
    pos = s.position()
    s.next()
    if_clauses = [p_if_clause(s)]
    while s.sy == 'elif':
        s.next()
        if_clauses.append(p_if_clause(s))
    else_clause = p_else_clause(s)
    return Nodes.IfStatNode(pos,
        if_clauses = if_clauses, else_clause = else_clause)

def p_if_clause(s):
    pos = s.position()
1849
    test = p_test(s)
William Stein's avatar
William Stein committed
1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864
    body = p_suite(s)
    return Nodes.IfClauseNode(pos,
        condition = test, body = body)

def p_else_clause(s):
    if s.sy == 'else':
        s.next()
        return p_suite(s)
    else:
        return None

def p_while_statement(s):
    # s.sy == 'while'
    pos = s.position()
    s.next()
1865
    test = p_test(s)
William Stein's avatar
William Stein committed
1866 1867
    body = p_suite(s)
    else_clause = p_else_clause(s)
1868 1869
    return Nodes.WhileStatNode(pos,
        condition = test, body = body,
William Stein's avatar
William Stein committed
1870 1871
        else_clause = else_clause)

1872 1873

def p_for_statement(s, is_async=False):
William Stein's avatar
William Stein committed
1874 1875 1876
    # s.sy == 'for'
    pos = s.position()
    s.next()
1877
    kw = p_for_bounds(s, allow_testlist=True, is_async=is_async)
1878 1879
    body = p_suite(s)
    else_clause = p_else_clause(s)
1880
    kw.update(body=body, else_clause=else_clause, is_async=is_async)
Robert Bradshaw's avatar
Robert Bradshaw committed
1881
    return Nodes.ForStatNode(pos, **kw)
1882

1883 1884

def p_for_bounds(s, allow_testlist=True, is_async=False):
William Stein's avatar
William Stein committed
1885 1886 1887
    target = p_for_target(s)
    if s.sy == 'in':
        s.next()
1888 1889 1890
        iterator = p_for_iterator(s, allow_testlist, is_async=is_async)
        return dict(target=target, iterator=iterator)
    elif not s.in_python_file and not is_async:
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
1891 1892 1893 1894 1895 1896
        if s.sy == 'from':
            s.next()
            bound1 = p_bit_expr(s)
        else:
            # Support shorter "for a <= x < b" syntax
            bound1, target = target, None
William Stein's avatar
William Stein committed
1897 1898 1899 1900 1901 1902
        rel1 = p_for_from_relation(s)
        name2_pos = s.position()
        name2 = p_ident(s)
        rel2_pos = s.position()
        rel2 = p_for_from_relation(s)
        bound2 = p_bit_expr(s)
1903
        step = p_for_from_step(s)
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
1904 1905 1906 1907
        if target is None:
            target = ExprNodes.NameNode(name2_pos, name = name2)
        else:
            if not target.is_name:
1908
                error(target.pos,
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
1909 1910 1911 1912
                    "Target of for-from statement must be a variable name")
            elif name2 != target.name:
                error(name2_pos,
                    "Variable name in for-from range does not match target")
Stefan Behnel's avatar
Stefan Behnel committed
1913
        if rel1[0] != rel2[0]:
William Stein's avatar
William Stein committed
1914 1915
            error(rel2_pos,
                "Relation directions in for-from do not match")
1916 1917 1918
        return dict(target = target,
                    bound1 = bound1,
                    relation1 = rel1,
1919 1920 1921 1922
                    relation2 = rel2,
                    bound2 = bound2,
                    step = step,
                    )
1923 1924 1925
    else:
        s.expect('in')
        return {}
William Stein's avatar
William Stein committed
1926 1927 1928 1929 1930 1931 1932 1933

def p_for_from_relation(s):
    if s.sy in inequality_relations:
        op = s.sy
        s.next()
        return op
    else:
        s.error("Expected one of '<', '<=', '>' '>='")
1934

1935
def p_for_from_step(s):
1936
    if s.sy == 'IDENT' and s.systring == 'by':
1937 1938 1939 1940 1941
        s.next()
        step = p_bit_expr(s)
        return step
    else:
        return None
William Stein's avatar
William Stein committed
1942

1943
inequality_relations = cython.declare(set, set(['<', '<=', '>', '>=']))
William Stein's avatar
William Stein committed
1944

1945
def p_target(s, terminator):
William Stein's avatar
William Stein committed
1946
    pos = s.position()
1947
    expr = p_starred_expr(s)
William Stein's avatar
William Stein committed
1948 1949 1950
    if s.sy == ',':
        s.next()
        exprs = [expr]
1951
        while s.sy != terminator:
1952
            exprs.append(p_starred_expr(s))
Stefan Behnel's avatar
Stefan Behnel committed
1953
            if s.sy != ',':
William Stein's avatar
William Stein committed
1954 1955 1956 1957 1958 1959
                break
            s.next()
        return ExprNodes.TupleNode(pos, args = exprs)
    else:
        return expr

1960

1961 1962 1963
def p_for_target(s):
    return p_target(s, 'in')

1964 1965

def p_for_iterator(s, allow_testlist=True, is_async=False):
William Stein's avatar
William Stein committed
1966
    pos = s.position()
1967 1968 1969 1970
    if allow_testlist:
        expr = p_testlist(s)
    else:
        expr = p_or_test(s)
1971 1972
    return (ExprNodes.AsyncIteratorNode if is_async else ExprNodes.IteratorNode)(pos, sequence=expr)

William Stein's avatar
William Stein committed
1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986

def p_try_statement(s):
    # s.sy == 'try'
    pos = s.position()
    s.next()
    body = p_suite(s)
    except_clauses = []
    else_clause = None
    if s.sy in ('except', 'else'):
        while s.sy == 'except':
            except_clauses.append(p_except_clause(s))
        if s.sy == 'else':
            s.next()
            else_clause = p_suite(s)
1987
        body = Nodes.TryExceptStatNode(pos,
William Stein's avatar
William Stein committed
1988 1989
            body = body, except_clauses = except_clauses,
            else_clause = else_clause)
1990 1991 1992 1993
        if s.sy != 'finally':
            return body
        # try-except-finally is equivalent to nested try-except/try-finally
    if s.sy == 'finally':
William Stein's avatar
William Stein committed
1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006
        s.next()
        finally_clause = p_suite(s)
        return Nodes.TryFinallyStatNode(pos,
            body = body, finally_clause = finally_clause)
    else:
        s.error("Expected 'except' or 'finally'")

def p_except_clause(s):
    # s.sy == 'except'
    pos = s.position()
    s.next()
    exc_type = None
    exc_value = None
2007
    is_except_as = False
Stefan Behnel's avatar
Stefan Behnel committed
2008
    if s.sy != ':':
2009
        exc_type = p_test(s)
2010 2011 2012 2013 2014
        # normalise into list of single exception tests
        if isinstance(exc_type, ExprNodes.TupleNode):
            exc_type = exc_type.args
        else:
            exc_type = [exc_type]
2015 2016
        if s.sy == ',' or (s.sy == 'IDENT' and s.systring == 'as'
                           and s.context.language_level == 2):
William Stein's avatar
William Stein committed
2017
            s.next()
2018
            exc_value = p_test(s)
2019
        elif s.sy == 'IDENT' and s.systring == 'as':
2020
            # Py3 syntax requires a name here
2021
            s.next()
2022 2023 2024
            pos2 = s.position()
            name = p_ident(s)
            exc_value = ExprNodes.NameNode(pos2, name = name)
2025
            is_except_as = True
William Stein's avatar
William Stein committed
2026 2027
    body = p_suite(s)
    return Nodes.ExceptClauseNode(pos,
2028 2029
        pattern = exc_type, target = exc_value,
        body = body, is_except_as=is_except_as)
William Stein's avatar
William Stein committed
2030

2031
def p_include_statement(s, ctx):
William Stein's avatar
William Stein committed
2032
    pos = s.position()
2033
    s.next()  # 'include'
2034
    unicode_include_file_name = p_string_literal(s, 'u')[2]
William Stein's avatar
William Stein committed
2035
    s.expect_newline("Syntax error in include statement")
2036
    if s.compile_time_eval:
2037
        include_file_name = unicode_include_file_name
2038 2039
        include_file_path = s.context.find_include_file(include_file_name, pos)
        if include_file_path:
2040
            s.included_files.append(include_file_name)
2041 2042 2043
            with Utils.open_source_file(include_file_path) as f:
                source_desc = FileSourceDescriptor(include_file_path)
                s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding, parse_comments=s.parse_comments)
2044
                tree = p_statement_list(s2, ctx)
2045 2046 2047
            return tree
        else:
            return None
William Stein's avatar
William Stein committed
2048
    else:
2049 2050
        return Nodes.PassStatNode(pos)

2051

2052
def p_with_statement(s):
2053
    s.next()  # 'with'
2054
    if s.systring == 'template' and not s.in_python_file:
2055 2056 2057 2058 2059
        node = p_with_template(s)
    else:
        node = p_with_items(s)
    return node

2060 2061

def p_with_items(s, is_async=False):
2062
    pos = s.position()
2063
    if not s.in_python_file and s.sy == 'IDENT' and s.systring in ('nogil', 'gil'):
2064 2065
        if is_async:
            s.error("with gil/nogil cannot be async")
2066 2067
        state = s.systring
        s.next()
2068 2069 2070 2071 2072 2073 2074 2075

        # support conditional gil/nogil
        condition = None
        if s.sy == '(':
            s.next()
            condition = p_test(s)
            s.expect(')')

2076
        if s.sy == ',':
Danilo Freitas's avatar
Danilo Freitas committed
2077
            s.next()
2078
            body = p_with_items(s)
Danilo Freitas's avatar
Danilo Freitas committed
2079
        else:
2080
            body = p_suite(s)
2081
        return Nodes.GILStatNode(pos, state=state, body=body, condition=condition)
2082
    elif not s.in_python_file and s.sy == 'IDENT' and s.systring in ('rlocked', 'wlocked'):
2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095
        state = s.systring
        s.next()

        if s.sy != 'IDENT':
            s.error("The with %s statement must be followed by the cypclass object to operate on" % state, pos)

        obj = p_starred_expr(s)
        if s.sy == ',':
            s.next()
            body = p_with_items(s, is_async=is_async)
        else:
            body = p_suite(s)
        return Nodes.LockCypclassNode(pos, state=state, obj=obj, body=body)
2096
    else:
2097
        manager = p_test(s)
2098 2099 2100
        target = None
        if s.sy == 'IDENT' and s.systring == 'as':
            s.next()
2101 2102 2103
            target = p_starred_expr(s)
        if s.sy == ',':
            s.next()
2104
            body = p_with_items(s, is_async=is_async)
2105 2106
        else:
            body = p_suite(s)
2107 2108
    return Nodes.WithStatNode(pos, manager=manager, target=target, body=body, is_async=is_async)

2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133

def p_with_template(s):
    pos = s.position()
    templates = []
    s.next()
    s.expect('[')
    templates.append(s.systring)
    s.next()
    while s.systring == ',':
        s.next()
        templates.append(s.systring)
        s.next()
    s.expect(']')
    if s.sy == ':':
        s.next()
        s.expect_newline("Syntax error in template function declaration")
        s.expect_indent()
        body_ctx = Ctx()
        body_ctx.templates = templates
        func_or_var = p_c_func_or_var_declaration(s, pos, body_ctx)
        s.expect_dedent()
        return func_or_var
    else:
        error(pos, "Syntax error in template function declaration")

Stefan Behnel's avatar
Stefan Behnel committed
2134
def p_simple_statement(s, first_statement = 0):
William Stein's avatar
William Stein committed
2135 2136 2137
    #print "p_simple_statement:", s.sy, s.systring ###
    if s.sy == 'global':
        node = p_global_statement(s)
2138 2139
    elif s.sy == 'nonlocal':
        node = p_nonlocal_statement(s)
William Stein's avatar
William Stein committed
2140 2141
    elif s.sy == 'print':
        node = p_print_statement(s)
2142 2143
    elif s.sy == 'exec':
        node = p_exec_statement(s)
William Stein's avatar
William Stein committed
2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156
    elif s.sy == 'del':
        node = p_del_statement(s)
    elif s.sy == 'break':
        node = p_break_statement(s)
    elif s.sy == 'continue':
        node = p_continue_statement(s)
    elif s.sy == 'return':
        node = p_return_statement(s)
    elif s.sy == 'raise':
        node = p_raise_statement(s)
    elif s.sy in ('import', 'cimport'):
        node = p_import_statement(s)
    elif s.sy == 'from':
Stefan Behnel's avatar
Stefan Behnel committed
2157
        node = p_from_import_statement(s, first_statement = first_statement)
2158
    elif s.sy == 'yield':
2159
        node = p_yield_statement(s)
William Stein's avatar
William Stein committed
2160 2161 2162 2163 2164 2165 2166 2167
    elif s.sy == 'assert':
        node = p_assert_statement(s)
    elif s.sy == 'pass':
        node = p_pass_statement(s)
    else:
        node = p_expression_or_assignment(s)
    return node

2168
def p_simple_statement_list(s, ctx, first_statement = 0):
William Stein's avatar
William Stein committed
2169 2170
    # Parse a series of simple statements on one line
    # separated by semicolons.
Stefan Behnel's avatar
Stefan Behnel committed
2171
    stat = p_simple_statement(s, first_statement = first_statement)
2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192
    pos = stat.pos
    stats = []
    if not isinstance(stat, Nodes.PassStatNode):
        stats.append(stat)
    while s.sy == ';':
        #print "p_simple_statement_list: maybe more to follow" ###
        s.next()
        if s.sy in ('NEWLINE', 'EOF'):
            break
        stat = p_simple_statement(s, first_statement = first_statement)
        if isinstance(stat, Nodes.PassStatNode):
            continue
        stats.append(stat)
        first_statement = False

    if not stats:
        stat = Nodes.PassStatNode(pos)
    elif len(stats) == 1:
        stat = stats[0]
    else:
        stat = Nodes.StatListNode(pos, stats = stats)
2193 2194 2195 2196 2197 2198

    if s.sy not in ('NEWLINE', 'EOF'):
        # provide a better error message for users who accidentally write Cython code in .py files
        if isinstance(stat, Nodes.ExprStatNode):
            if stat.expr.is_name and stat.expr.name == 'cdef':
                s.error("The 'cdef' keyword is only allowed in Cython files (pyx/pxi/pxd)", pos)
William Stein's avatar
William Stein committed
2199
    s.expect_newline("Syntax error in simple statement list")
2200

William Stein's avatar
William Stein committed
2201 2202
    return stat

2203 2204 2205
def p_compile_time_expr(s):
    old = s.compile_time_expr
    s.compile_time_expr = 1
2206
    expr = p_testlist(s)
2207 2208 2209 2210 2211 2212
    s.compile_time_expr = old
    return expr

def p_DEF_statement(s):
    pos = s.position()
    denv = s.compile_time_env
2213
    s.next()  # 'DEF'
2214 2215 2216
    name = p_ident(s)
    s.expect('=')
    expr = p_compile_time_expr(s)
2217 2218 2219 2220
    if s.compile_time_eval:
        value = expr.compile_time_value(denv)
        #print "p_DEF_statement: %s = %r" % (name, value) ###
        denv.declare(name, value)
2221
    s.expect_newline("Expected a newline", ignore_semicolon=True)
2222 2223
    return Nodes.PassStatNode(pos)

2224
def p_IF_statement(s, ctx):
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2225
    pos = s.position()
2226 2227 2228 2229 2230
    saved_eval = s.compile_time_eval
    current_eval = saved_eval
    denv = s.compile_time_env
    result = None
    while 1:
2231
        s.next()  # 'IF' or 'ELIF'
2232 2233
        expr = p_compile_time_expr(s)
        s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv))
2234
        body = p_suite(s, ctx)
2235 2236 2237
        if s.compile_time_eval:
            result = body
            current_eval = 0
Stefan Behnel's avatar
Stefan Behnel committed
2238
        if s.sy != 'ELIF':
2239 2240 2241 2242
            break
    if s.sy == 'ELSE':
        s.next()
        s.compile_time_eval = current_eval
2243
        body = p_suite(s, ctx)
2244 2245 2246
        if current_eval:
            result = body
    if not result:
Stefan Behnel's avatar
Stefan Behnel committed
2247
        result = Nodes.PassStatNode(pos)
2248 2249 2250
    s.compile_time_eval = saved_eval
    return result

2251 2252
def p_statement(s, ctx, first_statement = 0):
    cdef_flag = ctx.cdef_flag
Robert Bradshaw's avatar
Robert Bradshaw committed
2253
    decorators = None
William Stein's avatar
William Stein committed
2254
    if s.sy == 'ctypedef':
2255
        if ctx.level not in ('module', 'module_pxd'):
William Stein's avatar
William Stein committed
2256
            s.error("ctypedef statement not allowed here")
2257 2258
        #if ctx.api:
        #    error(s.position(), "'api' not allowed with 'ctypedef'")
2259
        return p_ctypedef_statement(s, ctx)
2260 2261 2262
    elif s.sy == 'DEF':
        return p_DEF_statement(s)
    elif s.sy == 'IF':
2263
        return p_IF_statement(s, ctx)
2264
    elif s.sy == '@':
2265
        if ctx.level not in ('module', 'class', 'c_class', 'function', 'property', 'module_pxd', 'c_class_pxd', 'other'):
2266 2267 2268
            s.error('decorator not allowed here')
        s.level = ctx.level
        decorators = p_decorators(s)
2269
        if not ctx.allow_struct_enum_decorator and s.sy not in ('def', 'cdef', 'cpdef', 'class', 'async'):
2270 2271 2272 2273
            if s.sy == 'IDENT' and s.systring == 'async':
                pass  # handled below
            else:
                s.error("Decorators can only be followed by functions or classes")
2274 2275
    elif s.sy == 'pass' and cdef_flag:
        # empty cdef block
2276
        return p_pass_statement(s, with_newline=1)
2277 2278

    overridable = 0
2279
    nogil_flag = ctx.nogil
2280 2281 2282
    if s.sy == 'cdef':
        cdef_flag = 1
        s.next()
2283
    elif s.sy == 'cpdef':
2284
        s.level = ctx.level
2285 2286 2287 2288
        cdef_flag = 1
        overridable = 1
        s.next()
    if cdef_flag:
2289
        if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):
2290 2291
            s.error('cdef statement not allowed here')
        s.level = ctx.level
2292
        node = p_cdef_statement(s, ctx(overridable=overridable, nogil=nogil_flag))
2293
        if decorators is not None:
2294
            tup = (Nodes.CFuncDefNode, Nodes.CVarDefNode, Nodes.CClassDefNode)
2295
            if ctx.allow_struct_enum_decorator:
2296
                tup += (Nodes.CStructOrUnionDefNode, Nodes.CEnumDefNode)
2297
            if not isinstance(node, tup):
2298
                s.error("Decorators can only be followed by functions or classes")
2299 2300
            node.decorators = decorators
        return node
William Stein's avatar
William Stein committed
2301
    else:
2302
        if ctx.api:
2303
            s.error("'api' not allowed with this statement", fatal=False)
2304
        elif s.sy == 'def':
2305 2306 2307
            # def statements aren't allowed in pxd files, except
            # as part of a cdef class
            if ('pxd' in ctx.level) and (ctx.level != 'c_class_pxd'):
2308
                s.error('def statement not allowed here')
2309
            s.level = ctx.level
2310 2311
            return p_def_statement(s, decorators)
        elif s.sy == 'class':
2312
            if ctx.level not in ('module', 'function', 'class', 'other'):
2313
                s.error("class definition not allowed here")
2314
            return p_class_statement(s, decorators)
2315 2316 2317 2318
        elif s.sy == 'include':
            if ctx.level not in ('module', 'module_pxd'):
                s.error("include statement not allowed here")
            return p_include_statement(s, ctx)
2319
        elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':
2320 2321
            return p_property_decl(s)
        elif s.sy == 'pass' and ctx.level != 'property':
2322
            return p_pass_statement(s, with_newline=True)
William Stein's avatar
William Stein committed
2323
        else:
2324
            if ctx.level in ('c_class_pxd', 'property'):
2325 2326 2327
                node = p_ignorable_statement(s)
                if node is not None:
                    return node
2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338
                s.error("Executable statement not allowed here")
            if s.sy == 'if':
                return p_if_statement(s)
            elif s.sy == 'while':
                return p_while_statement(s)
            elif s.sy == 'for':
                return p_for_statement(s)
            elif s.sy == 'try':
                return p_try_statement(s)
            elif s.sy == 'with':
                return p_with_statement(s)
2339 2340 2341
            elif s.sy == 'async':
                s.next()
                return p_async_statement(s, ctx, decorators)
2342
            else:
2343
                if s.sy == 'IDENT' and s.systring == 'async':
2344
                    ident_name = s.systring
2345 2346 2347
                    # PEP 492 enables the async/await keywords when it spots "async def ..."
                    s.next()
                    if s.sy == 'def':
2348
                        return p_async_statement(s, ctx, decorators)
2349 2350
                    elif decorators:
                        s.error("Decorators can only be followed by functions or classes")
2351
                    s.put_back('IDENT', ident_name)  # re-insert original token
2352 2353
                return p_simple_statement_list(s, ctx, first_statement=first_statement)

William Stein's avatar
William Stein committed
2354

2355
def p_statement_list(s, ctx, first_statement = 0):
William Stein's avatar
William Stein committed
2356 2357 2358 2359
    # Parse a series of statements separated by newlines.
    pos = s.position()
    stats = []
    while s.sy not in ('DEDENT', 'EOF'):
2360 2361 2362 2363 2364 2365 2366 2367
        stat = p_statement(s, ctx, first_statement = first_statement)
        if isinstance(stat, Nodes.PassStatNode):
            continue
        stats.append(stat)
        first_statement = False
    if not stats:
        return Nodes.PassStatNode(pos)
    elif len(stats) == 1:
2368 2369 2370
        return stats[0]
    else:
        return Nodes.StatListNode(pos, stats = stats)
William Stein's avatar
William Stein committed
2371

2372 2373 2374 2375 2376 2377

def p_suite(s, ctx=Ctx()):
    return p_suite_with_docstring(s, ctx, with_doc_only=False)[1]


def p_suite_with_docstring(s, ctx, with_doc_only=False):
William Stein's avatar
William Stein committed
2378 2379 2380 2381 2382
    s.expect(':')
    doc = None
    if s.sy == 'NEWLINE':
        s.next()
        s.expect_indent()
2383
        if with_doc_only:
2384
            doc = p_doc_string(s)
2385
        body = p_statement_list(s, ctx)
William Stein's avatar
William Stein committed
2386 2387
        s.expect_dedent()
    else:
2388
        if ctx.api:
2389
            s.error("'api' not allowed with this statement", fatal=False)
2390 2391
        if ctx.level in ('module', 'class', 'function', 'other'):
            body = p_simple_statement_list(s, ctx)
William Stein's avatar
William Stein committed
2392 2393
        else:
            body = p_pass_statement(s)
2394
            s.expect_newline("Syntax error in declarations", ignore_semicolon=True)
2395 2396 2397 2398
    if not with_doc_only:
        doc, body = _extract_docstring(body)
    return doc, body

William Stein's avatar
William Stein committed
2399

2400
def p_positional_and_keyword_args(s, end_sy_set, templates = None):
2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413
    """
    Parses positional and keyword arguments. end_sy_set
    should contain any s.sy that terminate the argument list.
    Argument expansion (* and **) are not allowed.

    Returns: (positional_args, keyword_args)
    """
    positional_args = []
    keyword_args = []
    pos_idx = 0

    while s.sy not in end_sy_set:
        if s.sy == '*' or s.sy == '**':
2414
            s.error('Argument expansion not allowed here.', fatal=False)
2415 2416

        parsed_type = False
2417
        if s.sy == 'IDENT' and s.peek()[0] == '=':
2418
            ident = s.systring
2419
            s.next()  # s.sy is '='
2420
            s.next()
2421
            if looking_at_expr(s):
2422
                arg = p_test(s)
2423 2424
            else:
                base_type = p_c_base_type(s, templates = templates)
2425
                declarator = p_c_declarator(s, empty = 1)
2426
                arg = Nodes.CComplexBaseTypeNode(base_type.pos,
2427 2428
                    base_type = base_type, declarator = declarator)
                parsed_type = True
2429
            keyword_node = ExprNodes.IdentifierStringNode(arg.pos, value=ident)
2430 2431
            keyword_args.append((keyword_node, arg))
            was_keyword = True
2432

2433
        else:
2434
            if looking_at_expr(s):
2435
                arg = p_test(s)
2436 2437
            else:
                base_type = p_c_base_type(s, templates = templates)
2438
                declarator = p_c_declarator(s, empty = 1)
2439
                arg = Nodes.CComplexBaseTypeNode(base_type.pos,
2440
                    base_type = base_type, declarator = declarator)
2441 2442 2443 2444 2445
                parsed_type = True
            positional_args.append(arg)
            pos_idx += 1
            if len(keyword_args) > 0:
                s.error("Non-keyword arg following keyword arg",
2446
                        pos=arg.pos)
2447 2448 2449 2450

        if s.sy != ',':
            if s.sy not in end_sy_set:
                if parsed_type:
2451
                    s.error("Unmatched %s" % " or ".join(end_sy_set))
2452 2453 2454 2455
            break
        s.next()
    return positional_args, keyword_args

Danilo Freitas's avatar
Danilo Freitas committed
2456
def p_c_base_type(s, self_flag = 0, nonempty = 0, templates = None):
William Stein's avatar
William Stein committed
2457 2458 2459
    # If self_flag is true, this is the base type for the
    # self argument of a C method of an extension type.
    if s.sy == '(':
2460
        return p_c_complex_base_type(s, templates = templates)
William Stein's avatar
William Stein committed
2461
    else:
Danilo Freitas's avatar
Danilo Freitas committed
2462
        return p_c_simple_base_type(s, self_flag, nonempty = nonempty, templates = templates)
William Stein's avatar
William Stein committed
2463

2464 2465 2466 2467 2468 2469 2470 2471
def p_calling_convention(s):
    if s.sy == 'IDENT' and s.systring in calling_convention_words:
        result = s.systring
        s.next()
        return result
    else:
        return ""

Stefan Behnel's avatar
Stefan Behnel committed
2472

2473 2474
calling_convention_words = cython.declare(
    set, set(["__stdcall", "__cdecl", "__fastcall"]))
2475

Stefan Behnel's avatar
Stefan Behnel committed
2476

2477
def p_c_complex_base_type(s, templates = None):
William Stein's avatar
William Stein committed
2478 2479 2480
    # s.sy == '('
    pos = s.position()
    s.next()
Stefan Behnel's avatar
Stefan Behnel committed
2481 2482 2483 2484
    base_type = p_c_base_type(s, templates=templates)
    declarator = p_c_declarator(s, empty=True)
    type_node = Nodes.CComplexBaseTypeNode(
        pos, base_type=base_type, declarator=declarator)
2485 2486 2487 2488 2489 2490
    if s.sy == ',':
        components = [type_node]
        while s.sy == ',':
            s.next()
            if s.sy == ')':
                break
Stefan Behnel's avatar
Stefan Behnel committed
2491 2492 2493 2494
            base_type = p_c_base_type(s, templates=templates)
            declarator = p_c_declarator(s, empty=True)
            components.append(Nodes.CComplexBaseTypeNode(
                pos, base_type=base_type, declarator=declarator))
2495 2496 2497
        type_node = Nodes.CTupleBaseTypeNode(pos, components = components)

    s.expect(')')
2498 2499 2500 2501 2502 2503
    if s.sy == '[':
        if is_memoryviewslice_access(s):
            type_node = p_memoryviewslice_access(s, type_node)
        else:
            type_node = p_buffer_or_template(s, type_node, templates)
    return type_node
2504

William Stein's avatar
William Stein committed
2505

Danilo Freitas's avatar
Danilo Freitas committed
2506
def p_c_simple_base_type(s, self_flag, nonempty, templates = None):
2507
    #print "p_c_simple_base_type: self_flag =", self_flag, nonempty
William Stein's avatar
William Stein committed
2508 2509 2510
    is_basic = 0
    signed = 1
    longness = 0
2511
    complex = 0
William Stein's avatar
William Stein committed
2512
    module_path = []
2513
    pos = s.position()
2514 2515 2516

    # Handle const/volatile
    is_const = is_volatile = 0
2517
    while s.sy == 'IDENT':
2518 2519 2520 2521 2522 2523 2524 2525
        if s.systring == 'const':
            if is_const: error(pos, "Duplicate 'const'")
            is_const = 1
        elif s.systring == 'volatile':
            if is_volatile: error(pos, "Duplicate 'volatile'")
            is_volatile = 1
        else:
            break
Robert Bradshaw's avatar
Robert Bradshaw committed
2526
        s.next()
2527
    if is_const or is_volatile:
2528 2529 2530
        base_type = p_c_base_type(s, self_flag=self_flag, nonempty=nonempty, templates=templates)
        if isinstance(base_type, Nodes.MemoryViewSliceTypeNode):
            # reverse order to avoid having to write "(const int)[:]"
2531 2532
            base_type.base_type_node = Nodes.CConstOrVolatileTypeNode(pos,
                base_type=base_type.base_type_node, is_const=is_const, is_volatile=is_volatile)
2533
            return base_type
2534 2535 2536
        return Nodes.CConstOrVolatileTypeNode(pos,
            base_type=base_type, is_const=is_const, is_volatile=is_volatile)

2537 2538 2539 2540 2541 2542 2543
    # Handle cypclass qualifiers
    if s.sy == 'IDENT' and s.systring in ('active',):
        qualifier = s.systring
        s.next()
        base_type = p_c_base_type(s, self_flag=self_flag, nonempty=nonempty, templates=templates)
        return Nodes.QualifiedCypclassNode(pos, base_type=base_type, qualifier=qualifier)

2544 2545
    if s.sy != 'IDENT':
        error(pos, "Expected an identifier, found '%s'" % s.sy)
William Stein's avatar
William Stein committed
2546 2547 2548
    if looking_at_base_type(s):
        #print "p_c_simple_base_type: looking_at_base_type at", s.position()
        is_basic = 1
2549 2550
        if s.sy == 'IDENT' and s.systring in special_basic_c_types:
            signed, longness = special_basic_c_types[s.systring]
William Stein's avatar
William Stein committed
2551 2552 2553
            name = s.systring
            s.next()
        else:
2554 2555 2556 2557 2558
            signed, longness = p_sign_and_longness(s)
            if s.sy == 'IDENT' and s.systring in basic_c_type_names:
                name = s.systring
                s.next()
            else:
Stefan Behnel's avatar
Stefan Behnel committed
2559
                name = 'int'  # long [int], short [int], long [int] complex, etc.
2560 2561 2562
        if s.sy == 'IDENT' and s.systring == 'complex':
            complex = 1
            s.next()
2563 2564
    elif looking_at_dotted_name(s):
        #print "p_c_simple_base_type: looking_at_type_name at", s.position()
2565
        name = s.systring
2566 2567 2568 2569 2570
        s.next()
        while s.sy == '.':
            module_path.append(name)
            s.next()
            name = p_ident(s)
2571
    else:
2572 2573 2574
        name = s.systring
        s.next()
        if nonempty and s.sy != 'IDENT':
2575
            # Make sure this is not a declaration of a variable or function.
2576 2577
            if s.sy == '(':
                s.next()
2578 2579
                if (s.sy == '*' or s.sy == '**' or s.sy == '&'
                        or (s.sy == 'IDENT' and s.systring in calling_convention_words)):
2580 2581 2582 2583 2584
                    s.put_back('(', '(')
                else:
                    s.put_back('(', '(')
                    s.put_back('IDENT', name)
                    name = None
Robert Bradshaw's avatar
Robert Bradshaw committed
2585
            elif s.sy not in ('*', '**', '[', '&'):
2586 2587
                s.put_back('IDENT', name)
                name = None
Danilo Freitas's avatar
Danilo Freitas committed
2588

2589
    type_node = Nodes.CSimpleBaseTypeNode(pos,
William Stein's avatar
William Stein committed
2590 2591
        name = name, module_path = module_path,
        is_basic_c_type = is_basic, signed = signed,
2592
        complex = complex, longness = longness,
Danilo Freitas's avatar
Danilo Freitas committed
2593
        is_self_arg = self_flag, templates = templates)
William Stein's avatar
William Stein committed
2594

2595
    #    declarations here.
2596
    if s.sy == '[':
2597
        if is_memoryviewslice_access(s):
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
2598
            type_node = p_memoryviewslice_access(s, type_node)
2599 2600
        else:
            type_node = p_buffer_or_template(s, type_node, templates)
2601

Robert Bradshaw's avatar
Robert Bradshaw committed
2602 2603 2604 2605
    if s.sy == '.':
        s.next()
        name = p_ident(s)
        type_node = Nodes.CNestedBaseTypeNode(pos, base_type = type_node, name = name)
2606

Robert Bradshaw's avatar
Robert Bradshaw committed
2607
    return type_node
2608

2609
def p_buffer_or_template(s, base_type_node, templates):
2610 2611 2612
    # s.sy == '['
    pos = s.position()
    s.next()
2613 2614
    # Note that buffer_positional_options_count=1, so the only positional argument is dtype.
    # For templated types, all parameters are types.
2615
    positional_args, keyword_args = (
2616
        p_positional_and_keyword_args(s, (']',), templates)
2617 2618
    )
    s.expect(']')
2619

2620 2621
    if s.sy == '[':
        base_type_node = p_buffer_or_template(s, base_type_node, templates)
2622 2623 2624 2625 2626 2627

    keyword_dict = ExprNodes.DictNode(pos,
        key_value_pairs = [
            ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)
            for key, value in keyword_args
        ])
2628
    result = Nodes.TemplatedTypeNode(pos,
2629 2630
        positional_args = positional_args,
        keyword_args = keyword_dict,
2631
        base_type_node = base_type_node)
2632
    return result
2633

2634 2635 2636 2637 2638 2639
def p_bracketed_base_type(s, base_type_node, nonempty, empty):
    # s.sy == '['
    if empty and not nonempty:
        # sizeof-like thing.  Only anonymous C arrays allowed (int[SIZE]).
        return base_type_node
    elif not empty and nonempty:
2640 2641 2642
        # declaration of either memoryview slice or buffer.
        if is_memoryviewslice_access(s):
            return p_memoryviewslice_access(s, base_type_node)
2643
        else:
Mark Florisson's avatar
Mark Florisson committed
2644 2645
            return p_buffer_or_template(s, base_type_node, None)
            # return p_buffer_access(s, base_type_node)
2646
    elif not empty and not nonempty:
2647 2648 2649 2650 2651
        # only anonymous C arrays and memoryview slice arrays here.  We
        # disallow buffer declarations for now, due to ambiguity with anonymous
        # C arrays.
        if is_memoryviewslice_access(s):
            return p_memoryviewslice_access(s, base_type_node)
2652 2653 2654
        else:
            return base_type_node

2655
def is_memoryviewslice_access(s):
2656
    # s.sy == '['
2657
    # a memoryview slice declaration is distinguishable from a buffer access
2658
    # declaration by the first entry in the bracketed list.  The buffer will
2659
    # not have an unnested colon in the first entry; the memoryview slice will.
2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670
    saved = [(s.sy, s.systring)]
    s.next()
    retval = False
    if s.systring == ':':
        retval = True
    elif s.sy == 'INT':
        saved.append((s.sy, s.systring))
        s.next()
        if s.sy == ':':
            retval = True

2671
    for sv in saved[::-1]:
2672 2673 2674 2675
        s.put_back(*sv)

    return retval

2676
def p_memoryviewslice_access(s, base_type_node):
2677 2678 2679
    # s.sy == '['
    pos = s.position()
    s.next()
2680
    subscripts, _ = p_subscript_list(s)
2681 2682 2683 2684 2685 2686
    # make sure each entry in subscripts is a slice
    for subscript in subscripts:
        if len(subscript) < 2:
            s.error("An axis specification in memoryview declaration does not have a ':'.")
    s.expect(']')
    indexes = make_slice_nodes(pos, subscripts)
2687
    result = Nodes.MemoryViewSliceTypeNode(pos,
2688 2689 2690
            base_type_node = base_type_node,
            axes = indexes)
    return result
2691

2692
def looking_at_name(s):
2693
    return s.sy == 'IDENT' and s.systring not in calling_convention_words
2694

2695
def looking_at_expr(s):
2696
    if s.systring in base_type_start_words:
2697
        return False
2698 2699 2700 2701 2702
    elif s.sy == 'IDENT':
        is_type = False
        name = s.systring
        dotted_path = []
        s.next()
2703

2704 2705 2706 2707
        while s.sy == '.':
            s.next()
            dotted_path.append(s.systring)
            s.expect('IDENT')
2708

2709
        saved = s.sy, s.systring
2710 2711 2712
        if s.sy == 'IDENT':
            is_type = True
        elif s.sy == '*' or s.sy == '**':
2713
            s.next()
2714
            is_type = s.sy in (')', ']')
2715 2716 2717 2718 2719 2720 2721 2722 2723
            s.put_back(*saved)
        elif s.sy == '(':
            s.next()
            is_type = s.sy == '*'
            s.put_back(*saved)
        elif s.sy == '[':
            s.next()
            is_type = s.sy == ']'
            s.put_back(*saved)
2724

2725 2726 2727 2728
        dotted_path.reverse()
        for p in dotted_path:
            s.put_back('IDENT', p)
            s.put_back('.', '.')
2729

2730
        s.put_back('IDENT', name)
2731
        return not is_type and saved[0]
2732
    else:
2733
        return True
William Stein's avatar
William Stein committed
2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747

def looking_at_base_type(s):
    #print "looking_at_base_type?", s.sy, s.systring, s.position()
    return s.sy == 'IDENT' and s.systring in base_type_start_words

def looking_at_dotted_name(s):
    if s.sy == 'IDENT':
        name = s.systring
        s.next()
        result = s.sy == '.'
        s.put_back('IDENT', name)
        return result
    else:
        return 0
2748

2749 2750 2751 2752 2753 2754
def looking_at_call(s):
    "See if we're looking at a.b.c("
    # Don't mess up the original position, so save and restore it.
    # Unfortunately there's no good way to handle this, as a subsequent call
    # to next() will not advance the position until it reads a new token.
    position = s.start_line, s.start_col
Mark Florisson's avatar
Mark Florisson committed
2755
    result = looking_at_expr(s) == u'('
2756 2757 2758 2759
    if not result:
        s.start_line, s.start_col = position
    return result

2760 2761
basic_c_type_names = cython.declare(
    set, set(["void", "char", "int", "float", "double", "bint"]))
2762

2763
special_basic_c_types = cython.declare(dict, {
2764
    # name : (signed, longness)
2765
    "Py_UNICODE" : (0, 0),
Stefan Behnel's avatar
Stefan Behnel committed
2766
    "Py_UCS4"    : (0, 0),
Stefan Behnel's avatar
Stefan Behnel committed
2767
    "Py_hash_t"  : (2, 0),
2768
    "Py_ssize_t" : (2, 0),
2769
    "ssize_t"    : (2, 0),
2770
    "size_t"     : (0, 0),
2771
    "ptrdiff_t"  : (2, 0),
2772
    "Py_tss_t"   : (1, 0),
2773
})
William Stein's avatar
William Stein committed
2774

2775 2776
sign_and_longness_words = cython.declare(
    set, set(["short", "long", "signed", "unsigned"]))
William Stein's avatar
William Stein committed
2777

2778 2779 2780 2781 2782
base_type_start_words = cython.declare(
    set,
    basic_c_type_names
    | sign_and_longness_words
    | set(special_basic_c_types))
William Stein's avatar
William Stein committed
2783

2784 2785
struct_enum_union = cython.declare(
    set, set(["struct", "union", "enum", "packed"]))
Mark Florisson's avatar
Mark Florisson committed
2786

William Stein's avatar
William Stein committed
2787 2788 2789 2790 2791 2792
def p_sign_and_longness(s):
    signed = 1
    longness = 0
    while s.sy == 'IDENT' and s.systring in sign_and_longness_words:
        if s.systring == 'unsigned':
            signed = 0
2793 2794
        elif s.systring == 'signed':
            signed = 2
William Stein's avatar
William Stein committed
2795 2796 2797 2798 2799 2800 2801 2802
        elif s.systring == 'short':
            longness = -1
        elif s.systring == 'long':
            longness += 1
        s.next()
    return signed, longness

def p_opt_cname(s):
2803 2804 2805
    literal = p_opt_string_literal(s, 'u')
    if literal is not None:
        cname = EncodedString(literal)
Stefan Behnel's avatar
Stefan Behnel committed
2806
        cname.encoding = s.source_encoding
William Stein's avatar
William Stein committed
2807 2808 2809 2810
    else:
        cname = None
    return cname

2811 2812 2813
def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0,
                   assignable = 0, nonempty = 0,
                   calling_convention_allowed = 0):
2814 2815
    # If empty is true, the declarator must be empty. If nonempty is true,
    # the declarator must be nonempty. Otherwise we don't care.
William Stein's avatar
William Stein committed
2816 2817 2818
    # If cmethod_flag is true, then if this declarator declares
    # a function, it's a C method of an extension type.
    pos = s.position()
2819 2820
    if s.sy == '(':
        s.next()
2821
        if s.sy == ')' or looking_at_name(s):
2822
            base = Nodes.CNameDeclaratorNode(pos, name=s.context.intern_ustring(u""), cname=None)
2823
            result = p_c_func_declarator(s, pos, ctx, base, cmethod_flag)
2824
        else:
2825 2826 2827 2828
            result = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
                                    cmethod_flag = cmethod_flag,
                                    nonempty = nonempty,
                                    calling_convention_allowed = 1)
2829 2830
            s.expect(')')
    else:
2831 2832
        result = p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
                                       assignable, nonempty)
Stefan Behnel's avatar
Stefan Behnel committed
2833
    if not calling_convention_allowed and result.calling_convention and s.sy != '(':
2834 2835 2836 2837 2838 2839
        error(s.position(), "%s on something that is not a function"
            % result.calling_convention)
    while s.sy in ('[', '('):
        pos = s.position()
        if s.sy == '[':
            result = p_c_array_declarator(s, result)
2840
        else:  # sy == '('
2841
            s.next()
2842
            result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag)
2843 2844 2845 2846 2847
        cmethod_flag = 0
    return result

def p_c_array_declarator(s, base):
    pos = s.position()
2848
    s.next()  # '['
Stefan Behnel's avatar
Stefan Behnel committed
2849
    if s.sy != ']':
2850
        dim = p_testlist(s)
2851 2852 2853 2854 2855
    else:
        dim = None
    s.expect(']')
    return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim)

2856 2857 2858 2859 2860 2861
def p_cpp_const_method(s, ctx):
    if s.sy == 'IDENT' and s.systring == 'const' and ctx.level == 'cpp_class':
        s.next()
        return 1
    return 0

root's avatar
root committed
2862
def p_c_func_declarator(s, pos, ctx, base, cmethod_flag):
2863
    # Opening paren has already been skipped
2864 2865
    args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag,
                        nonempty_declarators = 0)
2866 2867 2868 2869 2870
    ellipsis = p_optional_ellipsis(s)
    s.expect(')')
    nogil = p_nogil(s)
    exc_val, exc_check = p_exception_value_clause(s)
    with_gil = p_with_gil(s)
2871
    is_const_method = p_cpp_const_method(s, ctx)
2872 2873
    if 'mutable' in ctx.modifiers:
        s.error("Functions cannot be 'mutable'")
2874
    return Nodes.CFuncDeclaratorNode(pos,
2875 2876
        base = base, args = args, has_varargs = ellipsis,
        exception_value = exc_val, exception_check = exc_check,
2877 2878
        nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil,
        is_const_method = is_const_method)
2879

2880
supported_overloaded_operators = cython.declare(set, set([
2881
    '+', '-', '*', '/', '%',
Robert Bradshaw's avatar
Robert Bradshaw committed
2882
    '++', '--', '~', '|', '&', '^', '<<', '>>', ',',
Robert Bradshaw's avatar
Robert Bradshaw committed
2883
    '==', '!=', '>=', '>', '<=', '<',
2884
    '[]', '()', '!', '=',
2885
    '+=', '-=', '*=', '/=', '%=', '|=', '&=', '<<=', '>>=',
2886
    'bool',
2887
]))
2888

2889 2890
def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
                          assignable, nonempty):
2891 2892
    pos = s.position()
    calling_convention = p_calling_convention(s)
William Stein's avatar
William Stein committed
2893 2894
    if s.sy == '*':
        s.next()
2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907
        if s.systring == 'const':
            const_pos = s.position()
            s.next()
            const_base = p_c_declarator(s, ctx, empty = empty,
                                       is_type = is_type,
                                       cmethod_flag = cmethod_flag,
                                       assignable = assignable,
                                       nonempty = nonempty)
            base = Nodes.CConstDeclaratorNode(const_pos, base = const_base)
        else:
            base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
                                  cmethod_flag = cmethod_flag,
                                  assignable = assignable, nonempty = nonempty)
2908
        result = Nodes.CPtrDeclaratorNode(pos,
William Stein's avatar
William Stein committed
2909
            base = base)
2910
    elif s.sy == '**':  # scanner returns this as a single token
William Stein's avatar
William Stein committed
2911
        s.next()
2912 2913 2914
        base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
                              cmethod_flag = cmethod_flag,
                              assignable = assignable, nonempty = nonempty)
William Stein's avatar
William Stein committed
2915 2916 2917
        result = Nodes.CPtrDeclaratorNode(pos,
            base = Nodes.CPtrDeclaratorNode(pos,
                base = base))
Robert Bradshaw's avatar
merge  
Robert Bradshaw committed
2918
    elif s.sy == '&':
2919 2920 2921 2922 2923
        s.next()
        base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
                              cmethod_flag = cmethod_flag,
                              assignable = assignable, nonempty = nonempty)
        result = Nodes.CReferenceDeclaratorNode(pos, base = base)
William Stein's avatar
William Stein committed
2924
    else:
2925 2926
        rhs = None
        if s.sy == 'IDENT':
2927
            name = s.systring
2928 2929
            if empty:
                error(s.position(), "Declarator should be empty")
William Stein's avatar
William Stein committed
2930
            s.next()
2931
            cname = p_opt_cname(s)
2932
            if name != 'operator' and s.sy == '=' and assignable:
2933
                s.next()
2934
                rhs = p_test(s)
William Stein's avatar
William Stein committed
2935
        else:
2936 2937 2938 2939
            if nonempty:
                error(s.position(), "Empty declarator")
            name = ""
            cname = None
2940
        if cname is None and ctx.namespace is not None and nonempty:
2941
            cname = ctx.namespace + "::" + name
2942
        if name == 'operator' and (ctx.visibility == 'extern' or ctx.level == 'cpp_class') and nonempty:
2943
            op = s.sy
2944
            if [1 for c in op if c in '+-*/<=>!%&|([^~,']:
2945
                s.next()
2946 2947 2948 2949 2950 2951 2952
                # Handle diphthong operators.
                if op == '(':
                    s.expect(')')
                    op = '()'
                elif op == '[':
                    s.expect(']')
                    op = '[]'
Stefan Behnel's avatar
Stefan Behnel committed
2953 2954
                elif op in ('-', '+', '|', '&') and s.sy == op:
                    op *= 2       # ++, --, ...
2955
                    s.next()
Stefan Behnel's avatar
Stefan Behnel committed
2956 2957
                elif s.sy == '=':
                    op += s.sy    # +=, -=, ...
2958 2959
                    s.next()
                if op not in supported_overloaded_operators:
2960 2961 2962
                    s.error("Overloading operator '%s' not yet supported." % op,
                            fatal=False)
                name += op
2963
            elif op == 'IDENT':
2964
                op = s.systring
2965 2966 2967 2968 2969
                if op not in supported_overloaded_operators:
                    s.error("Overloading operator '%s' not yet supported." % op,
                            fatal=False)
                name = name + ' ' + op
                s.next()
2970
        result = Nodes.CNameDeclaratorNode(pos,
2971
            name = EncodedString(name), cname = cname, default = rhs)
2972
    result.calling_convention = calling_convention
William Stein's avatar
William Stein committed
2973 2974
    return result

2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989
def p_nogil(s):
    if s.sy == 'IDENT' and s.systring == 'nogil':
        s.next()
        return 1
    else:
        return 0

def p_with_gil(s):
    if s.sy == 'with':
        s.next()
        s.expect_keyword('gil')
        return 1
    else:
        return 0

William Stein's avatar
William Stein committed
2990 2991 2992 2993 2994 2995 2996 2997
def p_exception_value_clause(s):
    exc_val = None
    exc_check = 0
    if s.sy == 'except':
        s.next()
        if s.sy == '*':
            exc_check = 1
            s.next()
Felix Wu's avatar
Felix Wu committed
2998 2999 3000
        elif s.sy == '+':
            exc_check = '+'
            s.next()
Robert Bradshaw's avatar
Robert Bradshaw committed
3001 3002 3003 3004
            if s.sy == 'IDENT':
                name = s.systring
                s.next()
                exc_val = p_name(s, name)
3005 3006 3007
            elif s.sy == '*':
                exc_val = ExprNodes.CharNode(s.position(), value=u'*')
                s.next()
3008 3009 3010
        elif s.sy == '~':
            exc_check = '~'
            s.next()
William Stein's avatar
William Stein committed
3011 3012 3013 3014
        else:
            if s.sy == '?':
                exc_check = 1
                s.next()
3015
            exc_val = p_test(s)
William Stein's avatar
William Stein committed
3016 3017
    return exc_val, exc_check

3018
c_arg_list_terminators = cython.declare(set, set(['*', '**', '.', ')', ':', '/']))
William Stein's avatar
William Stein committed
3019

3020
def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0,
3021
                 nonempty_declarators = 0, kw_only = 0, annotated = 1):
3022 3023
    #  Comma-separated list of C argument declarations, possibly empty.
    #  May have a trailing comma.
William Stein's avatar
William Stein committed
3024
    args = []
3025 3026
    is_self_arg = cmethod_flag
    while s.sy not in c_arg_list_terminators:
3027
        args.append(p_c_arg_decl(s, ctx, in_pyfunc, is_self_arg,
3028 3029
            nonempty = nonempty_declarators, kw_only = kw_only,
            annotated = annotated))
3030 3031 3032 3033
        if s.sy != ',':
            break
        s.next()
        is_self_arg = 0
William Stein's avatar
William Stein committed
3034 3035 3036 3037 3038 3039 3040 3041 3042
    return args

def p_optional_ellipsis(s):
    if s.sy == '.':
        expect_ellipsis(s)
        return 1
    else:
        return 0

3043 3044
def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0,
                 kw_only = 0, annotated = 1):
William Stein's avatar
William Stein committed
3045
    pos = s.position()
3046
    not_none = or_none = 0
William Stein's avatar
William Stein committed
3047
    default = None
3048
    annotation = None
3049 3050 3051 3052 3053 3054 3055 3056 3057
    if s.in_python_file:
        # empty type declaration
        base_type = Nodes.CSimpleBaseTypeNode(pos,
            name = None, module_path = [],
            is_basic_c_type = 0, signed = 0,
            complex = 0, longness = 0,
            is_self_arg = cmethod_flag, templates = None)
    else:
        base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty)
3058
    declarator = p_c_declarator(s, ctx, nonempty = nonempty)
3059 3060
    if s.sy in ('not', 'or') and not s.in_python_file:
        kind = s.sy
William Stein's avatar
William Stein committed
3061 3062 3063 3064 3065 3066
        s.next()
        if s.sy == 'IDENT' and s.systring == 'None':
            s.next()
        else:
            s.error("Expected 'None'")
        if not in_pyfunc:
3067 3068 3069
            error(pos, "'%s None' only allowed in Python functions" % kind)
        or_none = kind == 'or'
        not_none = kind == 'not'
3070 3071
    if annotated and s.sy == ':':
        s.next()
3072
        annotation = p_annotation(s)
William Stein's avatar
William Stein committed
3073 3074
    if s.sy == '=':
        s.next()
3075 3076 3077 3078 3079 3080 3081 3082
        if 'pxd' in ctx.level:
            if s.sy in ['*', '?']:
                # TODO(github/1736): Make this an error for inline declarations.
                default = ExprNodes.NoneNode(pos)
                s.next()
            elif 'inline' in ctx.modifiers:
                default = p_test(s)
            else:
3083 3084
                error(pos, "default values cannot be specified in pxd files, use ? or *")
        else:
3085
            default = p_test(s)
William Stein's avatar
William Stein committed
3086 3087 3088 3089
    return Nodes.CArgDeclNode(pos,
        base_type = base_type,
        declarator = declarator,
        not_none = not_none,
3090
        or_none = or_none,
3091
        default = default,
3092
        annotation = annotation,
3093
        kw_only = kw_only)
William Stein's avatar
William Stein committed
3094

3095 3096 3097 3098 3099 3100 3101
def p_api(s):
    if s.sy == 'IDENT' and s.systring == 'api':
        s.next()
        return 1
    else:
        return 0

3102
def p_cdef_statement(s, ctx):
William Stein's avatar
William Stein committed
3103
    pos = s.position()
3104 3105 3106 3107 3108 3109 3110
    ctx.visibility = p_visibility(s, ctx.visibility)
    ctx.api = ctx.api or p_api(s)
    if ctx.api:
        if ctx.visibility not in ('private', 'public'):
            error(pos, "Cannot combine 'api' with '%s'" % ctx.visibility)
    if (ctx.visibility == 'extern') and s.sy == 'from':
        return p_cdef_extern_block(s, pos, ctx)
Robert Bradshaw's avatar
Robert Bradshaw committed
3111 3112
    elif s.sy == 'import':
        s.next()
3113
        return p_cdef_extern_block(s, pos, ctx)
3114
    elif p_nogil(s):
3115
        ctx.nogil = 1
3116 3117
        if ctx.overridable:
            error(pos, "cdef blocks cannot be declared cpdef")
3118
        return p_cdef_block(s, ctx)
3119 3120
    elif ctx.overridable and ctx.nogil:
        error(pos, "nogil blocks cannot be declared cpdef")
3121 3122 3123
    elif s.sy == ':':
        if ctx.overridable:
            error(pos, "cdef blocks cannot be declared cpdef")
3124
        return p_cdef_block(s, ctx)
William Stein's avatar
William Stein committed
3125
    elif s.sy == 'class':
3126
        if ctx.level not in ('module', 'module_pxd'):
William Stein's avatar
William Stein committed
3127
            error(pos, "Extension type definition not allowed here")
3128 3129
        if ctx.overridable:
            error(pos, "Extension types cannot be declared cpdef")
3130
        return p_c_class_definition(s, pos, ctx)
3131
    elif s.sy == 'IDENT' and s.systring in ('cppclass', 'cypclass'):
Robert Bradshaw's avatar
Robert Bradshaw committed
3132
        return p_cpp_class_definition(s, pos, ctx)
Mark Florisson's avatar
Mark Florisson committed
3133
    elif s.sy == 'IDENT' and s.systring in struct_enum_union:
3134
        if ctx.level not in ('module', 'module_pxd'):
William Stein's avatar
William Stein committed
3135
            error(pos, "C struct/union/enum definition not allowed here")
3136
        if ctx.overridable:
3137 3138
            if s.systring != 'enum':
                error(pos, "C struct/union cannot be declared cpdef")
Mark Florisson's avatar
Mark Florisson committed
3139 3140 3141
        return p_struct_enum(s, pos, ctx)
    elif s.sy == 'IDENT' and s.systring == 'fused':
        return p_fused_definition(s, pos, ctx)
William Stein's avatar
William Stein committed
3142
    else:
3143
        return p_c_func_or_var_declaration(s, pos, ctx)
3144

3145 3146
def p_cdef_block(s, ctx):
    return p_suite(s, ctx(cdef_flag = 1))
William Stein's avatar
William Stein committed
3147

3148
def p_cdef_extern_block(s, pos, ctx):
3149 3150
    if ctx.overridable:
        error(pos, "cdef extern blocks cannot be declared cpdef")
William Stein's avatar
William Stein committed
3151 3152 3153 3154 3155
    include_file = None
    s.expect('from')
    if s.sy == '*':
        s.next()
    else:
3156
        include_file = p_string_literal(s, 'u')[2]
3157
    ctx = ctx(cdef_flag = 1, visibility = 'extern')
3158 3159
    if s.systring == "namespace":
        s.next()
3160
        ctx.namespace = p_string_literal(s, 'u')[2]
3161 3162
    if p_nogil(s):
        ctx.nogil = 1
3163 3164 3165 3166

    # Use "docstring" as verbatim string to include
    verbatim_include, body = p_suite_with_docstring(s, ctx, True)

William Stein's avatar
William Stein committed
3167 3168
    return Nodes.CDefExternNode(pos,
        include_file = include_file,
3169
        verbatim_include = verbatim_include,
3170
        body = body,
Robert Bradshaw's avatar
Robert Bradshaw committed
3171
        namespace = ctx.namespace)
William Stein's avatar
William Stein committed
3172

3173
def p_c_enum_definition(s, pos, ctx):
William Stein's avatar
William Stein committed
3174 3175
    # s.sy == ident 'enum'
    s.next()
3176 3177 3178 3179 3180 3181

    scoped = False
    if s.context.cpp and (s.sy == 'class' or (s.sy == 'IDENT' and s.systring == 'struct')):
        scoped = True
        s.next()

William Stein's avatar
William Stein committed
3182 3183 3184 3185
    if s.sy == 'IDENT':
        name = s.systring
        s.next()
        cname = p_opt_cname(s)
3186 3187
        if cname is None and ctx.namespace is not None:
            cname = ctx.namespace + "::" + name
William Stein's avatar
William Stein committed
3188
    else:
3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207
        name = cname = None
        if scoped:
            s.error("Unnamed scoped enum not allowed")

    if scoped and s.sy == '(':
        s.next()
        underlying_type = p_c_base_type(s)
        s.expect(')')
    else:
        underlying_type = Nodes.CSimpleBaseTypeNode(
            pos,
            name="int",
            module_path = [],
            is_basic_c_type = True,
            signed = 1,
            complex = 0,
            longness = 0
        )

William Stein's avatar
William Stein committed
3208 3209
    s.expect(':')
    items = []
3210

Stefan Behnel's avatar
Stefan Behnel committed
3211
    if s.sy != 'NEWLINE':
3212
        p_c_enum_line(s, ctx, items)
William Stein's avatar
William Stein committed
3213
    else:
3214
        s.next()  # 'NEWLINE'
William Stein's avatar
William Stein committed
3215
        s.expect_indent()
3216

William Stein's avatar
William Stein committed
3217
        while s.sy not in ('DEDENT', 'EOF'):
3218
            p_c_enum_line(s, ctx, items)
3219

William Stein's avatar
William Stein committed
3220
        s.expect_dedent()
3221 3222 3223 3224

    if not items and ctx.visibility != "extern":
        error(pos, "Empty enum definition not allowed outside a 'cdef extern from' block")

3225
    return Nodes.CEnumDefNode(
3226 3227 3228 3229 3230 3231
        pos, name=name, cname=cname,
        scoped=scoped, items=items,
        underlying_type=underlying_type,
        typedef_flag=ctx.typedef_flag, visibility=ctx.visibility,
        create_wrapper=ctx.overridable,
        api=ctx.api, in_pxd=ctx.level == 'module_pxd')
William Stein's avatar
William Stein committed
3232

3233
def p_c_enum_line(s, ctx, items):
Stefan Behnel's avatar
Stefan Behnel committed
3234
    if s.sy != 'pass':
3235
        p_c_enum_item(s, ctx, items)
William Stein's avatar
William Stein committed
3236 3237 3238 3239
        while s.sy == ',':
            s.next()
            if s.sy in ('NEWLINE', 'EOF'):
                break
3240
            p_c_enum_item(s, ctx, items)
William Stein's avatar
William Stein committed
3241 3242 3243 3244
    else:
        s.next()
    s.expect_newline("Syntax error in enum item list")

3245
def p_c_enum_item(s, ctx, items):
William Stein's avatar
William Stein committed
3246 3247 3248
    pos = s.position()
    name = p_ident(s)
    cname = p_opt_cname(s)
3249 3250
    if cname is None and ctx.namespace is not None:
        cname = ctx.namespace + "::" + name
William Stein's avatar
William Stein committed
3251 3252 3253
    value = None
    if s.sy == '=':
        s.next()
3254
        value = p_test(s)
3255
    items.append(Nodes.CEnumDefItemNode(pos,
William Stein's avatar
William Stein committed
3256 3257
        name = name, cname = cname, value = value))

3258
def p_c_struct_or_union_definition(s, pos, ctx):
3259 3260 3261 3262
    packed = False
    if s.systring == 'packed':
        packed = True
        s.next()
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
3263
        if s.sy != 'IDENT' or s.systring != 'struct':
3264
            s.expected('struct')
William Stein's avatar
William Stein committed
3265 3266 3267 3268 3269
    # s.sy == ident 'struct' or 'union'
    kind = s.systring
    s.next()
    name = p_ident(s)
    cname = p_opt_cname(s)
3270 3271
    if cname is None and ctx.namespace is not None:
        cname = ctx.namespace + "::" + name
William Stein's avatar
William Stein committed
3272 3273 3274 3275
    attributes = None
    if s.sy == ':':
        s.next()
        attributes = []
3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290
        if s.sy == 'pass':
            s.next()
            s.expect_newline("Expected a newline", ignore_semicolon=True)
        else:
            s.expect('NEWLINE')
            s.expect_indent()
            body_ctx = Ctx()
            while s.sy != 'DEDENT':
                if s.sy != 'pass':
                    attributes.append(
                        p_c_func_or_var_declaration(s, s.position(), body_ctx))
                else:
                    s.next()
                    s.expect_newline("Expected a newline")
            s.expect_dedent()
3291 3292 3293

        if not attributes and ctx.visibility != "extern":
            error(pos, "Empty struct or union definition not allowed outside a 'cdef extern from' block")
William Stein's avatar
William Stein committed
3294 3295
    else:
        s.expect_newline("Syntax error in struct or union definition")
3296

Robert Bradshaw's avatar
Robert Bradshaw committed
3297
    return Nodes.CStructOrUnionDefNode(pos,
William Stein's avatar
William Stein committed
3298
        name = name, cname = cname, kind = kind, attributes = attributes,
3299
        typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,
3300
        api = ctx.api, in_pxd = ctx.level == 'module_pxd', packed = packed)
William Stein's avatar
William Stein committed
3301

Mark Florisson's avatar
Mark Florisson committed
3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322
def p_fused_definition(s, pos, ctx):
    """
    c(type)def fused my_fused_type:
        ...
    """
    # s.systring == 'fused'

    if ctx.level not in ('module', 'module_pxd'):
        error(pos, "Fused type definition not allowed here")

    s.next()
    name = p_ident(s)

    s.expect(":")
    s.expect_newline()
    s.expect_indent()

    types = []
    while s.sy != 'DEDENT':
        if s.sy != 'pass':
            #types.append(p_c_declarator(s))
3323
            types.append(p_c_base_type(s))  #, nonempty=1))
Mark Florisson's avatar
Mark Florisson committed
3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341
        else:
            s.next()

        s.expect_newline()

    s.expect_dedent()

    if not types:
        error(pos, "Need at least one type")

    return Nodes.FusedTypeNode(pos, name=name, types=types)

def p_struct_enum(s, pos, ctx):
    if s.systring == 'enum':
        return p_c_enum_definition(s, pos, ctx)
    else:
        return p_c_struct_or_union_definition(s, pos, ctx)

William Stein's avatar
William Stein committed
3342 3343 3344 3345 3346
def p_visibility(s, prev_visibility):
    pos = s.position()
    visibility = prev_visibility
    if s.sy == 'IDENT' and s.systring in ('extern', 'public', 'readonly'):
        visibility = s.systring
Stefan Behnel's avatar
Stefan Behnel committed
3347
        if prev_visibility != 'private' and visibility != prev_visibility:
William Stein's avatar
William Stein committed
3348
            s.error("Conflicting visibility options '%s' and '%s'"
3349
                % (prev_visibility, visibility), fatal=False)
William Stein's avatar
William Stein committed
3350 3351
        s.next()
    return visibility
3352

3353
def p_c_modifiers(s):
3354
    if s.sy == 'IDENT' and s.systring in ('inline', 'mutable'):
3355
        modifier = s.systring
3356
        s.next()
3357 3358
        return [modifier] + p_c_modifiers(s)
    return []
William Stein's avatar
William Stein committed
3359

3360
def p_c_func_or_var_declaration(s, pos, ctx):
3361
    cmethod_flag = ctx.level in ('c_class', 'c_class_pxd')
3362
    modifiers = p_c_modifiers(s)
Danilo Freitas's avatar
Danilo Freitas committed
3363
    base_type = p_c_base_type(s, nonempty = 1, templates = ctx.templates)
3364
    declarator = p_c_declarator(s, ctx(modifiers=modifiers), cmethod_flag = cmethod_flag,
3365 3366
                                assignable = 1, nonempty = 1)
    declarator.overridable = ctx.overridable
3367 3368 3369 3370 3371 3372 3373 3374
    if s.sy == '->':
        # Special enough to give a better error message and keep going.
        s.error(
            "Return type annotation is not allowed in cdef/cpdef signatures. "
            "Please define it before the function name, as in C signatures.",
            fatal=False)
        s.next()
        p_test(s)  # Keep going, but ignore result.
William Stein's avatar
William Stein committed
3375
    if s.sy == ':':
3376
        if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd', 'cpp_class') and not ctx.templates:
William Stein's avatar
William Stein committed
3377
            s.error("C function definition not allowed here")
3378
        doc, suite = p_suite_with_docstring(s, Ctx(level='function'))
3379 3380
        if 'mutable' in modifiers:
            s.error("Functions cannot be 'mutable'")
William Stein's avatar
William Stein committed
3381
        result = Nodes.CFuncDefNode(pos,
3382
            visibility = ctx.visibility,
William Stein's avatar
William Stein committed
3383
            base_type = base_type,
3384
            declarator = declarator,
3385
            body = suite,
3386
            doc = doc,
3387
            modifiers = modifiers,
3388
            api = ctx.api,
3389
            overridable = ctx.overridable)
William Stein's avatar
William Stein committed
3390
    else:
Stefan Behnel's avatar
Stefan Behnel committed
3391
        #if api:
3392
        #    s.error("'api' not allowed with variable declaration")
William Stein's avatar
William Stein committed
3393 3394 3395 3396 3397
        declarators = [declarator]
        while s.sy == ',':
            s.next()
            if s.sy == 'NEWLINE':
                break
3398 3399
            declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,
                                        assignable = 1, nonempty = 1)
William Stein's avatar
William Stein committed
3400
            declarators.append(declarator)
3401
        doc_line = s.start_line + 1
3402
        s.expect_newline("Syntax error in C variable declaration", ignore_semicolon=True)
3403
        if ctx.level in ('c_class', 'c_class_pxd') and s.start_line == doc_line:
3404 3405 3406
            doc = p_doc_string(s)
        else:
            doc = None
3407
        result = Nodes.CVarDefNode(pos,
3408 3409
            visibility = ctx.visibility,
            base_type = base_type,
3410
            declarators = declarators,
3411
            in_pxd = ctx.level in ('module_pxd', 'c_class_pxd'),
3412
            doc = doc,
3413
            api = ctx.api,
3414
            modifiers = modifiers,
3415
            overridable = ctx.overridable)
William Stein's avatar
William Stein committed
3416 3417
    return result

3418
def p_ctypedef_statement(s, ctx):
William Stein's avatar
William Stein committed
3419 3420 3421
    # s.sy == 'ctypedef'
    pos = s.position()
    s.next()
3422
    visibility = p_visibility(s, ctx.visibility)
3423
    api = p_api(s)
3424
    ctx = ctx(typedef_flag = 1, visibility = visibility)
3425 3426
    if api:
        ctx.api = 1
William Stein's avatar
William Stein committed
3427
    if s.sy == 'class':
3428
        return p_c_class_definition(s, pos, ctx)
Mark Florisson's avatar
Mark Florisson committed
3429 3430 3431 3432
    elif s.sy == 'IDENT' and s.systring in struct_enum_union:
        return p_struct_enum(s, pos, ctx)
    elif s.sy == 'IDENT' and s.systring == 'fused':
        return p_fused_definition(s, pos, ctx)
William Stein's avatar
William Stein committed
3433
    else:
3434
        base_type = p_c_base_type(s, nonempty = 1)
3435
        declarator = p_c_declarator(s, ctx, is_type = 1, nonempty = 1)
3436
        s.expect_newline("Syntax error in ctypedef statement", ignore_semicolon=True)
3437 3438
        return Nodes.CTypeDefNode(
            pos, base_type = base_type,
Robert Bradshaw's avatar
Robert Bradshaw committed
3439
            declarator = declarator,
3440
            visibility = visibility, api = api,
3441
            in_pxd = ctx.level == 'module_pxd')
William Stein's avatar
William Stein committed
3442

3443 3444
def p_decorators(s):
    decorators = []
3445
    while s.sy == '@':
3446 3447
        pos = s.position()
        s.next()
3448 3449
        decstring = p_dotted_name(s, as_allowed=0)[2]
        names = decstring.split('.')
3450
        decorator = ExprNodes.NameNode(pos, name=s.context.intern_ustring(names[0]))
3451
        for name in names[1:]:
3452 3453
            decorator = ExprNodes.AttributeNode(
                pos, attribute=s.context.intern_ustring(name), obj=decorator)
3454 3455 3456 3457 3458 3459
        if s.sy == '(':
            decorator = p_call(s, decorator)
        decorators.append(Nodes.DecoratorNode(pos, decorator=decorator))
        s.expect_newline("Expected a newline after decorator")
    return decorators

3460

3461 3462 3463 3464 3465 3466 3467 3468 3469 3470
def _reject_cdef_modifier_in_py(s, name):
    """Step over incorrectly placed cdef modifiers (@see _CDEF_MODIFIERS) to provide a good error message for them.
    """
    if s.sy == 'IDENT' and name in _CDEF_MODIFIERS:
        # Special enough to provide a good error message.
        s.error("Cannot use cdef modifier '%s' in Python function signature. Use a decorator instead." % name, fatal=False)
        return p_ident(s)  # Keep going, in case there are other errors.
    return name


3471
def p_def_statement(s, decorators=None, is_async_def=False):
William Stein's avatar
William Stein committed
3472
    # s.sy == 'def'
3473
    pos = decorators[0].pos if decorators else s.position()
3474 3475 3476
    # PEP 492 switches the async/await keywords on in "async def" functions
    if is_async_def:
        s.enter_async()
William Stein's avatar
William Stein committed
3477
    s.next()
3478
    name = _reject_cdef_modifier_in_py(s, p_ident(s))
3479 3480 3481 3482 3483
    s.expect(
        '(',
        "Expected '(', found '%s'. Did you use cdef syntax in a Python declaration? "
        "Use decorators and Python type annotations instead." % (
            s.systring if s.sy == 'IDENT' else s.sy))
Stefan Behnel's avatar
Stefan Behnel committed
3484 3485
    args, star_arg, starstar_arg = p_varargslist(s, terminator=')')
    s.expect(')')
3486
    _reject_cdef_modifier_in_py(s, s.systring)
3487 3488 3489
    return_type_annotation = None
    if s.sy == '->':
        s.next()
3490
        return_type_annotation = p_annotation(s)
3491
        _reject_cdef_modifier_in_py(s, s.systring)
3492

3493
    doc, body = p_suite_with_docstring(s, Ctx(level='function'))
3494
    if is_async_def:
3495
        s.exit_async()
3496

3497 3498 3499 3500 3501
    return Nodes.DefNode(
        pos, name=name, args=args, star_arg=star_arg, starstar_arg=starstar_arg,
        doc=doc, body=body, decorators=decorators, is_async_def=is_async_def,
        return_type_annotation=return_type_annotation)

Stefan Behnel's avatar
Stefan Behnel committed
3502

3503 3504 3505
def p_varargslist(s, terminator=')', annotated=1):
    args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1,
                        annotated = annotated)
William Stein's avatar
William Stein committed
3506 3507
    star_arg = None
    starstar_arg = None
3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521
    if s.sy == '/':
        if len(args) == 0:
            s.error("Got zero positional-only arguments despite presence of "
                    "positional-only specifier '/'")
        s.next()
        # Mark all args to the left as pos only
        for arg in args:
            arg.pos_only = 1
        if s.sy == ',':
            s.next()
            args.extend(p_c_arg_list(s, in_pyfunc = 1,
                nonempty_declarators = 1, annotated = annotated))
        elif s.sy != terminator:
            s.error("Syntax error in Python function argument list")
William Stein's avatar
William Stein committed
3522 3523
    if s.sy == '*':
        s.next()
3524
        if s.sy == 'IDENT':
3525
            star_arg = p_py_arg_decl(s, annotated=annotated)
William Stein's avatar
William Stein committed
3526 3527
        if s.sy == ',':
            s.next()
3528
            args.extend(p_c_arg_list(s, in_pyfunc = 1,
3529
                nonempty_declarators = 1, kw_only = 1, annotated = annotated))
Stefan Behnel's avatar
Stefan Behnel committed
3530
        elif s.sy != terminator:
3531 3532
            s.error("Syntax error in Python function argument list")
    if s.sy == '**':
William Stein's avatar
William Stein committed
3533
        s.next()
3534
        starstar_arg = p_py_arg_decl(s, annotated=annotated)
3535 3536
    if s.sy == ',':
        s.next()
Stefan Behnel's avatar
Stefan Behnel committed
3537
    return (args, star_arg, starstar_arg)
William Stein's avatar
William Stein committed
3538

3539
def p_py_arg_decl(s, annotated = 1):
William Stein's avatar
William Stein committed
3540 3541
    pos = s.position()
    name = p_ident(s)
3542
    annotation = None
3543
    if annotated and s.sy == ':':
3544
        s.next()
3545
        annotation = p_annotation(s)
3546
    return Nodes.PyArgDeclNode(pos, name = name, annotation = annotation)
William Stein's avatar
William Stein committed
3547

3548

3549
def p_class_statement(s, decorators):
William Stein's avatar
William Stein committed
3550 3551 3552
    # s.sy == 'class'
    pos = s.position()
    s.next()
3553 3554
    class_name = EncodedString(p_ident(s))
    class_name.encoding = s.source_encoding  # FIXME: why is this needed?
3555 3556
    arg_tuple = None
    keyword_dict = None
William Stein's avatar
William Stein committed
3557
    if s.sy == '(':
3558 3559
        positional_args, keyword_args = p_call_parse_args(s, allow_genexp=False)
        arg_tuple, keyword_dict = p_call_build_packed_args(pos, positional_args, keyword_args)
3560 3561
    if arg_tuple is None:
        # XXX: empty arg_tuple
3562 3563
        arg_tuple = ExprNodes.TupleNode(pos, args=[])
    doc, body = p_suite_with_docstring(s, Ctx(level='class'))
3564 3565 3566 3567 3568 3569
    return Nodes.PyClassDefNode(
        pos, name=class_name,
        bases=arg_tuple,
        keyword_args=keyword_dict,
        doc=doc, body=body, decorators=decorators,
        force_py3_semantics=s.context.language_level >= 3)
William Stein's avatar
William Stein committed
3570

3571

3572
def p_c_class_definition(s, pos,  ctx):
William Stein's avatar
William Stein committed
3573 3574 3575 3576 3577 3578 3579 3580
    # s.sy == 'class'
    s.next()
    module_path = []
    class_name = p_ident(s)
    while s.sy == '.':
        s.next()
        module_path.append(class_name)
        class_name = p_ident(s)
3581
    if module_path and ctx.visibility != 'extern':
William Stein's avatar
William Stein committed
3582 3583 3584 3585 3586 3587 3588 3589
        error(pos, "Qualified class name only allowed for 'extern' C class")
    if module_path and s.sy == 'IDENT' and s.systring == 'as':
        s.next()
        as_name = p_ident(s)
    else:
        as_name = class_name
    objstruct_name = None
    typeobj_name = None
3590
    bases = None
3591
    check_size = None
William Stein's avatar
William Stein committed
3592
    if s.sy == '(':
3593 3594
        positional_args, keyword_args = p_call_parse_args(s, allow_genexp=False)
        if keyword_args:
Robert Bradshaw's avatar
Robert Bradshaw committed
3595
            s.error("C classes cannot take keyword bases.")
3596 3597
        bases, _ = p_call_build_packed_args(pos, positional_args, keyword_args)
    if bases is None:
Robert Bradshaw's avatar
Robert Bradshaw committed
3598
        bases = ExprNodes.TupleNode(pos, args=[])
3599

William Stein's avatar
William Stein committed
3600
    if s.sy == '[':
3601 3602
        if ctx.visibility not in ('public', 'extern') and not ctx.api:
            error(s.position(), "Name options only allowed for 'public', 'api', or 'extern' C class")
3603
        objstruct_name, typeobj_name, check_size = p_c_class_options(s)
William Stein's avatar
William Stein committed
3604
    if s.sy == ':':
3605
        if ctx.level == 'module_pxd':
William Stein's avatar
William Stein committed
3606 3607 3608
            body_level = 'c_class_pxd'
        else:
            body_level = 'c_class'
3609
        doc, body = p_suite_with_docstring(s, Ctx(level=body_level))
William Stein's avatar
William Stein committed
3610 3611 3612 3613
    else:
        s.expect_newline("Syntax error in C class definition")
        doc = None
        body = None
3614
    if ctx.visibility == 'extern':
William Stein's avatar
William Stein committed
3615 3616 3617 3618
        if not module_path:
            error(pos, "Module name required for 'extern' C class")
        if typeobj_name:
            error(pos, "Type object name specification not allowed for 'extern' C class")
3619
    elif ctx.visibility == 'public':
William Stein's avatar
William Stein committed
3620 3621 3622 3623
        if not objstruct_name:
            error(pos, "Object struct name specification required for 'public' C class")
        if not typeobj_name:
            error(pos, "Type object name specification required for 'public' C class")
3624 3625
    elif ctx.visibility == 'private':
        if ctx.api:
3626 3627 3628 3629
            if not objstruct_name:
                error(pos, "Object struct name specification required for 'api' C class")
            if not typeobj_name:
                error(pos, "Type object name specification required for 'api' C class")
3630
    else:
3631
        error(pos, "Invalid class visibility '%s'" % ctx.visibility)
William Stein's avatar
William Stein committed
3632
    return Nodes.CClassDefNode(pos,
3633 3634 3635
        visibility = ctx.visibility,
        typedef_flag = ctx.typedef_flag,
        api = ctx.api,
William Stein's avatar
William Stein committed
3636 3637 3638
        module_name = ".".join(module_path),
        class_name = class_name,
        as_name = as_name,
3639
        bases = bases,
William Stein's avatar
William Stein committed
3640 3641
        objstruct_name = objstruct_name,
        typeobj_name = typeobj_name,
3642
        check_size = check_size,
3643
        in_pxd = ctx.level == 'module_pxd',
William Stein's avatar
William Stein committed
3644
        doc = doc,
3645
        body = body)
William Stein's avatar
William Stein committed
3646

3647

William Stein's avatar
William Stein committed
3648 3649 3650
def p_c_class_options(s):
    objstruct_name = None
    typeobj_name = None
3651
    check_size = None
William Stein's avatar
William Stein committed
3652 3653
    s.expect('[')
    while 1:
Stefan Behnel's avatar
Stefan Behnel committed
3654
        if s.sy != 'IDENT':
William Stein's avatar
William Stein committed
3655 3656 3657 3658 3659 3660 3661
            break
        if s.systring == 'object':
            s.next()
            objstruct_name = p_ident(s)
        elif s.systring == 'type':
            s.next()
            typeobj_name = p_ident(s)
3662 3663
        elif s.systring == 'check_size':
            s.next()
mattip's avatar
mattip committed
3664
            check_size = p_ident(s)
3665 3666
            if check_size not in ('ignore', 'warn', 'error'):
                s.error("Expected one of ignore, warn or error, found %r" % check_size)
Stefan Behnel's avatar
Stefan Behnel committed
3667
        if s.sy != ',':
William Stein's avatar
William Stein committed
3668 3669
            break
        s.next()
3670
    s.expect(']', "Expected 'object', 'type' or 'check_size'")
3671
    return objstruct_name, typeobj_name, check_size
William Stein's avatar
William Stein committed
3672

3673

William Stein's avatar
William Stein committed
3674 3675
def p_property_decl(s):
    pos = s.position()
3676
    s.next()  # 'property'
William Stein's avatar
William Stein committed
3677
    name = p_ident(s)
3678 3679 3680 3681
    doc, body = p_suite_with_docstring(
        s, Ctx(level='property'), with_doc_only=True)
    return Nodes.PropertyNode(pos, name=name, doc=doc, body=body)

William Stein's avatar
William Stein committed
3682

3683 3684 3685 3686 3687 3688 3689
def p_ignorable_statement(s):
    """
    Parses any kind of ignorable statement that is allowed in .pxd files.
    """
    if s.sy == 'BEGIN_STRING':
        pos = s.position()
        string_node = p_atom(s)
3690
        s.expect_newline("Syntax error in string", ignore_semicolon=True)
3691 3692 3693 3694
        return Nodes.ExprStatNode(pos, expr=string_node)
    return None


3695 3696 3697 3698
def p_doc_string(s):
    if s.sy == 'BEGIN_STRING':
        pos = s.position()
        kind, bytes_result, unicode_result = p_cat_string_literal(s)
3699
        s.expect_newline("Syntax error in doc string", ignore_semicolon=True)
3700 3701 3702 3703 3704 3705 3706
        if kind in ('u', ''):
            return unicode_result
        warning(pos, "Python 3 requires docstrings to be unicode strings")
        return bytes_result
    else:
        return None

3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742

def _extract_docstring(node):
    """
    Extract a docstring from a statement or from the first statement
    in a list.  Remove the statement if found.  Return a tuple
    (plain-docstring or None, node).
    """
    doc_node = None
    if node is None:
        pass
    elif isinstance(node, Nodes.ExprStatNode):
        if node.expr.is_string_literal:
            doc_node = node.expr
            node = Nodes.StatListNode(node.pos, stats=[])
    elif isinstance(node, Nodes.StatListNode) and node.stats:
        stats = node.stats
        if isinstance(stats[0], Nodes.ExprStatNode):
            if stats[0].expr.is_string_literal:
                doc_node = stats[0].expr
                del stats[0]

    if doc_node is None:
        doc = None
    elif isinstance(doc_node, ExprNodes.BytesNode):
        warning(node.pos,
                "Python 3 requires docstrings to be unicode strings")
        doc = doc_node.value
    elif isinstance(doc_node, ExprNodes.StringNode):
        doc = doc_node.unicode_value
        if doc is None:
            doc = doc_node.value
    else:
        doc = doc_node.value
    return doc, node


3743 3744
def p_code(s, level=None, ctx=Ctx):
    body = p_statement_list(s, ctx(level = level), first_statement = 1)
3745 3746 3747 3748
    if s.sy != 'EOF':
        s.error("Syntax error in statement [%s,%s]" % (
            repr(s.sy), repr(s.systring)))
    return body
William Stein's avatar
William Stein committed
3749

3750

3751 3752
_match_compiler_directive_comment = cython.declare(object, re.compile(
    r"^#\s*cython\s*:\s*((\w|[.])+\s*=.*)$").match)
3753

3754

3755
def p_compiler_directive_comments(s):
3756
    result = {}
3757
    while s.sy == 'commentline':
3758
        pos = s.position()
3759
        m = _match_compiler_directive_comment(s.systring)
3760
        if m:
3761
            directives_string = m.group(1).strip()
3762
            try:
3763
                new_directives = Options.parse_directive_list(directives_string, ignore_unknown=True)
3764
            except ValueError as e:
3765
                s.error(e.args[0], fatal=False)
3766
                s.next()
3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783
                continue

            for name in new_directives:
                if name not in result:
                    pass
                elif new_directives[name] == result[name]:
                    warning(pos, "Duplicate directive found: %s" % (name,))
                else:
                    s.error("Conflicting settings found for top-level directive %s: %r and %r" % (
                        name, result[name], new_directives[name]), pos=pos)

            if 'language_level' in new_directives:
                # Make sure we apply the language level already to the first token that follows the comments.
                s.context.set_language_level(new_directives['language_level'])

            result.update(new_directives)

3784 3785 3786
        s.next()
    return result

3787

3788
def p_module(s, pxd, full_module_name, ctx=Ctx):
William Stein's avatar
William Stein committed
3789
    pos = s.position()
3790

3791
    directive_comments = p_compiler_directive_comments(s)
3792 3793
    s.parse_comments = False

3794
    if s.context.language_level is None:
3795
        s.context.set_language_level('3str')
3796 3797 3798
        if pos[0].filename:
            import warnings
            warnings.warn(
3799 3800
                "Cython directive 'language_level' not set, using '3str' for now (Py3). "
                "This has changed from earlier releases! File: %s" % pos[0].filename,
3801 3802 3803 3804
                FutureWarning,
                stacklevel=1 if cython.compiled else 2,
            )

Stefan Behnel's avatar
Stefan Behnel committed
3805
    level = 'module_pxd' if pxd else 'module'
3806
    doc = p_doc_string(s)
3807
    body = p_statement_list(s, ctx(level=level), first_statement = 1)
Stefan Behnel's avatar
Stefan Behnel committed
3808
    if s.sy != 'EOF':
William Stein's avatar
William Stein committed
3809 3810
        s.error("Syntax error in statement [%s,%s]" % (
            repr(s.sy), repr(s.systring)))
3811 3812
    return ModuleNode(pos, doc = doc, body = body,
                      full_module_name = full_module_name,
3813
                      directive_comments = directive_comments)
William Stein's avatar
William Stein committed
3814

3815 3816 3817 3818 3819 3820 3821 3822 3823 3824
def p_template_definition(s):
    name = p_ident(s)
    if s.sy == '=':
        s.expect('=')
        s.expect('*')
        required = False
    else:
        required = True
    return name, required

3825
def p_cpp_class_definition(s, pos,  ctx):
3826 3827
    # s.sy in ('cppclass', 'cypclass')
    cypclass = s.systring == 'cypclass'
3828 3829
    s.next()
    class_name = p_ident(s)
3830 3831 3832
    cname = p_opt_cname(s)
    if cname is None and ctx.namespace is not None:
        cname = ctx.namespace + "::" + class_name
3833
    if s.sy == '.':
3834
        error(pos, "Qualified class name not allowed C++ class")
Danilo Freitas's avatar
Danilo Freitas committed
3835 3836
    if s.sy == '[':
        s.next()
3837
        templates = [p_template_definition(s)]
Danilo Freitas's avatar
Danilo Freitas committed
3838 3839
        while s.sy == ',':
            s.next()
3840
            templates.append(p_template_definition(s))
Danilo Freitas's avatar
Danilo Freitas committed
3841
        s.expect(']')
3842
        template_names = [name for name, required in templates]
3843 3844
    else:
        templates = None
3845
        template_names = None
3846
    if s.sy == '(':
3847
        s.next()
3848
        base_classes = [p_c_base_type(s, templates = template_names)]
3849
        while s.sy == ',':
3850
            s.next()
3851
            base_classes.append(p_c_base_type(s, templates = template_names))
3852
        s.expect(')')
3853 3854
    else:
        base_classes = []
3855
    if s.sy == '[':
3856
        error(s.position(), "Name options not allowed for C++ class")
3857
    nogil = p_nogil(s) or cypclass
3858
    lock_mode = None
gsamain's avatar
gsamain committed
3859
    activable = False
3860 3861
    if cypclass:
        lock_mode = p_cypclass_lock_mode(s)
gsamain's avatar
gsamain committed
3862
        activable = p_cypclass_activable(s)
3863
    if s.sy == ':':
3864 3865 3866
        s.next()
        s.expect('NEWLINE')
        s.expect_indent()
3867 3868 3869 3870
        # Allow a cppclass to have docstrings. It will be discarded as comment.
        # The goal of this is consistency: we can make docstrings inside cppclass methods,
        # so why not on the cppclass itself ?
        p_doc_string(s)
3871
        attributes = []
3872
        body_ctx = Ctx(visibility = ctx.visibility, level='cpp_class', nogil=nogil or ctx.nogil)
3873
        body_ctx.templates = template_names
3874
        while s.sy != 'DEDENT':
3875 3876
            if s.sy != 'pass':
                attributes.append(p_cpp_class_attribute(s, body_ctx))
3877 3878 3879 3880
            else:
                s.next()
                s.expect_newline("Expected a newline")
        s.expect_dedent()
3881
    else:
3882
        attributes = None
3883 3884 3885
        s.expect_newline("Syntax error in C++ class definition")
    return Nodes.CppClassNode(pos,
        name = class_name,
3886
        cname = cname,
3887
        base_classes = base_classes,
3888 3889
        visibility = ctx.visibility,
        in_pxd = ctx.level == 'module_pxd',
Danilo Freitas's avatar
Danilo Freitas committed
3890
        attributes = attributes,
gsamain's avatar
gsamain committed
3891
        templates = templates, cypclass=cypclass, lock_mode=lock_mode, activable=activable)
3892

3893 3894 3895 3896
def p_cpp_class_attribute(s, ctx):
    decorators = None
    if s.sy == '@':
        decorators = p_decorators(s)
gsamain's avatar
gsamain committed
3897
    if s.systring in ('cppclass', 'cypclass'):
3898
        return p_cpp_class_definition(s, s.position(), ctx)
3899
    elif s.systring == 'ctypedef':
3900
        return p_ctypedef_statement(s, ctx)
3901
    elif s.sy == 'IDENT' and s.systring in struct_enum_union:
3902 3903 3904 3905
        if s.systring != 'enum':
            return p_cpp_class_definition(s, s.position(), ctx)
        else:
            return p_struct_enum(s, s.position(), ctx)
3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916
    else:
        node = p_c_func_or_var_declaration(s, s.position(), ctx)
        if decorators is not None:
            tup = Nodes.CFuncDefNode, Nodes.CVarDefNode, Nodes.CClassDefNode
            if ctx.allow_struct_enum_decorator:
                tup += Nodes.CStructOrUnionDefNode, Nodes.CEnumDefNode
            if not isinstance(node, tup):
                s.error("Decorators can only be followed by functions or classes")
            node.decorators = decorators
        return node

3917 3918 3919 3920 3921 3922 3923 3924
def p_cypclass_lock_mode(s):
    if s.sy == 'IDENT' and s.systring in ('nolock', 'checklock', 'autolock'):
        mode = s.systring
        s.next()
        return mode
    else:
        return None

gsamain's avatar
gsamain committed
3925 3926 3927 3928 3929 3930
def p_cypclass_activable(s):
    if s.sy == 'IDENT' and s.systring == 'activable':
        s.next()
        return True
    else:
        return False
3931

William Stein's avatar
William Stein committed
3932 3933 3934 3935 3936 3937
#----------------------------------------------
#
#   Debugging
#
#----------------------------------------------

Stefan Behnel's avatar
Stefan Behnel committed
3938
def print_parse_tree(f, node, level, key = None):
William Stein's avatar
William Stein committed
3939 3940 3941 3942 3943 3944
    ind = "  " * level
    if node:
        f.write(ind)
        if key:
            f.write("%s: " % key)
        t = type(node)
Stefan Behnel's avatar
Stefan Behnel committed
3945
        if t is tuple:
William Stein's avatar
William Stein committed
3946
            f.write("(%s @ %s\n" % (node[0], node[1]))
3947
            for i in range(2, len(node)):
William Stein's avatar
William Stein committed
3948 3949 3950
                print_parse_tree(f, node[i], level+1)
            f.write("%s)\n" % ind)
            return
3951
        elif isinstance(node, Nodes.Node):
William Stein's avatar
William Stein committed
3952 3953 3954 3955 3956 3957
            try:
                tag = node.tag
            except AttributeError:
                tag = node.__class__.__name__
            f.write("%s @ %s\n" % (tag, node.pos))
            for name, value in node.__dict__.items():
Stefan Behnel's avatar
Stefan Behnel committed
3958
                if name != 'tag' and name != 'pos':
William Stein's avatar
William Stein committed
3959 3960
                    print_parse_tree(f, value, level+1, name)
            return
Stefan Behnel's avatar
Stefan Behnel committed
3961
        elif t is list:
William Stein's avatar
William Stein committed
3962
            f.write("[\n")
3963
            for i in range(len(node)):
William Stein's avatar
William Stein committed
3964 3965 3966 3967
                print_parse_tree(f, node[i], level+1)
            f.write("%s]\n" % ind)
            return
    f.write("%s%s\n" % (ind, node))
3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978

def p_annotation(s):
    """An annotation just has the "test" syntax, but also stores the string it came from

    Note that the string is *allowed* to be changed/processed (although isn't here)
    so may not exactly match the string generated by Python, and if it doesn't
    then it is not a bug.
    """
    pos = s.position()
    expr = p_test(s)
    return ExprNodes.AnnotationNode(pos, expr=expr)