Commit 6662537b authored by Stefan Behnel's avatar Stefan Behnel

cleanup in parser to make it a better match with the CPython grammar, fix some minor grammar issues

parent 8dbe6a2d
......@@ -6,8 +6,8 @@ from Cython.Compiler.Scanning cimport PyrexScanner
cpdef p_ident(PyrexScanner s, message =*)
cpdef p_ident_list(PyrexScanner s)
cpdef p_binop_operator(PyrexScanner s)
cpdef p_binop_expr(PyrexScanner s, ops, p_sub_expr)
cpdef p_simple_expr(PyrexScanner s)
cpdef p_lambdef(PyrexScanner s, bint allow_conditional=*)
cpdef p_lambdef_nocond(PyrexScanner s)
cpdef p_test(PyrexScanner s)
......@@ -17,9 +17,10 @@ cpdef p_rassoc_binop_expr(PyrexScanner s, ops, p_subexpr)
cpdef p_and_test(PyrexScanner s)
cpdef p_not_test(PyrexScanner s)
cpdef p_comparison(PyrexScanner s)
cpdef p_test_or_starred_expr(PyrexScanner s)
cpdef p_starred_expr(PyrexScanner s)
cpdef p_cascaded_cmp(PyrexScanner s)
cpdef p_cmp_op(PyrexScanner s)
cpdef p_starred_expr(PyrexScanner s)
cpdef p_bit_expr(PyrexScanner s)
cpdef p_xor_expr(PyrexScanner s)
cpdef p_and_expr(PyrexScanner s)
......@@ -30,6 +31,7 @@ cpdef p_factor(PyrexScanner s)
cpdef p_typecast(PyrexScanner s)
cpdef p_sizeof(PyrexScanner s)
cpdef p_yield_expression(PyrexScanner s)
cpdef p_yield_statement(PyrexScanner s)
cpdef p_power(PyrexScanner s)
cpdef p_new_expr(PyrexScanner s)
cpdef p_trailer(PyrexScanner s, node1)
......@@ -52,9 +54,12 @@ cpdef p_comp_for(PyrexScanner s, body)
cpdef p_comp_if(PyrexScanner s, body)
cpdef p_dict_or_set_maker(PyrexScanner s)
cpdef p_backquote_expr(PyrexScanner s)
cpdef p_simple_expr_list(PyrexScanner s)
cpdef p_expr(PyrexScanner s)
cpdef p_simple_expr_list(PyrexScanner s, expr=*)
cpdef p_test_or_starred_expr_list(s, expr=*)
cpdef p_testlist(PyrexScanner s)
cpdef p_testlist_star_expr(PyrexScanner s)
cpdef p_testlist_comp(PyrexScanner s)
cpdef p_genexp(PyrexScanner s, expr)
#-------------------------------------------------------
#
......@@ -83,12 +88,12 @@ cpdef p_if_clause(PyrexScanner s)
cpdef p_else_clause(PyrexScanner s)
cpdef p_while_statement(PyrexScanner s)
cpdef p_for_statement(PyrexScanner s)
cpdef p_for_bounds(PyrexScanner s)
cpdef p_for_bounds(PyrexScanner s, bint allow_testlist = *)
cpdef p_for_from_relation(PyrexScanner s)
cpdef p_for_from_step(PyrexScanner s)
cpdef p_target(PyrexScanner s, terminator)
cpdef p_for_target(PyrexScanner s)
cpdef p_for_iterator(PyrexScanner s)
cpdef p_for_iterator(PyrexScanner s, bint allow_testlist = *)
cpdef p_try_statement(PyrexScanner s)
cpdef p_except_clause(PyrexScanner s)
cpdef p_include_statement(PyrexScanner s, ctx)
......
......@@ -184,6 +184,12 @@ def p_comparison(s):
n1.cascade = p_cascaded_cmp(s)
return n1
def p_test_or_starred_expr(s):
if s.sy == '*':
return p_starred_expr(s)
else:
return p_test(s)
def p_starred_expr(s):
pos = s.position()
if s.sy == '*':
......@@ -326,7 +332,7 @@ def p_yield_expression(s):
pos = s.position()
s.next()
if s.sy != ')' and s.sy not in statement_terminators:
arg = p_expr(s)
arg = p_testlist(s)
else:
arg = None
return ExprNodes.YieldExprNode(pos, arg=arg)
......@@ -776,10 +782,11 @@ def p_list_maker(s):
return ExprNodes.ComprehensionNode(
pos, loop=loop, append=append, target=target)
else:
exprs = [expr]
if s.sy == ',':
s.next()
exprs += p_simple_expr_list(s)
exprs = p_simple_expr_list(s, expr)
else:
exprs = [expr]
s.expect(']')
return ExprNodes.ListNode(pos, args = exprs)
......@@ -796,7 +803,7 @@ def p_comp_for(s, body):
# s.sy == 'for'
pos = s.position()
s.next()
kw = p_for_bounds(s)
kw = p_for_bounds(s, allow_testlist=False)
kw['else_clause'] = None
kw['body'] = p_comp_iter(s, body)
return Nodes.ForStatNode(pos, **kw)
......@@ -874,44 +881,62 @@ def p_dict_or_set_maker(s):
s.expect('}')
return ExprNodes.DictNode(pos, key_value_pairs = [])
# NOTE: no longer in Py3 :)
def p_backquote_expr(s):
# s.sy == '`'
pos = s.position()
s.next()
arg = p_expr(s)
args = [p_test(s)]
while s.sy == ',':
s.next()
args.append(p_test(s))
s.expect('`')
if len(args) == 1:
arg = args[0]
else:
arg = ExprNodes.TupleNode(pos, args = args)
return ExprNodes.BackquoteNode(pos, arg = arg)
def p_simple_expr_list(s):
exprs = []
def p_simple_expr_list(s, expr=None):
exprs = expr is not None and [expr] or []
while s.sy not in expr_terminators:
expr = p_test(s)
exprs.append(expr)
exprs.append( p_test(s) )
if s.sy != ',':
break
s.next()
return exprs
def p_test_or_starred_expr_list(s, expr=None):
exprs = expr is not None and [expr] or []
while s.sy not in expr_terminators:
exprs.append( p_test_or_starred_expr(s) )
if s.sy != ',':
break
s.next()
return exprs
def p_expr(s):
#testlist: test (',' test)* [',']
def p_testlist(s):
pos = s.position()
expr = p_test(s)
if s.sy == ',':
s.next()
exprs = [expr] + p_simple_expr_list(s)
exprs = p_simple_expr_list(s, expr)
return ExprNodes.TupleNode(pos, args = exprs)
else:
return expr
# testlist_star_expr: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
#testlist: test (',' test)* [',']
# differs from p_expr only in the fact that it cannot contain conditional expressions
def p_testlist(s):
def p_testlist_star_expr(s):
pos = s.position()
expr = p_test(s)
expr = p_test_or_starred_expr(s)
if s.sy == ',':
s.next()
return p_testlist_tuple(s, pos, expr)
exprs = p_test_or_starred_expr_list(s, expr)
return ExprNodes.TupleNode(pos, args = exprs)
else:
return expr
......@@ -919,24 +944,16 @@ def p_testlist(s):
def p_testlist_comp(s):
pos = s.position()
expr = p_test(s)
expr = p_test_or_starred_expr(s)
if s.sy == ',':
s.next()
return p_testlist_tuple(s, pos, expr)
exprs = p_test_or_starred_expr_list(s, expr)
return ExprNodes.TupleNode(pos, args = exprs)
elif s.sy == 'for':
return p_genexp(s, expr)
else:
return expr
def p_testlist_tuple(s, pos, expr):
exprs = [expr]
while s.sy not in expr_terminators:
exprs.append(p_test(s))
if s.sy != ',':
break
s.next()
return ExprNodes.TupleNode(pos, args = exprs)
def p_genexp(s, expr):
# s.sy == 'for'
loop = p_comp_for(s, ExprNodes.YieldExprNode(expr.pos, arg=expr))
......@@ -958,10 +975,14 @@ def p_global_statement(s):
return Nodes.GlobalNode(pos, names = names)
def p_expression_or_assignment(s):
expr_list = [p_expr(s)]
expr_list = [p_testlist_star_expr(s)]
while s.sy == '=':
s.next()
expr_list.append(p_expr(s))
if s.sy == 'yield':
expr = p_yield_expression(s)
else:
expr = p_testlist_star_expr(s)
expr_list.append(expr)
if len(expr_list) == 1:
if re.match(r"([+*/\%^\&|-]|<<|>>|\*\*|//)=", s.sy):
lhs = expr_list[0]
......@@ -969,7 +990,10 @@ def p_expression_or_assignment(s):
error(lhs.pos, "Illegal operand for inplace operation.")
operator = s.sy[:-1]
s.next()
rhs = p_expr(s)
if s.sy == 'yield':
rhs = p_yield_expression(s)
else:
rhs = p_testlist(s)
return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs)
expr = expr_list[0]
if isinstance(expr, (ExprNodes.UnicodeNode, ExprNodes.StringNode, ExprNodes.BytesNode)):
......@@ -1059,7 +1083,7 @@ def p_return_statement(s):
pos = s.position()
s.next()
if s.sy not in statement_terminators:
value = p_expr(s)
value = p_testlist(s)
else:
value = None
return Nodes.ReturnStatNode(pos, value = value)
......@@ -1273,16 +1297,16 @@ def p_for_statement(s):
# s.sy == 'for'
pos = s.position()
s.next()
kw = p_for_bounds(s)
kw = p_for_bounds(s, allow_testlist=True)
kw['body'] = p_suite(s)
kw['else_clause'] = p_else_clause(s)
return Nodes.ForStatNode(pos, **kw)
def p_for_bounds(s):
def p_for_bounds(s, allow_testlist=True):
target = p_for_target(s)
if s.sy == 'in':
s.next()
iterator = p_for_iterator(s)
iterator = p_for_iterator(s, allow_testlist)
return { 'target': target, 'iterator': iterator }
else:
if s.sy == 'from':
......@@ -1353,9 +1377,12 @@ def p_target(s, terminator):
def p_for_target(s):
return p_target(s, 'in')
def p_for_iterator(s):
def p_for_iterator(s, allow_testlist=True):
pos = s.position()
if allow_testlist:
expr = p_testlist(s)
else:
expr = p_or_test(s)
return ExprNodes.IteratorNode(pos, sequence = expr)
def p_try_statement(s):
......@@ -1462,7 +1489,7 @@ def p_with_statement(s):
else:
error(pos, "Syntax error in template function declaration")
else:
manager = p_expr(s)
manager = p_test(s)
target = None
if s.sy == 'IDENT' and s.systring == 'as':
s.next()
......@@ -1525,7 +1552,7 @@ def p_simple_statement_list(s, ctx, first_statement = 0):
def p_compile_time_expr(s):
old = s.compile_time_expr
s.compile_time_expr = 1
expr = p_expr(s)
expr = p_testlist(s)
s.compile_time_expr = old
return expr
......@@ -1996,7 +2023,7 @@ def p_c_array_declarator(s, base):
pos = s.position()
s.next() # '['
if s.sy != ']':
dim = p_expr(s)
dim = p_testlist(s)
else:
dim = None
s.expect(']')
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment