Commit b6b5152f authored by Stefan Behnel's avatar Stefan Behnel

properly handle expressions at the beginning of func/class/etc. blocks that...

properly handle expressions at the beginning of func/class/etc. blocks that start with a string literal but are not docstrings
parent fc0a514a
......@@ -121,13 +121,20 @@ class NormalizeTree(CythonTransform):
def visit_CStructOrUnionDefNode(self, node):
return self.visit_StatNode(node, True)
# Eliminate PassStatNode
def visit_PassStatNode(self, node):
"""Eliminate PassStatNode"""
if not self.is_in_statlist:
return Nodes.StatListNode(pos=node.pos, stats=[])
else:
return []
def visit_ExprStatNode(self, node):
"""Eliminate useless string literals"""
if node.expr.is_string_literal:
return None
self.visitchildren(node)
return node
def visit_CDeclaratorNode(self, node):
return node
......
......@@ -125,7 +125,9 @@ cdef p_DEF_statement(PyrexScanner s)
cdef p_IF_statement(PyrexScanner s, ctx)
cdef p_statement(PyrexScanner s, ctx, bint first_statement = *)
cdef p_statement_list(PyrexScanner s, ctx, bint first_statement = *)
cdef p_suite(PyrexScanner s, ctx = *, bint with_doc = *, bint with_pseudo_doc = *)
cdef p_suite(PyrexScanner s, ctx = *)
cdef tuple p_suite_with_docstring(s, ctx, with_doc_only = *)
cdef tuple _extract_docstring(node)
cdef p_positional_and_keyword_args(PyrexScanner s, end_sy_set, templates = *)
cpdef p_c_base_type(PyrexScanner s, bint self_flag = *, bint nonempty = *, templates = *)
......
......@@ -1141,10 +1141,7 @@ def p_expression_or_assignment(s):
rhs = p_testlist(s)
return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs)
expr = expr_list[0]
if isinstance(expr, (ExprNodes.UnicodeNode, ExprNodes.StringNode, ExprNodes.BytesNode)):
return Nodes.PassStatNode(expr.pos)
else:
return Nodes.ExprStatNode(expr.pos, expr = expr)
return Nodes.ExprStatNode(expr.pos, expr=expr)
rhs = expr_list[-1]
if len(expr_list) == 2:
......@@ -1888,7 +1885,7 @@ def p_statement(s, ctx, first_statement = 0):
elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':
return p_property_decl(s)
elif s.sy == 'pass' and ctx.level != 'property':
return p_pass_statement(s, with_newline = 1)
return p_pass_statement(s, with_newline=True)
else:
if ctx.level in ('c_class_pxd', 'property'):
s.error("Executable statement not allowed here")
......@@ -1923,15 +1920,18 @@ def p_statement_list(s, ctx, first_statement = 0):
else:
return Nodes.StatListNode(pos, stats = stats)
def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):
pos = s.position()
def p_suite(s, ctx=Ctx()):
return p_suite_with_docstring(s, ctx, with_doc_only=False)[1]
def p_suite_with_docstring(s, ctx, with_doc_only=False):
s.expect(':')
doc = None
stmts = []
if s.sy == 'NEWLINE':
s.next()
s.expect_indent()
if with_doc or with_pseudo_doc:
if with_doc_only:
doc = p_doc_string(s)
body = p_statement_list(s, ctx)
s.expect_dedent()
......@@ -1943,10 +1943,10 @@ def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):
else:
body = p_pass_statement(s)
s.expect_newline("Syntax error in declarations")
if with_doc:
if not with_doc_only:
doc, body = _extract_docstring(body)
return doc, body
else:
return body
def p_positional_and_keyword_args(s, end_sy_set, templates = None):
"""
......@@ -2797,7 +2797,7 @@ def p_c_func_or_var_declaration(s, pos, ctx):
if s.sy == ':':
if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd', 'cpp_class') and not ctx.templates:
s.error("C function definition not allowed here")
doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1)
doc, suite = p_suite_with_docstring(s, Ctx(level='function'))
result = Nodes.CFuncDefNode(pos,
visibility = ctx.visibility,
base_type = base_type,
......@@ -2884,7 +2884,7 @@ def p_def_statement(s, decorators=None):
pos = s.position()
s.next()
name = EncodedString( p_ident(s) )
s.expect('(');
s.expect('(')
args, star_arg, starstar_arg = p_varargslist(s, terminator=')')
s.expect(')')
if p_nogil(s):
......@@ -2893,7 +2893,7 @@ def p_def_statement(s, decorators=None):
if s.sy == '->':
s.next()
return_type_annotation = p_test(s)
doc, body = p_suite(s, Ctx(level = 'function'), with_doc = 1)
doc, body = p_suite_with_docstring(s, Ctx(level='function'))
return Nodes.DefNode(pos, name = name, args = args,
star_arg = star_arg, starstar_arg = starstar_arg,
doc = doc, body = body, decorators = decorators,
......@@ -2944,8 +2944,8 @@ def p_class_statement(s, decorators):
pos, positional_args, keyword_args, star_arg, None)
if arg_tuple is None:
# XXX: empty arg_tuple
arg_tuple = ExprNodes.TupleNode(pos, args = [])
doc, body = p_suite(s, Ctx(level = 'class'), with_doc = 1)
arg_tuple = ExprNodes.TupleNode(pos, args=[])
doc, body = p_suite_with_docstring(s, Ctx(level='class'))
return Nodes.PyClassDefNode(pos,
name = class_name,
bases = arg_tuple,
......@@ -2993,7 +2993,7 @@ def p_c_class_definition(s, pos, ctx):
body_level = 'c_class_pxd'
else:
body_level = 'c_class'
doc, body = p_suite(s, Ctx(level = body_level), with_doc = 1)
doc, body = p_suite_with_docstring(s, Ctx(level=body_level))
else:
s.expect_newline("Syntax error in C class definition")
doc = None
......@@ -3050,12 +3050,15 @@ def p_c_class_options(s):
s.expect(']', "Expected 'object' or 'type'")
return objstruct_name, typeobj_name
def p_property_decl(s):
pos = s.position()
s.next() # 'property'
name = p_ident(s)
doc, body = p_suite(s, Ctx(level = 'property'), with_doc = 1)
return Nodes.PropertyNode(pos, name = name, doc = doc, body = body)
doc, body = p_suite_with_docstring(
s, Ctx(level='property'), with_doc_only=True)
return Nodes.PropertyNode(pos, name=name, doc=doc, body=body)
def p_doc_string(s):
if s.sy == 'BEGIN_STRING':
......@@ -3070,6 +3073,42 @@ def p_doc_string(s):
else:
return None
def _extract_docstring(node):
"""
Extract a docstring from a statement or from the first statement
in a list. Remove the statement if found. Return a tuple
(plain-docstring or None, node).
"""
doc_node = None
if node is None:
pass
elif isinstance(node, Nodes.ExprStatNode):
if node.expr.is_string_literal:
doc_node = node.expr
node = Nodes.StatListNode(node.pos, stats=[])
elif isinstance(node, Nodes.StatListNode) and node.stats:
stats = node.stats
if isinstance(stats[0], Nodes.ExprStatNode):
if stats[0].expr.is_string_literal:
doc_node = stats[0].expr
del stats[0]
if doc_node is None:
doc = None
elif isinstance(doc_node, ExprNodes.BytesNode):
warning(node.pos,
"Python 3 requires docstrings to be unicode strings")
doc = doc_node.value
elif isinstance(doc_node, ExprNodes.StringNode):
doc = doc_node.unicode_value
if doc is None:
doc = doc_node.value
else:
doc = doc_node.value
return doc, node
def p_code(s, level=None, ctx=Ctx):
body = p_statement_list(s, ctx(level = level), first_statement = 1)
if s.sy != 'EOF':
......
......@@ -3,7 +3,7 @@
# More comments
u'A module docstring'
'A module docstring'
doctest = u"""# Python 3 gets all of these right ...
>>> __doc__
......@@ -28,48 +28,44 @@ doctest = u"""# Python 3 gets all of these right ...
Compare with standard Python:
>>> def Pyf():
... u'''
... '''
... This is a function docstring.
... '''
>>> Pyf.__doc__
u'\\n This is a function docstring.\\n '
'\\n This is a function docstring.\\n '
>>> class PyC:
... u'''
... '''
... This is a class docstring.
... '''
>>> class PyCS(C):
... u'''
... '''
... This is a subclass docstring.
... '''
>>> class PyCSS(CS):
... pass
>>> PyC.__doc__
u'\\n This is a class docstring.\\n '
'\\n This is a class docstring.\\n '
>>> PyCS.__doc__
u'\\n This is a subclass docstring.\\n '
'\\n This is a subclass docstring.\\n '
>>> PyCSS.__doc__
"""
import sys
if sys.version_info[0] >= 3:
doctest = doctest.replace(u" u'", u" '")
__test__ = {u"test_docstrings" : doctest}
__test__ = {"test_docstrings" : doctest}
def f():
u"""
"""
This is a function docstring.
"""
class C:
u"""
"""
This is a class docstring.
"""
class CS(C):
u"""
"""
This is a subclass docstring.
"""
......@@ -77,14 +73,32 @@ class CSS(CS):
pass
cdef class T:
u"""
"""
This is an extension type docstring.
"""
cdef class TS(T):
u"""
"""
This is an extension subtype docstring.
"""
cdef class TSS(TS):
pass
def n():
"This is not a docstring".lower()
class PyN(object):
u"This is not a docstring".lower()
cdef class CN(object):
b"This is not a docstring".lower()
def test_non_docstrings():
"""
>>> n.__doc__
>>> PyN.__doc__
>>> CN.__doc__
"""
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment