Improve lineno handling

We've been a bit lazy about adding line numbers to all ast nodes.
This is an inconvenience when the tracebacks are bad/wrong, and it's
also causing issues for testing code that wants to look at the line
at which something failed.

So I added line numbers to a whole bunch more nodes.  There are still
quite a few that don't have line numbers, but many of them are harmless,
so I tried to identify the ones that matter.  I also added some assertions
to make sure that bad (non-positive) line numbers don't escape to the user.

src/codegen/ast_interpreter.cpp

@@ -1599,6 +1599,7 @@ Value ASTInterpreter::visit_repr(AST_Repr* node) {
 Value ASTInterpreter::visit_lambda(AST_Lambda* node) {
     AST_Return* expr = new AST_Return();
     expr->value = node->body;
+    expr->lineno = node->body->lineno;
 
     std::vector<AST_stmt*> body = { expr };
     return createFunction(node, node->args, body);

src/codegen/cpython_ast.cpp

@@ -499,6 +499,8 @@ public:
         r->type = convert(v.type);
         r->name = convert(v.name);
         r->body = convert<stmt_ty, AST_stmt*>(v.body);
+        r->lineno = eh->lineno;
+        r->col_offset = eh->col_offset;
         return r;
     }
 

src/codegen/irgen/hooks.cpp

@@ -403,6 +403,7 @@ static FunctionMetadata* compileEval(AST_Expression* parsedExpr, BoxedString* fn
     // We need body (list of statements) to compile.
     // Obtain this by simply making a single statement which contains the expression.
     AST_Return* stmt = new AST_Return();
+    stmt->lineno = parsedExpr->body->lineno;
     stmt->value = parsedExpr->body;
     std::vector<AST_stmt*> body = { stmt };
 

src/codegen/irgen/irgenerator.cpp

@@ -1240,6 +1240,7 @@ private:
     CompilerVariable* evalLambda(AST_Lambda* node, const UnwindInfo& unw_info) {
         AST_Return* expr = new AST_Return();
         expr->value = node->body;
+        expr->lineno = node->body->lineno;
 
         std::vector<AST_stmt*> body = { expr };
         CompilerVariable* func = _createFunction(node, unw_info, node->args, body);
@@ -2271,6 +2272,8 @@ private:
 
         ConcreteCompilerVariable* rtn = val->makeConverted(emitter, opt_rtn_type);
 
+        emitter.emitSetCurrentStmt(node);
+
         if (!irstate->getCurFunction()->entry_descriptor)
             emitter.getBuilder()->CreateCall(g.funcs.deinitFrame, irstate->getFrameInfoVar());
 

src/codegen/parser.cpp

@@ -1106,11 +1106,11 @@ AST_Module* parse_file(const char* fn, FutureFlags inherited_flags) {
 
 const char* getMagic() {
     if (ENABLE_CPYTHON_PARSER)
-        return "a\nCP";
+        return "a\nCQ";
     else if (ENABLE_PYPA_PARSER)
-        return "a\ncP";
+        return "a\ncQ";
     else
-        return "a\ncp";
+        return "a\ncq";
 }
 
 #define MAGIC_STRING_LENGTH 4

src/core/ast.h

@@ -1052,10 +1052,7 @@ public:
     virtual void accept(ASTVisitor* v);
     virtual void accept_stmt(StmtVisitor* v);
 
-    AST_Jump() : AST_stmt(AST_TYPE::Jump) {
-        lineno = -1;
-        col_offset = -1;
-    }
+    AST_Jump() : AST_stmt(AST_TYPE::Jump) {}
 
     static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Jump;
 };

src/runtime/frame.cpp

@@ -136,10 +136,13 @@ public:
     static Box* lineno(Box* obj, void*) noexcept {
         auto f = static_cast<BoxedFrame*>(obj);
 
-        if (f->hasExited())
+        if (f->hasExited()) {
+            ASSERT(f->_linenumber > 0 && f->_linenumber < 1000000, "%d", f->_linenumber);
             return boxInt(f->_linenumber);
+        }
 
         AST_stmt* stmt = f->frame_info->stmt;
+        ASSERT(stmt->lineno > 0 && stmt->lineno < 1000000, "%d", stmt->lineno);
         return boxInt(stmt->lineno);
     }
 
@@ -153,6 +156,7 @@ public:
         globals(this, NULL);
         assert(!_locals);
         _locals = incref(locals(this, NULL));
+        ASSERT(frame_info->stmt->lineno > 0 && frame_info->stmt->lineno < 1000000, "%d", frame_info->stmt->lineno);
         _linenumber = frame_info->stmt->lineno;
 
         frame_info = NULL; // this means exited == true

test/tests/lineno_tricky.py

+# Some cases that are tricky to get f_lineno right.
+# (It's not super critical to get them right in all these cases, since
+# in some of them it only shows up if you inspect the frame after-exit
+# which seems not very common.  But we should at least try to return
+# something reasonable [non-zero]).
+
+import sys
+
+fr = None
+def f():
+    global fr
+    fr = sys._getframe(0)
+
+f()
+print fr.f_lineno
+
+s = """
+import sys
+fr = sys._getframe(0)
+""".strip()
+
+exec s
+print fr.f_lineno
+
+
+def f2():
+    try:
+        1/1
+        1/0
+        1/1
+    except ImportError:
+        assert 0
+        pass
+    except ValueError:
+        assert 0
+        def f():
+            pass
+try:
+    f2()
+    assert 0
+except:
+    # These will be different!
+    print sys.exc_info()[2].tb_next.tb_frame.f_lineno
+    print sys.exc_info()[2].tb_next.tb_lineno
+
+def f5():
+    print "f5"
+    try:
+        1/0
+    finally:
+        pass
+try:
+    f5()
+    assert 0
+except:
+    # These will be different!
+    print sys.exc_info()[2].tb_next.tb_frame.f_lineno
+    print sys.exc_info()[2].tb_next.tb_lineno
+
+def f6():
+    print "f6"
+    with open("/dev/null"):
+        1/0
+        1/1
+try:
+    f6()
+    assert 0
+except:
+    # These will be different!
+    print sys.exc_info()[2].tb_next.tb_frame.f_lineno
+    print sys.exc_info()[2].tb_next.tb_lineno
+
+def f3(n):
+    global fr
+    fr = sys._getframe(0)
+    try:
+        1/n
+    except ZeroDivisionError:
+        pass
+    except:
+        pass
+
+f3(1)
+print fr.f_lineno
+f3(0)
+print fr.f_lineno
+
+def f4():
+    global fr
+    fr = sys._getframe(0)
+    yield 1
+    yield 2
+    yield 3
+g = f4()
+print g.next()
+print fr.f_lineno
+
+fr_l = []
+g = (i for i in [fr_l.append(sys._getframe(0)), xrange(10)][1])
+print g.next()
+print fr_l[0].f_lineno
+
+print repr(eval(u"\n\n__import__('sys')._getframe(0).f_lineno"))

test/tests/lineno_tricky_multiline.py

+# expected: fail
+# - see cfg.cpp::getLastLinenoSub()
+
+import sys
+
+def f():
+    1/ (len((1/1,
+            1/1,
+            1/1)) - 3)
+
+try:
+    f()
+    assert 0
+except ZeroDivisionError:
+    print sys.exc_info()[2].tb_next.tb_lineno
+    print sys.exc_info()[2].tb_next.tb_frame.f_lineno