Merge pull request #1253 from kmod/linenumbers

Add more line numbers

src/codegen/ast_interpreter.cpp

@@ -1599,6 +1599,7 @@ Value ASTInterpreter::visit_repr(AST_Repr* node) {
 Value ASTInterpreter::visit_lambda(AST_Lambda* node) {
     AST_Return* expr = new AST_Return();
     expr->value = node->body;
+    expr->lineno = node->body->lineno;
 
     std::vector<AST_stmt*> body = { expr };
     return createFunction(node, node->args, body);

src/codegen/cpython_ast.cpp

@@ -499,6 +499,8 @@ public:
         r->type = convert(v.type);
         r->name = convert(v.name);
         r->body = convert<stmt_ty, AST_stmt*>(v.body);
+        r->lineno = eh->lineno;
+        r->col_offset = eh->col_offset;
         return r;
     }
 

src/codegen/irgen/hooks.cpp

@@ -403,6 +403,7 @@ static FunctionMetadata* compileEval(AST_Expression* parsedExpr, BoxedString* fn
     // We need body (list of statements) to compile.
     // Obtain this by simply making a single statement which contains the expression.
     AST_Return* stmt = new AST_Return();
+    stmt->lineno = parsedExpr->body->lineno;
     stmt->value = parsedExpr->body;
     std::vector<AST_stmt*> body = { stmt };
 

src/codegen/irgen/irgenerator.cpp

@@ -1240,6 +1240,7 @@ private:
     CompilerVariable* evalLambda(AST_Lambda* node, const UnwindInfo& unw_info) {
         AST_Return* expr = new AST_Return();
         expr->value = node->body;
+        expr->lineno = node->body->lineno;
 
         std::vector<AST_stmt*> body = { expr };
         CompilerVariable* func = _createFunction(node, unw_info, node->args, body);
@@ -2271,6 +2272,8 @@ private:
 
         ConcreteCompilerVariable* rtn = val->makeConverted(emitter, opt_rtn_type);
 
+        emitter.emitSetCurrentStmt(node);
+
         if (!irstate->getCurFunction()->entry_descriptor)
             emitter.getBuilder()->CreateCall(g.funcs.deinitFrame, irstate->getFrameInfoVar());
 
@@ -2667,16 +2670,15 @@ private:
                 ConcreteCompilerVariable* v = p.second->makeConverted(emitter, phi_type);
                 symbol_table[p.first] = v;
             } else {
-#ifndef NDEBUG
                 if (myblock->successors.size()) {
                     // TODO getTypeAtBlockEnd will automatically convert up to the concrete type, which we don't
                     // want
                     // here, but this is just for debugging so I guess let it happen for now:
                     ConcreteCompilerType* ending_type = types->getTypeAtBlockEnd(p.first, myblock);
-                    ASSERT(p.second->canConvertTo(ending_type), "%s is supposed to be %s, but somehow is %s",
-                           p.first.c_str(), ending_type->debugName().c_str(), p.second->getType()->debugName().c_str());
+                    RELEASE_ASSERT(p.second->canConvertTo(ending_type), "%s is supposed to be %s, but somehow is %s",
+                                   p.first.c_str(), ending_type->debugName().c_str(),
+                                   p.second->getType()->debugName().c_str());
                 }
-#endif
             }
         }
 

src/codegen/parser.cpp

@@ -1106,11 +1106,11 @@ AST_Module* parse_file(const char* fn, FutureFlags inherited_flags) {
 
 const char* getMagic() {
     if (ENABLE_CPYTHON_PARSER)
-        return "a\nCP";
+        return "a\nCQ";
     else if (ENABLE_PYPA_PARSER)
-        return "a\ncP";
+        return "a\ncQ";
     else
-        return "a\ncp";
+        return "a\ncq";
 }
 
 #define MAGIC_STRING_LENGTH 4
@@ -1272,7 +1272,7 @@ AST_Module* caching_parse_file(const char* fn, FutureFlags inherited_flags) {
         if (good) {
             if (strncmp(&file_data[0], getMagic(), MAGIC_STRING_LENGTH) != 0) {
                 oss << "magic string did not match\n";
-                if (VERBOSITY() || tries == MAX_TRIES) {
+                if (VERBOSITY() >= 2 || tries == MAX_TRIES) {
                     fprintf(stderr, "Warning: corrupt or non-Pyston .pyc file found; ignoring\n");
                     fprintf(stderr, "%d %d %d %d\n", file_data[0], file_data[1], file_data[2], file_data[3]);
                     fprintf(stderr, "%d %d %d %d\n", getMagic()[0], getMagic()[1], getMagic()[2], getMagic()[3]);

src/core/ast.h

@@ -1052,10 +1052,7 @@ public:
     virtual void accept(ASTVisitor* v);
     virtual void accept_stmt(StmtVisitor* v);
 
-    AST_Jump() : AST_stmt(AST_TYPE::Jump) {
-        lineno = -1;
-        col_offset = -1;
-    }
+    AST_Jump() : AST_stmt(AST_TYPE::Jump) {}
 
     static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Jump;
 };

src/runtime/descr.cpp

@@ -608,6 +608,7 @@ void setupDescr() {
     property_cls->giveAttrMember("__doc__", T_OBJECT, offsetof(BoxedProperty, prop_doc));
     property_cls->freeze();
 
+    staticmethod_cls->giveAttrMember("__func__", T_OBJECT, offsetof(BoxedStaticmethod, sm_callable));
     staticmethod_cls->giveAttr(
         "__init__", new BoxedFunction(FunctionMetadata::create((void*)staticmethodInit, UNKNOWN, 5, false, false),
                                       { None, None, None, None }));
@@ -617,6 +618,7 @@ void setupDescr() {
     staticmethod_cls->freeze();
 
 
+    classmethod_cls->giveAttrMember("__func__", T_OBJECT, offsetof(BoxedClassmethod, cm_callable));
     classmethod_cls->giveAttr(
         "__init__", new BoxedFunction(FunctionMetadata::create((void*)classmethodInit, UNKNOWN, 5, false, false),
                                       { None, None, None, None }));

src/runtime/dict.cpp

@@ -800,7 +800,7 @@ Box* dictUpdate(BoxedDict* self, BoxedTuple* args, BoxedDict* kwargs) {
     if (args->size()) {
         Box* arg = args->elts[0];
         static BoxedString* keys_str = getStaticString("keys");
-        if (autoXDecref(getattrInternal<ExceptionStyle::CXX>(arg, keys_str))) {
+        if (PyObject_HasAttr(arg, keys_str)) {
             dictMerge(self, arg);
         } else {
             dictMergeFromSeq2(self, arg);

src/runtime/frame.cpp

@@ -136,10 +136,13 @@ public:
     static Box* lineno(Box* obj, void*) noexcept {
         auto f = static_cast<BoxedFrame*>(obj);
 
-        if (f->hasExited())
+        if (f->hasExited()) {
+            ASSERT(f->_linenumber > 0 && f->_linenumber < 1000000, "%d", f->_linenumber);
             return boxInt(f->_linenumber);
+        }
 
         AST_stmt* stmt = f->frame_info->stmt;
+        ASSERT(stmt->lineno > 0 && stmt->lineno < 1000000, "%d", stmt->lineno);
         return boxInt(stmt->lineno);
     }
 
@@ -153,6 +156,7 @@ public:
         globals(this, NULL);
         assert(!_locals);
         _locals = incref(locals(this, NULL));
+        ASSERT(frame_info->stmt->lineno > 0 && frame_info->stmt->lineno < 1000000, "%d", frame_info->stmt->lineno);
         _linenumber = frame_info->stmt->lineno;
 
         frame_info = NULL; // this means exited == true

src/runtime/generator.cpp

@@ -474,7 +474,7 @@ extern "C" BoxedGenerator::BoxedGenerator(BoxedFunctionBase* function, Box* arg1
         // Interestingly, it seems like MAP_GROWSDOWN will leave a page-size gap between the redzone and the growable
         // region.
 
-        if (VERBOSITY() >= 1) {
+        if (VERBOSITY() >= 3) {
             printf("Created new generator stack, starts at %p, currently extends to %p\n", (void*)stack_high,
                    initial_stack_limit);
             printf("Created a redzone from %p-%p\n", (void*)stack_low, (void*)(stack_low + STACK_REDZONE_SIZE));

test/tests/dict.py

@@ -240,3 +240,16 @@ print d
 # Remove an item using a different key:
 d = {1:1}
 d.pop(1L)
+
+
+# dict() will try to access the "keys" attribute, but it should swallow all exceptions
+class MyObj(object):
+    def __iter__(self):
+        print "iter!"
+        return [(1, 2)].__iter__()
+
+    def __getattr__(self, attr):
+        print "getattr", attr
+        1/0
+
+print dict(MyObj())

test/tests/lineno_tricky.py

+# Some cases that are tricky to get f_lineno right.
+# (It's not super critical to get them right in all these cases, since
+# in some of them it only shows up if you inspect the frame after-exit
+# which seems not very common.  But we should at least try to return
+# something reasonable [non-zero]).
+
+import sys
+
+fr = None
+def f():
+    global fr
+    fr = sys._getframe(0)
+
+f()
+print fr.f_lineno
+
+s = """
+import sys
+fr = sys._getframe(0)
+""".strip()
+
+exec s
+print fr.f_lineno
+
+
+def f2():
+    try:
+        1/1
+        1/0
+        1/1
+    except ImportError:
+        assert 0
+        pass
+    except ValueError:
+        assert 0
+        def f():
+            pass
+try:
+    f2()
+    assert 0
+except:
+    # These will be different!
+    print sys.exc_info()[2].tb_next.tb_frame.f_lineno
+    print sys.exc_info()[2].tb_next.tb_lineno
+
+def f5():
+    print "f5"
+    try:
+        1/0
+    finally:
+        pass
+try:
+    f5()
+    assert 0
+except:
+    # These will be different!
+    print sys.exc_info()[2].tb_next.tb_frame.f_lineno
+    print sys.exc_info()[2].tb_next.tb_lineno
+
+def f6():
+    print "f6"
+    with open("/dev/null"):
+        1/0
+        1/1
+try:
+    f6()
+    assert 0
+except:
+    # These will be different!
+    print sys.exc_info()[2].tb_next.tb_frame.f_lineno
+    print sys.exc_info()[2].tb_next.tb_lineno
+
+def f3(n):
+    global fr
+    fr = sys._getframe(0)
+    try:
+        1/n
+    except ZeroDivisionError:
+        pass
+    except:
+        pass
+
+f3(1)
+print fr.f_lineno
+f3(0)
+print fr.f_lineno
+
+def f4():
+    global fr
+    fr = sys._getframe(0)
+    yield 1
+    yield 2
+    yield 3
+g = f4()
+print g.next()
+print fr.f_lineno
+
+fr_l = []
+g = (i for i in [fr_l.append(sys._getframe(0)), xrange(10)][1])
+print g.next()
+print fr_l[0].f_lineno
+
+print repr(eval(u"\n\n__import__('sys')._getframe(0).f_lineno"))

test/tests/lineno_tricky_multiline.py

+# expected: fail
+# - see cfg.cpp::getLastLinenoSub()
+
+import sys
+
+def f():
+    1/ (len((1/1,
+            1/1,
+            1/1)) - 3)
+
+try:
+    f()
+    assert 0
+except ZeroDivisionError:
+    print sys.exc_info()[2].tb_next.tb_lineno
+    print sys.exc_info()[2].tb_next.tb_frame.f_lineno

test/tests/static_class_methods.py

@@ -11,6 +11,11 @@ c = C()
 c.f(1, 2, 3, 4)
 c.g(5, 6, 7, 8)
 
+assert C.__dict__['f'].__func__ is C.f
+assert C.__dict__['g'].__func__ is C.g.im_func
+c.f.__call__(1, 2, 3, 4)
+c.g.__call__(5, 6, 7, 8)
+
 C.f(9, 10, 11, 12)
 C.f(13, 14, 15, 16)
 

test/tests/type_phi_test.py

+# fail-if: '-n' not in EXTRA_JIT_ARGS and '-O' not in EXTRA_JIT_ARGS
+# - This test needs type profiling to be enabled to trigger the bug
+
+# This test throws an irgen assertion.
+# The issue is that type analysis is stronger than phi analysis and
+# they produce slightly different results.
+
+import thread
+l = thread.allocate_lock()
+
+class L(object):
+    def __enter__(self):
+        return self
+    def __exit__(self, *args):
+        pass
+L = thread.allocate_lock
+
+class C(object):
+    def __init__(self):
+        self.lock = L()
+    def f(self):
+        try:
+            with self.lock:
+                for i in []:
+                    pass
+        finally:
+            pass
+
+c = C()
+for i in xrange(10000):
+    c.f()