Add some lifetime information to temporaries in the cfg

By adding a 'is_kill' flag to AST_Name nodes, which say that
this is the last use of this name.  This handles some common cases
of keeping temporaries alive for too long.

For some rare cases, there is no AST_Name that is a last use:
for example, the iterator object of a for loop is live after
every time it is used, but dies when exiting the loop.

For those, we insert a `del #foo` instead.

The interpreter and bjit use these flags to remove temporaries
from the vregs.

The LLVM jit ignores them since it has its own way of handling
lifetime.  It ignores any `del` statements on temporary names
as an optimization.

This does not handle decref'ing temporaries on an exception.

src/analysis/function_analysis.cpp

@@ -136,8 +136,13 @@ public:
     bool visit_name(AST_Name* node) {
         if (node->ctx_type == AST_TYPE::Load)
             _doLoad(node->id, node);
-        else if (node->ctx_type == AST_TYPE::Store || node->ctx_type == AST_TYPE::Del
-                 || node->ctx_type == AST_TYPE::Param)
+        else if (node->ctx_type == AST_TYPE::Del) {
+            // Hack: we don't have a bytecode for temporary-kills:
+            if (node->id.s()[0] == '#')
+                return true;
+            _doLoad(node->id, node);
+            _doStore(node->id);
+        } else if (node->ctx_type == AST_TYPE::Store || node->ctx_type == AST_TYPE::Param)
             _doStore(node->id);
         else {
             ASSERT(0, "%d", node->ctx_type);
@@ -145,6 +150,7 @@ public:
         }
         return true;
     }
+
     bool visit_alias(AST_alias* node) {
         InternedString name = node->name;
         if (node->asname.s().size())

src/codegen/ast_interpreter.cpp

@@ -1080,6 +1080,24 @@ Value ASTInterpreter::visit_return(AST_Return* node) {
         finishJITing();
     }
 
+// Some day, we should make sure that all temporaries got deleted (and decrefed) at the right time:
+#if 0
+    bool temporaries_alive = false;
+#ifndef NDEBUG
+    for (auto&& v : getSymVRegMap()) {
+        if (v.first.s()[0] == '#' && vregs[v.second]) {
+            fprintf(stderr, "%s still alive\n", v.first.c_str());
+            temporaries_alive = true;
+        }
+    }
+
+    if (temporaries_alive)
+        source_info->cfg->print();
+
+    assert(!temporaries_alive);
+#endif
+#endif
+
     next_block = 0;
     return s;
 }
@@ -1332,14 +1350,21 @@ Value ASTInterpreter::visit_delete(AST_Delete* node) {
                         jit->emitDelName(target->id);
                     ASTInterpreterJitInterface::delNameHelper(this, target->id);
                 } else {
-                    abortJITing();
                     assert(vst == ScopeInfo::VarScopeType::FAST);
 
                     assert(getSymVRegMap().count(target->id));
                     assert(getSymVRegMap()[target->id] == target->vreg);
-                    if (vregs[target->vreg] == 0) {
-                        assertNameDefined(0, target->id.c_str(), NameError, true /* local_var_msg */);
-                        return Value();
+
+                    if (target->id.s()[0] == '#') {
+                        assert(vregs[target->vreg] != NULL);
+                        if (jit)
+                            jit->emitKillTemporary(target->id, target->vreg);
+                    } else {
+                        abortJITing();
+                        if (vregs[target->vreg] == 0) {
+                            assertNameDefined(0, target->id.c_str(), NameError, true /* local_var_msg */);
+                            return Value();
+                        }
                     }
 
                     Py_DECREF(vregs[target->vreg]);
@@ -1635,6 +1660,7 @@ Value ASTInterpreter::visit_name(AST_Name* node) {
 
     switch (node->lookup_type) {
         case ScopeInfo::VarScopeType::GLOBAL: {
+            assert(!node->is_kill);
             Value v;
             if (jit)
                 v.var = jit->emitGetGlobal(frame_info.globals, node->id.getBox());
@@ -1643,6 +1669,7 @@ Value ASTInterpreter::visit_name(AST_Name* node) {
             return v;
         }
         case ScopeInfo::VarScopeType::DEREF: {
+            assert(!node->is_kill);
             return Value(ASTInterpreterJitInterface::derefHelper(this, node->id),
                          jit ? jit->emitDeref(node->id) : NULL);
         }
@@ -1651,22 +1678,35 @@ Value ASTInterpreter::visit_name(AST_Name* node) {
             Value v;
             if (jit) {
                 bool is_live = true;
-                if (node->lookup_type == ScopeInfo::VarScopeType::FAST)
+                if (node->is_kill) {
+                    is_live = false;
+                    assert(!source_info->getLiveness()->isLiveAtEnd(node->id, current_block));
+                } else if (node->lookup_type == ScopeInfo::VarScopeType::FAST)
                     is_live = source_info->getLiveness()->isLiveAtEnd(node->id, current_block);
 
-                if (is_live)
+                if (is_live) {
+                    assert(!node->is_kill);
                     v.var = jit->emitGetLocal(node->id, node->vreg);
-                else
+                } else {
                     v.var = jit->emitGetBlockLocal(node->id, node->vreg);
+                    if (node->is_kill) {
+                        assert(node->id.s()[0] == '#');
+                        jit->emitKillTemporary(node->id, node->vreg);
+                    }
+                }
             }
 
             assert(node->vreg >= 0);
             assert(getSymVRegMap().count(node->id));
             assert(getSymVRegMap()[node->id] == node->vreg);
             Box* val = vregs[node->vreg];
+
             if (val) {
-                Py_INCREF(val);
                 v.o = val;
+                if (node->is_kill)
+                    vregs[node->vreg] = NULL;
+                else
+                    Py_INCREF(val);
                 return v;
             }
 
@@ -1674,6 +1714,7 @@ Value ASTInterpreter::visit_name(AST_Name* node) {
             RELEASE_ASSERT(0, "should be unreachable");
         }
         case ScopeInfo::VarScopeType::NAME: {
+            assert(!node->is_kill && "we might need to support this");
             Value v;
             if (jit)
                 v.var = jit->emitGetBoxedLocal(node->id.getBox());

src/codegen/baseline_jit.cpp

@@ -379,6 +379,10 @@ RewriterVar* JitFragmentWriter::emitGetBlockLocal(InternedString s, int vreg) {
     return it->second;
 }
 
+void JitFragmentWriter::emitKillTemporary(InternedString s, int vreg) {
+    emitSetLocal(s, vreg, false, imm(nullptr));
+}
+
 RewriterVar* JitFragmentWriter::emitGetBoxedLocal(BoxedString* s) {
     RewriterVar* boxed_locals = emitGetBoxedLocals();
     RewriterVar* globals = getInterp()->getAttr(ASTInterpreterJitInterface::getGlobalsOffset());

src/codegen/baseline_jit.h

@@ -230,6 +230,7 @@ public:
     RewriterVar* emitExceptionMatches(RewriterVar* v, RewriterVar* cls);
     RewriterVar* emitGetAttr(RewriterVar* obj, BoxedString* s, AST_expr* node);
     RewriterVar* emitGetBlockLocal(InternedString s, int vreg);
+    void emitKillTemporary(InternedString s, int vreg);
     RewriterVar* emitGetBoxedLocal(BoxedString* s);
     RewriterVar* emitGetBoxedLocals();
     RewriterVar* emitGetClsAttr(RewriterVar* obj, BoxedString* s);

src/codegen/irgen/irgenerator.cpp

@@ -2123,6 +2123,12 @@ private:
     }
 
     void _doDelName(AST_Name* target, const UnwindInfo& unw_info) {
+        // Hack: we don't have a bytecode for temporary-kills:
+        if (target->id.s()[0] == '#') {
+            // The refcounter will automatically delete this object.
+            return;
+        }
+
         auto scope_info = irstate->getScopeInfo();
         ScopeInfo::VarScopeType vst = scope_info->getScopeTypeOfName(target->id);
         if (vst == ScopeInfo::VarScopeType::GLOBAL) {

src/core/ast.cpp

@@ -1630,7 +1630,8 @@ bool PrintVisitor::visit_suite(AST_Suite* node) {
 
 bool PrintVisitor::visit_name(AST_Name* node) {
     stream << node->id.s();
-    // printf("%s(%d)", node->id.c_str(), node->ctx_type);
+    // Uncomment this line to see which names are kills:
+    // if (node->is_kill) stream << "<k>";
     return false;
 }
 

src/core/ast.h

@@ -735,6 +735,8 @@ public:
     // the zero based index of this variable inside the vregs array. If uninitialized it's value is -1.
     int vreg;
 
+    bool is_kill = false;
+
     virtual void accept(ASTVisitor* v);
     virtual void* accept_expr(ExprVisitor* v);
 

test/tests/local_lifetime.py

+from _weakref import ref
+
+# Test to make sure that we clear local variables at the right time:
+def f1():
+    def f():
+        pass
+    r = ref(f)
+    print type(r())
+    # del f
+    f = 1
+    assert not r()
+for i in xrange(40):
+    f1()
+
+def f3():
+    class MyIter(object):
+        def __init__(self):
+            self.n = 5
+        def __del__(self):
+            print "deleting iter"
+        def next(self):
+            if self.n:
+                self.n -= 1
+                return self.n
+            raise StopIteration()
+
+    class MyIterable(object):
+        def __iter__(self):
+            return MyIter()
+
+    for i in MyIterable():
+        print i
+    else:
+        print -1
+f3()

test/tests/resurrection.py

 # expected: fail
-#  - finalization (let alone resurrection) not implemented yet
-
 # Objects are allowed to resurrect themselves in their __del__ methods...
-# Note: the behavior here will differ from cPython and maybe PyPy
 
 x = None
 running = True