Merge pull request #1144 from kmod/bjit

Fix + reenable the bjit

pyston_gdbinit.txt

@@ -2,14 +2,18 @@ handle SIGUSR2 pass nostop noprint
 
 skip file /usr/include/c++/4.8/bits/stl_vector.h
 skip file /usr/include/c++/4.9/bits/stl_vector.h
+skip file /usr/include/c++/5/bits/stl_vector.h
 skip file /usr/include/c++/4.8/bits/unique_ptr.h
 skip file /usr/include/c++/4.9/bits/unique_ptr.h
+skip file /usr/include/c++/5/bits/unique_ptr.h
 skip file /usr/include/c++/4.8/bits/move.h
 skip file /usr/include/c++/4.9/bits/move.h
+skip file /usr/include/c++/5/bits/move.h
 
 skip pyston::ArgPassSpec::ArgPassSpec(int, int, bool, bool)
 skip pyston::InternedString::getBox() const
 skip pyston::BoxedString::s() const
+skip pyston::Value::operator pyston::RewriterVar*()
 
 skip pyston::autoDecref<pyston::Box, false>(pyston::Box*)
 skip pyston::autoDecref<pyston::Box, true>(pyston::Box*)

src/asm_writing/icinfo.cpp

@@ -23,6 +23,7 @@
 #include "asm_writing/assembler.h"
 #include "asm_writing/mc_writer.h"
 #include "codegen/patchpoints.h"
+#include "codegen/unwinding.h"
 #include "core/common.h"
 #include "core/options.h"
 #include "core/types.h"
@@ -53,6 +54,7 @@ void ICInvalidator::invalidateAll() {
 
 void ICSlotInfo::clear() {
     ic->clear(this);
+    decref_infos.clear();
 }
 
 ICSlotRewrite::ICSlotRewrite(ICInfo* ic, const char* debug_name)
@@ -82,7 +84,8 @@ uint8_t* ICSlotRewrite::getSlotStart() {
     return (uint8_t*)ic->start_addr + ic_entry->idx * ic->getSlotSize();
 }
 
-void ICSlotRewrite::commit(CommitHook* hook, std::vector<void*> gc_references) {
+void ICSlotRewrite::commit(CommitHook* hook, std::vector<void*> gc_references,
+                           std::vector<std::pair<uint64_t, std::vector<Location>>> decref_infos) {
     bool still_valid = true;
     for (int i = 0; i < dependencies.size(); i++) {
         int orig_version = dependencies[i].second;
@@ -130,6 +133,21 @@ void ICSlotRewrite::commit(CommitHook* hook, std::vector<void*> gc_references) {
         megamorphic_ics.log();
     }
 
+    // deregister old decref infos
+    ic_entry->decref_infos.clear();
+
+    // register new decref info
+    for (auto&& decref_info : decref_infos) {
+        // add decref locations which are always to decref inside this IC
+        auto&& merged_locations = decref_info.second;
+        merged_locations.insert(merged_locations.end(), ic->ic_global_decref_locations.begin(),
+                                ic->ic_global_decref_locations.end());
+        if (merged_locations.empty())
+            continue;
+
+        ic_entry->decref_infos.emplace_back(decref_info.first, std::move(merged_locations));
+    }
+
     llvm::sys::Memory::InvalidateInstructionCache(slot_start, ic->getSlotSize());
 }
 
@@ -207,7 +225,8 @@ void deregisterGCTrackedICInfo(ICInfo* ic) {
 
 ICInfo::ICInfo(void* start_addr, void* slowpath_rtn_addr, void* continue_addr, StackInfo stack_info, int num_slots,
                int slot_size, llvm::CallingConv::ID calling_conv, LiveOutSet _live_outs,
-               assembler::GenericRegister return_register, TypeRecorder* type_recorder)
+               assembler::GenericRegister return_register, TypeRecorder* type_recorder,
+               std::vector<Location> ic_global_decref_locations)
     : next_slot_to_try(0),
       stack_info(stack_info),
       num_slots(num_slots),
@@ -219,6 +238,7 @@ ICInfo::ICInfo(void* start_addr, void* slowpath_rtn_addr, void* continue_addr, S
       retry_in(0),
       retry_backoff(1),
       times_rewritten(0),
+      ic_global_decref_locations(std::move(ic_global_decref_locations)),
       start_addr(start_addr),
       slowpath_rtn_addr(slowpath_rtn_addr),
       continue_addr(continue_addr) {
@@ -226,6 +246,8 @@ ICInfo::ICInfo(void* start_addr, void* slowpath_rtn_addr, void* continue_addr, S
     for (int i = 0; i < num_slots; i++) {
         slots.emplace_back(this, i);
     }
+    if (slowpath_rtn_addr && !this->ic_global_decref_locations.empty())
+        slowpath_decref_info = DecrefInfo((uint64_t)slowpath_rtn_addr, this->ic_global_decref_locations);
 
 #if MOVING_GC
     assert(ics_list.count(this) == 0);
@@ -238,9 +260,21 @@ ICInfo::~ICInfo() {
 #endif
 }
 
+DecrefInfo::DecrefInfo(uint64_t ip, std::vector<Location> locations) : ip(ip) {
+    addDecrefInfoEntry(ip, std::move(locations));
+}
+
+void DecrefInfo::reset() {
+    if (ip) {
+        removeDecrefInfoEntry(ip);
+        ip = 0;
+    }
+}
+
 std::unique_ptr<ICInfo> registerCompiledPatchpoint(uint8_t* start_addr, uint8_t* slowpath_start_addr,
                                                    uint8_t* continue_addr, uint8_t* slowpath_rtn_addr,
-                                                   const ICSetupInfo* ic, StackInfo stack_info, LiveOutSet live_outs) {
+                                                   const ICSetupInfo* ic, StackInfo stack_info, LiveOutSet live_outs,
+                                                   std::vector<Location> decref_info) {
     assert(slowpath_start_addr - start_addr >= ic->num_slots * ic->slot_size);
     assert(slowpath_rtn_addr > slowpath_start_addr);
     assert(slowpath_rtn_addr <= start_addr + ic->totalSize());
@@ -276,8 +310,9 @@ std::unique_ptr<ICInfo> registerCompiledPatchpoint(uint8_t* start_addr, uint8_t*
         writer.jmp(JumpDestination::fromStart(slowpath_start_addr - start));
     }
 
-    ICInfo* icinfo = new ICInfo(start_addr, slowpath_rtn_addr, continue_addr, stack_info, ic->num_slots, ic->slot_size,
-                                ic->getCallingConvention(), std::move(live_outs), return_register, ic->type_recorder);
+    ICInfo* icinfo
+        = new ICInfo(start_addr, slowpath_rtn_addr, continue_addr, stack_info, ic->num_slots, ic->slot_size,
+                     ic->getCallingConvention(), std::move(live_outs), return_register, ic->type_recorder, decref_info);
 
     assert(!ics_by_return_addr.count(slowpath_rtn_addr));
     ics_by_return_addr[slowpath_rtn_addr] = icinfo;
@@ -355,6 +390,17 @@ ICInfo* ICInfo::getICInfoForNode(AST* node) {
 void ICInfo::associateNodeWithICInfo(AST* node) {
     ics_by_ast_node[node] = this;
 }
+void ICInfo::appendDecrefInfosTo(std::vector<DecrefInfo>& dest_decref_infos) {
+    if (slowpath_decref_info.ip)
+        dest_decref_infos.emplace_back(std::move(slowpath_decref_info));
+    for (auto&& slot : slots) {
+        for (DecrefInfo& decref_info : slot.decref_infos) {
+            dest_decref_infos.emplace_back(std::move(decref_info));
+            assert(decref_info.ip == 0 && "this can only happen if we copied instead of moved the value");
+        }
+        slot.decref_infos.clear();
+    }
+}
 
 void clearAllICs() {
     for (auto&& p : ics_by_ast_node) {

src/asm_writing/icinfo.h

@@ -35,6 +35,24 @@ class ICInvalidator;
 #define IC_INVALDITION_HEADER_SIZE 6
 #define IC_MEGAMORPHIC_THRESHOLD 100
 
+// This registers a decref info in the constructor and deregisters it in the destructor.
+struct DecrefInfo {
+    uint64_t ip;
+
+    DecrefInfo() : ip(0) {}
+    DecrefInfo(uint64_t ip, std::vector<Location> locations);
+
+    // we only allow moves
+    DecrefInfo(DecrefInfo&& other) : ip(0) { std::swap(other.ip, ip); }
+    DecrefInfo& operator=(DecrefInfo&& other) {
+        std::swap(other.ip, ip);
+        return *this;
+    }
+    ~DecrefInfo() { reset(); }
+
+    void reset();
+};
+
 struct ICSlotInfo {
 public:
     ICSlotInfo(ICInfo* ic, int idx) : ic(ic), idx(idx), num_inside(0) {}
@@ -44,6 +62,7 @@ public:
     int num_inside; // the number of stack frames that are currently inside this slot
 
     std::vector<void*> gc_references;
+    std::vector<DecrefInfo> decref_infos;
 
     void clear();
 };
@@ -85,7 +104,8 @@ public:
     ICSlotInfo* prepareEntry();
 
     void addDependenceOn(ICInvalidator&);
-    void commit(CommitHook* hook, std::vector<void*> gc_references);
+    void commit(CommitHook* hook, std::vector<void*> gc_references,
+                std::vector<std::pair<uint64_t, std::vector<Location>>> decref_infos);
     void abort();
 
     const ICInfo* getICInfo() { return ic; }
@@ -117,13 +137,20 @@ private:
     int retry_in, retry_backoff;
     int times_rewritten;
 
+    DecrefInfo slowpath_decref_info;
+    // This is a vector of locations which always need to get decrefed inside this IC.
+    // Calls inside the ICSlots may need to decref additional locations but they will always contain at least the IC
+    // global ones.
+    std::vector<Location> ic_global_decref_locations;
+
     // for ICSlotRewrite:
     ICSlotInfo* pickEntryForRewrite(const char* debug_name);
 
 public:
     ICInfo(void* start_addr, void* slowpath_rtn_addr, void* continue_addr, StackInfo stack_info, int num_slots,
            int slot_size, llvm::CallingConv::ID calling_conv, LiveOutSet live_outs,
-           assembler::GenericRegister return_register, TypeRecorder* type_recorder);
+           assembler::GenericRegister return_register, TypeRecorder* type_recorder,
+           std::vector<Location> ic_global_decref_locations);
     ~ICInfo();
     void* const start_addr, *const slowpath_rtn_addr, *const continue_addr;
 
@@ -152,6 +179,8 @@ public:
 
     static ICInfo* getICInfoForNode(AST* node);
     void associateNodeWithICInfo(AST* node);
+
+    void appendDecrefInfosTo(std::vector<DecrefInfo>& dest_decref_infos);
 };
 
 void registerGCTrackedICInfo(ICInfo* ic);
@@ -159,9 +188,10 @@ void deregisterGCTrackedICInfo(ICInfo* ic);
 
 class ICSetupInfo;
 struct CompiledFunction;
-std::unique_ptr<ICInfo> registerCompiledPatchpoint(uint8_t* start_addr, uint8_t* slowpath_start_addr,
-                                                   uint8_t* continue_addr, uint8_t* slowpath_rtn_addr,
-                                                   const ICSetupInfo*, StackInfo stack_info, LiveOutSet live_outs);
+std::unique_ptr<ICInfo>
+registerCompiledPatchpoint(uint8_t* start_addr, uint8_t* slowpath_start_addr, uint8_t* continue_addr,
+                           uint8_t* slowpath_rtn_addr, const ICSetupInfo*, StackInfo stack_info, LiveOutSet live_outs,
+                           std::vector<Location> ic_global_decref_locations = std::vector<Location>());
 void deregisterCompiledPatchpoint(ICInfo* ic);
 
 ICInfo* getICInfo(void* rtn_addr);

src/asm_writing/rewriter.cpp

@@ -681,9 +681,15 @@ void Rewriter::_toBool(RewriterVar* result, RewriterVar* var, Location dest) {
     assertConsistent();
 }
 
-void RewriterVar::setAttr(int offset, RewriterVar* val) {
+void RewriterVar::setAttr(int offset, RewriterVar* val, SetattrType type) {
     STAT_TIMER(t0, "us_timer_rewriter", 10);
 
+    // Check that the caller promises to handle lifetimes appropriately.
+    // We're only interested in OWNED references, since we are trying to
+    // prevent store-in-array-and-pass situations where the refcounter will
+    // decref between the store and the pass.
+    if (val->reftype == RefType::OWNED)
+        assert(type != SetattrType::UNKNOWN);
     rewriter->addAction([=]() { rewriter->_setAttr(this, offset, val); }, { this, val }, ActionType::MUTATION);
 }
 
@@ -1186,6 +1192,9 @@ void Rewriter::_call(RewriterVar* result, bool has_side_effects, void* func_addr
         assert(assembler->hasFailed() || asm_address == (uint64_t)assembler->curInstPointer());
     }
 
+    // TODO: we don't need to generate the decref info for calls which can't throw
+    registerDecrefInfoHere();
+
     if (!failed) {
         assert(vars_by_location.count(assembler::RAX) == 0);
 
@@ -1198,6 +1207,34 @@ void Rewriter::_call(RewriterVar* result, bool has_side_effects, void* func_addr
         result->releaseIfNoUses();
 }
 
+std::vector<Location> Rewriter::getDecrefLocations() {
+    std::vector<Location> decref_infos;
+    for (RewriterVar& var : vars) {
+        if (var.locations.size() && var.needsDecref()) {
+            // TODO: add code to handle other location types and choose best location if there are several
+            Location l = *var.locations.begin();
+            if (l.type == Location::Scratch) {
+                // convert to stack based location because later on we may not know the offset of the scratch area from
+                // the SP.
+                assert(indirectFor(l).offset % 8 == 0);
+                decref_infos.emplace_back(Location::Stack, indirectFor(l).offset / 8);
+            } else if (l.type == Location::Register) {
+                // CSRs shouldn't be getting allocated, and we should only be calling this at a callsite:
+                RELEASE_ASSERT(0, "we shouldn't be trying to decref anything in a register");
+            } else
+                RELEASE_ASSERT(0, "not implemented");
+        }
+    }
+    return decref_infos;
+}
+
+void Rewriter::registerDecrefInfoHere() {
+    std::vector<Location> decref_locations = getDecrefLocations();
+    auto call_offset = assembler->bytesWritten();
+    uint64_t ip = (uint64_t)rewrite->getSlotStart() + call_offset;
+    decref_infos.emplace_back(std::make_pair(ip, std::move(decref_locations)));
+}
+
 void Rewriter::abort() {
     STAT_TIMER(t0, "us_timer_rewriter", 10);
 
@@ -1224,23 +1261,6 @@ RewriterVar* RewriterVar::setType(RefType type) {
     assert(this->reftype == RefType::UNKNOWN || this->reftype == type);
 
     if (this->reftype == RefType::UNKNOWN) {
-        rewriter->addAction(
-            [=]() {
-                int num_needed_refs = this->num_refs_consumed - (this->refHandedOff() ? 1 : 0);
-                assert(num_needed_refs >= 0);
-                if (num_needed_refs > 0) {
-                    if (rewriter->isDoneGuarding()) {
-                        this->rewriter->_incref(this, num_needed_refs);
-                    } else {
-                        rewriter->pending_increfs.push_back(std::make_pair(this, num_needed_refs));
-                    }
-                }
-
-                this->bumpUse();
-                // Register this as a NORMAL (ie non-MUTATION) action since we will be careful to not do any mutations
-                // if we are still in the guarding phase.
-            },
-            { this }, ActionType::NORMAL);
         this->reftype = type;
     }
 
@@ -1272,9 +1292,22 @@ void RewriterVar::_release() {
     this->locations.clear();
 }
 
-void RewriterVar::refConsumed() {
+void RewriterVar::refConsumed(RewriterAction* action) {
+    assert(reftype != RefType::UNKNOWN || (isConstant() && constant_value == 0));
     num_refs_consumed++;
     last_refconsumed_numuses = uses.size();
+    if (!action)
+        action = rewriter->getLastAction();
+    action->consumed_refs.emplace_back(this);
+}
+
+void RewriterVar::refUsed() {
+    // TODO: This is a pretty silly implementation that might prevent other optimizations?
+    rewriter->addAction([=]() { this->bumpUse(); }, { this }, ActionType::NORMAL);
+}
+
+bool RewriterVar::needsDecref() {
+    return reftype == RefType::OWNED && !this->refHandedOff();
 }
 
 void RewriterVar::bumpUse() {
@@ -1374,12 +1407,6 @@ void Rewriter::commit() {
             }
         }
 
-        for (auto&& p : pending_increfs) {
-            // TODO use a single add rather than N inc's
-            for (int i = 0; i < p.second; i++) {
-                _incref(p.first);
-            }
-        }
         assertConsistent();
     };
 
@@ -1395,6 +1422,21 @@ void Rewriter::commit() {
 
     // Now, start emitting assembly; check if we're dong guarding after each.
     for (int i = 0; i < actions.size(); i++) {
+        // add increfs if required
+        for (auto&& var : actions[i].consumed_refs) {
+            if (var->refHandedOff()) {
+                // if this action is the one which the variable gets handed off we don't need todo anything
+                assert(var->last_refconsumed_numuses > 0 && var->last_refconsumed_numuses <= var->uses.size());
+                int last_used_action_id = var->uses[var->last_refconsumed_numuses - 1];
+                if (last_used_action_id == i)
+                    continue;
+                assert(last_used_action_id >= i);
+            }
+
+            assert(isDoneGuarding());
+            _incref(var, 1);
+        }
+
         actions[i].action();
 
         if (failed) {
@@ -1561,7 +1603,7 @@ void Rewriter::commit() {
     }
 #endif
 
-    rewrite->commit(this, std::move(gc_references));
+    rewrite->commit(this, std::move(gc_references), std::move(decref_infos));
     assert(gc_references.empty());
 
     if (assembler->hasFailed()) {
@@ -1823,10 +1865,13 @@ void Rewriter::_checkAndThrowCAPIException(RewriterVar* r, int64_t exc_val) {
     else
         assembler->cmp(var_reg, assembler::Immediate(exc_val));
 
+    _setupCall(false, RewriterVar::SmallVector(), RewriterVar::SmallVector());
     {
         assembler::ForwardJump jnz(*assembler, assembler::COND_NOT_ZERO);
         assembler->mov(assembler::Immediate((void*)throwCAPIException), assembler::R11);
         assembler->callq(assembler::R11);
+
+        registerDecrefInfoHere();
     }
 
     r->bumpUse();

src/asm_writing/rewriter.h

@@ -27,6 +27,7 @@
 
 #include "asm_writing/assembler.h"
 #include "asm_writing/icinfo.h"
+#include "asm_writing/types.h"
 #include "core/threading.h"
 #include "core/types.h"
 
@@ -41,82 +42,6 @@ class ICInvalidator;
 
 class RewriterVar;
 
-struct Location {
-public:
-    enum LocationType : uint8_t {
-        Register,
-        XMMRegister,
-        Stack,
-        Scratch, // stack location, relative to the scratch start
-
-        // For representing constants that fit in 32-bits, that can be encoded as immediates
-        AnyReg,        // special type for use when specifying a location as a destination
-        None,          // special type that represents the lack of a location, ex where a "ret void" gets returned
-        Uninitialized, // special type for an uninitialized (and invalid) location
-    };
-
-public:
-    LocationType type;
-
-    union {
-        // only valid if type==Register; uses X86 numbering, not dwarf numbering.
-        // also valid if type==XMMRegister
-        int32_t regnum;
-        // only valid if type==Stack; this is the offset from bottom of the original frame.
-        // ie argument #6 will have a stack_offset of 0, #7 will have a stack offset of 8, etc
-        int32_t stack_offset;
-        // only valid if type == Scratch; offset from the beginning of the scratch area
-        int32_t scratch_offset;
-
-        int32_t _data;
-    };
-
-    constexpr Location() noexcept : type(Uninitialized), _data(-1) {}
-    constexpr Location(const Location& r) = default;
-    Location& operator=(const Location& r) = default;
-
-    constexpr Location(LocationType type, int32_t data) : type(type), _data(data) {}
-
-    constexpr Location(assembler::Register reg) : type(Register), regnum(reg.regnum) {}
-
-    constexpr Location(assembler::XMMRegister reg) : type(XMMRegister), regnum(reg.regnum) {}
-
-    constexpr Location(assembler::GenericRegister reg)
-        : type(reg.type == assembler::GenericRegister::GP ? Register : reg.type == assembler::GenericRegister::XMM
-                                                                           ? XMMRegister
-                                                                           : None),
-          regnum(reg.type == assembler::GenericRegister::GP ? reg.gp.regnum : reg.xmm.regnum) {}
-
-    assembler::Register asRegister() const;
-    assembler::XMMRegister asXMMRegister() const;
-    bool isClobberedByCall() const;
-
-    static constexpr Location any() { return Location(AnyReg, 0); }
-    static constexpr Location none() { return Location(None, 0); }
-    static Location forArg(int argnum);
-    static Location forXMMArg(int argnum);
-
-    bool operator==(const Location rhs) const { return this->asInt() == rhs.asInt(); }
-
-    bool operator!=(const Location rhs) const { return !(*this == rhs); }
-
-    bool operator<(const Location& rhs) const { return this->asInt() < rhs.asInt(); }
-
-    uint64_t asInt() const { return (int)type + ((uint64_t)_data << 4); }
-
-    void dump() const;
-};
-static_assert(sizeof(Location) <= 8, "");
-}
-
-namespace std {
-template <> struct hash<pyston::Location> {
-    size_t operator()(const pyston::Location p) const { return p.asInt(); }
-};
-}
-
-namespace pyston {
-
 // Replacement for unordered_map<Location, T>
 template <class T> class LocMap {
 private:
@@ -224,7 +149,12 @@ public:
     // getAttrFloat casts to double (maybe I should make that separate?)
     RewriterVar* getAttrFloat(int offset, Location loc = Location::any());
     RewriterVar* getAttrDouble(int offset, Location loc = Location::any());
-    void setAttr(int offset, RewriterVar* other);
+    enum class SetattrType {
+        UNKNOWN,
+        HANDED_OFF,
+        REFUSED,
+    };
+    void setAttr(int offset, RewriterVar* other, SetattrType type = SetattrType::UNKNOWN);
     RewriterVar* cmp(AST_TYPE::AST_TYPE cmp_type, RewriterVar* other, Location loc = Location::any());
     RewriterVar* toBool(Location loc = Location::any());
 
@@ -240,7 +170,10 @@ public:
     // This should get called *after* the ref got consumed, ie something like
     //   r_array->setAttr(0, r_val);
     //   r_val->refConsumed()
-    void refConsumed();
+    // if no action is specified it will assume the last action consumed the reference
+    void refConsumed(RewriterAction* action = NULL);
+
+    void refUsed();
 
 
     template <typename Src, typename Dst> inline RewriterVar* getAttrCast(int offset, Location loc = Location::any());
@@ -278,6 +211,7 @@ private:
     bool isDoneUsing() { return next_use == uses.size(); }
     bool hasScratchAllocation() const { return scratch_allocation.second > 0; }
     void resetHasScratchAllocation() { scratch_allocation = std::make_pair(0, 0); }
+    bool needsDecref();
 
     // Indicates if this variable is an arg, and if so, what location the arg is from.
     bool is_arg;
@@ -363,6 +297,7 @@ public:
 class RewriterAction {
 public:
     SmallFunction<56> action;
+    std::vector<RewriterVar*> consumed_refs;
 
     template <typename F> RewriterAction(F&& action) : action(std::forward<F>(action)) {}
 
@@ -382,7 +317,7 @@ class Rewriter : public ICSlotRewrite::CommitHook {
 private:
     class RegionAllocator {
     public:
-        static const int BLOCK_SIZE = 200; // reserve a bit of space for list/malloc overhead
+        static const int BLOCK_SIZE = 1024; // reserve a bit of space for list/malloc overhead
         std::list<char[BLOCK_SIZE]> blocks;
 
         int cur_offset = BLOCK_SIZE + 1;
@@ -473,7 +408,7 @@ protected:
              bool needs_invalidation_support = true);
 
     std::deque<RewriterAction, RegionAllocatorAdaptor<RewriterAction>> actions;
-    template <typename F> void addAction(F&& action, llvm::ArrayRef<RewriterVar*> vars, ActionType type) {
+    template <typename F> RewriterAction* addAction(F&& action, llvm::ArrayRef<RewriterVar*> vars, ActionType type) {
         assertPhaseCollecting();
         for (RewriterVar* var : vars) {
             assert(var != NULL);
@@ -484,7 +419,7 @@ protected:
         } else if (type == ActionType::GUARD) {
             if (added_changing_action) {
                 failed = true;
-                return;
+                return NULL;
             }
             for (RewriterVar* arg : args) {
                 arg->uses.push_back(actions.size());
@@ -493,10 +428,17 @@ protected:
             last_guard_action = (int)actions.size();
         }
         actions.emplace_back(std::forward<F>(action));
+        return &actions.back();
+    }
+    RewriterAction* getLastAction() {
+        assert(!actions.empty());
+        return &actions.back();
     }
+
     bool added_changing_action;
     bool marked_inside_ic;
     std::vector<void*> gc_references;
+    std::vector<std::pair<uint64_t, std::vector<Location>>> decref_infos;
 
     bool done_guarding;
     bool isDoneGuarding() {
@@ -626,6 +568,10 @@ public:
 #else
     void comment(const llvm::Twine& msg) {}
 #endif
+    // returns a vector of locations of variables which need to get decrefed if the last action throwes
+    std::vector<Location> getDecrefLocations();
+    // calls getDecrefLocations and registers the current assembler address with the retrieved decref info
+    void registerDecrefInfoHere();
 
     void trap();
     RewriterVar* loadConst(int64_t val, Location loc = Location::any());

src/asm_writing/types.h

@@ -176,6 +176,79 @@ struct JumpDestination {
     static JumpDestination fromStart(int offset) { return JumpDestination(FROM_START, offset); }
 };
 }
+
+struct Location {
+public:
+    enum LocationType : uint8_t {
+        Register,
+        XMMRegister,
+        Stack,
+        Scratch, // stack location, relative to the scratch start
+
+        // For representing constants that fit in 32-bits, that can be encoded as immediates
+        AnyReg,        // special type for use when specifying a location as a destination
+        None,          // special type that represents the lack of a location, ex where a "ret void" gets returned
+        Uninitialized, // special type for an uninitialized (and invalid) location
+    };
+
+public:
+    LocationType type;
+
+    union {
+        // only valid if type==Register; uses X86 numbering, not dwarf numbering.
+        // also valid if type==XMMRegister
+        int32_t regnum;
+        // only valid if type==Stack; this is the offset from bottom of the original frame.
+        // ie argument #6 will have a stack_offset of 0, #7 will have a stack offset of 8, etc
+        int32_t stack_offset;
+        // only valid if type == Scratch; offset from the beginning of the scratch area
+        int32_t scratch_offset;
+
+        int32_t _data;
+    };
+
+    constexpr Location() noexcept : type(Uninitialized), _data(-1) {}
+    constexpr Location(const Location& r) = default;
+    Location& operator=(const Location& r) = default;
+
+    constexpr Location(LocationType type, int32_t data) : type(type), _data(data) {}
+
+    constexpr Location(assembler::Register reg) : type(Register), regnum(reg.regnum) {}
+
+    constexpr Location(assembler::XMMRegister reg) : type(XMMRegister), regnum(reg.regnum) {}
+
+    constexpr Location(assembler::GenericRegister reg)
+        : type(reg.type == assembler::GenericRegister::GP ? Register : reg.type == assembler::GenericRegister::XMM
+                                                                           ? XMMRegister
+                                                                           : None),
+          regnum(reg.type == assembler::GenericRegister::GP ? reg.gp.regnum : reg.xmm.regnum) {}
+
+    assembler::Register asRegister() const;
+    assembler::XMMRegister asXMMRegister() const;
+    bool isClobberedByCall() const;
+
+    static constexpr Location any() { return Location(AnyReg, 0); }
+    static constexpr Location none() { return Location(None, 0); }
+    static Location forArg(int argnum);
+    static Location forXMMArg(int argnum);
+
+    bool operator==(const Location rhs) const { return this->asInt() == rhs.asInt(); }
+
+    bool operator!=(const Location rhs) const { return !(*this == rhs); }
+
+    bool operator<(const Location& rhs) const { return this->asInt() < rhs.asInt(); }
+
+    uint64_t asInt() const { return (int)type + ((uint64_t)_data << 4); }
+
+    void dump() const;
+};
+static_assert(sizeof(Location) <= 8, "");
+}
+
+namespace std {
+template <> struct hash<pyston::Location> {
+    size_t operator()(const pyston::Location p) const { return p.asInt(); }
+};
 }
 
 #endif

src/codegen/ast_interpreter.cpp

@@ -66,7 +66,7 @@ extern "C" Box* executeInnerAndSetupFrame(ASTInterpreter& interpreter, CFGBlock*
  */
 class ASTInterpreter {
 public:
-    ASTInterpreter(FunctionMetadata* md, Box** vregs, int num_vregs);
+    ASTInterpreter(FunctionMetadata* md, Box** vregs, int num_vregs, FrameInfo* deopt_frame_info = NULL);
 
     void initArguments(BoxedClosure* closure, BoxedGenerator* generator, Box* arg1, Box* arg2, Box* arg3, Box** args);
 
@@ -183,7 +183,6 @@ public:
     void setPassedClosure(Box* closure);
     void setCreatedClosure(Box* closure);
     void setBoxedLocals(STOLEN(Box*));
-    void setFrameInfo(const FrameInfo* frame_info);
     void setGlobals(Box* globals);
 
     friend struct pyston::ASTInterpreterJitInterface;
@@ -215,7 +214,8 @@ void ASTInterpreter::setPassedClosure(Box* closure) {
 void ASTInterpreter::setCreatedClosure(Box* closure) {
     assert(!this->created_closure); // This should only used for initialization
     assert(closure->cls == closure_cls);
-    this->created_closure = static_cast<BoxedClosure*>(closure);
+    // we have to incref the closure because the interpreter destructor will decref it
+    this->created_closure = static_cast<BoxedClosure*>(incref(closure));
 }
 
 void ASTInterpreter::setBoxedLocals(Box* boxedLocals) {
@@ -223,20 +223,12 @@ void ASTInterpreter::setBoxedLocals(Box* boxedLocals) {
     this->frame_info.boxedLocals = boxedLocals;
 }
 
-void ASTInterpreter::setFrameInfo(const FrameInfo* frame_info) {
-    Box** vregs = this->frame_info.vregs;
-    int num_vregs = this->frame_info.num_vregs;
-    this->frame_info = *frame_info;
-    this->frame_info.vregs = vregs;
-    this->frame_info.num_vregs = num_vregs;
-}
-
 void ASTInterpreter::setGlobals(Box* globals) {
     assert(!this->frame_info.globals);
     this->frame_info.globals = incref(globals);
 }
 
-ASTInterpreter::ASTInterpreter(FunctionMetadata* md, Box** vregs, int num_vregs)
+ASTInterpreter::ASTInterpreter(FunctionMetadata* md, Box** vregs, int num_vregs, FrameInfo* deopt_frame_info)
     : current_block(0),
       frame_info(ExcInfo(NULL, NULL, NULL)),
       edgecount(0),
@@ -251,6 +243,22 @@ ASTInterpreter::ASTInterpreter(FunctionMetadata* md, Box** vregs, int num_vregs)
       should_jit(false) {
 
     scope_info = source_info->getScopeInfo();
+
+    if (deopt_frame_info) {
+        // copy over all fields and clear the deopt frame info
+        frame_info = *deopt_frame_info;
+
+        // Well, don't actually copy over the vregs.  We'll deal with them separately
+        // (using the locals dict), so just clear out and decref the old ones:
+        frame_info.vregs = NULL;
+        frame_info.num_vregs = 0;
+        for (int i = 0; i < deopt_frame_info->num_vregs; ++i)
+            Py_XDECREF(deopt_frame_info->vregs[i]);
+
+        // We are taking responsibility for calling deinit:
+        deopt_frame_info->disableDeinit(&this->frame_info);
+    }
+
     frame_info.vregs = vregs;
     frame_info.md = md;
     frame_info.num_vregs = num_vregs;
@@ -468,14 +476,14 @@ void ASTInterpreter::doStore(AST_Name* node, STOLEN(Value) value) {
     } else {
         bool closure = vst == ScopeInfo::VarScopeType::CLOSURE;
         if (jit) {
-            if (!closure) {
-                bool is_live = source_info->getLiveness()->isLiveAtEnd(name, current_block);
-                if (is_live)
-                    jit->emitSetLocal(name, node->vreg, closure, value);
-                else
-                    jit->emitSetBlockLocal(name, value);
-            } else
+            bool is_live = true;
+            // TODO: turn this optimization back on.
+            // if (!closure)
+            // is_live = source_info->getLiveness()->isLiveAtEnd(name, current_block);
+            if (is_live)
                 jit->emitSetLocal(name, node->vreg, closure, value);
+            else
+                jit->emitSetBlockLocal(name, value);
         }
 
         if (closure) {
@@ -1080,6 +1088,7 @@ Value ASTInterpreter::createFunction(AST* node, AST_arguments* args, const std::
     FunctionMetadata* md = wrapFunction(node, args, body, source_info);
 
     std::vector<Box*> defaults;
+    llvm::SmallVector<RewriterVar*, 4> defaults_vars;
 
     RewriterVar* defaults_var = NULL;
     if (jit)
@@ -1088,8 +1097,10 @@ Value ASTInterpreter::createFunction(AST* node, AST_arguments* args, const std::
     for (AST_expr* d : args->defaults) {
         Value v = visit_expr(d);
         defaults.push_back(v.o);
-        if (jit)
-            defaults_var->setAttr(i++ * sizeof(void*), v);
+        if (jit) {
+            defaults_var->setAttr(i++ * sizeof(void*), v, RewriterVar::SetattrType::REFUSED);
+            defaults_vars.push_back(v.var);
+        }
     }
     defaults.push_back(0);
     AUTO_XDECREF_ARRAY(defaults.data(), defaults.size());
@@ -1153,6 +1164,10 @@ Value ASTInterpreter::createFunction(AST* node, AST_arguments* args, const std::
             passed_globals_var = jit->imm(0ul);
         rtn.var = jit->call(false, (void*)createFunctionFromMetadata, jit->imm(md), closure_var, passed_globals_var,
                             defaults_var, jit->imm(args->defaults.size()))->setType(RefType::OWNED);
+
+        for (auto d_var : defaults_vars) {
+            d_var->refUsed();
+        }
     }
 
     rtn.o = createFunctionFromMetadata(md, closure, passed_globals, u.il);
@@ -1641,7 +1656,7 @@ Value ASTInterpreter::visit_name(AST_Name* node) {
         case ScopeInfo::VarScopeType::CLOSURE: {
             Value v;
             if (jit) {
-                bool is_live = false;
+                bool is_live = true;
                 if (node->lookup_type == ScopeInfo::VarScopeType::FAST)
                     is_live = source_info->getLiveness()->isLiveAtEnd(node->id, current_block);
 
@@ -1843,7 +1858,7 @@ const void* interpreter_instr_addr = (void*)&executeInnerAndSetupFrame;
 extern "C" Box* executeInnerFromASM(ASTInterpreter& interpreter, CFGBlock* start_block, AST_stmt* start_at) {
     initFrame(interpreter.getFrameInfo());
     Box* rtn = ASTInterpreter::executeInner(interpreter, start_block, start_at);
-    deinitFrame(interpreter.getFrameInfo());
+    deinitFrameMaybe(interpreter.getFrameInfo());
     return rtn;
 }
 
@@ -2006,9 +2021,7 @@ extern "C" Box* astInterpretDeoptFromASM(FunctionMetadata* md, AST_expr* after_e
         memset(vregs, 0, sizeof(Box*) * num_vregs);
     }
 
-    ASTInterpreter interpreter(md, vregs, num_vregs);
-    if (source_info->scoping->areGlobalsFromModule())
-        interpreter.setGlobals(source_info->parent_module);
+    ASTInterpreter interpreter(md, vregs, num_vregs, frame_state.frame_info);
 
     for (const auto& p : *frame_state.locals) {
         assert(p.first->cls == str_cls);
@@ -2016,7 +2029,8 @@ extern "C" Box* astInterpretDeoptFromASM(FunctionMetadata* md, AST_expr* after_e
         if (name == PASSED_GENERATOR_NAME) {
             interpreter.setGenerator(p.second);
         } else if (name == PASSED_CLOSURE_NAME) {
-            interpreter.setPassedClosure(p.second);
+            // this should have already got set because its stored in the frame info
+            assert(p.second == interpreter.getFrameInfo()->passed_closure);
         } else if (name == CREATED_CLOSURE_NAME) {
             interpreter.setCreatedClosure(p.second);
         } else {
@@ -2025,8 +2039,6 @@ extern "C" Box* astInterpretDeoptFromASM(FunctionMetadata* md, AST_expr* after_e
         }
     }
 
-    interpreter.setFrameInfo(frame_state.frame_info);
-
     CFGBlock* start_block = NULL;
     AST_stmt* starting_statement = NULL;
     while (true) {
@@ -2043,7 +2055,6 @@ extern "C" Box* astInterpretDeoptFromASM(FunctionMetadata* md, AST_expr* after_e
             auto expr = ast_cast<AST_Expr>(enclosing_stmt);
             RELEASE_ASSERT(expr->value == after_expr, "%p %p", expr->value, after_expr);
             assert(expr->value == after_expr);
-            assert(0 && "check refcounting");
             break;
         } else if (enclosing_stmt->type == AST_TYPE::Invoke) {
             auto invoke = ast_cast<AST_Invoke>(enclosing_stmt);
@@ -2080,7 +2091,7 @@ extern "C" Box* astInterpretDeoptFromASM(FunctionMetadata* md, AST_expr* after_e
     // We need to remove the old python frame created in the LLVM tier otherwise we would have a duplicate frame because
     // the interpreter will set the new state before executing the first statement.
     RELEASE_ASSERT(cur_thread_state.frame_info == frame_state.frame_info, "");
-    cur_thread_state.frame_info = frame_state.frame_info->back;
+    cur_thread_state.frame_info = interpreter.getFrameInfo()->back;
 
     Box* v = ASTInterpreter::execute(interpreter, start_block, starting_statement);
     return v ? v : incref(None);

src/codegen/baseline_jit.h

@@ -146,6 +146,9 @@ private:
     assembler::Assembler a;
     bool is_currently_writing;
     bool asm_failed;
+    // this contains all the decref infos the bjit generated inside the memory block,
+    // this allows us to deregister them when we release the code
+    std::vector<DecrefInfo> decref_infos;
 
 public:
     JitCodeBlock(llvm::StringRef name);
@@ -153,7 +156,7 @@ public:
     std::unique_ptr<JitFragmentWriter> newFragment(CFGBlock* block, int patch_jump_offset = 0);
     bool shouldCreateNewBlock() const { return asm_failed || a.bytesLeft() < 128; }
     void fragmentAbort(bool not_enough_space);
-    void fragmentFinished(int bytes_witten, int num_bytes_overlapping, void* next_fragment_start);
+    void fragmentFinished(int bytes_witten, int num_bytes_overlapping, void* next_fragment_start, ICInfo& ic_info);
 };
 
 class JitFragmentWriter : public Rewriter {
@@ -199,6 +202,7 @@ private:
         std::unique_ptr<ICSetupInfo> ic;
         StackInfo stack_info;
         AST* node;
+        std::vector<Location> decref_infos;
     };
 
     llvm::SmallVector<PPInfo, 8> pp_infos;
@@ -277,15 +281,22 @@ public:
     bool finishAssembly(int continue_offset) override;
 
 private:
-    RewriterVar* allocArgs(const llvm::ArrayRef<RewriterVar*> args);
+    RewriterVar* allocArgs(const llvm::ArrayRef<RewriterVar*> args, RewriterVar::SetattrType);
 #ifndef NDEBUG
     std::pair<uint64_t, uint64_t> asUInt(InternedString s);
 #else
     uint64_t asUInt(InternedString s);
 #endif
 
-    RewriterVar* emitPPCall(void* func_addr, llvm::ArrayRef<RewriterVar*> args, int num_slots, int slot_size,
-                            AST* ast_node = NULL, TypeRecorder* type_recorder = NULL);
+
+    // use this function when one emits a call where one argument is variable created with allocArgs(vars).
+    // it let's one specify the additional uses the call has which are unknown to the rewriter because it is hidden in
+    // the allocArgs call.
+    RewriterVar* emitCallWithAllocatedArgs(void* func_addr, const llvm::ArrayRef<RewriterVar*> args,
+                                           const llvm::ArrayRef<RewriterVar*> additional_uses);
+    std::pair<RewriterVar*, RewriterAction*> emitPPCall(void* func_addr, llvm::ArrayRef<RewriterVar*> args,
+                                                        int num_slots, int slot_size, AST* ast_node = NULL,
+                                                        TypeRecorder* type_recorder = NULL);
 
     static void assertNameDefinedHelper(const char* id);
     static Box* callattrHelper(Box* obj, BoxedString* attr, CallattrFlags flags, TypeRecorder* type_recorder,

src/codegen/compvars.cpp

@@ -498,7 +498,7 @@ public:
 
     Box* deserializeFromFrame(const FrameVals& vals) override {
         assert(vals.size() == 1);
-        return reinterpret_cast<Box*>(vals[0]);
+        return incref(reinterpret_cast<Box*>(vals[0]));
     }
 
     std::vector<CompilerVariable*> unpack(IREmitter& emitter, const OpInfo& info, VAR* var, int num_into) override {
@@ -1615,6 +1615,7 @@ public:
     std::string debugName() override { return "phony(" + ConcreteCompilerType::debugName() + ")"; }
 
     CompilerType* getUsableType() override { return usable_type; }
+    ConcreteCompilerType* getBoxType() override { return getUsableType()->getBoxType(); }
 
     llvm::Type* llvmType() override { return t; }
 
@@ -2190,7 +2191,7 @@ public:
 
     Box* deserializeFromFrame(const FrameVals& vals) override {
         assert(vals.size() == 1);
-        return reinterpret_cast<Box*>(vals[0]);
+        return incref(reinterpret_cast<Box*>(vals[0]));
     }
 };
 std::unordered_map<BoxedClass*, NormalObjectType*> NormalObjectType::made;
@@ -2221,7 +2222,7 @@ public:
 
     Box* deserializeFromFrame(const FrameVals& vals) override {
         assert(vals.size() == 1);
-        return reinterpret_cast<Box*>(vals[0]);
+        return incref(reinterpret_cast<Box*>(vals[0]));
     }
 } _CLOSURE;
 ConcreteCompilerType* CLOSURE = &_CLOSURE;
@@ -2236,7 +2237,7 @@ public:
 
     Box* deserializeFromFrame(const FrameVals& vals) override {
         assert(vals.size() == numFrameArgs());
-        return reinterpret_cast<Box*>(vals[0]);
+        return incref(reinterpret_cast<Box*>(vals[0]));
     }
 } _GENERATOR;
 ConcreteCompilerType* GENERATOR = &_GENERATOR;

src/codegen/irgen/irgenerator.cpp

@@ -841,6 +841,9 @@ private:
         curblock = deopt_bb;
         emitter.getBuilder()->SetInsertPoint(curblock);
         llvm::Value* v = emitter.createDeopt(current_statement, (AST_expr*)node, node_value);
+        // TODO: this might not be necessary since it should never fire?
+        if (!irstate->getCurFunction()->entry_descriptor)
+            emitter.getBuilder()->CreateCall(g.funcs.deinitFrameMaybe, irstate->getFrameInfoVar());
         llvm::Instruction* ret_inst = emitter.getBuilder()->CreateRet(v);
         irstate->getRefcounts()->refConsumed(v, ret_inst);
 
@@ -2428,6 +2431,8 @@ private:
             emitter.getBuilder()->CreateCall(l_free, malloc_save);
         }
 
+        if (!irstate->getCurFunction()->entry_descriptor)
+            emitter.getBuilder()->CreateCall(g.funcs.deinitFrame, irstate->getFrameInfoVar());
         emitter.getBuilder()->CreateRet(rtn);
 
         emitter.getBuilder()->SetInsertPoint(starting_block);
@@ -3008,7 +3013,8 @@ public:
                     emitter.getBuilder()->CreateCall(g.funcs.reraiseCapiExcAsCxx);
                     emitter.getBuilder()->CreateUnreachable();
                 } else {
-                    emitter.getBuilder()->CreateCall(g.funcs.deinitFrame, irstate->getFrameInfoVar());
+                    if (!irstate->getCurFunction()->entry_descriptor)
+                        emitter.getBuilder()->CreateCall(g.funcs.deinitFrame, irstate->getFrameInfoVar());
                     emitter.getBuilder()->CreateRet(getNullPtr(g.llvm_value_type_ptr));
                 }
             } else {
@@ -3097,7 +3103,8 @@ public:
             irstate->getRefcounts()->refConsumed(exc_type, call_inst);
             irstate->getRefcounts()->refConsumed(exc_value, call_inst);
             irstate->getRefcounts()->refConsumed(exc_traceback, call_inst);
-            emitter.getBuilder()->CreateCall(g.funcs.deinitFrame, irstate->getFrameInfoVar());
+            if (!irstate->getCurFunction()->entry_descriptor)
+                emitter.getBuilder()->CreateCall(g.funcs.deinitFrame, irstate->getFrameInfoVar());
             emitter.getBuilder()->CreateRet(getNullPtr(g.llvm_value_type_ptr));
         } else {
             // auto call_inst = emitter.createCall3(UnwindInfo(unw_info.current_stmt, NO_CXX_INTERCEPTION),

src/codegen/irgen/refcounts.cpp

@@ -651,8 +651,12 @@ void RefcountTracker::addRefcounts(IRGenState* irstate) {
         while (s->elements().size() > 0 && llvm::isa<llvm::StructType>(s->elements()[0]))
             s = llvm::cast<llvm::StructType>(s->elements()[0]);
 
+        if (isa<ConstantPointerNull>(v))
+            return;
+
         bool ok_type = false;
-        if (s->elements().size() >= 2 && s->elements()[0] == g.i64 && s->elements()[1] == g.llvm_class_type_ptr) {
+        if (s->elements().size() >= 2 + REFCOUNT_IDX && s->elements()[REFCOUNT_IDX] == g.i64
+            && s->elements()[REFCOUNT_IDX + 1] == g.llvm_class_type_ptr) {
             // printf("This looks likes a class\n");
             ok_type = true;
         }

src/codegen/runtime_hooks.cpp

@@ -198,6 +198,7 @@ void initGlobalFuncs(GlobalState& g) {
     GET(createSet);
     GET(initFrame);
     GET(deinitFrame);
+    GET(deinitFrameMaybe);
     GET(makePendingCalls);
     GET(setFrameExcInfo);
 

src/codegen/runtime_hooks.h

@@ -34,8 +34,8 @@ struct GlobalFuncs {
 
     llvm::Value* boxInt, *unboxInt, *boxFloat, *unboxFloat, *createFunctionFromMetadata, *getFunctionMetadata,
         *boxInstanceMethod, *boxBool, *unboxBool, *createTuple, *createDict, *createList, *createSlice,
-        *createUserClass, *createClosure, *createGenerator, *createSet, *initFrame, *deinitFrame, *makePendingCalls,
-        *setFrameExcInfo;
+        *createUserClass, *createClosure, *createGenerator, *createSet, *initFrame, *deinitFrame, *deinitFrameMaybe,
+        *makePendingCalls, *setFrameExcInfo;
     llvm::Value* getattr, *getattr_capi, *setattr, *delattr, *delitem, *delGlobal, *nonzero, *binop, *compare,
         *augbinop, *unboxedLen, *getitem, *getitem_capi, *getclsattr, *getGlobal, *setitem, *unaryop, *import,
         *importFrom, *importStar, *repr, *exceptionMatches, *yield, *getiterHelper, *hasnext, *setGlobal, *apply_slice;

src/codegen/unwinding.cpp

@@ -429,6 +429,9 @@ static inline unw_word_t get_cursor_ip(unw_cursor_t* cursor) {
 static inline unw_word_t get_cursor_bp(unw_cursor_t* cursor) {
     return get_cursor_reg(cursor, UNW_TDEP_BP);
 }
+static inline unw_word_t get_cursor_sp(unw_cursor_t* cursor) {
+    return get_cursor_reg(cursor, UNW_REG_SP);
+}
 
 // if the given ip/bp correspond to a jitted frame or
 // ASTInterpreter::execute_inner frame, return true and return the
@@ -507,6 +510,40 @@ static FrameInfo* getTopFrameInfo() {
     return (FrameInfo*)cur_thread_state.frame_info;
 }
 
+llvm::DenseMap<uint64_t /*ip*/, std::vector<Location>> decref_infos;
+void executeDecrefs(unw_cursor_t* cursor) {
+    unw_word_t ip = get_cursor_ip(cursor);
+    auto it = decref_infos.find(ip);
+    if (it == decref_infos.end())
+        return;
+
+    for (Location& l : it->second) {
+        Box* b = NULL;
+        if (l.type == Location::Stack) {
+            unw_word_t sp = get_cursor_sp(cursor);
+            b = ((Box**)sp)[l.stack_offset];
+        } else if (l.type == Location::Register) {
+            RELEASE_ASSERT(0, "untested");
+            // This branch should never get hit since we shouldn't generate Register locations,
+            // since we don't allow allocating callee-save registers.
+            // If we did, this code might be right:
+            // b = (Box*)get_cursor_reg(cursor, l.regnum);
+        } else {
+            RELEASE_ASSERT(0, "not implemented");
+        }
+
+        Py_XDECREF(b);
+    }
+}
+void addDecrefInfoEntry(uint64_t ip, std::vector<Location> location) {
+    assert(!decref_infos.count(ip) && "why is there already an entry??");
+    decref_infos[ip] = std::move(location);
+}
+void removeDecrefInfoEntry(uint64_t ip) {
+    decref_infos.erase(ip);
+}
+
+
 class PythonUnwindSession {
     ExcInfo exc_info;
     PythonStackExtractor pystack_extractor;
@@ -539,6 +576,8 @@ public:
             prev_frame_info = NULL;
         }
 
+        executeDecrefs(cursor);
+
         PythonFrameIteratorImpl frame_iter;
         bool found_frame = pystack_extractor.handleCFrame(cursor, &frame_iter);
         if (found_frame) {
@@ -830,7 +869,8 @@ DeoptState getDeoptState() {
                 if (!v)
                     continue;
 
-                d->d[incref(p.first.getBox())] = v;
+                assert(!d->d.count(p.first.getBox()));
+                d->d[incref(p.first.getBox())] = incref(v);
             }
 
             for (const auto& p : cf->location_map->names) {
@@ -851,6 +891,7 @@ DeoptState getDeoptState() {
                         vals.push_back(frame_iter->readLocation(loc));
                     }
 
+                    // this returns an owned reference so we don't incref it
                     Box* v = e->type->deserializeFromFrame(vals);
                     // printf("%s: (pp id %ld) %p\n", p.first().c_str(), e._debug_pp_id, v);
                     d->d[boxString(p.first())] = v;

src/codegen/unwinding.h

@@ -97,6 +97,9 @@ ExcInfo* getFrameExcInfo();
 // but just as slow if it's not.
 void updateFrameExcInfoIfNeeded(ExcInfo* latest);
 
+void addDecrefInfoEntry(uint64_t ip, std::vector<class Location> location);
+void removeDecrefInfoEntry(uint64_t ip);
+
 struct FrameStackState {
     // This includes all # variables (but not the ! ones).
     // Therefore, it's not the same as the BoxedLocals.

src/core/options.cpp

@@ -30,7 +30,7 @@ bool LOG_BJIT_ASSEMBLY = 0;
 bool FORCE_INTERPRETER = false;
 bool FORCE_OPTIMIZE = false;
 bool ENABLE_INTERPRETER = true;
-bool ENABLE_BASELINEJIT = false; // XXX
+bool ENABLE_BASELINEJIT = true;
 
 bool CONTINUE_AFTER_FATAL = false;
 bool SHOW_DISASM = false;
@@ -50,8 +50,8 @@ bool ENABLE_TRACEBACKS = true;
 bool FORCE_LLVM_CAPI_CALLS = false;
 bool FORCE_LLVM_CAPI_THROWS = false;
 
-int OSR_THRESHOLD_INTERPRETER = 5;   // XXX
-int REOPT_THRESHOLD_INTERPRETER = 1; // XXX
+int OSR_THRESHOLD_INTERPRETER = 25;
+int REOPT_THRESHOLD_INTERPRETER = 25;
 int OSR_THRESHOLD_BASELINE = 2500;
 int REOPT_THRESHOLD_BASELINE = 1500;
 int OSR_THRESHOLD_T2 = 10000;

src/core/types.h

@@ -1001,6 +1001,12 @@ struct FrameInfo {
 
     BORROWED(Box*) updateBoxedLocals();
 
+    static FrameInfo* const NO_DEINIT;
+
+    // Calling disableDeinit makes future deinitFrameMaybe() frames not call deinitFrame().
+    // For use by deopt(), which takes over deinit responsibility for its caller.
+    void disableDeinit(FrameInfo* replacement_frame);
+
     FrameInfo(ExcInfo exc)
         : exc(exc),
           boxedLocals(NULL),

src/runtime/frame.cpp

@@ -229,7 +229,37 @@ extern "C" void initFrame(FrameInfo* frame_info) {
     cur_thread_state.frame_info = frame_info;
 }
 
+FrameInfo* const FrameInfo::NO_DEINIT = (FrameInfo*)-2; // not -1 to not match memset(-1)
+
+void FrameInfo::disableDeinit(FrameInfo* replacement_frame) {
+    assert(replacement_frame->back == this->back);
+    assert(replacement_frame->frame_obj == this->frame_obj);
+
+    if (this->frame_obj) {
+        assert(this->frame_obj->frame_info == this);
+        this->frame_obj->frame_info = replacement_frame;
+    }
+
+#ifndef NDEBUG
+    // First, make sure this doesn't get used for anything else:
+    memset(this, -1, sizeof(*this));
+#endif
+
+    // Kinda hacky but maybe worth it to not store any extra bits:
+    back = NO_DEINIT;
+}
+
+extern "C" void deinitFrameMaybe(FrameInfo* frame_info) {
+    // Note: this has to match FrameInfo::disableDeinit
+    if (frame_info->back != FrameInfo::NO_DEINIT)
+        deinitFrame(frame_info);
+}
+
 extern "C" void deinitFrame(FrameInfo* frame_info) {
+    // This can fire if we have a call to deinitFrame() that should be to deinitFrameMaybe() instead
+    assert(frame_info->back != FrameInfo::NO_DEINIT);
+
+    assert(cur_thread_state.frame_info == frame_info);
     cur_thread_state.frame_info = frame_info->back;
     BoxedFrame* frame = frame_info->frame_obj;
     if (frame) {
@@ -239,6 +269,7 @@ extern "C" void deinitFrame(FrameInfo* frame_info) {
 
     assert(frame_info->vregs || frame_info->num_vregs == 0);
     int num_vregs = frame_info->num_vregs;
+    assert(num_vregs >= 0);
 
     decrefArray<true>(frame_info->vregs, num_vregs);
 

src/runtime/inline/link_forcer.cpp

@@ -73,6 +73,7 @@ void force() {
     FORCE(decodeUTF8StringPtr);
     FORCE(initFrame);
     FORCE(deinitFrame);
+    FORCE(deinitFrameMaybe);
     FORCE(makePendingCalls);
     FORCE(setFrameExcInfo);
 

src/runtime/objmodel.cpp

@@ -1422,7 +1422,8 @@ void Box::appendNewHCAttr(BORROWED(Box*) new_attr, SetattrRewriteArgs* rewrite_a
         if (!new_array)
             r_array = rewrite_args->obj->getAttr(cls->attrs_offset + offsetof(HCAttrs, attr_list));
 
-        r_array->setAttr(numattrs * sizeof(Box*) + offsetof(HCAttrs::AttrList, attrs), rewrite_args->attrval);
+        r_array->setAttr(numattrs * sizeof(Box*) + offsetof(HCAttrs::AttrList, attrs), rewrite_args->attrval,
+                         RewriterVar::SetattrType::HANDED_OFF);
         rewrite_args->attrval->refConsumed();
 
         if (new_array)
@@ -1515,8 +1516,8 @@ void Box::setattr(BoxedString* attr, BORROWED(Box*) val, SetattrRewriteArgs* rew
                     // will tell the auto-refcount system to decref it.
                     r_hattrs->getAttr(offset * sizeof(Box*) + offsetof(HCAttrs::AttrList, attrs))
                         ->setType(RefType::OWNED);
-                    r_hattrs->setAttr(offset * sizeof(Box*) + offsetof(HCAttrs::AttrList, attrs),
-                                      rewrite_args->attrval);
+                    r_hattrs->setAttr(offset * sizeof(Box*) + offsetof(HCAttrs::AttrList, attrs), rewrite_args->attrval,
+                                      RewriterVar::SetattrType::HANDED_OFF);
                     rewrite_args->attrval->refConsumed();
 
                     rewrite_args->out_success = true;
@@ -4357,7 +4358,9 @@ void rearrangeArgumentsInternal(ParamReceiveSpec paramspec, const ParamNames* pa
                 if (varargs_idx == 2)
                     rewrite_args->arg3 = varargs_val;
                 if (varargs_idx >= 3) {
-                    rewrite_args->args->setAttr((varargs_idx - 3) * sizeof(Box*), varargs_val);
+                    rewrite_args->args->setAttr(
+                        (varargs_idx - 3) * sizeof(Box*), varargs_val,
+                        (is_owned ? RewriterVar::SetattrType::HANDED_OFF : RewriterVar::SetattrType::UNKNOWN));
                     if (is_owned) {
                         oargs_owned[varargs_idx - 3] = true;
                         varargs_val->refConsumed();
@@ -4886,13 +4889,13 @@ Box* callCLFunc(FunctionMetadata* md, CallRewriteArgs* rewrite_args, int num_out
                 RewriterVar* arg_array = rewrite_args->rewriter->allocate(4);
                 arg_vec.push_back(arg_array);
                 if (num_output_args >= 1)
-                    arg_array->setAttr(0, rewrite_args->arg1);
+                    arg_array->setAttr(0, rewrite_args->arg1, RewriterVar::SetattrType::REFUSED);
                 if (num_output_args >= 2)
-                    arg_array->setAttr(8, rewrite_args->arg2);
+                    arg_array->setAttr(8, rewrite_args->arg2, RewriterVar::SetattrType::REFUSED);
                 if (num_output_args >= 3)
-                    arg_array->setAttr(16, rewrite_args->arg3);
+                    arg_array->setAttr(16, rewrite_args->arg3, RewriterVar::SetattrType::REFUSED);
                 if (num_output_args >= 4)
-                    arg_array->setAttr(24, rewrite_args->args);
+                    arg_array->setAttr(24, rewrite_args->args, RewriterVar::SetattrType::REFUSED);
 
                 if (S == CXX)
                     rewrite_args->out_rtn = rewrite_args->rewriter->call(true, (void*)astInterpretHelper, arg_vec)
@@ -4900,6 +4903,15 @@ Box* callCLFunc(FunctionMetadata* md, CallRewriteArgs* rewrite_args, int num_out
                 else
                     rewrite_args->out_rtn = rewrite_args->rewriter->call(true, (void*)astInterpretHelperCapi, arg_vec)
                                                 ->setType(RefType::OWNED);
+
+                if (num_output_args >= 1)
+                    rewrite_args->arg1->refUsed();
+                if (num_output_args >= 2)
+                    rewrite_args->arg2->refUsed();
+                if (num_output_args >= 3)
+                    rewrite_args->arg3->refUsed();
+                if (num_output_args >= 4)
+                    rewrite_args->args->refUsed();
             }
 
             rewrite_args->out_success = true;

src/runtime/types.h

@@ -1370,6 +1370,7 @@ BORROWED(Box*) getFrame(FrameInfo* frame_info);
 BORROWED(Box*) getFrame(int depth);
 void frameInvalidateBack(BoxedFrame* frame);
 extern "C" void deinitFrame(FrameInfo* frame_info);
+extern "C" void deinitFrameMaybe(FrameInfo* frame_info);
 int frameinfo_traverse(FrameInfo* frame_info, visitproc visit, void* arg) noexcept;
 extern "C" void initFrame(FrameInfo* frame_info);
 extern "C" void setFrameExcInfo(FrameInfo* frame_info, STOLEN(Box*) type, STOLEN(Box*) value, STOLEN(Box*) tb);

test/tests/deopt_tests2.py

-# expected: statfail
 # skip-if: '-O' in EXTRA_JIT_ARGS or '-n' in EXTRA_JIT_ARGS
 # statcheck: 4 == noninit_count('num_deopt')
 # this used to hit an abort in our LLVM tier codegen

test/tests/exception_inside_ic.py

+def f(x):
+    float(x)
+
+try:
+    f(1)
+    f(1)
+    f(1)
+    f("str")
+except Exception as e:
+    print e