Change how the llvm jit passes exceptions between blocks

Our IR doesn't explicitly represent the data transfer between an Invoke
statement and its corresponding LandingPad.  We use a couple different
techniques to pass it through: in the ast interpreter/bjit, we stash it
into an interpreter-local variable and then pull it back out.  Previous
to this change, in the LLVM tier we would pass it directly through
an exception, using either the C++ or CAPI exception-passing mechanism.

This works but is a pain, since it requires coordination between the invoke
and the landingpad.  These live in different basic blocks, so we ended up
having this other code that lives separate from the normal irgen that has
to decide which exception style to use, and it has to respect certain
restrictions within irgen (ie it has to be careful to not request CAPI
exceptions for cases that we haven't added support for yet).

This commit changes the approach so that the exception data is passed
directly as LLVM Values, and its up to the Invoke to figure out how
to get it into that form.  This adds a bit more complexity to the invoke,
but it should make the interface easier to extend (such as in the next commit).

src/codegen/compvars.cpp

@@ -337,9 +337,7 @@ public:
                               CompilerVariable* slice) override {
         ConcreteCompilerVariable* converted_slice = slice->makeConverted(emitter, slice->getBoxType());
 
-        ExceptionStyle target_exception_style = CXX;
-        if (FORCE_LLVM_CAPI || info.unw_info.capi_exc_dest)
-            target_exception_style = CAPI;
+        ExceptionStyle target_exception_style = info.preferredExceptionStyle();
 
         bool do_patchpoint = ENABLE_ICGETITEMS;
         llvm::Value* rtn;
@@ -500,9 +498,7 @@ CompilerVariable* UnknownType::getattr(IREmitter& emitter, const OpInfo& info, C
 
     llvm::Value* rtn_val = NULL;
 
-    ExceptionStyle target_exception_style = CXX;
-    if (info.unw_info.capi_exc_dest || (!cls_only && FORCE_LLVM_CAPI))
-        target_exception_style = CAPI;
+    ExceptionStyle target_exception_style = cls_only ? CXX : info.preferredExceptionStyle();
 
     llvm::Value* llvm_func;
     void* raw_func;
@@ -656,9 +652,7 @@ CompilerVariable* UnknownType::call(IREmitter& emitter, const OpInfo& info, Conc
     bool pass_keywords = (argspec.num_keywords != 0);
     int npassed_args = argspec.totalPassed();
 
-    ExceptionStyle exception_style = ((FORCE_LLVM_CAPI && !info.unw_info.cxx_exc_dest) || info.unw_info.capi_exc_dest)
-                                         ? ExceptionStyle::CAPI
-                                         : ExceptionStyle::CXX;
+    ExceptionStyle exception_style = info.preferredExceptionStyle();
 
     llvm::Value* func;
     if (pass_keywords)
@@ -691,10 +685,7 @@ CompilerVariable* UnknownType::callattr(IREmitter& emitter, const OpInfo& info,
     bool pass_keywords = (flags.argspec.num_keywords != 0);
     int npassed_args = flags.argspec.totalPassed();
 
-    ExceptionStyle exception_style = ((FORCE_LLVM_CAPI && !info.unw_info.cxx_exc_dest && !flags.null_on_nonexistent)
-                                      || info.unw_info.capi_exc_dest)
-                                         ? ExceptionStyle::CAPI
-                                         : ExceptionStyle::CXX;
+    ExceptionStyle exception_style = flags.null_on_nonexistent ? CXX : info.preferredExceptionStyle();
 
     if (exception_style == CAPI)
         assert(!flags.null_on_nonexistent); // Will conflict with CAPI's null-on-exception
@@ -1522,7 +1513,7 @@ public:
         if (canStaticallyResolveGetattrs()) {
             Box* rtattr = typeLookup(cls, attr, nullptr);
             if (rtattr == NULL) {
-                ExceptionStyle exception_style = (FORCE_LLVM_CAPI || info.unw_info.capi_exc_dest) ? CAPI : CXX;
+                ExceptionStyle exception_style = info.preferredExceptionStyle();
                 llvm::Value* raise_func = exception_style == CXX ? g.funcs.raiseAttributeErrorStr
                                                                  : g.funcs.raiseAttributeErrorStrCapi;
                 llvm::CallSite call = emitter.createCall3(
@@ -1717,10 +1708,8 @@ public:
     CompilerVariable* callattr(IREmitter& emitter, const OpInfo& info, ConcreteCompilerVariable* var, BoxedString* attr,
                                CallattrFlags flags, const std::vector<CompilerVariable*>& args,
                                const std::vector<BoxedString*>* keyword_names) override {
-        ExceptionStyle exception_style = CXX;
-        // Not safe to force-capi here since most of the functions won't have capi variants:
-        if (/*FORCE_LLVM_CAPI ||*/ info.unw_info.capi_exc_dest)
-            exception_style = CAPI;
+        // XXX investigate
+        ExceptionStyle exception_style = info.preferredExceptionStyle();
 
         ConcreteCompilerVariable* called_constant = tryCallattrConstant(
             emitter, info, var, attr, flags.cls_only, flags.argspec, args, keyword_names, NULL, exception_style);
@@ -1783,9 +1772,7 @@ public:
         static BoxedString* attr = internStringImmortal("__getitem__");
         bool no_attribute = false;
 
-        ExceptionStyle exception_style = CXX;
-        if (FORCE_LLVM_CAPI || info.unw_info.capi_exc_dest)
-            exception_style = CAPI;
+        ExceptionStyle exception_style = info.preferredExceptionStyle();
 
         ConcreteCompilerVariable* called_constant = tryCallattrConstant(
             emitter, info, var, attr, true, ArgPassSpec(1, 0, 0, 0), { slice }, NULL, &no_attribute, exception_style);
@@ -2293,9 +2280,7 @@ public:
                     rtn->incvref();
                     return rtn;
                 } else {
-                    ExceptionStyle target_exception_style = CXX;
-                    if (FORCE_LLVM_CAPI || info.unw_info.capi_exc_dest)
-                        target_exception_style = CAPI;
+                    ExceptionStyle target_exception_style = info.preferredExceptionStyle();
 
                     if (target_exception_style == CAPI) {
                         llvm::CallSite call = emitter.createCall(info.unw_info, g.funcs.raiseIndexErrorStrCapi,

src/codegen/irgen.cpp

@@ -495,6 +495,7 @@ static void emitBBs(IRGenState* irstate, TypeAnalysis* types, const OSREntryDesc
     std::unordered_map<CFGBlock*, ConcreteSymbolTable*> phi_ending_symbol_tables;
     typedef std::unordered_map<InternedString, std::pair<ConcreteCompilerType*, llvm::PHINode*>> PHITable;
     std::unordered_map<CFGBlock*, PHITable*> created_phis;
+    std::unordered_map<CFGBlock*, llvm::SmallVector<IRGenerator::ExceptionState, 2>> incoming_exception_state;
 
     CFGBlock* initial_block = NULL;
     if (entry_descriptor) {
@@ -759,6 +760,11 @@ static void emitBBs(IRGenState* irstate, TypeAnalysis* types, const OSREntryDesc
             }
         }
 
+        auto exc_it = incoming_exception_state.find(block);
+        if (exc_it != incoming_exception_state.end()) {
+            generator->setIncomingExceptionState(exc_it->second);
+        }
+
         // Generate loop safepoints on backedges.
         for (CFGBlock* predecessor : block->predecessors) {
             if (predecessor->idx > block->idx) {
@@ -777,6 +783,15 @@ static void emitBBs(IRGenState* irstate, TypeAnalysis* types, const OSREntryDesc
         phi_ending_symbol_tables[block] = ending_st.phi_symbol_table;
         llvm_exit_blocks[block] = ending_st.ending_block;
 
+        if (ending_st.exception_state.size()) {
+            AST_stmt* last_stmt = block->body.back();
+            assert(last_stmt->type == AST_TYPE::Invoke);
+            CFGBlock* exc_block = ast_cast<AST_Invoke>(last_stmt)->exc_dest;
+            assert(!incoming_exception_state.count(exc_block));
+
+            incoming_exception_state.insert(std::make_pair(exc_block, ending_st.exception_state));
+        }
+
         if (into_hax.count(block))
             ASSERT(ending_st.symbol_table->size() == 0, "%d", block->idx);
     }

src/codegen/irgen.h

@@ -20,6 +20,7 @@
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/IRBuilder.h"
 
+#include "core/options.h"
 #include "core/types.h"
 
 namespace pyston {
@@ -33,18 +34,18 @@ struct UnwindInfo {
 public:
     AST_stmt* current_stmt;
 
-    llvm::BasicBlock* capi_exc_dest;
-    llvm::BasicBlock* cxx_exc_dest;
+    llvm::BasicBlock* exc_dest;
 
-    bool hasHandler() const { return cxx_exc_dest != NULL || capi_exc_dest != NULL; }
+    bool hasHandler() const { return exc_dest != NULL; }
 
-    UnwindInfo(AST_stmt* current_stmt, llvm::BasicBlock* capi_exc_dest, llvm::BasicBlock* cxx_exc_dest)
-        : current_stmt(current_stmt), capi_exc_dest(capi_exc_dest), cxx_exc_dest(cxx_exc_dest) {}
+    UnwindInfo(AST_stmt* current_stmt, llvm::BasicBlock* exc_dest) : current_stmt(current_stmt), exc_dest(exc_dest) {}
+
+    ExceptionStyle preferredExceptionStyle() const;
 
     // Risky!  This means that we can't unwind from this location, and should be used in the
     // rare case that there are language-specific reasons that the statement should not unwind
     // (ex: loading function arguments into the appropriate scopes).
-    static UnwindInfo cantUnwind() { return UnwindInfo(NULL, NULL, NULL); }
+    static UnwindInfo cantUnwind() { return UnwindInfo(NULL, NULL); }
 };
 
 // TODO get rid of this
@@ -146,6 +147,8 @@ public:
         : effort(effort), type_recorder(type_recorder), unw_info(unw_info) {}
 
     TypeRecorder* getTypeRecorder() const { return type_recorder; }
+
+    ExceptionStyle preferredExceptionStyle() const { return unw_info.preferredExceptionStyle(); }
 };
 }
 

src/codegen/irgen/irgenerator.cpp

@@ -94,67 +94,24 @@ llvm::Value* IRGenState::getScratchSpace(int min_bytes) {
     return scratch_space;
 }
 
-// This function is where we decide whether to have a certain operation use CAPI or CXX exceptions.
-// FIXME It's a bit messy at the moment because this requires coordinating between a couple different
-// parts: we need to make sure that the associated landingpad will catch the right kind of exception,
-// and we need to make sure that we can actually emit this statement using capi-exceptions.
-// It doesn't really belong on the IRGenState, but it's here so that we can access this state from
-// separate basic blocks (the IRGenerator only exists for a single bb).
-ExceptionStyle IRGenState::getLandingpadStyle(AST_Invoke* invoke) {
-    assert(!landingpad_styles.count(invoke->exc_dest));
-    ExceptionStyle& r = landingpad_styles[invoke->exc_dest];
-    // printf("Added %d\n", invoke->exc_dest->idx);
-
-    r = CXX; // default
-
-    assert(invoke->stmt->cxx_exception_count == 0); // could be ok but would be unexpected
-
-    // First, check if we think it makes sense:
-    bool should = (invoke->cxx_exception_count >= 10 || invoke->stmt->type == AST_TYPE::Raise);
-    if (!should)
-        return r;
-
-    // Second, check if we are able to do it:
-    // (not all code paths support capi exceptions yet)
-    if (invoke->stmt->type == AST_TYPE::Raise) {
-        AST_Raise* raise_stmt = ast_cast<AST_Raise>(invoke->stmt);
-        // Currently can't do a re-raise with a capi exception:
-        if (raise_stmt->arg0 && !raise_stmt->arg2)
-            r = CAPI;
-        else
-            r = CXX;
-        return r;
-    }
-
-    AST_expr* expr = NULL;
-
-    if (invoke->stmt->type == AST_TYPE::Assign) {
-        expr = ast_cast<AST_Assign>(invoke->stmt)->value;
-    } else if (invoke->stmt->type == AST_TYPE::Expr) {
-        expr = ast_cast<AST_Expr>(invoke->stmt)->value;
-    }
-
-    if (!expr)
-        return r;
-
-    if (expr->type == AST_TYPE::Call) {
-        r = CAPI;
-        return r;
-    }
+ExceptionStyle UnwindInfo::preferredExceptionStyle() const {
+    if (FORCE_LLVM_CAPI)
+        return CAPI;
 
-    if (expr->type == AST_TYPE::Attribute || expr->type == AST_TYPE::Subscript) {
-        r = CAPI;
-        return r;
-    }
-
-    // Some expression type we haven't added yet -- might be worth looking into.
-    r = CXX;
-    return r;
-}
-
-ExceptionStyle IRGenState::getLandingpadStyle(CFGBlock* block) {
-    ASSERT(landingpad_styles.count(block), "%d", block->idx);
-    return landingpad_styles[block];
+    // TODO: I think this makes more sense as a relative percentage rather
+    // than an absolute threshold, but currently we don't count how many
+    // times a statement was executed but didn't throw.
+    //
+    // In theory this means that eventually anything that throws will be viewed
+    // as a highly-throwing statement, but I think that this is less bad than
+    // it might be because the denominator will be roughly fixed since we will
+    // tend to run this check after executing the statement a somewhat-fixed
+    // number of times.
+    // We might want to zero these out after we are done compiling, though.
+    if (current_stmt->cxx_exception_count >= 10)
+        return CAPI;
+
+    return CXX;
 }
 
 static llvm::Value* getClosureParentGep(IREmitter& emitter, llvm::Value* closure) {
@@ -297,12 +254,15 @@ private:
     llvm::CallSite emitCall(const UnwindInfo& unw_info, llvm::Value* callee, const std::vector<llvm::Value*>& args,
                             ExceptionStyle target_exception_style) {
         if (unw_info.hasHandler() && target_exception_style == CXX) {
-            assert(unw_info.cxx_exc_dest);
+            // Create the invoke:
             llvm::BasicBlock* normal_dest
                 = llvm::BasicBlock::Create(g.context, curblock->getName(), irstate->getLLVMFunction());
+            llvm::BasicBlock* exc_dest = irgenerator->getCXXExcDest(unw_info.exc_dest);
             normal_dest->moveAfter(curblock);
 
-            llvm::InvokeInst* rtn = getBuilder()->CreateInvoke(callee, normal_dest, unw_info.cxx_exc_dest, args);
+            llvm::InvokeInst* rtn = getBuilder()->CreateInvoke(callee, normal_dest, exc_dest, args);
+
+            // Normal case:
             getBuilder()->SetInsertPoint(normal_dest);
             curblock = normal_dest;
             return rtn;
@@ -493,37 +453,12 @@ public:
             = llvm::BasicBlock::Create(g.context, curblock->getName(), irstate->getLLVMFunction());
         normal_dest->moveAfter(curblock);
 
-        llvm::BasicBlock* exc_dest;
-        bool exc_caught;
-        if (unw_info.capi_exc_dest) {
-            exc_dest = unw_info.capi_exc_dest;
-            exc_caught = true;
-        } else {
-            exc_dest = llvm::BasicBlock::Create(g.context, curblock->getName() + "_exc", irstate->getLLVMFunction());
-            exc_dest->moveAfter(curblock);
-            exc_caught = false;
-        }
+        llvm::BasicBlock* exc_dest = irgenerator->getCAPIExcDest(unw_info.exc_dest, unw_info.current_stmt);
 
         assert(returned_val->getType() == exc_val->getType());
         llvm::Value* check_val = getBuilder()->CreateICmpEQ(returned_val, exc_val);
         llvm::BranchInst* nullcheck = getBuilder()->CreateCondBr(check_val, exc_dest, normal_dest);
 
-        setCurrentBasicBlock(exc_dest);
-        getBuilder()->CreateCall2(g.funcs.capiExcCaughtInJit,
-                                  embedRelocatablePtr(unw_info.current_stmt, g.llvm_aststmt_type_ptr),
-                                  embedRelocatablePtr(irstate->getSourceInfo(), g.i8_ptr));
-
-        if (!exc_caught) {
-            if (unw_info.cxx_exc_dest) {
-                // TODO: I'm not sure this gets the tracebacks quite right.  this is only for testing though:
-                assert(FORCE_LLVM_CAPI && "this shouldn't happen in non-FORCE mode");
-                createCall(unw_info, g.funcs.reraiseJitCapiExc);
-            } else {
-                getBuilder()->CreateCall(g.funcs.reraiseJitCapiExc);
-            }
-            getBuilder()->CreateUnreachable();
-        }
-
         setCurrentBasicBlock(normal_dest);
     }
 
@@ -582,6 +517,19 @@ private:
     CFGBlock* myblock;
     TypeAnalysis* types;
 
+    // These are some special values used for passing exception data between blocks;
+    // this transfer is not explicitly represented in the CFG which is why it has special
+    // handling here.  ie these variables are how we handle the special "invoke->landingpad"
+    // value transfer, which doesn't involve the normal symbol name handling.
+    //
+    // These are the values that are incoming to a landingpad block:
+    llvm::SmallVector<ExceptionState, 2> incoming_exc_state;
+    // These are the values that are outgoing of an invoke block:
+    llvm::SmallVector<ExceptionState, 2> outgoing_exc_state;
+    llvm::BasicBlock* cxx_exc_dest = NULL, * cxx_exc_final_dest = NULL;
+    llvm::BasicBlock* capi_exc_dest = NULL, * capi_exc_final_dest = NULL;
+    AST_stmt* capi_current_stmt = NULL;
+
     enum State {
         RUNNING,  // normal
         DEAD,     // passed a Return statement; still syntatically valid but the code should not be compiled
@@ -645,7 +593,7 @@ private:
 
         curblock = deopt_bb;
         emitter.getBuilder()->SetInsertPoint(curblock);
-        llvm::Value* v = emitter.createCall2(UnwindInfo(current_statement, NULL, NULL), g.funcs.deopt,
+        llvm::Value* v = emitter.createCall2(UnwindInfo(current_statement, NULL), g.funcs.deopt,
                                              embedRelocatablePtr(node, g.llvm_aststmt_type_ptr), node_value);
         emitter.getBuilder()->CreateRet(v);
 
@@ -695,62 +643,46 @@ private:
                 return boolFromI1(emitter, v);
             }
             case AST_LangPrimitive::LANDINGPAD: {
-                llvm::Value* exc_type;
-                llvm::Value* exc_value;
-                llvm::Value* exc_traceback;
-                if (irstate->getLandingpadStyle(myblock) == CXX) {
-                    // llvm::Function* _personality_func = g.stdlib_module->getFunction("__py_personality_v0");
-                    llvm::Function* _personality_func = g.stdlib_module->getFunction("__gxx_personality_v0");
-                    assert(_personality_func);
-                    llvm::Value* personality_func = g.cur_module->getOrInsertFunction(
-                        _personality_func->getName(), _personality_func->getFunctionType());
-                    assert(personality_func);
-                    llvm::LandingPadInst* landing_pad = emitter.getBuilder()->CreateLandingPad(
-                        llvm::StructType::create(std::vector<llvm::Type*>{ g.i8_ptr, g.i64 }), personality_func, 1);
-                    landing_pad->addClause(getNullPtr(g.i8_ptr));
-
-                    llvm::Value* cxaexc_pointer = emitter.getBuilder()->CreateExtractValue(landing_pad, { 0 });
-
-                    llvm::Function* std_module_catch = g.stdlib_module->getFunction("__cxa_begin_catch");
-                    auto begin_catch_func = g.cur_module->getOrInsertFunction(std_module_catch->getName(),
-                                                                              std_module_catch->getFunctionType());
-                    assert(begin_catch_func);
-
-                    llvm::Value* excinfo_pointer = emitter.getBuilder()->CreateCall(begin_catch_func, cxaexc_pointer);
-                    llvm::Value* excinfo_pointer_casted
-                        = emitter.getBuilder()->CreateBitCast(excinfo_pointer, g.llvm_excinfo_type->getPointerTo());
-
-                    auto* builder = emitter.getBuilder();
-                    exc_type = builder->CreateLoad(builder->CreateConstInBoundsGEP2_32(excinfo_pointer_casted, 0, 0));
-                    exc_value = builder->CreateLoad(builder->CreateConstInBoundsGEP2_32(excinfo_pointer_casted, 0, 1));
-                    exc_traceback
-                        = builder->CreateLoad(builder->CreateConstInBoundsGEP2_32(excinfo_pointer_casted, 0, 2));
+                ConcreteCompilerVariable* exc_type;
+                ConcreteCompilerVariable* exc_value;
+                ConcreteCompilerVariable* exc_tb;
+
+                if (this->incoming_exc_state.size()) {
+                    if (incoming_exc_state.size() == 1) {
+                        exc_type = this->incoming_exc_state[0].exc_type;
+                        exc_value = this->incoming_exc_state[0].exc_value;
+                        exc_tb = this->incoming_exc_state[0].exc_tb;
+                    } else {
+                        llvm::PHINode* phi_exc_type
+                            = emitter.getBuilder()->CreatePHI(g.llvm_value_type_ptr, incoming_exc_state.size());
+                        llvm::PHINode* phi_exc_value
+                            = emitter.getBuilder()->CreatePHI(g.llvm_value_type_ptr, incoming_exc_state.size());
+                        llvm::PHINode* phi_exc_tb
+                            = emitter.getBuilder()->CreatePHI(g.llvm_value_type_ptr, incoming_exc_state.size());
+                        for (auto e : this->incoming_exc_state) {
+                            phi_exc_type->addIncoming(e.exc_type->getValue(), e.from_block);
+                            phi_exc_value->addIncoming(e.exc_value->getValue(), e.from_block);
+                            phi_exc_tb->addIncoming(e.exc_tb->getValue(), e.from_block);
+                        }
+                        exc_type = new ConcreteCompilerVariable(UNKNOWN, phi_exc_type, true);
+                        exc_value = new ConcreteCompilerVariable(UNKNOWN, phi_exc_value, true);
+                        exc_tb = new ConcreteCompilerVariable(UNKNOWN, phi_exc_tb, true);
+                    }
                 } else {
-                    llvm::Value* exc_type_ptr
-                        = new llvm::AllocaInst(g.llvm_value_type_ptr, getConstantInt(1, g.i64), "exc_type",
-                                               irstate->getLLVMFunction()->getEntryBlock().getFirstInsertionPt());
-                    llvm::Value* exc_value_ptr
-                        = new llvm::AllocaInst(g.llvm_value_type_ptr, getConstantInt(1, g.i64), "exc_value",
-                                               irstate->getLLVMFunction()->getEntryBlock().getFirstInsertionPt());
-                    llvm::Value* exc_traceback_ptr
-                        = new llvm::AllocaInst(g.llvm_value_type_ptr, getConstantInt(1, g.i64), "exc_traceback",
-                                               irstate->getLLVMFunction()->getEntryBlock().getFirstInsertionPt());
-                    emitter.getBuilder()->CreateCall3(g.funcs.PyErr_Fetch, exc_type_ptr, exc_value_ptr,
-                                                      exc_traceback_ptr);
-                    // TODO: I think we should be doing this on a python raise() or when we enter a python catch:
-                    emitter.getBuilder()->CreateCall3(g.funcs.PyErr_NormalizeException, exc_type_ptr, exc_value_ptr,
-                                                      exc_traceback_ptr);
-                    exc_type = emitter.getBuilder()->CreateLoad(exc_type_ptr);
-                    exc_value = emitter.getBuilder()->CreateLoad(exc_value_ptr);
-                    exc_traceback = emitter.getBuilder()->CreateLoad(exc_traceback_ptr);
+                    // There can be no incoming exception if the irgenerator was able to prove that
+                    // an exception would not get thrown.
+                    // For example, the cfg code will conservatively assume that any name-access can
+                    // trigger an exception, but the irgenerator will know that definitely-defined
+                    // local symbols will not throw.
+                    exc_type = undefVariable();
+                    exc_value = undefVariable();
+                    exc_tb = undefVariable();
                 }
 
-                assert(exc_type->getType() == g.llvm_value_type_ptr);
-                assert(exc_value->getType() == g.llvm_value_type_ptr);
-                assert(exc_traceback->getType() == g.llvm_value_type_ptr);
-                return makeTuple({ new ConcreteCompilerVariable(UNKNOWN, exc_type, true),
-                                   new ConcreteCompilerVariable(UNKNOWN, exc_value, true),
-                                   new ConcreteCompilerVariable(UNKNOWN, exc_traceback, true) });
+                // clear this out to signal that we consumed them:
+                this->incoming_exc_state.clear();
+
+                return makeTuple({ exc_type, exc_value, exc_tb });
             }
             case AST_LangPrimitive::LOCALS: {
                 return new ConcreteCompilerVariable(UNKNOWN, irstate->getBoxedLocalsVar(), true);
@@ -2355,7 +2287,7 @@ private:
         // but ommitting the first argument is *not* the same as passing None.
 
         ExceptionStyle target_exception_style = CXX;
-        if (unw_info.capi_exc_dest || (FORCE_LLVM_CAPI && node->arg0 && !node->arg2))
+        if (unw_info.preferredExceptionStyle() == CAPI && (node->arg0 && !node->arg2))
             target_exception_style = CAPI;
 
         if (node->arg0 == NULL) {
@@ -2451,15 +2383,7 @@ private:
                 assert(!unw_info.hasHandler());
                 AST_Invoke* invoke = ast_cast<AST_Invoke>(node);
 
-                ExceptionStyle landingpad_style = irstate->getLandingpadStyle(invoke);
-
-                if (landingpad_style == CXX)
-                    doStmt(invoke->stmt, UnwindInfo(node, NULL, entry_blocks[invoke->exc_dest]));
-                else {
-                    // print_ast(invoke);
-                    // printf(" (%d exceptions)\n", invoke->cxx_exception_count);
-                    doStmt(invoke->stmt, UnwindInfo(node, entry_blocks[invoke->exc_dest], NULL));
-                }
+                doStmt(invoke->stmt, UnwindInfo(node, entry_blocks[invoke->exc_dest]));
 
                 assert(state == RUNNING || state == DEAD);
                 if (state == RUNNING) {
@@ -2634,17 +2558,20 @@ public:
         SymbolTable* st = new SymbolTable(symbol_table);
         ConcreteSymbolTable* phi_st = new ConcreteSymbolTable();
 
+        // This should have been consumed:
+        assert(incoming_exc_state.empty());
+
         if (myblock->successors.size() == 0) {
             for (auto& p : *st) {
                 p.second->decvref(emitter);
             }
             st->clear();
             symbol_table.clear();
-            return EndingState(st, phi_st, curblock);
+            return EndingState(st, phi_st, curblock, outgoing_exc_state);
         } else if (myblock->successors.size() > 1) {
             // Since there are no critical edges, all successors come directly from this node,
             // so there won't be any required phis.
-            return EndingState(st, phi_st, curblock);
+            return EndingState(st, phi_st, curblock, outgoing_exc_state);
         }
 
         assert(myblock->successors.size() == 1); // other cases should have been handled
@@ -2654,7 +2581,7 @@ public:
             // If the next block has a single predecessor, don't have to
             // emit any phis.
             // Should probably not emit no-op jumps like this though.
-            return EndingState(st, phi_st, curblock);
+            return EndingState(st, phi_st, curblock, outgoing_exc_state);
         }
 
         // We have one successor, but they have more than one predecessor.
@@ -2687,7 +2614,7 @@ public:
                 ++it;
             }
         }
-        return EndingState(st, phi_st, curblock);
+        return EndingState(st, phi_st, curblock, outgoing_exc_state);
     }
 
     void giveLocalSymbol(InternedString name, CompilerVariable* var) override {
@@ -2834,7 +2761,7 @@ public:
                 doSafePoint(block->body[i]);
 #endif
 
-            doStmt(block->body[i], UnwindInfo(block->body[i], NULL, NULL));
+            doStmt(block->body[i], UnwindInfo(block->body[i], NULL));
         }
         if (VERBOSITY("irgenerator") >= 2) { // print ending symbol table
             printf("  %d fini:", block->idx);
@@ -2847,7 +2774,122 @@ public:
     void doSafePoint(AST_stmt* next_statement) override {
         // Unwind info is always needed in allowGLReadPreemption if it has any chance of
         // running arbitrary code like finalizers.
-        emitter.createCall(UnwindInfo(next_statement, NULL, NULL), g.funcs.allowGLReadPreemption);
+        emitter.createCall(UnwindInfo(next_statement, NULL), g.funcs.allowGLReadPreemption);
+    }
+
+    llvm::BasicBlock* getCAPIExcDest(llvm::BasicBlock* final_dest, AST_stmt* current_stmt) {
+        if (capi_exc_dest) {
+            // We should only have one "final_dest"; we could support having multiple but
+            // for now it should be an invariante:
+            assert(capi_exc_final_dest == final_dest);
+            assert(capi_current_stmt == current_stmt);
+            return capi_exc_dest;
+        }
+
+        llvm::BasicBlock* orig_block = curblock;
+
+        capi_exc_dest = llvm::BasicBlock::Create(g.context, "", irstate->getLLVMFunction());
+        capi_exc_final_dest = final_dest;
+        capi_current_stmt = current_stmt;
+
+        emitter.setCurrentBasicBlock(capi_exc_dest);
+        emitter.getBuilder()->CreateCall2(g.funcs.capiExcCaughtInJit,
+                                          embedRelocatablePtr(current_stmt, g.llvm_aststmt_type_ptr),
+                                          embedRelocatablePtr(irstate->getSourceInfo(), g.i8_ptr));
+
+        if (!final_dest) {
+            emitter.getBuilder()->CreateCall(g.funcs.reraiseJitCapiExc);
+            emitter.getBuilder()->CreateUnreachable();
+        } else {
+            llvm::Value* exc_type_ptr
+                = new llvm::AllocaInst(g.llvm_value_type_ptr, getConstantInt(1, g.i64), "exc_type",
+                                       irstate->getLLVMFunction()->getEntryBlock().getFirstInsertionPt());
+            llvm::Value* exc_value_ptr
+                = new llvm::AllocaInst(g.llvm_value_type_ptr, getConstantInt(1, g.i64), "exc_value",
+                                       irstate->getLLVMFunction()->getEntryBlock().getFirstInsertionPt());
+            llvm::Value* exc_traceback_ptr
+                = new llvm::AllocaInst(g.llvm_value_type_ptr, getConstantInt(1, g.i64), "exc_traceback",
+                                       irstate->getLLVMFunction()->getEntryBlock().getFirstInsertionPt());
+            emitter.getBuilder()->CreateCall3(g.funcs.PyErr_Fetch, exc_type_ptr, exc_value_ptr, exc_traceback_ptr);
+            // TODO: I think we should be doing this on a python raise() or when we enter a python catch:
+            emitter.getBuilder()->CreateCall3(g.funcs.PyErr_NormalizeException, exc_type_ptr, exc_value_ptr,
+                                              exc_traceback_ptr);
+            llvm::Value* exc_type = emitter.getBuilder()->CreateLoad(exc_type_ptr);
+            llvm::Value* exc_value = emitter.getBuilder()->CreateLoad(exc_value_ptr);
+            llvm::Value* exc_traceback = emitter.getBuilder()->CreateLoad(exc_traceback_ptr);
+
+            addOutgoingExceptionState(
+                IRGenerator::ExceptionState(capi_exc_dest, new ConcreteCompilerVariable(UNKNOWN, exc_type, true),
+                                            new ConcreteCompilerVariable(UNKNOWN, exc_value, true),
+                                            new ConcreteCompilerVariable(UNKNOWN, exc_traceback, true)));
+
+            emitter.getBuilder()->CreateBr(final_dest);
+        }
+
+        emitter.setCurrentBasicBlock(orig_block);
+
+        return capi_exc_dest;
+    }
+
+    llvm::BasicBlock* getCXXExcDest(llvm::BasicBlock* final_dest) {
+        if (cxx_exc_dest) {
+            // We should only have one "final_dest"; we could support having multiple but
+            // for now it should be an invariante:
+            assert(cxx_exc_final_dest == final_dest);
+            return cxx_exc_dest;
+        }
+
+        llvm::BasicBlock* orig_block = curblock;
+
+        cxx_exc_dest = llvm::BasicBlock::Create(g.context, "", irstate->getLLVMFunction());
+        cxx_exc_final_dest = final_dest;
+
+        emitter.getBuilder()->SetInsertPoint(cxx_exc_dest);
+
+        llvm::Function* _personality_func = g.stdlib_module->getFunction("__gxx_personality_v0");
+        assert(_personality_func);
+        llvm::Value* personality_func
+            = g.cur_module->getOrInsertFunction(_personality_func->getName(), _personality_func->getFunctionType());
+        assert(personality_func);
+        llvm::LandingPadInst* landing_pad = emitter.getBuilder()->CreateLandingPad(
+            llvm::StructType::create(std::vector<llvm::Type*>{ g.i8_ptr, g.i64 }), personality_func, 1);
+        landing_pad->addClause(getNullPtr(g.i8_ptr));
+
+        llvm::Value* cxaexc_pointer = emitter.getBuilder()->CreateExtractValue(landing_pad, { 0 });
+
+        llvm::Function* std_module_catch = g.stdlib_module->getFunction("__cxa_begin_catch");
+        auto begin_catch_func
+            = g.cur_module->getOrInsertFunction(std_module_catch->getName(), std_module_catch->getFunctionType());
+        assert(begin_catch_func);
+
+        llvm::Value* excinfo_pointer = emitter.getBuilder()->CreateCall(begin_catch_func, cxaexc_pointer);
+        llvm::Value* excinfo_pointer_casted
+            = emitter.getBuilder()->CreateBitCast(excinfo_pointer, g.llvm_excinfo_type->getPointerTo());
+
+        auto* builder = emitter.getBuilder();
+        llvm::Value* exc_type = builder->CreateLoad(builder->CreateConstInBoundsGEP2_32(excinfo_pointer_casted, 0, 0));
+        llvm::Value* exc_value = builder->CreateLoad(builder->CreateConstInBoundsGEP2_32(excinfo_pointer_casted, 0, 1));
+        llvm::Value* exc_traceback
+            = builder->CreateLoad(builder->CreateConstInBoundsGEP2_32(excinfo_pointer_casted, 0, 2));
+
+        addOutgoingExceptionState(ExceptionState(cxx_exc_dest, new ConcreteCompilerVariable(UNKNOWN, exc_type, true),
+                                                 new ConcreteCompilerVariable(UNKNOWN, exc_value, true),
+                                                 new ConcreteCompilerVariable(UNKNOWN, exc_traceback, true)));
+
+        builder->CreateBr(final_dest);
+
+        emitter.setCurrentBasicBlock(orig_block);
+
+        return cxx_exc_dest;
+    }
+
+    void addOutgoingExceptionState(ExceptionState exception_state) override {
+        this->outgoing_exc_state.push_back(exception_state);
+    }
+
+    void setIncomingExceptionState(llvm::SmallVector<ExceptionState, 2> exc_state) override {
+        assert(this->incoming_exc_state.empty());
+        this->incoming_exc_state = std::move(exc_state);
     }
 };
 

src/codegen/irgen/irgenerator.h

@@ -72,8 +72,6 @@ private:
     llvm::Value* frame_info_arg;
     int scratch_size;
 
-    llvm::DenseMap<CFGBlock*, ExceptionStyle> landingpad_styles;
-
 public:
     IRGenState(CLFunction* clfunc, CompiledFunction* cf, SourceInfo* source_info, std::unique_ptr<PhiAnalysis> phis,
                ParamNames* param_names, GCBuilder* gc, llvm::MDNode* func_dbg_info);
@@ -107,15 +105,19 @@ public:
     ParamNames* getParamNames() { return param_names; }
 
     void setFrameInfoArgument(llvm::Value* v) { frame_info_arg = v; }
-
-    ExceptionStyle getLandingpadStyle(AST_Invoke* invoke);
-    ExceptionStyle getLandingpadStyle(CFGBlock* block);
 };
 
 // turns CFGBlocks into LLVM IR
 class IRGenerator {
 private:
 public:
+    struct ExceptionState {
+        llvm::BasicBlock* from_block;
+        ConcreteCompilerVariable* exc_type, *exc_value, *exc_tb;
+        ExceptionState(llvm::BasicBlock* from_block, ConcreteCompilerVariable* exc_type,
+                       ConcreteCompilerVariable* exc_value, ConcreteCompilerVariable* exc_tb)
+            : from_block(from_block), exc_type(exc_type), exc_value(exc_value), exc_tb(exc_tb) {}
+    };
     struct EndingState {
         // symbol_table records which Python variables are bound to what CompilerVariables at the end of this block.
         // phi_symbol_table records the ones that will need to be `phi'd.
@@ -123,8 +125,14 @@ public:
         SymbolTable* symbol_table;
         ConcreteSymbolTable* phi_symbol_table;
         llvm::BasicBlock* ending_block;
-        EndingState(SymbolTable* symbol_table, ConcreteSymbolTable* phi_symbol_table, llvm::BasicBlock* ending_block)
-            : symbol_table(symbol_table), phi_symbol_table(phi_symbol_table), ending_block(ending_block) {}
+        llvm::SmallVector<ExceptionState, 2> exception_state;
+
+        EndingState(SymbolTable* symbol_table, ConcreteSymbolTable* phi_symbol_table, llvm::BasicBlock* ending_block,
+                    llvm::ArrayRef<ExceptionState> exception_state)
+            : symbol_table(symbol_table),
+              phi_symbol_table(phi_symbol_table),
+              ending_block(ending_block),
+              exception_state(exception_state.begin(), exception_state.end()) {}
     };
 
     virtual ~IRGenerator() {}
@@ -139,6 +147,10 @@ public:
     virtual void doSafePoint(AST_stmt* next_statement) = 0;
     virtual void addFrameStackmapArgs(PatchpointInfo* pp, AST_stmt* current_stmt,
                                       std::vector<llvm::Value*>& stackmap_args) = 0;
+    virtual void addOutgoingExceptionState(ExceptionState exception_state) = 0;
+    virtual void setIncomingExceptionState(llvm::SmallVector<ExceptionState, 2> exc_state) = 0;
+    virtual llvm::BasicBlock* getCXXExcDest(llvm::BasicBlock* final_dest) = 0;
+    virtual llvm::BasicBlock* getCAPIExcDest(llvm::BasicBlock* final_dest, AST_stmt* current_stmt) = 0;
 };
 
 class IREmitter;