Have the llvm tier be able to throw capi exceptions

(Not enabled yet)

src/codegen/ast_interpreter.cpp

@@ -743,12 +743,19 @@ Box* ASTInterpreter::doOSR(AST_Jump* node) {
 
     UNAVOIDABLE_STAT_TIMER(t0, "us_timer_in_jitted_code");
     CompiledFunction* partial_func = compilePartialFuncInternal(&exit);
+
     auto arg_tuple = getTupleFromArgsArray(&arg_array[0], arg_array.size());
     Box* r = partial_func->call(std::get<0>(arg_tuple), std::get<1>(arg_tuple), std::get<2>(arg_tuple),
                                 std::get<3>(arg_tuple));
 
-    assert(r);
-    return r;
+    if (partial_func->exception_style == CXX) {
+        assert(r);
+        return r;
+    } else {
+        if (!r)
+            throwCAPIException();
+        return r;
+    }
 }
 
 Value ASTInterpreter::visit_invoke(AST_Invoke* node) {
@@ -1725,14 +1732,24 @@ Box* astInterpretFunction(CLFunction* clfunc, int nargs, Box* closure, Box* gene
         clfunc->dependent_interp_callsites.invalidateAll();
 
         UNAVOIDABLE_STAT_TIMER(t0, "us_timer_in_jitted_code");
+        Box* r;
         if (closure && generator)
-            return optimized->closure_generator_call((BoxedClosure*)closure, (BoxedGenerator*)generator, arg1, arg2,
-                                                     arg3, args);
+            r = optimized->closure_generator_call((BoxedClosure*)closure, (BoxedGenerator*)generator, arg1, arg2, arg3,
+                                                  args);
         else if (closure)
-            return optimized->closure_call((BoxedClosure*)closure, arg1, arg2, arg3, args);
+            r = optimized->closure_call((BoxedClosure*)closure, arg1, arg2, arg3, args);
         else if (generator)
-            return optimized->generator_call((BoxedGenerator*)generator, arg1, arg2, arg3, args);
-        return optimized->call(arg1, arg2, arg3, args);
+            r = optimized->generator_call((BoxedGenerator*)generator, arg1, arg2, arg3, args);
+        else
+            r = optimized->call(arg1, arg2, arg3, args);
+
+        if (optimized->exception_style == CXX)
+            return r;
+        else {
+            if (!r)
+                throwCAPIException();
+            return r;
+        }
     }
 
     Box** vregs = NULL;

src/codegen/compvars.cpp

@@ -1610,15 +1610,13 @@ public:
                        "%d", info.unw_info.current_stmt->lineno);
 
         CompiledFunction* cf = NULL;
+        CompiledFunction* best_exception_mismatch = NULL;
         bool found = false;
         // TODO have to find the right version.. similar to resolveclfunc?
         for (int i = 0; i < cl->versions.size(); i++) {
             cf = cl->versions[i];
             assert(cf->spec->arg_types.size() == cl->numReceivedArgs());
 
-            if (cf->exception_style != exception_style)
-                continue;
-
             bool fits = true;
             for (int j = 0; j < args.size(); j++) {
                 if (!args[j]->canConvertTo(cf->spec->arg_types[j + 1])) {
@@ -1629,14 +1627,22 @@ public:
             if (!fits)
                 continue;
 
+            if (cf->exception_style != exception_style) {
+                if (!best_exception_mismatch)
+                    best_exception_mismatch = cf;
+                continue;
+            }
+
             found = true;
             break;
         }
 
-        if (!found && exception_style == CAPI) {
-            std::string name = g.func_addr_registry.getFuncNameAtAddress(cl->versions[0]->code, true);
-            RELEASE_ASSERT(0, "Please define a capi variant for %s", name.c_str());
+        if (!found) {
+            assert(best_exception_mismatch);
+            cf = best_exception_mismatch;
+            found = true;
         }
+
         RELEASE_ASSERT(found, "");
         RELEASE_ASSERT(cf->code, "");
 
@@ -1708,7 +1714,6 @@ public:
     CompilerVariable* callattr(IREmitter& emitter, const OpInfo& info, ConcreteCompilerVariable* var, BoxedString* attr,
                                CallattrFlags flags, const std::vector<CompilerVariable*>& args,
                                const std::vector<BoxedString*>* keyword_names) override {
-        // XXX investigate
         ExceptionStyle exception_style = info.preferredExceptionStyle();
 
         ConcreteCompilerVariable* called_constant = tryCallattrConstant(

src/codegen/irgen.cpp

@@ -967,7 +967,7 @@ static std::string getUniqueFunctionName(std::string nameprefix, EffortLevel eff
 
 CompiledFunction* doCompile(CLFunction* clfunc, SourceInfo* source, ParamNames* param_names,
                             const OSREntryDescriptor* entry_descriptor, EffortLevel effort,
-                            FunctionSpecialization* spec, std::string nameprefix) {
+                            ExceptionStyle exception_style, FunctionSpecialization* spec, std::string nameprefix) {
     Timer _t("in doCompile");
     Timer _t2;
     long irgen_us = 0;
@@ -1030,8 +1030,7 @@ CompiledFunction* doCompile(CLFunction* clfunc, SourceInfo* source, ParamNames*
         }
     }
 
-
-    CompiledFunction* cf = new CompiledFunction(NULL, spec, NULL, effort, ExceptionStyle::CXX, entry_descriptor);
+    CompiledFunction* cf = new CompiledFunction(NULL, spec, NULL, effort, exception_style, entry_descriptor);
 
     // Make sure that the instruction memory keeps the module object alive.
     // TODO: implement this for real

src/codegen/irgen.h

@@ -113,7 +113,7 @@ bool isIsDefinedName(llvm::StringRef name);
 
 CompiledFunction* doCompile(CLFunction* clfunc, SourceInfo* source, ParamNames* param_names,
                             const OSREntryDescriptor* entry_descriptor, EffortLevel effort,
-                            FunctionSpecialization* spec, std::string nameprefix);
+                            ExceptionStyle exception_style, FunctionSpecialization* spec, std::string nameprefix);
 
 // A common pattern is to branch based off whether a variable is defined but only if it is
 // potentially-undefined.  If it is potentially-undefined, we have to generate control-flow

src/codegen/irgen/hooks.cpp

@@ -197,6 +197,10 @@ CompiledFunction* compileFunction(CLFunction* f, FunctionSpecialization* spec, E
 
     ASSERT(f->versions.size() < 20, "%s %ld", name.c_str(), f->versions.size());
 
+    ExceptionStyle exception_style = CXX;
+    if (FORCE_LLVM_CAPI_THROWS)
+        exception_style = CAPI;
+
     if (VERBOSITY("irgen") >= 1) {
         std::string s;
         llvm::raw_string_ostream ss(s);
@@ -223,7 +227,8 @@ CompiledFunction* compileFunction(CLFunction* f, FunctionSpecialization* spec, E
             ss << "\033[" << colors[(int)effort] << ";1mDoing OSR-entry partial compile of " << source->fn << ":"
                << name << ", starting with backedge to block " << entry_descriptor->backedge->target->idx;
         }
-        ss << " at effort level " << (int)effort << '\n';
+        ss << " at effort level " << (int)effort << " with exception style "
+           << (exception_style == CXX ? "C++" : "CAPI") << '\n';
 
         if (entry_descriptor && VERBOSITY("irgen") >= 2) {
             for (const auto& p : entry_descriptor->args) {
@@ -241,8 +246,7 @@ CompiledFunction* compileFunction(CLFunction* f, FunctionSpecialization* spec, E
     }
 
 
-
-    CompiledFunction* cf = doCompile(f, source, &f->param_names, entry_descriptor, effort, spec, name);
+    CompiledFunction* cf = doCompile(f, source, &f->param_names, entry_descriptor, effort, exception_style, spec, name);
     compileIR(cf, effort);
 
     f->addVersion(cf);
@@ -822,7 +826,9 @@ extern "C" CompiledFunction* reoptCompiledFuncInternal(CompiledFunction* cf) {
 
 
 extern "C" char* reoptCompiledFunc(CompiledFunction* cf) {
-    return (char*)reoptCompiledFuncInternal(cf)->code;
+    CompiledFunction* new_cf = reoptCompiledFuncInternal(cf);
+    assert(new_cf->exception_style == cf->exception_style);
+    return (char*)new_cf->code;
 }
 
 CLFunction* createRTFunction(int num_args, int num_defaults, bool takes_varargs, bool takes_kwargs,

src/codegen/irgen/irgenerator.cpp

@@ -95,7 +95,7 @@ llvm::Value* IRGenState::getScratchSpace(int min_bytes) {
 }
 
 ExceptionStyle UnwindInfo::preferredExceptionStyle() const {
-    if (FORCE_LLVM_CAPI)
+    if (FORCE_LLVM_CAPI_CALLS)
         return CAPI;
 
     // TODO: I think this makes more sense as a relative percentage rather
@@ -253,11 +253,21 @@ private:
 
     llvm::CallSite emitCall(const UnwindInfo& unw_info, llvm::Value* callee, const std::vector<llvm::Value*>& args,
                             ExceptionStyle target_exception_style) {
-        if (unw_info.hasHandler() && target_exception_style == CXX) {
+        if (target_exception_style == CXX && (unw_info.hasHandler() || irstate->getExceptionStyle() == CAPI)) {
             // Create the invoke:
             llvm::BasicBlock* normal_dest
                 = llvm::BasicBlock::Create(g.context, curblock->getName(), irstate->getLLVMFunction());
-            llvm::BasicBlock* exc_dest = irgenerator->getCXXExcDest(unw_info.exc_dest);
+
+            llvm::BasicBlock* final_exc_dest;
+            if (unw_info.hasHandler()) {
+                final_exc_dest = unw_info.exc_dest;
+            } else {
+                assert(irstate->getExceptionStyle() == CAPI && "shoudn't have bothered creating an invoke");
+
+                final_exc_dest = NULL; // signal to reraise as a capi exception
+            }
+
+            llvm::BasicBlock* exc_dest = irgenerator->getCXXExcDest(final_exc_dest);
             normal_dest->moveAfter(curblock);
 
             llvm::InvokeInst* rtn = getBuilder()->CreateInvoke(callee, normal_dest, exc_dest, args);
@@ -526,9 +536,9 @@ private:
     llvm::SmallVector<ExceptionState, 2> incoming_exc_state;
     // These are the values that are outgoing of an invoke block:
     llvm::SmallVector<ExceptionState, 2> outgoing_exc_state;
-    llvm::BasicBlock* cxx_exc_dest = NULL, * cxx_exc_final_dest = NULL;
-    llvm::BasicBlock* capi_exc_dest = NULL, * capi_exc_final_dest = NULL;
-    AST_stmt* capi_current_stmt = NULL;
+    llvm::DenseMap<llvm::BasicBlock*, llvm::BasicBlock*> cxx_exc_dests;
+    llvm::DenseMap<llvm::BasicBlock*, llvm::BasicBlock*> capi_exc_dests;
+    llvm::DenseMap<llvm::BasicBlock*, AST_stmt*> capi_current_statements;
 
     enum State {
         RUNNING,  // normal
@@ -2777,20 +2787,22 @@ public:
         emitter.createCall(UnwindInfo(next_statement, NULL), g.funcs.allowGLReadPreemption);
     }
 
+    // Create a (or reuse an existing) block that will catch a CAPI exception, and then forward
+    // it to the "final_dest" block.  ie final_dest is a block corresponding to the IR level
+    // LANDINGPAD, and this function will createa  helper block that fetches the exception.
+    // As a special-case, a NULL value for final_dest means that this helper block should
+    // instead propagate the exception out of the function.
     llvm::BasicBlock* getCAPIExcDest(llvm::BasicBlock* final_dest, AST_stmt* current_stmt) {
+        llvm::BasicBlock*& capi_exc_dest = capi_exc_dests[final_dest];
         if (capi_exc_dest) {
-            // We should only have one "final_dest"; we could support having multiple but
-            // for now it should be an invariante:
-            assert(capi_exc_final_dest == final_dest);
-            assert(capi_current_stmt == current_stmt);
+            assert(capi_current_statements[final_dest] == current_stmt);
             return capi_exc_dest;
         }
 
         llvm::BasicBlock* orig_block = curblock;
 
         capi_exc_dest = llvm::BasicBlock::Create(g.context, "", irstate->getLLVMFunction());
-        capi_exc_final_dest = final_dest;
-        capi_current_stmt = current_stmt;
+        capi_current_statements[final_dest] = current_stmt;
 
         emitter.setCurrentBasicBlock(capi_exc_dest);
         emitter.getBuilder()->CreateCall2(g.funcs.capiExcCaughtInJit,
@@ -2798,9 +2810,15 @@ public:
                                           embedRelocatablePtr(irstate->getSourceInfo(), g.i8_ptr));
 
         if (!final_dest) {
-            emitter.getBuilder()->CreateCall(g.funcs.reraiseJitCapiExc);
-            emitter.getBuilder()->CreateUnreachable();
+            // Propagate the exception out of the function:
+            if (irstate->getExceptionStyle() == CXX) {
+                emitter.getBuilder()->CreateCall(g.funcs.reraiseJitCapiExc);
+                emitter.getBuilder()->CreateUnreachable();
+            } else {
+                emitter.getBuilder()->CreateRet(getNullPtr(g.llvm_value_type_ptr));
+            }
         } else {
+            // Catch the exception and forward to final_dest:
             llvm::Value* exc_type_ptr
                 = new llvm::AllocaInst(g.llvm_value_type_ptr, getConstantInt(1, g.i64), "exc_type",
                                        irstate->getLLVMFunction()->getEntryBlock().getFirstInsertionPt());
@@ -2832,17 +2850,13 @@ public:
     }
 
     llvm::BasicBlock* getCXXExcDest(llvm::BasicBlock* final_dest) {
-        if (cxx_exc_dest) {
-            // We should only have one "final_dest"; we could support having multiple but
-            // for now it should be an invariante:
-            assert(cxx_exc_final_dest == final_dest);
+        llvm::BasicBlock*& cxx_exc_dest = cxx_exc_dests[final_dest];
+        if (cxx_exc_dest)
             return cxx_exc_dest;
-        }
 
         llvm::BasicBlock* orig_block = curblock;
 
         cxx_exc_dest = llvm::BasicBlock::Create(g.context, "", irstate->getLLVMFunction());
-        cxx_exc_final_dest = final_dest;
 
         emitter.getBuilder()->SetInsertPoint(cxx_exc_dest);
 
@@ -2872,11 +2886,22 @@ public:
         llvm::Value* exc_traceback
             = builder->CreateLoad(builder->CreateConstInBoundsGEP2_32(excinfo_pointer_casted, 0, 2));
 
-        addOutgoingExceptionState(ExceptionState(cxx_exc_dest, new ConcreteCompilerVariable(UNKNOWN, exc_type, true),
-                                                 new ConcreteCompilerVariable(UNKNOWN, exc_value, true),
-                                                 new ConcreteCompilerVariable(UNKNOWN, exc_traceback, true)));
+        if (final_dest) {
+            // Catch the exception and forward to final_dest:
+            addOutgoingExceptionState(ExceptionState(cxx_exc_dest,
+                                                     new ConcreteCompilerVariable(UNKNOWN, exc_type, true),
+                                                     new ConcreteCompilerVariable(UNKNOWN, exc_value, true),
+                                                     new ConcreteCompilerVariable(UNKNOWN, exc_traceback, true)));
 
-        builder->CreateBr(final_dest);
+            builder->CreateBr(final_dest);
+        } else {
+            // Propagate the exception out of the function.
+            // We shouldn't be hitting this case if the current function is CXX-style; then we should have
+            // just not created an Invoke and let the exception machinery propagate it for us.
+            assert(irstate->getExceptionStyle() == CAPI);
+            builder->CreateCall3(g.funcs.PyErr_Restore, exc_type, exc_value, exc_traceback);
+            builder->CreateRet(getNullPtr(g.llvm_value_type_ptr));
+        }
 
         emitter.setCurrentBasicBlock(orig_block);
 

src/codegen/irgen/irgenerator.h

@@ -80,6 +80,8 @@ public:
     CompiledFunction* getCurFunction() { return cf; }
     CLFunction* getCL() { return clfunc; }
 
+    ExceptionStyle getExceptionStyle() { return cf->exception_style; }
+
     llvm::Function* getLLVMFunction() { return cf->func; }
 
     EffortLevel getEffortLevel() { return cf->effort; }

src/codegen/runtime_hooks.cpp

@@ -311,6 +311,7 @@ void initGlobalFuncs(GlobalState& g) {
     GET(raise3_capi);
     GET(PyErr_Fetch);
     GET(PyErr_NormalizeException);
+    GET(PyErr_Restore);
     GET(capiExcCaughtInJit);
     GET(reraiseJitCapiExc);
     GET(deopt);

src/codegen/runtime_hooks.h

@@ -51,7 +51,7 @@ struct GlobalFuncs {
 
     llvm::Value* __cxa_end_catch;
     llvm::Value* raise0, *raise3, *raise3_capi;
-    llvm::Value* PyErr_Fetch, *PyErr_NormalizeException, *capiExcCaughtInJit, *reraiseJitCapiExc;
+    llvm::Value* PyErr_Fetch, *PyErr_NormalizeException, *PyErr_Restore, *capiExcCaughtInJit, *reraiseJitCapiExc;
     llvm::Value* deopt;
 
     llvm::Value* div_float_float, *floordiv_float_float, *mod_float_float, *pow_float_float;

src/core/options.cpp

@@ -45,8 +45,10 @@ bool USE_REGALLOC_BASIC = true;
 bool PAUSE_AT_ABORT = false;
 bool ENABLE_TRACEBACKS = true;
 // Forces the llvm jit to use capi exceptions whenever it can, as opposed to whenever it thinks
-// it is faster:
-bool FORCE_LLVM_CAPI = false;
+// it is faster.  The CALLS version is for calls that the llvm jit will make, and the THROWS version
+// is for the exceptions it will throw.
+bool FORCE_LLVM_CAPI_CALLS = false;
+bool FORCE_LLVM_CAPI_THROWS = false;
 
 int OSR_THRESHOLD_INTERPRETER = 25;
 int REOPT_THRESHOLD_INTERPRETER = 25;

src/core/options.h

@@ -39,7 +39,7 @@ extern int MAX_OBJECT_CACHE_ENTRIES;
 
 extern bool SHOW_DISASM, FORCE_INTERPRETER, FORCE_OPTIMIZE, PROFILE, DUMPJIT, TRAP, USE_STRIPPED_STDLIB,
     CONTINUE_AFTER_FATAL, ENABLE_INTERPRETER, ENABLE_BASELINEJIT, ENABLE_PYPA_PARSER, USE_REGALLOC_BASIC,
-    PAUSE_AT_ABORT, ENABLE_TRACEBACKS, ASSEMBLY_LOGGING, FORCE_LLVM_CAPI;
+    PAUSE_AT_ABORT, ENABLE_TRACEBACKS, ASSEMBLY_LOGGING, FORCE_LLVM_CAPI_CALLS, FORCE_LLVM_CAPI_THROWS;
 
 extern bool ENABLE_ICS, ENABLE_ICGENERICS, ENABLE_ICGETITEMS, ENABLE_ICSETITEMS, ENABLE_ICDELITEMS, ENABLE_ICBINEXPS,
     ENABLE_ICNONZEROS, ENABLE_ICCALLSITES, ENABLE_ICSETATTRS, ENABLE_ICGETATTRS, ENALBE_ICDELATTRS, ENABLE_ICGETGLOBALS,

src/runtime/inline/link_forcer.cpp

@@ -129,6 +129,7 @@ void force() {
     FORCE(raise3_capi);
     FORCE(PyErr_Fetch);
     FORCE(PyErr_NormalizeException);
+    FORCE(PyErr_Restore);
     FORCE(capiExcCaughtInJit);
     FORCE(reraiseJitCapiExc);
     FORCE(deopt);

src/runtime/util.cpp

@@ -262,6 +262,10 @@ extern "C" void dumpEx(void* p, int levels) {
             printf("Has %ld function versions\n", cl->versions.size());
             for (CompiledFunction* cf : cl->versions) {
                 bool got_name;
+                if (cf->exception_style == CXX)
+                    printf("CXX style: ");
+                else
+                    printf("CAPI style: ");
                 std::string name = g.func_addr_registry.getFuncNameAtAddress(cf->code, true, &got_name);
                 if (got_name)
                     printf("%s\n", name.c_str());