Merge commit 'dc91299a' into refcounting

builds but has some issues running

from_cpython/Lib/test/test_signal.py

-# expected: fail
 import unittest
 from test import test_support
 from contextlib import closing
@@ -80,7 +79,8 @@ class InterProcessSignalTests(unittest.TestCase):
         # don't worry about re-setting the default handlers.
         signal.signal(signal.SIGHUP, self.handlerA)
         signal.signal(signal.SIGUSR1, self.handlerB)
-        signal.signal(signal.SIGUSR2, signal.SIG_IGN)
+        # Pyston change: pyston uses SIGUSR2 internally
+        # signal.signal(signal.SIGUSR2, signal.SIG_IGN)
         signal.signal(signal.SIGALRM, signal.default_int_handler)
 
         # Variables the signals will modify:
@@ -117,9 +117,11 @@ class InterProcessSignalTests(unittest.TestCase):
             if test_support.verbose:
                 print "HandlerBCalled exception caught"
 
-        child = ignoring_eintr(subprocess.Popen, ['kill', '-USR2', str(pid)])
-        if child:
-            self.wait(child)  # Nothing should happen.
+
+        # Pyston change: pyston uses SIGUSR2 internally
+        # child = ignoring_eintr(subprocess.Popen, ['kill', '-USR2', str(pid)])
+        # if child:
+        #     self.wait(child)  # Nothing should happen.
 
         try:
             signal.alarm(1)

from_cpython/Modules/signalmodule.c

@@ -895,10 +895,6 @@ PyErr_CheckSignals(void)
     if (!is_tripped)
         return 0;
 
-    // Pyston change:
-    Py_FatalError("TODO");
-
-#if 0
     int i;
     PyObject *f;
 
@@ -943,7 +939,6 @@ PyErr_CheckSignals(void)
             Py_DECREF(result);
         }
     }
-#endif
 
     return 0;
 }

src/codegen/ast_interpreter.cpp

@@ -990,43 +990,66 @@ Value ASTInterpreter::visit_stmt(AST_stmt* node) {
         printf("\n");
     }
 
+    Value rtn;
     switch (node->type) {
         case AST_TYPE::Assert:
-            return visit_assert((AST_Assert*)node);
+            rtn = visit_assert((AST_Assert*)node);
+            ASTInterpreterJitInterface::pendingCallsCheckHelper();
+            break;
         case AST_TYPE::Assign:
-            return visit_assign((AST_Assign*)node);
+            rtn = visit_assign((AST_Assign*)node);
+            ASTInterpreterJitInterface::pendingCallsCheckHelper();
+            break;
         case AST_TYPE::Delete:
-            return visit_delete((AST_Delete*)node);
+            rtn = visit_delete((AST_Delete*)node);
+            ASTInterpreterJitInterface::pendingCallsCheckHelper();
+            break;
         case AST_TYPE::Exec:
-            return visit_exec((AST_Exec*)node);
+            rtn = visit_exec((AST_Exec*)node);
+            ASTInterpreterJitInterface::pendingCallsCheckHelper();
+            break;
         case AST_TYPE::Expr:
             // docstrings are str constant expression statements.
             // ignore those while interpreting.
-            if ((((AST_Expr*)node)->value)->type != AST_TYPE::Str)
-                return visit_expr((AST_Expr*)node);
+            if ((((AST_Expr*)node)->value)->type != AST_TYPE::Str) {
+                rtn = visit_expr((AST_Expr*)node);
+                ASTInterpreterJitInterface::pendingCallsCheckHelper();
+            }
             break;
         case AST_TYPE::Pass:
-            return Value(); // nothing todo
+            ASTInterpreterJitInterface::pendingCallsCheckHelper();
+            break; // nothing todo
         case AST_TYPE::Print:
-            return visit_print((AST_Print*)node);
+            rtn = visit_print((AST_Print*)node);
+            ASTInterpreterJitInterface::pendingCallsCheckHelper();
+            break;
         case AST_TYPE::Raise:
-            return visit_raise((AST_Raise*)node);
+            rtn = visit_raise((AST_Raise*)node);
+            ASTInterpreterJitInterface::pendingCallsCheckHelper();
+            break;
         case AST_TYPE::Return:
-            return visit_return((AST_Return*)node);
+            rtn = visit_return((AST_Return*)node);
+            ASTInterpreterJitInterface::pendingCallsCheckHelper();
+            break;
         case AST_TYPE::Global:
-            return visit_global((AST_Global*)node);
+            rtn = visit_global((AST_Global*)node);
+            ASTInterpreterJitInterface::pendingCallsCheckHelper();
+            break;
 
         // pseudo
         case AST_TYPE::Branch:
-            return visit_branch((AST_Branch*)node);
+            rtn = visit_branch((AST_Branch*)node);
+            break;
         case AST_TYPE::Jump:
-            return visit_jump((AST_Jump*)node);
+            rtn = visit_jump((AST_Jump*)node);
+            break;
         case AST_TYPE::Invoke:
-            return visit_invoke((AST_Invoke*)node);
+            rtn = visit_invoke((AST_Invoke*)node);
+            break;
         default:
             RELEASE_ASSERT(0, "not implemented");
     };
-    return Value();
+    return rtn;
 }
 
 Value ASTInterpreter::visit_return(AST_Return* node) {
@@ -1750,6 +1773,11 @@ Box* ASTInterpreterJitInterface::landingpadHelper(void* _interpreter) {
     return rtn;
 }
 
+void ASTInterpreterJitInterface::pendingCallsCheckHelper() {
+    if (unlikely(_pendingcalls_to_do))
+        makePendingCalls();
+}
+
 void ASTInterpreterJitInterface::setExcInfoHelper(void* _interpreter, Box* type, Box* value, Box* traceback) {
     ASTInterpreter* interpreter = (ASTInterpreter*)_interpreter;
     interpreter->getFrameInfo()->exc = ExcInfo(type, value, traceback);

src/codegen/ast_interpreter.h

@@ -45,6 +45,7 @@ struct ASTInterpreterJitInterface {
     static Box* derefHelper(void* interp, InternedString s);
     static Box* doOSRHelper(void* interp, AST_Jump* node);
     static Box* landingpadHelper(void* interp);
+    static void pendingCallsCheckHelper();
     static void setExcInfoHelper(void* interp, Box* type, Box* value, Box* traceback);
     static void setLocalClosureHelper(void* interp, long vreg, InternedString id, Box* v);
     static void uncacheExcInfoHelper(void* interp);

src/codegen/baseline_jit.cpp

@@ -497,6 +497,10 @@ void JitFragmentWriter::emitOSRPoint(AST_Jump* node) {
     if (LOG_BJIT_ASSEMBLY) comment("BJIT: emitOSRPoint() end");
 }
 
+void JitFragmentWriter::emitPendingCallsCheck() {
+    call(false, (void*)ASTInterpreterJitInterface::pendingCallsCheckHelper);
+}
+
 void JitFragmentWriter::emitPrint(RewriterVar* dest, RewriterVar* var, bool nl) {
     if (LOG_BJIT_ASSEMBLY) comment("BJIT: emitPrint() start");
     if (!dest)
@@ -769,13 +773,17 @@ RewriterVar* JitFragmentWriter::emitPPCall(void* func_addr, llvm::ArrayRef<Rewri
         RewriterVar* obj_cls_var = result->getAttr(offsetof(Box, cls));
         addAction([=]() { _emitRecordType(type_recorder_var, obj_cls_var); }, { type_recorder_var, obj_cls_var },
                   ActionType::NORMAL);
+
+        emitPendingCallsCheck();
         return result;
     }
+    emitPendingCallsCheck();
     if (LOG_BJIT_ASSEMBLY) comment("BJIT: emitPPCall() end");
     return result;
 #else
     assert(args_vec.size() < 7);
-    auto result = call(false, func_addr, args_vec);
+    RewriterVar* result = call(false, func_addr, args_vec);
+    emitPendingCallsCheck();
     if (LOG_BJIT_ASSEMBLY) comment("BJIT: emitPPCall() end");
     return result;
 #endif

src/codegen/baseline_jit.h

@@ -253,6 +253,7 @@ public:
     void emitExec(RewriterVar* code, RewriterVar* globals, RewriterVar* locals, FutureFlags flags);
     void emitJump(CFGBlock* b);
     void emitOSRPoint(AST_Jump* node);
+    void emitPendingCallsCheck();
     void emitPrint(RewriterVar* dest, RewriterVar* var, bool nl);
     void emitRaise0();
     void emitRaise3(RewriterVar* arg0, RewriterVar* arg1, RewriterVar* arg2);

src/codegen/entry.cpp

@@ -378,16 +378,6 @@ static void handle_sigprof_investigate_stattimer(int signum) {
 }
 #endif
 
-static void handle_sigint(int signum) {
-    assert(signum == SIGINT);
-    // TODO: this should set a flag saying a KeyboardInterrupt is pending.
-    // For now, just call abort(), so that we get a traceback at least.
-    fprintf(stderr, "SIGINT!\n");
-    Py_Finalize();
-    Stats::dump(false);
-    abort();
-}
-
 extern "C" void Py_Initialize() noexcept {
     llvm::InitializeNativeTarget();
     llvm::InitializeNativeTargetAsmPrinter();
@@ -481,9 +471,8 @@ extern "C" void Py_Initialize() noexcept {
 
     setupRuntime();
 
-    // signal(SIGFPE, &handle_sigfpe);
-    signal(SIGUSR1, &handle_sigusr1);
-    signal(SIGINT, &handle_sigint);
+// signal(SIGFPE, &handle_sigfpe);
+// signal(SIGUSR1, &handle_sigusr1);
 
 #if ENABLE_SAMPLING_PROFILER
     struct itimerval prof_timer;

src/codegen/irgen/irgenerator.cpp

@@ -357,11 +357,53 @@ private:
     llvm::BasicBlock*& curblock;
     IRGenerator* irgenerator;
 
+    void emitPendingCallsCheck(llvm::BasicBlock* exc_dest) {
+        auto&& builder = *getBuilder();
+
+        llvm::GlobalVariable* pendingcalls_to_do_gv = g.cur_module->getGlobalVariable("_pendingcalls_to_do");
+        if (!pendingcalls_to_do_gv) {
+            static_assert(sizeof(_pendingcalls_to_do) == 4, "");
+            pendingcalls_to_do_gv = new llvm::GlobalVariable(
+                *g.cur_module, g.i32, false, llvm::GlobalValue::ExternalLinkage, 0, "_pendingcalls_to_do");
+            pendingcalls_to_do_gv->setAlignment(4);
+        }
+
+        llvm::BasicBlock* cur_block = builder.GetInsertBlock();
+
+        llvm::BasicBlock* pendingcalls_set = createBasicBlock("_pendingcalls_set");
+        pendingcalls_set->moveAfter(cur_block);
+        llvm::BasicBlock* join_block = createBasicBlock("continue_after_pendingcalls_check");
+        join_block->moveAfter(pendingcalls_set);
+
+        llvm::Value* pendingcalls_to_do_val = builder.CreateLoad(pendingcalls_to_do_gv, true /* volatile */);
+        llvm::Value* is_zero
+            = builder.CreateICmpEQ(pendingcalls_to_do_val, getConstantInt(0, pendingcalls_to_do_val->getType()));
+
+        llvm::Metadata* md_vals[]
+            = { llvm::MDString::get(g.context, "branch_weights"), llvm::ConstantAsMetadata::get(getConstantInt(1000)),
+                llvm::ConstantAsMetadata::get(getConstantInt(1)) };
+        llvm::MDNode* branch_weights = llvm::MDNode::get(g.context, llvm::ArrayRef<llvm::Metadata*>(md_vals));
+
+
+        builder.CreateCondBr(is_zero, join_block, pendingcalls_set, branch_weights);
+        {
+            setCurrentBasicBlock(pendingcalls_set);
+
+            if (exc_dest) {
+                builder.CreateInvoke(g.funcs.makePendingCalls, join_block, exc_dest);
+            } else {
+                builder.CreateCall(g.funcs.makePendingCalls);
+                builder.CreateBr(join_block);
+            }
+        }
+
+        cur_block = join_block;
+        setCurrentBasicBlock(join_block);
+    }
+
     llvm::CallSite emitCall(const UnwindInfo& unw_info, llvm::Value* callee, const std::vector<llvm::Value*>& args,
                             ExceptionStyle target_exception_style) {
-        llvm::Value* stmt = unw_info.current_stmt ? embedRelocatablePtr(unw_info.current_stmt, g.llvm_aststmt_type_ptr)
-                                                  : getNullPtr(g.llvm_aststmt_type_ptr);
-        getBuilder()->CreateStore(stmt, irstate->getStmtVar());
+        emitSetCurrentStmt(unw_info.current_stmt);
 
         bool needs_cxx_interception;
         if (unw_info.exc_dest == NO_CXX_INTERCEPTION) {
@@ -389,12 +431,24 @@ private:
 
             llvm::InvokeInst* rtn = getBuilder()->CreateInvoke(callee, normal_dest, exc_dest, args);
 
+            // Note -- this code can often create critical edges between LLVM blocks.
+            // The refcounting system has some support for handling this, but if we start generating
+            // IR that it can't handle, we might have to break the critical edges here (or teach the
+            // refcounting system how to do that.)
+
             // Normal case:
             getBuilder()->SetInsertPoint(normal_dest);
             curblock = normal_dest;
+            if (unw_info.hasHandler())
+                emitPendingCallsCheck(irgenerator->getCXXExcDest(unw_info));
+            else
+                emitPendingCallsCheck(NULL);
             return rtn;
         } else {
+
             llvm::CallInst* cs = getBuilder()->CreateCall(callee, args);
+            if (target_exception_style == CXX)
+                emitPendingCallsCheck(NULL);
             return cs;
         }
     }
@@ -496,8 +550,18 @@ public:
         return llvm::BasicBlock::Create(g.context, name, irstate->getLLVMFunction());
     }
 
-    llvm::Instruction* createCall(const UnwindInfo& unw_info, llvm::Value* callee, const std::vector<llvm::Value*>& args,
-                            ExceptionStyle target_exception_style = CXX) override {
+    // Our current frame introspection approach requires that we update the currently executed stmt before doing a call
+    // to a function which could throw an exception, inspect the python call frame,...
+    // Only patchpoint don't need to set the current statement because the stmt will be inluded in the stackmap args.
+    void emitSetCurrentStmt(AST_stmt* stmt) {
+        getBuilder()->CreateStore(stmt ? embedRelocatablePtr(stmt, g.llvm_aststmt_type_ptr)
+                                       : getNullPtr(g.llvm_aststmt_type_ptr),
+                                  irstate->getStmtVar());
+    }
+
+    llvm::Instruction* createCall(const UnwindInfo& unw_info, llvm::Value* callee,
+                                  const std::vector<llvm::Value*>& args,
+                                  ExceptionStyle target_exception_style = CXX) override {
 #ifndef NDEBUG
         // Copied the argument-type-checking from CallInst::init, since the patchpoint arguments don't
         // get checked.
@@ -2107,7 +2171,7 @@ private:
         ConcreteCompilerVariable* rtn = val->makeConverted(emitter, opt_rtn_type);
 
         if (!irstate->getCurFunction()->entry_descriptor)
-            emitter.createCall(unw_info, g.funcs.deinitFrame, irstate->getFrameInfoVar());
+            emitter.getBuilder()->CreateCall(g.funcs.deinitFrame, irstate->getFrameInfoVar());
 
         assert(rtn->getValue());
         auto ret_inst = emitter.getBuilder()->CreateRet(rtn->getValue());
@@ -2841,9 +2905,10 @@ public:
     }
 
     void doSafePoint(AST_stmt* next_statement) override {
-        // Unwind info is always needed in allowGLReadPreemption if it has any chance of
-        // running arbitrary code like finalizers.
-        emitter.createCall(UnwindInfo(next_statement, NULL), g.funcs.allowGLReadPreemption, NOEXC);
+        // We need to setup frame introspection by updating the current stmt because we can run can run arbitrary code
+        // like finalizers inside allowGLReadPreemption.
+        emitter.emitSetCurrentStmt(next_statement);
+        emitter.getBuilder()->CreateCall(g.funcs.allowGLReadPreemption);
     }
 
     // Create a (or reuse an existing) block that will catch a CAPI exception, and then forward
@@ -2865,8 +2930,9 @@ public:
             assert(!phi_node);
             phi_node = emitter.getBuilder()->CreatePHI(g.llvm_aststmt_type_ptr, 0);
 
-            emitter.createCall(UnwindInfo(current_stmt, NULL), g.funcs.caughtCapiException,
-                               { phi_node, embedRelocatablePtr(irstate->getSourceInfo(), g.i8_ptr) });
+            emitter.emitSetCurrentStmt(current_stmt);
+            emitter.getBuilder()->CreateCall(g.funcs.caughtCapiException,
+                                             { phi_node, embedRelocatablePtr(irstate->getSourceInfo(), g.i8_ptr) });
 
             if (!final_dest) {
                 // Propagate the exception out of the function:
@@ -2874,7 +2940,7 @@ public:
                     emitter.getBuilder()->CreateCall(g.funcs.reraiseCapiExcAsCxx);
                     emitter.getBuilder()->CreateUnreachable();
                 } else {
-                    emitter.createCall(UnwindInfo(current_stmt, NULL), g.funcs.deinitFrame, irstate->getFrameInfoVar());
+                    emitter.getBuilder()->CreateCall(g.funcs.deinitFrame, irstate->getFrameInfoVar());
                     emitter.getBuilder()->CreateRet(getNullPtr(g.llvm_value_type_ptr));
                 }
             } else {
@@ -2915,6 +2981,8 @@ public:
         assert(capi_exc_dest);
         assert(phi_node);
 
+        // Break a likely critical edge, for the benefit of the refcounter.
+        // We should probably just teach the refcounter to break the edges on-demand though.
         llvm::BasicBlock* critedge_breaker = llvm::BasicBlock::Create(g.context, "", irstate->getLLVMFunction());
         critedge_breaker->moveBefore(capi_exc_dest);
         llvm::BranchInst::Create(capi_exc_dest, critedge_breaker);

src/codegen/irgen/refcounts.cpp

@@ -42,6 +42,14 @@
 
 namespace pyston {
 
+static int numSuccessors(llvm::BasicBlock* b) {
+    return std::distance(llvm::succ_begin(b), llvm::succ_end(b));
+}
+
+static int numPredecessors(llvm::BasicBlock* b) {
+    return std::distance(llvm::pred_begin(b), llvm::pred_end(b));
+}
+
 llvm::Value* RefcountTracker::setType(llvm::Value* v, RefType reftype) {
     assert(!llvm::isa<llvm::UndefValue>(v));
 
@@ -69,10 +77,11 @@ void RefcountTracker::refConsumed(llvm::Value* v, llvm::Instruction* inst) {
     //var.ref_consumers.push_back(inst);
 }
 
-llvm::Instruction* findInsertionPoint(llvm::BasicBlock* BB) {
-    ASSERT(pred_begin(BB) == pred_end(BB) || pred_end(BB) == ++pred_begin(BB),
-           "We shouldn't be inserting anything at the beginning of blocks with multiple predecessors (%s)",
-           BB->getName().data());
+llvm::Instruction* findInsertionPoint(llvm::BasicBlock* BB, bool multipred_ok = false) {
+    if (!multipred_ok)
+        ASSERT(numPredecessors(BB) <= 1,
+               "We shouldn't be inserting anything at the beginning of blocks with multiple predecessors (%s)",
+               BB->getName().data());
 
     if (llvm::isa<llvm::LandingPadInst>(*BB->begin())) {
         // Don't split up the landingpad+extract+cxa_begin_catch
@@ -522,9 +531,14 @@ void RefcountTracker::addRefcounts(IRGenState* irstate) {
                 for (auto SBB : successors) {
                     auto it = states[SBB].ending_refs.find(v);
                     if (it != states[SBB].ending_refs.end()) {
+                        //llvm::outs() << "Going from " << BB.getName() << " to " << SBB->getName() << ", have "
+                                     //<< it->second << " refs on " << *v << '\n';
                         min_refs = std::min(it->second, min_refs);
-                    } else
+                    } else {
+                        //llvm::outs() << "Going from " << BB.getName() << " to " << SBB->getName()
+                                     //<< ", have 0 (missing) refs on " << *v << '\n';
                         min_refs = 0;
+                    }
                 }
 
                 if (refstate.reftype == RefType::OWNED)
@@ -536,7 +550,10 @@ void RefcountTracker::addRefcounts(IRGenState* irstate) {
                     if (it != states[SBB].ending_refs.end()) {
                         this_refs = it->second;
                     }
+
                     if (this_refs > min_refs) {
+                        //llvm::outs() << "Going from " << BB.getName() << " to " << SBB->getName() << ", need to add "
+                                     //<< (this_refs - min_refs) << " refs to " << *v << '\n';
                         state.increfs.push_back(RefOp({v, refstate.nullable, this_refs - min_refs, findInsertionPoint(SBB)}));
                     } else if (this_refs < min_refs) {
                         assert(refstate.reftype == RefType::OWNED);
@@ -590,9 +607,8 @@ void RefcountTracker::addRefcounts(IRGenState* irstate) {
                 if (num_times_as_op[op] > num_consumed) {
                     if (rt->vars[op].reftype == RefType::OWNED) {
                         if (state.ending_refs[op] == 0) {
-                            // llvm::outs() << "Last use of " << *op << " is at " << I << "; adding a decref after\n";
+                             //llvm::outs() << "Last use of " << *op << " is at " << I << "; adding a decref after\n";
 
-                            // Don't do any updates now since we are iterating over the bb
                             if (llvm::InvokeInst* invoke = llvm::dyn_cast<llvm::InvokeInst>(&I)) {
                                 state.decrefs.push_back(RefOp({op, rt->vars[op].nullable, 1, findInsertionPoint(invoke->getNormalDest())}));
                                 state.decrefs.push_back(RefOp({op, rt->vars[op].nullable, 1, findInsertionPoint(invoke->getUnwindDest())}));
@@ -652,6 +668,43 @@ void RefcountTracker::addRefcounts(IRGenState* irstate) {
             }
         }
 
+        // If we have any predecessor blocks with multiple successors, then let's just zero-out our pending refs.
+        // If we avoid critical edges, we have have the predecessor take care of this, but to allow critical edges
+        // we need to have a shared "interface" that we adhere to.
+        //
+        // Note: I'm not sure if the "try to coalesce refcounting operations" optimizations actually do anything.
+        // In that case there isn't any cost to hitting this case, and since it's simpler maybe we should do
+        // this for everything?
+        bool have_multisucc_predecessor = false;
+        for (auto PBB : llvm::predecessors(&BB)) {
+            if (numSuccessors(PBB) > 1) {
+                have_multisucc_predecessor = true;
+                break;
+            }
+        }
+
+        if (have_multisucc_predecessor) {
+            for (auto&& p : state.ending_refs) {
+                llvm::Value* v = p.first;
+                auto var = rt->vars[v];
+
+                int starting_refs = (var.reftype == RefType::OWNED ? 1 : 0);
+
+                assert(starting_refs <= state.ending_refs[v]);
+                if (state.ending_refs[v] == starting_refs)
+                    continue;
+
+                llvm::Instruction* insertion_pt = findInsertionPoint(&BB, /* multipred_ok */ true);
+
+                state.increfs.push_back(RefOp({ v, var.nullable, state.ending_refs[v] - starting_refs, insertion_pt }));
+
+                if (starting_refs == 0)
+                    state.ending_refs.erase(p.first);
+                else
+                    state.ending_refs[p.first] = starting_refs;
+            }
+        }
+
         // If this is the entry block, finish dealing with the ref state rather than handing off to a predecessor
         if (&BB == &BB.getParent()->front()) {
             for (auto&& p : state.ending_refs) {

src/codegen/runtime_hooks.cpp

@@ -200,6 +200,7 @@ void initGlobalFuncs(GlobalState& g) {
     GET(createSet);
     GET(initFrame);
     GET(deinitFrame);
+    GET(makePendingCalls);
 
     GET(getattr);
     GET(getattr_capi);

src/codegen/runtime_hooks.h

@@ -34,7 +34,7 @@ struct GlobalFuncs {
 
     llvm::Value* boxInt, *unboxInt, *boxFloat, *unboxFloat, *createFunctionFromMetadata, *getFunctionMetadata,
         *boxInstanceMethod, *boxBool, *unboxBool, *createTuple, *createDict, *createList, *createSlice,
-        *createUserClass, *createClosure, *createGenerator, *createSet, *initFrame, *deinitFrame;
+        *createUserClass, *createClosure, *createGenerator, *createSet, *initFrame, *deinitFrame, *makePendingCalls;
     llvm::Value* getattr, *getattr_capi, *setattr, *delattr, *delitem, *delGlobal, *nonzero, *binop, *compare,
         *augbinop, *unboxedLen, *getitem, *getitem_capi, *getclsattr, *getGlobal, *setitem, *unaryop, *import,
         *importFrom, *importStar, *repr, *exceptionMatches, *yield, *getiterHelper, *hasnext, *setGlobal, *apply_slice;

src/core/threading.cpp

@@ -330,9 +330,13 @@ static void* find_stack() {
     return NULL; /* not found =^P */
 }
 
+static long main_thread_id;
+
 void registerMainThread() {
     LOCK_REGION(&threading_lock);
 
+    main_thread_id = pthread_self();
+
     assert(!current_internal_thread_state);
     current_internal_thread_state = new ThreadStateInternal(find_stack(), pthread_self(), &cur_thread_state);
     current_threads[pthread_self()] = current_internal_thread_state;
@@ -345,6 +349,10 @@ void finishMainThread() {
     // TODO maybe this is the place to wait for non-daemon threads?
 }
 
+bool isMainThread() {
+    return pthread_self() == main_thread_id;
+}
+
 
 // For the "AllowThreads" regions, let's save the thread state at the beginning of the region.
 // This means that the thread won't get interrupted by the signals we would otherwise need to

src/core/threading.h

@@ -45,6 +45,8 @@ intptr_t start_thread(void* (*start_func)(Box*, Box*, Box*), Box* arg1, Box* arg
 void registerMainThread();
 void finishMainThread();
 
+bool isMainThread();
+
 // Some hooks to keep track of the list of stacks that this thread has been using.
 // Every time we switch to a new generator, we need to pass a reference to the generator
 // itself (so we can access the registers it is saving), the location of the new stack, and

src/runtime/capi.cpp

@@ -17,6 +17,7 @@
 #include <string.h>
 
 #include "Python.h"
+#include "pythread.h"
 
 #include "codegen/cpython_ast.h"
 #include "grammar.h"
@@ -1543,9 +1544,123 @@ extern "C" PyOS_sighandler_t PyOS_setsig(int sig, PyOS_sighandler_t handler) noe
 #endif
 }
 
+static PyThread_type_lock pending_lock = 0; /* for pending calls */
+
+/* The WITH_THREAD implementation is thread-safe.  It allows
+   scheduling to be made from any thread, and even from an executing
+   callback.
+ */
+
+#define NPENDINGCALLS 32
+static struct {
+    int (*func)(void*);
+    void* arg;
+} pendingcalls[NPENDINGCALLS];
+static int pendingfirst = 0;
+static int pendinglast = 0;
+// Pyston change
+// static volatile int pendingcalls_to_do = 1; /* trigger initialization of lock */
+extern "C" {
+volatile int _pendingcalls_to_do = 1;
+}
+static char pendingbusy = 0;
+
 extern "C" int Py_AddPendingCall(int (*func)(void*), void* arg) noexcept {
-    fatalOrError(PyExc_NotImplementedError, "unimplemented");
-    return -1;
+    int i, j, result = 0;
+    PyThread_type_lock lock = pending_lock;
+
+    /* try a few times for the lock.  Since this mechanism is used
+     * for signal handling (on the main thread), there is a (slim)
+     * chance that a signal is delivered on the same thread while we
+     * hold the lock during the Py_MakePendingCalls() function.
+     * This avoids a deadlock in that case.
+     * Note that signals can be delivered on any thread.  In particular,
+     * on Windows, a SIGINT is delivered on a system-created worker
+     * thread.
+     * We also check for lock being NULL, in the unlikely case that
+     * this function is called before any bytecode evaluation takes place.
+     */
+    if (lock != NULL) {
+        for (i = 0; i < 100; i++) {
+            if (PyThread_acquire_lock(lock, NOWAIT_LOCK))
+                break;
+        }
+        if (i == 100)
+            return -1;
+    }
+
+    i = pendinglast;
+    j = (i + 1) % NPENDINGCALLS;
+    if (j == pendingfirst) {
+        result = -1; /* Queue full */
+    } else {
+        pendingcalls[i].func = func;
+        pendingcalls[i].arg = arg;
+        pendinglast = j;
+    }
+    /* signal main loop */
+    // Pyston change: we don't have a _Py_Ticker
+    // _Py_Ticker = 0;
+
+    _pendingcalls_to_do = 1;
+    if (lock != NULL)
+        PyThread_release_lock(lock);
+    return result;
+}
+
+extern "C" int Py_MakePendingCalls(void) noexcept {
+    int i;
+    int r = 0;
+
+    if (!pending_lock) {
+        /* initial allocation of the lock */
+        pending_lock = PyThread_allocate_lock();
+        if (pending_lock == NULL)
+            return -1;
+    }
+
+    /* only service pending calls on main thread */
+    // Pyston change:
+    // if (main_thread && PyThread_get_thread_ident() != main_thread)
+    if (!threading::isMainThread())
+        return 0;
+    /* don't perform recursive pending calls */
+    if (pendingbusy)
+        return 0;
+    pendingbusy = 1;
+    /* perform a bounded number of calls, in case of recursion */
+    for (i = 0; i < NPENDINGCALLS; i++) {
+        int j;
+        int (*func)(void*);
+        void* arg = NULL;
+
+        /* pop one item off the queue while holding the lock */
+        PyThread_acquire_lock(pending_lock, WAIT_LOCK);
+        j = pendingfirst;
+        if (j == pendinglast) {
+            func = NULL; /* Queue empty */
+        } else {
+            func = pendingcalls[j].func;
+            arg = pendingcalls[j].arg;
+            pendingfirst = (j + 1) % NPENDINGCALLS;
+        }
+        _pendingcalls_to_do = pendingfirst != pendinglast;
+        PyThread_release_lock(pending_lock);
+        /* having released the lock, perform the callback */
+        if (func == NULL)
+            break;
+        r = func(arg);
+        if (r)
+            break;
+    }
+    pendingbusy = 0;
+    return r;
+}
+
+extern "C" void makePendingCalls() {
+    int ret = Py_MakePendingCalls();
+    if (ret != 0)
+        throwCAPIException();
 }
 
 extern "C" PyObject* _PyImport_FixupExtension(char* name, char* filename) noexcept {
@@ -1712,6 +1827,16 @@ extern "C" void PyEval_RestoreThread(PyThreadState* tstate) noexcept {
     endAllowThreads();
 }
 
+extern "C" struct _frame* PyEval_GetFrame(void) noexcept {
+    Box* frame = NULL;
+    try {
+        frame = getFrame(0);
+    } catch (ExcInfo) {
+        RELEASE_ASSERT(0, "untested");
+    }
+    return (struct _frame*)frame;
+}
+
 extern "C" char* PyModule_GetName(PyObject* m) noexcept {
     PyObject* d;
     PyObject* nameobj;

src/runtime/inline/link_forcer.cpp

@@ -74,6 +74,7 @@ void force() {
     FORCE(decodeUTF8StringPtr);
     FORCE(initFrame);
     FORCE(deinitFrame);
+    FORCE(makePendingCalls);
 
     FORCE(getattr);
     FORCE(getattr_capi);

src/runtime/types.h

@@ -162,6 +162,7 @@ extern "C" Box* createDict();
 extern "C" Box* createList();
 extern "C" Box* createSlice(Box* start, Box* stop, Box* step);
 extern "C" Box* createTuple(int64_t nelts, Box** elts);
+extern "C" void makePendingCalls();
 
 Box* objectStr(Box*);
 Box* objectRepr(Box*);
@@ -1312,6 +1313,8 @@ inline BoxedString* getStaticString(llvm::StringRef s) {
     return r;
 }
 
+extern "C" volatile int _pendingcalls_to_do;
+
 inline Box* Box::getattrString(const char* attr) {
     // XXX need to auto-decref
     BoxedString* s = internStringMortal(attr);

test/CPYTHON_TEST_NOTES.md

@@ -183,7 +183,6 @@ test_scope              eval of code object from existing function (not currentl
 test_scriptpackages     [unknown]
 test_shelve             [unknown]
 test_shlex              [unknown]
-test_signal             [unknown]
 test_site               [unknown]
 test_smtpnet            [unknown]
 test_socketserver       [unknown]

test/tests/signal_test.py

@@ -6,3 +6,23 @@ for k in sorted(dir(signal)):
     print k, getattr(signal, k)
 
 print hasattr(signal, "alarm")
+
+
+import time
+import signal
+
+def sig_handler(signum, stack):
+    print "inside sig_handler"
+    import sys, traceback
+    traceback.print_stack(stack)
+    sys.exit(0)
+
+def f(lst):
+    signal.signal(signal.SIGALRM, sig_handler)
+    signal.setitimer(signal.ITIMER_REAL, 2, 1)
+    for x in lst:
+        time.sleep(x)  #1
+        time.sleep(x)  #2
+
+f([0] * 100 + [10])
+assert False, "shuld not get executed"