Merge pull request #1832 from scoder/gen_exc_handling

Repair some issues with coroutine exception handling

Cython/Compiler/Code.pxd

@@ -33,6 +33,7 @@ cdef class FunctionState:
     cdef public object return_from_error_cleanup_label # not used in __init__ ?
 
     cdef public object exc_vars
+    cdef public object current_except
     cdef public bint in_try_finally
     cdef public bint can_trace
     cdef public bint gil_owned

Cython/Compiler/Code.py

@@ -6,10 +6,11 @@
 from __future__ import absolute_import
 
 import cython
-cython.declare(os=object, re=object, operator=object,
-               Naming=object, Options=object, StringEncoding=object,
+cython.declare(os=object, re=object, operator=object, textwrap=object,
+               Template=object, Naming=object, Options=object, StringEncoding=object,
                Utils=object, SourceDescriptor=object, StringIOTree=object,
-               DebugFlags=object, basestring=object)
+               DebugFlags=object, basestring=object, defaultdict=object,
+               closing=object, partial=object)
 
 import os
 import re
@@ -602,6 +603,7 @@ class FunctionState(object):
 
         self.in_try_finally = 0
         self.exc_vars = None
+        self.current_except = None
         self.can_trace = False
         self.gil_owned = True
 

Cython/Compiler/ExprNodes.py

@@ -9477,6 +9477,13 @@ class YieldExprNode(ExprNode):
                                   nogil=not code.funcstate.gil_owned)
         code.put_finish_refcount_context()
 
+        if code.funcstate.current_except is not None:
+            # inside of an except block => save away currently handled exception
+            code.putln("__Pyx_Coroutine_SwapException(%s);" % Naming.generator_cname)
+        else:
+            # no exceptions being handled => restore exception state of caller
+            code.putln("__Pyx_Coroutine_ResetAndClearException(%s);" % Naming.generator_cname)
+
         code.putln("/* return from %sgenerator, %sing value */" % (
             'async ' if self.in_async_gen else '',
             'await' if self.is_await else 'yield'))
@@ -9540,7 +9547,7 @@ class _YieldDelegationExprNode(YieldExprNode):
         code.put_gotref(self.result())
 
     def handle_iteration_exception(self, code):
-        code.putln("PyObject* exc_type = PyErr_Occurred();")
+        code.putln("PyObject* exc_type = __Pyx_PyErr_Occurred();")
         code.putln("if (exc_type) {")
         code.putln("if (likely(exc_type == PyExc_StopIteration || (exc_type != PyExc_GeneratorExit &&"
                    " __Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration)))) PyErr_Clear();")
@@ -9590,7 +9597,7 @@ class AwaitIterNextExprNode(AwaitExprNode):
 
     def _generate_break(self, code):
         code.globalstate.use_utility_code(UtilityCode.load_cached("StopAsyncIteration", "Coroutine.c"))
-        code.putln("PyObject* exc_type = PyErr_Occurred();")
+        code.putln("PyObject* exc_type = __Pyx_PyErr_Occurred();")
         code.putln("if (unlikely(exc_type && (exc_type == __Pyx_PyExc_StopAsyncIteration || ("
                    " exc_type != PyExc_StopIteration && exc_type != PyExc_GeneratorExit &&"
                    " __Pyx_PyErr_GivenExceptionMatches(exc_type, __Pyx_PyExc_StopAsyncIteration))))) {")

Cython/Compiler/Nodes.py

@@ -4054,9 +4054,10 @@ class GeneratorBodyDefNode(DefNode):
         self.declare_generator_body(env)
 
     def generate_function_header(self, code, proto=False):
-        header = "static PyObject *%s(__pyx_CoroutineObject *%s, PyObject *%s)" % (
+        header = "static PyObject *%s(__pyx_CoroutineObject *%s, CYTHON_UNUSED PyThreadState *%s, PyObject *%s)" % (
             self.entry.func_cname,
             Naming.generator_cname,
+            Naming.local_tstate_cname,
             Naming.sent_value_cname)
         if proto:
             code.putln('%s; /* proto */' % header)
@@ -4157,6 +4158,7 @@ class GeneratorBodyDefNode(DefNode):
             code.put_xgiveref(Naming.retval_cname)
         else:
             code.put_xdecref_clear(Naming.retval_cname, py_object_type)
+        code.putln("__Pyx_Coroutine_ResetAndClearException(%s);" % Naming.generator_cname)
         code.putln('%s->resume_label = -1;' % Naming.generator_cname)
         # clean up as early as possible to help breaking any reference cycles
         code.putln('__Pyx_Coroutine_clear((PyObject*)%s);' % Naming.generator_cname)
@@ -6696,6 +6698,7 @@ class TryExceptStatNode(StatNode):
     #  else_clause      StatNode or None
 
     child_attrs = ["body", "except_clauses", "else_clause"]
+    in_generator = False
 
     def analyse_declarations(self, env):
         self.body.analyse_declarations(env)
@@ -6755,8 +6758,9 @@ class TryExceptStatNode(StatNode):
         if can_raise:
             # inject code before the try block to save away the exception state
             code.globalstate.use_utility_code(reset_exception_utility_code)
-            save_exc.putln("__Pyx_PyThreadState_declare")
-            save_exc.putln("__Pyx_PyThreadState_assign")
+            if not self.in_generator:
+                save_exc.putln("__Pyx_PyThreadState_declare")
+                save_exc.putln("__Pyx_PyThreadState_assign")
             save_exc.putln("__Pyx_ExceptionSave(%s);" % (
                 ', '.join(['&%s' % var for var in exc_save_vars])))
             for var in exc_save_vars:
@@ -6794,11 +6798,16 @@ class TryExceptStatNode(StatNode):
                     code.put_xdecref_clear(var, py_object_type)
                 code.put_goto(try_end_label)
             code.put_label(our_error_label)
-            code.putln("__Pyx_PyThreadState_assign")  # re-assign in case a generator yielded
             for temp_name, temp_type in temps_to_clean_up:
                 code.put_xdecref_clear(temp_name, temp_type)
+
+            outer_except = code.funcstate.current_except
+            # Currently points to self, but the ExceptClauseNode would also be ok. Change if needed.
+            code.funcstate.current_except = self
             for except_clause in self.except_clauses:
                 except_clause.generate_handling_code(code, except_end_label)
+            code.funcstate.current_except = outer_except
+
             if not self.has_default_clause:
                 code.put_goto(except_error_label)
 
@@ -6813,7 +6822,6 @@ class TryExceptStatNode(StatNode):
                 code.put_label(exit_label)
                 code.mark_pos(self.pos, trace=False)
                 if can_raise:
-                    code.putln("__Pyx_PyThreadState_assign")  # re-assign in case a generator yielded
                     restore_saved_exception()
                 code.put_goto(old_label)
 
@@ -6822,7 +6830,6 @@ class TryExceptStatNode(StatNode):
                 code.put_goto(try_end_label)
             code.put_label(except_end_label)
             if can_raise:
-                code.putln("__Pyx_PyThreadState_assign")  # re-assign in case a generator yielded
                 restore_saved_exception()
         if code.label_used(try_end_label):
             code.put_label(try_end_label)
@@ -6939,8 +6946,8 @@ class ExceptClauseNode(Node):
         exc_args = "&%s, &%s, &%s" % tuple(exc_vars)
         code.putln("if (__Pyx_GetException(%s) < 0) %s" % (
             exc_args, code.error_goto(self.pos)))
-        for x in exc_vars:
-            code.put_gotref(x)
+        for var in exc_vars:
+            code.put_gotref(var)
         if self.target:
             self.exc_value.set_var(exc_vars[1])
             self.exc_value.generate_evaluation_code(code)
@@ -6957,6 +6964,7 @@ class ExceptClauseNode(Node):
         code.funcstate.exc_vars = exc_vars
         self.body.generate_execution_code(code)
         code.funcstate.exc_vars = old_exc_vars
+
         if not self.body.is_terminator:
             for var in exc_vars:
                 code.put_decref_clear(var, py_object_type)
@@ -7086,7 +7094,8 @@ class TryFinallyStatNode(StatNode):
 
         if preserve_error:
             code.putln('/*exception exit:*/{')
-            code.putln("__Pyx_PyThreadState_declare")
+            if not self.in_generator:
+                code.putln("__Pyx_PyThreadState_declare")
             if self.is_try_finally_in_nogil:
                 code.declare_gilstate()
             if needs_success_cleanup:
@@ -7148,7 +7157,6 @@ class TryFinallyStatNode(StatNode):
             if old_label == return_label:
                 # return actually raises an (uncatchable) exception in generators that we must preserve
                 if self.in_generator:
-                    code.putln("__Pyx_PyThreadState_declare")
                     exc_vars = tuple([
                         code.funcstate.allocate_temp(py_object_type, manage_ref=False)
                         for _ in range(6)])
@@ -7229,8 +7237,6 @@ class TryFinallyStatNode(StatNode):
 
         if self.is_try_finally_in_nogil:
             code.put_ensure_gil(declare_gilstate=False)
-        if self.in_generator:
-            code.putln("__Pyx_PyThreadState_assign")  # re-assign in case a generator yielded
 
         # not using preprocessor here to avoid warnings about
         # unused utility functions and/or temps
@@ -7257,8 +7263,6 @@ class TryFinallyStatNode(StatNode):
         code.globalstate.use_utility_code(reset_exception_utility_code)
         if self.is_try_finally_in_nogil:
             code.put_ensure_gil(declare_gilstate=False)
-        if self.in_generator:
-            code.putln("__Pyx_PyThreadState_assign")  # re-assign in case a generator yielded
 
         # not using preprocessor here to avoid warnings about
         # unused utility functions and/or temps

Cython/Compiler/ParseTreeTransforms.pxd

@@ -54,6 +54,7 @@ cdef class YieldNodeCollector(TreeVisitor):
     cdef public list yields
     cdef public list returns
     cdef public list finallys
+    cdef public list excepts
     cdef public bint has_return_value
     cdef public bint has_yield
     cdef public bint has_await

Cython/Compiler/ParseTreeTransforms.py

@@ -2475,6 +2475,7 @@ class YieldNodeCollector(TreeVisitor):
         self.yields = []
         self.returns = []
         self.finallys = []
+        self.excepts = []
         self.has_return_value = False
         self.has_yield = False
         self.has_await = False
@@ -2502,6 +2503,10 @@ class YieldNodeCollector(TreeVisitor):
         self.visitchildren(node)
         self.finallys.append(node)
 
+    def visit_TryExceptStatNode(self, node):
+        self.visitchildren(node)
+        self.excepts.append(node)
+
     def visit_ClassDefNode(self, node):
         pass
 
@@ -2552,7 +2557,7 @@ class MarkClosureVisitor(CythonTransform):
 
         for i, yield_expr in enumerate(collector.yields, 1):
             yield_expr.label_num = i
-        for retnode in collector.returns + collector.finallys:
+        for retnode in collector.returns + collector.finallys + collector.excepts:
             retnode.in_generator = True
 
         gbody = Nodes.GeneratorBodyDefNode(
@@ -2665,6 +2670,9 @@ class CreateClosureClasses(CythonTransform):
         class_scope = entry.type.scope
         class_scope.is_internal = True
         class_scope.is_closure_class_scope = True
+        if node.is_async_def or node.is_generator:
+            # Generators need their closure intact during cleanup as they resume to handle GeneratorExit
+            class_scope.directives['no_gc_clear'] = True
         if Options.closure_freelist_size:
             class_scope.directives['freelist'] = Options.closure_freelist_size
 

Cython/Utility/Coroutine.c

@@ -355,8 +355,9 @@ static void __Pyx_Generator_Replace_StopIteration(CYTHON_UNUSED int in_async_gen
 
 
 //////////////////// CoroutineBase.proto ////////////////////
+//@substitute: naming
 
-typedef PyObject *(*__pyx_coroutine_body_t)(PyObject *, PyObject *);
+typedef PyObject *(*__pyx_coroutine_body_t)(PyObject *, PyThreadState *, PyObject *);
 
 typedef struct {
     PyObject_HEAD
@@ -389,11 +390,25 @@ static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value); /*proto*
 static PyObject *__Pyx_Coroutine_Close(PyObject *self); /*proto*/
 static PyObject *__Pyx_Coroutine_Throw(PyObject *gen, PyObject *args); /*proto*/
 
-#if 1 || PY_VERSION_HEX < 0x030300B0
-static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue); /*proto*/
+// macros for exception state swapping instead of inline functions to make use of the local thread state context
+#define __Pyx_Coroutine_SwapException(self) { \
+    __Pyx_ExceptionSwap(&(self)->exc_type, &(self)->exc_value, &(self)->exc_traceback); \
+    __Pyx_Coroutine_ResetFrameBackpointer(self); \
+    }
+#define __Pyx_Coroutine_ResetAndClearException(self) { \
+    __Pyx_ExceptionReset((self)->exc_type, (self)->exc_value, (self)->exc_traceback); \
+    (self)->exc_type = (self)->exc_value = (self)->exc_traceback = NULL; \
+    }
+
+#if CYTHON_FAST_THREAD_STATE
+#define __Pyx_PyGen_FetchStopIterationValue(pvalue) \
+    __Pyx_PyGen__FetchStopIterationValue($local_tstate_cname, pvalue)
 #else
-#define __Pyx_PyGen_FetchStopIterationValue(pvalue) PyGen_FetchStopIterationValue(pvalue)
+#define __Pyx_PyGen_FetchStopIterationValue(pvalue) \
+    __Pyx_PyGen__FetchStopIterationValue(__Pyx_PyThreadState_Current, pvalue)
 #endif
+static int __Pyx_PyGen__FetchStopIterationValue(PyThreadState *tstate, PyObject **pvalue); /*proto*/
+static CYTHON_INLINE void __Pyx_Coroutine_ResetFrameBackpointer(__pyx_CoroutineObject *self); /*proto*/
 
 
 //////////////////// Coroutine.proto ////////////////////
@@ -443,6 +458,7 @@ static int __pyx_Generator_init(void); /*proto*/
 //@requires: Exceptions.c::PyThreadStateGet
 //@requires: Exceptions.c::SwapException
 //@requires: Exceptions.c::RaiseException
+//@requires: Exceptions.c::SaveResetException
 //@requires: ObjectHandling.c::PyObjectCallMethod1
 //@requires: ObjectHandling.c::PyObjectGetAttrStr
 //@requires: CommonStructures.c::FetchCommonType
@@ -458,12 +474,9 @@ static int __pyx_Generator_init(void); /*proto*/
 //   Returns 0 if no exception or StopIteration is set.
 //   If any other exception is set, returns -1 and leaves
 //   pvalue unchanged.
-#if 1 || PY_VERSION_HEX < 0x030300B0
-static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue) {
+static int __Pyx_PyGen__FetchStopIterationValue(CYTHON_UNUSED PyThreadState *$local_tstate_cname, PyObject **pvalue) {
     PyObject *et, *ev, *tb;
     PyObject *value = NULL;
-    __Pyx_PyThreadState_declare
-    __Pyx_PyThreadState_assign
 
     __Pyx_ErrFetch(&et, &ev, &tb);
 
@@ -550,7 +563,6 @@ static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue) {
     *pvalue = value;
     return 0;
 }
-#endif
 
 static CYTHON_INLINE
 void __Pyx_Coroutine_ExceptionClear(__pyx_CoroutineObject *self) {
@@ -627,8 +639,9 @@ static void __Pyx__Coroutine_AlreadyTerminatedError(CYTHON_UNUSED PyObject *gen,
 
 static
 PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value, int closing) {
-    PyObject *retval;
     __Pyx_PyThreadState_declare
+    PyThreadState *tstate;
+    PyObject *retval;
 
     assert(!self->is_running);
 
@@ -642,8 +655,21 @@ PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value, i
         return __Pyx_Coroutine_AlreadyTerminatedError((PyObject*)self, value, closing);
     }
 
+#if CYTHON_FAST_THREAD_STATE
     __Pyx_PyThreadState_assign
-    if (value) {
+    tstate = $local_tstate_cname;
+#else
+    tstate = __Pyx_PyThreadState_Current;
+#endif
+
+    // Traceback/Frame rules:
+    // - on entry, save external exception state in self->exc_*, restore it on exit
+    // - on exit, keep internally generated exceptions in self->exc_*, clear everything else
+    // - on entry, set "f_back" pointer of internal exception traceback to (current) outer call frame
+    // - on exit, clear "f_back" of internal exception traceback
+    // - do not touch external frames and tracebacks
+
+    if (self->exc_type) {
 #if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON
         // FIXME: what to do in PyPy?
 #else
@@ -653,41 +679,42 @@ PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value, i
             PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback;
             PyFrameObject *f = tb->tb_frame;
 
-            Py_XINCREF($local_tstate_cname->frame);
+            Py_XINCREF(tstate->frame);
             assert(f->f_back == NULL);
-            f->f_back = $local_tstate_cname->frame;
+            f->f_back = tstate->frame;
         }
 #endif
+        // We were in an except handler when we left,
+        // restore the exception state which was put aside.
         __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value,
                             &self->exc_traceback);
+        // self->exc_* now holds the exception state of the caller
     } else {
+        // save away the exception state of the caller
         __Pyx_Coroutine_ExceptionClear(self);
+        __Pyx_ExceptionSave(&self->exc_type, &self->exc_value, &self->exc_traceback);
     }
 
     self->is_running = 1;
-    retval = self->body((PyObject *) self, value);
+    retval = self->body((PyObject *) self, tstate, value);
     self->is_running = 0;
 
-    if (retval) {
-        __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value,
-                            &self->exc_traceback);
+    return retval;
+}
+
+static CYTHON_INLINE void __Pyx_Coroutine_ResetFrameBackpointer(__pyx_CoroutineObject *self) {
+    // Don't keep the reference to f_back any longer than necessary.  It
+    // may keep a chain of frames alive or it could create a reference
+    // cycle.
+    if (likely(self->exc_traceback)) {
 #if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON
-        // FIXME: what to do in PyPy?
+    // FIXME: what to do in PyPy?
 #else
-        // Don't keep the reference to f_back any longer than necessary.  It
-        // may keep a chain of frames alive or it could create a reference
-        // cycle.
-        if (self->exc_traceback) {
-            PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback;
-            PyFrameObject *f = tb->tb_frame;
-            Py_CLEAR(f->f_back);
-        }
+        PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback;
+        PyFrameObject *f = tb->tb_frame;
+        Py_CLEAR(f->f_back);
 #endif
-    } else {
-        __Pyx_Coroutine_ExceptionClear(self);
     }
-
-    return retval;
 }
 
 static CYTHON_INLINE
@@ -709,7 +736,7 @@ PyObject *__Pyx_Coroutine_FinishDelegation(__pyx_CoroutineObject *gen) {
     PyObject *ret;
     PyObject *val = NULL;
     __Pyx_Coroutine_Undelegate(gen);
-    __Pyx_PyGen_FetchStopIterationValue(&val);
+    __Pyx_PyGen__FetchStopIterationValue(__Pyx_PyThreadState_Current, &val);
     // val == NULL on failure => pass on exception
     ret = __Pyx_Coroutine_SendEx(gen, val, 0);
     Py_XDECREF(val);
@@ -876,10 +903,10 @@ static PyObject *__Pyx_Coroutine_Close(PyObject *self) {
     if (err == 0)
         PyErr_SetNone(PyExc_GeneratorExit);
     retval = __Pyx_Coroutine_SendEx(gen, NULL, 1);
-    if (retval) {
+    if (unlikely(retval)) {
         const char *msg;
         Py_DECREF(retval);
-        if (0) {
+        if ((0)) {
         #ifdef __Pyx_Coroutine_USED
         } else if (__Pyx_Coroutine_CheckExact(self)) {
             msg = "coroutine ignored GeneratorExit";
@@ -899,7 +926,7 @@ static PyObject *__Pyx_Coroutine_Close(PyObject *self) {
         return NULL;
     }
     raised_exception = PyErr_Occurred();
-    if (!raised_exception || __Pyx_PyErr_GivenExceptionMatches2(raised_exception, PyExc_GeneratorExit, PyExc_StopIteration)) {
+    if (likely(!raised_exception || __Pyx_PyErr_GivenExceptionMatches2(raised_exception, PyExc_GeneratorExit, PyExc_StopIteration))) {
         // ignore these errors
         if (raised_exception) PyErr_Clear();
         Py_INCREF(Py_None);

Cython/Utility/Exceptions.c

@@ -11,6 +11,7 @@
 
 #if CYTHON_FAST_THREAD_STATE
 #define __Pyx_PyThreadState_declare  PyThreadState *$local_tstate_cname;
+#define __Pyx_PyErr_Occurred()  $local_tstate_cname->curexc_type
 #if PY_VERSION_HEX >= 0x03050000
   #define __Pyx_PyThreadState_assign  $local_tstate_cname = _PyThreadState_UncheckedGet();
 #elif PY_VERSION_HEX >= 0x03000000
@@ -23,6 +24,7 @@
 #else
 #define __Pyx_PyThreadState_declare
 #define __Pyx_PyThreadState_assign
+#define __Pyx_PyErr_Occurred()  PyErr_Occurred()
 #endif
 
 

Tools/cevaltrace.py

+#!/usr/bin/env python3
+
+"""
+Translate the byte code of a Python function into the corresponding
+sequences of C code in CPython's "ceval.c".
+"""
+
+from __future__ import print_function, absolute_import
+
+import re
+import os.path
+
+from dis import get_instructions  # requires Python 3.4+
+
+# collapse some really boring byte codes
+_COLLAPSE = {'NOP', 'LOAD_CONST', 'POP_TOP', 'JUMP_FORWARD'}
+#_COLLAPSE.clear()
+
+_is_start = re.compile(r"\s* switch \s* \( opcode \)", re.VERBOSE).match
+# Py3: TARGET(XX), Py2: case XX
+_match_target = re.compile(r"\s* (?: TARGET \s* \( | case \s* ) \s* (\w+) \s* [:)]", re.VERBOSE).match
+_ignored = re.compile(r"\s* PREDICTED[A-Z_]*\(", re.VERBOSE).match
+_is_end = re.compile(r"\s* } \s* /\* \s* switch \s* \*/", re.VERBOSE).match
+
+_find_pyversion = re.compile(r'\#define \s+ PY_VERSION \s+ "([^"]+)"', re.VERBOSE).findall
+
+class ParseError(Exception):
+    def __init__(self, message="Failed to parse ceval.c"):
+        super(ParseError, self).__init__(message)
+
+
+def parse_ceval(file_path):
+    snippets = {}
+    with open(file_path) as f:
+        lines = iter(f)
+
+        for line in lines:
+            if _is_start(line):
+                break
+        else:
+            raise ParseError()
+
+        targets = []
+        code_lines = []
+        for line in lines:
+            target_match = _match_target(line)
+            if target_match:
+                if code_lines:
+                    code = ''.join(code_lines).rstrip()
+                    for target in targets:
+                        snippets[target] = code
+                    del code_lines[:], targets[:]
+                targets.append(target_match.group(1))
+            elif _ignored(line):
+                pass
+            elif _is_end(line):
+                break
+            else:
+                code_lines.append(line)
+        else:
+            if not snippets:
+                raise ParseError()
+    return snippets
+
+
+def translate(func, ceval_snippets):
+    start_offset = 0
+    code_obj = getattr(func, '__code__', None)
+    if code_obj and os.path.exists(code_obj.co_filename):
+        start_offset = code_obj.co_firstlineno
+        with open(code_obj.co_filename) as f:
+            code_line_at = {
+                i: line.strip()
+                for i, line in enumerate(f, 1)
+                if line.strip()
+            }.get
+    else:
+        code_line_at = lambda _: None
+
+    for instr in get_instructions(func):
+        code_line = code_line_at(instr.starts_line)
+        line_no = (instr.starts_line or start_offset) - start_offset
+        yield line_no, code_line, instr, ceval_snippets.get(instr.opname)
+
+
+def main():
+    import sys
+    import importlib.util
+
+    if len(sys.argv) < 3:
+        print("Usage:  %s  path/to/Python/ceval.c  script.py ..." % sys.argv[0], file=sys.stderr)
+        return
+
+    ceval_source_file = sys.argv[1]
+    version_header = os.path.join(os.path.dirname(ceval_source_file), '..', 'Include', 'patchlevel.h')
+    if os.path.exists(version_header):
+        with open(version_header) as f:
+            py_version = _find_pyversion(f.read())
+        if py_version:
+            py_version = py_version[0]
+            if not sys.version.startswith(py_version + ' '):
+                print("Warning:  disassembling with Python %s, but ceval.c has version %s" % (
+                    sys.version.split(None, 1)[0],
+                    py_version,
+                ), file=sys.stderr)
+
+    snippets = parse_ceval(ceval_source_file)
+
+    for code in _COLLAPSE:
+        if code in snippets:
+            snippets[code] = ''
+
+    for file_path in sys.argv[2:]:
+        module_name = os.path.basename(file_path)
+        print("/*######## MODULE %s ########*/" % module_name)
+        print('')
+
+        spec = importlib.util.spec_from_file_location(module_name, file_path)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+
+        for func_name, item in sorted(vars(module).items()):
+            if not callable(item):
+                continue
+            print("/* FUNCTION %s */" % func_name)
+            print("static void")  # assuming that it highlights in editors
+            print("%s() {" % func_name)
+
+            last_line = None
+            for line_no, code_line, instr, snippet in translate(item, snippets):
+                if last_line != line_no:
+                    if code_line:
+                        print('')
+                        print('/*# %3d  %s */' % (line_no, code_line))
+                        print('')
+                    last_line = line_no
+
+                print("  %s:%s {%s" % (
+                    instr.opname,
+                    ' /* %s */' % instr.argrepr if instr.arg is not None else '',
+                    ' /* ??? */' if snippet is None else ' /* ... */ }' if snippet == '' else '',
+                ))
+                print(snippet or '')
+
+            print("} /* FUNCTION %s */" % func_name)
+
+
+if __name__ == '__main__':
+    main()

runtests.py

@@ -2164,6 +2164,9 @@ def runtests(options, cmd_args, coverage=None):
         pyximport.install(pyimport=True, build_dir=os.path.join(WORKDIR, '_pyximport'),
                           load_py_module_on_import_failure=True, inplace=True)
 
+    import gc
+    gc.set_debug(gc.DEBUG_UNCOLLECTABLE)
+
     result = test_runner.run(test_suite)
 
     if common_utility_dir and options.shard_num < 0 and options.cleanup_workdir:

tests/run/generator_frame_cycle.py

 # mode: run
 # tag: generator
 
+import cython
 import sys
 
-def _next(it):
-    if sys.version_info[0] >= 3:
-        return next(it)
-    else:
-        return it.next()
 
 def test_generator_frame_cycle():
     """
@@ -23,8 +19,42 @@ def test_generator_frame_cycle():
         finally:
             testit.append("I'm done")
     g = whoo()
-    _next(g)
+    next(g)
+
     # Frame object cycle
     eval('g.throw(ValueError)', {'g': g})
     del g
+
+    return tuple(testit)
+
+
+def test_generator_frame_cycle_with_outer_exc():
+    """
+    >>> test_generator_frame_cycle_with_outer_exc()
+    ("I'm done",)
+    """
+    testit = []
+    def whoo():
+        try:
+            yield
+        except:
+            yield
+        finally:
+            testit.append("I'm done")
+    g = whoo()
+    next(g)
+
+    try:
+        raise ValueError()
+    except ValueError as exc:
+        assert sys.exc_info()[1] is exc, sys.exc_info()
+        # Frame object cycle
+        eval('g.throw(ValueError)', {'g': g})
+        # CPython 3.3 handles this incorrectly itself :)
+        if cython.compiled or sys.version_info[:2] not in [(3, 2), (3, 3)]:
+            assert sys.exc_info()[1] is exc, sys.exc_info()
+        del g
+        if cython.compiled or sys.version_info[:2] not in [(3, 2), (3, 3)]:
+            assert sys.exc_info()[1] is exc, sys.exc_info()
+
     return tuple(testit)

tests/run/generators_GH1731.pyx

+# mode: run
+# ticket: gh1731
+
+
+def cygen():
+    yield 1
+
+
+def test_from_cython(g):
+    """
+    >>> def pygen(): yield 1
+    >>> test_from_cython(pygen)
+    Traceback (most recent call last):
+    ZeroDivisionError: integer division or modulo by zero
+
+    >>> test_from_cython(cygen)
+    Traceback (most recent call last):
+    ZeroDivisionError: integer division or modulo by zero
+    """
+    try:
+        1 / 0
+    except:
+        for _ in g():
+            pass
+        raise
+
+
+def test_from_python():
+    """
+    >>> def test(g):
+    ...     try:
+    ...         1 / 0
+    ...     except:
+    ...         for _ in g():
+    ...             pass
+    ...         raise
+
+    >>> def pygen():
+    ...     yield 1
+    >>> test(pygen)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: ...division ...by zero
+
+    >>> test(cygen)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: ...division ...by zero
+    """
+
+
+def test_from_console():
+    """
+    >>> def pygen(): yield 1
+    >>> try:  # doctest: +ELLIPSIS
+    ...     1 / 0
+    ... except:
+    ...     for _ in pygen():
+    ...         pass
+    ...     raise
+    Traceback (most recent call last):
+    ZeroDivisionError: ...division ...by zero
+
+    >>> try:  # doctest: +ELLIPSIS
+    ...     1 / 0
+    ... except:
+    ...     for _ in cygen():
+    ...         pass
+    ...     raise
+    Traceback (most recent call last):
+    ZeroDivisionError: ...division ...by zero
+    """

tests/run/generators_py.py

 # mode: run
 # tag: generators
 
+import sys
 import cython
 
 
@@ -147,25 +148,39 @@ def check_throw():
         except ValueError:
             pass
 
+
 def check_yield_in_except():
     """
-    >>> import sys
-    >>> orig_exc = sys.exc_info()[0]
-    >>> g = check_yield_in_except()
-    >>> next(g)
-    >>> next(g)
-    >>> orig_exc is sys.exc_info()[0] or sys.exc_info()[0]
+    >>> if sys.version_info[0] == 2: sys.exc_clear()
+    >>> try:
+    ...     raise TypeError("RAISED !")
+    ... except TypeError as orig_exc:
+    ...     assert isinstance(orig_exc, TypeError), orig_exc
+    ...     g = check_yield_in_except()
+    ...     print(orig_exc is sys.exc_info()[1] or sys.exc_info())
+    ...     next(g)
+    ...     print(orig_exc is sys.exc_info()[1] or sys.exc_info())
+    ...     next(g)
+    ...     print(orig_exc is sys.exc_info()[1] or sys.exc_info())
     True
+    True
+    True
+    >>> next(g)
+    Traceback (most recent call last):
+    StopIteration
     """
     try:
         yield
         raise ValueError
-    except ValueError:
+    except ValueError as exc:
+        assert sys.exc_info()[1] is exc, sys.exc_info()
         yield
+        if cython.compiled or sys.version_info[0] > 2:
+            assert sys.exc_info()[1] is exc, sys.exc_info()
+
 
 def yield_in_except_throw_exc_type():
     """
-    >>> import sys
     >>> g = yield_in_except_throw_exc_type()
     >>> next(g)
     >>> g.throw(TypeError)
@@ -177,12 +192,14 @@ def yield_in_except_throw_exc_type():
     """
     try:
         raise ValueError
-    except ValueError:
+    except ValueError as exc:
+        assert sys.exc_info()[1] is exc, sys.exc_info()
         yield
+        assert sys.exc_info()[1] is exc, sys.exc_info()
+
 
 def yield_in_except_throw_instance():
     """
-    >>> import sys
     >>> g = yield_in_except_throw_instance()
     >>> next(g)
     >>> g.throw(TypeError())
@@ -194,8 +211,11 @@ def yield_in_except_throw_instance():
     """
     try:
         raise ValueError
-    except ValueError:
+    except ValueError as exc:
+        assert sys.exc_info()[1] is exc, sys.exc_info()
         yield
+        assert sys.exc_info()[1] is exc, sys.exc_info()
+
 
 def test_swap_assignment():
     """