Merge remote branch 'markflorisson/withgil'

Conflicts:
	tests/errors/nogil.pyx

Cython/Compiler/Code.pxd

@@ -35,6 +35,7 @@ cdef class FunctionState:
     cdef public size_t temp_counter
 
     cdef public object closure_temps
+    cdef public bint should_declare_error_indicator
 
     @cython.locals(n=size_t)
     cpdef new_label(self, name=*)

Cython/Compiler/Code.py

@@ -134,6 +134,12 @@ class FunctionState(object):
         self.temp_counter = 0
         self.closure_temps = None
 
+        # This is used for the error indicator, which needs to be local to the
+        # function. It used to be global, which relies on the GIL being held.
+        # However, exceptions may need to be propagated through 'nogil'
+        # sections, in which case we introduce a race condition.
+        self.should_declare_error_indicator = False
+
     # labels
 
     def new_label(self, name=None):
@@ -1362,6 +1368,52 @@ class CCodeWriter(object):
                     doc_code,
                     term))
 
+    # GIL methods
+
+    def put_ensure_gil(self, declare_gilstate=True):
+        """
+        Acquire the GIL. The generated code is safe even when no PyThreadState
+        has been allocated for this thread (for threads not initialized by
+        using the Python API). Additionally, the code generated by this method
+        may be called recursively.
+        """
+        from Cython.Compiler import Nodes
+
+        self.globalstate.use_utility_code(Nodes.force_init_threads_utility_code)
+
+        self.putln("#ifdef WITH_THREAD")
+        if declare_gilstate:
+            self.put("PyGILState_STATE ")
+        self.putln("_save = PyGILState_Ensure();")
+        self.putln("#endif")
+
+    def put_release_ensured_gil(self):
+        """
+        Releases the GIL, corresponds to `put_ensure_gil`.
+        """
+        self.putln("#ifdef WITH_THREAD")
+        self.putln("PyGILState_Release(_save);")
+        self.putln("#endif")
+
+    def put_acquire_gil(self):
+        """
+        Acquire the GIL. The thread's thread state must have been initialized
+        by a previous `put_release_gil`
+        """
+        self.putln("Py_BLOCK_THREADS")
+
+    def put_release_gil(self):
+        "Release the GIL, corresponds to `put_acquire_gil`."
+        self.putln("#ifdef WITH_THREAD")
+        self.putln("PyThreadState *_save = NULL;")
+        self.putln("#endif")
+        self.putln("Py_UNBLOCK_THREADS")
+
+    def declare_gilstate(self):
+        self.putln("#ifdef WITH_THREAD")
+        self.putln("PyGILState_STATE _save;")
+        self.putln("#endif")
+
     # error handling
 
     def put_error_if_neg(self, pos, value):
@@ -1369,10 +1421,12 @@ class CCodeWriter(object):
         return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos)))
 
     def set_error_info(self, pos):
+        self.funcstate.should_declare_error_indicator = True
         if self.c_line_in_traceback:
             cinfo = " %s = %s;" % (Naming.clineno_cname, Naming.line_c_macro)
         else:
             cinfo = ""
+
         return "%s = %s[%s]; %s = %s;%s" % (
             Naming.filename_cname,
             Naming.filetable_cname,
@@ -1412,6 +1466,20 @@ class CCodeWriter(object):
     def put_finish_refcount_context(self):
         self.putln("__Pyx_RefNannyFinishContext();")
 
+    def put_add_traceback(self, qualified_name):
+        """
+        Build a Python traceback for propagating exceptions.
+
+        qualified_name should be the qualified name of the function
+        """
+        format_tuple = (
+            qualified_name,
+            Naming.clineno_cname,
+            Naming.lineno_cname,
+            Naming.filename_cname,
+        )
+        self.putln('__Pyx_AddTraceback("%s", %s, %s, %s);' % format_tuple)
+
     def put_trace_declarations(self):
         self.putln('__Pyx_TraceDeclarations');
 

Cython/Compiler/ExprNodes.py

@@ -6308,7 +6308,12 @@ class DivNode(NumBinopNode):
                                 self.operand1.result(),
                                 self.operand2.result()))
                 code.putln(code.set_error_info(self.pos));
-                code.put("if (__Pyx_cdivision_warning()) ")
+                code.put("if (__Pyx_cdivision_warning(%(FILENAME)s, "
+                                                     "%(LINENO)s)) " % {
+                    'FILENAME': Naming.filename_cname,
+                    'LINENO':  Naming.lineno_cname,
+                    })
+
                 code.put_goto(code.error_label)
                 code.putln("}")
 
@@ -8643,21 +8648,18 @@ static CYTHON_INLINE %(type)s __Pyx_mod_%(type_name)s(%(type)s a, %(type)s b) {
 
 cdivision_warning_utility_code = UtilityCode(
 proto="""
-static int __Pyx_cdivision_warning(void); /* proto */
+static int __Pyx_cdivision_warning(const char *, int); /* proto */
 """,
 impl="""
-static int __Pyx_cdivision_warning(void) {
+static int __Pyx_cdivision_warning(const char *filename, int lineno) {
     return PyErr_WarnExplicit(PyExc_RuntimeWarning,
                               "division with oppositely signed operands, C and Python semantics differ",
-                              %(FILENAME)s,
-                              %(LINENO)s,
+                              filename,
+                              lineno,
                               __Pyx_MODULE_NAME,
                               NULL);
 }
-""" % {
-    'FILENAME': Naming.filename_cname,
-    'LINENO':  Naming.lineno_cname,
-})
+""")
 
 # from intobject.c
 division_overflow_test_code = UtilityCode(

Cython/Compiler/ModuleNode.py

@@ -1802,7 +1802,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         if code.label_used(code.error_label):
             code.put_label(code.error_label)
             # This helps locate the offending name.
-            code.putln('__Pyx_AddTraceback("%s");' % self.full_module_name);
+            code.put_add_traceback(self.full_module_name)
         code.error_label = old_error_label
         code.putln("bad:")
         code.putln("return -1;")
@@ -1919,7 +1919,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         for cname, type in code.funcstate.all_managed_temps():
             code.put_xdecref(cname, type)
         code.putln('if (%s) {' % env.module_cname)
-        code.putln('__Pyx_AddTraceback("init %s");' % env.qualified_name)
+        code.put_add_traceback("init %s" % env.qualified_name)
         env.use_utility_code(Nodes.traceback_utility_code)
         code.put_decref_clear(env.module_cname, py_object_type, nanny=False)
         code.putln('} else if (!PyErr_Occurred()) {')

Cython/Compiler/Nodes.py

@@ -1321,9 +1321,11 @@ class FuncDefNode(StatNode, BlockNode):
             code.put(cenv.scope_class.type.declaration_code(Naming.outer_scope_cname))
             code.putln(";")
         self.generate_argument_declarations(lenv, code)
+
         for entry in lenv.var_entries:
             if not entry.in_closure:
                 code.put_var_declaration(entry)
+
         init = ""
         if not self.return_type.is_void:
             if self.return_type.is_pyobject:
@@ -1333,23 +1335,31 @@ class FuncDefNode(StatNode, BlockNode):
                     (self.return_type.declaration_code(Naming.retval_cname),
                      init))
         tempvardecl_code = code.insertion_point()
-        if not lenv.nogil:
-            code.put_declare_refcount_context()
         self.generate_keyword_list(code)
+
         if profile:
             code.put_trace_declarations()
+
         # ----- Extern library function declarations
         lenv.generate_library_function_declarations(code)
+
         # ----- GIL acquisition
         acquire_gil = self.acquire_gil
-        if acquire_gil:
-            env.use_utility_code(force_init_threads_utility_code)
-            code.putln("#ifdef WITH_THREAD")
-            code.putln("PyGILState_STATE _save = PyGILState_Ensure();")
-            code.putln("#endif")
+
+        # See if we need to acquire the GIL for variable declarations and
+        acquire_gil_for_var_decls_only = (lenv.nogil and
+                                          lenv.has_with_gil_block)
+
+        use_refnanny = not lenv.nogil or acquire_gil_for_var_decls_only
+
+        if acquire_gil or acquire_gil_for_var_decls_only:
+            code.put_ensure_gil()
+
         # ----- set up refnanny
-        if not lenv.nogil:
+        if use_refnanny:
+            tempvardecl_code.put_declare_refcount_context()
             code.put_setup_refcount_context(self.entry.name)
+
         # ----- Automatic lead-ins for certain special functions
         if is_getbuffer_slot:
             self.getbuffer_init(code)
@@ -1364,8 +1374,12 @@ class FuncDefNode(StatNode, BlockNode):
             code.putln("if (unlikely(!%s)) {" % Naming.cur_scope_cname)
             if is_getbuffer_slot:
                 self.getbuffer_error_cleanup(code)
-            if not lenv.nogil:
+
+            if use_refnanny:
                 code.put_finish_refcount_context()
+                if acquire_gil_for_var_decls_only:
+                    code.put_release_ensured_gil()
+
             # FIXME: what if the error return value is a Python value?
             code.putln("return %s;" % self.error_value())
             code.putln("}")
@@ -1411,6 +1425,9 @@ class FuncDefNode(StatNode, BlockNode):
             if entry.type.is_buffer:
                 Buffer.put_acquire_arg_buffer(entry, code, self.pos)
 
+        if acquire_gil_for_var_decls_only:
+            code.put_release_ensured_gil()
+
         # -------------------------
         # ----- Function body -----
         # -------------------------
@@ -1453,7 +1470,7 @@ class FuncDefNode(StatNode, BlockNode):
                 # TODO: Fix exception tracing (though currently unused by cProfile).
                 # code.globalstate.use_utility_code(get_exception_tuple_utility_code)
                 # code.put_trace_exception()
-                code.putln('__Pyx_AddTraceback("%s");' % self.entry.qualified_name)
+                code.put_add_traceback(self.entry.qualified_name)
             else:
                 warning(self.entry.pos, "Unraisable exception in function '%s'." \
                             % self.entry.qualified_name, 0)
@@ -1477,7 +1494,6 @@ class FuncDefNode(StatNode, BlockNode):
             if buffers_present or is_getbuffer_slot:
                 code.put_goto(code.return_from_error_cleanup_label)
 
-
         # ----- Non-error return cleanup
         code.put_label(code.return_label)
         for entry in lenv.buffer_entries:
@@ -1525,13 +1541,13 @@ class FuncDefNode(StatNode, BlockNode):
                 code.put_trace_return(Naming.retval_cname)
             else:
                 code.put_trace_return("Py_None")
+
         if not lenv.nogil:
+            # GIL holding funcion
             code.put_finish_refcount_context()
 
-        if acquire_gil:
-            code.putln("#ifdef WITH_THREAD")
-            code.putln("PyGILState_Release(_save);")
-            code.putln("#endif")
+        if acquire_gil or acquire_gil_for_var_decls_only:
+            code.put_release_ensured_gil()
 
         if not self.return_type.is_void:
             code.putln("return %s;" % Naming.retval_cname)
@@ -1543,6 +1559,14 @@ class FuncDefNode(StatNode, BlockNode):
 
         # ----- Go back and insert temp variable declarations
         tempvardecl_code.put_temp_declarations(code.funcstate)
+        if code.funcstate.should_declare_error_indicator:
+            # Initialize these variables to shut up compiler warnings
+            tempvardecl_code.putln("int %s = 0;" % Naming.lineno_cname)
+            tempvardecl_code.putln("const char *%s = NULL;" %
+                                                    Naming.filename_cname)
+            if code.c_line_in_traceback:
+                tempvardecl_code.putln("int %s = 0;" % Naming.clineno_cname)
+
         # ----- Python version
         code.exit_cfunc_scope()
         if self.py_func:
@@ -1764,7 +1788,7 @@ class CFuncDefNode(FuncDefNode):
                 error(self.pos,
                       "Function with Python return type cannot be declared nogil")
             for entry in self.local_scope.var_entries:
-                if entry.type.is_pyobject:
+                if entry.type.is_pyobject and not entry.in_with_gil_block:
                     error(self.pos, "Function declared nogil has Python locals or temporaries")
 
     def analyse_expressions(self, env):
@@ -2436,7 +2460,7 @@ class DefNode(FuncDefNode):
                         code.put_var_xdecref_clear(self.starstar_arg.entry)
                     else:
                         code.put_var_decref_clear(self.starstar_arg.entry)
-            code.putln('__Pyx_AddTraceback("%s");' % self.entry.qualified_name)
+            code.put_add_traceback(self.entry.qualified_name)
             # The arguments are put into the closure one after the
             # other, so when type errors are found, all references in
             # the closure instance must be properly ref-counted to
@@ -3060,7 +3084,7 @@ class GeneratorBodyDefNode(DefNode):
             code.put_label(code.error_label)
             for cname, type in code.funcstate.all_managed_temps():
                 code.put_xdecref(cname, type)
-            code.putln('__Pyx_AddTraceback("%s");' % self.entry.qualified_name)
+            code.put_add_traceback(self.entry.qualified_name)
 
         # ----- Non-error return cleanup
         code.put_label(code.return_label)
@@ -5256,7 +5280,7 @@ class ExceptClauseNode(Node):
         exc_vars = [code.funcstate.allocate_temp(py_object_type,
                                                  manage_ref=True)
                     for i in xrange(3)]
-        code.putln('__Pyx_AddTraceback("%s");' % self.function_name)
+        code.put_add_traceback(self.function_name)
         # We always have to fetch the exception value even if
         # there is no target, because this also normalises the
         # exception and stores it in the thread state.
@@ -5357,6 +5381,8 @@ class TryFinallyStatNode(StatNode):
     # continue in the try block, since we have no problem
     # handling it.
 
+    is_try_finally_in_nogil = False
+
     def create_analysed(pos, env, body, finally_clause):
         node = TryFinallyStatNode(pos, body=body, finally_clause=finally_clause)
         return node
@@ -5388,20 +5414,24 @@ class TryFinallyStatNode(StatNode):
         if not self.handle_error_case:
             code.error_label = old_error_label
         catch_label = code.new_label()
-        code.putln(
-            "/*try:*/ {")
+
+        code.putln("/*try:*/ {")
+
         if self.disallow_continue_in_try_finally:
             was_in_try_finally = code.funcstate.in_try_finally
             code.funcstate.in_try_finally = 1
+
         self.body.generate_execution_code(code)
+
         if self.disallow_continue_in_try_finally:
             code.funcstate.in_try_finally = was_in_try_finally
-        code.putln(
-            "}")
+
+        code.putln("}")
+
         temps_to_clean_up = code.funcstate.all_free_managed_temps()
         code.mark_pos(self.finally_clause.pos)
-        code.putln(
-            "/*finally:*/ {")
+        code.putln("/*finally:*/ {")
+
         cases_used = []
         error_label_used = 0
         for i, new_label in enumerate(new_labels):
@@ -5410,22 +5440,25 @@ class TryFinallyStatNode(StatNode):
                 if new_label == new_error_label:
                     error_label_used = 1
                     error_label_case = i
+
         if cases_used:
-            code.putln(
-                    "int __pyx_why;")
+            code.putln("int __pyx_why;")
+
             if error_label_used and self.preserve_exception:
-                code.putln(
-                    "PyObject *%s, *%s, *%s;" % Naming.exc_vars)
-                code.putln(
-                    "int %s;" % Naming.exc_lineno_name)
-                exc_var_init_zero = ''.join(["%s = 0; " % var for var in Naming.exc_vars])
+                code.putln("PyObject *%s, *%s, *%s;" % Naming.exc_vars)
+                code.putln("int %s;" % Naming.exc_lineno_name)
+                exc_var_init_zero = ''.join(
+                                ["%s = 0; " % var for var in Naming.exc_vars])
                 exc_var_init_zero += '%s = 0;' % Naming.exc_lineno_name
                 code.putln(exc_var_init_zero)
+
+                if self.is_try_finally_in_nogil:
+                    code.declare_gilstate()
             else:
                 exc_var_init_zero = None
+
             code.use_label(catch_label)
-            code.putln(
-                    "__pyx_why = 0; goto %s;" % catch_label)
+            code.putln("__pyx_why = 0; goto %s;" % catch_label)
             for i in cases_used:
                 new_label = new_labels[i]
                 #if new_label and new_label != "<try>":
@@ -5436,27 +5469,36 @@ class TryFinallyStatNode(StatNode):
                     code.put('%s: ' % new_label)
                     if exc_var_init_zero:
                         code.putln(exc_var_init_zero)
-                    code.putln("__pyx_why = %s; goto %s;" % (
-                            i+1,
-                            catch_label))
+                    code.putln("__pyx_why = %s; goto %s;" % (i+1, catch_label))
             code.put_label(catch_label)
+
         code.set_all_labels(old_labels)
         if error_label_used:
             code.new_error_label()
             finally_error_label = code.error_label
+
         self.finally_clause.generate_execution_code(code)
+
         if error_label_used:
             if finally_error_label in code.labels_used and self.preserve_exception:
                 over_label = code.new_label()
-                code.put_goto(over_label);
+                code.put_goto(over_label)
                 code.put_label(finally_error_label)
+
                 code.putln("if (__pyx_why == %d) {" % (error_label_case + 1))
+                if self.is_try_finally_in_nogil:
+                    code.put_ensure_gil(declare_gilstate=False)
                 for var in Naming.exc_vars:
                     code.putln("Py_XDECREF(%s);" % var)
+                if self.is_try_finally_in_nogil:
+                    code.put_release_ensured_gil()
                 code.putln("}")
+
                 code.put_goto(old_error_label)
                 code.put_label(over_label)
+
             code.error_label = old_error_label
+
         if cases_used:
             code.putln(
                 "switch (__pyx_why) {")
@@ -5466,12 +5508,13 @@ class TryFinallyStatNode(StatNode):
                     self.put_error_uncatcher(code, i+1, old_error_label)
                 else:
                     code.use_label(old_label)
-                    code.putln(
-                        "case %s: goto %s;" % (
-                            i+1,
-                            old_label))
+                    code.putln("case %s: goto %s;" % (i+1, old_label))
+
+            # End the switch
             code.putln(
                 "}")
+
+        # End finally
         code.putln(
             "}")
 
@@ -5479,40 +5522,45 @@ class TryFinallyStatNode(StatNode):
         self.body.generate_function_definitions(env, code)
         self.finally_clause.generate_function_definitions(env, code)
 
-    def put_error_catcher(self, code, error_label, i, catch_label, temps_to_clean_up):
+    def put_error_catcher(self, code, error_label, i, catch_label,
+                          temps_to_clean_up):
         code.globalstate.use_utility_code(restore_exception_utility_code)
-        code.putln(
-            "%s: {" %
-                error_label)
-        code.putln(
-                "__pyx_why = %s;" %
-                    i)
+        code.putln("%s: {" % error_label)
+        code.putln("__pyx_why = %s;" % i)
+
+        if self.is_try_finally_in_nogil:
+            code.put_ensure_gil(declare_gilstate=False)
+
         for temp_name, type in temps_to_clean_up:
             code.put_xdecref_clear(temp_name, type)
-        code.putln(
-                "__Pyx_ErrFetch(&%s, &%s, &%s);" %
-                    Naming.exc_vars)
-        code.putln(
-                "%s = %s;" % (
-                    Naming.exc_lineno_name, Naming.lineno_cname))
+
+        code.putln("__Pyx_ErrFetch(&%s, &%s, &%s);" % Naming.exc_vars)
+        code.putln("%s = %s;" % (Naming.exc_lineno_name, Naming.lineno_cname))
+
+        if self.is_try_finally_in_nogil:
+            code.put_release_ensured_gil()
+
         code.put_goto(catch_label)
         code.putln("}")
 
     def put_error_uncatcher(self, code, i, error_label):
         code.globalstate.use_utility_code(restore_exception_utility_code)
         code.putln(
-            "case %s: {" %
-                i)
-        code.putln(
-                "__Pyx_ErrRestore(%s, %s, %s);" %
-                    Naming.exc_vars)
-        code.putln(
-                "%s = %s;" % (
-                    Naming.lineno_cname, Naming.exc_lineno_name))
+            "case %s: {" % i)
+
+        if self.is_try_finally_in_nogil:
+            code.put_ensure_gil(declare_gilstate=False)
+
+        code.putln("__Pyx_ErrRestore(%s, %s, %s);" % Naming.exc_vars)
+        code.putln("%s = %s;" % (Naming.lineno_cname, Naming.exc_lineno_name))
+
+        if self.is_try_finally_in_nogil:
+            code.put_release_ensured_gil()
+
         for var in Naming.exc_vars:
             code.putln(
-                "%s = 0;" %
-                    var)
+                   "%s = 0;" % var)
+
         code.put_goto(error_label)
         code.putln(
             "}")
@@ -5522,50 +5570,59 @@ class TryFinallyStatNode(StatNode):
         self.finally_clause.annotate(code)
 
 
-class GILStatNode(TryFinallyStatNode):
+class NogilTryFinallyStatNode(TryFinallyStatNode):
+    """
+    A try/finally statement that may be used in nogil code sections.
+    """
+
+    preserve_exception = False
+    nogil_check = None
+
+
+class GILStatNode(NogilTryFinallyStatNode):
     #  'with gil' or 'with nogil' statement
     #
     #   state   string   'gil' or 'nogil'
 
-#    child_attrs = []
-
-    preserve_exception = 0
-
     def __init__(self, pos, state, body):
         self.state = state
         TryFinallyStatNode.__init__(self, pos,
             body = body,
             finally_clause = GILExitNode(pos, state = state))
 
+    def analyse_declarations(self, env):
+        env._in_with_gil_block = (self.state == 'gil')
+        if self.state == 'gil':
+            env.has_with_gil_block = True
+
+        return super(GILStatNode, self).analyse_declarations(env)
+
     def analyse_expressions(self, env):
         env.use_utility_code(force_init_threads_utility_code)
         was_nogil = env.nogil
-        env.nogil = 1
+        env.nogil = self.state == 'nogil'
         TryFinallyStatNode.analyse_expressions(self, env)
         env.nogil = was_nogil
 
-    nogil_check = None
-
     def generate_execution_code(self, code):
         code.mark_pos(self.pos)
-        code.putln("{")
+        code.begin_block()
+
         if self.state == 'gil':
-            code.putln("#ifdef WITH_THREAD")
-            code.putln("PyGILState_STATE _save = PyGILState_Ensure();")
-            code.putln("#endif")
+            code.put_ensure_gil()
         else:
-            code.putln("#ifdef WITH_THREAD")
-            code.putln("PyThreadState *_save = NULL;")
-            code.putln("#endif")
-            code.putln("Py_UNBLOCK_THREADS")
+            code.put_release_gil()
+
         TryFinallyStatNode.generate_execution_code(self, code)
-        code.putln("}")
+        code.end_block()
 
 
 class GILExitNode(StatNode):
-    #  Used as the 'finally' block in a GILStatNode
-    #
-    #  state   string   'gil' or 'nogil'
+    """
+    Used as the 'finally' block in a GILStatNode
+
+    state   string   'gil' or 'nogil'
+    """
 
     child_attrs = []
 
@@ -5574,11 +5631,18 @@ class GILExitNode(StatNode):
 
     def generate_execution_code(self, code):
         if self.state == 'gil':
-            code.putln("#ifdef WITH_THREAD")
-            code.putln("PyGILState_Release(_save);")
-            code.putln("#endif")
+            code.put_release_ensured_gil()
         else:
-            code.putln("Py_BLOCK_THREADS")
+            code.put_acquire_gil()
+
+
+class EnsureGILNode(GILExitNode):
+    """
+    Ensure the GIL in nogil functions for cleanup before returning.
+    """
+
+    def generate_execution_code(self, code):
+        code.put_ensure_gil(declare_gilstate=False)
 
 
 class CImportStatNode(StatNode):
@@ -7082,15 +7146,22 @@ requires=[raise_double_keywords_utility_code])
 #------------------------------------------------------------------------------------
 
 traceback_utility_code = UtilityCode(
-proto = """
-static void __Pyx_AddTraceback(const char *funcname); /*proto*/
-""",
-impl = """
+    proto = """
+static void __Pyx_AddTraceback(const char *funcname, int %(CLINENO)s,
+                               int %(LINENO)s, const char *%(FILENAME)s); /*proto*/
+""" % {
+    'FILENAME': Naming.filename_cname,
+    'LINENO':  Naming.lineno_cname,
+    'CLINENO':  Naming.clineno_cname,
+},
+
+    impl = """
 #include "compile.h"
 #include "frameobject.h"
 #include "traceback.h"
 
-static void __Pyx_AddTraceback(const char *funcname) {
+static void __Pyx_AddTraceback(const char *funcname, int %(CLINENO)s,
+                               int %(LINENO)s, const char *%(FILENAME)s) {
     PyObject *py_srcfile = 0;
     PyObject *py_funcname = 0;
     PyObject *py_globals = 0;

Cython/Compiler/ParseTreeTransforms.py

@@ -638,6 +638,8 @@ class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
                                         'is not allowed in %s scope' % (directive, scope)))
             return False
         else:
+            if directive not in Options.directive_defaults:
+                error(pos, "Invalid directive: '%s'." % (directive,))
             return True
 
     # Set up processing and handle the cython: comments.
@@ -946,9 +948,9 @@ class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
                         PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
                 else:
                     name, value = directive
-                    if name == 'nogil':
+                    if name in ('nogil', 'gil'):
                         # special case: in pure mode, "with nogil" spells "with cython.nogil"
-                        node = Nodes.GILStatNode(node.pos, state = "nogil", body = node.body)
+                        node = Nodes.GILStatNode(node.pos, state = name, body = node.body)
                         return self.visit_Node(node)
                     if self.check_directive_scope(node.pos, name, 'with statement'):
                         directive_dict[name] = value
@@ -1361,6 +1363,18 @@ if VALUE is not None:
                 else:
                     error(type_node.pos, "Not a type")
         node.body.analyse_declarations(lenv)
+
+        if lenv.nogil and lenv.has_with_gil_block:
+            # Acquire the GIL for cleanup in 'nogil' functions, by wrapping
+            # the entire function body in try/finally.
+            # The corresponding release will be taken care of by
+            # Nodes.FuncDefNode.generate_function_definitions()
+            node.body = Nodes.NogilTryFinallyStatNode(
+                node.body.pos,
+                body = node.body,
+                finally_clause = Nodes.EnsureGILNode(node.body.pos),
+            )
+
         self.env_stack.append(lenv)
         self.visitchildren(node)
         self.env_stack.pop()
@@ -1532,6 +1546,7 @@ if VALUE is not None:
         # ---------------------------------------
         return property
 
+
 class AnalyseExpressionsTransform(CythonTransform):
 
     def visit_ModuleNode(self, node):
@@ -1553,6 +1568,7 @@ class AnalyseExpressionsTransform(CythonTransform):
         self.visitchildren(node)
         return node
 
+
 class ExpandInplaceOperators(EnvTransform):
 
     def visit_InPlaceAssignmentNode(self, node):
@@ -2016,10 +2032,18 @@ class GilCheck(VisitorTransform):
     Call `node.gil_check(env)` on each node to make sure we hold the
     GIL when we need it.  Raise an error when on Python operations
     inside a `nogil` environment.
+
+    Additionally, raise exceptions for closely nested with gil or with nogil
+    statements. The latter would abort Python.
     """
+
     def __call__(self, root):
         self.env_stack = [root.scope]
         self.nogil = False
+
+        # True for 'cdef func() nogil:' functions, as the GIL may be held while
+        # calling this function (thus contained 'nogil' blocks may be valid).
+        self.nogil_declarator_only = False
         return super(GilCheck, self).__call__(root)
 
     def visit_FuncDefNode(self, node):
@@ -2027,10 +2051,17 @@ class GilCheck(VisitorTransform):
         was_nogil = self.nogil
         self.nogil = node.local_scope.nogil
 
+        if self.nogil:
+            self.nogil_declarator_only = True
+
         if self.nogil and node.nogil_check:
             node.nogil_check(node.local_scope)
 
         self.visitchildren(node)
+
+        # This cannot be nested, so it doesn't need backup/restore
+        self.nogil_declarator_only = False
+
         self.env_stack.pop()
         self.nogil = was_nogil
         return node
@@ -2041,6 +2072,23 @@ class GilCheck(VisitorTransform):
 
         was_nogil = self.nogil
         self.nogil = (node.state == 'nogil')
+
+        if was_nogil == self.nogil and not self.nogil_declarator_only:
+            if not was_nogil:
+                error(node.pos, "Trying to acquire the GIL while it is "
+                                "already held.")
+            else:
+                error(node.pos, "Trying to release the GIL while it was "
+                                "previously released.")
+
+        if isinstance(node.finally_clause, Nodes.StatListNode):
+            # The finally clause of the GILStatNode is a GILExitNode,
+            # which is wrapped in a StatListNode. Just unpack that.
+            node.finally_clause, = node.finally_clause.stats
+
+        if node.state == 'gil':
+            self.seen_with_gil_statement = True
+
         self.visitchildren(node)
         self.nogil = was_nogil
         return node
@@ -2074,6 +2122,28 @@ class GilCheck(VisitorTransform):
         self.visitchildren(node)
         return node
 
+    def visit_TryFinallyStatNode(self, node):
+        """
+        Take care of try/finally statements in nogil code sections. The
+        'try' must contain a 'with gil:' statement somewhere.
+        """
+        if not self.nogil or isinstance(node, Nodes.GILStatNode):
+            return self.visit_Node(node)
+
+        node.nogil_check = None
+        node.is_try_finally_in_nogil = True
+
+        # First, visit the body and check for errors
+        self.seen_with_gil_statement = False
+        self.visitchildren(node.body)
+
+        if not self.seen_with_gil_statement:
+            error(node.pos, "Cannot use try/finally in nogil sections unless "
+                            "it contains a 'with gil' statement.")
+
+        self.visitchildren(node.finally_clause)
+        return node
+
     def visit_Node(self, node):
         if self.env_stack and self.nogil and node.nogil_check:
             node.nogil_check(self.env_stack[-1])

Cython/Compiler/Parsing.py

@@ -1580,7 +1580,7 @@ def p_with_statement(s):
 
 def p_with_items(s):
     pos = s.position()
-    if not s.in_python_file and s.sy == 'IDENT' and s.systring == 'nogil':
+    if not s.in_python_file and s.sy == 'IDENT' and s.systring in ('nogil', 'gil'):
         state = s.systring
         s.next()
         if s.sy == ',':

Cython/Compiler/Symtab.py

@@ -176,6 +176,7 @@ class Entry(object):
     buffer_aux = None
     prev_entry = None
     might_overflow = 0
+    in_with_gil_block = 0
 
     def __init__(self, name, cname, type, pos = None, init = None):
         self.name = name
@@ -1349,6 +1350,12 @@ class ModuleScope(Scope):
 
 class LocalScope(Scope):
 
+    # Does the function have a 'with gil:' block?
+    has_with_gil_block = False
+
+    # Transient attribute, used for symbol table variable declarations
+    _in_with_gil_block = False
+
     def __init__(self, name, outer_scope, parent_scope = None):
         if parent_scope is None:
             parent_scope = outer_scope
@@ -1383,6 +1390,8 @@ class LocalScope(Scope):
             entry.init = "0"
         entry.init_to_none = (type.is_pyobject or type.is_unspecified) and Options.init_local_none
         entry.is_local = 1
+
+        entry.in_with_gil_block = self._in_with_gil_block
         self.var_entries.append(entry)
         return entry
 

Cython/Includes/cpython/__init__.pxd

@@ -146,6 +146,7 @@ from cpython.method cimport *
 from cpython.weakref cimport *
 from cpython.getargs cimport *
 from cpython.pythread cimport *
+from cpython.pystate cimport *
 
 # Python <= 2.x
 from cpython.cobject cimport *

Cython/Includes/cpython/pystate.pxd

+# Thread and interpreter state structures and their interfaces
+
+from cpython.ref cimport PyObject
+
+cdef extern from "Python.h":
+
+    # We make these an opague types. If the user wants specific attributes,
+    # they can be declared manually.
+
+    ctypedef struct PyInterpreterState:
+        pass
+
+    ctypedef struct PyThreadState:
+        pass
+
+    ctypedef struct PyFrameObject:
+        pass
+
+    # This is not actually a struct, but make sure it can never be coerced to
+    # an int or used in arithmetic expressions
+    ctypedef struct PyGILState_STATE
+
+    # The type of the trace function registered using PyEval_SetProfile() and
+    # PyEval_SetTrace().
+    # Py_tracefunc return -1 when raising an exception, or 0 for success.
+    ctypedef int (*Py_tracefunc)(PyObject *, PyFrameObject *, int, PyObject *)
+
+    # The following values are used for 'what' for tracefunc functions
+    enum:
+        PyTrace_CALL
+        PyTrace_EXCEPTION
+        PyTrace_LINE
+        PyTrace_RETURN
+        PyTrace_C_CALL
+        PyTrace_C_EXCEPTION
+        PyTrace_C_RETURN
+
+
+    PyInterpreterState * PyInterpreterState_New()
+    void PyInterpreterState_Clear(PyInterpreterState *)
+    void PyInterpreterState_Delete(PyInterpreterState *)
+
+    PyThreadState * PyThreadState_New(PyInterpreterState *)
+    void PyThreadState_Clear(PyThreadState *)
+    void PyThreadState_Delete(PyThreadState *)
+
+    PyThreadState * PyThreadState_Get()
+    PyThreadState * PyThreadState_Swap(PyThreadState *)
+    PyObject * PyThreadState_GetDict()
+    int PyThreadState_SetAsyncExc(long, PyObject *)
+
+    # Ensure that the current thread is ready to call the Python
+    # C API, regardless of the current state of Python, or of its
+    # thread lock.  This may be called as many times as desired
+    # by a thread so long as each call is matched with a call to
+    # PyGILState_Release().  In general, other thread-state APIs may
+    # be used between _Ensure() and _Release() calls, so long as the
+    # thread-state is restored to its previous state before the Release().
+    # For example, normal use of the Py_BEGIN_ALLOW_THREADS/
+    # Py_END_ALLOW_THREADS macros are acceptable.
+
+    # The return value is an opaque "handle" to the thread state when
+    # PyGILState_Ensure() was called, and must be passed to
+    # PyGILState_Release() to ensure Python is left in the same state. Even
+    # though recursive calls are allowed, these handles can *not* be shared -
+    # each unique call to PyGILState_Ensure must save the handle for its
+    # call to PyGILState_Release.
+
+    # When the function returns, the current thread will hold the GIL.
+
+    # Failure is a fatal error.
+    PyGILState_STATE PyGILState_Ensure()
+
+    # Release any resources previously acquired.  After this call, Python's
+    # state will be the same as it was prior to the corresponding
+    # PyGILState_Ensure() call (but generally this state will be unknown to
+    # the caller, hence the use of the GILState API.)
+
+    # Every call to PyGILState_Ensure must be matched by a call to
+    # PyGILState_Release on the same thread.
+    void PyGILState_Release(PyGILState_STATE)
+
+    # Routines for advanced debuggers, requested by David Beazley.
+    # Don't use unless you know what you are doing!
+    PyInterpreterState * PyInterpreterState_Head()
+    PyInterpreterState * PyInterpreterState_Next(PyInterpreterState *)
+    PyThreadState * PyInterpreterState_ThreadHead(PyInterpreterState *)
+    PyThreadState * PyThreadState_Next(PyThreadState *)

Cython/Runtime/refnanny.pyx

 from cpython.ref cimport PyObject, Py_INCREF, Py_DECREF, Py_XDECREF
 from cpython.exc cimport PyErr_Fetch, PyErr_Restore
+from cpython.pystate cimport PyThreadState_Get
 
 
 loglevel = 0
@@ -80,6 +81,7 @@ cdef PyObject* SetupContext(char* funcname, int lineno, char* filename) except N
         return NULL
     cdef PyObject* type = NULL, *value = NULL, *tb = NULL
     cdef PyObject* result = NULL
+    PyThreadState_Get()
     PyErr_Fetch(&type, &value, &tb)
     try:
         ctx = Context(funcname, lineno, filename)
@@ -131,16 +133,19 @@ cdef void GIVEREF(PyObject* ctx, PyObject* p_obj, int lineno):
 
 cdef void INCREF(PyObject* ctx, PyObject* obj, int lineno):
     if obj is not NULL: Py_INCREF(<object>obj)
+    PyThreadState_Get()
     GOTREF(ctx, obj, lineno)
 
 cdef void DECREF(PyObject* ctx, PyObject* obj, int lineno):
     if GIVEREF_and_report(ctx, obj, lineno):
         if obj is not NULL: Py_DECREF(<object>obj)
+    PyThreadState_Get()
 
 cdef void FinishContext(PyObject** ctx):
     if ctx == NULL or ctx[0] == NULL: return
     cdef PyObject* type = NULL, *value = NULL, *tb = NULL
     cdef object errors = None
+    PyThreadState_Get()
     PyErr_Fetch(&type, &value, &tb)
     try:
         try:

Cython/Shadow.py

@@ -89,6 +89,7 @@ class _nogil(object):
         return exc_class is None
 
 nogil = _nogil()
+gil = _nogil()
 del _nogil
 
 # Emulated types

tests/errors/incorrectly_nested_gil_blocks.pyx

+# mode: error
+
+with gil:
+    pass
+
+with nogil:
+    with nogil:
+        pass
+
+cdef void without_gil() nogil:
+   # This is not an error, as 'func' *may* be called without the GIL, but it
+   # may also be held.
+    with nogil:
+        pass
+
+cdef void with_gil() with gil:
+    # This is an error, as the GIL is acquired already
+    with gil:
+        pass
+
+def func():
+    with gil:
+        pass
+
+_ERRORS = u'''
+3:5: Trying to acquire the GIL while it is already held.
+7:9: Trying to release the GIL while it was previously released.
+18:9: Trying to acquire the GIL while it is already held.
+22:9: Trying to acquire the GIL while it is already held.
+'''

tests/errors/nogil.pyx

@@ -158,6 +158,6 @@ _ERRORS = u"""
 60:17: Truth-testing Python object not allowed without gil
 62:8: For-loop using object bounds or target not allowed without gil
 64:8: Try-except statement not allowed without gil
-68:8: Try-finally statement not allowed without gil
+68:8: Cannot use try/finally in nogil sections unless it contains a 'with gil' statement.
 85:8: For-loop using object bounds or target not allowed without gil
 """

tests/run/parallel.pyx

@@ -24,4 +24,4 @@ def test_parallel():
 
     free(buf)
 
-#include "sequential_parallel.pyx"
+include "sequential_parallel.pyx"

tests/run/sequential_parallel.pyx

@@ -50,7 +50,7 @@ def test_propagation():
     >>> test_propagation()
     (9, 9, 9, 9, 450, 450)
     """
-    cdef int i, j, x, y
+    cdef int i = 0, j = 0, x = 0, y = 0
     cdef int sum1 = 0, sum2 = 0
 
     for i in prange(10, nogil=True):

tests/run/with_gil.pyx

+"""
+Test the 'with gil:' statement.
+"""
+
+cimport cython
+
+#from libc.stdio cimport printf, puts
+from cpython cimport PyObject, Py_INCREF
+
+import sys
+
+
+def redirect_stderr(func, *args, **kwargs):
+    """
+    Helper function that redirects stderr to stdout for doctest.
+    """
+    stderr, sys.stderr = sys.stderr, sys.stdout
+    func(*args, **kwargs)
+    sys.stderr = stderr
+
+cdef void puts(char *string) with gil:
+    """
+    We need this for doctest, used from nogil sections.
+    """
+    print string.decode('ascii')
+
+
+# Start with some normal Python functions
+
+def test_simple():
+    """
+    >>> test_simple()
+    ['spam', 'ham']
+    """
+    with nogil:
+        with gil:
+            print ['spam', 'ham']
+
+def test_nested_gil_blocks():
+    """
+    >>> test_nested_gil_blocks()
+    entered outer nogil section
+    entered outer gil section
+    entered inner nogil section
+    entered inner gil section
+    leaving inner gil section
+    leaving inner nogil section
+    leaving outer gil section
+    leaving outer nogil section
+    """
+
+    with nogil:
+        puts("entered outer nogil section")
+
+        with gil:
+            print 'entered outer gil section'
+
+            with nogil:
+                puts("entered inner nogil section")
+                with gil:
+                    print 'entered inner gil section'
+                    print 'leaving inner gil section'
+                puts("leaving inner nogil section")
+
+            print "leaving outer gil section"
+        puts("leaving outer nogil section")
+
+def test_propagate_exception():
+    """
+    >>> test_propagate_exception()
+    Traceback (most recent call last):
+        ...
+    Exception: This exception propagates!
+    """
+    # Note, doctest doesn't support both output and exceptions
+    with nogil:
+        with gil:
+            raise Exception("This exception propagates!")
+
+def test_catch_exception():
+    """
+    >>> test_catch_exception()
+    This is executed
+    Exception value
+    This is also executed
+    """
+    try:
+        with nogil:
+            with gil:
+                print "This is executed"
+                raise Exception("Exception value")
+                print "This is not executed"
+            puts("This is also not executed")
+    except Exception, e:
+        print e
+    print "This is also executed"
+
+def test_try_finally_and_outer_except():
+    """
+    >>> test_try_finally_and_outer_except()
+    First finally clause
+    Second finally clause
+    Caught: Some Exception
+    End of function
+    """
+    try:
+
+        with nogil:
+            with gil:
+                try:
+                    with nogil:
+                        with gil:
+                            try:
+                                raise Exception("Some Exception")
+                            finally:
+                                puts("First finally clause")
+                finally:
+                    puts("Second finally clause")
+            puts("This is not executed")
+
+    except Exception, e:
+        print "Caught:", e
+
+    print "End of function"
+
+def test_restore_exception():
+    """
+    >>> test_restore_exception()
+    Traceback (most recent call last):
+        ...
+    Exception: Override the raised exception
+    """
+    with nogil:
+        with gil:
+            try:
+                with nogil:
+                    with gil:
+                        raise Exception("Override this please")
+            finally:
+                raise Exception("Override the raised exception")
+
+def test_declared_variables():
+    """
+    >>> test_declared_variables()
+    None
+    None
+    ['s', 'p', 'a', 'm']
+    ['s', 'p', 'a', 'm']
+    """
+    cdef object somevar
+
+    print somevar
+
+    with nogil:
+        with gil:
+            print somevar
+            somevar = list("spam")
+            print somevar
+
+    print somevar
+
+def test_undeclared_variables():
+    """
+    >>> test_undeclared_variables()
+    None
+    None
+    ['s', 'p', 'a', 'm']
+    ['s', 'p', 'a', 'm']
+    """
+    print somevar
+    with nogil:
+        with gil:
+            print somevar
+            somevar = list("spam")
+            print somevar
+
+    print somevar
+
+def test_loops_and_boxing():
+    """
+    >>> test_loops_and_boxing()
+    spamham
+    h
+    a
+    m
+    done looping
+    """
+    cdef char c, *string = "spamham"
+
+    with nogil:
+        with gil:
+            print string
+            for c in string[4:]:
+                print "%c" % c
+            else:
+                print "done looping"
+
+cdef class SomeExtClass(object):
+    cdef int some_attribute
+
+@cython.infer_types(True)
+def test_infer_types():
+    """
+    >>> test_infer_types()
+    10
+    """
+    with nogil:
+        with gil:
+            obj = SomeExtClass()
+            obj.some_attribute = 10
+
+    print obj.some_attribute
+
+def test_closure():
+    """
+    >>> test_closure()
+    Traceback (most recent call last):
+        ...
+    Exception: {'twinkle': 'little star'}
+    """
+    a = dict(twinkle='little star')
+
+    def inner_function():
+        with nogil:
+            with gil:
+                raise Exception(a)
+
+    with nogil:
+        with gil:
+            inner_function()
+
+    raise Exception("This should not be raised!")
+
+cpdef test_cpdef():
+    """
+    >>> test_cpdef()
+    Seems to work!
+    Or does it?
+    """
+    with nogil:
+        with gil:
+            print "Seems to work!"
+        puts("Or does it?")
+
+
+# Now test some cdef functions with different return types
+
+cdef void void_nogil_ignore_exception() nogil:
+    with gil:
+        raise Exception("This is swallowed")
+
+    puts("unreachable")
+    with gil:
+        print "unreachable"
+
+cdef void void_nogil_nested_gil() nogil:
+    with gil:
+        with nogil:
+            with gil:
+                print 'Inner gil section'
+            puts("nogil section")
+        raise Exception("Swallow this")
+    puts("Don't print this")
+
+def test_nogil_void_funcs_with_gil():
+    """
+    >>> redirect_stderr(test_nogil_void_funcs_with_gil)
+    Exception Exception: Exception('This is swallowed',) in 'with_gil.void_nogil_ignore_exception' ignored
+    Inner gil section
+    nogil section
+    Exception Exception: Exception('Swallow this',) in 'with_gil.void_nogil_nested_gil' ignored
+    """
+    void_nogil_ignore_exception()
+    void_nogil_nested_gil()
+
+def test_nogil_void_funcs_with_nogil():
+    """
+    >>> redirect_stderr(test_nogil_void_funcs_with_nogil)
+    Exception Exception: Exception('This is swallowed',) in 'with_gil.void_nogil_ignore_exception' ignored
+    Inner gil section
+    nogil section
+    Exception Exception: Exception('Swallow this',) in 'with_gil.void_nogil_nested_gil' ignored
+    """
+    with nogil:
+        void_nogil_ignore_exception()
+        void_nogil_nested_gil()
+
+
+cdef PyObject *nogil_propagate_exception() nogil except NULL:
+    with nogil:
+        with gil:
+            raise Exception("This exception propagates!")
+    return <PyObject *> 1
+
+def test_nogil_propagate_exception():
+    """
+    >>> test_nogil_propagate_exception()
+    Traceback (most recent call last):
+        ...
+    Exception: This exception propagates!
+    """
+    nogil_propagate_exception()
+
+
+cdef with_gil_raise() with gil:
+    raise Exception("This exception propagates!")
+
+def test_release_gil_call_gil_func():
+    """
+    >>> test_release_gil_call_gil_func()
+    Traceback (most recent call last):
+        ...
+    Exception: This exception propagates!
+    """
+    with nogil:
+        with gil:
+            with_gil_raise()
+
+
+# Test try/finally in nogil blocks
+
+def test_try_finally_in_nogil():
+    """
+    >>> test_try_finally_in_nogil()
+    Traceback (most recent call last):
+        ...
+    Exception: Override exception!
+    """
+    with nogil:
+        try:
+            with gil:
+                raise Exception("This will be overridden")
+        finally:
+            with gil:
+                raise Exception("Override exception!")
+
+            with gil:
+                raise Exception("This code should not be executed!")
+
+def test_nogil_try_finally_no_exception():
+    """
+    >>> test_nogil_try_finally_no_exception()
+    first nogil try
+    nogil try gil
+    second nogil try
+    nogil finally
+    ------
+    First with gil block
+    Second with gil block
+    finally block
+    """
+    with nogil:
+        try:
+            puts("first nogil try")
+            with gil:
+                print "nogil try gil"
+            puts("second nogil try")
+        finally:
+            puts("nogil finally")
+
+    print '------'
+
+    with nogil:
+        try:
+            with gil:
+                print "First with gil block"
+
+            with gil:
+                print "Second with gil block"
+        finally:
+            puts("finally block")
+
+def test_nogil_try_finally_propagate_exception():
+    """
+    >>> test_nogil_try_finally_propagate_exception()
+    Execute finally clause
+    Propagate this!
+    """
+    try:
+        with nogil:
+            try:
+                with gil:
+                    raise Exception("Propagate this!")
+                with gil:
+                    raise Exception("Don't reach this section!")
+            finally:
+                puts("Execute finally clause")
+    except Exception, e:
+        print e
+
+def test_nogil_try_finally_return_in_with_gil(x):
+    """
+    >>> test_nogil_try_finally_return_in_with_gil(10)
+    print me
+    10
+    """
+    with nogil:
+        try:
+            with gil:
+                raise Exception("Swallow me!")
+        finally:
+            with gil:
+                print "print me"
+                return x
+
+    print "I am not executed"
+
+cdef void nogil_try_finally_return() nogil:
+    try:
+        with gil:
+            raise Exception("I am swallowed in nogil code... right?")
+    finally:
+        with gil:
+            print "print me first"
+
+        return
+
+    with gil:
+        print "I am not executed"
+
+def test_nogil_try_finally_return():
+    """
+    >>> test_nogil_try_finally_return()
+    print me first
+    """
+    with nogil:
+        nogil_try_finally_return()