Commit 4d4bb4aa authored by Robert Bradshaw's avatar Robert Bradshaw

Merge remote branch 'markflorisson/withgil'

Conflicts:
	tests/errors/nogil.pyx
parents ca1592c5 5f6667c7
......@@ -35,6 +35,7 @@ cdef class FunctionState:
cdef public size_t temp_counter
cdef public object closure_temps
cdef public bint should_declare_error_indicator
@cython.locals(n=size_t)
cpdef new_label(self, name=*)
......
......@@ -134,6 +134,12 @@ class FunctionState(object):
self.temp_counter = 0
self.closure_temps = None
# This is used for the error indicator, which needs to be local to the
# function. It used to be global, which relies on the GIL being held.
# However, exceptions may need to be propagated through 'nogil'
# sections, in which case we introduce a race condition.
self.should_declare_error_indicator = False
# labels
def new_label(self, name=None):
......@@ -1362,6 +1368,52 @@ class CCodeWriter(object):
doc_code,
term))
# GIL methods
def put_ensure_gil(self, declare_gilstate=True):
"""
Acquire the GIL. The generated code is safe even when no PyThreadState
has been allocated for this thread (for threads not initialized by
using the Python API). Additionally, the code generated by this method
may be called recursively.
"""
from Cython.Compiler import Nodes
self.globalstate.use_utility_code(Nodes.force_init_threads_utility_code)
self.putln("#ifdef WITH_THREAD")
if declare_gilstate:
self.put("PyGILState_STATE ")
self.putln("_save = PyGILState_Ensure();")
self.putln("#endif")
def put_release_ensured_gil(self):
"""
Releases the GIL, corresponds to `put_ensure_gil`.
"""
self.putln("#ifdef WITH_THREAD")
self.putln("PyGILState_Release(_save);")
self.putln("#endif")
def put_acquire_gil(self):
"""
Acquire the GIL. The thread's thread state must have been initialized
by a previous `put_release_gil`
"""
self.putln("Py_BLOCK_THREADS")
def put_release_gil(self):
"Release the GIL, corresponds to `put_acquire_gil`."
self.putln("#ifdef WITH_THREAD")
self.putln("PyThreadState *_save = NULL;")
self.putln("#endif")
self.putln("Py_UNBLOCK_THREADS")
def declare_gilstate(self):
self.putln("#ifdef WITH_THREAD")
self.putln("PyGILState_STATE _save;")
self.putln("#endif")
# error handling
def put_error_if_neg(self, pos, value):
......@@ -1369,10 +1421,12 @@ class CCodeWriter(object):
return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos)))
def set_error_info(self, pos):
self.funcstate.should_declare_error_indicator = True
if self.c_line_in_traceback:
cinfo = " %s = %s;" % (Naming.clineno_cname, Naming.line_c_macro)
else:
cinfo = ""
return "%s = %s[%s]; %s = %s;%s" % (
Naming.filename_cname,
Naming.filetable_cname,
......@@ -1412,6 +1466,20 @@ class CCodeWriter(object):
def put_finish_refcount_context(self):
self.putln("__Pyx_RefNannyFinishContext();")
def put_add_traceback(self, qualified_name):
"""
Build a Python traceback for propagating exceptions.
qualified_name should be the qualified name of the function
"""
format_tuple = (
qualified_name,
Naming.clineno_cname,
Naming.lineno_cname,
Naming.filename_cname,
)
self.putln('__Pyx_AddTraceback("%s", %s, %s, %s);' % format_tuple)
def put_trace_declarations(self):
self.putln('__Pyx_TraceDeclarations');
......
......@@ -6308,7 +6308,12 @@ class DivNode(NumBinopNode):
self.operand1.result(),
self.operand2.result()))
code.putln(code.set_error_info(self.pos));
code.put("if (__Pyx_cdivision_warning()) ")
code.put("if (__Pyx_cdivision_warning(%(FILENAME)s, "
"%(LINENO)s)) " % {
'FILENAME': Naming.filename_cname,
'LINENO': Naming.lineno_cname,
})
code.put_goto(code.error_label)
code.putln("}")
......@@ -8643,21 +8648,18 @@ static CYTHON_INLINE %(type)s __Pyx_mod_%(type_name)s(%(type)s a, %(type)s b) {
cdivision_warning_utility_code = UtilityCode(
proto="""
static int __Pyx_cdivision_warning(void); /* proto */
static int __Pyx_cdivision_warning(const char *, int); /* proto */
""",
impl="""
static int __Pyx_cdivision_warning(void) {
static int __Pyx_cdivision_warning(const char *filename, int lineno) {
return PyErr_WarnExplicit(PyExc_RuntimeWarning,
"division with oppositely signed operands, C and Python semantics differ",
%(FILENAME)s,
%(LINENO)s,
filename,
lineno,
__Pyx_MODULE_NAME,
NULL);
}
""" % {
'FILENAME': Naming.filename_cname,
'LINENO': Naming.lineno_cname,
})
""")
# from intobject.c
division_overflow_test_code = UtilityCode(
......
......@@ -1802,7 +1802,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
if code.label_used(code.error_label):
code.put_label(code.error_label)
# This helps locate the offending name.
code.putln('__Pyx_AddTraceback("%s");' % self.full_module_name);
code.put_add_traceback(self.full_module_name)
code.error_label = old_error_label
code.putln("bad:")
code.putln("return -1;")
......@@ -1919,7 +1919,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
for cname, type in code.funcstate.all_managed_temps():
code.put_xdecref(cname, type)
code.putln('if (%s) {' % env.module_cname)
code.putln('__Pyx_AddTraceback("init %s");' % env.qualified_name)
code.put_add_traceback("init %s" % env.qualified_name)
env.use_utility_code(Nodes.traceback_utility_code)
code.put_decref_clear(env.module_cname, py_object_type, nanny=False)
code.putln('} else if (!PyErr_Occurred()) {')
......
......@@ -1321,9 +1321,11 @@ class FuncDefNode(StatNode, BlockNode):
code.put(cenv.scope_class.type.declaration_code(Naming.outer_scope_cname))
code.putln(";")
self.generate_argument_declarations(lenv, code)
for entry in lenv.var_entries:
if not entry.in_closure:
code.put_var_declaration(entry)
init = ""
if not self.return_type.is_void:
if self.return_type.is_pyobject:
......@@ -1333,23 +1335,31 @@ class FuncDefNode(StatNode, BlockNode):
(self.return_type.declaration_code(Naming.retval_cname),
init))
tempvardecl_code = code.insertion_point()
if not lenv.nogil:
code.put_declare_refcount_context()
self.generate_keyword_list(code)
if profile:
code.put_trace_declarations()
# ----- Extern library function declarations
lenv.generate_library_function_declarations(code)
# ----- GIL acquisition
acquire_gil = self.acquire_gil
if acquire_gil:
env.use_utility_code(force_init_threads_utility_code)
code.putln("#ifdef WITH_THREAD")
code.putln("PyGILState_STATE _save = PyGILState_Ensure();")
code.putln("#endif")
# See if we need to acquire the GIL for variable declarations and
acquire_gil_for_var_decls_only = (lenv.nogil and
lenv.has_with_gil_block)
use_refnanny = not lenv.nogil or acquire_gil_for_var_decls_only
if acquire_gil or acquire_gil_for_var_decls_only:
code.put_ensure_gil()
# ----- set up refnanny
if not lenv.nogil:
if use_refnanny:
tempvardecl_code.put_declare_refcount_context()
code.put_setup_refcount_context(self.entry.name)
# ----- Automatic lead-ins for certain special functions
if is_getbuffer_slot:
self.getbuffer_init(code)
......@@ -1364,8 +1374,12 @@ class FuncDefNode(StatNode, BlockNode):
code.putln("if (unlikely(!%s)) {" % Naming.cur_scope_cname)
if is_getbuffer_slot:
self.getbuffer_error_cleanup(code)
if not lenv.nogil:
if use_refnanny:
code.put_finish_refcount_context()
if acquire_gil_for_var_decls_only:
code.put_release_ensured_gil()
# FIXME: what if the error return value is a Python value?
code.putln("return %s;" % self.error_value())
code.putln("}")
......@@ -1411,6 +1425,9 @@ class FuncDefNode(StatNode, BlockNode):
if entry.type.is_buffer:
Buffer.put_acquire_arg_buffer(entry, code, self.pos)
if acquire_gil_for_var_decls_only:
code.put_release_ensured_gil()
# -------------------------
# ----- Function body -----
# -------------------------
......@@ -1453,7 +1470,7 @@ class FuncDefNode(StatNode, BlockNode):
# TODO: Fix exception tracing (though currently unused by cProfile).
# code.globalstate.use_utility_code(get_exception_tuple_utility_code)
# code.put_trace_exception()
code.putln('__Pyx_AddTraceback("%s");' % self.entry.qualified_name)
code.put_add_traceback(self.entry.qualified_name)
else:
warning(self.entry.pos, "Unraisable exception in function '%s'." \
% self.entry.qualified_name, 0)
......@@ -1477,7 +1494,6 @@ class FuncDefNode(StatNode, BlockNode):
if buffers_present or is_getbuffer_slot:
code.put_goto(code.return_from_error_cleanup_label)
# ----- Non-error return cleanup
code.put_label(code.return_label)
for entry in lenv.buffer_entries:
......@@ -1525,13 +1541,13 @@ class FuncDefNode(StatNode, BlockNode):
code.put_trace_return(Naming.retval_cname)
else:
code.put_trace_return("Py_None")
if not lenv.nogil:
# GIL holding funcion
code.put_finish_refcount_context()
if acquire_gil:
code.putln("#ifdef WITH_THREAD")
code.putln("PyGILState_Release(_save);")
code.putln("#endif")
if acquire_gil or acquire_gil_for_var_decls_only:
code.put_release_ensured_gil()
if not self.return_type.is_void:
code.putln("return %s;" % Naming.retval_cname)
......@@ -1543,6 +1559,14 @@ class FuncDefNode(StatNode, BlockNode):
# ----- Go back and insert temp variable declarations
tempvardecl_code.put_temp_declarations(code.funcstate)
if code.funcstate.should_declare_error_indicator:
# Initialize these variables to shut up compiler warnings
tempvardecl_code.putln("int %s = 0;" % Naming.lineno_cname)
tempvardecl_code.putln("const char *%s = NULL;" %
Naming.filename_cname)
if code.c_line_in_traceback:
tempvardecl_code.putln("int %s = 0;" % Naming.clineno_cname)
# ----- Python version
code.exit_cfunc_scope()
if self.py_func:
......@@ -1764,7 +1788,7 @@ class CFuncDefNode(FuncDefNode):
error(self.pos,
"Function with Python return type cannot be declared nogil")
for entry in self.local_scope.var_entries:
if entry.type.is_pyobject:
if entry.type.is_pyobject and not entry.in_with_gil_block:
error(self.pos, "Function declared nogil has Python locals or temporaries")
def analyse_expressions(self, env):
......@@ -2436,7 +2460,7 @@ class DefNode(FuncDefNode):
code.put_var_xdecref_clear(self.starstar_arg.entry)
else:
code.put_var_decref_clear(self.starstar_arg.entry)
code.putln('__Pyx_AddTraceback("%s");' % self.entry.qualified_name)
code.put_add_traceback(self.entry.qualified_name)
# The arguments are put into the closure one after the
# other, so when type errors are found, all references in
# the closure instance must be properly ref-counted to
......@@ -3060,7 +3084,7 @@ class GeneratorBodyDefNode(DefNode):
code.put_label(code.error_label)
for cname, type in code.funcstate.all_managed_temps():
code.put_xdecref(cname, type)
code.putln('__Pyx_AddTraceback("%s");' % self.entry.qualified_name)
code.put_add_traceback(self.entry.qualified_name)
# ----- Non-error return cleanup
code.put_label(code.return_label)
......@@ -5256,7 +5280,7 @@ class ExceptClauseNode(Node):
exc_vars = [code.funcstate.allocate_temp(py_object_type,
manage_ref=True)
for i in xrange(3)]
code.putln('__Pyx_AddTraceback("%s");' % self.function_name)
code.put_add_traceback(self.function_name)
# We always have to fetch the exception value even if
# there is no target, because this also normalises the
# exception and stores it in the thread state.
......@@ -5357,6 +5381,8 @@ class TryFinallyStatNode(StatNode):
# continue in the try block, since we have no problem
# handling it.
is_try_finally_in_nogil = False
def create_analysed(pos, env, body, finally_clause):
node = TryFinallyStatNode(pos, body=body, finally_clause=finally_clause)
return node
......@@ -5388,20 +5414,24 @@ class TryFinallyStatNode(StatNode):
if not self.handle_error_case:
code.error_label = old_error_label
catch_label = code.new_label()
code.putln(
"/*try:*/ {")
code.putln("/*try:*/ {")
if self.disallow_continue_in_try_finally:
was_in_try_finally = code.funcstate.in_try_finally
code.funcstate.in_try_finally = 1
self.body.generate_execution_code(code)
if self.disallow_continue_in_try_finally:
code.funcstate.in_try_finally = was_in_try_finally
code.putln(
"}")
code.putln("}")
temps_to_clean_up = code.funcstate.all_free_managed_temps()
code.mark_pos(self.finally_clause.pos)
code.putln(
"/*finally:*/ {")
code.putln("/*finally:*/ {")
cases_used = []
error_label_used = 0
for i, new_label in enumerate(new_labels):
......@@ -5410,22 +5440,25 @@ class TryFinallyStatNode(StatNode):
if new_label == new_error_label:
error_label_used = 1
error_label_case = i
if cases_used:
code.putln(
"int __pyx_why;")
code.putln("int __pyx_why;")
if error_label_used and self.preserve_exception:
code.putln(
"PyObject *%s, *%s, *%s;" % Naming.exc_vars)
code.putln(
"int %s;" % Naming.exc_lineno_name)
exc_var_init_zero = ''.join(["%s = 0; " % var for var in Naming.exc_vars])
code.putln("PyObject *%s, *%s, *%s;" % Naming.exc_vars)
code.putln("int %s;" % Naming.exc_lineno_name)
exc_var_init_zero = ''.join(
["%s = 0; " % var for var in Naming.exc_vars])
exc_var_init_zero += '%s = 0;' % Naming.exc_lineno_name
code.putln(exc_var_init_zero)
if self.is_try_finally_in_nogil:
code.declare_gilstate()
else:
exc_var_init_zero = None
code.use_label(catch_label)
code.putln(
"__pyx_why = 0; goto %s;" % catch_label)
code.putln("__pyx_why = 0; goto %s;" % catch_label)
for i in cases_used:
new_label = new_labels[i]
#if new_label and new_label != "<try>":
......@@ -5436,27 +5469,36 @@ class TryFinallyStatNode(StatNode):
code.put('%s: ' % new_label)
if exc_var_init_zero:
code.putln(exc_var_init_zero)
code.putln("__pyx_why = %s; goto %s;" % (
i+1,
catch_label))
code.putln("__pyx_why = %s; goto %s;" % (i+1, catch_label))
code.put_label(catch_label)
code.set_all_labels(old_labels)
if error_label_used:
code.new_error_label()
finally_error_label = code.error_label
self.finally_clause.generate_execution_code(code)
if error_label_used:
if finally_error_label in code.labels_used and self.preserve_exception:
over_label = code.new_label()
code.put_goto(over_label);
code.put_goto(over_label)
code.put_label(finally_error_label)
code.putln("if (__pyx_why == %d) {" % (error_label_case + 1))
if self.is_try_finally_in_nogil:
code.put_ensure_gil(declare_gilstate=False)
for var in Naming.exc_vars:
code.putln("Py_XDECREF(%s);" % var)
if self.is_try_finally_in_nogil:
code.put_release_ensured_gil()
code.putln("}")
code.put_goto(old_error_label)
code.put_label(over_label)
code.error_label = old_error_label
if cases_used:
code.putln(
"switch (__pyx_why) {")
......@@ -5466,12 +5508,13 @@ class TryFinallyStatNode(StatNode):
self.put_error_uncatcher(code, i+1, old_error_label)
else:
code.use_label(old_label)
code.putln(
"case %s: goto %s;" % (
i+1,
old_label))
code.putln("case %s: goto %s;" % (i+1, old_label))
# End the switch
code.putln(
"}")
# End finally
code.putln(
"}")
......@@ -5479,40 +5522,45 @@ class TryFinallyStatNode(StatNode):
self.body.generate_function_definitions(env, code)
self.finally_clause.generate_function_definitions(env, code)
def put_error_catcher(self, code, error_label, i, catch_label, temps_to_clean_up):
def put_error_catcher(self, code, error_label, i, catch_label,
temps_to_clean_up):
code.globalstate.use_utility_code(restore_exception_utility_code)
code.putln(
"%s: {" %
error_label)
code.putln(
"__pyx_why = %s;" %
i)
code.putln("%s: {" % error_label)
code.putln("__pyx_why = %s;" % i)
if self.is_try_finally_in_nogil:
code.put_ensure_gil(declare_gilstate=False)
for temp_name, type in temps_to_clean_up:
code.put_xdecref_clear(temp_name, type)
code.putln(
"__Pyx_ErrFetch(&%s, &%s, &%s);" %
Naming.exc_vars)
code.putln(
"%s = %s;" % (
Naming.exc_lineno_name, Naming.lineno_cname))
code.putln("__Pyx_ErrFetch(&%s, &%s, &%s);" % Naming.exc_vars)
code.putln("%s = %s;" % (Naming.exc_lineno_name, Naming.lineno_cname))
if self.is_try_finally_in_nogil:
code.put_release_ensured_gil()
code.put_goto(catch_label)
code.putln("}")
def put_error_uncatcher(self, code, i, error_label):
code.globalstate.use_utility_code(restore_exception_utility_code)
code.putln(
"case %s: {" %
i)
code.putln(
"__Pyx_ErrRestore(%s, %s, %s);" %
Naming.exc_vars)
code.putln(
"%s = %s;" % (
Naming.lineno_cname, Naming.exc_lineno_name))
"case %s: {" % i)
if self.is_try_finally_in_nogil:
code.put_ensure_gil(declare_gilstate=False)
code.putln("__Pyx_ErrRestore(%s, %s, %s);" % Naming.exc_vars)
code.putln("%s = %s;" % (Naming.lineno_cname, Naming.exc_lineno_name))
if self.is_try_finally_in_nogil:
code.put_release_ensured_gil()
for var in Naming.exc_vars:
code.putln(
"%s = 0;" %
var)
"%s = 0;" % var)
code.put_goto(error_label)
code.putln(
"}")
......@@ -5522,50 +5570,59 @@ class TryFinallyStatNode(StatNode):
self.finally_clause.annotate(code)
class GILStatNode(TryFinallyStatNode):
class NogilTryFinallyStatNode(TryFinallyStatNode):
"""
A try/finally statement that may be used in nogil code sections.
"""
preserve_exception = False
nogil_check = None
class GILStatNode(NogilTryFinallyStatNode):
# 'with gil' or 'with nogil' statement
#
# state string 'gil' or 'nogil'
# child_attrs = []
preserve_exception = 0
def __init__(self, pos, state, body):
self.state = state
TryFinallyStatNode.__init__(self, pos,
body = body,
finally_clause = GILExitNode(pos, state = state))
def analyse_declarations(self, env):
env._in_with_gil_block = (self.state == 'gil')
if self.state == 'gil':
env.has_with_gil_block = True
return super(GILStatNode, self).analyse_declarations(env)
def analyse_expressions(self, env):
env.use_utility_code(force_init_threads_utility_code)
was_nogil = env.nogil
env.nogil = 1
env.nogil = self.state == 'nogil'
TryFinallyStatNode.analyse_expressions(self, env)
env.nogil = was_nogil
nogil_check = None
def generate_execution_code(self, code):
code.mark_pos(self.pos)
code.putln("{")
code.begin_block()
if self.state == 'gil':
code.putln("#ifdef WITH_THREAD")
code.putln("PyGILState_STATE _save = PyGILState_Ensure();")
code.putln("#endif")
code.put_ensure_gil()
else:
code.putln("#ifdef WITH_THREAD")
code.putln("PyThreadState *_save = NULL;")
code.putln("#endif")
code.putln("Py_UNBLOCK_THREADS")
code.put_release_gil()
TryFinallyStatNode.generate_execution_code(self, code)
code.putln("}")
code.end_block()
class GILExitNode(StatNode):
# Used as the 'finally' block in a GILStatNode
#
# state string 'gil' or 'nogil'
"""
Used as the 'finally' block in a GILStatNode
state string 'gil' or 'nogil'
"""
child_attrs = []
......@@ -5574,11 +5631,18 @@ class GILExitNode(StatNode):
def generate_execution_code(self, code):
if self.state == 'gil':
code.putln("#ifdef WITH_THREAD")
code.putln("PyGILState_Release(_save);")
code.putln("#endif")
code.put_release_ensured_gil()
else:
code.putln("Py_BLOCK_THREADS")
code.put_acquire_gil()
class EnsureGILNode(GILExitNode):
"""
Ensure the GIL in nogil functions for cleanup before returning.
"""
def generate_execution_code(self, code):
code.put_ensure_gil(declare_gilstate=False)
class CImportStatNode(StatNode):
......@@ -7082,15 +7146,22 @@ requires=[raise_double_keywords_utility_code])
#------------------------------------------------------------------------------------
traceback_utility_code = UtilityCode(
proto = """
static void __Pyx_AddTraceback(const char *funcname); /*proto*/
""",
impl = """
proto = """
static void __Pyx_AddTraceback(const char *funcname, int %(CLINENO)s,
int %(LINENO)s, const char *%(FILENAME)s); /*proto*/
""" % {
'FILENAME': Naming.filename_cname,
'LINENO': Naming.lineno_cname,
'CLINENO': Naming.clineno_cname,
},
impl = """
#include "compile.h"
#include "frameobject.h"
#include "traceback.h"
static void __Pyx_AddTraceback(const char *funcname) {
static void __Pyx_AddTraceback(const char *funcname, int %(CLINENO)s,
int %(LINENO)s, const char *%(FILENAME)s) {
PyObject *py_srcfile = 0;
PyObject *py_funcname = 0;
PyObject *py_globals = 0;
......
......@@ -638,6 +638,8 @@ class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
'is not allowed in %s scope' % (directive, scope)))
return False
else:
if directive not in Options.directive_defaults:
error(pos, "Invalid directive: '%s'." % (directive,))
return True
# Set up processing and handle the cython: comments.
......@@ -946,9 +948,9 @@ class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
else:
name, value = directive
if name == 'nogil':
if name in ('nogil', 'gil'):
# special case: in pure mode, "with nogil" spells "with cython.nogil"
node = Nodes.GILStatNode(node.pos, state = "nogil", body = node.body)
node = Nodes.GILStatNode(node.pos, state = name, body = node.body)
return self.visit_Node(node)
if self.check_directive_scope(node.pos, name, 'with statement'):
directive_dict[name] = value
......@@ -1361,6 +1363,18 @@ if VALUE is not None:
else:
error(type_node.pos, "Not a type")
node.body.analyse_declarations(lenv)
if lenv.nogil and lenv.has_with_gil_block:
# Acquire the GIL for cleanup in 'nogil' functions, by wrapping
# the entire function body in try/finally.
# The corresponding release will be taken care of by
# Nodes.FuncDefNode.generate_function_definitions()
node.body = Nodes.NogilTryFinallyStatNode(
node.body.pos,
body = node.body,
finally_clause = Nodes.EnsureGILNode(node.body.pos),
)
self.env_stack.append(lenv)
self.visitchildren(node)
self.env_stack.pop()
......@@ -1532,6 +1546,7 @@ if VALUE is not None:
# ---------------------------------------
return property
class AnalyseExpressionsTransform(CythonTransform):
def visit_ModuleNode(self, node):
......@@ -1553,6 +1568,7 @@ class AnalyseExpressionsTransform(CythonTransform):
self.visitchildren(node)
return node
class ExpandInplaceOperators(EnvTransform):
def visit_InPlaceAssignmentNode(self, node):
......@@ -2016,10 +2032,18 @@ class GilCheck(VisitorTransform):
Call `node.gil_check(env)` on each node to make sure we hold the
GIL when we need it. Raise an error when on Python operations
inside a `nogil` environment.
Additionally, raise exceptions for closely nested with gil or with nogil
statements. The latter would abort Python.
"""
def __call__(self, root):
self.env_stack = [root.scope]
self.nogil = False
# True for 'cdef func() nogil:' functions, as the GIL may be held while
# calling this function (thus contained 'nogil' blocks may be valid).
self.nogil_declarator_only = False
return super(GilCheck, self).__call__(root)
def visit_FuncDefNode(self, node):
......@@ -2027,10 +2051,17 @@ class GilCheck(VisitorTransform):
was_nogil = self.nogil
self.nogil = node.local_scope.nogil
if self.nogil:
self.nogil_declarator_only = True
if self.nogil and node.nogil_check:
node.nogil_check(node.local_scope)
self.visitchildren(node)
# This cannot be nested, so it doesn't need backup/restore
self.nogil_declarator_only = False
self.env_stack.pop()
self.nogil = was_nogil
return node
......@@ -2041,6 +2072,23 @@ class GilCheck(VisitorTransform):
was_nogil = self.nogil
self.nogil = (node.state == 'nogil')
if was_nogil == self.nogil and not self.nogil_declarator_only:
if not was_nogil:
error(node.pos, "Trying to acquire the GIL while it is "
"already held.")
else:
error(node.pos, "Trying to release the GIL while it was "
"previously released.")
if isinstance(node.finally_clause, Nodes.StatListNode):
# The finally clause of the GILStatNode is a GILExitNode,
# which is wrapped in a StatListNode. Just unpack that.
node.finally_clause, = node.finally_clause.stats
if node.state == 'gil':
self.seen_with_gil_statement = True
self.visitchildren(node)
self.nogil = was_nogil
return node
......@@ -2074,6 +2122,28 @@ class GilCheck(VisitorTransform):
self.visitchildren(node)
return node
def visit_TryFinallyStatNode(self, node):
"""
Take care of try/finally statements in nogil code sections. The
'try' must contain a 'with gil:' statement somewhere.
"""
if not self.nogil or isinstance(node, Nodes.GILStatNode):
return self.visit_Node(node)
node.nogil_check = None
node.is_try_finally_in_nogil = True
# First, visit the body and check for errors
self.seen_with_gil_statement = False
self.visitchildren(node.body)
if not self.seen_with_gil_statement:
error(node.pos, "Cannot use try/finally in nogil sections unless "
"it contains a 'with gil' statement.")
self.visitchildren(node.finally_clause)
return node
def visit_Node(self, node):
if self.env_stack and self.nogil and node.nogil_check:
node.nogil_check(self.env_stack[-1])
......
......@@ -1580,7 +1580,7 @@ def p_with_statement(s):
def p_with_items(s):
pos = s.position()
if not s.in_python_file and s.sy == 'IDENT' and s.systring == 'nogil':
if not s.in_python_file and s.sy == 'IDENT' and s.systring in ('nogil', 'gil'):
state = s.systring
s.next()
if s.sy == ',':
......
......@@ -176,6 +176,7 @@ class Entry(object):
buffer_aux = None
prev_entry = None
might_overflow = 0
in_with_gil_block = 0
def __init__(self, name, cname, type, pos = None, init = None):
self.name = name
......@@ -1349,6 +1350,12 @@ class ModuleScope(Scope):
class LocalScope(Scope):
# Does the function have a 'with gil:' block?
has_with_gil_block = False
# Transient attribute, used for symbol table variable declarations
_in_with_gil_block = False
def __init__(self, name, outer_scope, parent_scope = None):
if parent_scope is None:
parent_scope = outer_scope
......@@ -1383,6 +1390,8 @@ class LocalScope(Scope):
entry.init = "0"
entry.init_to_none = (type.is_pyobject or type.is_unspecified) and Options.init_local_none
entry.is_local = 1
entry.in_with_gil_block = self._in_with_gil_block
self.var_entries.append(entry)
return entry
......
......@@ -146,6 +146,7 @@ from cpython.method cimport *
from cpython.weakref cimport *
from cpython.getargs cimport *
from cpython.pythread cimport *
from cpython.pystate cimport *
# Python <= 2.x
from cpython.cobject cimport *
......
# Thread and interpreter state structures and their interfaces
from cpython.ref cimport PyObject
cdef extern from "Python.h":
# We make these an opague types. If the user wants specific attributes,
# they can be declared manually.
ctypedef struct PyInterpreterState:
pass
ctypedef struct PyThreadState:
pass
ctypedef struct PyFrameObject:
pass
# This is not actually a struct, but make sure it can never be coerced to
# an int or used in arithmetic expressions
ctypedef struct PyGILState_STATE
# The type of the trace function registered using PyEval_SetProfile() and
# PyEval_SetTrace().
# Py_tracefunc return -1 when raising an exception, or 0 for success.
ctypedef int (*Py_tracefunc)(PyObject *, PyFrameObject *, int, PyObject *)
# The following values are used for 'what' for tracefunc functions
enum:
PyTrace_CALL
PyTrace_EXCEPTION
PyTrace_LINE
PyTrace_RETURN
PyTrace_C_CALL
PyTrace_C_EXCEPTION
PyTrace_C_RETURN
PyInterpreterState * PyInterpreterState_New()
void PyInterpreterState_Clear(PyInterpreterState *)
void PyInterpreterState_Delete(PyInterpreterState *)
PyThreadState * PyThreadState_New(PyInterpreterState *)
void PyThreadState_Clear(PyThreadState *)
void PyThreadState_Delete(PyThreadState *)
PyThreadState * PyThreadState_Get()
PyThreadState * PyThreadState_Swap(PyThreadState *)
PyObject * PyThreadState_GetDict()
int PyThreadState_SetAsyncExc(long, PyObject *)
# Ensure that the current thread is ready to call the Python
# C API, regardless of the current state of Python, or of its
# thread lock. This may be called as many times as desired
# by a thread so long as each call is matched with a call to
# PyGILState_Release(). In general, other thread-state APIs may
# be used between _Ensure() and _Release() calls, so long as the
# thread-state is restored to its previous state before the Release().
# For example, normal use of the Py_BEGIN_ALLOW_THREADS/
# Py_END_ALLOW_THREADS macros are acceptable.
# The return value is an opaque "handle" to the thread state when
# PyGILState_Ensure() was called, and must be passed to
# PyGILState_Release() to ensure Python is left in the same state. Even
# though recursive calls are allowed, these handles can *not* be shared -
# each unique call to PyGILState_Ensure must save the handle for its
# call to PyGILState_Release.
# When the function returns, the current thread will hold the GIL.
# Failure is a fatal error.
PyGILState_STATE PyGILState_Ensure()
# Release any resources previously acquired. After this call, Python's
# state will be the same as it was prior to the corresponding
# PyGILState_Ensure() call (but generally this state will be unknown to
# the caller, hence the use of the GILState API.)
# Every call to PyGILState_Ensure must be matched by a call to
# PyGILState_Release on the same thread.
void PyGILState_Release(PyGILState_STATE)
# Routines for advanced debuggers, requested by David Beazley.
# Don't use unless you know what you are doing!
PyInterpreterState * PyInterpreterState_Head()
PyInterpreterState * PyInterpreterState_Next(PyInterpreterState *)
PyThreadState * PyInterpreterState_ThreadHead(PyInterpreterState *)
PyThreadState * PyThreadState_Next(PyThreadState *)
from cpython.ref cimport PyObject, Py_INCREF, Py_DECREF, Py_XDECREF
from cpython.exc cimport PyErr_Fetch, PyErr_Restore
from cpython.pystate cimport PyThreadState_Get
loglevel = 0
......@@ -80,6 +81,7 @@ cdef PyObject* SetupContext(char* funcname, int lineno, char* filename) except N
return NULL
cdef PyObject* type = NULL, *value = NULL, *tb = NULL
cdef PyObject* result = NULL
PyThreadState_Get()
PyErr_Fetch(&type, &value, &tb)
try:
ctx = Context(funcname, lineno, filename)
......@@ -131,16 +133,19 @@ cdef void GIVEREF(PyObject* ctx, PyObject* p_obj, int lineno):
cdef void INCREF(PyObject* ctx, PyObject* obj, int lineno):
if obj is not NULL: Py_INCREF(<object>obj)
PyThreadState_Get()
GOTREF(ctx, obj, lineno)
cdef void DECREF(PyObject* ctx, PyObject* obj, int lineno):
if GIVEREF_and_report(ctx, obj, lineno):
if obj is not NULL: Py_DECREF(<object>obj)
PyThreadState_Get()
cdef void FinishContext(PyObject** ctx):
if ctx == NULL or ctx[0] == NULL: return
cdef PyObject* type = NULL, *value = NULL, *tb = NULL
cdef object errors = None
PyThreadState_Get()
PyErr_Fetch(&type, &value, &tb)
try:
try:
......
......@@ -89,6 +89,7 @@ class _nogil(object):
return exc_class is None
nogil = _nogil()
gil = _nogil()
del _nogil
# Emulated types
......
# mode: error
with gil:
pass
with nogil:
with nogil:
pass
cdef void without_gil() nogil:
# This is not an error, as 'func' *may* be called without the GIL, but it
# may also be held.
with nogil:
pass
cdef void with_gil() with gil:
# This is an error, as the GIL is acquired already
with gil:
pass
def func():
with gil:
pass
_ERRORS = u'''
3:5: Trying to acquire the GIL while it is already held.
7:9: Trying to release the GIL while it was previously released.
18:9: Trying to acquire the GIL while it is already held.
22:9: Trying to acquire the GIL while it is already held.
'''
......@@ -158,6 +158,6 @@ _ERRORS = u"""
60:17: Truth-testing Python object not allowed without gil
62:8: For-loop using object bounds or target not allowed without gil
64:8: Try-except statement not allowed without gil
68:8: Try-finally statement not allowed without gil
68:8: Cannot use try/finally in nogil sections unless it contains a 'with gil' statement.
85:8: For-loop using object bounds or target not allowed without gil
"""
......@@ -24,4 +24,4 @@ def test_parallel():
free(buf)
#include "sequential_parallel.pyx"
include "sequential_parallel.pyx"
......@@ -50,7 +50,7 @@ def test_propagation():
>>> test_propagation()
(9, 9, 9, 9, 450, 450)
"""
cdef int i, j, x, y
cdef int i = 0, j = 0, x = 0, y = 0
cdef int sum1 = 0, sum2 = 0
for i in prange(10, nogil=True):
......
"""
Test the 'with gil:' statement.
"""
cimport cython
#from libc.stdio cimport printf, puts
from cpython cimport PyObject, Py_INCREF
import sys
def redirect_stderr(func, *args, **kwargs):
"""
Helper function that redirects stderr to stdout for doctest.
"""
stderr, sys.stderr = sys.stderr, sys.stdout
func(*args, **kwargs)
sys.stderr = stderr
cdef void puts(char *string) with gil:
"""
We need this for doctest, used from nogil sections.
"""
print string.decode('ascii')
# Start with some normal Python functions
def test_simple():
"""
>>> test_simple()
['spam', 'ham']
"""
with nogil:
with gil:
print ['spam', 'ham']
def test_nested_gil_blocks():
"""
>>> test_nested_gil_blocks()
entered outer nogil section
entered outer gil section
entered inner nogil section
entered inner gil section
leaving inner gil section
leaving inner nogil section
leaving outer gil section
leaving outer nogil section
"""
with nogil:
puts("entered outer nogil section")
with gil:
print 'entered outer gil section'
with nogil:
puts("entered inner nogil section")
with gil:
print 'entered inner gil section'
print 'leaving inner gil section'
puts("leaving inner nogil section")
print "leaving outer gil section"
puts("leaving outer nogil section")
def test_propagate_exception():
"""
>>> test_propagate_exception()
Traceback (most recent call last):
...
Exception: This exception propagates!
"""
# Note, doctest doesn't support both output and exceptions
with nogil:
with gil:
raise Exception("This exception propagates!")
def test_catch_exception():
"""
>>> test_catch_exception()
This is executed
Exception value
This is also executed
"""
try:
with nogil:
with gil:
print "This is executed"
raise Exception("Exception value")
print "This is not executed"
puts("This is also not executed")
except Exception, e:
print e
print "This is also executed"
def test_try_finally_and_outer_except():
"""
>>> test_try_finally_and_outer_except()
First finally clause
Second finally clause
Caught: Some Exception
End of function
"""
try:
with nogil:
with gil:
try:
with nogil:
with gil:
try:
raise Exception("Some Exception")
finally:
puts("First finally clause")
finally:
puts("Second finally clause")
puts("This is not executed")
except Exception, e:
print "Caught:", e
print "End of function"
def test_restore_exception():
"""
>>> test_restore_exception()
Traceback (most recent call last):
...
Exception: Override the raised exception
"""
with nogil:
with gil:
try:
with nogil:
with gil:
raise Exception("Override this please")
finally:
raise Exception("Override the raised exception")
def test_declared_variables():
"""
>>> test_declared_variables()
None
None
['s', 'p', 'a', 'm']
['s', 'p', 'a', 'm']
"""
cdef object somevar
print somevar
with nogil:
with gil:
print somevar
somevar = list("spam")
print somevar
print somevar
def test_undeclared_variables():
"""
>>> test_undeclared_variables()
None
None
['s', 'p', 'a', 'm']
['s', 'p', 'a', 'm']
"""
print somevar
with nogil:
with gil:
print somevar
somevar = list("spam")
print somevar
print somevar
def test_loops_and_boxing():
"""
>>> test_loops_and_boxing()
spamham
h
a
m
done looping
"""
cdef char c, *string = "spamham"
with nogil:
with gil:
print string
for c in string[4:]:
print "%c" % c
else:
print "done looping"
cdef class SomeExtClass(object):
cdef int some_attribute
@cython.infer_types(True)
def test_infer_types():
"""
>>> test_infer_types()
10
"""
with nogil:
with gil:
obj = SomeExtClass()
obj.some_attribute = 10
print obj.some_attribute
def test_closure():
"""
>>> test_closure()
Traceback (most recent call last):
...
Exception: {'twinkle': 'little star'}
"""
a = dict(twinkle='little star')
def inner_function():
with nogil:
with gil:
raise Exception(a)
with nogil:
with gil:
inner_function()
raise Exception("This should not be raised!")
cpdef test_cpdef():
"""
>>> test_cpdef()
Seems to work!
Or does it?
"""
with nogil:
with gil:
print "Seems to work!"
puts("Or does it?")
# Now test some cdef functions with different return types
cdef void void_nogil_ignore_exception() nogil:
with gil:
raise Exception("This is swallowed")
puts("unreachable")
with gil:
print "unreachable"
cdef void void_nogil_nested_gil() nogil:
with gil:
with nogil:
with gil:
print 'Inner gil section'
puts("nogil section")
raise Exception("Swallow this")
puts("Don't print this")
def test_nogil_void_funcs_with_gil():
"""
>>> redirect_stderr(test_nogil_void_funcs_with_gil)
Exception Exception: Exception('This is swallowed',) in 'with_gil.void_nogil_ignore_exception' ignored
Inner gil section
nogil section
Exception Exception: Exception('Swallow this',) in 'with_gil.void_nogil_nested_gil' ignored
"""
void_nogil_ignore_exception()
void_nogil_nested_gil()
def test_nogil_void_funcs_with_nogil():
"""
>>> redirect_stderr(test_nogil_void_funcs_with_nogil)
Exception Exception: Exception('This is swallowed',) in 'with_gil.void_nogil_ignore_exception' ignored
Inner gil section
nogil section
Exception Exception: Exception('Swallow this',) in 'with_gil.void_nogil_nested_gil' ignored
"""
with nogil:
void_nogil_ignore_exception()
void_nogil_nested_gil()
cdef PyObject *nogil_propagate_exception() nogil except NULL:
with nogil:
with gil:
raise Exception("This exception propagates!")
return <PyObject *> 1
def test_nogil_propagate_exception():
"""
>>> test_nogil_propagate_exception()
Traceback (most recent call last):
...
Exception: This exception propagates!
"""
nogil_propagate_exception()
cdef with_gil_raise() with gil:
raise Exception("This exception propagates!")
def test_release_gil_call_gil_func():
"""
>>> test_release_gil_call_gil_func()
Traceback (most recent call last):
...
Exception: This exception propagates!
"""
with nogil:
with gil:
with_gil_raise()
# Test try/finally in nogil blocks
def test_try_finally_in_nogil():
"""
>>> test_try_finally_in_nogil()
Traceback (most recent call last):
...
Exception: Override exception!
"""
with nogil:
try:
with gil:
raise Exception("This will be overridden")
finally:
with gil:
raise Exception("Override exception!")
with gil:
raise Exception("This code should not be executed!")
def test_nogil_try_finally_no_exception():
"""
>>> test_nogil_try_finally_no_exception()
first nogil try
nogil try gil
second nogil try
nogil finally
------
First with gil block
Second with gil block
finally block
"""
with nogil:
try:
puts("first nogil try")
with gil:
print "nogil try gil"
puts("second nogil try")
finally:
puts("nogil finally")
print '------'
with nogil:
try:
with gil:
print "First with gil block"
with gil:
print "Second with gil block"
finally:
puts("finally block")
def test_nogil_try_finally_propagate_exception():
"""
>>> test_nogil_try_finally_propagate_exception()
Execute finally clause
Propagate this!
"""
try:
with nogil:
try:
with gil:
raise Exception("Propagate this!")
with gil:
raise Exception("Don't reach this section!")
finally:
puts("Execute finally clause")
except Exception, e:
print e
def test_nogil_try_finally_return_in_with_gil(x):
"""
>>> test_nogil_try_finally_return_in_with_gil(10)
print me
10
"""
with nogil:
try:
with gil:
raise Exception("Swallow me!")
finally:
with gil:
print "print me"
return x
print "I am not executed"
cdef void nogil_try_finally_return() nogil:
try:
with gil:
raise Exception("I am swallowed in nogil code... right?")
finally:
with gil:
print "print me first"
return
with gil:
print "I am not executed"
def test_nogil_try_finally_return():
"""
>>> test_nogil_try_finally_return()
print me first
"""
with nogil:
nogil_try_finally_return()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment