Skip to content

C
cython
Commit 3b6ec99c authored by Stefan Behnel's avatar Stefan Behnel
Browse files
Options

enable tracing for nogil functions/sections

parent 02d86d70
Hide whitespace changes
Inline Side-by-side
Showing with 133 additions and 57 deletions
CHANGES.rst
View file @ 3b6ec99c
......@@ -11,6 +11,8 @@ Features added
* Support for coverage.py 4.0+ can be enabled by adding the plugin
"Cython.Coverage" to the ".coveragerc" config file.
* Tracing is supported in ``nogil`` functions/sections.
Bugs fixed
----------
......
Cython/Compiler/Code.pxd
View file @ 3b6ec99c
......@@ -31,9 +31,10 @@ cdef class FunctionState:
cdef public object return_from_error_cleanup_label # not used in __init__ ?
cdef public bint in_try_finally
cdef public object exc_vars
cdef public bint in_try_finally
cdef public bint can_trace
cdef public bint gil_owned
cdef public list temps_allocated
cdef public dict temps_free
......
Cython/Compiler/Code.py
View file @ 3b6ec99c
......@@ -518,6 +518,7 @@ class FunctionState(object):
self.in_try_finally = 0
self.exc_vars = None
self.can_trace = False
self.gil_owned = True
self.temps_allocated = [] # of (name, type, manage_ref, static)
self.temps_free = {} # (type, manage_ref) -> list of free vars with same type/managed status
......@@ -1572,7 +1573,8 @@ class CCodeWriter(object):
if (self.funcstate and self.funcstate.can_trace
and self.globalstate.directives['linetrace']):
self.indent()
self.write('__Pyx_TraceLine(%d)\n' % self.marker[0])
self.write('__Pyx_TraceLine(%d,%d)\n' % (
self.marker[0], not self.funcstate.gil_owned))
self.last_marker_line = self.marker[0]
self.marker = None
......@@ -2093,17 +2095,18 @@ class CCodeWriter(object):
self.globalstate.use_utility_code(
UtilityCode.load_cached("WriteUnraisableException", "Exceptions.c"))
def put_trace_declarations(self, codeobj=None):
self.putln('__Pyx_TraceDeclarations(%s)' % (codeobj or 'NULL'))
def put_trace_declarations(self, codeobj=None, nogil=False):
self.putln('__Pyx_TraceDeclarations(%s, %d)' % (codeobj or 'NULL', nogil))
def put_trace_call(self, name, pos):
self.putln('__Pyx_TraceCall("%s", %s[%s], %s);' % (name, Naming.filetable_cname, self.lookup_filename(pos[0]), pos[1]))
def put_trace_call(self, name, pos, nogil=False):
self.putln('__Pyx_TraceCall("%s", %s[%s], %s, %d);' % (
name, Naming.filetable_cname, self.lookup_filename(pos[0]), pos[1], nogil))
def put_trace_exception(self):
self.putln("__Pyx_TraceException();")
def put_trace_return(self, retvalue_cname):
self.putln("__Pyx_TraceReturn(%s);" % retvalue_cname)
def put_trace_return(self, retvalue_cname, nogil=False):
self.putln("__Pyx_TraceReturn(%s, %d);" % (retvalue_cname, nogil))
def putln_openmp(self, string):
self.putln("#ifdef _OPENMP")
......
Cython/Compiler/Nodes.py
View file @ 3b6ec99c
......@@ -1698,9 +1698,6 @@ class FuncDefNode(StatNode, BlockNode):
profile = code.globalstate.directives['profile']
linetrace = code.globalstate.directives['linetrace']
if (linetrace or profile) and lenv.nogil:
warning(self.pos, "Cannot profile nogil function.", 1)
profile = linetrace = False
if profile or linetrace:
code.globalstate.use_utility_code(
UtilityCode.load_cached("Profile", "Profile.c"))
......@@ -1708,6 +1705,7 @@ class FuncDefNode(StatNode, BlockNode):
# Generate C code for header and body of function
code.enter_cfunc_scope()
code.return_from_error_cleanup_label = code.new_label()
code.funcstate.gil_owned = not lenv.nogil
# ----- Top-level constants used by this function
code.mark_pos(self.pos)
......@@ -1764,7 +1762,7 @@ class FuncDefNode(StatNode, BlockNode):
if profile or linetrace:
code_object = self.code_object.calculate_result_code(code) if self.code_object else None
code.put_trace_declarations(code_object)
code.put_trace_declarations(code_object, nogil=not code.funcstate.gil_owned)
# ----- Extern library function declarations
lenv.generate_library_function_declarations(code)
......@@ -1775,10 +1773,9 @@ class FuncDefNode(StatNode, BlockNode):
# See if we need to acquire the GIL for variable declarations, or for
# refnanny only
# Profiling or closures are not currently possible for cdef nogil
# functions, but check them anyway
have_object_args = (self.needs_closure or self.needs_outer_scope or
profile or linetrace)
# Closures are not currently possible for cdef nogil functions,
# but check them anyway
have_object_args = self.needs_closure or self.needs_outer_scope
for arg in lenv.arg_entries:
if arg.type.is_pyobject:
have_object_args = True
......@@ -1796,6 +1793,7 @@ class FuncDefNode(StatNode, BlockNode):
if acquire_gil or acquire_gil_for_var_decls_only:
code.put_ensure_gil()
code.funcstate.gil_owned = True
elif lenv.nogil and lenv.has_with_gil_block:
code.declare_gilstate()
......@@ -1855,7 +1853,7 @@ class FuncDefNode(StatNode, BlockNode):
if profile or linetrace:
# this looks a bit late, but if we don't get here due to a
# fatal error before hand, it's not really worth tracing
code.put_trace_call(self.entry.name, self.pos)
code.put_trace_call(self.entry.name, self.pos, nogil=not code.funcstate.gil_owned)
code.funcstate.can_trace = True
# ----- Fetch arguments
self.generate_argument_parsing_code(env, code)
......@@ -1874,8 +1872,7 @@ class FuncDefNode(StatNode, BlockNode):
# incref our arguments
elif (is_cdef and entry.type.is_memoryviewslice and
len(entry.cf_assignments) > 1):
code.put_incref_memoryviewslice(entry.cname,
have_gil=not lenv.nogil)
code.put_incref_memoryviewslice(entry.cname, have_gil=code.funcstate.gil_owned)
for entry in lenv.var_entries:
if entry.is_arg and len(entry.cf_assignments) > 1:
code.put_var_incref(entry)
......@@ -1894,6 +1891,7 @@ class FuncDefNode(StatNode, BlockNode):
if acquire_gil_for_var_decls_only:
code.put_release_ensured_gil()
code.funcstate.gil_owned = False
# -------------------------
# ----- Function body -----
......@@ -2054,9 +2052,9 @@ class FuncDefNode(StatNode, BlockNode):
if profile or linetrace:
code.funcstate.can_trace = False
if self.return_type.is_pyobject:
code.put_trace_return(Naming.retval_cname)
code.put_trace_return(Naming.retval_cname, nogil=not code.funcstate.gil_owned)
else:
code.put_trace_return("Py_None")
code.put_trace_return("Py_None", nogil=not code.funcstate.gil_owned)
if not lenv.nogil:
# GIL holding function
......@@ -2065,6 +2063,7 @@ class FuncDefNode(StatNode, BlockNode):
if acquire_gil or (lenv.nogil and lenv.has_with_gil_block):
# release the GIL (note that with-gil blocks acquire it on exit in their EnsureGILNode)
code.put_release_ensured_gil()
code.funcstate.gil_owned = False
if not self.return_type.is_void:
code.putln("return %s;" % Naming.retval_cname)
......@@ -7073,21 +7072,20 @@ class GILStatNode(NogilTryFinallyStatNode):
else:
variable = None
old_trace_config = code.funcstate.can_trace
old_gil_config = code.funcstate.gil_owned
if self.state == 'gil':
code.put_ensure_gil(variable=variable)
# FIXME: not that easy, tracing may not be possible at all here
#code.funcstate.can_trace = True
code.funcstate.gil_owned = True
else:
code.put_release_gil(variable=variable)
code.funcstate.can_trace = False
code.funcstate.gil_owned = False
TryFinallyStatNode.generate_execution_code(self, code)
if self.state_temp:
self.state_temp.release(code)
code.funcstate.can_trace = old_trace_config
code.funcstate.gil_owned = old_gil_config
code.end_block()
......
Cython/Utility/Profile.c
View file @ 3b6ec99c
......@@ -12,6 +12,10 @@
#define CYTHON_TRACE 0
#endif
#ifndef CYTHON_TRACE_NOGIL
#define CYTHON_TRACE_NOGIL 0
#endif
#if CYTHON_TRACE
#undef CYTHON_PROFILE_REUSE_FRAME
#endif
......@@ -28,25 +32,47 @@
#if CYTHON_PROFILE_REUSE_FRAME
#define CYTHON_FRAME_MODIFIER static
#define CYTHON_FRAME_DEL
#define CYTHON_FRAME_DEL(frame)
#else
#define CYTHON_FRAME_MODIFIER
#define CYTHON_FRAME_DEL Py_CLEAR($frame_cname)
#define CYTHON_FRAME_DEL(frame) Py_CLEAR(frame)
#endif
#define __Pyx_TraceDeclarations(codeobj) \
#define __Pyx_TraceDeclarations(codeobj, nogil) \
static PyCodeObject *$frame_code_cname = NULL; \
CYTHON_FRAME_MODIFIER PyFrameObject *$frame_cname = NULL; \
int __Pyx_use_tracing = 0; \
if (codeobj) $frame_code_cname = (PyCodeObject*) codeobj;
#define __Pyx_TraceCall(funcname, srcfile, firstlineno) \
{ PyThreadState* tstate = PyThreadState_GET(); \
if (unlikely(tstate->use_tracing) && !tstate->tracing && \
(tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) { \
#ifdef WITH_THREAD
#define __Pyx_TraceCall(funcname, srcfile, firstlineno, nogil) \
if (nogil) { \
if (CYTHON_TRACE_NOGIL) { \
PyThreadState *tstate; \
PyGILState_STATE state = PyGILState_Ensure(); \
tstate = PyThreadState_GET(); \
if (unlikely(tstate->use_tracing) && !tstate->tracing && \
(tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) { \
__Pyx_use_tracing = __Pyx_TraceSetupAndCall(&$frame_code_cname, &$frame_cname, funcname, srcfile, firstlineno); \
} \
PyGILState_Release(state); \
} \
} else { \
PyThreadState* tstate = PyThreadState_GET(); \
if (unlikely(tstate->use_tracing) && !tstate->tracing && \
(tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) { \
__Pyx_use_tracing = __Pyx_TraceSetupAndCall(&$frame_code_cname, &$frame_cname, funcname, srcfile, firstlineno); \
} \
} \
}
#else
#define __Pyx_TraceCall(funcname, srcfile, firstlineno, nogil) \
{ PyThreadState* tstate = PyThreadState_GET(); \
if (unlikely(tstate->use_tracing) && !tstate->tracing && \
(tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) { \
__Pyx_use_tracing = __Pyx_TraceSetupAndCall(&$frame_code_cname, &$frame_cname, funcname, srcfile, firstlineno); \
} \
}
#endif
#define __Pyx_TraceException() \
if (likely(!__Pyx_use_tracing)); else { \
......@@ -69,36 +95,60 @@
} \
}
#define __Pyx_TraceReturn(result) \
static void __Pyx_call_return_trace_func(PyThreadState *tstate, PyFrameObject *frame, PyObject *result) {
PyObject *type, *value, *traceback;
PyErr_Fetch(&type, &value, &traceback);
tstate->tracing++;
tstate->use_tracing = 0;
if (CYTHON_TRACE && tstate->c_tracefunc)
tstate->c_tracefunc(tstate->c_traceobj, frame, PyTrace_RETURN, result);
if (tstate->c_profilefunc)
tstate->c_profilefunc(tstate->c_profileobj, frame, PyTrace_RETURN, result);
CYTHON_FRAME_DEL(frame);
tstate->use_tracing = 1;
tstate->tracing--;
PyErr_Restore(type, value, traceback);
}
#ifdef WITH_THREAD
#define __Pyx_TraceReturn(result, nogil) \
if (likely(!__Pyx_use_tracing)); else { \
if (nogil) { \
if (CYTHON_TRACE_NOGIL) { \
PyThreadState *tstate; \
PyGILState_STATE state = PyGILState_Ensure(); \
tstate = PyThreadState_GET(); \
if (tstate->use_tracing) { \
__Pyx_call_return_trace_func(tstate, $frame_cname, (PyObject*)result); \
} \
PyGILState_Release(state); \
} \
} else { \
PyThreadState* tstate = PyThreadState_GET(); \
if (tstate->use_tracing) { \
__Pyx_call_return_trace_func(tstate, $frame_cname, (PyObject*)result); \
} \
} \
}
#else
#define __Pyx_TraceReturn(result, nogil) \
if (likely(!__Pyx_use_tracing)); else { \
PyThreadState* tstate = PyThreadState_GET(); \
if (tstate->use_tracing) { \
PyObject *type, *value, *traceback; \
PyErr_Fetch(&type, &value, &traceback); \
tstate->tracing++; \
tstate->use_tracing = 0; \
if (CYTHON_TRACE && tstate->c_tracefunc) \
tstate->c_tracefunc( \
tstate->c_traceobj, $frame_cname, PyTrace_RETURN, (PyObject*)result); \
if (tstate->c_profilefunc) \
tstate->c_profilefunc( \
tstate->c_profileobj, $frame_cname, PyTrace_RETURN, (PyObject*)result); \
CYTHON_FRAME_DEL; \
tstate->use_tracing = 1; \
tstate->tracing--; \
PyErr_Restore(type, value, traceback); \
__Pyx_call_return_trace_func(tstate, $frame_cname, (PyObject*)result); \
} \
}
#endif
static PyCodeObject *__Pyx_createFrameCodeObject(const char *funcname, const char *srcfile, int firstlineno); /*proto*/
static int __Pyx_TraceSetupAndCall(PyCodeObject** code, PyFrameObject** frame, const char *funcname, const char *srcfile, int firstlineno); /*proto*/
#else
#define __Pyx_TraceDeclarations(codeobj)
#define __Pyx_TraceCall(funcname, srcfile, firstlineno)
#define __Pyx_TraceDeclarations(codeobj, nogil)
#define __Pyx_TraceCall(funcname, srcfile, firstlineno, nogil)
#define __Pyx_TraceException()
#define __Pyx_TraceReturn(result)
#define __Pyx_TraceReturn(result, nogil)
#endif /* CYTHON_PROFILE */
......@@ -117,15 +167,37 @@
PyErr_Restore(type, value, traceback);
}
#define __Pyx_TraceLine(lineno) \
#ifdef WITH_THREAD
#define __Pyx_TraceLine(lineno, nogil) \
if (likely(!__Pyx_use_tracing)); else { \
if (nogil) { \
if (CYTHON_TRACE_NOGIL) { \
PyThreadState *tstate; \
PyGILState_STATE state = PyGILState_Ensure(); \
tstate = PyThreadState_GET(); \
if (unlikely(tstate->use_tracing && tstate->c_tracefunc)) { \
__Pyx_call_line_trace_func(tstate, $frame_cname, lineno); \
} \
PyGILState_Release(state); \
} \
} else { \
PyThreadState* tstate = PyThreadState_GET(); \
if (unlikely(tstate->use_tracing && tstate->c_tracefunc)) { \
__Pyx_call_line_trace_func(tstate, $frame_cname, lineno); \
} \
} \
}
#else
#define __Pyx_TraceLine(lineno, nogil) \
if (likely(!__Pyx_use_tracing)); else { \
PyThreadState* tstate = PyThreadState_GET(); \
if (unlikely(tstate->use_tracing && tstate->c_tracefunc)) { \
__Pyx_call_line_trace_func(tstate, $frame_cname, lineno); \
} \
}
#endif
#else
#define __Pyx_TraceLine(lineno)
#define __Pyx_TraceLine(lineno, nogil)
#endif
/////////////// Profile ///////////////
......
tests/run/coverage_nogil.srctree
View file @ 3b6ec99c
......@@ -23,7 +23,7 @@ plugins = Cython.Coverage
######## coverage_test_nogil.pyx ########
# cython: linetrace=True
# distutils: define_macros=CYTHON_TRACE=1
# distutils: define_macros=CYTHON_TRACE=1 CYTHON_TRACE_NOGIL=1
cdef int func1(int a, int b) nogil:
cdef int x # 5
......@@ -85,8 +85,8 @@ def run_coverage(module):
executed = set(exec_lines) - set(missing_lines)
# check that everything that runs with the gil owned was executed
assert all(line in executed for line in [13, 17, 18, 20]), '%s / %s' % (exec_lines, missing_lines)
# currently, we do not trace nogil code lines, but that should eventually be implemented
# we also don't trace 'with gil' blocks in 'nogil' functions
# check that everything that runs in nogil sections was executed
assert all(line in executed for line in [6, 7, 8, 9]), '%s / %s' % (exec_lines, missing_lines)
if __name__ == '__main__':
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7