Skip to content

C
cython
Commit 3b6ec99c authored by Stefan Behnel's avatar Stefan Behnel
Browse files
Options

enable tracing for nogil functions/sections

parent 02d86d70
Hide whitespace changes
Inline Side-by-side
Showing with 133 additions and 57 deletions
CHANGES.rst
View file @ 3b6ec99c
...@@ -11,6 +11,8 @@ Features added ...@@ -11,6 +11,8 @@ Features added
* Support for coverage.py 4.0+ can be enabled by adding the plugin * Support for coverage.py 4.0+ can be enabled by adding the plugin
"Cython.Coverage" to the ".coveragerc" config file. "Cython.Coverage" to the ".coveragerc" config file.
* Tracing is supported in ``nogil`` functions/sections.
Bugs fixed Bugs fixed
---------- ----------
......
Cython/Compiler/Code.pxd
View file @ 3b6ec99c
...@@ -31,9 +31,10 @@ cdef class FunctionState: ...@@ -31,9 +31,10 @@ cdef class FunctionState:
cdef public object return_from_error_cleanup_label # not used in __init__ ? cdef public object return_from_error_cleanup_label # not used in __init__ ?
cdef public bint in_try_finally
cdef public object exc_vars cdef public object exc_vars
cdef public bint in_try_finally
cdef public bint can_trace cdef public bint can_trace
cdef public bint gil_owned
cdef public list temps_allocated cdef public list temps_allocated
cdef public dict temps_free cdef public dict temps_free
......
Cython/Compiler/Code.py
View file @ 3b6ec99c
...@@ -518,6 +518,7 @@ class FunctionState(object): ...@@ -518,6 +518,7 @@ class FunctionState(object):
self.in_try_finally = 0 self.in_try_finally = 0
self.exc_vars = None self.exc_vars = None
self.can_trace = False self.can_trace = False
self.gil_owned = True
self.temps_allocated = [] # of (name, type, manage_ref, static) self.temps_allocated = [] # of (name, type, manage_ref, static)
self.temps_free = {} # (type, manage_ref) -> list of free vars with same type/managed status self.temps_free = {} # (type, manage_ref) -> list of free vars with same type/managed status
...@@ -1572,7 +1573,8 @@ class CCodeWriter(object): ...@@ -1572,7 +1573,8 @@ class CCodeWriter(object):
if (self.funcstate and self.funcstate.can_trace if (self.funcstate and self.funcstate.can_trace
and self.globalstate.directives['linetrace']): and self.globalstate.directives['linetrace']):
self.indent() self.indent()
self.write('__Pyx_TraceLine(%d)\n' % self.marker[0]) self.write('__Pyx_TraceLine(%d,%d)\n' % (
self.marker[0], not self.funcstate.gil_owned))
self.last_marker_line = self.marker[0] self.last_marker_line = self.marker[0]
self.marker = None self.marker = None
...@@ -2093,17 +2095,18 @@ class CCodeWriter(object): ...@@ -2093,17 +2095,18 @@ class CCodeWriter(object):
self.globalstate.use_utility_code( self.globalstate.use_utility_code(
UtilityCode.load_cached("WriteUnraisableException", "Exceptions.c")) UtilityCode.load_cached("WriteUnraisableException", "Exceptions.c"))
def put_trace_declarations(self, codeobj=None): def put_trace_declarations(self, codeobj=None, nogil=False):
self.putln('__Pyx_TraceDeclarations(%s)' % (codeobj or 'NULL')) self.putln('__Pyx_TraceDeclarations(%s, %d)' % (codeobj or 'NULL', nogil))
def put_trace_call(self, name, pos): def put_trace_call(self, name, pos, nogil=False):
self.putln('__Pyx_TraceCall("%s", %s[%s], %s);' % (name, Naming.filetable_cname, self.lookup_filename(pos[0]), pos[1])) self.putln('__Pyx_TraceCall("%s", %s[%s], %s, %d);' % (
name, Naming.filetable_cname, self.lookup_filename(pos[0]), pos[1], nogil))
def put_trace_exception(self): def put_trace_exception(self):
self.putln("__Pyx_TraceException();") self.putln("__Pyx_TraceException();")
def put_trace_return(self, retvalue_cname): def put_trace_return(self, retvalue_cname, nogil=False):
self.putln("__Pyx_TraceReturn(%s);" % retvalue_cname) self.putln("__Pyx_TraceReturn(%s, %d);" % (retvalue_cname, nogil))
def putln_openmp(self, string): def putln_openmp(self, string):
self.putln("#ifdef _OPENMP") self.putln("#ifdef _OPENMP")
......
Cython/Compiler/Nodes.py
View file @ 3b6ec99c
...@@ -1698,9 +1698,6 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1698,9 +1698,6 @@ class FuncDefNode(StatNode, BlockNode):
profile = code.globalstate.directives['profile'] profile = code.globalstate.directives['profile']
linetrace = code.globalstate.directives['linetrace'] linetrace = code.globalstate.directives['linetrace']
if (linetrace or profile) and lenv.nogil:
warning(self.pos, "Cannot profile nogil function.", 1)
profile = linetrace = False
if profile or linetrace: if profile or linetrace:
code.globalstate.use_utility_code( code.globalstate.use_utility_code(
UtilityCode.load_cached("Profile", "Profile.c")) UtilityCode.load_cached("Profile", "Profile.c"))
...@@ -1708,6 +1705,7 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1708,6 +1705,7 @@ class FuncDefNode(StatNode, BlockNode):
# Generate C code for header and body of function # Generate C code for header and body of function
code.enter_cfunc_scope() code.enter_cfunc_scope()
code.return_from_error_cleanup_label = code.new_label() code.return_from_error_cleanup_label = code.new_label()
code.funcstate.gil_owned = not lenv.nogil
# ----- Top-level constants used by this function # ----- Top-level constants used by this function
code.mark_pos(self.pos) code.mark_pos(self.pos)
...@@ -1764,7 +1762,7 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1764,7 +1762,7 @@ class FuncDefNode(StatNode, BlockNode):
if profile or linetrace: if profile or linetrace:
code_object = self.code_object.calculate_result_code(code) if self.code_object else None code_object = self.code_object.calculate_result_code(code) if self.code_object else None
code.put_trace_declarations(code_object) code.put_trace_declarations(code_object, nogil=not code.funcstate.gil_owned)
# ----- Extern library function declarations # ----- Extern library function declarations
lenv.generate_library_function_declarations(code) lenv.generate_library_function_declarations(code)
...@@ -1775,10 +1773,9 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1775,10 +1773,9 @@ class FuncDefNode(StatNode, BlockNode):
# See if we need to acquire the GIL for variable declarations, or for # See if we need to acquire the GIL for variable declarations, or for
# refnanny only # refnanny only
# Profiling or closures are not currently possible for cdef nogil # Closures are not currently possible for cdef nogil functions,
# functions, but check them anyway # but check them anyway
have_object_args = (self.needs_closure or self.needs_outer_scope or have_object_args = self.needs_closure or self.needs_outer_scope
profile or linetrace)
for arg in lenv.arg_entries: for arg in lenv.arg_entries:
if arg.type.is_pyobject: if arg.type.is_pyobject:
have_object_args = True have_object_args = True
...@@ -1796,6 +1793,7 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1796,6 +1793,7 @@ class FuncDefNode(StatNode, BlockNode):
if acquire_gil or acquire_gil_for_var_decls_only: if acquire_gil or acquire_gil_for_var_decls_only:
code.put_ensure_gil() code.put_ensure_gil()
code.funcstate.gil_owned = True
elif lenv.nogil and lenv.has_with_gil_block: elif lenv.nogil and lenv.has_with_gil_block:
code.declare_gilstate() code.declare_gilstate()
...@@ -1855,7 +1853,7 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1855,7 +1853,7 @@ class FuncDefNode(StatNode, BlockNode):
if profile or linetrace: if profile or linetrace:
# this looks a bit late, but if we don't get here due to a # this looks a bit late, but if we don't get here due to a
# fatal error before hand, it's not really worth tracing # fatal error before hand, it's not really worth tracing
code.put_trace_call(self.entry.name, self.pos) code.put_trace_call(self.entry.name, self.pos, nogil=not code.funcstate.gil_owned)
code.funcstate.can_trace = True code.funcstate.can_trace = True
# ----- Fetch arguments # ----- Fetch arguments
self.generate_argument_parsing_code(env, code) self.generate_argument_parsing_code(env, code)
...@@ -1874,8 +1872,7 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1874,8 +1872,7 @@ class FuncDefNode(StatNode, BlockNode):
# incref our arguments # incref our arguments
elif (is_cdef and entry.type.is_memoryviewslice and elif (is_cdef and entry.type.is_memoryviewslice and
len(entry.cf_assignments) > 1): len(entry.cf_assignments) > 1):
code.put_incref_memoryviewslice(entry.cname, code.put_incref_memoryviewslice(entry.cname, have_gil=code.funcstate.gil_owned)
have_gil=not lenv.nogil)
for entry in lenv.var_entries: for entry in lenv.var_entries:
if entry.is_arg and len(entry.cf_assignments) > 1: if entry.is_arg and len(entry.cf_assignments) > 1:
code.put_var_incref(entry) code.put_var_incref(entry)
...@@ -1894,6 +1891,7 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1894,6 +1891,7 @@ class FuncDefNode(StatNode, BlockNode):
if acquire_gil_for_var_decls_only: if acquire_gil_for_var_decls_only:
code.put_release_ensured_gil() code.put_release_ensured_gil()
code.funcstate.gil_owned = False
# ------------------------- # -------------------------
# ----- Function body ----- # ----- Function body -----
...@@ -2054,9 +2052,9 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -2054,9 +2052,9 @@ class FuncDefNode(StatNode, BlockNode):
if profile or linetrace: if profile or linetrace:
code.funcstate.can_trace = False code.funcstate.can_trace = False
if self.return_type.is_pyobject: if self.return_type.is_pyobject:
code.put_trace_return(Naming.retval_cname) code.put_trace_return(Naming.retval_cname, nogil=not code.funcstate.gil_owned)
else: else:
code.put_trace_return("Py_None") code.put_trace_return("Py_None", nogil=not code.funcstate.gil_owned)
if not lenv.nogil: if not lenv.nogil:
# GIL holding function # GIL holding function
...@@ -2065,6 +2063,7 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -2065,6 +2063,7 @@ class FuncDefNode(StatNode, BlockNode):
if acquire_gil or (lenv.nogil and lenv.has_with_gil_block): if acquire_gil or (lenv.nogil and lenv.has_with_gil_block):
# release the GIL (note that with-gil blocks acquire it on exit in their EnsureGILNode) # release the GIL (note that with-gil blocks acquire it on exit in their EnsureGILNode)
code.put_release_ensured_gil() code.put_release_ensured_gil()
code.funcstate.gil_owned = False
if not self.return_type.is_void: if not self.return_type.is_void:
code.putln("return %s;" % Naming.retval_cname) code.putln("return %s;" % Naming.retval_cname)
...@@ -7073,21 +7072,20 @@ class GILStatNode(NogilTryFinallyStatNode): ...@@ -7073,21 +7072,20 @@ class GILStatNode(NogilTryFinallyStatNode):
else: else:
variable = None variable = None
old_trace_config = code.funcstate.can_trace old_gil_config = code.funcstate.gil_owned
if self.state == 'gil': if self.state == 'gil':
code.put_ensure_gil(variable=variable) code.put_ensure_gil(variable=variable)
# FIXME: not that easy, tracing may not be possible at all here code.funcstate.gil_owned = True
#code.funcstate.can_trace = True
else: else:
code.put_release_gil(variable=variable) code.put_release_gil(variable=variable)
code.funcstate.can_trace = False code.funcstate.gil_owned = False
TryFinallyStatNode.generate_execution_code(self, code) TryFinallyStatNode.generate_execution_code(self, code)
if self.state_temp: if self.state_temp:
self.state_temp.release(code) self.state_temp.release(code)
code.funcstate.can_trace = old_trace_config code.funcstate.gil_owned = old_gil_config
code.end_block() code.end_block()
......
Cython/Utility/Profile.c
View file @ 3b6ec99c
...@@ -12,6 +12,10 @@ ...@@ -12,6 +12,10 @@
#define CYTHON_TRACE 0 #define CYTHON_TRACE 0
#endif #endif
#ifndef CYTHON_TRACE_NOGIL
#define CYTHON_TRACE_NOGIL 0
#endif
#if CYTHON_TRACE #if CYTHON_TRACE
#undef CYTHON_PROFILE_REUSE_FRAME #undef CYTHON_PROFILE_REUSE_FRAME
#endif #endif
...@@ -28,25 +32,47 @@ ...@@ -28,25 +32,47 @@
#if CYTHON_PROFILE_REUSE_FRAME #if CYTHON_PROFILE_REUSE_FRAME
#define CYTHON_FRAME_MODIFIER static #define CYTHON_FRAME_MODIFIER static
#define CYTHON_FRAME_DEL #define CYTHON_FRAME_DEL(frame)
#else #else
#define CYTHON_FRAME_MODIFIER #define CYTHON_FRAME_MODIFIER
#define CYTHON_FRAME_DEL Py_CLEAR($frame_cname) #define CYTHON_FRAME_DEL(frame) Py_CLEAR(frame)
#endif #endif
#define __Pyx_TraceDeclarations(codeobj) \ #define __Pyx_TraceDeclarations(codeobj, nogil) \
static PyCodeObject *$frame_code_cname = NULL; \ static PyCodeObject *$frame_code_cname = NULL; \
CYTHON_FRAME_MODIFIER PyFrameObject *$frame_cname = NULL; \ CYTHON_FRAME_MODIFIER PyFrameObject *$frame_cname = NULL; \
int __Pyx_use_tracing = 0; \ int __Pyx_use_tracing = 0; \
if (codeobj) $frame_code_cname = (PyCodeObject*) codeobj; if (codeobj) $frame_code_cname = (PyCodeObject*) codeobj;
#define __Pyx_TraceCall(funcname, srcfile, firstlineno) \ #ifdef WITH_THREAD
{ PyThreadState* tstate = PyThreadState_GET(); \ #define __Pyx_TraceCall(funcname, srcfile, firstlineno, nogil) \
if (unlikely(tstate->use_tracing) && !tstate->tracing && \ if (nogil) { \
(tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) { \ if (CYTHON_TRACE_NOGIL) { \
PyThreadState *tstate; \
PyGILState_STATE state = PyGILState_Ensure(); \
tstate = PyThreadState_GET(); \
if (unlikely(tstate->use_tracing) && !tstate->tracing && \
(tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) { \
__Pyx_use_tracing = __Pyx_TraceSetupAndCall(&$frame_code_cname, &$frame_cname, funcname, srcfile, firstlineno); \
} \
PyGILState_Release(state); \
} \
} else { \
PyThreadState* tstate = PyThreadState_GET(); \
if (unlikely(tstate->use_tracing) && !tstate->tracing && \
(tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) { \
__Pyx_use_tracing = __Pyx_TraceSetupAndCall(&$frame_code_cname, &$frame_cname, funcname, srcfile, firstlineno); \ __Pyx_use_tracing = __Pyx_TraceSetupAndCall(&$frame_code_cname, &$frame_cname, funcname, srcfile, firstlineno); \
} \ } \
} }
#else
#define __Pyx_TraceCall(funcname, srcfile, firstlineno, nogil) \
{ PyThreadState* tstate = PyThreadState_GET(); \
if (unlikely(tstate->use_tracing) && !tstate->tracing && \
(tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) { \
__Pyx_use_tracing = __Pyx_TraceSetupAndCall(&$frame_code_cname, &$frame_cname, funcname, srcfile, firstlineno); \
} \
}
#endif
#define __Pyx_TraceException() \ #define __Pyx_TraceException() \
if (likely(!__Pyx_use_tracing)); else { \ if (likely(!__Pyx_use_tracing)); else { \
...@@ -69,36 +95,60 @@ ...@@ -69,36 +95,60 @@
} \ } \
} }
#define __Pyx_TraceReturn(result) \ static void __Pyx_call_return_trace_func(PyThreadState *tstate, PyFrameObject *frame, PyObject *result) {
PyObject *type, *value, *traceback;
PyErr_Fetch(&type, &value, &traceback);
tstate->tracing++;
tstate->use_tracing = 0;
if (CYTHON_TRACE && tstate->c_tracefunc)
tstate->c_tracefunc(tstate->c_traceobj, frame, PyTrace_RETURN, result);
if (tstate->c_profilefunc)
tstate->c_profilefunc(tstate->c_profileobj, frame, PyTrace_RETURN, result);
CYTHON_FRAME_DEL(frame);
tstate->use_tracing = 1;
tstate->tracing--;
PyErr_Restore(type, value, traceback);
}
#ifdef WITH_THREAD
#define __Pyx_TraceReturn(result, nogil) \
if (likely(!__Pyx_use_tracing)); else { \
if (nogil) { \
if (CYTHON_TRACE_NOGIL) { \
PyThreadState *tstate; \
PyGILState_STATE state = PyGILState_Ensure(); \
tstate = PyThreadState_GET(); \
if (tstate->use_tracing) { \
__Pyx_call_return_trace_func(tstate, $frame_cname, (PyObject*)result); \
} \
PyGILState_Release(state); \
} \
} else { \
PyThreadState* tstate = PyThreadState_GET(); \
if (tstate->use_tracing) { \
__Pyx_call_return_trace_func(tstate, $frame_cname, (PyObject*)result); \
} \
} \
}
#else
#define __Pyx_TraceReturn(result, nogil) \
if (likely(!__Pyx_use_tracing)); else { \ if (likely(!__Pyx_use_tracing)); else { \
PyThreadState* tstate = PyThreadState_GET(); \ PyThreadState* tstate = PyThreadState_GET(); \
if (tstate->use_tracing) { \ if (tstate->use_tracing) { \
PyObject *type, *value, *traceback; \ __Pyx_call_return_trace_func(tstate, $frame_cname, (PyObject*)result); \
PyErr_Fetch(&type, &value, &traceback); \
tstate->tracing++; \
tstate->use_tracing = 0; \
if (CYTHON_TRACE && tstate->c_tracefunc) \
tstate->c_tracefunc( \
tstate->c_traceobj, $frame_cname, PyTrace_RETURN, (PyObject*)result); \
if (tstate->c_profilefunc) \
tstate->c_profilefunc( \
tstate->c_profileobj, $frame_cname, PyTrace_RETURN, (PyObject*)result); \
CYTHON_FRAME_DEL; \
tstate->use_tracing = 1; \
tstate->tracing--; \
PyErr_Restore(type, value, traceback); \
} \ } \
} }
#endif
static PyCodeObject *__Pyx_createFrameCodeObject(const char *funcname, const char *srcfile, int firstlineno); /*proto*/ static PyCodeObject *__Pyx_createFrameCodeObject(const char *funcname, const char *srcfile, int firstlineno); /*proto*/
static int __Pyx_TraceSetupAndCall(PyCodeObject** code, PyFrameObject** frame, const char *funcname, const char *srcfile, int firstlineno); /*proto*/ static int __Pyx_TraceSetupAndCall(PyCodeObject** code, PyFrameObject** frame, const char *funcname, const char *srcfile, int firstlineno); /*proto*/
#else #else
#define __Pyx_TraceDeclarations(codeobj) #define __Pyx_TraceDeclarations(codeobj, nogil)
#define __Pyx_TraceCall(funcname, srcfile, firstlineno) #define __Pyx_TraceCall(funcname, srcfile, firstlineno, nogil)
#define __Pyx_TraceException() #define __Pyx_TraceException()
#define __Pyx_TraceReturn(result) #define __Pyx_TraceReturn(result, nogil)
#endif /* CYTHON_PROFILE */ #endif /* CYTHON_PROFILE */
...@@ -117,15 +167,37 @@ ...@@ -117,15 +167,37 @@
PyErr_Restore(type, value, traceback); PyErr_Restore(type, value, traceback);
} }
#define __Pyx_TraceLine(lineno) \ #ifdef WITH_THREAD
#define __Pyx_TraceLine(lineno, nogil) \
if (likely(!__Pyx_use_tracing)); else { \
if (nogil) { \
if (CYTHON_TRACE_NOGIL) { \
PyThreadState *tstate; \
PyGILState_STATE state = PyGILState_Ensure(); \
tstate = PyThreadState_GET(); \
if (unlikely(tstate->use_tracing && tstate->c_tracefunc)) { \
__Pyx_call_line_trace_func(tstate, $frame_cname, lineno); \
} \
PyGILState_Release(state); \
} \
} else { \
PyThreadState* tstate = PyThreadState_GET(); \
if (unlikely(tstate->use_tracing && tstate->c_tracefunc)) { \
__Pyx_call_line_trace_func(tstate, $frame_cname, lineno); \
} \
} \
}
#else
#define __Pyx_TraceLine(lineno, nogil) \
if (likely(!__Pyx_use_tracing)); else { \ if (likely(!__Pyx_use_tracing)); else { \
PyThreadState* tstate = PyThreadState_GET(); \ PyThreadState* tstate = PyThreadState_GET(); \
if (unlikely(tstate->use_tracing && tstate->c_tracefunc)) { \ if (unlikely(tstate->use_tracing && tstate->c_tracefunc)) { \
__Pyx_call_line_trace_func(tstate, $frame_cname, lineno); \ __Pyx_call_line_trace_func(tstate, $frame_cname, lineno); \
} \ } \
} }
#endif
#else #else
#define __Pyx_TraceLine(lineno) #define __Pyx_TraceLine(lineno, nogil)
#endif #endif
/////////////// Profile /////////////// /////////////// Profile ///////////////
......
tests/run/coverage_nogil.srctree
View file @ 3b6ec99c
...@@ -23,7 +23,7 @@ plugins = Cython.Coverage ...@@ -23,7 +23,7 @@ plugins = Cython.Coverage
######## coverage_test_nogil.pyx ######## ######## coverage_test_nogil.pyx ########
# cython: linetrace=True # cython: linetrace=True
# distutils: define_macros=CYTHON_TRACE=1 # distutils: define_macros=CYTHON_TRACE=1 CYTHON_TRACE_NOGIL=1
cdef int func1(int a, int b) nogil: cdef int func1(int a, int b) nogil:
cdef int x # 5 cdef int x # 5
...@@ -85,8 +85,8 @@ def run_coverage(module): ...@@ -85,8 +85,8 @@ def run_coverage(module):
executed = set(exec_lines) - set(missing_lines) executed = set(exec_lines) - set(missing_lines)
# check that everything that runs with the gil owned was executed # check that everything that runs with the gil owned was executed
assert all(line in executed for line in [13, 17, 18, 20]), '%s / %s' % (exec_lines, missing_lines) assert all(line in executed for line in [13, 17, 18, 20]), '%s / %s' % (exec_lines, missing_lines)
# currently, we do not trace nogil code lines, but that should eventually be implemented # check that everything that runs in nogil sections was executed
# we also don't trace 'with gil' blocks in 'nogil' functions assert all(line in executed for line in [6, 7, 8, 9]), '%s / %s' % (exec_lines, missing_lines)
if __name__ == '__main__': if __name__ == '__main__':
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7