Make asyncio.iscoroutinefunction() recognise Cython compiled coroutines. (GH-3427)

Python's asyncio.coroutines uses an object to tag objects as coroutine functions. We now read this object and use it to tag Cython compiled coroutines as well.

It also includes tests to make sure `asyncio.iscoroutinefunction()` works as expected.
This doesn't fix `inspect.iscouroutinefunction()` (which uses a flag that can trigger undesirable behaviour for cython functions).

Closes https://github.com/cython/cython/issues/2273

Cython/Compiler/ExprNodes.py

@@ -9486,6 +9486,9 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin):
         if def_node.local_scope.parent_scope.is_c_class_scope and not def_node.entry.is_anonymous:
             flags.append('__Pyx_CYFUNCTION_CCLASS')
 
+        if def_node.is_coroutine:
+            flags.append('__Pyx_CYFUNCTION_COROUTINE')
+
         if flags:
             flags = ' | '.join(flags)
         else:

Cython/Compiler/Nodes.py

@@ -1691,6 +1691,8 @@ class FuncDefNode(StatNode, BlockNode):
     needs_outer_scope = False
     pymethdef_required = False
     is_generator = False
+    is_coroutine = False
+    is_asyncgen = False
     is_generator_body = False
     is_async_def = False
     modifiers = []
@@ -4336,9 +4338,7 @@ class GeneratorDefNode(DefNode):
     #
 
     is_generator = True
-    is_coroutine = False
     is_iterable_coroutine = False
-    is_asyncgen = False
     gen_type_name = 'Generator'
     needs_closure = True
 

Cython/Utility/CythonFunction.c

@@ -6,6 +6,7 @@
 #define __Pyx_CYFUNCTION_STATICMETHOD  0x01
 #define __Pyx_CYFUNCTION_CLASSMETHOD   0x02
 #define __Pyx_CYFUNCTION_CCLASS        0x04
+#define __Pyx_CYFUNCTION_COROUTINE     0x08
 
 #define __Pyx_CyFunction_GetClosure(f) \
     (((__pyx_CyFunctionObject *) (f))->func_closure)
@@ -47,6 +48,9 @@ typedef struct {
     PyObject *defaults_kwdict;  /* Const kwonly defaults dict */
     PyObject *(*defaults_getter)(PyObject *);
     PyObject *func_annotations; /* function annotations dict */
+
+    // Coroutine marker
+    PyObject *func_is_coroutine;
 } __pyx_CyFunctionObject;
 
 #if !CYTHON_COMPILING_IN_LIMITED_API
@@ -92,6 +96,7 @@ static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func,
 //@requires: CommonStructures.c::FetchCommonType
 //@requires: ObjectHandling.c::PyMethodNew
 //@requires: ObjectHandling.c::PyVectorcallFastCallDict
+//@requires: ObjectHandling.c::PyObjectGetAttrStr
 
 #include <structmember.h>
 
@@ -362,6 +367,35 @@ __Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, CYTHON_UNUSED void
     return result;
 }
 
+static PyObject *
+__Pyx_CyFunction_get_is_coroutine(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) {
+    if (op->func_is_coroutine)
+        return __Pyx_NewRef(op->func_is_coroutine);
+
+    int is_coroutine = op->flags & __Pyx_CYFUNCTION_COROUTINE;
+
+#if PY_VERSION_HEX >= 0x03050000
+    if (is_coroutine) {
+        PyObject *module, *fromlist, *marker = PYIDENT("_is_coroutine");
+        fromlist = PyList_New(1);
+        if (unlikely(!fromlist)) return NULL;
+        PyList_SET_ITEM(fromlist, 0, marker);
+        module = PyImport_ImportModuleLevelObject(PYIDENT("asyncio.coroutines"), NULL, NULL, fromlist, 0);
+        Py_DECREF(fromlist);
+        if (unlikely(!module)) goto ignore;
+        op->func_is_coroutine = __Pyx_PyObject_GetAttrStr(module, marker);
+        Py_DECREF(module);
+        if (unlikely(!op->func_is_coroutine)) goto ignore;
+        return __Pyx_NewRef(op->func_is_coroutine);
+    }
+ignore:
+    PyErr_Clear();
+#endif
+
+    op->func_is_coroutine = __Pyx_PyBool_FromLong(is_coroutine);
+    return __Pyx_NewRef(op->func_is_coroutine);
+}
+
 //#if PY_VERSION_HEX >= 0x030400C1
 //static PyObject *
 //__Pyx_CyFunction_get_signature(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) {
@@ -406,6 +440,7 @@ static PyGetSetDef __pyx_CyFunction_getsets[] = {
     {(char *) "__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0},
     {(char *) "__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0},
     {(char *) "__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0},
+    {(char *) "_is_coroutine", (getter)__Pyx_CyFunction_get_is_coroutine, 0, 0, 0},
 //#if PY_VERSION_HEX >= 0x030400C1
 //    {(char *) "__signature__", (getter)__Pyx_CyFunction_get_signature, 0, 0, 0},
 //#endif
@@ -478,6 +513,7 @@ static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject *op, PyMethodDef *
     op->defaults_kwdict = NULL;
     op->defaults_getter = NULL;
     op->func_annotations = NULL;
+    op->func_is_coroutine = NULL;
 #if CYTHON_METH_FASTCALL
     switch (ml->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS)) {
     case METH_NOARGS:
@@ -518,6 +554,7 @@ __Pyx_CyFunction_clear(__pyx_CyFunctionObject *m)
     Py_CLEAR(m->defaults_tuple);
     Py_CLEAR(m->defaults_kwdict);
     Py_CLEAR(m->func_annotations);
+    Py_CLEAR(m->func_is_coroutine);
 
     if (m->defaults) {
         PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m);
@@ -560,6 +597,7 @@ static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit,
     Py_VISIT(m->func_classobj);
     Py_VISIT(m->defaults_tuple);
     Py_VISIT(m->defaults_kwdict);
+    Py_VISIT(m->func_is_coroutine);
 
     if (m->defaults) {
         PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m);

tests/run/py35_asyncio_async_def.srctree

 # mode: run
-# tag: asyncio, gh1685
+# tag: asyncio, gh1685, gh2273
 
 PYTHON setup.py build_ext -i
 PYTHON main.py
@@ -19,6 +19,7 @@ setup(
 
 import asyncio
 import cy_test
+import py_test
 from contextlib import closing
 
 async def main():
@@ -31,6 +32,12 @@ with closing(asyncio.get_event_loop()) as loop:
     print("Running Cython coroutine ...")
     loop.run_until_complete(cy_test.say())
 
+assert asyncio.iscoroutinefunction(cy_test.cy_async_def_example) == True
+assert asyncio.iscoroutinefunction(cy_test.cy_async_def_example) == True
+assert asyncio.iscoroutinefunction(py_test.py_async_def_example) == True
+assert asyncio.iscoroutinefunction(py_test.py_async_def_example) == True
+assert asyncio.iscoroutinefunction(cy_test.cy_def_example) == False
+assert asyncio.iscoroutinefunction(py_test.py_def_example) == False
 
 ######## cy_test.pyx ########
 
@@ -51,8 +58,19 @@ async def cb():
     await asyncio.sleep(0.5)
     print("done!")
 
+async def cy_async_def_example():
+    return 1
+
+def cy_def_example():
+    return 1
 
 ######## py_test.py ########
 
 async def py_async():
     print("- and this one is from Python")
+
+async def py_async_def_example():
+    return 1
+
+def py_def_example():
+    return 1