Do not hijack "@asyncio.coroutine" to make async-def functions iterable, since...

Do not hijack "@asyncio.coroutine" to make async-def functions iterable, since this is really just a legacy feature that users should not overuse. Instead, provide a dedicated and explicit "cython.iterable_coroutine" directive.

CHANGES.rst

@@ -27,10 +27,10 @@ Features added
   them to depend on module declarations themselves.
   Patch by Jeroen Demeyer.  (Github issue #1896)
 
-* Decorating an async coroutine with ``@types.coroutine`` or ``@asyncio.coroutine``
-  changes its type at compile time to make it iterable.  While this is not
-  strictly in line with PEP-492, it improves the interoperability with old-style
-  coroutines that use ``yield from`` instead of ``await``.
+* Decorating an async coroutine with ``@cython.iterable_coroutine`` changes its
+  type at compile time to make it iterable.  While this is not strictly in line
+  with PEP-492, it improves the interoperability with old-style coroutines that
+  use ``yield from`` instead of ``await``.
 
 * The new TSS C-API in CPython 3.7 is supported and has been backported.
   Patch by Naotoshi Seo.  (Github issue #1932)

Cython/Compiler/Options.py

@@ -171,6 +171,7 @@ _directive_defaults = {
     'language_level': 2,
     'fast_getattr': False, # Undocumented until we come up with a better way to handle this everywhere.
     'py2_import': False, # For backward compatibility of Cython's source code in Py3 source mode
+    'iterable_coroutine': False,  # Make async coroutines backwards compatible with the old asyncio yield-from syntax.
     'c_string_type': 'bytes',
     'c_string_encoding': '',
     'type_version_tag': True,   # enables Py_TPFLAGS_HAVE_VERSION_TAG on extension types
@@ -320,6 +321,7 @@ directive_scopes = { # defaults to available everywhere
     'old_style_globals': ('module',),
     'np_pythran': ('module',),
     'fast_gil': ('module',),
+    'iterable_coroutine': ('module', 'function'),
 }
 
 

Cython/Compiler/ParseTreeTransforms.py

@@ -2601,21 +2601,7 @@ class MarkClosureVisitor(CythonTransform):
                 coroutine_type = Nodes.AsyncGenNode
                 for yield_expr in collector.yields + collector.returns:
                     yield_expr.in_async_gen = True
-            elif node.decorators:
-                # evaluate @asyncio.coroutine() decorator at compile time if it's the inner-most one
-                # TODO: better decorator validation: should come from imported module
-                decorator = node.decorators[-1].decorator
-                if decorator.is_name and decorator.name == 'coroutine':
-                    pass
-                elif decorator.is_attribute and decorator.attribute == 'coroutine':
-                    if decorator.obj.is_name and decorator.obj.name in ('types', 'asyncio'):
-                        pass
-                    else:
-                        decorator = None
-                else:
-                    decorator = None
-                if decorator is not None:
-                    node.decorators.pop()
+            elif self.current_directives['iterable_coroutine']:
                 coroutine_type = Nodes.IterableAsyncDefNode
         elif collector.has_await:
             found = next(y for y in collector.yields if y.is_await)

docs/src/reference/compilation.rst

@@ -577,6 +577,16 @@ Cython code.  Here is the list of currently supported directives:
 ``unraisable_tracebacks`` (True / False)
     Whether to print tracebacks when suppressing unraisable exceptions.
 
+``iterable_coroutine`` (True / False)
+    `PEP 492 <https://www.python.org/dev/peps/pep-0492/>`_ specifies that async-def
+    coroutines must not be iterable, in order to prevent accidental misuse in
+    non-async contexts.  However, this makes it difficult and inefficient to write
+    backwards compatible code that uses async-def coroutines in Cython but needs to
+    interact with async Python code that uses the older yield-from syntax, such as
+    asyncio before Python 3.5.  This directive can be applied in modules or
+    selectively as decorator on an async-def coroutine to make the affected
+    coroutine(s) iterable and thus directly interoperable with yield-from.
+
 
 Configurable optimisations
 --------------------------

tests/run/py35_pep492_interop.pyx

@@ -3,7 +3,8 @@
 # tag: pep492, asyncfor, await
 
 
-def run_async(coro):
+def run_async(coro, ignore_type=False):
+    if not ignore_type:
         #assert coro.__class__ is types.GeneratorType
         assert coro.__class__.__name__ in ('coroutine', 'GeneratorWrapper'), coro.__class__.__name__
 
@@ -87,3 +88,69 @@ def await_cyobject():
         return await awaitable
 
     return simple, awaiting
+
+
+cimport cython
+
+def yield_from_cyobject():
+    """
+    >>> async def py_simple_nonit():
+    ...     return 10
+
+    >>> async def run_await(awaitable):
+    ...     return await awaitable
+
+    >>> def run_yield_from(it):
+    ...     return (yield from it)
+
+    >>> simple_nonit, simple_it, awaiting, yield_from = yield_from_cyobject()
+
+    >>> buffer, result = run_async(run_await(simple_it()))
+    >>> result
+    10
+    >>> buffer, result = run_async(run_await(awaiting(simple_it())))
+    >>> result
+    10
+    >>> buffer, result = run_async(awaiting(run_await(simple_it())), ignore_type=True)
+    >>> result
+    10
+    >>> buffer, result = run_async(run_await(py_simple_nonit()))
+    >>> result
+    10
+
+    >>> buffer, result = run_async(run_yield_from(awaiting(run_await(simple_it()))), ignore_type=True)
+    >>> result
+    10
+
+    >>> buffer, result = run_async(run_yield_from(simple_it()), ignore_type=True)
+    >>> result
+    10
+    >>> buffer, result = run_async(yield_from(simple_it()), ignore_type=True)
+    >>> result
+    10
+
+    >>> next(run_yield_from(simple_nonit()))  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...
+    >>> next(run_yield_from(py_simple_nonit()))  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...
+    >>> next(yield_from(py_simple_nonit()))
+    Traceback (most recent call last):
+    TypeError: 'coroutine' object is not iterable
+    """
+    async def simple_nonit():
+        return 10
+
+    @cython.iterable_coroutine
+    async def simple_it():
+        return 10
+
+    @cython.iterable_coroutine
+    async def awaiting(awaitable):
+        return await awaitable
+
+    def yield_from(it):
+        return (yield from it)
+
+    return simple_nonit, simple_it, awaiting, yield_from