Boxiang: Create nogil extension to cdef classes

Cython/Compiler/Code.py

@@ -2050,6 +2050,8 @@ class CCodeWriter(object):
                 self.putln("%s = NULL;" % decl)
             elif type.is_memoryviewslice:
                 self.putln("%s = %s;" % (decl, type.literal_code(type.default_value)))
+            elif type.is_struct and type.is_extension_type and type.nogil:
+                self.putln("%s;" % decl)
             else:
                 self.putln("%s%s;" % (static and "static " or "", decl))
 

Cython/Compiler/ExprNodes.py

@@ -704,6 +704,8 @@ class ExprNode(Node):
             self.gil_error()
 
     def gil_assignment_check(self, env):
+        if self.type == PyrexTypes.PyExtensionType and env.nogil and self.type.nogil:
+            error("No gil type in no gil function")
         if env.nogil and self.type.is_pyobject:
             error(self.pos, "Assignment of Python object not allowed without gil")
 
@@ -1719,6 +1721,9 @@ class StringNode(PyConstNode):
     is_identifier = None
     unicode_value = None
 
+    # XXX: Let the StringNode can be used in nogil extension initializing
+    nogil_check = None
+
     def calculate_constant_result(self):
         if self.unicode_value is not None:
             # only the Unicode value is portable across Py2/3
@@ -2314,6 +2319,9 @@ class NameNode(AtomicExprNode):
                         code.error_goto_if_null(self.result(), self.pos)))
             self.generate_gotref(code)
 
+        elif entry.is_local and isinstance(entry.type, PyrexTypes.CythonExtensionType):
+            pass
+            # code.putln(entry.cname)
         elif entry.is_local or entry.in_closure or entry.from_closure or entry.type.is_memoryviewslice:
             # Raise UnboundLocalError for objects and memoryviewslices
             raise_unbound = (
@@ -5547,6 +5555,18 @@ class CallNode(ExprNode):
             self.analyse_c_function_call(env)
             self.type = type
             return True
+        elif type and type.is_struct and type.nogil:
+            args, kwds = self.explicit_args_kwds()
+            items = []
+            for arg, member in zip(args, type.scope.var_entries):
+                items.append(DictItemNode(pos=arg.pos, key=StringNode(pos=arg.pos, value=member.name), value=arg))
+            if kwds:
+                items += kwds.key_value_pairs
+            self.key_value_pairs = items
+            self.__class__ = DictNode
+            self.analyse_types(env)    # FIXME
+            self.coerce_to(type, env)
+            return True
 
     def is_lvalue(self):
         return self.type.is_reference
@@ -7203,6 +7223,9 @@ class AttributeNode(ExprNode):
                     # (AnalyseExpressionsTransform)
                     self.member = self.entry.cname
 
+                if obj.type.nogil:
+                    return "%s" % self.entry.func_cname
+                else:
                     return "((struct %s *)%s%s%s)->%s" % (
                         obj.type.vtabstruct_cname, obj_code, self.op,
                         obj.type.vtabslot_cname, self.member)
@@ -8804,6 +8827,8 @@ class DictNode(ExprNode):
         #  pairs are evaluated and used one at a time.
         code.mark_pos(self.pos)
         self.allocate_temp_result(code)
+        if hasattr(self.type, 'nogil') and self.type.nogil:
+            code.putln("%s = (struct %s *)malloc(sizeof(struct %s));" % (self.result(), self.type.objstruct_cname, self.type.objstruct_cname))
 
         is_dict = self.type.is_pyobject
         if is_dict:
@@ -8861,6 +8886,11 @@ class DictNode(ExprNode):
                     code.putln('}')
                 if self.exclude_null_values:
                     code.putln('}')
+            elif self.type.nogil:
+                code.putln("%s->%s = %s;" % (
+                        self.result(),
+                        item.key.value,
+                        item.value.result()))
             else:
                 code.putln("%s.%s = %s;" % (
                         self.result(),

Cython/Compiler/ModuleNode.py

@@ -1197,6 +1197,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             self.sue_header_footer(type, "struct", type.objstruct_cname)
         code.putln(header)
         base_type = type.base_type
+        nogil = type.nogil
         if base_type:
             basestruct_cname = base_type.objstruct_cname
             if basestruct_cname == "PyTypeObject":
@@ -1207,6 +1208,16 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     ("struct ", "")[base_type.typedef_flag],
                     basestruct_cname,
                     Naming.obj_base_cname))
+        elif nogil:
+            # Extension type with nogil keyword indicate it is a CPython-free struct
+            code.globalstate.use_utility_code(
+                UtilityCode.load_cached("CythonReferenceCounting", "ObjectHandling.c"))
+            code.putln(
+                "// nogil"
+            )
+            code.putln(
+                "int ob_refcnt;" # "CyObject_HEAD;" Sometimes the CythonReferenceCounting was put after the nogil extension declaration, WTF!!!
+            )
         else:
             code.putln(
                 "PyObject_HEAD")

Cython/Compiler/Nodes.py

@@ -3376,6 +3376,8 @@ class DefNodeWrapper(FuncDefNode):
         # different code types.
         for arg in self.args:
             if not arg.type.is_pyobject:
+                if arg.type is PyrexTypes.PyExtensionType and arg.type.nogil:
+                    continue # XXX maybe here is not the correct place to put it...
                 if not arg.type.create_from_py_utility_code(env):
                     pass # will fail later
             elif arg.hdr_type and not arg.hdr_type.is_pyobject:
@@ -4906,6 +4908,7 @@ class CClassDefNode(ClassDefNode):
     #  doc                string or None
     #  body               StatNode or None
     #  entry              Symtab.Entry
+    #  nogil              boolean
     #  base_type          PyExtensionType or None
     #  buffer_defaults_node DictNode or None Declares defaults for a buffer
     #  buffer_defaults_pos
@@ -4915,6 +4918,7 @@ class CClassDefNode(ClassDefNode):
     buffer_defaults_pos = None
     typedef_flag = False
     api = False
+    nogil = False
     objstruct_name = None
     typeobj_name = None
     check_size = None
@@ -4959,6 +4963,7 @@ class CClassDefNode(ClassDefNode):
             typedef_flag=self.typedef_flag,
             check_size = self.check_size,
             api=self.api,
+            nogil=self.nogil,
             buffer_defaults=self.buffer_defaults(env),
             shadow=self.shadow)
 
@@ -5047,6 +5052,7 @@ class CClassDefNode(ClassDefNode):
             visibility=self.visibility,
             typedef_flag=self.typedef_flag,
             api=self.api,
+            nogil=self.nogil,
             buffer_defaults=self.buffer_defaults(env),
             shadow=self.shadow)
 
@@ -6186,6 +6192,8 @@ class DelStatNode(StatNode):
                     error(arg.pos, "Deletion of global C variable")
             elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
                 self.cpp_check(env)
+            elif arg.type.is_struct_or_union and arg.type.nogil:
+                pass # del nogil extension
             elif arg.type.is_cpp_class:
                 error(arg.pos, "Deletion of non-heap C++ object")
             elif arg.is_subscript and arg.base.type is Builtin.bytearray_type:
@@ -6214,6 +6222,8 @@ class DelStatNode(StatNode):
                 arg.generate_evaluation_code(code)
                 code.putln("delete %s;" % arg.result())
                 arg.generate_disposal_code(code)
+            elif arg.type.is_struct_or_union and hasattr(arg.type, "nogil") and arg.type.nogil:
+                code.putln("free(&%s);" % arg.result())
             # else error reported earlier
 
     def annotate(self, code):

Cython/Compiler/ParseTreeTransforms.py

@@ -1720,6 +1720,7 @@ if VALUE is not None:
             if stats:
                 node.body.stats += stats
             if (node.visibility != 'extern'
+                and not node.nogil
                 and not node.scope.lookup('__reduce__')
                 and not node.scope.lookup('__reduce_ex__')):
                 self._inject_pickle_methods(node)

Cython/Compiler/Parsing.py

@@ -2235,7 +2235,7 @@ def p_statement(s, ctx, first_statement = 0):
     elif s.sy == 'IF':
         return p_IF_statement(s, ctx)
     elif s.sy == '@':
-        if ctx.level not in ('module', 'class', 'c_class', 'function', 'property', 'module_pxd', 'c_class_pxd', 'other'):
+        if ctx.level not in ('module', 'class', 'c_class', 'class_nogil', 'function', 'property', 'module_pxd', 'c_class_pxd', 'other'):
             s.error('decorator not allowed here')
         s.level = ctx.level
         decorators = p_decorators(s)
@@ -2253,11 +2253,14 @@ def p_statement(s, ctx, first_statement = 0):
         cdef_flag = 1
         s.next()
     elif s.sy == 'cpdef':
+        if ctx.level == 'c_class_nogil':
+            s.error('cpdef statement not allowed in nogil extension type')
+        s.level = ctx.level
         cdef_flag = 1
         overridable = 1
         s.next()
     if cdef_flag:
-        if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):
+        if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_nogil', 'c_class_pxd'):
             s.error('cdef statement not allowed here')
         s.level = ctx.level
         node = p_cdef_statement(s, ctx(overridable=overridable))
@@ -2277,6 +2280,8 @@ def p_statement(s, ctx, first_statement = 0):
             # as part of a cdef class
             if ('pxd' in ctx.level) and (ctx.level != 'c_class_pxd'):
                 s.error('def statement not allowed here')
+            if ctx.level == 'c_class_nogil':
+               s.error('def statement not allowed in nogil extension type, only cdef with nogil is allowed')
             s.level = ctx.level
             return p_def_statement(s, decorators)
         elif s.sy == 'class':
@@ -2287,7 +2292,7 @@ def p_statement(s, ctx, first_statement = 0):
             if ctx.level not in ('module', 'module_pxd'):
                 s.error("include statement not allowed here")
             return p_include_statement(s, ctx)
-        elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':
+        elif ctx.level in ('c_class', 'c_class_nogil') and s.sy == 'IDENT' and s.systring == 'property':
             return p_property_decl(s)
         elif s.sy == 'pass' and ctx.level != 'property':
             return p_pass_statement(s, with_newline=True)
@@ -3271,7 +3276,7 @@ def p_c_modifiers(s):
     return []
 
 def p_c_func_or_var_declaration(s, pos, ctx):
-    cmethod_flag = ctx.level in ('c_class', 'c_class_pxd')
+    cmethod_flag = ctx.level in ('c_class', 'c_class_pxd', 'c_class_nogil')
     modifiers = p_c_modifiers(s)
     base_type = p_c_base_type(s, nonempty = 1, templates = ctx.templates)
     declarator = p_c_declarator(s, ctx(modifiers=modifiers), cmethod_flag = cmethod_flag,
@@ -3290,8 +3295,13 @@ def p_c_func_or_var_declaration(s, pos, ctx):
             fatal=False)
         s.next()
         p_test(s)  # Keep going, but ignore result.
+    if s.sy == 'nogil':
+        nogil = p_nogil(s)
+        s.next()
+        if ctx.level == 'c_class_nogil' and not nogil:
+            s.error("Only C function with nogil allowed in nogil extension")
     if s.sy == ':':
-        if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd', 'cpp_class') and not ctx.templates:
+        if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd', 'cpp_class', 'c_class_nogil') and not ctx.templates:
             s.error("C function definition not allowed here")
         doc, suite = p_suite_with_docstring(s, Ctx(level='function'))
         result = Nodes.CFuncDefNode(pos,
@@ -3319,7 +3329,7 @@ def p_c_func_or_var_declaration(s, pos, ctx):
             declarators.append(declarator)
         doc_line = s.start_line + 1
         s.expect_newline("Syntax error in C variable declaration", ignore_semicolon=True)
-        if ctx.level in ('c_class', 'c_class_pxd') and s.start_line == doc_line:
+        if ctx.level in ('c_class', 'c_class_pxd', 'c_class_nogil') and s.start_line == doc_line:
             doc = p_doc_string(s)
         else:
             doc = None
@@ -3520,9 +3530,12 @@ def p_c_class_definition(s, pos,  ctx):
         if ctx.visibility not in ('public', 'extern') and not ctx.api:
             error(s.position(), "Name options only allowed for 'public', 'api', or 'extern' C class")
         objstruct_name, typeobj_name, check_size = p_c_class_options(s)
+    nogil = p_nogil(s)
     if s.sy == ':':
         if ctx.level == 'module_pxd':
             body_level = 'c_class_pxd'
+        elif nogil:
+            body_level = 'c_class_nogil'
         else:
             body_level = 'c_class'
         doc, body = p_suite_with_docstring(s, Ctx(level=body_level))
@@ -3561,7 +3574,8 @@ def p_c_class_definition(s, pos,  ctx):
         check_size = check_size,
         in_pxd = ctx.level == 'module_pxd',
         doc = doc,
-        body = body)
+        body = body,
+        nogil = nogil or ctx.nogil)
 
 
 def p_c_class_options(s):

Cython/Compiler/PyrexTypes.py

@@ -1325,6 +1325,215 @@ class PyObjectType(PyrexType):
         return cname
 
 
+class CythonObjectType(PyrexType):
+    #
+    #  Base class for all nogil extension object types (reference-counted).
+    #
+
+    name = "cythonobject"
+    is_pyobject = 0
+    default_value = "0"
+    declaration_value = "0"
+    buffer_defaults = None
+    is_extern = False
+    is_subclassed = False
+    is_gc_simple = False
+
+    def __str__(self):
+        return "Cython object"
+
+    def __repr__(self):
+        return "<CythonObjectType>"
+
+    def can_coerce_to_pyobject(self, env):
+        return False
+
+    def can_coerce_from_pyobject(self, env):
+        return False
+
+    def default_coerced_ctype(self):
+        """The default C type that this Python type coerces to, or None."""
+        return None
+
+    def assignable_from(self, src_type):
+        # except for pointers, conversion will be attempted
+        # return not src_type.is_ptr or src_type.is_string or src_type.is_pyunicode_ptr
+        return False
+
+    def declaration_code(self, entity_code,
+            for_display = 0, dll_linkage = None, pyrex = 0):
+        raise NotImplementedError("Calling declartion_code on Cython object")
+        if pyrex or for_display:
+            base_code = "object"
+        else:
+            base_code = public_decl("PyObject", dll_linkage)
+            entity_code = "*%s" % entity_code
+        return self.base_declaration_code(base_code, entity_code)
+
+    def as_pyobject(self, cname):
+        raise NotImplementedError("Calling as_pyobject on Cython object")
+        if (not self.is_complete()) or self.is_extension_type:
+            return "(PyObject *)" + cname
+        else:
+            return cname
+
+    def py_type_name(self):
+        raise NotImplementedError("Calling py_type_name on Cython object")
+        return "cythonobject"
+
+    def __lt__(self, other):
+        """
+        Make sure we sort highest, as instance checking on py_type_name
+        ('object') is always true
+        """
+        return False
+
+    def global_init_code(self, entry, code):
+        raise NotImplementedError("Calling global_init_code on Cython object")
+        code.put_init_var_to_py_none(entry, nanny=False)
+
+    def check_for_null_code(self, cname):
+        raise NotImplementedError("Calling check_for_null_code on Cython object")
+        return cname
+
+
+class CythonExtensionType(CythonObjectType):
+    #
+    #  A Cython extension type with nogil flag
+    # TODO: This type may need big amend
+    #
+    #  name             string
+    #  scope            CClassScope      Attribute namespace
+    #  visibility       string
+    #  typedef_flag     boolean
+    #  base_type        PyExtensionType or None
+    #  nogil            boolean
+    #  module_name      string or None   Qualified name of defining module
+    #  objstruct_cname  string           Name of PyObject struct
+    #  objtypedef_cname string           Name of PyObject struct typedef
+    #  typeobj_cname    string or None   C code fragment referring to type object
+    #  typeptr_cname    string or None   Name of pointer to external type object
+    #  vtabslot_cname   string           Name of C method table member
+    #  vtabstruct_cname string           Name of C method table struct
+    #  vtabptr_cname    string           Name of pointer to C method table
+    #  vtable_cname     string           Name of C method table definition
+    #  early_init       boolean          Whether to initialize early (as opposed to during module execution).
+    #  defered_declarations [thunk]      Used to declare class hierarchies in order
+    #  check_size       'warn', 'error', 'ignore'    What to do if tp_basicsize does not match
+
+    is_extension_type = 1
+    is_struct = 1
+    is_struct_or_union = 1
+    nogil = 1
+    is_pyobject = 0
+    has_attributes = 1
+    early_init = 1
+
+    objtypedef_cname = None
+
+    def __init__(self, name, typedef_flag, base_type, is_external=0, check_size=None):
+        self.name = name
+        self.scope = None
+        self.typedef_flag = typedef_flag
+        if base_type is not None:
+            base_type.is_subclassed = True
+        self.base_type = base_type
+        self.nogil = False
+        self.module_name = None
+        self.objstruct_cname = None
+        self.typeobj_cname = None
+        self.typeptr_cname = None
+        self.vtabslot_cname = None
+        self.vtabstruct_cname = None
+        self.vtabptr_cname = None
+        self.vtable_cname = None
+        self.is_external = is_external
+        self.check_size = check_size or 'warn'
+        self.defered_declarations = []
+
+    def set_scope(self, scope):
+        self.scope = scope
+        if scope:
+            scope.parent_type = self
+
+    def needs_nonecheck(self):
+        return True
+
+    def subtype_of_resolved_type(self, other_type):
+        if other_type.is_extension_type or other_type.is_builtin_type:
+            return self is other_type or (
+                self.base_type and self.base_type.subtype_of(other_type))
+        else:
+            return other_type is py_object_type
+
+    def typeobj_is_available(self):
+        # Do we have a pointer to the type object?
+        return self.typeptr_cname
+
+    def typeobj_is_imported(self):
+        # If we don't know the C name of the type object but we do
+        # know which module it's defined in, it will be imported.
+        return self.typeobj_cname is None and self.module_name is not None
+
+    def assignable_from(self, src_type):
+        if self == src_type:
+            return True
+        if isinstance(src_type, PyExtensionType):
+            if src_type.base_type is not None:
+                return self.assignable_from(src_type.base_type)
+        if isinstance(src_type, BuiltinObjectType):
+            # FIXME: This is an ugly special case that we currently
+            # keep supporting.  It allows users to specify builtin
+            # types as external extension types, while keeping them
+            # compatible with the real builtin types.  We already
+            # generate a warning for it.  Big TODO: remove!
+            return (self.module_name == '__builtin__' and
+                    self.name == src_type.name)
+        return False
+
+    def declaration_code(self, entity_code,
+            for_display = 0, dll_linkage = None, pyrex = 0, deref = 0):
+        if pyrex or for_display:
+            base_code = self.name
+        else:
+            if self.typedef_flag:
+                objstruct = self.objstruct_cname
+            else:
+                objstruct = "struct %s" % self.objstruct_cname
+            base_code = public_decl(objstruct, dll_linkage)
+            if deref:
+                assert not entity_code
+            else:
+                entity_code = "*%s" % entity_code
+        return self.base_declaration_code(base_code, entity_code)
+
+    def type_test_code(self, py_arg, notnone=False):
+
+        none_check = "((%s) == Py_None)" % py_arg
+        type_check = "likely(__Pyx_TypeTest(%s, %s))" % (
+            py_arg, self.typeptr_cname)
+        if notnone:
+            return type_check
+        else:
+            return "likely(%s || %s)" % (none_check, type_check)
+
+    def attributes_known(self):
+        return self.scope is not None
+
+    def __str__(self):
+        return self.name
+
+    def __repr__(self):
+        return "<CythonExtensionType %s%s>" % (self.scope.class_name,
+            ("", " typedef")[self.typedef_flag])
+
+    def py_type_name(self):
+        if not self.module_name:
+            return self.name
+
+        return "__import__(%r, None, None, ['']).%s" % (self.module_name,
+                                                        self.name)
+
 builtin_types_that_cannot_create_refcycles = set([
     'object', 'bool', 'int', 'long', 'float', 'complex',
     'bytearray', 'bytes', 'unicode', 'str', 'basestring'
@@ -1483,6 +1692,7 @@ class PyExtensionType(PyObjectType):
     #  visibility       string
     #  typedef_flag     boolean
     #  base_type        PyExtensionType or None
+    #  nogil            boolean
     #  module_name      string or None   Qualified name of defining module
     #  objstruct_cname  string           Name of PyObject struct
     #  objtypedef_cname string           Name of PyObject struct typedef
@@ -1509,6 +1719,7 @@ class PyExtensionType(PyObjectType):
         if base_type is not None:
             base_type.is_subclassed = True
         self.base_type = base_type
+        self.nogil = False
         self.module_name = None
         self.objstruct_cname = None
         self.typeobj_cname = None

Cython/Compiler/Symtab.py

@@ -1573,7 +1573,7 @@ class ModuleScope(Scope):
     def declare_c_class(self, name, pos, defining=0, implementing=0,
             module_name=None, base_type=None, objstruct_cname=None,
             typeobj_cname=None, typeptr_cname=None, visibility='private',
-            typedef_flag=0, api=0, check_size=None,
+            typedef_flag=0, api=0, nogil=0, check_size=None,
             buffer_defaults=None, shadow=0):
         # If this is a non-extern typedef class, expose the typedef, but use
         # the non-typedef struct internally to avoid needing forward
@@ -1606,8 +1606,15 @@ class ModuleScope(Scope):
         #  Make a new entry if needed
         #
         if not entry or shadow:
+            if nogil:
+                pass
+            if nogil:
+                type = PyrexTypes.CythonExtensionType(
+                        name, typedef_flag, base_type, visibility == 'extern', check_size=check_size)
+            else:
                 type = PyrexTypes.PyExtensionType(
                         name, typedef_flag, base_type, visibility == 'extern', check_size=check_size)
+            type.nogil = nogil
             type.pos = pos
             type.buffer_defaults = buffer_defaults
             if objtypedef_cname is not None:
@@ -2327,6 +2334,7 @@ class CClassScope(ClassScope):
                           defining=0, modifiers=(), utility_code=None, overridable=False):
         name = self.mangle_class_private_name(name)
         if get_special_method_signature(name) and not self.parent_type.is_builtin_type:
+            if not(hasattr(self.parent_type, "nogil") and self.parent_type.nogil and self.parent_type.is_struct_or_union):
                 error(pos, "Special methods must be declared with 'def', not 'cdef'")
         args = type.args
         if not type.is_static_method:

Cython/Utility/ObjectHandling.c

@@ -1851,6 +1851,51 @@ try_unpack:
     return 0;
 }
 
+/////////////// CythonReferenceCounting.proto ///////////////
+
+#include <stdlib.h>
+#include <stddef.h>
+
+#if !defined(__GNUC__)
+#define GCC_VERSION (__GNUC__ * 10000 \
+                     + __GNUC_MINOR__ * 100 \
+                     + __GNUC_PATCHLEVEL__)
+/* Test for GCC > 4.9.0 */
+#if GCC_VERSION < 40900
+#error atomic.h works only with GCC newer than version 4.9
+#endif /* GNUC >= 4.9 */
+
+#endif /* Has GCC */
+
+// #include <stdatomic.h>
+
+/* CyObject_HEAD defines the initial segment of every CyObject. */
+#define CyObject_HEAD                   \
+    int ob_refcnt;               \
+    void (*cdealloc)();
+
+struct CyObject {
+  CyObject_HEAD
+};
+
+/* Cast argument to PyObject* type. */
+#define _CyObject_CAST(op) ((struct CyObject*)(op))
+
+// XXX: Without scope analysis this is useless...
+/*
+static inline void _Cy_DECREF(struct CyObject *op) {
+  if (atomic_fetch_sub(&(op->ob_refcnt), 1) == 1) {
+    op->cdealloc(op);
+  }
+}
+
+static inline void _Cy_INCREF(struct CyObject *op) {
+  atomic_fetch_add(&(op->ob_refcnt), 1);
+}
+
+#define Cy_INCREF(op) _Cy_INCREF(_CyObject_CAST(op))
+#define Cy_DECREF(op) _Cy_DECREF(_CyObject_CAST(op))
+*/
 
 /////////////// UnpackUnboundCMethod.proto ///////////////
 

nogil_test/basic_test.py

+import subprocess
+import importlib
+import build_extension
+
+def run(env, python_path):
+  source = subprocess.check_output([python_path + " -c 'import nogil_extension; a = nogil_extension.bag(); print(a)'"],
+          env=env,
+          shell=True,
+          )
+  failure_count = 0
+  if source != b'4.0\n42.0\n':
+    failure_count = 1
+  
+  result_dict = {'failed': failure_count, 'stdout': source}
+  return result_dict

nogil_test/build_extension.py

+import subprocess
+import importlib
+import sys
+
+def build_ext(env):
+  # Build the cython_lwan_coro when installing it in slapos
+  # subprocess.check_output(['python3', 'setup.py', 'build_ext', '--inplace'], cwd='./cython_lwan_coro/', env=env)
+  source = ''
+  failure_count = 0
+  try:
+    sys.path.append('./cython_lwan_coro/')
+    wrapper = importlib.import_module('wrapper')
+  except ImportError as e:
+    failure_count = 1
+    source = str(e)
+    print(source) # Will be captured by check_output

nogil_test/lwan_coro_test.py

+import io
+import subprocess
+import importlib
+import sys
+import build_extension
+
+def run(env, python_path):
+  # build the extension
+  # build_extension.build_ext(env)
+  source = subprocess.check_output([python_path + " -c 'import wrapper; wrapper.main()'"],
+          # cwd='./cython_lwan_coro/',
+          env=env,
+          shell=True,
+          )
+  failure_count = 0
+  expected_result = b'2\n' * 5 + b'3\n' * 5 + b'4\n' * 5
+  if source != expected_result:
+    failure_count = 1
+  
+  result_dict = {'failed': failure_count, 'stdout': source}
+  return result_dict

nogil_test/nogil_extension.pyx

+#cython: language_level = 3
+from cython.parallel import parallel
+from libc.stdio cimport printf
+"""
+    GOAL: implement nogil option in cdef class (extension types)
+    and native memory manager (refcount based) that does not
+    depend on cpython's memory manager and that does not require GIL.
+    
+    HINT: look at C++ standard library (that works very nicely with Cython)
+
+    Cython documentation if here: http://docs.cython.org/en/latest/
+    
+    Basic usage: http://docs.cython.org/en/latest/src/quickstart/build.html
+    
+    Tutorial: http://docs.cython.org/en/latest/src/tutorial/cython_tutorial.html
+    
+    Language: http://docs.cython.org/en/latest/src/userguide/language_basics.html
+    
+    Extension types: http://docs.cython.org/en/latest/src/userguide/extension_types.html
+        
+        Extension Types are the "pure cython" classes that I want to be able to
+        use without depending on cpython GIL (and in essence runtime, memory manager, etc.)
+        
+    Cython memory allocation: http://docs.cython.org/en/latest/src/tutorial/memory_allocation.html
+    
+    Parallelism: http://docs.cython.org/en/latest/src/userguide/parallelism.html
+    
+        Explains how nogil is posisble in cython for anything that
+        only relies on C libraries that are multi-threaded
+"""
+
+
+
+# cdef class SomeMemory:
+cdef class SomeMemory nogil:
+  """
+  This is a cdef class which is also called
+  a extensino type. It is a kind of C struct
+  that also acts as a python object.
+  
+  We would like to be able to define "nogil"
+  extension types:
+  
+  cdef class SomeMemory nogil:
+  
+  where all methods are "nogil" and memory
+  allocation does not depend on python runtime
+  """
+  cdef int a;
+  cdef double b;
+
+  cdef void cinit(self, a, b) nogil:
+      # self.a = a
+      # self.b = b
+      pass
+
+  cdef void cdealloc(self) nogil:
+      pass
+
+  cdef void foo1(self) nogil:
+    """
+    It is possible to define native C/Cython methods
+    that release the GIL (cool...)
+    """
+    self.a += 3
+
+  cdef void foo2(self) nogil:
+    """
+    It is possible to define native C/Cython methods
+    that release the GIL (cool...)
+    """
+    self.a = 42
+    
+  # Not allowed to define pure Python function in the extension type with nogil option now
+  # since we want this extension type is CPython free
+  # def baz(self):
+  #  """
+  #  It is also possible to define standard python
+  #  methods
+  #  """
+  #  pass
+    
+    
+# cdef bar(): # it is currently impossible to release GIL
+cdef double bar1() nogil: # yet this is what we would like to
+    """
+    This is a pure "cython method" which we would like to
+    be able to declare with nogil option but this requires
+    to first introduce the concept of nogil in cdef class
+    """
+    cdef SomeMemory o1 = SomeMemory(1, 1.0) # Call init and get an object
+
+    o1.foo1()
+    
+    return o1.a
+
+cdef double bar2() nogil: # yet this is what we would like to
+    cdef SomeMemory o2 = SomeMemory(2, 2.0)
+
+    o2.foo2()
+    
+    return o2.a
+
+cpdef baz1():
+    """
+    This method is both callable from python and pure "cython".
+    It can call both cdef methods and usual python functions
+    """
+    return bar1()
+
+cpdef baz2():
+    """
+    This method is both callable from python and pure "cython".
+    It can call both cdef methods and usual python functions
+    """
+    return bar2()
+
+def bag():
+    return str(baz1()) + '\n' + str(baz2())

nogil_test/nogil_extension_test.py

+from distutils.core import setup
+from distutils.extension import Extension
+from Cython.Build import cythonize
+
+setup(
+    ext_modules = cythonize([
+        Extension(
+            'nogil_extension',
+            ['nogil_extension.pyx'],
+        ),
+    ])
+)

test.py

+from distutils.core import setup
+from Cython.Build import cythonize
+
+setup(
+    ext_modules = cythonize("test.pyx")
+)
\ No newline at end of file

test.pyx

+#cython: language_level = 3
+from libc.stdio cimport printf
+"""
+    GOAL: implement nogil option in cdef class (extension types)
+    and native memory manager (refcount based) that does not
+    depend on cpython's memory manager and that does not require GIL.
+    
+    HINT: look at C++ standard library (that works very nicely with Cython)
+
+    Cython documentation if here: http://docs.cython.org/en/latest/
+    
+    Basic usage: http://docs.cython.org/en/latest/src/quickstart/build.html
+    
+    Tutorial: http://docs.cython.org/en/latest/src/tutorial/cython_tutorial.html
+    
+    Language: http://docs.cython.org/en/latest/src/userguide/language_basics.html
+    
+    Extension types: http://docs.cython.org/en/latest/src/userguide/extension_types.html
+        
+        Extension Types are the "pure cython" classes that I want to be able to
+        use without depending on cpython GIL (and in essence runtime, memory manager, etc.)
+        
+    Cython memory allocation: http://docs.cython.org/en/latest/src/tutorial/memory_allocation.html
+    
+    Parallelism: http://docs.cython.org/en/latest/src/userguide/parallelism.html
+    
+        Explains how nogil is posisble in cython for anything that
+        only relies on C libraries that are multi-threaded
+"""
+
+
+
+# cdef class SomeMemory:
+cdef class SomeMemory nogil:
+  """
+  This is a cdef class which is also called
+  a extensino type. It is a kind of C struct
+  that also acts as a python object.
+  
+  We would like to be able to define "nogil"
+  extension types:
+  
+  cdef class SomeMemory nogil:
+  
+  where all methods are "nogil" and memory
+  allocation does not depend on python runtime
+  """
+  cdef double a;
+  cdef double b;
+    
+  cdef void foo(self) nogil:
+    """
+    It is possible to define native C/Cython methods
+    that release the GIL (cool...)
+    """
+    self.a = self.b
+    
+  cdef void foo1(self, int a) nogil:
+    """
+    It is possible to define native C/Cython methods
+    that release the GIL (cool...)
+    """
+    self.a = a
+
+  cdef void foo3(self) nogil:
+    """
+    It is possible to define native C/Cython methods
+    that release the GIL (cool...)
+    """
+    pass
+  # Not allowed to define pure Python function in the extension type with nogil option now
+  # since we want this extension type is CPython free
+  # def baz(self):
+  #  """
+  #  It is also possible to define standard python
+  #  methods
+  #  """
+  #  pass
+    
+    
+# cdef bar(): # it is currently impossible to release GIL
+cdef int bar() nogil: # yet this is what we would like to
+    """
+    This is a pure "cython method" which we would like to
+    be able to declare with nogil option but this requires
+    to first introduce the concept of nogil in cdef class
+    """
+    cdef SomeMemory o = SomeMemory(42.0, 3.14) # for this we need class allocation to handle memory without libpython
+    o.foo() # and we need method selection to be independent of libpython
+    o.foo1(2)
+    o.a = 1.0
+    return 0
+    
+cpdef baz():
+    """
+    This method is both callable from python and pure "cython".
+    It can call both cdef methods and usual python functions
+    """
+    bar()
+    
+    
+# We call here a cpdef function, which calls a def function
+# which then allocates cdef class SomeMemory
+baz()
+print("done")