ccdef CyRefCount

Cython/Compiler/Code.py

@@ -2025,24 +2025,42 @@ class CCodeWriter(object):
     def put_gotref(self, cname):
         self.putln("__Pyx_GOTREF(%s);" % cname)
 
+    def put_cygotref(self, cname):
+        self.putln("Cy_GOTREF(%s);" % cname)
+
     def put_giveref(self, cname):
         self.putln("__Pyx_GIVEREF(%s);" % cname)
 
+    def put_cygiveref(self, cname):
+        self.putln("Cy_GIVEREF(%s);" % cname)
+
     def put_xgiveref(self, cname):
         self.putln("__Pyx_XGIVEREF(%s);" % cname)
 
+    def put_cyxgiveref(self, cname):
+        self.putln("Cy_XGIVEREF(%s);" % cname)
+
     def put_xgotref(self, cname):
         self.putln("__Pyx_XGOTREF(%s);" % cname)
 
+    def put_cyxgotref(self, cname):
+        self.putln("Cy_XGOTREF(%s);" % cname)
+
     def put_incref(self, cname, type, nanny=True):
         if nanny:
             self.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname, type))
         else:
             self.putln("Py_INCREF(%s);" % self.as_pyobject(cname, type))
 
+    def put_cyincref(self, cname):
+        self.putln("Cy_INCREF(%s);" % cname)
+
     def put_decref(self, cname, type, nanny=True):
         self._put_decref(cname, type, nanny, null_check=False, clear=False)
 
+    def put_cydecref(self, cname):
+        self.putln("Cy_DECREF(%s);" % cname)
+
     def put_var_gotref(self, entry):
         if entry.type.is_pyobject:
             self.putln("__Pyx_GOTREF(%s);" % self.entry_as_pyobject(entry))
@@ -2078,6 +2096,9 @@ class CCodeWriter(object):
         self._put_decref(cname, type, nanny, null_check=True,
                          have_gil=have_gil, clear=False)
 
+    def put_cyxdecref(self, cname):
+        self.putln("Cy_XDECREF(%s);" % cname)
+
     def put_xdecref_clear(self, cname, type, nanny=True, clear_before_decref=False):
         self._put_decref(cname, type, nanny, null_check=True,
                          clear=True, clear_before_decref=clear_before_decref)

Cython/Compiler/ExprNodes.py

@@ -753,6 +753,8 @@ class ExprNode(Node):
         """
         if self.type.is_pyobject and not self.result_in_temp():
             code.put_incref(self.result(), self.ctype())
+        elif self.type.is_extension_type and not self.type.is_pyobject and not self.result_in_temp():
+            code.put_cyincref(self.result())
 
     def make_owned_memoryviewslice(self, code):
         """
@@ -800,6 +802,8 @@ class ExprNode(Node):
                             self.result(), have_gil=not self.in_nogil_context)
                     code.putln("%s.memview = NULL;" % self.result())
                     code.putln("%s.data = NULL;" % self.result())
+                elif self.type.is_extension_type and not self.type.is_pyobject:
+                    code.put_cydecref(self.result())
         else:
             # Already done if self.is_temp
             self.generate_subexpr_disposal_code(code)
@@ -821,6 +825,8 @@ class ExprNode(Node):
             elif self.type.is_memoryviewslice:
                 code.putln("%s.memview = NULL;" % self.result())
                 code.putln("%s.data = NULL;" % self.result())
+            elif self.type.is_extension_type and not self.type.is_pyobject:
+                code.putln("%s = 0;" % self.result())
         else:
             self.generate_subexpr_disposal_code(code)
 
@@ -1784,6 +1790,8 @@ class ImagNode(AtomicExprNode):
                     float(self.value),
                     code.error_goto_if_null(self.result(), self.pos)))
             code.put_gotref(self.py_result())
+        elif self.type.is_extension_type:
+            code.put_cygotref(self.result())
 
 
 class NewExprNode(AtomicExprNode):
@@ -2263,8 +2271,9 @@ class NameNode(AtomicExprNode):
                         code.error_goto_if_null(self.result(), self.pos)))
             code.put_gotref(self.py_result())
 
-        elif entry.is_local and isinstance(entry.type, PyrexTypes.CythonExtensionType):
-            pass
+        elif entry.is_local and entry.type.is_extension_type and not entry.type.is_pyobject:
+            code.put_cygotref(self.result())
+            #pass
             # code.putln(entry.cname)
         elif entry.is_local or entry.in_closure or entry.from_closure or entry.type.is_memoryviewslice:
             # Raise UnboundLocalError for objects and memoryviewslices
@@ -2484,6 +2493,22 @@ class NameNode(AtomicExprNode):
                 else:
                     code.put_xdecref_memoryviewslice(self.entry.cname,
                                                      have_gil=not self.nogil)
+        elif self.entry.type.is_extension_type:
+            if not self.cf_is_null:
+                if self.cf_maybe_null and not ignore_nonexisting:
+                    code.put_error_if_unbound(self.pos, self.entry)
+
+                if self.entry.in_closure:
+                    # generator
+                    if ignore_nonexisting and self.cf_maybe_null:
+                        code.put_cyxgotref(self.result())
+                    else:
+                        code.put_cygotref(self.result())
+                if ignore_nonexisting and self.cf_maybe_null:
+                    code.put_cyxdecref(self.result())
+                else:
+                    code.put_cydecref(self.result())
+                code.putln('%s = NULL;' % self.result())
         else:
             error(self.pos, "Deletion of C names not supported")
 
@@ -4079,6 +4104,8 @@ class IndexNode(_IndexingBaseNode):
                     code.error_goto_if(error_check % self.result(), self.pos)))
         if self.type.is_pyobject:
             code.put_gotref(self.py_result())
+        elif self.type.is_extension_type:
+            code.put_cygotref(self.result())
 
     def generate_setitem_code(self, value_code, code):
         if self.index.type.is_int:
@@ -5982,8 +6009,11 @@ class SimpleCallNode(CallNode):
                     else:
                         goto_error = ""
                     code.putln("%s%s; %s" % (lhs, rhs, goto_error))
-                if self.type.is_pyobject and self.result():
-                    code.put_gotref(self.py_result())
+                if self.result():
+                    if self.type.is_pyobject:
+                        code.put_gotref(self.py_result())
+                    elif self.type.is_extension_type:
+                        code.put_cygotref(self.result())
             if self.has_optional_args:
                 code.funcstate.release_temp(self.opt_arg_struct)
 
@@ -8815,6 +8845,7 @@ class DictNode(ExprNode):
         self.allocate_temp_result(code)
         if hasattr(self.type, 'nogil') and self.type.nogil:
             code.putln("%s = (struct %s *)malloc(sizeof(struct %s));" % (self.result(), self.type.objstruct_cname, self.type.objstruct_cname))
+            code.put_cyincref(self.result()) # FIXME: or gotref ?
 
         is_dict = self.type.is_pyobject
         if is_dict:

Cython/Compiler/ModuleNode.py

@@ -654,6 +654,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             self._put_setup_code(code, "CInitCode")
         self._put_setup_code(code, "PythonCompatibility")
         self._put_setup_code(code, "MathInitCode")
+        self._put_setup_code(code, "CythonExtensionTypes")
 
         if options.c_line_in_traceback:
             cinfo = "%s = %s; " % (Naming.clineno_cname, Naming.line_c_macro)
@@ -1142,7 +1143,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 "// nogil"
             )
             code.putln(
-                "int ob_refcnt;" # "CyObject_HEAD;" Sometimes the CythonReferenceCounting was put after the nogil extension declaration, WTF!!!
+                "CyObject_HEAD"
             )
         else:
             code.putln(

Cython/Compiler/Nodes.py

@@ -960,6 +960,8 @@ class CArgDeclNode(Node):
         code.putln("%s = %s;" % (target, result))
         if self.type.is_pyobject:
             code.put_giveref(default.result())
+        elif self.type.is_extension_type:
+            code.put_cygiveref(default.result())
         default.generate_post_assignment_code(code)
         default.free_temps(code)
 
@@ -1972,6 +1974,9 @@ class FuncDefNode(StatNode, BlockNode):
             #       incref our arguments
             elif is_cdef and entry.type.is_memoryviewslice and len(entry.cf_assignments) > 1:
                 code.put_incref_memoryviewslice(entry.cname, have_gil=code.funcstate.gil_owned)
+            # We have to Cy_INCREF the nogil classes (ccdef'ed ones)
+            elif entry.type.is_extension_type and not entry.type.is_pyobject and len(entry.cf_assignments) > 1:
+                code.put_cyincref(entry.cname)
         for entry in lenv.var_entries:
             if entry.is_arg and len(entry.cf_assignments) > 1 and not entry.in_closure:
                 if entry.xdecref_cleanup:
@@ -2123,6 +2128,9 @@ class FuncDefNode(StatNode, BlockNode):
                         code.put_var_xdecref(entry)
                     else:
                         code.put_var_decref(entry)
+            elif entry.type.is_extension_type and not entry.type.is_pyobject and \
+                 (not entry.is_arg or len(entry.cf_assignments) > 1):
+                code.put_cydecref(entry.cname)
 
         # Decref any increfed args
         for entry in lenv.arg_entries:
@@ -2135,6 +2143,8 @@ class FuncDefNode(StatNode, BlockNode):
                 # functions, but not borrowed slices from cdef functions.
                 code.put_xdecref_memoryviewslice(entry.cname,
                                                  have_gil=not lenv.nogil)
+            elif entry.type.is_extension_type and not entry.type.is_pyobject and len(entry.cf_assignments) > 1:
+                code.put_cydecref(entry.cname)
         if self.needs_closure:
             code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type)
 
@@ -2147,6 +2157,9 @@ class FuncDefNode(StatNode, BlockNode):
                 err_val = default_retval  # FIXME: why is err_val not used?
             if self.return_type.is_pyobject:
                 code.put_xgiveref(self.return_type.as_pyobject(Naming.retval_cname))
+        # We can always return a CythonExtensionType as it is nogil-compliant
+        if self.return_type.is_extension_type and not self.return_type.is_pyobject:
+            code.put_cyxgiveref(Naming.retval_cname)
 
         if self.entry.is_special and self.entry.name == "__hash__":
             # Returning -1 for __hash__ is supposed to signal an error
@@ -5946,7 +5959,9 @@ class DelStatNode(StatNode):
                 code.putln("delete %s;" % arg.result())
                 arg.generate_disposal_code(code)
             elif arg.type.is_struct_or_union and hasattr(arg.type, "nogil") and arg.type.nogil:
-                code.putln("free(&%s);" % arg.result())
+                # Whenever there will be a proper GC:
+                code.putln("Cy_DECREF(%s);" % arg.result())
+                #code.putln("free(&%s);" % arg.result())
             # else error reported earlier
 
     def annotate(self, code):
@@ -6064,6 +6079,9 @@ class ReturnStatNode(StatNode):
                 # Use specialised default handling for "return None".
                 value = None
 
+        if self.return_type.is_extension_type and not self.return_type.is_pyobject:
+            code.put_cyxdecref(Naming.retval_cname)
+
         if value:
             value.generate_evaluation_code(code)
             if self.return_type.is_memoryviewslice:

Cython/Compiler/UtilNodes.py

@@ -198,10 +198,13 @@ class ResultRefNode(AtomicExprNode):
         pass
 
     def generate_assignment_code(self, rhs, code, overloaded_assignment=False):
-        if self.type.is_pyobject:
+        if self.type.is_pyobject or self.type.is_extension_type:
             rhs.make_owned_reference(code)
             if not self.lhs_of_first_assignment:
-                code.put_decref(self.result(), self.ctype())
+                if self.type.is_pyobject:
+                    code.put_decref(self.result(), self.ctype())
+                elif self.type.is_extension_type:
+                    code.put_cydecref(self.result())
         code.putln('%s = %s;' % (
             self.result(),
             rhs.result() if overloaded_assignment else rhs.result_as(self.ctype()),
@@ -248,6 +251,9 @@ class LetNodeMixin:
         else:
             if self.temp_type.is_pyobject:
                 code.put_decref_clear(self.temp, self.temp_type)
+            elif self.temp_type.is_extension_type:
+                pass
+                #code.put_decref_clear(self.temp, self.temp_type)
             code.funcstate.release_temp(self.temp)
 
 

Cython/Utility/ModuleSetupCode.c

@@ -1515,3 +1515,31 @@ static void __Pyx_FastGilFuncInit(void) {
 }
 
 #endif
+
+/////////////// CythonExtensionTypes ///////////////
+
+#if !defined(__GNUC__)
+#define GCC_VERSION (__GNUC__ * 10000 \
+                     + __GNUC_MINOR__ * 100 \
+                     + __GNUC_PATCHLEVEL__)
+/* Test for GCC > 4.9.0 */
+#if GCC_VERSION < 40900
+#error atomic.h works only with GCC newer than version 4.9
+#endif /* GNUC >= 4.9 */
+
+#endif /* Has GCC */
+
+#ifdef __cplusplus
+  #include <atomic>
+  using namespace std;
+  #define CyObject_ATOMIC_REFCOUNT_TYPE atomic_int
+#else
+  #include <stdatomic.h>
+  #define CyObject_ATOMIC_REFCOUNT_TYPE int
+#endif /* __cplusplus */
+
+/* CyObject_HEAD defines the initial segment of every CyObject. */
+#define CyObject_HEAD                  \
+    CyObject_ATOMIC_REFCOUNT_TYPE ob_refcnt;                  \
+    void (*cdealloc)(void* self);
+

Cython/Utility/ObjectHandling.c

@@ -1574,26 +1574,22 @@ try_unpack:
 
 /////////////// CythonReferenceCounting.proto ///////////////
 
-#include <stdlib.h>
-#include <stddef.h>
-
-#if !defined(__GNUC__)
-#define GCC_VERSION (__GNUC__ * 10000 \
-                     + __GNUC_MINOR__ * 100 \
-                     + __GNUC_PATCHLEVEL__)
-/* Test for GCC > 4.9.0 */
-#if GCC_VERSION < 40900
-#error atomic.h works only with GCC newer than version 4.9
-#endif /* GNUC >= 4.9 */
-
-#endif /* Has GCC */
-
-// #include <stdatomic.h>
+#ifdef __cplusplus
+  #include <cstdlib>
+  #include <cstddef>
+// atomic is already included in ModuleSetupCode
+//  #include <atomic>
+#else
+  #include <stdlib.h>
+  #include <stddef.h>
+//  #include <stdatomic.h>
+#endif /* __cplusplus */
 
+// Defined in ModuleSetupCode.c
 /* CyObject_HEAD defines the initial segment of every CyObject. */
-#define CyObject_HEAD                   \
-    int ob_refcnt;               \
-    void (*cdealloc)();
+//#define CyObject_HEAD                   \
+//    int ob_refcnt;               \
+//    void (*cdealloc)(void * self);
 
 struct CyObject {
   CyObject_HEAD
@@ -1602,12 +1598,16 @@ struct CyObject {
 /* Cast argument to PyObject* type. */
 #define _CyObject_CAST(op) ((struct CyObject*)(op))
 
-// XXX: Without scope analysis this is useless...
-/*
 static inline void _Cy_DECREF(struct CyObject *op) {
+//  int f = open("log_nogil", O_WRONLY|O_APPEND);
+//  dprintf(f, "DECREF ob_refcnt (before decref) = %d\n", op->ob_refcnt);
   if (atomic_fetch_sub(&(op->ob_refcnt), 1) == 1) {
-    op->cdealloc(op);
+    //op->cdealloc(op);
+    // DEBUG
+//    dprintf(f, "Freeing memory\n");
+    free(op);
   }
+//  close(f);
 }
 
 static inline void _Cy_INCREF(struct CyObject *op) {
@@ -1616,7 +1616,11 @@ static inline void _Cy_INCREF(struct CyObject *op) {
 
 #define Cy_INCREF(op) _Cy_INCREF(_CyObject_CAST(op))
 #define Cy_DECREF(op) _Cy_DECREF(_CyObject_CAST(op))
-*/
+#define Cy_XDECREF(op) do {if (op != NULL) {Cy_DECREF(op);}} while(0)
+#define Cy_GOTREF(op)
+#define Cy_XGOTREF(op)
+#define Cy_GIVEREF(op)
+#define Cy_XGIVEREF(op)
 
 /////////////// UnpackUnboundCMethod.proto ///////////////