Starting to add traverse and clear functions

from_cpython/Modules/gcmodule.c

@@ -382,6 +382,13 @@ subtract_refs(PyGC_Head *containers)
     PyGC_Head *gc = containers->gc.gc_next;
     for (; gc != containers; gc=gc->gc.gc_next) {
         traverse = Py_TYPE(FROM_GC(gc))->tp_traverse;
+        // Pyston addition: some extra checking for our transition
+#ifndef NDEBUG
+        if (!traverse) {
+            fprintf(stderr, "%s needs a tp_traverse\n", Py_TYPE(FROM_GC(gc))->tp_name);
+            assert(0);
+        }
+#endif
         (void) traverse(FROM_GC(gc),
                        (visitproc)visit_decref,
                        NULL);

src/capi/types.h

@@ -62,6 +62,32 @@ public:
             return boxString(s);
         return None;
     }
+
+    static void dealloc(Box* _o) noexcept {
+        BoxedCApiFunction* o = (BoxedCApiFunction*)_o;
+
+        _PyObject_GC_UNTRACK(o);
+        Py_XDECREF(o->module);
+        Py_XDECREF(o->passthrough);
+        o->cls->tp_free(o);
+    }
+
+    static int traverse(Box* _o, visitproc visit, void* arg) noexcept {
+        BoxedCApiFunction* o = (BoxedCApiFunction*)_o;
+
+        Py_VISIT(o->module);
+        Py_VISIT(o->passthrough);
+        return 0;
+    }
+
+    static int clear(Box* _o) noexcept {
+        BoxedCApiFunction* o = (BoxedCApiFunction*)_o;
+
+        Py_CLEAR(o->module);
+        Py_CLEAR(o->passthrough);
+        return 0;
+    }
+
 };
 static_assert(sizeof(BoxedCApiFunction) == sizeof(PyCFunctionObject), "");
 static_assert(offsetof(BoxedCApiFunction, method_def) == offsetof(PyCFunctionObject, m_ml), "");

src/core/types.h

@@ -590,11 +590,25 @@ public:
     AttrList* attr_list;
 
     HCAttrs(HiddenClass* hcls = root_hcls) : hcls(hcls), attr_list(nullptr) {}
+
+    int traverse(visitproc visit, void* arg) noexcept;
 };
 static_assert(sizeof(HCAttrs) == sizeof(struct _hcattrs), "");
 
+#define Py_VISIT_HCATTRS(hcattrs)                                                                                      \
+    do {                                                                                                               \
+        int vret = hcattrs.traverse(visit, arg);                                                                      \
+        if (vret)                                                                                                      \
+            return vret;                                                                                               \
+    } while (0)
+
+
 extern std::vector<BoxedClass*> classes;
 
+// Debugging helper: pass this as a tp_clear function to say that you have explicitly verified
+// that you don't need a tp_clear (as opposed to haven't added one yet)
+#define NOCLEAR ((inquiry)-1)
+
 class BoxedDict;
 class BoxedString;
 

src/runtime/builtin_modules/sys.cpp

@@ -712,7 +712,8 @@ void setupSys() {
     sys_module->giveAttr("maxint", boxInt(PYSTON_INT_MAX));
     sys_module->giveAttr("maxsize", boxInt(PY_SSIZE_T_MAX));
 
-    sys_flags_cls = BoxedClass::create(type_cls, object_cls, 0, 0, sizeof(BoxedSysFlags), false, "flags");
+    sys_flags_cls
+        = BoxedClass::create(type_cls, object_cls, 0, 0, sizeof(BoxedSysFlags), false, "flags", NULL, NULL, false);
     sys_flags_cls->giveAttr(
         "__new__", new BoxedFunction(FunctionMetadata::create((void*)BoxedSysFlags::__new__, UNKNOWN, 1, true, true)));
     sys_flags_cls->tp_dealloc = (destructor)BoxedSysFlags::dealloc;

src/runtime/descr.cpp

@@ -430,6 +430,38 @@ Box* BoxedMethodDescriptor::descr_get(BoxedMethodDescriptor* self, Box* inst, Bo
         return boxInstanceMethod(inst, self, self->type);
 }
 
+void BoxedMethodDescriptor::dealloc(Box* _self) noexcept {
+    BoxedMethodDescriptor* self = static_cast<BoxedMethodDescriptor*>(_self);
+
+    PyObject_GC_UnTrack(self);
+    Py_XDECREF(self->type);
+    self->cls->tp_free(self);
+}
+
+int BoxedMethodDescriptor::traverse(Box* _self, visitproc visit, void *arg) noexcept {
+    BoxedMethodDescriptor* self = static_cast<BoxedMethodDescriptor*>(_self);
+
+    Py_VISIT(self->type);
+    return 0;
+}
+
+void BoxedWrapperObject::dealloc(Box* _self) noexcept {
+    BoxedWrapperObject* self = static_cast<BoxedWrapperObject*>(_self);
+
+    PyObject_GC_UnTrack(self);
+    Py_XDECREF(self->obj);
+    Py_XDECREF(self->descr);
+    self->cls->tp_free(self);
+}
+
+int BoxedWrapperObject::traverse(Box* _self, visitproc visit, void *arg) noexcept {
+    BoxedWrapperObject* self = static_cast<BoxedWrapperObject*>(_self);
+
+    Py_VISIT(self->obj);
+    Py_VISIT(self->descr);
+    return 0;
+}
+
 Box* BoxedWrapperDescriptor::descr_get(Box* _self, Box* inst, Box* owner) noexcept {
     STAT_TIMER(t0, "us_timer_boxedwrapperdescriptor_descr_get", 20);
 

src/runtime/dict.cpp

@@ -763,6 +763,23 @@ void BoxedDict::dealloc(Box* b) noexcept {
     static_cast<BoxedDict*>(b)->d.freeAllMemory();
 }
 
+int BoxedDict::traverse(PyObject* op, visitproc visit, void* arg) noexcept {
+    Py_ssize_t i = 0;
+    PyObject* pk;
+    PyObject* pv;
+
+    while (PyDict_Next(op, &i, &pk, &pv)) {
+        Py_VISIT(pk);
+        Py_VISIT(pv);
+    }
+    return 0;
+}
+
+int BoxedDict::clear(PyObject* op) noexcept {
+    PyDict_Clear(op);
+    return 0;
+}
+
 extern "C" void _PyDict_MaybeUntrack(PyObject* op) noexcept {
     PyDictObject* mp;
     PyObject* value;

src/runtime/list.cpp

@@ -1258,6 +1258,63 @@ extern "C" int PyList_SetSlice(PyObject* a, Py_ssize_t ilow, Py_ssize_t ihigh, P
     }
 }
 
+void BoxedList::dealloc(Box* b) noexcept {
+    BoxedList* op = static_cast<BoxedList*>(b);
+
+    Py_ssize_t i;
+    PyObject_GC_UnTrack(op);
+    Py_TRASHCAN_SAFE_BEGIN(op)
+    if (op->elts != NULL) {
+        /* Do it backwards, for Christian Tismer.
+           There's a simple test case where somehow this reduces
+           thrashing when a *very* large list is created and
+           immediately deleted. */
+        i = Py_SIZE(op);
+        while (--i >= 0) {
+            Py_XDECREF(op->elts->elts[i]);
+        }
+        //PyMem_FREE(op->elts);
+    }
+#if 0
+    if (numfree < PyList_MAXFREELIST && PyList_CheckExact(op))
+        free_list[numfree++] = op;
+    else
+#endif
+    Py_TYPE(op)->tp_free((PyObject*)op);
+    Py_TRASHCAN_SAFE_END(op)
+}
+
+int BoxedList::traverse(Box* _o, visitproc visit, void* arg) noexcept {
+    PyListObject* o = (PyListObject*)_o;
+    Py_ssize_t i;
+
+    for (i = Py_SIZE(o); --i >= 0; )
+        Py_VISIT(o->ob_item[i]);
+    return 0;
+}
+
+int BoxedList::clear(Box* _a) noexcept {
+    PyListObject* a = (PyListObject*)_a;
+    Py_ssize_t i;
+    PyObject **item = a->ob_item;
+    if (item != NULL) {
+        /* Because XDECREF can recursively invoke operations on
+           this list, we make it empty first. */
+        i = Py_SIZE(a);
+        Py_SIZE(a) = 0;
+        a->ob_item = NULL;
+        a->allocated = 0;
+        while (--i >= 0) {
+            Py_XDECREF(item[i]);
+        }
+        PyMem_FREE(item);
+    }
+    /* Never fails; the return value can be ignored.
+       Note that there is no guarantee that the list is actually empty
+       at this point, because XDECREF may have populated it again! */
+    return 0;
+}
+
 void setupList() {
     static PySequenceMethods list_as_sequence;
     list_cls->tp_as_sequence = &list_as_sequence;

src/runtime/objmodel.cpp

@@ -400,7 +400,8 @@ void BoxedClass::freeze() {
 
 std::vector<BoxedClass*> classes;
 BoxedClass::BoxedClass(BoxedClass* base, int attrs_offset, int weaklist_offset, int instance_size, bool is_user_defined,
-                       const char* name, destructor dealloc, freefunc free, bool is_gc)
+                       const char* name, destructor dealloc, freefunc free, bool is_gc, traverseproc traverse,
+                       inquiry clear)
     : attrs(HiddenClass::makeSingleton()),
       attrs_offset(attrs_offset),
       is_constant(false),
@@ -410,6 +411,15 @@ BoxedClass::BoxedClass(BoxedClass* base, int attrs_offset, int weaklist_offset,
       has_instancecheck(false),
       tpp_call(NULL, NULL) {
 
+    bool ok_noclear = (clear == NOCLEAR);
+    if (ok_noclear)
+        clear = NULL;
+    if (is_gc) {
+        ASSERT(traverse, "%s", name);
+        ASSERT(dealloc, "%s", name);
+    }
+    ASSERT(((bool)traverse == (bool)clear) || ok_noclear, "%s", name);
+
     classes.push_back(this);
 
     // Zero out the CPython tp_* slots:
@@ -453,6 +463,22 @@ BoxedClass::BoxedClass(BoxedClass* base, int attrs_offset, int weaklist_offset,
         assert(cls == type_cls || PyType_Check(this));
     }
 
+    tp_traverse = traverse;
+    tp_clear = clear;
+    if (base && !PyType_IS_GC(this) & PyType_IS_GC(base) && !traverse && !tp_clear) {
+        assert(tp_flags & Py_TPFLAGS_HAVE_RICHCOMPARE);
+
+        tp_flags |= Py_TPFLAGS_HAVE_GC;
+        assert(tp_free != PyObject_Del);
+        if (!tp_traverse)
+            tp_traverse = base->tp_traverse;
+        if (!tp_clear)
+            tp_clear = base->tp_clear;
+    }
+
+    ASSERT((bool)tp_traverse == PyType_IS_GC(this), "%s missing traverse", tp_name);
+    ASSERT(((bool)tp_clear == PyType_IS_GC(this)) || ok_noclear, "%s missing clear", tp_name);
+
     if (dealloc)
         tp_dealloc = dealloc;
     else {
@@ -470,9 +496,6 @@ BoxedClass::BoxedClass(BoxedClass* base, int attrs_offset, int weaklist_offset,
             this->tp_free = PyObject_GC_Del;
     }
 
-    if (PyType_IS_GC(this))
-        assert(tp_free != PyObject_Del);
-
     assert(tp_dealloc);
     assert(tp_free);
 
@@ -500,10 +523,10 @@ BoxedClass::BoxedClass(BoxedClass* base, int attrs_offset, int weaklist_offset,
 
 BoxedClass* BoxedClass::create(BoxedClass* metaclass, BoxedClass* base, int attrs_offset, int weaklist_offset,
                                int instance_size, bool is_user_defined, const char* name, destructor dealloc,
-                               freefunc free, bool is_gc) {
+                               freefunc free, bool is_gc, traverseproc traverse, inquiry clear) {
     assert(!is_user_defined);
-    BoxedClass* made = new (metaclass, 0)
-        BoxedClass(base, attrs_offset, weaklist_offset, instance_size, is_user_defined, name, dealloc, free, is_gc);
+    BoxedClass* made = new (metaclass, 0) BoxedClass(base, attrs_offset, weaklist_offset, instance_size,
+                                                     is_user_defined, name, dealloc, free, is_gc, traverse, clear);
 
     // While it might be ok if these were set, it'd indicate a difference in
     // expectations as to who was going to calculate them:
@@ -518,13 +541,6 @@ BoxedClass* BoxedClass::create(BoxedClass* metaclass, BoxedClass* base, int attr
 }
 
 void BoxedClass::finishInitialization() {
-    assert(!tp_traverse);
-    assert(!tp_clear);
-    if (tp_base) {
-        tp_traverse = tp_base->tp_traverse;
-        tp_clear = tp_base->tp_clear;
-    }
-
     assert(!this->tp_dict);
     this->tp_dict = incref(this->getAttrWrapper());
 

src/runtime/types.h

@@ -277,10 +277,12 @@ public:
     // These should only be used for builtin types:
     static BoxedClass* create(BoxedClass* metatype, BoxedClass* base, int attrs_offset, int weaklist_offset,
                               int instance_size, bool is_user_defined, const char* name, destructor dealloc = NULL,
-                              freefunc free = NULL, bool is_gc = true);
+                              freefunc free = NULL, bool is_gc = true, traverseproc traverse = NULL,
+                              inquiry clear = NULL);
 
-    BoxedClass(BoxedClass* base, int attrs_offset, int weaklist_offset, int instance_size,
-               bool is_user_defined, const char* name, destructor dealloc, freefunc free, bool is_gc = true);
+    BoxedClass(BoxedClass* base, int attrs_offset, int weaklist_offset, int instance_size, bool is_user_defined,
+               const char* name, destructor dealloc, freefunc free, bool is_gc = true, traverseproc traverse = NULL,
+               inquiry clear = NULL);
 
 
     DEFAULT_CLASS_VAR(type_cls, sizeof(SlotOffset));
@@ -498,6 +500,7 @@ public:
     DEFAULT_CLASS_SIMPLE(instancemethod_cls, true);
 
     static void dealloc(Box* self) noexcept;
+    static int traverse(Box* self, visitproc visit, void *arg) noexcept;
 };
 
 class GCdArray {
@@ -534,6 +537,8 @@ public:
     DEFAULT_CLASS_SIMPLE(list_cls, true);
 
     static void dealloc(Box* self) noexcept;
+    static int traverse(Box* self, visitproc visit, void *arg) noexcept;
+    static int clear(Box* self) noexcept;
 };
 static_assert(sizeof(BoxedList) <= sizeof(PyListObject), "");
 static_assert(sizeof(BoxedList) >= sizeof(PyListObject), "");
@@ -824,6 +829,8 @@ public:
     iterator end() { return iterator(d.end()); }
 
     static void dealloc(Box* b) noexcept;
+    static int traverse(Box* self, visitproc visit, void *arg) noexcept;
+    static int clear(Box* self) noexcept;
 };
 static_assert(sizeof(BoxedDict) == sizeof(PyDictObject), "");
 
@@ -896,6 +903,7 @@ public:
     Box* getLongConstant(llvm::StringRef s);
 
     static void dealloc(Box* b) noexcept;
+    static int traverse(Box* self, visitproc visit, void *arg) noexcept;
 
 private:
     ContiguousMap<llvm::StringRef, BoxedString*, llvm::StringMap<int>> str_constants;
@@ -920,6 +928,8 @@ public:
     Box* start, *stop, *step;
     BoxedSlice(Box* lower, Box* upper, Box* step) : start(lower), stop(upper), step(step) {}
 
+    static void dealloc(Box* b) noexcept;
+
     DEFAULT_CLASS_SIMPLE(slice_cls, true);
 };
 static_assert(sizeof(BoxedSlice) == sizeof(PySliceObject), "");
@@ -959,7 +969,7 @@ public:
     BoxedMemberDescriptor(PyMemberDef* member)
         : type((MemberType)member->type), offset(member->offset), readonly(member->flags & READONLY) {}
 
-    DEFAULT_CLASS_SIMPLE(member_descriptor_cls, true);
+    DEFAULT_CLASS_SIMPLE(member_descriptor_cls, false);
 };
 
 class BoxedGetsetDescriptor : public Box {
@@ -971,6 +981,9 @@ public:
     BoxedGetsetDescriptor(Box* (*get)(Box*, void*), void (*set)(Box*, Box*, void*), void* closure)
         : get(get), set(set), closure(closure) {}
 
+    static void dealloc(Box* b) noexcept;
+    static int traverse(Box* self, visitproc visit, void *arg) noexcept;
+
     // No DEFAULT_CLASS annotation here -- force callers to explicitly specifiy pyston_getset_cls or capi_getset_cls
 };
 
@@ -1097,6 +1110,9 @@ public:
     template <ExceptionStyle S>
     static Box* tppCall(Box* _self, CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2, Box* arg3,
                         Box** args, const std::vector<BoxedString*>* keyword_names) noexcept(S == CAPI);
+
+    static void dealloc(Box* self) noexcept;
+    static int traverse(Box* self, visitproc visit, void *arg) noexcept;
 };
 
 class BoxedMethodDescriptor : public Box {
@@ -1113,6 +1129,9 @@ public:
     template <ExceptionStyle S>
     static Box* tppCall(Box* _self, CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2, Box* arg3,
                         Box** args, const std::vector<BoxedString*>* keyword_names) noexcept(S == CAPI);
+
+    static void dealloc(Box* self) noexcept;
+    static int traverse(Box* self, visitproc visit, void *arg) noexcept;
 };
 
 Box* objectSetattr(Box* obj, Box* attr, Box* value);