It runs!

from_cpython/Objects/weakrefobject.c

@@ -35,9 +35,7 @@ new_weakref(PyObject *ob, PyObject *callback)
 {
     PyWeakReference *result;
 
-    // Pyston change: We can't use PyObject_GC_New because it will conservatively scan the memory.
-    // result = PyObject_GC_New(PyWeakReference, &_PyWeakref_RefType);
-    result = (PyWeakReference*)_PyWeakref_RefType.tp_alloc(&_PyWeakref_RefType, 0);
+    result = PyObject_GC_New(PyWeakReference, &_PyWeakref_RefType);
     if (result) {
         init_weakref(result, ob, callback);
         PyObject_GC_Track(result);
@@ -544,9 +542,7 @@ proxy_dealloc(PyWeakReference *self)
     if (self->wr_callback != NULL)
         PyObject_GC_UnTrack((PyObject *)self);
     clear_weakref(self);
-    // Pyston change: we monkey-patch this function to use the Pyston GC
-    // Shouldn't need to call tp_free either.
-    // PyObject_GC_Del(self);
+    PyObject_GC_Del(self);
 }
 
 /* sequence slots */
@@ -922,8 +918,7 @@ PyObject_ClearWeakRefs(PyObject *object)
 
     if (object == NULL
         || !PyType_SUPPORTS_WEAKREFS(Py_TYPE(object))
-        //|| object->ob_refcnt != 0
-        ) {
+        || object->ob_refcnt != 0) {
         PyErr_BadInternalCall();
         return;
     }
@@ -948,10 +943,7 @@ PyObject_ClearWeakRefs(PyObject *object)
             current->wr_callback = NULL;
             clear_weakref(current);
             if (callback != NULL) {
-                // Pyston change:
-                // current is a stack reference to a GC allocated object.  If it wasn't null when we fetched it from *list, it won't
-                // be collected, and we can trust that it's still valid here.
-                if (1 /*current->ob_refcnt > 0*/)
+                if (current->ob_refcnt > 0)
                     handle_callback(current, callback);
                 Py_DECREF(callback);
             }
@@ -970,10 +962,7 @@ PyObject_ClearWeakRefs(PyObject *object)
             for (i = 0; i < count; ++i) {
                 PyWeakReference *next = current->wr_next;
 
-                // Pyston change:
-                // current is a stack reference to a GC allocated object.  If it wasn't null when we fetched it from *list, it won't
-                // be collected, and we can trust that it's still valid here.
-                if (1 /*current->ob_refcnt > 0*/)
+                if (current->ob_refcnt > 0)
                 {
                     Py_INCREF(current);
                     PyTuple_SET_ITEM(tuple, i * 2, (PyObject *) current);

src/capi/typeobject.cpp

@@ -3473,9 +3473,6 @@ extern "C" int PyType_Ready(PyTypeObject* cls) noexcept {
             cls->tp_as_buffer = base->tp_as_buffer;
     }
 
-    if (cls->tp_alloc == &PystonType_GenericAlloc)
-        cls->tp_alloc = &PyType_GenericAlloc;
-
     cls->is_user_defined = true;
 
 

src/codegen/unwinding.cpp

@@ -538,7 +538,7 @@ class PythonUnwindSession : public Box {
     Timer t;
 
 public:
-    DEFAULT_CLASS_SIMPLE(unwind_session_cls);
+    DEFAULT_CLASS_SIMPLE(unwind_session_cls, true);
 
     PythonUnwindSession() : exc_info(NULL, NULL, NULL), t(/*min_usec=*/10000) {}
 

src/core/types.h

@@ -672,9 +672,6 @@ public:
 };
 static_assert(offsetof(Box, cls) == offsetof(struct _object, ob_type), "");
 
-// Our default for tp_alloc:
-extern "C" PyObject* PystonType_GenericAlloc(BoxedClass* cls, Py_ssize_t nitems) noexcept;
-
 // These are some macros for tying the C++ type hiercharchy to the Pyston type hiercharchy.
 // Classes that inherit from Box have a special operator new() that takes a class object (as
 // a BoxedClass*) since the class is necessary for object allocation.
@@ -726,33 +723,46 @@ extern "C" PyObject* PystonType_GenericAlloc(BoxedClass* cls, Py_ssize_t nitems)
 // In the simple cases, we can inline the fast paths of the following methods and improve allocation speed quite a bit:
 // - Box::operator new
 // - cls->tp_alloc
-// - PystonType_GenericAlloc
+// - PyType_GenericAlloc
 // - PyObject_Init
 // The restrictions on when you can use the SIMPLE (ie fast) variant are encoded as
 // asserts in the 1-arg operator new function:
-#define DEFAULT_CLASS_SIMPLE(default_cls)                                                                              \
+#define DEFAULT_CLASS_SIMPLE(default_cls, is_gc)                                                                       \
     void* operator new(size_t size, BoxedClass * cls) __attribute__((visibility("default"))) {                         \
         return Box::operator new(size, cls);                                                                           \
     }                                                                                                                  \
     void* operator new(size_t size) __attribute__((visibility("default"))) {                                           \
         ALLOC_STATS(default_cls);                                                                                      \
-        assert(default_cls->tp_alloc == PystonType_GenericAlloc);                                                      \
+        assert(default_cls->tp_alloc == PyType_GenericAlloc);                                                          \
         assert(default_cls->tp_itemsize == 0);                                                                         \
         assert(default_cls->tp_basicsize == size);                                                                     \
         assert(default_cls->is_pyston_class);                                                                          \
         assert(default_cls->attrs_offset == 0);                                                                        \
+        assert(is_gc == PyType_IS_GC(default_cls));                                                                    \
+        bool is_heaptype = false;                                                                                      \
+        assert(is_heaptype == (bool)(default_cls->tp_flags & Py_TPFLAGS_HEAPTYPE));                                    \
                                                                                                                        \
         /* Don't allocate classes through this -- we need to keep track of all class objects. */                       \
         assert(default_cls != type_cls);                                                                               \
                                                                                                                        \
         /* note: we want to use size instead of tp_basicsize, since size is a compile-time constant */                 \
-        void* mem = PyObject_MALLOC(size);                                                                \
+        void* mem;                                                                                                     \
+        if (is_gc)                                                                                                     \
+            mem = _PyObject_GC_Malloc(size);                                                                           \
+        else                                                                                                           \
+            mem = PyObject_MALLOC(size);                                                                               \
         assert(mem);                                                                                                   \
                                                                                                                        \
         Box* rtn = static_cast<Box*>(mem);                                                                             \
                                                                                                                        \
-        rtn->cls = default_cls;                                                                                        \
-        _Py_NewReference(rtn);                                                                                         \
+        if (is_heaptype)                                                                                               \
+            Py_INCREF(default_cls);                                                                                    \
+                                                                                                                       \
+        PyObject_INIT(rtn, default_cls);                                                                               \
+                                                                                                                       \
+        if (is_gc)                                                                                                     \
+            _PyObject_GC_TRACK(rtn);                                                                                   \
+                                                                                                                       \
         return rtn;                                                                                                    \
         /* TODO: there should be a way to not have to do this nested inlining by hand */                               \
     }
@@ -801,7 +811,7 @@ static_assert(offsetof(BoxVar, ob_size) == offsetof(struct _varobject, ob_size),
     }                                                                                                                  \
     void* operator new(size_t size, size_t nitems) __attribute__((visibility("default"))) {                            \
         ALLOC_STATS_VAR(default_cls)                                                                                   \
-        assert(default_cls->tp_alloc == PystonType_GenericAlloc);                                                      \
+        assert(default_cls->tp_alloc == PyType_GenericAlloc);                                                      \
         assert(default_cls->tp_itemsize == itemsize);                                                                  \
         assert(default_cls->tp_basicsize == size);                                                                     \
         assert(default_cls->is_pyston_class);                                                                          \

src/runtime/objmodel.cpp

@@ -439,7 +439,7 @@ BoxedClass::BoxedClass(BoxedClass* base, int attrs_offset, int weaklist_offset,
         tp_alloc = tp_base->tp_alloc;
     } else {
         assert(this == object_cls);
-        tp_alloc = PystonType_GenericAlloc;
+        tp_alloc = PyType_GenericAlloc;
     }
 
     if (cls == NULL) {
@@ -5989,10 +5989,7 @@ Box* _typeNew(BoxedClass* metatype, BoxedString* name, BoxedTuple* bases, BoxedD
 
     fixup_slot_dispatchers(made);
 
-    if (base->tp_alloc == &PystonType_GenericAlloc)
-        made->tp_alloc = PystonType_GenericAlloc;
-    else
-        made->tp_alloc = PyType_GenericAlloc;
+    made->tp_alloc = PyType_GenericAlloc;
 
     // On some occasions, Python-implemented classes inherit from C-implement classes. For
     // example, KeyedRef inherits from weakref, and needs to have it's finalizer called

src/runtime/types.cpp

@@ -95,62 +95,6 @@ void setupGC();
 
 std::vector<BoxedClass*> exception_types;
 
-// Analogue of PyType_GenericAlloc (default tp_alloc), but should only be used for Pyston classes!
-extern "C" PyObject* PystonType_GenericAlloc(BoxedClass* cls, Py_ssize_t nitems) noexcept {
-    assert(cls);
-
-    // See PyType_GenericAlloc for note about the +1 here:
-    const size_t size = _PyObject_VAR_SIZE(cls, nitems + 1);
-
-#ifndef NDEBUG
-#if 0
-    assert(cls->tp_bases);
-    // TODO this should iterate over all subclasses
-    for (auto e : cls->tp_bases->pyElements()) {
-        assert(e->cls == type_cls); // what about old style classes?
-        assert(static_cast<BoxedClass*>(e)->is_pyston_class);
-    }
-#endif
-    if (!cls->tp_mro) {
-        // wrapperdescr_cls is the last class to be set up during bootstrapping:
-        ASSERT(!wrapperdescr_cls, "looks like we need to set up the mro for %s manually", cls->tp_name);
-    } else {
-        assert(cls->tp_mro && "maybe we should just skip these checks if !mro");
-        assert(cls->tp_mro->cls == tuple_cls);
-        for (auto b : *static_cast<BoxedTuple*>(cls->tp_mro)) {
-            // old-style classes are always pyston classes:
-            if (b->cls == classobj_cls)
-                continue;
-            assert(PyType_Check(b));
-            ASSERT(static_cast<BoxedClass*>(b)->is_pyston_class, "%s (%s)", cls->tp_name,
-                   static_cast<BoxedClass*>(b)->tp_name);
-        }
-    }
-#endif
-
-    void* mem;
-    if (PyType_IS_GC(cls))
-        mem = _PyObject_GC_Malloc(size);
-    else
-        mem = PyObject_MALLOC(size);
-    RELEASE_ASSERT(mem, "");
-
-    // Not sure if we can get away with not initializing this memory.
-    // I think there are small optimizations we can do, like not initializing cls (always
-    // the first 8 bytes) since it will get written by PyObject_Init.
-    memset(mem, '\0', size);
-
-    Box* rtn = static_cast<Box*>(mem);
-
-    if (cls->tp_itemsize != 0)
-        static_cast<BoxVar*>(rtn)->ob_size = nitems;
-
-    PyObject_INIT(rtn, cls);
-    assert(rtn->cls);
-
-    return rtn;
-}
-
 extern "C" PyObject* PyType_GenericAlloc(PyTypeObject* type, Py_ssize_t nitems) noexcept {
     PyObject* obj;
     const size_t size = _PyObject_VAR_SIZE(type, nitems + 1);
@@ -3558,11 +3502,16 @@ void setupRuntime() {
     file_cls->tp_dealloc = file_dealloc;
     int_cls = new (0) BoxedClass(object_cls, 0, 0, sizeof(BoxedInt), false, "int");
     int_cls->tp_flags |= Py_TPFLAGS_INT_SUBCLASS;
+    int_cls->tp_flags &= ~Py_TPFLAGS_HAVE_GC;
     bool_cls = new (0) BoxedClass(int_cls, 0, 0, sizeof(BoxedBool), false, "bool");
+    bool_cls->tp_flags &= ~Py_TPFLAGS_HAVE_GC;
     complex_cls = new (0) BoxedClass(object_cls, 0, 0, sizeof(BoxedComplex), false, "complex");
+    complex_cls->tp_flags &= ~Py_TPFLAGS_HAVE_GC;
     long_cls = new (0) BoxedClass(object_cls, 0, 0, sizeof(BoxedLong), false, "long");
     long_cls->tp_flags |= Py_TPFLAGS_LONG_SUBCLASS;
+    long_cls->tp_flags &= ~Py_TPFLAGS_HAVE_GC;
     float_cls = new (0) BoxedClass(object_cls, 0, 0, sizeof(BoxedFloat), false, "float");
+    float_cls->tp_flags &= ~Py_TPFLAGS_HAVE_GC;
     function_cls = new (0)
         BoxedClass(object_cls, offsetof(BoxedFunction, attrs),
                    offsetof(BoxedFunction, in_weakreflist), sizeof(BoxedFunction), false, "function");

src/runtime/types.h

@@ -379,7 +379,7 @@ public:
 
     BoxedInt(int64_t n) __attribute__((visibility("default"))) : n(n) {}
 
-    DEFAULT_CLASS_SIMPLE(int_cls);
+    DEFAULT_CLASS_SIMPLE(int_cls, false);
 };
 
 class BoxedFloat : public Box {
@@ -388,7 +388,7 @@ public:
 
     BoxedFloat(double d) __attribute__((visibility("default"))) : d(d) {}
 
-    DEFAULT_CLASS_SIMPLE(float_cls);
+    DEFAULT_CLASS_SIMPLE(float_cls, false);
 };
 static_assert(sizeof(BoxedFloat) == sizeof(PyFloatObject), "");
 static_assert(offsetof(BoxedFloat, d) == offsetof(PyFloatObject, ob_fval), "");
@@ -400,14 +400,14 @@ public:
 
     BoxedComplex(double r, double i) __attribute__((visibility("default"))) : real(r), imag(i) {}
 
-    DEFAULT_CLASS_SIMPLE(complex_cls);
+    DEFAULT_CLASS_SIMPLE(complex_cls, false);
 };
 
 class BoxedBool : public BoxedInt {
 public:
     BoxedBool(bool b) __attribute__((visibility("default"))) : BoxedInt(b ? 1 : 0) {}
 
-    DEFAULT_CLASS_SIMPLE(bool_cls);
+    DEFAULT_CLASS_SIMPLE(bool_cls, false);
 };
 
 class BoxedString : public BoxVar {
@@ -436,7 +436,7 @@ public:
     void* operator new(size_t size, size_t nitems) __attribute__((visibility("default"))) {
         ALLOC_STATS_VAR(str_cls)
 
-        assert(str_cls->tp_alloc == PystonType_GenericAlloc);
+        assert(str_cls->tp_alloc == PyType_GenericAlloc);
         assert(str_cls->tp_itemsize == 1);
         assert(str_cls->tp_basicsize == offsetof(BoxedString, s_data) + 1);
         assert(str_cls->is_pyston_class);
@@ -491,7 +491,7 @@ public:
     BoxedInstanceMethod(Box* obj, Box* func, Box* im_class) __attribute__((visibility("default")))
     : in_weakreflist(NULL), obj(obj), func(func), im_class(im_class) {}
 
-    DEFAULT_CLASS_SIMPLE(instancemethod_cls);
+    DEFAULT_CLASS_SIMPLE(instancemethod_cls, true);
 };
 
 class GCdArray {
@@ -525,7 +525,7 @@ public:
     void shrink();
     static const int INITIAL_CAPACITY;
 
-    DEFAULT_CLASS_SIMPLE(list_cls);
+    DEFAULT_CLASS_SIMPLE(list_cls, true);
 
     static void dealloc(Box* self) noexcept;
 };
@@ -678,19 +678,15 @@ public:
     void* operator new(size_t size, size_t nitems) __attribute__((visibility("default"))) {
         ALLOC_STATS_VAR(tuple_cls)
 
-        assert(tuple_cls->tp_alloc == PystonType_GenericAlloc);
+        assert(tuple_cls->tp_alloc == PyType_GenericAlloc);
         assert(tuple_cls->tp_itemsize == sizeof(Box*));
         assert(tuple_cls->tp_basicsize == offsetof(BoxedTuple, elts));
         assert(tuple_cls->is_pyston_class);
         assert(tuple_cls->attrs_offset == 0);
 
-        void* mem = PyObject_MALLOC(offsetof(BoxedTuple, elts) + nitems * sizeof(Box*));
-        assert(mem);
+        BoxVar* rtn = static_cast<BoxVar*>(PyObject_GC_NewVar(BoxedTuple, &PyTuple_Type, nitems));
+        assert(rtn);
 
-        BoxVar* rtn = static_cast<BoxVar*>(mem);
-        rtn->cls = tuple_cls;
-        rtn->ob_size = nitems;
-        _Py_NewReference(rtn);
         return rtn;
     }
 
@@ -792,7 +788,7 @@ public:
 
     BoxedDict() __attribute__((visibility("default"))) {}
 
-    DEFAULT_CLASS_SIMPLE(dict_cls);
+    DEFAULT_CLASS_SIMPLE(dict_cls, true);
 
     Box* getOrNull(Box* k) {
         const auto& p = d.find(BoxAndHash(k));
@@ -918,7 +914,7 @@ public:
     Box* start, *stop, *step;
     BoxedSlice(Box* lower, Box* upper, Box* step) : start(lower), stop(upper), step(step) {}
 
-    DEFAULT_CLASS_SIMPLE(slice_cls);
+    DEFAULT_CLASS_SIMPLE(slice_cls, true);
 };
 static_assert(sizeof(BoxedSlice) == sizeof(PySliceObject), "");
 static_assert(offsetof(BoxedSlice, start) == offsetof(PySliceObject, start), "");
@@ -957,7 +953,7 @@ public:
     BoxedMemberDescriptor(PyMemberDef* member)
         : type((MemberType)member->type), offset(member->offset), readonly(member->flags & READONLY) {}
 
-    DEFAULT_CLASS_SIMPLE(member_descriptor_cls);
+    DEFAULT_CLASS_SIMPLE(member_descriptor_cls, true);
 };
 
 class BoxedGetsetDescriptor : public Box {
@@ -983,7 +979,7 @@ public:
     BoxedProperty(Box* get, Box* set, Box* del, Box* doc)
         : prop_get(get), prop_set(set), prop_del(del), prop_doc(doc) {}
 
-    DEFAULT_CLASS_SIMPLE(property_cls);
+    DEFAULT_CLASS_SIMPLE(property_cls, true);
 };
 
 class BoxedStaticmethod : public Box {
@@ -992,7 +988,7 @@ public:
 
     BoxedStaticmethod(Box* callable) : sm_callable(callable){};
 
-    DEFAULT_CLASS_SIMPLE(staticmethod_cls);
+    DEFAULT_CLASS_SIMPLE(staticmethod_cls, true);
 };
 
 class BoxedClassmethod : public Box {
@@ -1001,7 +997,7 @@ public:
 
     BoxedClassmethod(Box* callable) : cm_callable(callable){};
 
-    DEFAULT_CLASS_SIMPLE(classmethod_cls);
+    DEFAULT_CLASS_SIMPLE(classmethod_cls, true);
 };
 
 // TODO is there any particular reason to make this a Box, i.e. a python-level object?
@@ -1123,9 +1119,6 @@ BoxedDict* attrwrapperToDict(Box* b);
 Box* boxAst(AST* ast);
 AST* unboxAst(Box* b);
 
-// Our default for tp_alloc:
-extern "C" PyObject* PystonType_GenericAlloc(BoxedClass* cls, Py_ssize_t nitems) noexcept;
-
 #define fatalOrError(exception, message)                                                                               \
     do {                                                                                                               \
         if (CONTINUE_AFTER_FATAL)                                                                                      \

src/runtime/util.cpp

@@ -133,8 +133,22 @@ extern "C" void dumpEx(void* p, int levels) {
     printf("\n");
     printf("Raw address: %p\n", p);
 
-    if (true) {
-        printf("Python object\n");
+    if ((((intptr_t)p) & 0x7) == 0) {
+        uint8_t lowbyte = *reinterpret_cast<uint8_t*>(p);
+        if (lowbyte == 0xcb) {
+            printf("Uninitialized memory\n");
+            return;
+        }
+        if (lowbyte == 0xdb) {
+            printf("Freed memory\n");
+            return;
+        }
+        if (lowbyte == 0xcb) {
+            printf("Forbidden (redzone) memory\n");
+            return;
+        }
+
+        printf("Guessing that it's a Python object\n");
         Box* b = (Box*)p;
 
         printf("Class: %s", getFullTypeName(b).c_str());