Bunch more fixes

- change up interpreter-shutdown to better deal with objects with custom finalizers
- fix a bunch of mixed decrefs for oldstyle-classes
- a couple other misc things

file.py is now working

from_cpython/Include/Python.h

@@ -113,7 +113,7 @@ class HCAttrs;
 typedef pyston::HCAttrs PyHcAttrs;
 #endif
 PyAPI_FUNC(void) PyObject_InitHcAttrs(PyHcAttrs*) PYSTON_NOEXCEPT;
-PyAPI_FUNC(PyObject*) PyObject_GetAttrWrapper(PyObject*) PYSTON_NOEXCEPT;
+PyAPI_FUNC(BORROWED(PyObject*)) PyObject_GetAttrWrapper(PyObject*) PYSTON_NOEXCEPT;
 PyAPI_FUNC(void) PyType_RequestHcAttrs(PyTypeObject*, int offset) PYSTON_NOEXCEPT;
 // Sets a descriptor on the type so that the attrs are available via __dict__
 PyAPI_FUNC(void) PyType_GiveHcAttrsDictDescr(PyTypeObject*) PYSTON_NOEXCEPT;

from_cpython/Modules/_io/iobase.c

@@ -201,12 +201,10 @@ _PyIOBase_finalize(PyObject *self)
 
     /* If _PyIOBase_finalize() is called from a destructor, we need to
        resurrect the object as calling close() can invoke arbitrary code. */
-    // Pyston change:
-    is_zombie = 0;
-    // is_zombie = (Py_REFCNT(self) == 0);
-    //if (is_zombie) {
-    //    ++Py_REFCNT(self);
-    //}
+    is_zombie = (Py_REFCNT(self) == 0);
+    if (is_zombie) {
+        ++Py_REFCNT(self);
+    }
     PyErr_Fetch(&tp, &v, &tb);
     /* If `closed` doesn't exist or can't be evaluated as bool, then the
        object is probably in an unusable state, so ignore. */
@@ -232,8 +230,6 @@ _PyIOBase_finalize(PyObject *self)
     }
     PyErr_Restore(tp, v, tb);
     if (is_zombie) {
-// Pyston change:
-#if 0
         if (--Py_REFCNT(self) != 0) {
             /* The object lives again. The following code is taken from
                slot_tp_del in typeobject.c. */
@@ -249,14 +245,12 @@ _PyIOBase_finalize(PyObject *self)
              * _Py_NewReference bumped tp_allocs:  both of those need to be
              * undone.
              */
-
 #ifdef COUNT_ALLOCS
             --Py_TYPE(self)->tp_frees;
             --Py_TYPE(self)->tp_allocs;
 #endif
             return -1;
         }
-#endif
     }
     return 0;
 }

from_cpython/Objects/exceptions.c

@@ -243,6 +243,7 @@ BaseException_repr(PyBaseExceptionObject *self)
 static PyObject *
 BaseException_reduce(PyBaseExceptionObject *self)
 {
+    assert(0 && "check refcounting");
     /* Pyston change:
     if (self->args && self->dict)
         return PyTuple_Pack(3, Py_TYPE(self), self->args, self->dict);
@@ -849,6 +850,7 @@ EnvironmentError_reduce(PyEnvironmentErrorObject *self)
     } else
         Py_INCREF(args);
 
+    assert(0 && "check refcounting");
     /* Pyston change:
     if (self->dict)
         res = PyTuple_Pack(3, Py_TYPE(self), args, self->dict);

src/capi/abstract.cpp

@@ -705,7 +705,6 @@ extern "C" PyObject* PyObject_CallMethodObjArgs(PyObject* callable, PyObject* na
 
 
     Py_DECREF(args);
-    Py_DECREF(callable);
 
     return tmp;
 }

src/core/types.h

@@ -307,6 +307,7 @@ class LocationMap;
 class JitCodeBlock;
 
 extern std::vector<Box*> constants;
+extern std::vector<Box*> late_constants; // constants that should be freed after normal constants
 
 // A specific compilation of a FunctionMetadata.  Usually these will be created by the LLVM JIT, which will take a FunctionMetadata
 // and some compilation settings, and produce a CompiledFunction
@@ -696,7 +697,7 @@ public:
     HCAttrs* getHCAttrsPtr();
     void setDictBacked(STOLEN(Box*) d);
     // For instances with dict attrs:
-    BoxedDict* getDict();
+    BORROWED(BoxedDict*) getDict();
     void setDict(STOLEN(BoxedDict*) d);
 
 

src/runtime/classobj.cpp

@@ -554,7 +554,7 @@ void instanceSetattroInternal(Box* _inst, Box* _attr, STOLEN(Box*) value, Setatt
 
         if (setattr) {
             setattr = processDescriptor(setattr, inst, inst->inst_cls);
-            runtimeCall(setattr, ArgPassSpec(2), _attr, value, NULL, NULL, NULL);
+            autoDecref(runtimeCall(setattr, ArgPassSpec(2), _attr, value, NULL, NULL, NULL));
             return;
         }
 
@@ -567,7 +567,7 @@ void instanceSetattroInternal(Box* _inst, Box* _attr, STOLEN(Box*) value, Setatt
         Box* setattr = classLookup(inst->inst_cls, setattr_str);
         if (setattr) {
             setattr = processDescriptor(setattr, inst, inst->inst_cls);
-            runtimeCall(setattr, ArgPassSpec(2), _attr, value, NULL, NULL, NULL);
+            autoDecref(runtimeCall(setattr, ArgPassSpec(2), _attr, value, NULL, NULL, NULL));
             return;
         }
 
@@ -633,6 +633,7 @@ Box* instanceRepr(Box* _inst) {
     Box* repr_func = _instanceGetattribute(inst, repr_str, false);
 
     if (repr_func) {
+        AUTO_DECREF(repr_func);
         return runtimeCall(repr_func, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
     } else {
         Box* class_str = classobjStr(inst->inst_cls);
@@ -652,6 +653,7 @@ Box* instanceStr(Box* _inst) {
     Box* str_func = _instanceGetattribute(inst, str_str, false);
 
     if (str_func) {
+        AUTO_DECREF(str_func);
         return runtimeCall(str_func, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
     } else {
         return instanceRepr(inst);
@@ -684,6 +686,7 @@ Box* instanceNonzero(Box* _inst) {
     }
 
     if (nonzero_func) {
+        AUTO_DECREF(nonzero_func);
         return runtimeCall(nonzero_func, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
     } else {
         Py_RETURN_TRUE;
@@ -696,6 +699,7 @@ Box* instanceLen(Box* _inst) {
 
     static BoxedString* len_str = getStaticString("__len__");
     Box* len_func = _instanceGetattribute(inst, len_str, true);
+    AUTO_DECREF(len_func);
     return runtimeCall(len_func, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
 }
 
@@ -705,6 +709,7 @@ Box* instanceGetitem(Box* _inst, Box* key) {
 
     static BoxedString* getitem_str = getStaticString("__getitem__");
     Box* getitem_func = _instanceGetattribute(inst, getitem_str, true);
+    AUTO_DECREF(getitem_func);
     return runtimeCall(getitem_func, ArgPassSpec(1), key, NULL, NULL, NULL, NULL);
 }
 
@@ -714,6 +719,7 @@ Box* instanceSetitem(Box* _inst, Box* key, Box* value) {
 
     static BoxedString* setitem_str = getStaticString("__setitem__");
     Box* setitem_func = _instanceGetattribute(inst, setitem_str, true);
+    AUTO_DECREF(setitem_func);
     return runtimeCall(setitem_func, ArgPassSpec(2), key, value, NULL, NULL, NULL);
 }
 
@@ -723,6 +729,7 @@ Box* instanceDelitem(Box* _inst, Box* key) {
 
     static BoxedString* delitem_str = getStaticString("__delitem__");
     Box* delitem_func = _instanceGetattribute(inst, delitem_str, true);
+    AUTO_DECREF(delitem_func);
     return runtimeCall(delitem_func, ArgPassSpec(1), key, NULL, NULL, NULL, NULL);
 }
 
@@ -741,7 +748,9 @@ Box* instanceGetslice(Box* _inst, Box* i, Box* j) {
     }
 
     if (getslice_func == NULL) {
+        AUTO_DECREF(getslice_func);
         Box* slice = static_cast<Box*>(createSlice(i, j, None));
+        AUTO_DECREF(slice);
         return instanceGetitem(inst, slice);
     }
 
@@ -763,7 +772,9 @@ Box* instanceSetslice(Box* _inst, Box* i, Box* j, Box** sequence) {
     }
 
     if (setslice_func == NULL) {
+        AUTO_DECREF(setslice_func);
         Box* slice = static_cast<Box*>(createSlice(i, j, None));
+        AUTO_DECREF(slice);
         return instanceSetitem(inst, slice, *sequence);
     }
 
@@ -785,7 +796,9 @@ Box* instanceDelslice(Box* _inst, Box* i, Box* j) {
     }
 
     if (delslice_func == NULL) {
+        AUTO_DECREF(delslice_func);
         Box* slice = static_cast<Box*>(createSlice(i, j, None));
+        AUTO_DECREF(slice);
         return instanceDelitem(inst, slice);
     }
     try {
@@ -840,6 +853,8 @@ static int half_cmp(PyObject* v, PyObject* w) noexcept {
     }
 #endif
 
+    AUTO_DECREF(cmp_func);
+
     args = PyTuple_Pack(1, w);
     if (args == NULL) {
         Py_DECREF(cmp_func);
@@ -946,7 +961,10 @@ Box* instanceContains(Box* _inst, Box* key) {
         return boxBool(result);
     }
 
+    AUTO_DECREF(contains_func);
+
     Box* r = runtimeCall(contains_func, ArgPassSpec(1), key, NULL, NULL, NULL, NULL);
+    AUTO_DECREF(r);
     return boxBool(nonzero(r));
 }
 
@@ -970,12 +988,18 @@ static Box* instanceHash(BoxedInstance* inst) {
             func = _instanceGetattribute(inst, cmp_str, false);
             if (func == NULL) {
                 return boxInt(_Py_HashPointer(inst));
+            } else {
+                Py_DECREF(func);
             }
+        } else {
+            Py_DECREF(func);
         }
         raiseExcHelper(TypeError, "unhashable instance");
     }
+    AUTO_DECREF(func);
 
     res = runtimeCall(func, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
+    AUTO_DECREF(res);
     if (PyInt_Check(res) || PyLong_Check(res)) {
         CallattrFlags callattr_flags{.cls_only = true, .null_on_nonexistent = false, .argspec = ArgPassSpec(0) };
         return callattr(res, hash_str, callattr_flags, nullptr, nullptr, nullptr, nullptr, nullptr);
@@ -992,6 +1016,7 @@ static Box* instanceIter(BoxedInstance* self) {
     static BoxedString* iter_str = getStaticString("__iter__");
     static BoxedString* getitem_str = getStaticString("__getitem__");
     if ((func = _instanceGetattribute(self, iter_str, false)) != NULL) {
+        AUTO_DECREF(func);
         PyObject* res = PyEval_CallObject(func, (PyObject*)NULL);
         if (!res)
             throwCAPIException();
@@ -1002,6 +1027,7 @@ static Box* instanceIter(BoxedInstance* self) {
     }
 
     if ((func = _instanceGetattribute(self, getitem_str, false)) == NULL) {
+        AUTO_DECREF(func);
         raiseExcHelper(TypeError, "iteration over non-sequence");
     }
 
@@ -1021,6 +1047,7 @@ static Box* instanceNext(BoxedInstance* inst) {
         // not 100% sure why this is a different error:
         raiseExcHelper(TypeError, "instance has no next() method");
     }
+    AUTO_DECREF(next_func);
 
     Box* r = runtimeCall(next_func, ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
     return r;

src/runtime/objmodel.cpp

@@ -970,7 +970,7 @@ void Box::setDict(STOLEN(BoxedDict*) d) {
     *getDictPtr() = d;
 }
 
-BoxedDict* Box::getDict() {
+BORROWED(BoxedDict*) Box::getDict() {
     assert(cls->instancesHaveDictAttrs());
 
     BoxedDict** d_ptr = getDictPtr();
@@ -980,7 +980,7 @@ BoxedDict* Box::getDict() {
     }
 
     assert(d->cls == dict_cls);
-    return incref(d);
+    return d;
 }
 
 static StatCounter box_getattr_slowpath("slowpath_box_getattr");
@@ -2226,14 +2226,19 @@ extern "C" Box* getclsattr(Box* obj, BoxedString* attr) {
         GetattrRewriteArgs rewrite_args(rewriter.get(), rewriter->getArg(0), rewriter->getReturnDestination());
         gotten = getclsattrInternal<REWRITABLE>(obj, attr, &rewrite_args);
 
-        if (rewrite_args.isSuccessful() && gotten) {
-            RewriterVar* r_rtn;
-            ReturnConvention return_convention;
-            std::tie(r_rtn, return_convention) = rewrite_args.getReturn();
+        if (rewrite_args.isSuccessful()) {
+            if (gotten) {
+                RewriterVar* r_rtn;
+                ReturnConvention return_convention;
+                std::tie(r_rtn, return_convention) = rewrite_args.getReturn();
 
-            assert(return_convention == ReturnConvention::HAS_RETURN
-                   || return_convention == ReturnConvention::MAYBE_EXC);
-            rewriter->commitReturning(r_rtn);
+                assert(return_convention == ReturnConvention::HAS_RETURN
+                       || return_convention == ReturnConvention::MAYBE_EXC);
+                rewriter->commitReturning(r_rtn);
+            } else {
+                rewrite_args.getReturn(); // just to make the asserts happy
+                rewriter.reset(NULL);
+            }
         }
     } else
 #endif

src/runtime/tuple.cpp

@@ -54,7 +54,7 @@ Box* _tupleSlice(BoxedTuple* self, i64 start, i64 stop, i64 step, i64 length) {
     return rtn;
 }
 
-Box* tupleGetitemUnboxed(BoxedTuple* self, i64 n) {
+BORROWED(Box*) tupleGetitemUnboxedBorrowed(BoxedTuple* self, i64 n) {
     i64 size = self->size();
 
     if (n < 0)
@@ -63,19 +63,22 @@ Box* tupleGetitemUnboxed(BoxedTuple* self, i64 n) {
         raiseExcHelper(IndexError, "tuple index out of range");
 
     Box* rtn = self->elts[n];
-    Py_INCREF(rtn);
     return rtn;
 }
 
+Box* tupleGetitemUnboxed(BoxedTuple* self, i64 n) {
+    return incref(tupleGetitemUnboxedBorrowed(self, n));
+}
+
 Box* tupleGetitemInt(BoxedTuple* self, BoxedInt* slice) {
     return tupleGetitemUnboxed(self, slice->n);
 }
 
-extern "C" PyObject* PyTuple_GetItem(PyObject* op, Py_ssize_t i) noexcept {
+extern "C" BORROWED(PyObject*) PyTuple_GetItem(PyObject* op, Py_ssize_t i) noexcept {
     RELEASE_ASSERT(PyTuple_Check(op), "");
     RELEASE_ASSERT(i >= 0, ""); // unlike tuple.__getitem__, PyTuple_GetItem doesn't do index wrapping
     try {
-        return tupleGetitemUnboxed(static_cast<BoxedTuple*>(op), i);
+        return tupleGetitemUnboxedBorrowed(static_cast<BoxedTuple*>(op), i);
     } catch (ExcInfo e) {
         abort();
     }

src/runtime/types.cpp

@@ -2776,7 +2776,8 @@ BORROWED(Box*) Box::getAttrWrapper() {
     return attrs->attr_list->attrs[offset];
 }
 
-extern "C" PyObject* PyObject_GetAttrWrapper(PyObject* obj) noexcept {
+extern "C" BORROWED(PyObject*) PyObject_GetAttrWrapper(PyObject* obj) noexcept {
+    assert(0 && "check refcounting");
     return obj->getAttrWrapper();
 }
 
@@ -4046,6 +4047,7 @@ void BoxedGetsetDescriptor::dealloc(Box* _o) noexcept {
 #endif
 
 std::vector<Box*> constants;
+std::vector<Box*> late_constants;
 extern "C" PyObject* PyGC_RegisterStaticConstant(Box* b) noexcept {
     constants.push_back(b);
     return b;
@@ -4185,7 +4187,7 @@ void setupRuntime() {
     tuple_cls->tp_itemsize = sizeof(Box*);
     tuple_cls->tp_mro = BoxedTuple::create({ tuple_cls, object_cls });
     EmptyTuple = BoxedTuple::create({});
-    constants.push_back(EmptyTuple);
+    late_constants.push_back(EmptyTuple);
     list_cls = new (0) BoxedClass(object_cls, 0, 0, sizeof(BoxedList), false, "list", true, BoxedList::dealloc, NULL,
                                   true, BoxedList::traverse, BoxedList::clear);
     list_cls->tp_flags |= Py_TPFLAGS_LIST_SUBCLASS;
@@ -4753,18 +4755,7 @@ extern "C" void Py_Finalize() noexcept {
     // initialized = 0;
 
 #ifdef Py_REF_DEBUG
-    PyGC_Collect(); // To make sure it creates any static objects
     IN_SHUTDOWN = true;
-    PyOS_FiniInterrupts();
-    _PyCodecRegistry_Deinit();
-    // TODO: we might have to do this in a loop:
-    PyType_ClearCache();
-    _PyUnicode_Fini();
-    PyThreadState_Clear(NULL);
-    for (auto b : constants) {
-        Py_DECREF(b);
-    }
-    constants.clear();
 
     // May need to run multiple collections to collect everything:
     while (true) {
@@ -4782,6 +4773,18 @@ extern "C" void Py_Finalize() noexcept {
             break;
     }
 
+    PyType_ClearCache();
+    PyOS_FiniInterrupts();
+    _PyCodecRegistry_Deinit();
+    // TODO: we might have to do this in a loop:
+    _PyUnicode_Fini();
+    PyThreadState_Clear(NULL);
+
+    for (auto b : late_constants) {
+        Py_DECREF(b);
+    }
+    late_constants.clear();
+
     _Py_ReleaseInternedStrings();
     for (auto b : classes) {
         if (!PyObject_IS_GC(b)) {

test/tests/assign_literal.py

-# expected: reffail
 # skip-if: '-x' in EXTRA_JIT_ARGS
 # - CPython parser backend doesn't handle well the errors
 # should_error

test/tests/dictcomp.py

-# expected: reffail
 def dict2str(d):
     result = ''
     for k, v in sorted(d.items()):

test/tests/file.py

-# expected: reffail
 import sys
 import tempfile
 

test/tests/gcj_2014_2_c.py

-# expected: reffail
 import sys
 
 if __name__ == "__main__":

test/tests/gcj_2014_2_d.py

-# expected: reffail
 import sys
 
 P = 1000000007

test/tests/instance_methods.py

-# expected: reffail
 class C(object):
     def foo(self):
         pass

test/tests/oldstyle_classes.py

@@ -532,3 +532,16 @@ for i in range(2):
     except AttributeError, e:
         print e
     C._ = (lambda s: 42)
+
+class C:
+    pass
+def __getattr__(self, attr):
+    print attr
+    return lambda *args: 1
+
+c = C()
+c.__getattr__ = __getattr__
+try:
+    print c[1]
+except Exception as e:
+    print e.message

test/tests/pwd_test.py

-# expected: reffail
 import pwd
 import os
 print pwd.getpwuid(os.getuid())[5] == os.environ["HOME"]

test/tests/socket_test.py

-# expected: reffail
 import socket
 
 s = socket.socket()

test/tests/tempfile_test.py

-# expected: reffail
 import os
 import tempfile
 

test/tests/zipimport_hook_test.py

-# expected: reffail
 import sys, os
 
 # this zip contains two pkgs: test1 and test2

test/tests/zipimport_test.py

-# expected: reffail
 import os
 import zipimport