Merge commit 'f1e9b5' into refcounting

from_cpython/CMakeLists.txt

@@ -86,6 +86,7 @@ file(GLOB_RECURSE STDOBJECT_SRCS Objects
     capsule.c
     cobject.c
     complexobject.c
+    dictobject.c
     dictproxy.c
     exceptions.c
     floatobject.c

from_cpython/Include/dictobject.h

@@ -103,10 +103,11 @@ PyAPI_DATA(PyTypeObject) PyDict_Type;
 PyAPI_DATA(PyTypeObject) PyDictIterKey_Type;
 PyAPI_DATA(PyTypeObject) PyDictIterValue_Type;
 PyAPI_DATA(PyTypeObject) PyDictIterItem_Type;
+#endif
 PyAPI_DATA(PyTypeObject) PyDictKeys_Type;
 PyAPI_DATA(PyTypeObject) PyDictItems_Type;
 PyAPI_DATA(PyTypeObject) PyDictValues_Type;
-#endif
+
 PyAPI_DATA(PyTypeObject*) dict_cls;
 #define PyDict_Type (*dict_cls)
 PyAPI_DATA(PyTypeObject*) dictiterkey_cls;
@@ -115,12 +116,6 @@ PyAPI_DATA(PyTypeObject*) dictitervalue_cls;
 #define PyDictIterValue_Type (*dictitervalue_cls)
 PyAPI_DATA(PyTypeObject*) dictiteritem_cls;
 #define PyDictIterItem_Type (*dictiteritem_cls)
-PyAPI_DATA(PyTypeObject*) dictkeys_cls;
-#define PyDictKeys_Type (*dictkeys_cls)
-PyAPI_DATA(PyTypeObject*) dictitems_cls;
-#define PyDictItems_Type (*dictitems_cls)
-PyAPI_DATA(PyTypeObject*) dictvalues_cls;
-#define PyDictValues_Type (*dictvalues_cls)
 
 #define PyDict_Check(op) \
                  PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_DICT_SUBCLASS)

src/capi/typeobject.cpp

@@ -3408,6 +3408,9 @@ extern "C" int PyType_Ready(PyTypeObject* cls) noexcept {
     // RELEASE_ASSERT(cls->tp_traverse == NULL, "");
     // RELEASE_ASSERT(cls->tp_clear == NULL, "");
 
+    // set this flag early because some function check if it is set pretty early
+    cls->is_user_defined = true;
+
     assert(cls->attrs.hcls == NULL);
     new (&cls->attrs) HCAttrs(HiddenClass::makeSingleton());
 #define INITIALIZE(a) new (&(a)) decltype(a)
@@ -3483,7 +3486,6 @@ extern "C" int PyType_Ready(PyTypeObject* cls) noexcept {
             cls->tp_as_buffer = base->tp_as_buffer;
     }
 
-    cls->is_user_defined = true;
 
 
     if (!cls->instancesHaveHCAttrs() && cls->tp_base) {

src/runtime/dict.cpp

@@ -28,10 +28,11 @@
 
 namespace pyston {
 
-BoxedClass* dict_iterator_cls = NULL;
-BoxedClass* dict_keys_cls = NULL;
-BoxedClass* dict_values_cls = NULL;
-BoxedClass* dict_items_cls = NULL;
+extern "C" {
+BoxedClass* dictiterkey_cls = NULL;
+BoxedClass* dictitervalue_cls = NULL;
+BoxedClass* dictiteritem_cls = NULL;
+}
 
 Box* dictRepr(BoxedDict* self) {
     std::vector<char> chars;
@@ -143,33 +144,6 @@ extern "C" PyObject* PyDict_Items(PyObject* mp) noexcept {
     return dict_helper(mp, dictItems);
 }
 
-Box* dictViewKeys(BoxedDict* self) {
-    if (!PyDict_Check(self)) {
-        raiseExcHelper(TypeError, "descriptor 'viewkeys' requires a 'dict' object but received a '%s'",
-                       getTypeName(self));
-    }
-    BoxedDictView* rtn = new (dict_keys_cls) BoxedDictView(self);
-    return rtn;
-}
-
-Box* dictViewValues(BoxedDict* self) {
-    if (!PyDict_Check(self)) {
-        raiseExcHelper(TypeError, "descriptor 'viewvalues' requires a 'dict' object but received a '%s'",
-                       getTypeName(self));
-    }
-    BoxedDictView* rtn = new (dict_values_cls) BoxedDictView(self);
-    return rtn;
-}
-
-Box* dictViewItems(BoxedDict* self) {
-    if (!PyDict_Check(self)) {
-        raiseExcHelper(TypeError, "descriptor 'viewitems' requires a 'dict' object but received a '%s'",
-                       getTypeName(self));
-    }
-    BoxedDictView* rtn = new (dict_items_cls) BoxedDictView(self);
-    return rtn;
-}
-
 // Analoguous to CPython's, used for sq_ slots.
 static Py_ssize_t dict_length(PyDictObject* mp) {
     return ((BoxedDict*)mp)->d.size();
@@ -845,28 +819,52 @@ extern "C" void _PyDict_MaybeUntrack(PyObject* op) noexcept {
     _PyObject_GC_UNTRACK(op);
 }
 
+// We use cpythons dictview implementation from dictobject.c
+extern "C" PyObject* dictview_new(PyObject* dict, PyTypeObject* type) noexcept;
+Box* dictViewKeys(BoxedDict* d) {
+    Box* rtn = dictview_new(d, &PyDictKeys_Type);
+    if (!rtn)
+        throwCAPIException();
+    return rtn;
+}
+Box* dictViewValues(BoxedDict* d) {
+    Box* rtn = dictview_new(d, &PyDictValues_Type);
+    if (!rtn)
+        throwCAPIException();
+    return rtn;
+}
+Box* dictViewItems(BoxedDict* d) {
+    Box* rtn = dictview_new(d, &PyDictItems_Type);
+    if (!rtn)
+        throwCAPIException();
+    return rtn;
+}
+
+
+// This function gets called from dictobject.c
+extern "C" PyObject* dictiter_new(PyDictObject* dict, PyTypeObject* iter_type) noexcept {
+    return new (iter_type) BoxedDictIterator((BoxedDict*)dict);
+}
+
+
 void setupDict() {
     static PyMappingMethods dict_as_mapping;
     dict_cls->tp_as_mapping = &dict_as_mapping;
     static PySequenceMethods dict_as_sequence;
     dict_cls->tp_as_sequence = &dict_as_sequence;
 
-    dict_iterator_cls = BoxedClass::create(type_cls, object_cls, 0, 0, sizeof(BoxedDictIterator), false,
-                                           "dictionary-itemiterator", (destructor)BoxedDictIterator::dealloc, NULL,
+    dictiterkey_cls = BoxedClass::create(type_cls, object_cls, 0, 0, sizeof(BoxedDictIterator), false,
+                                         "dictionary-keyiterator", (destructor)BoxedDictIterator::dealloc, NULL, true,
+                                         (traverseproc)BoxedDictIterator::traverse, NOCLEAR);
+    dictitervalue_cls = BoxedClass::create(type_cls, object_cls, 0, 0, sizeof(BoxedDictIterator), false,
+                                           "dictionary-valueiterator", (destructor)BoxedDictIterator::dealloc, NULL,
                                            true, (traverseproc)BoxedDictIterator::traverse, NOCLEAR);
+    dictiteritem_cls = BoxedClass::create(type_cls, object_cls, 0, 0, sizeof(BoxedDictIterator), false,
+                                          "dictionary-itemiterator", (destructor)BoxedDictIterator::dealloc, NULL, true,
+                                          (traverseproc)BoxedDictIterator::traverse, NOCLEAR);
 
-    dict_keys_cls = BoxedClass::create(type_cls, object_cls, 0, 0, sizeof(BoxedDictView), false, "dict_keys",
-                                       (destructor)BoxedDictView::dealloc, NULL, true,
-                                       (traverseproc)BoxedDictView::traverse, NOCLEAR);
-    dict_values_cls = BoxedClass::create(type_cls, object_cls, 0, 0, sizeof(BoxedDictView), false, "dict_values",
-                                         (destructor)BoxedDictView::dealloc, NULL, true,
-                                         (traverseproc)BoxedDictView::traverse, NOCLEAR);
-    dict_items_cls = BoxedClass::create(type_cls, object_cls, 0, 0, sizeof(BoxedDictView), false, "dict_items",
-                                        (destructor)BoxedDictView::dealloc, NULL, true,
-                                        (traverseproc)BoxedDictView::traverse, NOCLEAR);
-
-    dict_iterator_cls->instances_are_nonzero = dict_keys_cls->instances_are_nonzero
-        = dict_values_cls->instances_are_nonzero = dict_items_cls->instances_are_nonzero = true;
+    dictiterkey_cls->instances_are_nonzero = dictitervalue_cls->instances_are_nonzero
+        = dictiteritem_cls->instances_are_nonzero = true;
 
     dict_cls->has_safe_tp_dealloc = true;
 
@@ -879,7 +877,7 @@ void setupDict() {
     dict_cls->giveAttr("__ne__", new BoxedFunction(FunctionMetadata::create((void*)dictNe, UNKNOWN, 2)));
 
     dict_cls->giveAttr("__iter__", new BoxedFunction(FunctionMetadata::create((void*)dictIterKeys,
-                                                                              typeFromClass(dict_iterator_cls), 1)));
+                                                                              typeFromClass(dictiterkey_cls), 1)));
 
     dict_cls->giveAttr("update", new BoxedFunction(FunctionMetadata::create((void*)dictUpdate, NONE, 1, true, true)));
 
@@ -889,11 +887,11 @@ void setupDict() {
     dict_cls->giveAttr("has_key", new BoxedFunction(FunctionMetadata::create((void*)dictContains, BOXED_BOOL, 2)));
     dict_cls->giveAttr("items", new BoxedFunction(FunctionMetadata::create((void*)dictItems, LIST, 1)));
     dict_cls->giveAttr("iteritems", new BoxedFunction(FunctionMetadata::create((void*)dictIterItems,
-                                                                               typeFromClass(dict_iterator_cls), 1)));
+                                                                               typeFromClass(dictiteritem_cls), 1)));
 
     dict_cls->giveAttr("values", new BoxedFunction(FunctionMetadata::create((void*)dictValues, LIST, 1)));
     dict_cls->giveAttr("itervalues", new BoxedFunction(FunctionMetadata::create((void*)dictIterValues,
-                                                                                typeFromClass(dict_iterator_cls), 1)));
+                                                                                typeFromClass(dictitervalue_cls), 1)));
 
     dict_cls->giveAttr("keys", new BoxedFunction(FunctionMetadata::create((void*)dictKeys, LIST, 1)));
     dict_cls->giveAttrBorrowed("iterkeys", dict_cls->getattr(autoDecref(internStringMortal("__iter__"))));
@@ -929,15 +927,20 @@ void setupDict() {
 
     dict_cls->freeze();
 
-    FunctionMetadata* hasnext = FunctionMetadata::create((void*)dictIterHasnextUnboxed, BOOL, 1);
-    hasnext->addVersion((void*)dictIterHasnext, BOXED_BOOL);
-    dict_iterator_cls->giveAttr("__hasnext__", new BoxedFunction(hasnext));
-    dict_iterator_cls->giveAttr("__iter__", new BoxedFunction(FunctionMetadata::create(
-                                                (void*)dictIterIter, typeFromClass(dict_iterator_cls), 1)));
-    dict_iterator_cls->giveAttr("next", new BoxedFunction(FunctionMetadata::create((void*)dictIterNext, UNKNOWN, 1)));
-    dict_iterator_cls->freeze();
-    dict_iterator_cls->tp_iter = PyObject_SelfIter;
-    dict_iterator_cls->tp_iternext = dictiter_next;
+    // create the dictonary iterator types
+    for (BoxedClass* iter_type : { dictiterkey_cls, dictitervalue_cls, dictiteritem_cls }) {
+        FunctionMetadata* hasnext = FunctionMetadata::create((void*)dictIterHasnextUnboxed, BOOL, 1);
+        hasnext->addVersion((void*)dictIterHasnext, BOXED_BOOL);
+        iter_type->giveAttr("__hasnext__", new BoxedFunction(hasnext));
+        iter_type->giveAttr(
+            "__iter__", new BoxedFunction(FunctionMetadata::create((void*)dictIterIter, typeFromClass(iter_type), 1)));
+        iter_type->giveAttr("next", new BoxedFunction(FunctionMetadata::create((void*)dictIterNext, UNKNOWN, 1)));
+        iter_type->freeze();
+        iter_type->tp_iter = PyObject_SelfIter;
+        iter_type->tp_iternext = dictiter_next;
+        iter_type->tp_flags &= ~Py_TPFLAGS_BASETYPE; // subclassing is not allowed
+    }
+
 
     // Manually set some tp_* slots *after* calling freeze() -> fixup_slot_dispatchers().
     // fixup_slot_dispatchers will insert a wrapper like slot_tp_init into tp_init, which calls the python-level
@@ -960,14 +963,8 @@ void setupDict() {
 
     dict_cls->tp_as_sequence->sq_contains = (objobjproc)PyDict_Contains;
 
-    dict_keys_cls->giveAttr("__iter__", new BoxedFunction(FunctionMetadata::create(
-                                            (void*)dictViewKeysIter, typeFromClass(dict_iterator_cls), 1)));
-    dict_keys_cls->freeze();
-    dict_values_cls->giveAttr("__iter__", new BoxedFunction(FunctionMetadata::create(
-                                              (void*)dictViewValuesIter, typeFromClass(dict_iterator_cls), 1)));
-    dict_values_cls->freeze();
-    dict_items_cls->giveAttr("__iter__", new BoxedFunction(FunctionMetadata::create(
-                                             (void*)dictViewItemsIter, typeFromClass(dict_iterator_cls), 1)));
-    dict_items_cls->freeze();
+    PyType_Ready(&PyDictKeys_Type);
+    PyType_Ready(&PyDictValues_Type);
+    PyType_Ready(&PyDictItems_Type);
 }
 }

src/runtime/dict.h

@@ -20,23 +20,13 @@
 
 namespace pyston {
 
-extern BoxedClass* dict_iterator_cls;
-extern BoxedClass* dict_keys_cls;
-extern BoxedClass* dict_values_cls;
-extern BoxedClass* dict_items_cls;
-
 class BoxedDictIterator : public Box {
 public:
-    enum IteratorType { KeyIterator, ValueIterator, ItemIterator };
-
     BoxedDict* d;
     BoxedDict::DictMap::iterator it;
     const BoxedDict::DictMap::iterator itEnd;
-    const IteratorType type;
-
-    BoxedDictIterator(BoxedDict* d, IteratorType type);
 
-    DEFAULT_CLASS(dict_iterator_cls);
+    BoxedDictIterator(BoxedDict* d);
 
     static void dealloc(BoxedDictIterator* o) noexcept {
         PyObject_GC_UnTrack(o);
@@ -62,26 +52,7 @@ llvm_compat_bool dictIterHasnextUnboxed(Box* self);
 Box* dictiter_next(Box* self) noexcept;
 Box* dictIterNext(Box* self);
 
-class BoxedDictView : public Box {
-public:
-    BoxedDict* d;
-    BoxedDictView(BoxedDict* d);
-
-    static void dealloc(BoxedDictIterator* o) noexcept {
-        PyObject_GC_UnTrack(o);
-        Py_DECREF(o->d);
-        o->cls->tp_free(o);
-    }
-
-    static int traverse(BoxedDictIterator* self, visitproc visit, void *arg) noexcept {
-        Py_VISIT(self->d);
-        return 0;
-    }
-};
 
-Box* dictViewKeysIter(Box* self);
-Box* dictViewValuesIter(Box* self);
-Box* dictViewItemsIter(Box* self);
 void dictMerge(BoxedDict* self, Box* other);
 Box* dictUpdate(BoxedDict* self, BoxedTuple* args, BoxedDict* kwargs);
 }

src/runtime/inline/dict.cpp

@@ -19,14 +19,13 @@
 
 namespace pyston {
 
-BoxedDictIterator::BoxedDictIterator(BoxedDict* d, IteratorType type)
-    : d(d), it(d->d.begin()), itEnd(d->d.end()), type(type) {
+BoxedDictIterator::BoxedDictIterator(BoxedDict* d) : d(d), it(d->d.begin()), itEnd(d->d.end()) {
 }
 
 Box* dict_iter(Box* s) noexcept {
     assert(PyDict_Check(s));
     BoxedDict* self = static_cast<BoxedDict*>(s);
-    return new BoxedDictIterator(self, BoxedDictIterator::KeyIterator);
+    return new (&PyDictIterKey_Type) BoxedDictIterator(self);
 }
 
 Box* dictIterKeys(Box* s) {
@@ -36,13 +35,13 @@ Box* dictIterKeys(Box* s) {
 Box* dictIterValues(Box* s) {
     assert(PyDict_Check(s));
     BoxedDict* self = static_cast<BoxedDict*>(s);
-    return new BoxedDictIterator(self, BoxedDictIterator::ValueIterator);
+    return new (&PyDictIterValue_Type) BoxedDictIterator(self);
 }
 
 Box* dictIterItems(Box* s) {
     assert(PyDict_Check(s));
     BoxedDict* self = static_cast<BoxedDict*>(s);
-    return new BoxedDictIterator(self, BoxedDictIterator::ItemIterator);
+    return new (&PyDictIterItem_Type) BoxedDictIterator(self);
 }
 
 Box* dictIterIter(Box* s) {
@@ -50,7 +49,6 @@ Box* dictIterIter(Box* s) {
 }
 
 llvm_compat_bool dictIterHasnextUnboxed(Box* s) {
-    assert(s->cls == dict_iterator_cls);
     BoxedDictIterator* self = static_cast<BoxedDictIterator*>(s);
 
     return self->it != self->itEnd;
@@ -61,19 +59,20 @@ Box* dictIterHasnext(Box* s) {
 }
 
 Box* dictiter_next(Box* s) noexcept {
-    assert(s->cls == dict_iterator_cls);
     BoxedDictIterator* self = static_cast<BoxedDictIterator*>(s);
 
     if (self->it == self->itEnd)
         return NULL;
 
     Box* rtn = nullptr;
-    if (self->type == BoxedDictIterator::KeyIterator) {
+    if (self->cls == &PyDictIterKey_Type) {
         rtn = self->it->first.value;
-    } else if (self->type == BoxedDictIterator::ValueIterator) {
+    } else if (self->cls == &PyDictIterValue_Type) {
         rtn = self->it->second;
-    } else if (self->type == BoxedDictIterator::ItemIterator) {
+    } else if (self->cls == &PyDictIterItem_Type) {
         rtn = BoxedTuple::create({ self->it->first.value, self->it->second });
+    } else {
+        RELEASE_ASSERT(0, "");
     }
     ++self->it;
     return rtn;
@@ -85,25 +84,4 @@ Box* dictIterNext(Box* s) {
         raiseExcHelper(StopIteration, "");
     return rtn;
 }
-
-BoxedDictView::BoxedDictView(BoxedDict* d) : d(d) {
-}
-
-Box* dictViewKeysIter(Box* s) {
-    assert(s->cls == dict_keys_cls);
-    BoxedDictView* self = static_cast<BoxedDictView*>(s);
-    return dictIterKeys(self->d);
-}
-
-Box* dictViewValuesIter(Box* s) {
-    assert(s->cls == dict_values_cls);
-    BoxedDictView* self = static_cast<BoxedDictView*>(s);
-    return dictIterValues(self->d);
-}
-
-Box* dictViewItemsIter(Box* s) {
-    assert(s->cls == dict_items_cls);
-    BoxedDictView* self = static_cast<BoxedDictView*>(s);
-    return dictIterItems(self->d);
-}
 }

src/runtime/types.cpp

@@ -2344,6 +2344,12 @@ public:
         return r ? True : False;
     }
 
+    static Box* hasKey(Box* _self, Box* _key) {
+        if (PyErr_WarnPy3k("dict.has_key() not supported in 3.x; use the in operator", 1) < 0)
+            throwCAPIException();
+        return contains<CXX>(_self, _key);
+    }
+
     static int sq_contains(Box* _self, Box* _key) noexcept {
         Box* rtn = contains<CAPI>(_self, _key);
         if (!rtn)
@@ -3968,8 +3974,10 @@ void setupRuntime() {
         = BoxedClass::create(type_cls, object_cls, 0, 0, sizeof(AttrWrapperIter), false, "attrwrapperiter",
                              AttrWrapperIter::dealloc, NULL, true, AttrWrapperIter::traverse, NOCLEAR);
 
-    pyston_getset_cls->giveAttr("__get__", new BoxedFunction(FunctionMetadata::create((void*)getsetGet, UNKNOWN, 3)));
-    capi_getset_cls->giveAttr("__get__", new BoxedFunction(FunctionMetadata::create((void*)getsetGet, UNKNOWN, 3)));
+    pyston_getset_cls->giveAttr("__get__",
+                                new BoxedFunction(FunctionMetadata::create((void*)getsetGet, UNKNOWN, 3), { None }));
+    capi_getset_cls->giveAttr("__get__",
+                              new BoxedFunction(FunctionMetadata::create((void*)getsetGet, UNKNOWN, 3), { None }));
     pyston_getset_cls->giveAttr("__set__", new BoxedFunction(FunctionMetadata::create((void*)getsetSet, UNKNOWN, 3)));
     capi_getset_cls->giveAttr("__set__", new BoxedFunction(FunctionMetadata::create((void*)getsetSet, UNKNOWN, 3)));
     pyston_getset_cls->giveAttr("__delete__",
@@ -4177,7 +4185,9 @@ void setupRuntime() {
     attrwrapper_cls->giveAttr("__str__",
                               new BoxedFunction(FunctionMetadata::create((void*)AttrWrapper::str, UNKNOWN, 1)));
     attrwrapper_cls->giveAttr(
-        "__contains__", new BoxedFunction(FunctionMetadata::create((void*)AttrWrapper::contains<CXX>, UNKNOWN, 2)));
+        "__contains__", new BoxedFunction(FunctionMetadata::create((void*)AttrWrapper::contains<CXX>, BOXED_BOOL, 2)));
+    attrwrapper_cls->giveAttr("has_key",
+                              new BoxedFunction(FunctionMetadata::create((void*)AttrWrapper::hasKey, BOXED_BOOL, 2)));
     attrwrapper_cls->giveAttr("__eq__",
                               new BoxedFunction(FunctionMetadata::create((void*)AttrWrapper::eq, UNKNOWN, 2)));
     attrwrapper_cls->giveAttr("__ne__",

test/tests/builtins.py

@@ -72,6 +72,7 @@ except TypeError, e:
 
 print globals().get("not a real variable")
 print globals().get("not a real variable", 1)
+print globals().has_key("C"), globals().has_key("CC")
 
 print hex(12345)
 print oct(234)

test/tests/capi_slots.py

@@ -44,6 +44,14 @@ print slots_test.SlotsTesterNum(0) == slots_test.SlotsTesterNum(1)
 for i in slots_test.SlotsTesterSeq(6):
     print i
 
+try:
+    # seqiter.tp_new is NULL so we should not be allowed to create an instance
+    slot_tester_seqiter = type(iter(slots_test.SlotsTesterSeq(6)))
+    print slot_tester_seqiter
+    slot_tester_seqiter()
+except Exception as e:
+    print e
+
 su = slots_test.SlotsTesterSub(5)
 print su
 

test/tests/descriptors_double.py

@@ -117,3 +117,7 @@ print type(f.at) # should not call __get__
 f.at = 12 # should not call __set__, should print nothing
 #TODO uncomment this:
 #print f.__dict__['at']
+
+
+# test if we support getset.__get__ with default args
+print type(5).__dict__["real"].__get__(42)