Support the types in the operator module

Add tp_call handling, a couple random functions, and
allow GC-aware objects (though we don't do much with them).

lib_python/2.7_Modules/operator.c

@@ -914,9 +914,6 @@ initoperator(void)
     if (m == NULL)
         return;
 
-    /* Pyston TODO: un-comment these when we have the functionality to
-     * support them.
-
     if (PyType_Ready(&itemgetter_type) < 0)
         return;
     Py_INCREF(&itemgetter_type);
@@ -931,5 +928,4 @@ initoperator(void)
         return;
     Py_INCREF(&methodcaller_type);
     PyModule_AddObject(m, "methodcaller", (PyObject *)&methodcaller_type);
-    */
 }

src/runtime/capi.cpp

@@ -143,15 +143,21 @@ extern "C" PyObject* PyType_GenericAlloc(PyTypeObject* cls, Py_ssize_t nitems) {
     return rtn;
 }
 
-/*
-// These are for supporting things like tp_call.
-// tp_new is handled differently, which is the only one supported right now,
-// which is why these are (temporarily) commented out.
 BoxedClass* wrapperdescr_cls, *wrapperobject_cls;
+struct wrapper_def {
+    const char* name;
+    int offset;
+    int flags;
+};
+
+wrapper_def call_wrapper = { "__call__", offsetof(PyTypeObject, tp_call), PyWrapperFlag_KEYWORDS };
+
 class BoxedWrapperDescriptor : public Box {
 public:
-    const int offset;
-    BoxedWrapperDescriptor(int offset) : Box(wrapperdescr_cls), offset(offset) {}
+    const wrapper_def* wrapper;
+    BoxedClass* type;
+    BoxedWrapperDescriptor(const wrapper_def* wrapper, BoxedClass* type)
+        : Box(wrapperdescr_cls), wrapper(wrapper), type(type) {}
 
     static Box* __get__(BoxedWrapperDescriptor* self, Box* inst, Box* owner);
 };
@@ -164,9 +170,19 @@ public:
     BoxedWrapperObject(BoxedWrapperDescriptor* descr, Box* obj) : Box(wrapperobject_cls), descr(descr), obj(obj) {}
 
     static Box* __call__(BoxedWrapperObject* self, Box* args, Box* kwds) {
+        assert(self->cls == wrapperobject_cls);
         assert(args->cls == tuple_cls);
         assert(kwds->cls == dict_cls);
 
+        int flags = self->descr->wrapper->flags;
+        char* ptr = (char*)self->descr->type + self->descr->wrapper->offset;
+
+        if (flags & PyWrapperFlag_KEYWORDS) {
+            PyCFunctionWithKeywords f = *(PyCFunctionWithKeywords*)ptr;
+            return f(self->obj, args, kwds);
+        } else {
+            abort();
+        }
         abort();
     }
 };
@@ -177,9 +193,12 @@ Box* BoxedWrapperDescriptor::__get__(BoxedWrapperDescriptor* self, Box* inst, Bo
     if (inst == None)
         return self;
 
+    if (!isSubclass(inst->cls, self->type))
+        raiseExcHelper(TypeError, "Descriptor '' for '%s' objects doesn't apply to '%s' object",
+                       getFullNameOfClass(self->type).c_str(), getFullTypeName(inst).c_str());
+
     return new BoxedWrapperObject(self, inst);
 }
-*/
 
 PyObject* tp_new_wrapper(PyTypeObject* self, BoxedTuple* args, Box* kwds) {
     RELEASE_ASSERT(isSubclass(self->cls, type_cls), "");
@@ -212,14 +231,16 @@ extern "C" int PyType_Ready(PyTypeObject* cls) {
     RELEASE_ASSERT(cls->tp_as_sequence == NULL, "");
     RELEASE_ASSERT(cls->tp_as_mapping == NULL, "");
     RELEASE_ASSERT(cls->tp_hash == NULL, "");
-    RELEASE_ASSERT(cls->tp_call == NULL, "");
     RELEASE_ASSERT(cls->tp_str == NULL, "");
     RELEASE_ASSERT(cls->tp_getattro == NULL || cls->tp_getattro == PyObject_GenericGetAttr, "");
     RELEASE_ASSERT(cls->tp_setattro == NULL, "");
     RELEASE_ASSERT(cls->tp_as_buffer == NULL, "");
-    RELEASE_ASSERT((cls->tp_flags & ~(Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE)) == 0, "");
-    RELEASE_ASSERT(cls->tp_traverse == NULL, "");
-    RELEASE_ASSERT(cls->tp_clear == NULL, "");
+
+    int ALLOWABLE_FLAGS = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC;
+    RELEASE_ASSERT((cls->tp_flags & ~ALLOWABLE_FLAGS) == 0, "");
+    // RELEASE_ASSERT(cls->tp_traverse == NULL, "");
+    // RELEASE_ASSERT(cls->tp_clear == NULL, "");
+
     RELEASE_ASSERT(cls->tp_richcompare == NULL, "");
     RELEASE_ASSERT(cls->tp_iter == NULL, "");
     RELEASE_ASSERT(cls->tp_iternext == NULL, "");
@@ -265,6 +286,10 @@ extern "C" int PyType_Ready(PyTypeObject* cls) {
                       new BoxedCApiFunction(METH_VARARGS | METH_KEYWORDS, cls, "__new__", (PyCFunction)tp_new_wrapper));
     }
 
+    if (cls->tp_call) {
+        cls->giveAttr("__call__", new BoxedWrapperDescriptor(&call_wrapper, cls));
+    }
+
     if (!cls->tp_alloc) {
         cls->tp_alloc = reinterpret_cast<decltype(cls->tp_alloc)>(PyType_GenericAlloc);
     }
@@ -405,18 +430,23 @@ extern "C" void PyObject_Free(void* p) {
 extern "C" PyObject* _PyObject_GC_Malloc(size_t) {
     Py_FatalError("unimplemented");
 }
-extern "C" PyObject* _PyObject_GC_New(PyTypeObject*) {
-    Py_FatalError("unimplemented");
+
+extern "C" PyObject* _PyObject_GC_New(PyTypeObject* cls) {
+    return _PyObject_New(cls);
 }
+
 extern "C" PyVarObject* _PyObject_GC_NewVar(PyTypeObject*, Py_ssize_t) {
     Py_FatalError("unimplemented");
 }
+
 extern "C" void PyObject_GC_Track(void*) {
-    Py_FatalError("unimplemented");
+    // TODO do we have to do anything to support the C API GC protocol?
 }
+
 extern "C" void PyObject_GC_UnTrack(void*) {
-    Py_FatalError("unimplemented");
+    // TODO do we have to do anything to support the C API GC protocol?
 }
+
 extern "C" void PyObject_GC_Del(void*) {
     Py_FatalError("unimplemented");
 }
@@ -470,7 +500,16 @@ extern "C" PyObject* PyObject_GetIter(PyObject*) {
 }
 
 extern "C" PyObject* PyObject_GetAttr(PyObject* o, PyObject* attr_name) {
-    Py_FatalError("unimplemented");
+    if (!isSubclass(attr_name->cls, str_cls)) {
+        PyErr_Format(PyExc_TypeError, "attribute name must be string, not '%.200s'", Py_TYPE(attr_name)->tp_name);
+        return NULL;
+    }
+
+    try {
+        return getattr(o, static_cast<BoxedString*>(attr_name)->s.c_str());
+    } catch (Box* b) {
+        Py_FatalError("unimplemented");
+    }
 }
 
 extern "C" PyObject* PyObject_GenericGetAttr(PyObject* o, PyObject* name) {
@@ -519,7 +558,11 @@ extern "C" int PyObject_Not(PyObject* o) {
 }
 
 extern "C" PyObject* PyObject_Call(PyObject* callable_object, PyObject* args, PyObject* kw) {
-    Py_FatalError("unimplemented");
+    try {
+        return runtimeCall(callable_object, ArgPassSpec(0, 0, true, true), args, kw, NULL, NULL, NULL);
+    } catch (Box* b) {
+        Py_FatalError("unimplemented");
+    }
 }
 
 extern "C" void PyObject_ClearWeakRefs(PyObject* object) {
@@ -945,7 +988,6 @@ void setupCAPI() {
                                                                      0, true, true)));
     method_cls->freeze();
 
-    /*
     wrapperdescr_cls = new BoxedClass(type_cls, object_cls, NULL, 0, sizeof(BoxedWrapperDescriptor), false);
     wrapperdescr_cls->giveAttr("__name__", boxStrConstant("wrapper_descriptor"));
     wrapperdescr_cls->giveAttr("__get__",
@@ -957,7 +999,6 @@ void setupCAPI() {
     wrapperobject_cls->giveAttr(
         "__call__", new BoxedFunction(boxRTFunction((void*)BoxedWrapperObject::__call__, UNKNOWN, 1, 0, true, true)));
     wrapperobject_cls->freeze();
-    */
 }
 
 void teardownCAPI() {

src/runtime/objmodel.cpp

@@ -412,7 +412,7 @@ extern "C" Box** unpackIntoArray(Box* obj, int64_t expected_size) {
 
 PyObject* Py_CallPythonNew(PyTypeObject* self, PyObject* args, PyObject* kwds) {
     try {
-        Py_FatalError("this function is unverified");
+        Py_FatalError("this function is untested");
 
         Box* new_attr = typeLookup(self, _new_str, NULL);
         assert(new_attr);
@@ -423,16 +423,33 @@ PyObject* Py_CallPythonNew(PyTypeObject* self, PyObject* args, PyObject* kwds) {
     }
 }
 
+PyObject* Py_CallPythonCall(PyObject* self, PyObject* args, PyObject* kwds) {
+    try {
+        Py_FatalError("this function is untested");
+
+        return runtimeCallInternal(self, NULL, ArgPassSpec(0, 0, true, true), args, kwds, NULL, NULL, NULL);
+    } catch (Box* e) {
+        abort();
+    }
+}
+
 void BoxedClass::freeze() {
     assert(!is_constant);
     assert(getattr("__name__")); // otherwise debugging will be very hard
 
+    // This will probably share a lot in common with Py_TypeReady:
     if (!tp_new) {
         this->tp_new = &Py_CallPythonNew;
     } else if (tp_new != Py_CallPythonNew) {
         ASSERT(0, "need to set __new__?");
     }
 
+    if (!tp_call) {
+        this->tp_call = &Py_CallPythonCall;
+    } else if (tp_call != Py_CallPythonCall) {
+        ASSERT(0, "need to set __call__?");
+    }
+
     is_constant = true;
 }
 
@@ -565,7 +582,7 @@ Box::Box(BoxedClass* cls) : cls(cls) {
     // the only way cls should be NULL is if we're creating the type_cls
     // object itself:
     if (cls == NULL) {
-        assert(type_cls == NULL);
+        ASSERT(type_cls == NULL, "should pass a non-null cls here");
     } else {
     }
 }
@@ -1482,6 +1499,13 @@ void setattrInternal(Box* obj, const std::string& attr, Box* val, SetattrRewrite
 
             rewrite_args = NULL;
         }
+
+        if (attr == "__call__") {
+            self->tp_call = &Py_CallPythonCall;
+            // TODO update subclasses
+
+            rewrite_args = NULL;
+        }
     }
 
     Box* _set_ = NULL;
@@ -1854,6 +1878,11 @@ extern "C" void dump(void* p) {
         if (isSubclass(b->cls, str_cls)) {
             printf("String value: %s\n", static_cast<BoxedString*>(b)->s.c_str());
         }
+
+        if (isSubclass(b->cls, tuple_cls)) {
+            printf("%ld elements\n", static_cast<BoxedTuple*>(b)->elts.size());
+        }
+
         return;
     }
 

src/runtime/tuple.cpp

@@ -101,7 +101,21 @@ Box* tupleGetitemSlice(BoxedTuple* self, BoxedSlice* slice) {
 }
 
 extern "C" PyObject* PyTuple_GetSlice(PyObject* p, Py_ssize_t low, Py_ssize_t high) {
-    Py_FatalError("unimplemented");
+    RELEASE_ASSERT(p->cls == tuple_cls, ""); // could it be a subclass or something else?
+    BoxedTuple* t = static_cast<BoxedTuple*>(p);
+
+    Py_ssize_t n = t->elts.size();
+    if (low < 0)
+        low = 0;
+    if (high > n)
+        high = n;
+    if (high < low)
+        high = low;
+
+    if (low == 0 && high == n)
+        return p;
+
+    return new BoxedTuple(BoxedTuple::GCVector(&t->elts[low], &t->elts[high]));
 }
 
 Box* tupleGetitem(BoxedTuple* self, Box* slice) {

test/tests/operator.py → test/tests/operator_test.py

-# expected: fail
 import operator
 
 print operator.methodcaller
 print operator.itemgetter
 print operator.attrgetter
+
+f = operator.methodcaller("__repr__")
+print f(1)
+print f("hello world")
+
+f = operator.itemgetter(0)
+print f("i")
+print f((None,))
+
+f = operator.attrgetter("count")
+print f(["a", "a"])("a")
+print f("ababa")("a")
+