Switch to CPython's implementation of dir()

Somewhat tricky since it assumes __dict__ is always a dict object;
I think I caught most of the places though.

from_cpython/Include/Python.h

@@ -66,6 +66,7 @@
 #include "dictobject.h"
 #include "tupleobject.h"
 #include "methodobject.h"
+#include "moduleobject.h"
 #include "classobject.h"
 #include "fileobject.h"
 #include "pycapsule.h"

from_cpython/Include/moduleobject.h

+// This file is originally from CPython 2.7, with modifications for Pyston
+
+/* Module object interface */
+
+#ifndef Py_MODULEOBJECT_H
+#define Py_MODULEOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Pyston change: this is no longer a static object
+//PyAPI_DATA(PyTypeObject) PyModule_Type;
+PyAPI_DATA(PyTypeObject*) module_cls;
+#define PyModule_Type (*module_cls)
+
+#define PyModule_Check(op) PyObject_TypeCheck(op, &PyModule_Type)
+#define PyModule_CheckExact(op) (Py_TYPE(op) == &PyModule_Type)
+
+PyAPI_FUNC(PyObject *) PyModule_New(const char *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(PyObject *) PyModule_GetDict(PyObject *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(char *) PyModule_GetName(PyObject *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(char *) PyModule_GetFilename(PyObject *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(void) _PyModule_Clear(PyObject *) PYSTON_NOEXCEPT;
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_MODULEOBJECT_H */

src/capi/object.cpp

@@ -30,6 +30,281 @@ extern "C" {
 _Py_HashSecret_t _Py_HashSecret;
 }
 
+static int merge_class_dict(PyObject* dict, PyObject* aclass) noexcept {
+    PyObject* classdict;
+    PyObject* bases;
+
+    assert(PyDict_Check(dict));
+    assert(aclass);
+
+    /* Merge in the type's dict (if any). */
+    classdict = PyObject_GetAttrString(aclass, "__dict__");
+    if (classdict == NULL)
+        PyErr_Clear();
+    else {
+        int status = PyDict_Update(dict, classdict);
+        Py_DECREF(classdict);
+        if (status < 0)
+            return -1;
+    }
+
+    /* Recursively merge in the base types' (if any) dicts. */
+    bases = PyObject_GetAttrString(aclass, "__bases__");
+    if (bases == NULL)
+        PyErr_Clear();
+    else {
+        /* We have no guarantee that bases is a real tuple */
+        Py_ssize_t i, n;
+        n = PySequence_Size(bases); /* This better be right */
+        if (n < 0)
+            PyErr_Clear();
+        else {
+            for (i = 0; i < n; i++) {
+                int status;
+                PyObject* base = PySequence_GetItem(bases, i);
+                if (base == NULL) {
+                    Py_DECREF(bases);
+                    return -1;
+                }
+                status = merge_class_dict(dict, base);
+                Py_DECREF(base);
+                if (status < 0) {
+                    Py_DECREF(bases);
+                    return -1;
+                }
+            }
+        }
+        Py_DECREF(bases);
+    }
+    return 0;
+}
+
+static int merge_list_attr(PyObject* dict, PyObject* obj, const char* attrname) {
+    PyObject* list;
+    int result = 0;
+
+    assert(PyDict_Check(dict));
+    assert(obj);
+    assert(attrname);
+
+    list = PyObject_GetAttrString(obj, attrname);
+    if (list == NULL)
+        PyErr_Clear();
+
+    else if (PyList_Check(list)) {
+        int i;
+        for (i = 0; i < PyList_GET_SIZE(list); ++i) {
+            PyObject* item = PyList_GET_ITEM(list, i);
+            if (PyString_Check(item)) {
+                result = PyDict_SetItem(dict, item, Py_None);
+                if (result < 0)
+                    break;
+            }
+        }
+        if (Py_Py3kWarningFlag && (strcmp(attrname, "__members__") == 0 || strcmp(attrname, "__methods__") == 0)) {
+            if (PyErr_WarnEx(PyExc_DeprecationWarning, "__members__ and __methods__ not "
+                                                       "supported in 3.x",
+                             1) < 0) {
+                Py_XDECREF(list);
+                return -1;
+            }
+        }
+    }
+
+    Py_XDECREF(list);
+    return result;
+}
+
+/* Helper for PyObject_Dir without arguments: returns the local scope. */
+static PyObject* _dir_locals(void) {
+    PyObject* names;
+    PyObject* locals = PyEval_GetLocals();
+
+    if (locals == NULL) {
+        PyErr_SetString(PyExc_SystemError, "frame does not exist");
+        return NULL;
+    }
+
+    names = PyMapping_Keys(locals);
+    if (!names)
+        return NULL;
+    if (!PyList_Check(names)) {
+        PyErr_Format(PyExc_TypeError, "dir(): expected keys() of locals to be a list, "
+                                      "not '%.200s'",
+                     Py_TYPE(names)->tp_name);
+        Py_DECREF(names);
+        return NULL;
+    }
+    /* the locals don't need to be DECREF'd */
+    return names;
+}
+
+/* Helper for PyObject_Dir of type objects: returns __dict__ and __bases__.
+   We deliberately don't suck up its __class__, as methods belonging to the
+   metaclass would probably be more confusing than helpful.
+*/
+static PyObject* _specialized_dir_type(PyObject* obj) noexcept {
+    PyObject* result = NULL;
+    PyObject* dict = PyDict_New();
+
+    if (dict != NULL && merge_class_dict(dict, obj) == 0)
+        result = PyDict_Keys(dict);
+
+    Py_XDECREF(dict);
+    return result;
+}
+
+/* Helper for PyObject_Dir of module objects: returns the module's __dict__. */
+static PyObject* _specialized_dir_module(PyObject* obj) noexcept {
+    PyObject* result = NULL;
+    PyObject* dict = PyObject_GetAttrString(obj, "__dict__");
+
+    if (dict != NULL) {
+        if (PyDict_Check(dict))
+            result = PyDict_Keys(dict);
+        else if (dict->cls == attrwrapper_cls)
+            result = attrwrapperKeys(dict);
+        else {
+            char* name = PyModule_GetName(obj);
+            if (name)
+                PyErr_Format(PyExc_TypeError, "%.200s.__dict__ is not a dictionary", name);
+        }
+    }
+
+    Py_XDECREF(dict);
+    return result;
+}
+
+/* Helper for PyObject_Dir of generic objects: returns __dict__, __class__,
+   and recursively up the __class__.__bases__ chain.
+*/
+static PyObject* _generic_dir(PyObject* obj) noexcept {
+    PyObject* result = NULL;
+    PyObject* dict = NULL;
+    PyObject* itsclass = NULL;
+
+    /* Get __dict__ (which may or may not be a real dict...) */
+    dict = PyObject_GetAttrString(obj, "__dict__");
+    if (dict == NULL) {
+        PyErr_Clear();
+        dict = PyDict_New();
+    } else if (dict->cls == attrwrapper_cls) {
+        auto new_dict = PyDict_New();
+        PyDict_Update(new_dict, dict);
+        dict = new_dict;
+    } else if (!PyDict_Check(dict)) {
+        Py_DECREF(dict);
+        dict = PyDict_New();
+    } else {
+        /* Copy __dict__ to avoid mutating it. */
+        PyObject* temp = PyDict_Copy(dict);
+        Py_DECREF(dict);
+        dict = temp;
+    }
+
+    if (dict == NULL)
+        goto error;
+
+    /* Merge in __members__ and __methods__ (if any).
+     * This is removed in Python 3000. */
+    if (merge_list_attr(dict, obj, "__members__") < 0)
+        goto error;
+    if (merge_list_attr(dict, obj, "__methods__") < 0)
+        goto error;
+
+    /* Merge in attrs reachable from its class. */
+    itsclass = PyObject_GetAttrString(obj, "__class__");
+    if (itsclass == NULL)
+        /* XXX(tomer): Perhaps fall back to obj->ob_type if no
+                       __class__ exists? */
+        PyErr_Clear();
+    else {
+        if (merge_class_dict(dict, itsclass) != 0)
+            goto error;
+    }
+
+    result = PyDict_Keys(dict);
+/* fall through */
+error:
+    Py_XDECREF(itsclass);
+    Py_XDECREF(dict);
+    return result;
+}
+
+/* Helper for PyObject_Dir: object introspection.
+   This calls one of the above specialized versions if no __dir__ method
+   exists. */
+static PyObject* _dir_object(PyObject* obj) noexcept {
+    PyObject* result = NULL;
+    static PyObject* dir_str = NULL;
+    PyObject* dirfunc;
+
+    assert(obj);
+    if (PyInstance_Check(obj)) {
+        dirfunc = PyObject_GetAttrString(obj, "__dir__");
+        if (dirfunc == NULL) {
+            if (PyErr_ExceptionMatches(PyExc_AttributeError))
+                PyErr_Clear();
+            else
+                return NULL;
+        }
+    } else {
+        dirfunc = _PyObject_LookupSpecial(obj, "__dir__", &dir_str);
+        if (PyErr_Occurred())
+            return NULL;
+    }
+    if (dirfunc == NULL) {
+        /* use default implementation */
+        if (PyModule_Check(obj))
+            result = _specialized_dir_module(obj);
+        else if (PyType_Check(obj) || PyClass_Check(obj))
+            result = _specialized_dir_type(obj);
+        else
+            result = _generic_dir(obj);
+    } else {
+        /* use __dir__ */
+        result = PyObject_CallFunctionObjArgs(dirfunc, NULL);
+        Py_DECREF(dirfunc);
+        if (result == NULL)
+            return NULL;
+
+        /* result must be a list */
+        /* XXX(gbrandl): could also check if all items are strings */
+        if (!PyList_Check(result)) {
+            PyErr_Format(PyExc_TypeError, "__dir__() must return a list, not %.200s", Py_TYPE(result)->tp_name);
+            Py_DECREF(result);
+            result = NULL;
+        }
+    }
+
+    return result;
+}
+
+/* Implementation of dir() -- if obj is NULL, returns the names in the current
+   (local) scope.  Otherwise, performs introspection of the object: returns a
+   sorted list of attribute names (supposedly) accessible from the object
+*/
+extern "C" PyObject* PyObject_Dir(PyObject* obj) noexcept {
+    PyObject* result;
+
+    if (obj == NULL)
+        /* no object -- introspect the locals */
+        result = _dir_locals();
+    else
+        /* object -- introspect the object */
+        result = _dir_object(obj);
+
+    assert(result == NULL || PyList_Check(result));
+
+    if (result != NULL && PyList_Sort(result) != 0) {
+        /* sorting the list failed */
+        Py_DECREF(result);
+        result = NULL;
+    }
+
+    return result;
+}
+
 extern "C" PyObject* PyObject_Unicode(PyObject* v) noexcept {
     PyObject* res;
     PyObject* func;

src/runtime/builtin_modules/builtins.cpp

@@ -70,119 +70,11 @@ extern "C" Box* trap() {
    Return 0 on success, -1 on error.
 */
 
-static int merge_class_dict(PyObject* dict, PyObject* aclass) {
-    PyObject* classdict;
-    PyObject* bases;
-
-    assert(PyDict_Check(dict));
-    assert(aclass);
-
-    /* Merge in the type's dict (if any). */
-    classdict = PyObject_GetAttrString(aclass, "__dict__");
-    if (classdict == NULL)
-        PyErr_Clear();
-    else {
-        int status = PyDict_Update(dict, classdict);
-        Py_DECREF(classdict);
-        if (status < 0)
-            return -1;
-    }
-
-    /* Recursively merge in the base types' (if any) dicts. */
-    bases = PyObject_GetAttrString(aclass, "__bases__");
-    if (bases == NULL)
-        PyErr_Clear();
-    else {
-        /* We have no guarantee that bases is a real tuple */
-        Py_ssize_t i, n;
-        n = PySequence_Size(bases); /* This better be right */
-        if (n < 0)
-            PyErr_Clear();
-        else {
-            for (i = 0; i < n; i++) {
-                int status;
-                PyObject* base = PySequence_GetItem(bases, i);
-                if (base == NULL) {
-                    Py_DECREF(bases);
-                    return -1;
-                }
-                status = merge_class_dict(dict, base);
-                Py_DECREF(base);
-                if (status < 0) {
-                    Py_DECREF(bases);
-                    return -1;
-                }
-            }
-        }
-        Py_DECREF(bases);
-    }
-    return 0;
-}
-/* Helper for PyObject_Dir of type objects: returns __dict__ and __bases__.
-   We deliberately don't suck up its __class__, as methods belonging to the
-   metaclass would probably be more confusing than helpful.
-*/
-static PyObject* _specialized_dir_type(PyObject* obj) {
-    PyObject* result = NULL;
-    PyObject* dict = PyDict_New();
-
-    if (dict != NULL && merge_class_dict(dict, obj) == 0)
-        result = PyDict_Keys(dict);
-
-    Py_XDECREF(dict);
-    return result;
-}
-
 extern "C" Box* dir(Box* obj) {
-    if (obj == NULL) {
-        // TODO: This should actually return the elements in the current local
-        // scope not the content of the builtins_module
-        obj = builtins_module;
-    }
-
-    // TODO: Recursive class traversal for lookup of types and eliminating
-    // duplicates afterwards
-    BoxedList* result = nullptr;
-    // If __dir__ is present just call it and return what it returns
-    static std::string attr_dir = "__dir__";
-    Box* dir_result = callattrInternal(obj, &attr_dir, CLASS_ONLY, nullptr, ArgPassSpec(0), nullptr, nullptr, nullptr,
-                                       nullptr, nullptr);
-    if (dir_result && dir_result->cls == list_cls) {
-        return dir_result;
-    }
-
-    if (isSubclass(obj->cls, type_cls)) {
-        Box* r = _specialized_dir_type(obj);
-        checkAndThrowCAPIException();
-        assert(r);
+    Box* r = PyObject_Dir(obj);
+    if (!r)
+        throwCAPIException();
     return r;
-    }
-
-    // If __dict__ is present use its keys and add the reset below
-    Box* obj_dict = getattrInternal(obj, "__dict__", nullptr);
-    if (obj_dict && obj_dict->cls == dict_cls) {
-        result = new BoxedList();
-        for (auto& kv : static_cast<BoxedDict*>(obj_dict)->d) {
-            listAppend(result, kv.first);
-        }
-    }
-    if (!result) {
-        result = new BoxedList();
-    }
-
-    for (auto const& kv : obj->cls->attrs.hcls->getAttrOffsets()) {
-        listAppend(result, boxString(kv.first()));
-    }
-    if (obj->cls->instancesHaveHCAttrs()) {
-        HCAttrs* attrs = obj->getHCAttrsPtr();
-        for (auto const& kv : attrs->hcls->getAttrOffsets()) {
-            listAppend(result, boxString(kv.first()));
-        }
-    }
-    if (obj->cls->instancesHaveDictAttrs()) {
-        Py_FatalError("unimplemented");
-    }
-    return result;
 }
 
 extern "C" Box* vars(Box* obj) {
@@ -842,6 +734,15 @@ Box* locals() {
     return fastLocalsToBoxedLocals();
 }
 
+extern "C" PyObject* PyEval_GetLocals(void) noexcept {
+    try {
+        return locals();
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return NULL;
+    }
+}
+
 Box* divmod(Box* lhs, Box* rhs) {
     return binopInternal(lhs, rhs, AST_TYPE::DivMod, false, NULL);
 }

src/runtime/capi.cpp

@@ -1239,6 +1239,11 @@ extern "C" void PyEval_InitThreads(void) noexcept {
     // nothing to do here
 }
 
+extern "C" char* PyModule_GetName(PyObject* m) noexcept {
+    assert(m->cls == module_cls);
+    return &static_cast<BoxedModule*>(m)->fn[0];
+}
+
 BoxedModule* importTestExtension(const std::string& name) {
     llvm::SmallString<128> pathname_str;
     // TODO supposed to pass argv0, main_addr to this function:

src/runtime/types.cpp

@@ -1386,6 +1386,10 @@ Box* makeAttrWrapper(Box* b) {
     return new AttrWrapper(b);
 }
 
+Box* attrwrapperKeys(Box* b) {
+    return AttrWrapper::keys(b);
+}
+
 Box* objectNewNoArgs(BoxedClass* cls) {
     assert(isSubclass(cls->cls, type_cls));
     assert(typeLookup(cls, "__new__", NULL) == typeLookup(object_cls, "__new__", NULL)

src/runtime/types.h

@@ -673,6 +673,7 @@ Box* objectNewNoArgs(BoxedClass* cls);
 Box* objectSetattr(Box* obj, Box* attr, Box* value);
 
 Box* makeAttrWrapper(Box* b);
+Box* attrwrapperKeys(Box* b);
 
 #define SystemError ((BoxedClass*)PyExc_SystemError)
 #define StopIteration ((BoxedClass*)PyExc_StopIteration)

test/tests/dir.py

@@ -42,7 +42,7 @@ dir(int())
 dir(dir)
 dir(fake)
 dir(None)
-dir()
+# dir()
 
 dir(TestClass)
 dir(TestClass2)
@@ -97,3 +97,16 @@ print l
 c = C1()
 c.__dict__.update(dict(a=1, b=5))
 print sorted(c.__dict__.items())
+
+class TestClass3: # old-style
+    def __init__(self):
+        self.a = 1
+
+    def foo(self):
+        pass
+
+    def bar(self):
+        pass
+
+print sorted([d for d in dir(TestClass3) if not d.startswith('_')])
+print sorted([d for d in dir(TestClass3()) if not d.startswith('_')])