Merge pull request #965 from undingen/method_cache

Add cpythons method cache

from_cpython/CMakeLists.txt

@@ -111,7 +111,7 @@ file(GLOB_RECURSE STDPARSER_SRCS Parser
     myreadline.c
     )
 
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers -Wno-tautological-compare -Wno-type-limits -Wno-unused-result -Wno-strict-aliasing")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers -Wno-tautological-compare -Wno-type-limits -Wno-unused-result -Wno-strict-aliasing -DPy_BUILD_CORE")
 add_library(FROM_CPYTHON OBJECT ${STDMODULE_SRCS} ${STDOBJECT_SRCS} ${STDPYTHON_SRCS} ${STDPARSER_SRCS})
 add_dependencies(FROM_CPYTHON copy_stdlib)
 

src/CMakeLists.txt

 set(CMAKE_INCLUDE_CURRENT_DIR ON)
-set(CMAKE_CXX_FLAGS_DEBUG "-g -DBINARY_SUFFIX= -DBINARY_STRIPPED_SUFFIX=_stripped")
-set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fstrict-aliasing -enable-tbaa -DNVALGRIND -DBINARY_SUFFIX=_release -DBINARY_STRIPPED_SUFFIX=")
+set(CMAKE_CXX_FLAGS_DEBUG "-g -DBINARY_SUFFIX= -DBINARY_STRIPPED_SUFFIX=_stripped -DPy_BUILD_CORE")
+set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fstrict-aliasing -enable-tbaa -DNVALGRIND -DBINARY_SUFFIX=_release -DBINARY_STRIPPED_SUFFIX= -DPy_BUILD_CORE")
 
 execute_process(COMMAND git rev-parse HEAD WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE GITREV OUTPUT_STRIP_TRAILING_WHITESPACE)
 set_source_files_properties(jit.cpp PROPERTIES COMPILE_DEFINITIONS "GITREV=${GITREV}")

src/capi/typeobject.cpp

@@ -3301,7 +3301,24 @@ void commonClassSetup(BoxedClass* cls) {
 }
 
 extern "C" void PyType_Modified(PyTypeObject* type) noexcept {
-    // We don't cache anything yet that would need to be invalidated:
+    PyObject* raw, *ref;
+    Py_ssize_t i, n;
+
+    if (!PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG))
+        return;
+
+    raw = type->tp_subclasses;
+    if (raw != NULL) {
+        n = PyList_GET_SIZE(raw);
+        for (i = 0; i < n; i++) {
+            ref = PyList_GET_ITEM(raw, i);
+            ref = PyWeakref_GET_OBJECT(ref);
+            if (ref != Py_None) {
+                PyType_Modified((PyTypeObject*)ref);
+            }
+        }
+    }
+    type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG;
 }
 
 template <ExceptionStyle S>
@@ -3504,6 +3521,7 @@ extern "C" int PyType_Ready(PyTypeObject* cls) noexcept {
         exception_types.push_back(cls);
     }
 
+    cls->tp_flags |= Py_TPFLAGS_READY;
     return 0;
 }
 

src/jit.cpp

@@ -455,7 +455,7 @@ static int main(int argc, char** argv) {
             main_module = createModule(boxString("__main__"), "<string>");
             rtncode = (RunModule(module, 1) != 0);
         } else {
-            main_module = createModule(boxString("__main__"), fn);
+            main_module = createModule(boxString("__main__"), fn ? fn : "<string>");
             rtncode = 0;
             if (fn != NULL) {
                 rtncode = RunMainFromImporter(fn);

src/runtime/objmodel.cpp

@@ -120,45 +120,6 @@ static inline Box* callattrInternal3(Box* obj, BoxedString* attr, LookupScope sc
     return callattrInternal<S, rewritable>(obj, attr, scope, rewrite_args, argspec, arg1, arg2, arg3, NULL, NULL);
 }
 
-#if STAT_TIMERS
-static uint64_t* pyhasher_timer_counter = Stats::getStatCounter("us_timer_PyHasher");
-static uint64_t* pyeq_timer_counter = Stats::getStatCounter("us_timer_PyEq");
-static uint64_t* pylt_timer_counter = Stats::getStatCounter("us_timer_PyLt");
-#endif
-size_t PyHasher::operator()(Box* b) const {
-#if EXPENSIVE_STAT_TIMERS
-    ScopedStatTimer _st(pyhasher_timer_counter, 10);
-#endif
-    if (b->cls == str_cls) {
-        auto s = static_cast<BoxedString*>(b);
-        return strHashUnboxed(s);
-    }
-
-    return hashUnboxed(b);
-}
-
-bool PyEq::operator()(Box* lhs, Box* rhs) const {
-#if EXPENSIVE_STAT_TIMERS
-    ScopedStatTimer _st(pyeq_timer_counter, 10);
-#endif
-
-    int r = PyObject_RichCompareBool(lhs, rhs, Py_EQ);
-    if (r == -1)
-        throwCAPIException();
-    return (bool)r;
-}
-
-bool PyLt::operator()(Box* lhs, Box* rhs) const {
-#if EXPENSIVE_STAT_TIMERS
-    ScopedStatTimer _st(pylt_timer_counter, 10);
-#endif
-
-    int r = PyObject_RichCompareBool(lhs, rhs, Py_LT);
-    if (r == -1)
-        throwCAPIException();
-    return (bool)r;
-}
-
 extern "C" Box* deopt(AST_expr* expr, Box* value) {
     STAT_TIMER(t0, "us_timer_deopt", 10);
 
@@ -465,7 +426,7 @@ BoxedClass::BoxedClass(BoxedClass* base, gcvisit_func gc_visit, int attrs_offset
     tp_weaklistoffset = weaklist_offset;
     tp_name = name;
 
-    tp_flags |= Py_TPFLAGS_DEFAULT_EXTERNAL;
+    tp_flags |= Py_TPFLAGS_DEFAULT_CORE;
     tp_flags |= Py_TPFLAGS_CHECKTYPES;
     tp_flags |= Py_TPFLAGS_BASETYPE;
     tp_flags |= Py_TPFLAGS_HAVE_GC;
@@ -574,6 +535,7 @@ void BoxedClass::finishInitialization() {
     this->tp_dict = this->getAttrWrapper();
 
     commonClassSetup(this);
+    tp_flags |= Py_TPFLAGS_READY;
 }
 
 BoxedHeapClass::BoxedHeapClass(BoxedClass* base, gcvisit_func gc_visit, int attrs_offset, int weaklist_offset,
@@ -987,13 +949,75 @@ extern "C" PyObject* _PyType_Lookup(PyTypeObject* type, PyObject* name) noexcept
     }
 }
 
+#define MCACHE_MAX_ATTR_SIZE 100
+#define MCACHE_SIZE_EXP 10
+#define MCACHE_HASH(version, name_hash)                                                                                \
+    (((unsigned int)(version) * (unsigned int)(name_hash)) >> (8 * sizeof(unsigned int) - MCACHE_SIZE_EXP))
+#define MCACHE_HASH_METHOD(type, name) MCACHE_HASH((type)->tp_version_tag, ((BoxedString*)(name))->hash)
+#define MCACHE_CACHEABLE_NAME(name) PyString_CheckExact(name) && PyString_GET_SIZE(name) <= MCACHE_MAX_ATTR_SIZE
+
+struct method_cache_entry {
+    unsigned int version;
+    PyObject* name;  /* reference to exactly a str or None */
+    PyObject* value; /* borrowed */
+};
+
+static struct method_cache_entry method_cache[1 << MCACHE_SIZE_EXP];
+static unsigned int next_version_tag = 0;
+
+int assign_version_tag(PyTypeObject* type) noexcept {
+    /* Ensure that the tp_version_tag is valid and set
+       Py_TPFLAGS_VALID_VERSION_TAG.  To respect the invariant, this
+       must first be done on all super classes.  Return 0 if this
+       cannot be done, 1 if Py_TPFLAGS_VALID_VERSION_TAG.
+    */
+    Py_ssize_t i, n;
+    PyObject* bases;
+
+    if (PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG))
+        return 1;
+    if (!PyType_HasFeature(type, Py_TPFLAGS_HAVE_VERSION_TAG))
+        return 0;
+    if (!PyType_HasFeature(type, Py_TPFLAGS_READY))
+        return 0;
+
+    type->tp_version_tag = next_version_tag++;
+    /* for stress-testing: next_version_tag &= 0xFF; */
+
+    if (type->tp_version_tag == 0) {
+        /* wrap-around or just starting Python - clear the whole
+           cache by filling names with references to Py_None.
+           Values are also set to NULL for added protection, as they
+           are borrowed reference */
+        for (i = 0; i < (1 << MCACHE_SIZE_EXP); i++) {
+            method_cache[i].value = NULL;
+            Py_XDECREF(method_cache[i].name);
+            method_cache[i].name = Py_None;
+            Py_INCREF(Py_None);
+        }
+        /* mark all version tags as invalid */
+        PyType_Modified(&PyBaseObject_Type);
+        return 1;
+    }
+    bases = type->tp_bases;
+    n = PyTuple_GET_SIZE(bases);
+    for (i = 0; i < n; i++) {
+        PyObject* b = PyTuple_GET_ITEM(bases, i);
+        assert(PyType_Check(b));
+        if (!assign_version_tag((PyTypeObject*)b))
+            return 0;
+    }
+    type->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG;
+    return 1;
+}
+
 template <Rewritable rewritable> Box* typeLookup(BoxedClass* cls, BoxedString* attr, GetattrRewriteArgs* rewrite_args) {
     if (rewritable == NOT_REWRITABLE) {
         assert(!rewrite_args);
         rewrite_args = NULL;
     }
 
-    Box* val;
+    Box* val = NULL;
 
     if (rewrite_args) {
         assert(!rewrite_args->isSuccessful());
@@ -1048,6 +1072,17 @@ template <Rewritable rewritable> Box* typeLookup(BoxedClass* cls, BoxedString* a
         assert(attr->interned_state != SSTATE_NOT_INTERNED);
         assert(cls->tp_mro);
         assert(cls->tp_mro->cls == tuple_cls);
+
+        if (MCACHE_CACHEABLE_NAME(attr) && PyType_HasFeature(cls, Py_TPFLAGS_VALID_VERSION_TAG)) {
+            if (attr->hash == -1)
+                strHashUnboxed(attr);
+
+            /* fast path */
+            unsigned int h = MCACHE_HASH_METHOD(cls, attr);
+            if (method_cache[h].version == cls->tp_version_tag && method_cache[h].name == attr)
+                return method_cache[h].value;
+        }
+
         for (auto b : *static_cast<BoxedTuple*>(cls->tp_mro)) {
             // object_cls will get checked very often, but it only
             // has attributes that start with an underscore.
@@ -1060,9 +1095,18 @@ template <Rewritable rewritable> Box* typeLookup(BoxedClass* cls, BoxedString* a
 
             val = b->getattr(attr);
             if (val)
-                return val;
+                break;
         }
-        return NULL;
+
+        if (MCACHE_CACHEABLE_NAME(attr) && assign_version_tag(cls)) {
+            unsigned int h = MCACHE_HASH_METHOD(cls, attr);
+            method_cache[h].version = cls->tp_version_tag;
+            method_cache[h].value = val; /* borrowed */
+            Py_INCREF(attr);
+            Py_DECREF(method_cache[h].name);
+            method_cache[h].name = attr;
+        }
+        return val;
     }
 }
 template Box* typeLookup<REWRITABLE>(BoxedClass*, BoxedString*, GetattrRewriteArgs*);
@@ -2321,6 +2365,10 @@ void setattrGeneric(Box* obj, BoxedString* attr, Box* val, SetattrRewriteArgs* r
             rewrite_args = NULL;
             REWRITE_ABORTED("");
         }
+
+        // update_slot() calls PyType_Modified() internally so we only have to explicitly call it inside the IC
+        if (rewrite_args)
+            rewrite_args->rewriter->call(true, (void*)PyType_Modified, rewrite_args->obj);
     }
 }
 

src/runtime/str.cpp

@@ -55,33 +55,33 @@ namespace pyston {
 BoxedString* EmptyString;
 BoxedString* characters[UCHAR_MAX + 1];
 
-BoxedString::BoxedString(const char* s, size_t n) : interned_state(SSTATE_NOT_INTERNED) {
+BoxedString::BoxedString(const char* s, size_t n) : hash(-1), interned_state(SSTATE_NOT_INTERNED) {
     assert(s);
     RELEASE_ASSERT(n != llvm::StringRef::npos, "");
-    memmove(data(), s, n);
+    memcpy(data(), s, n);
     data()[n] = 0;
 }
 
-BoxedString::BoxedString(llvm::StringRef lhs, llvm::StringRef rhs) : interned_state(SSTATE_NOT_INTERNED) {
+BoxedString::BoxedString(llvm::StringRef lhs, llvm::StringRef rhs) : hash(-1), interned_state(SSTATE_NOT_INTERNED) {
     RELEASE_ASSERT(lhs.size() + rhs.size() != llvm::StringRef::npos, "");
-    memmove(data(), lhs.data(), lhs.size());
-    memmove(data() + lhs.size(), rhs.data(), rhs.size());
+    memcpy(data(), lhs.data(), lhs.size());
+    memcpy(data() + lhs.size(), rhs.data(), rhs.size());
     data()[lhs.size() + rhs.size()] = 0;
 }
 
-BoxedString::BoxedString(llvm::StringRef s) : interned_state(SSTATE_NOT_INTERNED) {
+BoxedString::BoxedString(llvm::StringRef s) : hash(-1), interned_state(SSTATE_NOT_INTERNED) {
     RELEASE_ASSERT(s.size() != llvm::StringRef::npos, "");
-    memmove(data(), s.data(), s.size());
+    memcpy(data(), s.data(), s.size());
     data()[s.size()] = 0;
 }
 
-BoxedString::BoxedString(size_t n, char c) : interned_state(SSTATE_NOT_INTERNED) {
+BoxedString::BoxedString(size_t n, char c) : hash(-1), interned_state(SSTATE_NOT_INTERNED) {
     RELEASE_ASSERT(n != llvm::StringRef::npos, "");
     memset(data(), c, n);
     data()[n] = 0;
 }
 
-BoxedString::BoxedString(size_t n) : interned_state(SSTATE_NOT_INTERNED) {
+BoxedString::BoxedString(size_t n) : hash(-1), interned_state(SSTATE_NOT_INTERNED) {
     RELEASE_ASSERT(n != llvm::StringRef::npos, "");
     // Note: no memset.  add the null-terminator for good measure though
     // (CPython does the same thing).
@@ -1586,13 +1586,15 @@ extern "C" size_t strHashUnboxed(BoxedString* self) {
 #ifdef Py_DEBUG
     assert(_Py_HashSecret_Initialized);
 #endif
-
+    if (self->hash != -1)
+        return self->hash;
     long len = Py_SIZE(self);
     /*
       We make the hash of the empty string be 0, rather than using
       (prefix ^ suffix), since this slightly obfuscates the hash secret
     */
     if (len == 0) {
+        self->hash = 0;
         return 0;
     }
     p = self->s().data();
@@ -1604,7 +1606,7 @@ extern "C" size_t strHashUnboxed(BoxedString* self) {
     x ^= _Py_HashSecret.suffix;
     if (x == -1)
         x = -2;
-
+    self->hash = x;
     return x;
 }
 
@@ -1662,6 +1664,11 @@ Box* _strSlice(BoxedString* self, i64 start, i64 stop, i64 step, i64 length) {
     if (length == 0)
         return EmptyString;
 
+    if (length == 1) {
+        char c = self->s()[start];
+        return characters[c & UCHAR_MAX];
+    }
+
     BoxedString* bs = BoxedString::createUninitializedString(length);
     copySlice(bs->data(), s.data(), start, step, length);
     return bs;
@@ -2531,19 +2538,18 @@ extern "C" int _PyString_Resize(PyObject** pv, Py_ssize_t newsize) noexcept {
     if (newsize < s->size()) {
         // XXX resize the box (by reallocating) smaller if it makes sense
         s->ob_size = newsize;
+        s->hash = -1; /* invalidate cached hash value */
         s->data()[newsize] = 0;
         return 0;
     }
 
     BoxedString* resized;
-
     if (s->cls == str_cls)
-        resized = new (newsize) BoxedString(newsize, 0); // we need an uninitialized string, but this will memset
+        resized = BoxedString::createUninitializedString(newsize);
     else
-        resized = new (s->cls, newsize)
-            BoxedString(newsize, 0); // we need an uninitialized string, but this will memset
-    memmove(resized->data(), s->data(), s->size());
-
+        resized = BoxedString::createUninitializedString(s->cls, newsize);
+    memcpy(resized->data(), s->data(), s->size());
+    resized->data()[newsize] = 0;
     *pv = resized;
     return 0;
 }

src/runtime/types.h

@@ -175,6 +175,12 @@ extern "C" void printFloat(double d);
 Box* objectStr(Box*);
 Box* objectRepr(Box*);
 
+void checkAndThrowCAPIException();
+void throwCAPIException() __attribute__((noreturn));
+void ensureCAPIExceptionSet();
+struct ExcInfo;
+void setCAPIException(const ExcInfo& e);
+
 // In Pyston, this is the same type as CPython's PyTypeObject (they are interchangeable, but we
 // use BoxedClass in Pyston wherever possible as a convention).
 class BoxedClass : public BoxVar {
@@ -387,6 +393,7 @@ public:
     // optimizations and inlining, creating a new one each time shouldn't have any cost.
     llvm::StringRef s() const { return llvm::StringRef(s_data, ob_size); };
 
+    long hash; // -1 means not yet computed
     char interned_state;
 
     char* data() { return s_data; }
@@ -430,6 +437,7 @@ public:
     // creates an uninitialized string of length n; useful for directly constructing into the string and avoiding
     // copies:
     static BoxedString* createUninitializedString(ssize_t n) { return new (n) BoxedString(n); }
+    static BoxedString* createUninitializedString(BoxedClass* cls, ssize_t n) { return new (cls, n) BoxedString(n); }
 
     // Gets a writeable pointer to the contents of a string.
     // Is only meant to be used with something just created from createUninitializedString(), though
@@ -447,6 +455,7 @@ private:
 };
 
 extern "C" size_t strHashUnboxed(BoxedString* self);
+extern "C" int64_t hashUnboxed(Box* obj);
 
 class BoxedInstanceMethod : public Box {
 public:
@@ -678,15 +687,33 @@ static_assert(offsetof(BoxedTuple, elts) == offsetof(PyTupleObject, ob_item), ""
 extern BoxedString* characters[UCHAR_MAX + 1];
 
 struct PyHasher {
-    size_t operator()(Box*) const;
+    size_t operator()(Box* b) const {
+        if (b->cls == str_cls) {
+            auto s = static_cast<BoxedString*>(b);
+            if (s->hash != -1)
+                return s->hash;
+            return strHashUnboxed(s);
+        }
+        return hashUnboxed(b);
+    }
 };
 
 struct PyEq {
-    bool operator()(Box*, Box*) const;
+    bool operator()(Box* lhs, Box* rhs) const {
+        int r = PyObject_RichCompareBool(lhs, rhs, Py_EQ);
+        if (r == -1)
+            throwCAPIException();
+        return (bool)r;
+    }
 };
 
 struct PyLt {
-    bool operator()(Box*, Box*) const;
+    bool operator()(Box* lhs, Box* rhs) const {
+        int r = PyObject_RichCompareBool(lhs, rhs, Py_LT);
+        if (r == -1)
+            throwCAPIException();
+        return (bool)r;
+    }
 };
 
 // llvm::DenseMap doesn't store the original hash values, choosing to instead
@@ -1077,12 +1104,6 @@ AST* unboxAst(Box* b);
 // Our default for tp_alloc:
 extern "C" PyObject* PystonType_GenericAlloc(BoxedClass* cls, Py_ssize_t nitems) noexcept;
 
-void checkAndThrowCAPIException();
-void throwCAPIException() __attribute__((noreturn));
-void ensureCAPIExceptionSet();
-struct ExcInfo;
-void setCAPIException(const ExcInfo& e);
-
 #define fatalOrError(exception, message)                                                                               \
     do {                                                                                                               \
         if (CONTINUE_AFTER_FATAL)                                                                                      \