Merge pull request #1059 from undingen/cleanup_numpy

enable PyStringObject and remove some workarounds inside the numpy patch

from_cpython/Include/stringobject.h

@@ -33,8 +33,6 @@ functions should be applied to nil objects.
    interning of any string.
    Together, these sped the interpreter by up to 20%. */
 
-// Pyston change: comment this out since this is not the format we're using
-#if 0
 typedef struct {
     PyObject_VAR_HEAD
     long ob_shash;
@@ -50,9 +48,6 @@ typedef struct {
      *       from 'interned' to this object are *not counted* in ob_refcnt.
      */
 } PyStringObject;
-#endif
-struct _PyStringObject;
-typedef struct _PyStringObject PyStringObject;
 
 #define SSTATE_NOT_INTERNED 0
 #define SSTATE_INTERNED_MORTAL 1

src/capi/typeobject.cpp

@@ -3402,14 +3402,14 @@ extern "C" void PyType_GiveHcAttrsDictDescr(PyTypeObject* cls) noexcept {
 }
 
 extern "C" int PyType_Ready(PyTypeObject* cls) noexcept {
-    ASSERT(!cls->is_pyston_class, "should not call this on Pyston classes");
-
     // if this type is already in ready state we are finished.
     if (cls->tp_flags & Py_TPFLAGS_READY) {
         assert(cls->tp_dict != NULL);
         return 0;
     }
 
+    ASSERT(!cls->is_pyston_class, "should not call this on Pyston classes");
+
     gc::registerNonheapRootObject(cls, sizeof(PyTypeObject));
 
     // unhandled fields:

src/runtime/types.h

@@ -408,7 +408,8 @@ public:
     llvm::StringRef s() const { return llvm::StringRef(s_data, ob_size); };
 
     long hash; // -1 means not yet computed
-    char interned_state;
+    int interned_state;
+    char s_data[0];
 
     char* data() { return s_data; }
     const char* c_str() {
@@ -463,10 +464,13 @@ private:
 
     BoxedString(size_t n); // non-initializing constructor
 
-    char s_data[0];
-
     friend void setupRuntime();
 };
+static_assert(sizeof(BoxedString) == sizeof(PyStringObject), "");
+static_assert(offsetof(BoxedString, ob_size) == offsetof(PyStringObject, ob_size), "");
+static_assert(offsetof(BoxedString, hash) == offsetof(PyStringObject, ob_shash), "");
+static_assert(offsetof(BoxedString, interned_state) == offsetof(PyStringObject, ob_sstate), "");
+static_assert(offsetof(BoxedString, s_data) == offsetof(PyStringObject, ob_sval), "");
 
 extern "C" size_t strHashUnboxed(BoxedString* self);
 extern "C" int64_t hashUnboxed(Box* obj);

test/integration/numpy_patch.patch

@@ -147,78 +147,11 @@ index d025901..27c8718 100644
                  }
                  PyDict_SetItem(fields, title, tup);
              }
-diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c
-index 6155551..d488816 100644
---- a/numpy/core/src/multiarray/multiarraymodule.c
-+++ b/numpy/core/src/multiarray/multiarraymodule.c
-@@ -4206,6 +4206,7 @@ setup_scalartypes(PyObject *NPY_UNUSED(dict))
-     initialize_casting_tables();
-     initialize_numeric_types();
- 
-+    /* Pyston assumes PyType_Ready doesn't get called on Pyston classes
-     if (PyType_Ready(&PyBool_Type) < 0) {
-         return -1;
-     }
-@@ -4226,6 +4227,7 @@ setup_scalartypes(PyObject *NPY_UNUSED(dict))
-     if (PyType_Ready(&PyUnicode_Type) < 0) {
-         return -1;
-     }
-+    */
- 
- #define SINGLE_INHERIT(child, parent)                                   \
-     Py##child##ArrType_Type.tp_base = &Py##parent##ArrType_Type;        \
-diff --git a/numpy/core/src/multiarray/scalarapi.c b/numpy/core/src/multiarray/scalarapi.c
-index 71a82d7..15a0b36 100644
---- a/numpy/core/src/multiarray/scalarapi.c
-+++ b/numpy/core/src/multiarray/scalarapi.c
-@@ -712,11 +712,18 @@ PyArray_Scalar(void *data, PyArray_Descr *descr, PyObject *base)
-     if (PyTypeNum_ISFLEXIBLE(type_num)) {
-         if (type_num == NPY_STRING) {
-             destptr = PyString_AS_STRING(obj);
-+/*
-+ * Pyston change: commented out for now. Two problems:
-+ * 1) We don't have ob_shash and ob_sstate since we have different structs
-+ * 2) NumPy tries to bypass the CPython API and use a memcpy here. I'm not
-+ *    too sure what all the implications of this are but at the very least,
-+ *    it would require resetting the hash and changing the state to not interned.
-             ((PyStringObject *)obj)->ob_shash = -1;
- #if !defined(NPY_PY3K)
-             ((PyStringObject *)obj)->ob_sstate = SSTATE_NOT_INTERNED;
- #endif
-             memcpy(destptr, data, itemsize);
-+*/
-             return obj;
-         }
- #if PY_VERSION_HEX < 0x03030000
 diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src
 index b5e0fde..13784b3 100644
 --- a/numpy/core/src/multiarray/scalartypes.c.src
 +++ b/numpy/core/src/multiarray/scalartypes.c.src
-@@ -1673,7 +1673,7 @@ voidtype_setfield(PyVoidScalarObject *self, PyObject *args, PyObject *kwds)
-      * However, as a special case, void-scalar assignment broadcasts
-      * differently from ndarrays when assigning to an object field: Assignment
-      * to an ndarray object field broadcasts, but assignment to a void-scalar
--     * object-field should not, in order to allow nested ndarrays. 
-+     * object-field should not, in order to allow nested ndarrays.
-      * These lines should then behave identically:
-      *
-      *     b = np.zeros(1, dtype=[('x', 'O')])
-@@ -3479,7 +3479,13 @@ NPY_NO_EXPORT PyTypeObject Py@NAME@ArrType_Type = {
-     0,                                          /* ob_size */
- #endif
-     "numpy." NAME_@name@ "@ex@",                /* tp_name*/
--    sizeof(Py@NAME@ScalarObject),               /* tp_basicsize*/
-+    // Pyston change: We don't have PyStringObject defined (incomplete type)
-+    // so we can't use sizeof. As a temporary workaround, just put some
-+    // other reasonable value but we'll want to proper value at some point.
-+    // More doesn't break things, too little does (allocated memory is not
-+    // large enough so it writes past the object).
-+    // sizeof(Py@NAME@ScalarObject),               /* tp_basicsize*/
-+    sizeof(PyObjectScalarObject) * 4,               /* tp_basicsize*/
-     0,                                          /* tp_itemsize */
-     0,                                          /* tp_dealloc */
-     0,                                          /* tp_print */
-@@ -4060,6 +4066,10 @@ static void init_basetypes(void);
+@@ -4060,6 +4060,10 @@ static void init_basetypes(void);
  NPY_NO_EXPORT void
  initialize_numeric_types(void)
  {

test/integration/numpy_test.py

@@ -135,6 +135,15 @@ m = mandelbrot(complex(400),complex(400))
 print m
 """
 
+# this is just a small test which checks if numpy is able to manually create a pyston string
+string_test = """
+import numpy as np
+a = np.array([1, 2**16, 2**32], dtype=int)
+s = a.astype(str)
+print s
+assert repr(s) == "array(['1', '65536', '4294967296'], \n      dtype='|S21')"
+"""
+
 numpy_test = "import numpy as np; np.test(verbose=2)"
 
 print_progress_header("Running NumPy linear algebra test...")
@@ -143,6 +152,10 @@ subprocess.check_call([PYTHON_EXE, "-c", script], cwd=CYTHON_DIR)
 print_progress_header("Running NumPy mandelbrot test...")
 subprocess.check_call([PYTHON_EXE, "-c", mandelbrot], cwd=CYTHON_DIR)
 
+
+print_progress_header("Running NumPy str test...")
+subprocess.check_call([PYTHON_EXE, "-c", mandelbrot], cwd=CYTHON_DIR)
+
 print_progress_header("Running NumPy test suite...")
 # Currently we crash running the test suite. Uncomment for testing or
 # when all the crashes are fixed.