Add support for the marshal module

from_cpython/Include/Python.h

@@ -62,9 +62,10 @@
 #include "bufferobject.h"
 #include "bytesobject.h"
 #include "bytearrayobject.h"
+#include "tupleobject.h"
 #include "listobject.h"
 #include "dictobject.h"
-#include "tupleobject.h"
+#include "setobject.h"
 #include "methodobject.h"
 #include "moduleobject.h"
 #include "classobject.h"

from_cpython/Include/pyport.h

@@ -350,5 +350,30 @@ typedef ssize_t         Py_ssize_t;
 #endif /* !TIME_WITH_SYS_TIME */
 
 
+/* Py_ARITHMETIC_RIGHT_SHIFT
+ * C doesn't define whether a right-shift of a signed integer sign-extends
+ * or zero-fills.  Here a macro to force sign extension:
+ * Py_ARITHMETIC_RIGHT_SHIFT(TYPE, I, J)
+ *    Return I >> J, forcing sign extension.  Arithmetically, return the
+ *    floor of I/2**J.
+ * Requirements:
+ *    I should have signed integer type.  In the terminology of C99, this can
+ *    be either one of the five standard signed integer types (signed char,
+ *    short, int, long, long long) or an extended signed integer type.
+ *    J is an integer >= 0 and strictly less than the number of bits in the
+ *    type of I (because C doesn't define what happens for J outside that
+ *    range either).
+ *    TYPE used to specify the type of I, but is now ignored.  It's been left
+ *    in for backwards compatibility with versions <= 2.6 or 3.0.
+ * Caution:
+ *    I may be evaluated more than once.
+ */
+#ifdef SIGNED_RIGHT_SHIFT_ZERO_FILLS
+#define Py_ARITHMETIC_RIGHT_SHIFT(TYPE, I, J) \
+    ((I) < 0 ? -1-((-1-(I)) >> (J)) : (I) >> (J))
+#else
+#define Py_ARITHMETIC_RIGHT_SHIFT(TYPE, I, J) ((I) >> (J))
+#endif
+
 #endif /* Py_PYPORT_H */
 

from_cpython/Include/setobject.h

+// This file is originally from CPython 2.7, with modifications for Pyston
+
+/* Set object interface */
+
+#ifndef Py_SETOBJECT_H
+#define Py_SETOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Pyston change: comment this out since this is not the format we're using
+#if 0
+/*
+There are three kinds of slots in the table:
+
+1. Unused:  key == NULL
+2. Active:  key != NULL and key != dummy
+3. Dummy:   key == dummy
+
+Note: .pop() abuses the hash field of an Unused or Dummy slot to
+hold a search finger.  The hash field of Unused or Dummy slots has
+no meaning otherwise.
+*/
+
+#define PySet_MINSIZE 8
+
+typedef struct {
+    long hash;      /* cached hash code for the entry key */
+    PyObject *key;
+} setentry;
+
+
+/*
+This data structure is shared by set and frozenset objects.
+*/
+
+typedef struct _setobject PySetObject;
+struct _setobject {
+    PyObject_HEAD
+
+    Py_ssize_t fill;  /* # Active + # Dummy */
+    Py_ssize_t used;  /* # Active */
+
+    /* The table contains mask + 1 slots, and that's a power of 2.
+     * We store the mask instead of the size because the mask is more
+     * frequently needed.
+     */
+    Py_ssize_t mask;
+
+    /* table points to smalltable for small tables, else to
+     * additional malloc'ed memory.  table is never NULL!  This rule
+     * saves repeated runtime null-tests.
+     */
+    setentry *table;
+    setentry *(*lookup)(PySetObject *so, PyObject *key, long hash);
+    setentry smalltable[PySet_MINSIZE];
+
+    long hash;                  /* only used by frozenset objects */
+    PyObject *weakreflist;      /* List of weak references */
+};
+#endif
+struct _PySetObject;
+typedef struct _PySetObject PySetObject;
+
+// Pyston change: these are no longer static objects:
+#if 0
+PyAPI_DATA(PyTypeObject) PySet_Type;
+PyAPI_DATA(PyTypeObject) PyFrozenSet_Type;
+#endif
+PyAPI_DATA(PyTypeObject*) set_cls;
+#define PySet_Type (*set_cls)
+PyAPI_DATA(PyTypeObject*) frozenset_cls;
+#define PyFrozenSet_Type (*frozenset_cls)
+
+/* Invariants for frozensets:
+ *     data is immutable.
+ *     hash is the hash of the frozenset or -1 if not computed yet.
+ * Invariants for sets:
+ *     hash is -1
+ */
+
+#define PyFrozenSet_CheckExact(ob) (Py_TYPE(ob) == &PyFrozenSet_Type)
+#define PyAnySet_CheckExact(ob) \
+    (Py_TYPE(ob) == &PySet_Type || Py_TYPE(ob) == &PyFrozenSet_Type)
+#define PyAnySet_Check(ob) \
+    (Py_TYPE(ob) == &PySet_Type || Py_TYPE(ob) == &PyFrozenSet_Type || \
+      PyType_IsSubtype(Py_TYPE(ob), &PySet_Type) || \
+      PyType_IsSubtype(Py_TYPE(ob), &PyFrozenSet_Type))
+#define PySet_Check(ob) \
+    (Py_TYPE(ob) == &PySet_Type || \
+    PyType_IsSubtype(Py_TYPE(ob), &PySet_Type))
+#define   PyFrozenSet_Check(ob) \
+    (Py_TYPE(ob) == &PyFrozenSet_Type || \
+      PyType_IsSubtype(Py_TYPE(ob), &PyFrozenSet_Type))
+
+PyAPI_FUNC(PyObject *) PySet_New(PyObject *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(PyObject *) PyFrozenSet_New(PyObject *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(Py_ssize_t) PySet_Size(PyObject *anyset) PYSTON_NOEXCEPT;
+
+// Pyston change
+//#define PySet_GET_SIZE(so) (((PySetObject *)(so))->used)
+#define PySet_GET_SIZE(so) PySet_Size(so)
+
+PyAPI_FUNC(int) PySet_Clear(PyObject *set) PYSTON_NOEXCEPT;
+PyAPI_FUNC(int) PySet_Contains(PyObject *anyset, PyObject *key) PYSTON_NOEXCEPT;
+PyAPI_FUNC(int) PySet_Discard(PyObject *set, PyObject *key) PYSTON_NOEXCEPT;
+PyAPI_FUNC(int) PySet_Add(PyObject *set, PyObject *key) PYSTON_NOEXCEPT;
+PyAPI_FUNC(int) _PySet_Next(PyObject *set, Py_ssize_t *pos, PyObject **key) PYSTON_NOEXCEPT;
+PyAPI_FUNC(int) _PySet_NextEntry(PyObject *set, Py_ssize_t *pos, PyObject **key, long *hash) PYSTON_NOEXCEPT;
+PyAPI_FUNC(PyObject *) PySet_Pop(PyObject *set) PYSTON_NOEXCEPT;
+PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable) PYSTON_NOEXCEPT;
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_SETOBJECT_H */

from_cpython/Python/marshal.c

@@ -9,9 +9,9 @@
 
 #include "Python.h"
 
-// Pyston change: not yet ported
-#if 0
-#include "longintrepr.h"
+// Pyston change: not needed
+//#include "longintrepr.h"
+
 #include "code.h"
 #include "marshal.h"
 
@@ -52,7 +52,6 @@
 #define WFERR_UNMARSHALLABLE 1
 #define WFERR_NESTEDTOODEEP 2
 #define WFERR_NOMEMORY 3
-#endif // Pyston change
 
 typedef struct {
     FILE *fp;
@@ -66,8 +65,6 @@ typedef struct {
     int version;
 } WFILE;
 
-// Pyston change: not yet ported
-#if 0
 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
                       else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
                            else w_more(c, p)
@@ -155,6 +152,8 @@ w_pstring(const char *s, Py_ssize_t n, WFILE *p)
         w_string(s, n, p);
 }
 
+// Pyston change: not yet ported
+#if 0
 /* We assume that Python longs are stored internally in base some power of
    2**15; for the sake of portability we'll always read and write them in base
    exactly 2**15. */
@@ -209,6 +208,14 @@ w_PyLong(const PyLongObject *ob, WFILE *p)
         d >>= PyLong_MARSHAL_SHIFT;
     } while (d != 0);
 }
+#else
+static void
+w_PyLong(const PyLongObject *ob, WFILE *p)
+{
+    assert(0 && "not implemented");
+    abort();
+}
+#endif
 
 static void
 w_object(PyObject *v, WFILE *p)
@@ -327,6 +334,8 @@ w_object(PyObject *v, WFILE *p)
     }
 #endif
     else if (PyString_CheckExact(v)) {
+        // Pyston change: I think we don't need this
+        /*
         if (p->strings && PyString_CHECK_INTERNED(v)) {
             PyObject *o = PyDict_GetItem(p->strings, v);
             if (o) {
@@ -352,6 +361,8 @@ w_object(PyObject *v, WFILE *p)
         else {
             w_byte(TYPE_STRING, p);
         }
+        */
+        w_byte(TYPE_STRING, p);
         w_pstring(PyBytes_AS_STRING(v), PyString_GET_SIZE(v), p);
     }
 #ifdef Py_USING_UNICODE
@@ -428,6 +439,10 @@ w_object(PyObject *v, WFILE *p)
         }
     }
     else if (PyCode_Check(v)) {
+        // Pyston change: not implemented
+        assert(0 && "not implemented");
+        abort();
+        /*
         PyCodeObject *co = (PyCodeObject *)v;
         w_byte(TYPE_CODE, p);
         w_long(co->co_argcount, p);
@@ -444,6 +459,7 @@ w_object(PyObject *v, WFILE *p)
         w_object(co->co_name, p);
         w_long(co->co_firstlineno, p);
         w_object(co->co_lnotab, p);
+        */
     }
     else if (PyObject_CheckReadBuffer(v)) {
         /* Write unknown buffer-style objects as a string */
@@ -486,7 +502,6 @@ PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
     w_object(x, &wf);
     Py_XDECREF(wf.strings);
 }
-#endif // Pyston change
 
 typedef WFILE RFILE; /* Same struct with different invariants */
 
@@ -633,6 +648,14 @@ r_PyLong(RFILE *p)
                     "bad marshal data (digit out of range in long)");
     return NULL;
 }
+#else
+static PyObject *
+r_PyLong(RFILE *p)
+{
+    assert(0 && "not implemented");
+    abort();
+}
+#endif
 
 static PyObject *
 r_object(RFILE *p)
@@ -1096,7 +1119,6 @@ read_object(RFILE *p)
         PyErr_SetString(PyExc_TypeError, "NULL object in marshal data for object");
     return v;
 }
-#endif // Pyston change
 
 int
 PyMarshal_ReadShortFromFile(FILE *fp)
@@ -1132,8 +1154,6 @@ getfilesize(FILE *fp)
 }
 #endif
 
-// Pyston change: not yet ported
-#if 0
 /* If we can get the size of the file up-front, and it's reasonably small,
  * read it in one gulp and delegate to ...FromString() instead.  Much quicker
  * than reading a byte at a time from file; speeds .pyc imports.
@@ -1421,4 +1441,3 @@ PyMarshal_Init(void)
         return;
     PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION);
 }
-#endif // Pyston change

src/capi/abstract.cpp

@@ -500,6 +500,15 @@ extern "C" int PyObject_AsCharBuffer(PyObject* obj, const char** buffer, Py_ssiz
     return 0;
 }
 
+extern "C" int PyObject_CheckReadBuffer(PyObject* obj) noexcept {
+    PyBufferProcs* pb = obj->cls->tp_as_buffer;
+
+    if (pb == NULL || pb->bf_getreadbuffer == NULL || pb->bf_getsegcount == NULL
+        || (*pb->bf_getsegcount)(obj, NULL) != 1)
+        return 0;
+    return 1;
+}
+
 extern "C" int PyObject_AsReadBuffer(PyObject* obj, const void** buffer, Py_ssize_t* buffer_len) noexcept {
     fatalOrError(PyExc_NotImplementedError, "unimplemented");
     return -1;

src/runtime/capi.cpp

@@ -531,15 +531,22 @@ extern "C" PyObject* PySequence_Tuple(PyObject* o) noexcept {
 }
 
 extern "C" PyObject* PyIter_Next(PyObject* iter) noexcept {
-    static const std::string next_str("next");
     try {
-        return callattr(iter, &next_str, CallattrFlags({.cls_only = true, .null_on_nonexistent = false }),
-                        ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
+        Box* hasnext = iter->hasnextOrNullIC();
+        if (hasnext) {
+            if (hasnext->nonzeroIC())
+                return iter->nextIC();
+            else
+                return NULL;
+        } else {
+            return iter->nextIC();
+        }
     } catch (ExcInfo e) {
-        if (!e.matches(StopIteration))
-            setCAPIException(e);
-        return NULL;
+        if (e.matches(StopIteration))
+            return NULL;
+        setCAPIException(e);
     }
+    return NULL;
 }
 
 extern "C" int PyCallable_Check(PyObject* x) noexcept {

src/runtime/code.cpp

@@ -24,7 +24,9 @@
 
 namespace pyston {
 
+extern "C" {
 BoxedClass* code_cls;
+}
 
 class BoxedCode : public Box {
 public:
@@ -94,6 +96,11 @@ Box* codeForCLFunction(CLFunction* f) {
     return new BoxedCode(f);
 }
 
+extern "C" PyCodeObject* PyCode_New(int, int, int, int, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*,
+                                    PyObject*, PyObject*, PyObject*, int, PyObject*) noexcept {
+    RELEASE_ASSERT(0, "not implemented");
+}
+
 void setupCode() {
     code_cls
         = BoxedHeapClass::create(type_cls, object_cls, &BoxedCode::gcHandler, 0, 0, sizeof(BoxedCode), false, "code");

src/runtime/complex.cpp

@@ -50,6 +50,11 @@ extern "C" double PyComplex_ImagAsDouble(PyObject* op) noexcept {
     }
 }
 
+extern "C" PyObject* PyComplex_FromCComplex(Py_complex val) noexcept {
+    return new BoxedComplex(val.real, val.imag);
+}
+
+
 // addition
 
 extern "C" Box* complexAddComplex(BoxedComplex* lhs, BoxedComplex* rhs) {

src/runtime/set.cpp

@@ -17,12 +17,12 @@
 #include <llvm/Support/raw_ostream.h>
 
 #include "gc/collector.h"
+#include "runtime/capi.h"
 #include "runtime/objmodel.h"
 
 namespace pyston {
 
-BoxedClass* set_cls, *set_iterator_cls;
-BoxedClass* frozenset_cls;
+BoxedClass* set_iterator_cls;
 
 extern "C" Box* createSet() {
     return new BoxedSet();
@@ -196,7 +196,7 @@ Box* setLen(BoxedSet* self) {
 }
 
 Box* setAdd(BoxedSet* self, Box* v) {
-    assert(self->cls == set_cls);
+    assert(self->cls == set_cls || self->cls == frozenset_cls);
     self->s.insert(v);
     return None;
 }
@@ -391,6 +391,42 @@ Box* setHash(BoxedSet* self) {
     return boxInt(rtn);
 }
 
+extern "C" PyObject* PySet_New(PyObject* iterable) noexcept {
+    if (!iterable)
+        return new BoxedSet(); // Fast path for empty set.
+
+    try {
+        return runtimeCall(set_cls, ArgPassSpec(iterable ? 1 : 0), iterable, NULL, NULL, NULL, NULL);
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return NULL;
+    }
+}
+
+extern "C" PyObject* PyFrozenSet_New(PyObject* iterable) noexcept {
+    try {
+        return runtimeCall(frozenset_cls, ArgPassSpec(iterable ? 1 : 0), iterable, NULL, NULL, NULL, NULL);
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return NULL;
+    }
+}
+
+extern "C" int PySet_Add(PyObject* set, PyObject* key) noexcept {
+    if (!PySet_Check(set) && !PyFrozenSet_Check(set)) {
+        PyErr_BadInternalCall();
+        return -1;
+    }
+
+    try {
+        setAdd((BoxedSet*)set, key);
+        return 0;
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return -1;
+    }
+}
+
 } // namespace set
 
 using namespace pyston::set;

src/runtime/set.h

@@ -25,8 +25,6 @@ namespace pyston {
 void setupSet();
 void teardownSet();
 
-extern BoxedClass* set_cls, *frozenset_cls;
-
 extern "C" Box* createSet();
 
 class BoxedSet : public Box {

src/runtime/str.cpp

@@ -343,7 +343,6 @@ extern "C" PyObject* PyString_InternFromString(const char* s) noexcept {
 }
 
 extern "C" void PyString_InternInPlace(PyObject**) noexcept {
-    Py_FatalError("unimplemented");
 }
 
 /* Format codes

src/runtime/types.cpp

@@ -77,6 +77,7 @@ extern "C" void initzipimport();
 extern "C" void init_csv();
 extern "C" void init_ssl();
 extern "C" void init_sqlite3();
+extern "C" void PyMarshal_Init();
 
 namespace pyston {
 
@@ -655,7 +656,7 @@ BoxedClass* object_cls, *type_cls, *none_cls, *bool_cls, *int_cls, *float_cls,
     * str_cls = NULL, *function_cls, *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls,
       *file_cls, *member_cls, *closure_cls, *generator_cls, *complex_cls, *basestring_cls, *property_cls,
       *staticmethod_cls, *classmethod_cls, *attrwrapper_cls, *pyston_getset_cls, *capi_getset_cls,
-      *builtin_function_or_method_cls, *attrwrapperiter_cls;
+      *builtin_function_or_method_cls, *attrwrapperiter_cls, *set_cls, *frozenset_cls;
 
 BoxedTuple* EmptyTuple;
 BoxedString* EmptyString;
@@ -2270,6 +2271,7 @@ void setupRuntime() {
     init_csv();
     init_ssl();
     init_sqlite3();
+    PyMarshal_Init();
 
     // some additional setup to ensure weakrefs participate in our GC
     BoxedClass* weakref_ref_cls = &_PyWeakref_RefType;

src/runtime/types.h

@@ -86,7 +86,7 @@ extern BoxedClass* object_cls, *type_cls, *bool_cls, *int_cls, *long_cls, *float
     *none_cls, *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls, *file_cls,
     *enumerate_cls, *xrange_cls, *member_cls, *method_cls, *closure_cls, *generator_cls, *complex_cls, *basestring_cls,
     *property_cls, *staticmethod_cls, *classmethod_cls, *attrwrapper_cls, *pyston_getset_cls, *capi_getset_cls,
-    *builtin_function_or_method_cls;
+    *builtin_function_or_method_cls, *set_cls, *frozenset_cls;
 }
 #define unicode_cls (&PyUnicode_Type)
 #define memoryview_cls (&PyMemoryView_Type)

test/tests/marshal_test.py

+import marshal
+o = [-1, 1.23456789, complex(1.2, 3.4)]
+o += [True, False, None]
+o += ["Hello World!", u"Hello World!"]
+o += [{ "Key" : "Value" }, set(["Set"]), frozenset(["FrozenSet"]), (1, 2, 3), [1, 2, 3]]
+for i in o:
+    s = marshal.dumps(i)
+    r = marshal.loads(s)
+    print "Dumping:", i, "Loaded", r