Merge pull request #446 from undingen/builtin_func2

Implement misc stuff

from_cpython/Include/Python.h

@@ -62,9 +62,10 @@
 #include "bufferobject.h"
 #include "bytesobject.h"
 #include "bytearrayobject.h"
+#include "tupleobject.h"
 #include "listobject.h"
 #include "dictobject.h"
-#include "tupleobject.h"
+#include "setobject.h"
 #include "methodobject.h"
 #include "moduleobject.h"
 #include "classobject.h"

from_cpython/Include/pyconfig.h

@@ -94,6 +94,7 @@
 #define HAVE_POLL_H 1
 #define HAVE_PTHREAD_H 1
 #define HAVE_PTY_H 1
+#define HAVE_SETVBUF 1
 #define HAVE_SHADOW_H 1
 #define HAVE_SIGNAL_H 1
 #define HAVE_SPAWN_H 1

from_cpython/Include/pyport.h

@@ -320,6 +320,31 @@ typedef ssize_t         Py_ssize_t;
 #endif
 #endif
 
+/* uintptr_t is the C9X name for an unsigned integral type such that a
+ * legitimate void* can be cast to uintptr_t and then back to void* again
+ * without loss of information.  Similarly for intptr_t, wrt a signed
+ * integral type.
+ */
+#ifdef HAVE_UINTPTR_T
+typedef uintptr_t       Py_uintptr_t;
+typedef intptr_t        Py_intptr_t;
+
+#elif SIZEOF_VOID_P <= SIZEOF_INT
+typedef unsigned int    Py_uintptr_t;
+typedef int             Py_intptr_t;
+
+#elif SIZEOF_VOID_P <= SIZEOF_LONG
+typedef unsigned long   Py_uintptr_t;
+typedef long            Py_intptr_t;
+
+#elif defined(HAVE_LONG_LONG) && (SIZEOF_VOID_P <= SIZEOF_LONG_LONG)
+typedef unsigned PY_LONG_LONG   Py_uintptr_t;
+typedef PY_LONG_LONG            Py_intptr_t;
+
+#else
+#   error "Python needs a typedef for Py_uintptr_t in pyport.h."
+#endif /* HAVE_UINTPTR_T */
+
 #if defined(_MSC_VER)
 #define Py_MEMCPY(target, source, length) do {                          \
         size_t i_, n_ = (length);                                       \
@@ -350,5 +375,30 @@ typedef ssize_t         Py_ssize_t;
 #endif /* !TIME_WITH_SYS_TIME */
 
 
+/* Py_ARITHMETIC_RIGHT_SHIFT
+ * C doesn't define whether a right-shift of a signed integer sign-extends
+ * or zero-fills.  Here a macro to force sign extension:
+ * Py_ARITHMETIC_RIGHT_SHIFT(TYPE, I, J)
+ *    Return I >> J, forcing sign extension.  Arithmetically, return the
+ *    floor of I/2**J.
+ * Requirements:
+ *    I should have signed integer type.  In the terminology of C99, this can
+ *    be either one of the five standard signed integer types (signed char,
+ *    short, int, long, long long) or an extended signed integer type.
+ *    J is an integer >= 0 and strictly less than the number of bits in the
+ *    type of I (because C doesn't define what happens for J outside that
+ *    range either).
+ *    TYPE used to specify the type of I, but is now ignored.  It's been left
+ *    in for backwards compatibility with versions <= 2.6 or 3.0.
+ * Caution:
+ *    I may be evaluated more than once.
+ */
+#ifdef SIGNED_RIGHT_SHIFT_ZERO_FILLS
+#define Py_ARITHMETIC_RIGHT_SHIFT(TYPE, I, J) \
+    ((I) < 0 ? -1-((-1-(I)) >> (J)) : (I) >> (J))
+#else
+#define Py_ARITHMETIC_RIGHT_SHIFT(TYPE, I, J) ((I) >> (J))
+#endif
+
 #endif /* Py_PYPORT_H */
 

from_cpython/Include/setobject.h

+// This file is originally from CPython 2.7, with modifications for Pyston
+
+/* Set object interface */
+
+#ifndef Py_SETOBJECT_H
+#define Py_SETOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Pyston change: comment this out since this is not the format we're using
+#if 0
+/*
+There are three kinds of slots in the table:
+
+1. Unused:  key == NULL
+2. Active:  key != NULL and key != dummy
+3. Dummy:   key == dummy
+
+Note: .pop() abuses the hash field of an Unused or Dummy slot to
+hold a search finger.  The hash field of Unused or Dummy slots has
+no meaning otherwise.
+*/
+
+#define PySet_MINSIZE 8
+
+typedef struct {
+    long hash;      /* cached hash code for the entry key */
+    PyObject *key;
+} setentry;
+
+
+/*
+This data structure is shared by set and frozenset objects.
+*/
+
+typedef struct _setobject PySetObject;
+struct _setobject {
+    PyObject_HEAD
+
+    Py_ssize_t fill;  /* # Active + # Dummy */
+    Py_ssize_t used;  /* # Active */
+
+    /* The table contains mask + 1 slots, and that's a power of 2.
+     * We store the mask instead of the size because the mask is more
+     * frequently needed.
+     */
+    Py_ssize_t mask;
+
+    /* table points to smalltable for small tables, else to
+     * additional malloc'ed memory.  table is never NULL!  This rule
+     * saves repeated runtime null-tests.
+     */
+    setentry *table;
+    setentry *(*lookup)(PySetObject *so, PyObject *key, long hash);
+    setentry smalltable[PySet_MINSIZE];
+
+    long hash;                  /* only used by frozenset objects */
+    PyObject *weakreflist;      /* List of weak references */
+};
+#endif
+struct _PySetObject;
+typedef struct _PySetObject PySetObject;
+
+// Pyston change: these are no longer static objects:
+#if 0
+PyAPI_DATA(PyTypeObject) PySet_Type;
+PyAPI_DATA(PyTypeObject) PyFrozenSet_Type;
+#endif
+PyAPI_DATA(PyTypeObject*) set_cls;
+#define PySet_Type (*set_cls)
+PyAPI_DATA(PyTypeObject*) frozenset_cls;
+#define PyFrozenSet_Type (*frozenset_cls)
+
+/* Invariants for frozensets:
+ *     data is immutable.
+ *     hash is the hash of the frozenset or -1 if not computed yet.
+ * Invariants for sets:
+ *     hash is -1
+ */
+
+#define PyFrozenSet_CheckExact(ob) (Py_TYPE(ob) == &PyFrozenSet_Type)
+#define PyAnySet_CheckExact(ob) \
+    (Py_TYPE(ob) == &PySet_Type || Py_TYPE(ob) == &PyFrozenSet_Type)
+#define PyAnySet_Check(ob) \
+    (Py_TYPE(ob) == &PySet_Type || Py_TYPE(ob) == &PyFrozenSet_Type || \
+      PyType_IsSubtype(Py_TYPE(ob), &PySet_Type) || \
+      PyType_IsSubtype(Py_TYPE(ob), &PyFrozenSet_Type))
+#define PySet_Check(ob) \
+    (Py_TYPE(ob) == &PySet_Type || \
+    PyType_IsSubtype(Py_TYPE(ob), &PySet_Type))
+#define   PyFrozenSet_Check(ob) \
+    (Py_TYPE(ob) == &PyFrozenSet_Type || \
+      PyType_IsSubtype(Py_TYPE(ob), &PyFrozenSet_Type))
+
+PyAPI_FUNC(PyObject *) PySet_New(PyObject *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(PyObject *) PyFrozenSet_New(PyObject *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(Py_ssize_t) PySet_Size(PyObject *anyset) PYSTON_NOEXCEPT;
+
+// Pyston change
+//#define PySet_GET_SIZE(so) (((PySetObject *)(so))->used)
+#define PySet_GET_SIZE(so) PySet_Size(so)
+
+PyAPI_FUNC(int) PySet_Clear(PyObject *set) PYSTON_NOEXCEPT;
+PyAPI_FUNC(int) PySet_Contains(PyObject *anyset, PyObject *key) PYSTON_NOEXCEPT;
+PyAPI_FUNC(int) PySet_Discard(PyObject *set, PyObject *key) PYSTON_NOEXCEPT;
+PyAPI_FUNC(int) PySet_Add(PyObject *set, PyObject *key) PYSTON_NOEXCEPT;
+PyAPI_FUNC(int) _PySet_Next(PyObject *set, Py_ssize_t *pos, PyObject **key) PYSTON_NOEXCEPT;
+PyAPI_FUNC(int) _PySet_NextEntry(PyObject *set, Py_ssize_t *pos, PyObject **key, long *hash) PYSTON_NOEXCEPT;
+PyAPI_FUNC(PyObject *) PySet_Pop(PyObject *set) PYSTON_NOEXCEPT;
+PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable) PYSTON_NOEXCEPT;
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_SETOBJECT_H */

from_cpython/Python/marshal.c

@@ -9,9 +9,9 @@
 
 #include "Python.h"
 
-// Pyston change: not yet ported
-#if 0
-#include "longintrepr.h"
+// Pyston change: not needed
+//#include "longintrepr.h"
+
 #include "code.h"
 #include "marshal.h"
 
@@ -52,7 +52,6 @@
 #define WFERR_UNMARSHALLABLE 1
 #define WFERR_NESTEDTOODEEP 2
 #define WFERR_NOMEMORY 3
-#endif // Pyston change
 
 typedef struct {
     FILE *fp;
@@ -66,8 +65,6 @@ typedef struct {
     int version;
 } WFILE;
 
-// Pyston change: not yet ported
-#if 0
 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
                       else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
                            else w_more(c, p)
@@ -155,6 +152,8 @@ w_pstring(const char *s, Py_ssize_t n, WFILE *p)
         w_string(s, n, p);
 }
 
+// Pyston change: not yet ported
+#if 0
 /* We assume that Python longs are stored internally in base some power of
    2**15; for the sake of portability we'll always read and write them in base
    exactly 2**15. */
@@ -209,6 +208,14 @@ w_PyLong(const PyLongObject *ob, WFILE *p)
         d >>= PyLong_MARSHAL_SHIFT;
     } while (d != 0);
 }
+#else
+static void
+w_PyLong(const PyLongObject *ob, WFILE *p)
+{
+    assert(0 && "not implemented");
+    abort();
+}
+#endif
 
 static void
 w_object(PyObject *v, WFILE *p)
@@ -327,6 +334,8 @@ w_object(PyObject *v, WFILE *p)
     }
 #endif
     else if (PyString_CheckExact(v)) {
+        // Pyston change: I think we don't need this
+        /*
         if (p->strings && PyString_CHECK_INTERNED(v)) {
             PyObject *o = PyDict_GetItem(p->strings, v);
             if (o) {
@@ -352,6 +361,8 @@ w_object(PyObject *v, WFILE *p)
         else {
             w_byte(TYPE_STRING, p);
         }
+        */
+        w_byte(TYPE_STRING, p);
         w_pstring(PyBytes_AS_STRING(v), PyString_GET_SIZE(v), p);
     }
 #ifdef Py_USING_UNICODE
@@ -428,6 +439,10 @@ w_object(PyObject *v, WFILE *p)
         }
     }
     else if (PyCode_Check(v)) {
+        // Pyston change: not implemented
+        assert(0 && "not implemented");
+        abort();
+        /*
         PyCodeObject *co = (PyCodeObject *)v;
         w_byte(TYPE_CODE, p);
         w_long(co->co_argcount, p);
@@ -444,6 +459,7 @@ w_object(PyObject *v, WFILE *p)
         w_object(co->co_name, p);
         w_long(co->co_firstlineno, p);
         w_object(co->co_lnotab, p);
+        */
     }
     else if (PyObject_CheckReadBuffer(v)) {
         /* Write unknown buffer-style objects as a string */
@@ -486,7 +502,6 @@ PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
     w_object(x, &wf);
     Py_XDECREF(wf.strings);
 }
-#endif // Pyston change
 
 typedef WFILE RFILE; /* Same struct with different invariants */
 
@@ -633,6 +648,14 @@ r_PyLong(RFILE *p)
                     "bad marshal data (digit out of range in long)");
     return NULL;
 }
+#else
+static PyObject *
+r_PyLong(RFILE *p)
+{
+    assert(0 && "not implemented");
+    abort();
+}
+#endif
 
 static PyObject *
 r_object(RFILE *p)
@@ -1096,7 +1119,6 @@ read_object(RFILE *p)
         PyErr_SetString(PyExc_TypeError, "NULL object in marshal data for object");
     return v;
 }
-#endif // Pyston change
 
 int
 PyMarshal_ReadShortFromFile(FILE *fp)
@@ -1132,8 +1154,6 @@ getfilesize(FILE *fp)
 }
 #endif
 
-// Pyston change: not yet ported
-#if 0
 /* If we can get the size of the file up-front, and it's reasonably small,
  * read it in one gulp and delegate to ...FromString() instead.  Much quicker
  * than reading a byte at a time from file; speeds .pyc imports.
@@ -1421,4 +1441,3 @@ PyMarshal_Init(void)
         return;
     PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION);
 }
-#endif // Pyston change

src/analysis/scoping_analysis.cpp

@@ -473,6 +473,7 @@ public:
     bool visit_dict(AST_Dict* node) override { return false; }
     bool visit_excepthandler(AST_ExceptHandler* node) override { return false; }
     bool visit_expr(AST_Expr* node) override { return false; }
+    bool visit_extslice(AST_ExtSlice* node) override { return false; }
     bool visit_for(AST_For* node) override { return false; }
     // bool visit_functiondef(AST_FunctionDef *node) override { return false; }
     // bool visit_global(AST_Global *node) override { return false; }

src/analysis/type_analysis.cpp

@@ -453,6 +453,14 @@ private:
 
     void* visit_slice(AST_Slice* node) override { return SLICE; }
 
+    void* visit_extslice(AST_ExtSlice* node) override {
+        std::vector<CompilerType*> elt_types;
+        for (auto* e : node->dims) {
+            elt_types.push_back(getType(e));
+        }
+        return makeTupleType(elt_types);
+    }
+
     void* visit_str(AST_Str* node) override {
         if (node->str_type == AST_Str::STR)
             return STR;

src/capi/abstract.cpp

@@ -500,6 +500,15 @@ extern "C" int PyObject_AsCharBuffer(PyObject* obj, const char** buffer, Py_ssiz
     return 0;
 }
 
+extern "C" int PyObject_CheckReadBuffer(PyObject* obj) noexcept {
+    PyBufferProcs* pb = obj->cls->tp_as_buffer;
+
+    if (pb == NULL || pb->bf_getreadbuffer == NULL || pb->bf_getsegcount == NULL
+        || (*pb->bf_getsegcount)(obj, NULL) != 1)
+        return 0;
+    return 1;
+}
+
 extern "C" int PyObject_AsReadBuffer(PyObject* obj, const void** buffer, Py_ssize_t* buffer_len) noexcept {
     fatalOrError(PyExc_NotImplementedError, "unimplemented");
     return -1;

src/codegen/ast_interpreter.cpp

@@ -104,6 +104,7 @@ private:
     Value visit_dict(AST_Dict* node);
     Value visit_expr(AST_expr* node);
     Value visit_expr(AST_Expr* node);
+    Value visit_extslice(AST_ExtSlice* node);
     Value visit_index(AST_Index* node);
     Value visit_lambda(AST_Lambda* node);
     Value visit_list(AST_List* node);
@@ -430,6 +431,14 @@ Value ASTInterpreter::visit_slice(AST_Slice* node) {
     return createSlice(lower.o, upper.o, step.o);
 }
 
+Value ASTInterpreter::visit_extslice(AST_ExtSlice* node) {
+    int num_slices = node->dims.size();
+    BoxedTuple* rtn = BoxedTuple::create(num_slices);
+    for (int i = 0; i < num_slices; ++i)
+        rtn->elts[i] = visit_expr(node->dims[i]).o;
+    return rtn;
+}
+
 Value ASTInterpreter::visit_branch(AST_Branch* node) {
     Value v = visit_expr(node->test);
     ASSERT(v.o == True || v.o == False, "Should have called NONZERO before this branch");
@@ -942,6 +951,8 @@ Value ASTInterpreter::visit_expr(AST_expr* node) {
             return visit_compare((AST_Compare*)node);
         case AST_TYPE::Dict:
             return visit_dict((AST_Dict*)node);
+        case AST_TYPE::ExtSlice:
+            return visit_extslice((AST_ExtSlice*)node);
         case AST_TYPE::Index:
             return visit_index((AST_Index*)node);
         case AST_TYPE::Lambda:

src/codegen/irgen/irgenerator.cpp

@@ -1092,6 +1092,20 @@ private:
         return new ConcreteCompilerVariable(SLICE, rtn, true);
     }
 
+    CompilerVariable* evalExtSlice(AST_ExtSlice* node, UnwindInfo unw_info) {
+        std::vector<CompilerVariable*> elts;
+        for (auto* e : node->dims) {
+            elts.push_back(evalExpr(e, unw_info));
+        }
+
+        // TODO makeTuple should probably just transfer the vref, but I want to keep things consistent
+        CompilerVariable* rtn = makeTuple(elts);
+        for (auto* e : elts) {
+            e->decvref(emitter);
+        }
+        return rtn;
+    }
+
     CompilerVariable* evalStr(AST_Str* node, UnwindInfo unw_info) {
         if (node->str_type == AST_Str::STR) {
             llvm::Value* rtn = embedConstantPtr(
@@ -1350,6 +1364,9 @@ private:
             case AST_TYPE::Dict:
                 rtn = evalDict(ast_cast<AST_Dict>(node), unw_info);
                 break;
+            case AST_TYPE::ExtSlice:
+                rtn = evalExtSlice(ast_cast<AST_ExtSlice>(node), unw_info);
+                break;
             case AST_TYPE::Index:
                 rtn = evalIndex(ast_cast<AST_Index>(node), unw_info);
                 break;

src/codegen/parser.cpp

@@ -403,6 +403,15 @@ AST_Expr* read_expr(BufferedReader* reader) {
     return rtn;
 }
 
+AST_ExtSlice* read_extslice(BufferedReader* reader) {
+    AST_ExtSlice* rtn = new AST_ExtSlice();
+
+    rtn->col_offset = -1;
+    rtn->lineno = -1;
+    readExprVector(rtn->dims, reader);
+    return rtn;
+}
+
 AST_For* read_for(BufferedReader* reader) {
     AST_For* rtn = new AST_For();
 
@@ -798,6 +807,8 @@ AST_expr* readASTExpr(BufferedReader* reader) {
             return read_dict(reader);
         case AST_TYPE::DictComp:
             return read_dictcomp(reader);
+        case AST_TYPE::ExtSlice:
+            return read_extslice(reader);
         case AST_TYPE::GeneratorExp:
             return read_generatorexp(reader);
         case AST_TYPE::IfExp:

src/codegen/serialize_ast.cpp

@@ -285,6 +285,10 @@ private:
         writeExpr(node->value);
         return true;
     }
+    virtual bool visit_extslice(AST_ExtSlice* node) {
+        writeExprVector(node->dims);
+        return true;
+    }
     virtual bool visit_for(AST_For* node) {
         writeStmtVector(node->body);
         writeColOffset(node->col_offset);

src/core/ast.cpp

@@ -1997,6 +1997,10 @@ public:
         output->push_back(node);
         return false;
     }
+    virtual bool visit_extslice(AST_ExtSlice* node) {
+        output->push_back(node);
+        return false;
+    }
     virtual bool visit_for(AST_For* node) {
         output->push_back(node);
         return !expand_scopes;

src/core/cfg.cpp

@@ -863,6 +863,16 @@ private:
         return rtn;
     }
 
+    AST_expr* remapExtSlice(AST_ExtSlice* node) {
+        AST_ExtSlice* rtn = new AST_ExtSlice();
+        rtn->lineno = node->lineno;
+        rtn->col_offset = node->col_offset;
+
+        for (auto* e : node->dims)
+            rtn->dims.push_back(remapExpr(e));
+        return rtn;
+    }
+
     // This is a helper function used for generators expressions and comprehensions.
     //
     // Generates a FunctionDef which produces scope for `node'. The function produced is empty, so you'd better fill it.
@@ -1174,6 +1184,9 @@ private:
             case AST_TYPE::DictComp:
                 rtn = remapScopedComprehension<AST_Dict>(ast_cast<AST_DictComp>(node));
                 break;
+            case AST_TYPE::ExtSlice:
+                rtn = remapExtSlice(ast_cast<AST_ExtSlice>(node));
+                break;
             case AST_TYPE::GeneratorExp:
                 rtn = remapGeneratorExp(ast_cast<AST_GeneratorExp>(node));
                 break;

src/runtime/builtin_modules/builtins.cpp

@@ -235,10 +235,11 @@ extern "C" Box* id(Box* arg) {
 }
 
 
-Box* open(Box* arg1, Box* arg2) {
+Box* open(Box* arg1, Box* arg2, Box* arg3) {
     assert(arg2);
+    assert(arg3);
     // This could be optimized quite a bit if it ends up being important:
-    return runtimeCall(file_cls, ArgPassSpec(2), arg1, arg2, NULL, NULL, NULL);
+    return runtimeCall(file_cls, ArgPassSpec(3), arg1, arg2, arg3, NULL, NULL);
 }
 
 extern "C" Box* chr(Box* arg) {
@@ -972,6 +973,18 @@ Box* builtinCmp(Box* lhs, Box* rhs) {
     throwCAPIException();
 }
 
+Box* builtinApply(Box* func, Box* args, Box* keywords) {
+    if (!PyTuple_Check(args)) {
+        if (!PySequence_Check(args))
+            raiseExcHelper(TypeError, "apply() arg 2 expected sequence, found %s", getTypeName(args));
+        args = PySequence_Tuple(args);
+        checkAndThrowCAPIException();
+    }
+    if (keywords && !PyDict_Check(keywords))
+        raiseExcHelper(TypeError, "apply() arg 3 expected dictionary, found %s", getTypeName(keywords));
+    return runtimeCall(func, ArgPassSpec(0, 0, true, keywords != NULL), args, keywords, NULL, NULL, NULL);
+}
+
 void setupBuiltins() {
     builtins_module = createModule("__builtin__", "__builtin__",
                                    "Built-in functions, exceptions, and other objects.\n\nNoteworthy: None is "
@@ -1004,6 +1017,10 @@ void setupBuiltins() {
     builtins_module->giveAttr("all", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)all, BOXED_BOOL, 1), "all"));
     builtins_module->giveAttr("any", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)any, BOXED_BOOL, 1), "any"));
 
+    builtins_module->giveAttr(
+        "apply", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)builtinApply, UNKNOWN, 3, 1, false, false),
+                                                  "apply", { NULL }));
+
     repr_obj = new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)repr, UNKNOWN, 1), "repr");
     builtins_module->giveAttr("repr", repr_obj);
 
@@ -1108,9 +1125,9 @@ void setupBuiltins() {
     setupXrange();
     builtins_module->giveAttr("xrange", xrange_cls);
 
-    open_obj = new BoxedBuiltinFunctionOrMethod(
-        boxRTFunction((void*)open, typeFromClass(file_cls), 2, 1, false, false, ParamNames({ "name", "mode" }, "", "")),
-        "open", { boxStrConstant("r") });
+    open_obj = new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)open, typeFromClass(file_cls), 3, 2, false, false,
+                                                              ParamNames({ "name", "mode", "buffering" }, "", "")),
+                                                "open", { boxStrConstant("r"), boxInt(-1) });
     builtins_module->giveAttr("open", open_obj);
 
     builtins_module->giveAttr("globals", new BoxedBuiltinFunctionOrMethod(

src/runtime/capi.cpp

@@ -531,15 +531,22 @@ extern "C" PyObject* PySequence_Tuple(PyObject* o) noexcept {
 }
 
 extern "C" PyObject* PyIter_Next(PyObject* iter) noexcept {
-    static const std::string next_str("next");
     try {
-        return callattr(iter, &next_str, CallattrFlags({.cls_only = true, .null_on_nonexistent = false }),
-                        ArgPassSpec(0), NULL, NULL, NULL, NULL, NULL);
+        Box* hasnext = iter->hasnextOrNullIC();
+        if (hasnext) {
+            if (hasnext->nonzeroIC())
+                return iter->nextIC();
+            else
+                return NULL;
+        } else {
+            return iter->nextIC();
+        }
     } catch (ExcInfo e) {
-        if (!e.matches(StopIteration))
-            setCAPIException(e);
+        if (e.matches(StopIteration))
             return NULL;
+        setCAPIException(e);
     }
+    return NULL;
 }
 
 extern "C" int PyCallable_Check(PyObject* x) noexcept {

src/runtime/code.cpp

@@ -24,7 +24,9 @@
 
 namespace pyston {
 
+extern "C" {
 BoxedClass* code_cls;
+}
 
 class BoxedCode : public Box {
 public:
@@ -94,6 +96,11 @@ Box* codeForCLFunction(CLFunction* f) {
     return new BoxedCode(f);
 }
 
+extern "C" PyCodeObject* PyCode_New(int, int, int, int, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*,
+                                    PyObject*, PyObject*, PyObject*, int, PyObject*) noexcept {
+    RELEASE_ASSERT(0, "not implemented");
+}
+
 void setupCode() {
     code_cls
         = BoxedHeapClass::create(type_cls, object_cls, &BoxedCode::gcHandler, 0, 0, sizeof(BoxedCode), false, "code");

src/runtime/complex.cpp

@@ -50,6 +50,11 @@ extern "C" double PyComplex_ImagAsDouble(PyObject* op) noexcept {
     }
 }
 
+extern "C" PyObject* PyComplex_FromCComplex(Py_complex val) noexcept {
+    return new BoxedComplex(val.real, val.imag);
+}
+
+
 // addition
 
 extern "C" Box* complexAddComplex(BoxedComplex* lhs, BoxedComplex* rhs) {

src/runtime/dict.cpp

@@ -96,20 +96,32 @@ Box* dictKeys(BoxedDict* self) {
     return rtn;
 }
 
-extern "C" PyObject* PyDict_Keys(PyObject* mp) noexcept {
+static PyObject* dict_helper(PyObject* mp, std::function<Box*(BoxedDict*)> f) noexcept {
     if (mp == NULL || !PyDict_Check(mp)) {
         PyErr_BadInternalCall();
         return NULL;
     }
 
     try {
-        return dictKeys(static_cast<BoxedDict*>(mp));
+        return f(static_cast<BoxedDict*>(mp));
     } catch (ExcInfo e) {
         setCAPIException(e);
         return NULL;
     }
 }
 
+extern "C" PyObject* PyDict_Keys(PyObject* mp) noexcept {
+    return dict_helper(mp, dictKeys);
+}
+
+extern "C" PyObject* PyDict_Values(PyObject* mp) noexcept {
+    return dict_helper(mp, dictValues);
+}
+
+extern "C" PyObject* PyDict_Items(PyObject* mp) noexcept {
+    return dict_helper(mp, dictItems);
+}
+
 Box* dictViewKeys(BoxedDict* self) {
     if (!isSubclass(self->cls, dict_cls)) {
         raiseExcHelper(TypeError, "descriptor 'viewkeys' requires a 'dict' object but received a '%s'",

src/runtime/file.cpp

@@ -805,7 +805,9 @@ Box* fileExit(BoxedFile* self, Box* exc_type, Box* exc_val, Box** args) {
 }
 
 // This differs very significantly from CPython:
-Box* fileNew(BoxedClass* cls, Box* s, Box* m) {
+Box* fileNew(BoxedClass* cls, Box* s, Box* m, Box** args) {
+    BoxedInt* buffering = (BoxedInt*)args[0];
+
     assert(cls == file_cls);
 
     if (s->cls == unicode_cls)
@@ -823,6 +825,9 @@ Box* fileNew(BoxedClass* cls, Box* s, Box* m) {
         raiseExcHelper(TypeError, "");
     }
 
+    if (!PyInt_Check(buffering))
+        raiseExcHelper(TypeError, "an integer is required");
+
     auto fn = static_cast<BoxedString*>(s);
     auto mode = static_cast<BoxedString*>(m);
 
@@ -840,7 +845,9 @@ Box* fileNew(BoxedClass* cls, Box* s, Box* m) {
         abort(); // unreachable;
     }
 
-    return new BoxedFile(f, fn->s, PyString_AsString(m));
+    auto file = new BoxedFile(f, fn->s, PyString_AsString(m));
+    PyFile_SetBufSize(file, buffering->n);
+    return file;
 }
 
 static PyObject* file_readlines(BoxedFile* f, PyObject* args) noexcept {
@@ -1111,12 +1118,38 @@ extern "C" int PyFile_WriteString(const char* s, PyObject* f) noexcept {
 
 extern "C" void PyFile_SetBufSize(PyObject* f, int bufsize) noexcept {
     assert(f->cls == file_cls);
+    BoxedFile* file = (BoxedFile*)f;
     if (bufsize >= 0) {
-        if (bufsize == 0) {
-            setvbuf(static_cast<BoxedFile*>(f)->f_fp, NULL, _IONBF, 0);
+        int type;
+        switch (bufsize) {
+            case 0:
+                type = _IONBF;
+                break;
+#ifdef HAVE_SETVBUF
+            case 1:
+                type = _IOLBF;
+                bufsize = BUFSIZ;
+                break;
+#endif
+            default:
+                type = _IOFBF;
+#ifndef HAVE_SETVBUF
+                bufsize = BUFSIZ;
+#endif
+                break;
+        }
+        fflush(file->f_fp);
+        if (type == _IONBF) {
+            PyMem_Free(file->f_setbuf);
+            file->f_setbuf = NULL;
         } else {
-            Py_FatalError("unimplemented");
+            file->f_setbuf = (char*)PyMem_Realloc(file->f_setbuf, bufsize);
         }
+#ifdef HAVE_SETVBUF
+        setvbuf(file->f_fp, file->f_setbuf, type, bufsize);
+#else  /* !HAVE_SETVBUF */
+        setbuf(file->f_fp, file->f_setbuf);
+#endif /* !HAVE_SETVBUF */
     }
 }
 
@@ -1426,6 +1459,7 @@ void BoxedFile::gcHandler(GCVisitor* v, Box* b) {
     v->visit(f->f_mode);
     v->visit(f->f_encoding);
     v->visit(f->f_errors);
+    v->visit(f->f_setbuf);
 }
 
 void setupFile() {
@@ -1455,8 +1489,8 @@ void setupFile() {
     file_cls->giveAttr("softspace",
                        new BoxedMemberDescriptor(BoxedMemberDescriptor::INT, offsetof(BoxedFile, f_softspace), false));
 
-    file_cls->giveAttr("__new__", new BoxedFunction(boxRTFunction((void*)fileNew, UNKNOWN, 3, 1, false, false),
-                                                    { boxStrConstant("r") }));
+    file_cls->giveAttr("__new__", new BoxedFunction(boxRTFunction((void*)fileNew, UNKNOWN, 4, 2, false, false),
+                                                    { boxStrConstant("r"), boxInt(-1) }));
 
     for (auto& md : file_methods) {
         file_cls->giveAttr(md.ml_name, new BoxedMethodDescriptor(&md, file_cls));

src/runtime/objmodel.h

@@ -65,7 +65,6 @@ extern "C" BoxedString* strOrNull(Box* obj);  // similar to str, but returns NUL
 extern "C" bool exceptionMatches(Box* obj, Box* cls);
 extern "C" BoxedInt* hash(Box* obj);
 extern "C" Box* abs_(Box* obj);
-Box* open(Box* arg1, Box* arg2);
 // extern "C" Box* chr(Box* arg);
 extern "C" Box* compare(Box*, Box*, int);
 extern "C" BoxedInt* len(Box* obj);

src/runtime/set.cpp

@@ -17,12 +17,12 @@
 #include <llvm/Support/raw_ostream.h>
 
 #include "gc/collector.h"
+#include "runtime/capi.h"
 #include "runtime/objmodel.h"
 
 namespace pyston {
 
-BoxedClass* set_cls, *set_iterator_cls;
-BoxedClass* frozenset_cls;
+BoxedClass* set_iterator_cls;
 
 extern "C" Box* createSet() {
     return new BoxedSet();
@@ -196,7 +196,7 @@ Box* setLen(BoxedSet* self) {
 }
 
 Box* setAdd(BoxedSet* self, Box* v) {
-    assert(self->cls == set_cls);
+    assert(self->cls == set_cls || self->cls == frozenset_cls);
     self->s.insert(v);
     return None;
 }
@@ -391,6 +391,42 @@ Box* setHash(BoxedSet* self) {
     return boxInt(rtn);
 }
 
+extern "C" PyObject* PySet_New(PyObject* iterable) noexcept {
+    if (!iterable)
+        return new BoxedSet(); // Fast path for empty set.
+
+    try {
+        return runtimeCall(set_cls, ArgPassSpec(iterable ? 1 : 0), iterable, NULL, NULL, NULL, NULL);
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return NULL;
+    }
+}
+
+extern "C" PyObject* PyFrozenSet_New(PyObject* iterable) noexcept {
+    try {
+        return runtimeCall(frozenset_cls, ArgPassSpec(iterable ? 1 : 0), iterable, NULL, NULL, NULL, NULL);
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return NULL;
+    }
+}
+
+extern "C" int PySet_Add(PyObject* set, PyObject* key) noexcept {
+    if (!PySet_Check(set) && !PyFrozenSet_Check(set)) {
+        PyErr_BadInternalCall();
+        return -1;
+    }
+
+    try {
+        setAdd((BoxedSet*)set, key);
+        return 0;
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return -1;
+    }
+}
+
 } // namespace set
 
 using namespace pyston::set;

src/runtime/set.h

@@ -25,8 +25,6 @@ namespace pyston {
 void setupSet();
 void teardownSet();
 
-extern BoxedClass* set_cls, *frozenset_cls;
-
 extern "C" Box* createSet();
 
 class BoxedSet : public Box {

src/runtime/str.cpp

@@ -343,7 +343,6 @@ extern "C" PyObject* PyString_InternFromString(const char* s) noexcept {
 }
 
 extern "C" void PyString_InternInPlace(PyObject**) noexcept {
-    Py_FatalError("unimplemented");
 }
 
 /* Format codes

src/runtime/types.cpp

@@ -77,6 +77,7 @@ extern "C" void initzipimport();
 extern "C" void init_csv();
 extern "C" void init_ssl();
 extern "C" void init_sqlite3();
+extern "C" void PyMarshal_Init();
 
 namespace pyston {
 
@@ -655,7 +656,7 @@ BoxedClass* object_cls, *type_cls, *none_cls, *bool_cls, *int_cls, *float_cls,
     * str_cls = NULL, *function_cls, *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls,
       *file_cls, *member_cls, *closure_cls, *generator_cls, *complex_cls, *basestring_cls, *property_cls,
       *staticmethod_cls, *classmethod_cls, *attrwrapper_cls, *pyston_getset_cls, *capi_getset_cls,
-      *builtin_function_or_method_cls, *attrwrapperiter_cls;
+      *builtin_function_or_method_cls, *attrwrapperiter_cls, *set_cls, *frozenset_cls;
 
 BoxedTuple* EmptyTuple;
 BoxedString* EmptyString;
@@ -2270,6 +2271,7 @@ void setupRuntime() {
     init_csv();
     init_ssl();
     init_sqlite3();
+    PyMarshal_Init();
 
     // some additional setup to ensure weakrefs participate in our GC
     BoxedClass* weakref_ref_cls = &_PyWeakref_RefType;

src/runtime/types.h

@@ -86,7 +86,7 @@ extern BoxedClass* object_cls, *type_cls, *bool_cls, *int_cls, *long_cls, *float
     *none_cls, *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls, *file_cls,
     *enumerate_cls, *xrange_cls, *member_cls, *method_cls, *closure_cls, *generator_cls, *complex_cls, *basestring_cls,
     *property_cls, *staticmethod_cls, *classmethod_cls, *attrwrapper_cls, *pyston_getset_cls, *capi_getset_cls,
-    *builtin_function_or_method_cls;
+    *builtin_function_or_method_cls, *set_cls, *frozenset_cls;
 }
 #define unicode_cls (&PyUnicode_Type)
 #define memoryview_cls (&PyMemoryView_Type)

test/tests/builtins.py

@@ -109,3 +109,7 @@ print round(0.5), round(-0.5)
 print list(iter(xrange(100).__iter__().next, 20))
 
 print bytearray(xrange(256))
+
+l = [2, 1, 3]
+print apply(sorted, [l])
+print apply(sorted, [l], { "reverse" : True })

test/tests/file.py

@@ -75,6 +75,8 @@ with open(fn) as f:
 with open(fn, "rU") as f:
     print len(f.readlines())
 
+with open(fn, "r", 1) as f:
+    print len(f.readlines())
 
 fd, fn = tempfile.mkstemp()
 try:

test/tests/marshal_test.py

+import marshal
+o = [-1, 1.23456789, complex(1.2, 3.4)]
+o += [True, False, None]
+o += ["Hello World!", u"Hello World!"]
+o += [{ "Key" : "Value" }, set(["Set"]), frozenset(["FrozenSet"]), (1, 2, 3), [1, 2, 3]]
+for i in o:
+    s = marshal.dumps(i)
+    r = marshal.loads(s)
+    print "Dumping:", i, "Loaded", r

test/tests/slice.py

@@ -13,3 +13,7 @@ print sl
 
 sl = slice(1, 2, "hello")
 print sl
+
+C()[:,:]
+C()[1:2,3:4]
+C()[1:2:3,3:4:5]