Add a number of C API functions

Working towards getting _sre.c to link (it compiles with the previous include/ additions)

src/Makefile

@@ -262,20 +262,20 @@ CLANG_CXX := $(CXX_ENV) $(CLANG_CXX)
 # Not sure if ccache_basedir actually helps at all (I think the generated files make them different?)
 LLVM_BUILD_ENV += CCACHE_DIR=$(HOME)/.ccache_llvm CCACHE_BASEDIR=$(LLVM_SRC)
 
-MAIN_SRCS := $(wildcard codegen/*.cpp) $(wildcard asm_writing/*.cpp) $(wildcard codegen/irgen/*.cpp) $(wildcard codegen/opt/*.cpp) $(wildcard analysis/*.cpp) $(wildcard core/*.cpp) jit.cpp codegen/profiling/profiling.cpp codegen/profiling/dumprof.cpp $(wildcard runtime/*.cpp) $(wildcard runtime/builtin_modules/*.cpp) $(wildcard gc/*.cpp)
+MAIN_SRCS := $(wildcard codegen/*.cpp) $(wildcard asm_writing/*.cpp) $(wildcard codegen/irgen/*.cpp) $(wildcard codegen/opt/*.cpp) $(wildcard analysis/*.cpp) $(wildcard core/*.cpp) jit.cpp codegen/profiling/profiling.cpp codegen/profiling/dumprof.cpp $(wildcard runtime/*.cpp) $(wildcard runtime/builtin_modules/*.cpp) $(wildcard gc/*.cpp) $(wildcard capi/*.cpp)
 STDLIB_SRCS := $(wildcard runtime/inline/*.cpp)
 SRCS := $(MAIN_SRCS) $(STDLIB_SRCS)
 STDLIB_OBJS := stdlib.bc.o stdlib.stripped.bc.o
 STDLIB_RELEASE_OBJS := stdlib.release.bc.o
 
 STDMODULE_SRCS := errnomodule.c shamodule.c sha256module.c sha512module.c md5.c md5module.c $(EXTRA_STDMODULE_SRCS)
-STDMODULE_SRCS := $(addprefix ../lib_python/2.7_Modules/,$(STDMODULE_SRCS))
+FROM_CPYTHON_SRCS := $(addprefix ../lib_python/2.7_Modules/,$(STDMODULE_SRCS)) $(wildcard capi/*.c)
 
 # The stdlib objects have slightly longer dependency chains,
 # so put them first in the list:
-OBJS := $(STDLIB_OBJS) $(SRCS:.cpp=.o) $(STDMODULE_SRCS:.c=.o)
-PROFILE_OBJS := $(STDLIB_RELEASE_OBJS) $(MAIN_SRCS:.cpp=.prof.o) $(STDLIB_SRCS:.cpp=.release.o) $(STDMODULE_SRCS:.c=.release.o)
-OPT_OBJS := $(STDLIB_RELEASE_OBJS) $(SRCS:.cpp=.release.o) $(STDMODULE_SRCS:.c=.release.o)
+OBJS := $(STDLIB_OBJS) $(SRCS:.cpp=.o) $(FROM_CPYTHON_SRCS:.c=.o)
+PROFILE_OBJS := $(STDLIB_RELEASE_OBJS) $(MAIN_SRCS:.cpp=.prof.o) $(STDLIB_SRCS:.cpp=.release.o) $(FROM_CPYTHON_SRCS:.c=.release.o)
+OPT_OBJS := $(STDLIB_RELEASE_OBJS) $(SRCS:.cpp=.release.o) $(FROM_CPYTHON_SRCS:.c=.release.o)
 
 OPTIONAL_SRCS := codegen/profiling/oprofile.cpp codegen/profiling/pprof.cpp
 TOOL_SRCS := $(wildcard $(TOOLS_DIR)/*.cpp)
@@ -881,10 +881,14 @@ ext: ../test/test_extension/test.so
 ../test/test_extension/test.o: ../test/test_extension/test.c $(wildcard ../include/*.h) $(BUILD_SYSTEM_DEPS)
 	$(CLANG_EXE) -O2 -fPIC -Wimplicit -I../include -c $< -o $@ -g
 
-../lib_python/2.7_Modules/%.o: ../lib_python/2.7_Modules/%.c $(BUILD_SYSTEM_DEPS)
-	$(ECHO) Compiling extension file $@
+$(FROM_CPYTHON_SRCS:.c=.o): %.o: %.c $(BUILD_SYSTEM_DEPS)
+	$(ECHO) Compiling C file to $@
 	$(VERB) $(CC) $(EXT_CFLAGS) -c $< -o $@ -g -MMD -MP -MF $<.d -O0
 
-../lib_python/2.7_Modules/%.release.o: ../lib_python/2.7_Modules/%.c $(BUILD_SYSTEM_DEPS)
-	$(ECHO) Compiling extension file $@
+$(FROM_CPYTHON_SRCS:.c=.release.o): %.release.o: %.c $(BUILD_SYSTEM_DEPS)
+	$(ECHO) Compiling C file to $@
 	$(VERB) $(CC) $(EXT_CFLAGS) -c $< -o $@ -g -MMD -MP -MF $<.d
+
+# These are necessary until we support unicode:
+../lib_python/2.7_Modules/_sre.o: EXT_CFLAGS += -Wno-sometimes-uninitialized
+../lib_python/2.7_Modules/_sre.release.o: EXT_CFLAGS += -Wno-sometimes-uninitialized

src/capi/getargs.cpp

+// Copyright (c) 2014 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Hopefully soon we will be able to switch to CPython's getargs.c, instead of having
+// to reimplement it.  For now, it's easier to create simple versions of the functions
+// instead of trying to support all of the internals of getargs.c
+
+#include <dlfcn.h>
+#include <stdarg.h>
+#include <string.h>
+
+#include "Python.h"
+
+#include "codegen/compvars.h"
+#include "core/threading.h"
+#include "core/types.h"
+#include "runtime/objmodel.h"
+#include "runtime/types.h"
+
+namespace pyston {
+
+
+#define FLAG_COMPAT 1
+#define FLAG_SIZE_T 2
+
+// This function is named after the CPython one:
+static int vgetargs1(PyObject* _tuple, const char* fmt, va_list* ap, int flags) {
+    RELEASE_ASSERT(_tuple->cls == tuple_cls, "");
+    BoxedTuple* tuple = static_cast<BoxedTuple*>(_tuple);
+
+    bool now_optional = false;
+    int arg_idx = 0;
+
+    int tuple_size = tuple->elts.size();
+
+    while (char c = *fmt) {
+        fmt++;
+
+        if (c == ':') {
+            break;
+        } else if (c == '|') {
+            now_optional = true;
+            continue;
+        } else {
+            if (arg_idx >= tuple_size) {
+                RELEASE_ASSERT(now_optional, "");
+                break;
+            }
+
+            PyObject* arg = tuple->elts[arg_idx];
+
+            switch (c) {
+                case 's': {
+                    if (*fmt == '*') {
+                        Py_buffer* p = (Py_buffer*)va_arg(*ap, Py_buffer*);
+
+                        RELEASE_ASSERT(arg->cls == str_cls, "");
+                        PyBuffer_FillInfo(p, arg, PyString_AS_STRING(arg), PyString_GET_SIZE(arg), 1, 0);
+                        fmt++;
+                    } else if (*fmt == ':') {
+                        break;
+                    } else {
+                        RELEASE_ASSERT(0, "");
+                    }
+                    break;
+                }
+                case 'O': {
+                    PyObject** p = (PyObject**)va_arg(*ap, PyObject**);
+                    *p = arg;
+                    break;
+                }
+                default:
+                    RELEASE_ASSERT(0, "Unhandled format character: '%c'", c);
+            }
+        }
+    }
+    return 1;
+}
+
+extern "C" int PyArg_VaParse(PyObject* _tuple, const char* fmt, va_list ap) {
+    va_list lva;
+    __va_copy(lva, ap);
+    return vgetargs1(_tuple, fmt, &lva, 0);
+}
+
+extern "C" int PyArg_ParseTuple(PyObject* _tuple, const char* fmt, ...) {
+    va_list ap;
+    va_start(ap, fmt);
+
+    int r = vgetargs1(_tuple, fmt, &ap, 0);
+
+    va_end(ap);
+
+    return r;
+}
+
+extern "C" int _PyArg_ParseTuple_SizeT(PyObject* args, char* format, ...) {
+    int retval;
+    va_list va;
+
+    va_start(va, format);
+    retval = vgetargs1(args, format, &va, FLAG_SIZE_T);
+    va_end(va);
+    return retval;
+}
+
+extern "C" int PyArg_ParseTupleAndKeywords(PyObject* args, PyObject* kwargs, const char* format, char** kwlist, ...) {
+    assert(kwargs->cls == dict_cls);
+    RELEASE_ASSERT(static_cast<BoxedDict*>(kwargs)->d.size() == 0, "");
+
+    va_list ap;
+    va_start(ap, kwlist);
+
+    int r = vgetargs1(args, format, &ap, 0);
+
+    va_end(ap);
+
+    return r;
+}
+
+extern "C" int _PyArg_ParseTupleAndKeywords_SizeT(PyObject* args, PyObject* keywords, const char* format, char** kwlist,
+                                                  ...) {
+    if ((args == NULL || !PyTuple_Check(args)) || (keywords != NULL && !PyDict_Check(keywords)) || format == NULL
+        || kwlist == NULL) {
+        PyErr_BadInternalCall();
+        return 0;
+    }
+
+    assert(keywords->cls == dict_cls);
+    RELEASE_ASSERT(static_cast<BoxedDict*>(keywords)->d.size() == 0, "");
+
+    va_list ap;
+    va_start(ap, kwlist);
+
+    int r = vgetargs1(args, format, &ap, FLAG_SIZE_T);
+
+    va_end(ap);
+
+    return r;
+}
+
+extern "C" int PyArg_UnpackTuple(PyObject* args, const char* name, Py_ssize_t min, Py_ssize_t max, ...) {
+    RELEASE_ASSERT(args->cls == tuple_cls, "");
+    BoxedTuple* t = static_cast<BoxedTuple*>(args);
+
+    RELEASE_ASSERT(min <= t->elts.size() && t->elts.size() <= max, "");
+
+    va_list ap;
+    va_start(ap, max);
+
+    for (auto e : t->elts) {
+        PyObject** p = (PyObject**)va_arg(ap, PyObject**);
+        *p = e;
+    }
+
+    va_end(ap);
+
+    return true;
+}
+
+} // namespace pyston

src/core/types.h

@@ -376,13 +376,23 @@ public:
     Box* getattr(const std::string& attr) { return getattr(attr, NULL); }
     void delattr(const std::string& attr, DelattrRewriteArgs* rewrite_args);
 };
+static_assert(offsetof(Box, cls) == offsetof(struct _object, ob_type), "");
+
+// CPython C API compatibility class:
+class BoxVar : public Box {
+public:
+    Py_ssize_t ob_size;
+
+    BoxVar(BoxedClass* cls, Py_ssize_t ob_size) : Box(cls), ob_size(ob_size) {}
+};
+static_assert(offsetof(BoxVar, ob_size) == offsetof(struct _varobject, ob_size), "");
+
 extern "C" const std::string* getTypeName(Box* o);
 
 
 
-class BoxedClass : public Box {
+class BoxedClass : public BoxVar {
 public:
-    Py_ssize_t ob_size; // CPython API compatibility
     PyTypeObject_BODY;
 
     HCAttrs attrs;

src/runtime/builtin_modules/builtins.cpp

@@ -395,9 +395,11 @@ BoxedClass* notimplemented_cls;
 BoxedModule* builtins_module;
 
 // TODO looks like CPython and pypy put this into an "exceptions" module:
+extern "C" {
 BoxedClass* Exception, *AssertionError, *AttributeError, *GeneratorExit, *TypeError, *NameError, *KeyError, *IndexError,
     *IOError, *OSError, *ZeroDivisionError, *ValueError, *UnboundLocalError, *RuntimeError, *ImportError,
-    *StopIteration, *Warning, *SyntaxError;
+    *StopIteration, *Warning, *SyntaxError, *OverflowError, *DeprecationWarning;
+}
 
 Box* exceptionNew1(BoxedClass* cls) {
     return exceptionNew2(cls, boxStrConstant(""));
@@ -554,9 +556,10 @@ void setupBuiltins() {
     StopIteration = makeBuiltinException(Exception, "StopIteration");
     Warning = makeBuiltinException(Exception, "Warning");
     SyntaxError = makeBuiltinException(Exception, "SyntaxError");
+    OverflowError = makeBuiltinException(Exception, "OverflowError");
     /*ImportWarning =*/makeBuiltinException(Warning, "ImportWarning");
     /*PendingDeprecationWarning =*/makeBuiltinException(Warning, "PendingDeprecationWarning");
-    /*DeprecationWarning =*/makeBuiltinException(Warning, "DeprecationWarning");
+    DeprecationWarning = makeBuiltinException(Warning, "DeprecationWarning");
     /*BytesWarning =*/makeBuiltinException(Warning, "BytesWarning");
 
     repr_obj = new BoxedFunction(boxRTFunction((void*)repr, UNKNOWN, 1));

src/runtime/capi.cpp

@@ -74,6 +74,17 @@ extern "C" int PyModule_AddIntConstant(PyObject* _m, const char* name, long valu
     return 0;
 }
 
+#define MAKE_CHECK(NAME, cls_name)                                                                                     \
+    extern "C" bool Py##NAME##_Check(PyObject* op) { return isSubclass(op->cls, cls_name); }
+
+MAKE_CHECK(Int, int_cls)
+MAKE_CHECK(String, str_cls)
+MAKE_CHECK(Long, long_cls)
+MAKE_CHECK(List, list_cls)
+MAKE_CHECK(Tuple, tuple_cls)
+MAKE_CHECK(Dict, dict_cls)
+#undef MAKE_CHECK
+
 extern "C" PyObject* PyDict_New() {
     return new BoxedDict();
 }
@@ -333,100 +344,112 @@ extern "C" void PyBuffer_Release(Py_buffer* view) {
     view->obj = NULL;
 }
 
-int vPyArg_ParseTuple(PyObject* _tuple, const char* fmt, va_list ap) {
-    RELEASE_ASSERT(_tuple->cls == tuple_cls, "");
-    BoxedTuple* tuple = static_cast<BoxedTuple*>(_tuple);
+extern "C" void Py_FatalError(const char* msg) {
+    fprintf(stderr, "%s", msg);
+    abort();
+}
 
-    bool now_optional = false;
-    int arg_idx = 0;
+extern "C" void _PyErr_BadInternalCall(const char* filename, int lineno) {
+    abort();
+}
 
-    int tuple_size = tuple->elts.size();
+extern "C" PyObject* PyObject_Init(PyObject* op, PyTypeObject* tp) {
+    RELEASE_ASSERT(op, "");
+    RELEASE_ASSERT(tp, "");
+    Py_TYPE(op) = tp;
+    return op;
+}
 
-    while (char c = *fmt) {
-        fmt++;
+extern "C" PyVarObject* PyObject_InitVar(PyVarObject* op, PyTypeObject* tp, Py_ssize_t size) {
+    abort(); // "var" objects not tested yet
 
-        if (c == ':') {
-            break;
-        } else if (c == '|') {
-            now_optional = true;
-            continue;
-        } else {
-            if (arg_idx >= tuple_size) {
-                RELEASE_ASSERT(now_optional, "");
-                break;
-            }
-
-            PyObject* arg = tuple->elts[arg_idx];
-
-            switch (c) {
-                case 's': {
-                    if (*fmt == '*') {
-                        Py_buffer* p = (Py_buffer*)va_arg(ap, Py_buffer*);
-
-                        RELEASE_ASSERT(arg->cls == str_cls, "");
-                        PyBuffer_FillInfo(p, arg, PyString_AS_STRING(arg), PyString_GET_SIZE(arg), 1, 0);
-                        fmt++;
-                    } else if (*fmt == ':') {
-                        break;
-                    } else {
-                        RELEASE_ASSERT(0, "");
-                    }
-                    break;
-                }
-                case 'O': {
-                    PyObject** p = (PyObject**)va_arg(ap, PyObject**);
-                    *p = arg;
-                    break;
-                }
-                default:
-                    RELEASE_ASSERT(0, "Unhandled format character: '%c'", c);
-            }
-        }
+    RELEASE_ASSERT(op, "");
+    RELEASE_ASSERT(tp, "");
+    Py_TYPE(op) = tp;
+    op->ob_size = size;
+    return op;
+}
+
+extern "C" PyObject* _PyObject_New(PyTypeObject* cls) {
+    assert(cls->tp_itemsize == 0);
+    auto rtn = (PyObject*)gc_alloc(cls->tp_basicsize, gc::GCKind::PYTHON);
+    rtn->cls = cls;
+    return rtn;
+}
+
+extern "C" void PyObject_Free(void* p) {
+    gc::gc_free(p);
+    ASSERT(0, "I think this is good enough but I'm not sure; should test");
+}
+
+extern "C" PyObject* PyObject_CallObject(PyObject* obj, PyObject* args) {
+    RELEASE_ASSERT(args, ""); // actually it looks like this is allowed to be NULL
+    RELEASE_ASSERT(args->cls == tuple_cls, "");
+
+    // TODO do something like this?  not sure if this is safe; will people expect that calling into a known function
+    // won't end up doing a GIL check?
+    // threading::GLDemoteRegion _gil_demote;
+
+    try {
+        Box* r = runtimeCall(obj, ArgPassSpec(0, 0, true, false), args, NULL, NULL, NULL, NULL);
+        return r;
+    } catch (Box* b) {
+        abort();
     }
-    return 1;
 }
 
-extern "C" int PyArg_ParseTuple(PyObject* _tuple, const char* fmt, ...) {
-    va_list ap;
-    va_start(ap, fmt);
+extern "C" PyObject* PyObject_GetAttrString(PyObject* o, const char* attr) {
+    // TODO do something like this?  not sure if this is safe; will people expect that calling into a known function
+    // won't end up doing a GIL check?
+    // threading::GLDemoteRegion _gil_demote;
 
-    int r = vPyArg_ParseTuple(_tuple, fmt, ap);
+    try {
+        return getattr(o, attr);
+    } catch (Box* b) {
+        abort();
+    }
+}
 
-    va_end(ap);
 
-    return r;
+extern "C" void PyErr_Restore(PyObject* type, PyObject* value, PyObject* traceback) {
+    abort();
 }
 
-extern "C" int PyArg_ParseTupleAndKeywords(PyObject* args, PyObject* kwargs, const char* format, char** kwlist, ...) {
-    assert(kwargs->cls == dict_cls);
-    RELEASE_ASSERT(static_cast<BoxedDict*>(kwargs)->d.size() == 0, "");
+extern "C" void PyErr_Clear() {
+    PyErr_Restore(NULL, NULL, NULL);
+}
 
-    va_list ap;
-    va_start(ap, kwlist);
+extern "C" void PyErr_SetString(PyObject* exception, const char* string) {
+    PyErr_SetObject(exception, boxStrConstant(string));
+}
 
-    int r = vPyArg_ParseTuple(args, format, ap);
+extern "C" void PyErr_SetObject(PyObject* exception, PyObject* value) {
+    PyErr_Restore(exception, value, NULL);
+}
 
-    va_end(ap);
+extern "C" PyObject* PyErr_Format(PyObject* exception, const char* format, ...) {
+    abort();
+}
 
-    return r;
+extern "C" PyObject* PyErr_NoMemory() {
+    abort();
 }
 
-extern "C" PyObject* _PyObject_New(PyTypeObject* cls) {
-    assert(cls->tp_itemsize == 0);
-    auto rtn = (PyObject*)gc_alloc(cls->tp_basicsize, gc::GCKind::PYTHON);
-    rtn->cls = cls;
-    return rtn;
+extern "C" int PyErr_CheckSignals() {
+    abort();
 }
 
-extern "C" void PyObject_Free(void* p) {
-    gc::gc_free(p);
-    ASSERT(0, "I think this is good enough but I'm not sure; should test");
+extern "C" int PyErr_ExceptionMatches(PyObject* exc) {
+    abort();
 }
 
 extern "C" PyObject* PyErr_Occurred() {
+    abort();
+    /*
     printf("need to hook exception handling -- make sure errors dont propagate into C code, and error codes get "
            "checked coming out\n");
     return NULL;
+    */
 }
 
 BoxedModule* importTestExtension() {

src/runtime/dict.cpp

@@ -82,6 +82,11 @@ Box* dictLen(BoxedDict* self) {
     return boxInt(self->d.size());
 }
 
+extern "C" Py_ssize_t PyDict_Size(PyObject* op) {
+    RELEASE_ASSERT(PyDict_Check(op), "");
+    return static_cast<BoxedDict*>(op)->d.size();
+}
+
 Box* dictGetitem(BoxedDict* self, Box* k) {
     assert(self->cls == dict_cls);
 

src/runtime/int.cpp

@@ -31,9 +31,23 @@
 
 namespace pyston {
 
-extern "C" long PyInt_AsLong(PyObject* obj) {
-    assert(obj->cls == int_cls);
-    return static_cast<BoxedInt*>(obj)->n;
+extern "C" long PyInt_AsLong(PyObject* op) {
+    RELEASE_ASSERT(op->cls == int_cls, "");
+    return static_cast<BoxedInt*>(op)->n;
+}
+
+extern "C" Py_ssize_t PyInt_AsSsize_t(PyObject* op) {
+    RELEASE_ASSERT(op->cls == int_cls, "");
+    return static_cast<BoxedInt*>(op)->n;
+}
+
+extern "C" PyObject* PyInt_FromSize_t(size_t ival) {
+    RELEASE_ASSERT(ival <= LONG_MAX, "");
+    return boxInt(ival);
+}
+
+extern "C" PyObject* PyInt_FromSsize_t(Py_ssize_t ival) {
+    return boxInt(ival);
 }
 
 BoxedInt* interned_ints[NUM_INTERNED_INTS];

src/runtime/list.cpp

@@ -30,6 +30,15 @@
 
 namespace pyston {
 
+extern "C" int PyList_Append(PyObject* op, PyObject* newitem) {
+    try {
+        listAppend(op, newitem);
+    } catch (Box* b) {
+        abort();
+    }
+    return 0;
+}
+
 extern "C" Box* listRepr(BoxedList* self) {
     LOCK_REGION(self->lock.asRead());
 
@@ -87,7 +96,13 @@ extern "C" Box* listPop(BoxedList* self, Box* idx) {
     return rtn;
 }
 
+extern "C" Py_ssize_t PyList_Size(PyObject* self) {
+    RELEASE_ASSERT(self->cls == list_cls, "");
+    return static_cast<BoxedList*>(self)->size;
+}
+
 extern "C" Box* listLen(BoxedList* self) {
+    assert(self->cls == list_cls);
     return boxInt(self->size);
 }
 
@@ -117,12 +132,10 @@ Box* _listSlice(BoxedList* self, i64 start, i64 stop, i64 step) {
     return rtn;
 }
 
-extern "C" Box* listGetitemInt(BoxedList* self, BoxedInt* slice) {
+extern "C" Box* listGetitemUnboxed(BoxedList* self, int64_t n) {
     LOCK_REGION(self->lock.asRead());
 
     assert(self->cls == list_cls);
-    assert(slice->cls == int_cls);
-    int64_t n = slice->n;
     if (n < 0)
         n = self->size + n;
 
@@ -133,6 +146,21 @@ extern "C" Box* listGetitemInt(BoxedList* self, BoxedInt* slice) {
     return rtn;
 }
 
+extern "C" Box* listGetitemInt(BoxedList* self, BoxedInt* slice) {
+    assert(slice->cls == int_cls);
+    return listGetitemUnboxed(self, slice->n);
+}
+
+extern "C" PyObject* PyList_GetItem(PyObject* op, Py_ssize_t i) {
+    RELEASE_ASSERT(PyList_Check(op), "");
+    RELEASE_ASSERT(i >= 0, ""); // unlike list.__getitem__, PyList_GetItem doesn't do index wrapping
+    try {
+        return listGetitemUnboxed(static_cast<BoxedList*>(op), i);
+    } catch (Box* b) {
+        abort();
+    }
+}
+
 extern "C" Box* listGetitemSlice(BoxedList* self, BoxedSlice* slice) {
     LOCK_REGION(self->lock.asRead());
 
@@ -466,6 +494,18 @@ extern "C" Box* listNew(Box* cls, Box* container) {
     return rtn;
 }
 
+extern "C" PyObject* PyList_New(Py_ssize_t size) {
+    // This function is supposed to return a list of `size` NULL elements.
+    // That will probably trip an assert somewhere if we try to create that (ex
+    // I think the GC will expect them to be real objects so they can be relocated).
+    RELEASE_ASSERT(size == 0, "");
+    try {
+        return new BoxedList();
+    } catch (Box* b) {
+        abort();
+    }
+}
+
 Box* _listCmp(BoxedList* lhs, BoxedList* rhs, AST_TYPE::AST_TYPE op_type) {
     int lsz = lhs->size;
     int rsz = rhs->size;

src/runtime/long.cpp

@@ -33,6 +33,21 @@ namespace pyston {
 
 BoxedClass* long_cls;
 
+extern "C" unsigned long PyLong_AsUnsignedLong(PyObject* vv) {
+    RELEASE_ASSERT(PyLong_Check(vv), "");
+    BoxedLong* l = static_cast<BoxedLong*>(vv);
+
+    // TODO Will this error on negative values?
+    RELEASE_ASSERT(mpz_fits_ulong_p(l->n), "");
+    return mpz_get_ui(l->n);
+}
+
+extern "C" PyObject* PyLong_FromUnsignedLong(unsigned long ival) {
+    BoxedLong* rtn = new BoxedLong(long_cls);
+    mpz_init_set_ui(rtn->n, ival);
+    return rtn;
+}
+
 extern "C" Box* createLong(const std::string* s) {
     BoxedLong* rtn = new BoxedLong(long_cls);
     int r = mpz_init_set_str(rtn->n, s->c_str(), 10);

src/runtime/objmodel.cpp

@@ -382,7 +382,7 @@ extern "C" void checkUnpackingLength(i64 expected, i64 given) {
 
 BoxedClass::BoxedClass(BoxedClass* base, gcvisit_func gc_visit, int attrs_offset, int instance_size,
                        bool is_user_defined)
-    : Box(type_cls), tp_basicsize(instance_size), tp_dealloc(NULL), base(base), gc_visit(gc_visit),
+    : BoxVar(type_cls, 0), tp_basicsize(instance_size), tp_dealloc(NULL), base(base), gc_visit(gc_visit),
       attrs_offset(attrs_offset), is_constant(false), is_user_defined(is_user_defined) {
     assert(tp_dealloc == NULL);
 

src/runtime/tuple.cpp

@@ -65,8 +65,7 @@ Box* _tupleSlice(BoxedTuple* self, i64 start, i64 stop, i64 step) {
     return new BoxedTuple(std::move(velts));
 }
 
-Box* tupleGetitemInt(BoxedTuple* self, BoxedInt* slice) {
-    i64 n = slice->n;
+Box* tupleGetitemUnboxed(BoxedTuple* self, i64 n) {
     i64 size = self->elts.size();
 
     if (n < 0)
@@ -78,6 +77,20 @@ Box* tupleGetitemInt(BoxedTuple* self, BoxedInt* slice) {
     return rtn;
 }
 
+Box* tupleGetitemInt(BoxedTuple* self, BoxedInt* slice) {
+    return tupleGetitemUnboxed(self, slice->n);
+}
+
+extern "C" PyObject* PyTuple_GetItem(PyObject* op, Py_ssize_t i) {
+    RELEASE_ASSERT(PyTuple_Check(op), "");
+    RELEASE_ASSERT(i >= 0, ""); // unlike tuple.__getitem__, PyTuple_GetItem doesn't do index wrapping
+    try {
+        return tupleGetitemUnboxed(static_cast<BoxedTuple*>(op), i);
+    } catch (Box* b) {
+        abort();
+    }
+}
+
 Box* tupleGetitemSlice(BoxedTuple* self, BoxedSlice* slice) {
     assert(self->cls == tuple_cls);
     assert(slice->cls == slice_cls);
@@ -115,6 +128,11 @@ Box* tupleLen(BoxedTuple* t) {
     return boxInt(t->elts.size());
 }
 
+extern "C" Py_ssize_t PyTuple_Size(PyObject* op) {
+    RELEASE_ASSERT(PyTuple_Check(op), "");
+    return static_cast<BoxedTuple*>(op)->elts.size();
+}
+
 Box* tupleRepr(BoxedTuple* t) {
     assert(t->cls == tuple_cls);
 
@@ -286,6 +304,40 @@ extern "C" Box* tupleNew(Box* _cls, BoxedTuple* args, BoxedDict* kwargs) {
     return new BoxedTuple(std::move(velts));
 }
 
+extern "C" int PyTuple_SetItem(PyObject* op, Py_ssize_t i, PyObject* newitem) {
+    RELEASE_ASSERT(PyTuple_Check(op), "");
+
+    BoxedTuple* t = static_cast<BoxedTuple*>(op);
+    RELEASE_ASSERT(i >= 0 && i < t->elts.size(), "");
+    t->elts[i] = newitem;
+    return 0;
+}
+
+extern "C" PyObject* PyTuple_Pack(Py_ssize_t n, ...) {
+    va_list vargs;
+
+    va_start(vargs, n);
+    PyObject* result = PyTuple_New(n);
+
+    if (result == NULL) {
+        va_end(vargs);
+        return NULL;
+    }
+
+    for (Py_ssize_t i = 0; i < n; i++) {
+        PyObject* o = va_arg(vargs, PyObject*);
+        PyTuple_SetItem(result, i, o);
+    }
+    va_end(vargs);
+    return result;
+}
+
+extern "C" PyObject* PyTuple_New(Py_ssize_t size) {
+    RELEASE_ASSERT(size >= 0, "");
+
+    return new BoxedTuple(BoxedTuple::GCVector(size, NULL));
+}
+
 
 BoxedClass* tuple_iterator_cls = NULL;
 extern "C" void tupleIteratorGCHandler(GCVisitor* v, Box* b) {

src/runtime/types.h

@@ -74,9 +74,9 @@ BoxedList* getSysPath();
 Box* getSysStdout();
 
 extern "C" {
-extern BoxedClass* object_cls, *type_cls, *bool_cls, *int_cls, *float_cls, *str_cls, *function_cls, *none_cls,
-    *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls, *file_cls, *xrange_cls, *member_cls,
-    *method_cls, *closure_cls, *generator_cls;
+extern BoxedClass* object_cls, *type_cls, *bool_cls, *int_cls, *long_cls, *float_cls, *str_cls, *function_cls,
+    *none_cls, *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls, *file_cls, *xrange_cls,
+    *member_cls, *method_cls, *closure_cls, *generator_cls;
 }
 extern "C" { extern Box* None, *NotImplemented, *True, *False; }
 extern "C" {
@@ -253,12 +253,10 @@ public:
 class BoxedTuple : public Box {
 public:
     typedef std::vector<Box*, StlCompatAllocator<Box*> > GCVector;
-    const GCVector elts;
+    GCVector elts;
 
-    BoxedTuple(std::vector<Box*, StlCompatAllocator<Box*> >& elts) __attribute__((visibility("default")))
-    : Box(tuple_cls), elts(elts) {}
-    BoxedTuple(std::vector<Box*, StlCompatAllocator<Box*> >&& elts) __attribute__((visibility("default")))
-    : Box(tuple_cls), elts(std::move(elts)) {}
+    BoxedTuple(GCVector& elts) __attribute__((visibility("default"))) : Box(tuple_cls), elts(elts) {}
+    BoxedTuple(GCVector&& elts) __attribute__((visibility("default"))) : Box(tuple_cls), elts(std::move(elts)) {}
 };
 extern "C" BoxedTuple* EmptyTuple;
 
@@ -370,7 +368,7 @@ extern "C" void boxGCHandler(GCVisitor* v, Box* b);
 Box* exceptionNew1(BoxedClass* cls);
 Box* exceptionNew2(BoxedClass* cls, Box* message);
 
-extern BoxedClass* Exception, *AssertionError, *AttributeError, *TypeError, *NameError, *KeyError, *IndexError,
+extern "C" BoxedClass* Exception, *AssertionError, *AttributeError, *TypeError, *NameError, *KeyError, *IndexError,
     *IOError, *OSError, *ZeroDivisionError, *ValueError, *UnboundLocalError, *RuntimeError, *ImportError,
     *StopIteration, *GeneratorExit, *SyntaxError;
 

tools/lint.py

 import os
 import sys
 
-def verify_include(_, dir, files):
+def file_is_from_cpython(fn):
+    return '2.7' in fn
+
+def verify_include_guard(_, dir, files):
     for bn in files:
         fn = os.path.join(dir, bn)
 
         if not bn.endswith(".h"):
             continue
 
+        if file_is_from_cpython(fn):
+            continue
+
         expected_guard = "PYSTON" + fn[1:-2].replace('_', '').replace('/', '_').upper() + "_H"
         with open(fn) as f:
             while True:
@@ -24,8 +30,11 @@ def verify_license(_, dir, files):
 
         if bn.endswith(".h") or bn.endswith(".cpp"):
             s = open(fn).read(1024)
-            assert "Copyright (c) 2014 Dropbox, Inc." in s, fn
-            assert "Apache License, Version 2.0" in s, fn
+            if file_is_from_cpython(fn):
+                assert "This file is originally from CPython 2.7, with modifications for Pyston" in s, fn
+            else:
+                assert "Copyright (c) 2014 Dropbox, Inc." in s, fn
+                assert "Apache License, Version 2.0" in s, fn
 
 PYSTON_SRC_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../src"))
 PYSTON_SRC_SUBDIRS = [bn for bn in os.listdir(PYSTON_SRC_DIR) if os.path.isdir(os.path.join(PYSTON_SRC_DIR, bn))]
@@ -147,7 +156,7 @@ def verify_include_order(_, dir, files):
 
 
 if __name__ == "__main__":
-    os.path.walk('.', verify_include, None)
+    os.path.walk('.', verify_include_guard, None)
     os.path.walk('.', verify_include_order, None)
     os.path.walk('.', verify_license, None)
     os.path.walk('../tools', verify_license, None)