Merge commit '74f7ff8a' into refcounting

Conflicts:
	src/runtime/code.cpp

added BoxedCode::traverse and BoxedCode::dealloc

from_cpython/Include/Python.h

@@ -83,7 +83,6 @@
 #include "classobject.h"
 #include "cobject.h"
 #include "fileobject.h"
-#include "frameobject.h"
 #include "pycapsule.h"
 #include "traceback.h"
 #include "sliceobject.h"

from_cpython/Include/frameobject.h

@@ -58,12 +58,11 @@ PyAPI_DATA(PyTypeObject) PyFrame_Type;
 
 #define PyFrame_Check(op) ((op)->ob_type == &PyFrame_Type)
 #define PyFrame_IsRestricted(f) \
-	((f)->f_builtins != (f)->f_tstate->interp->builtins)
+    ((f)->f_builtins != (f)->f_tstate->interp->builtins)
 
 PyAPI_FUNC(PyFrameObject *) PyFrame_New(PyThreadState *, PyCodeObject *,
                                        PyObject *, PyObject *);
 
-
 /* The rest of the interface is specific for frame objects */
 
 /* Block management functions */
@@ -89,8 +88,12 @@ PyAPI_DATA(PyTypeObject*) frame_cls;
 #define PyFrame_Type (*frame_cls)
 #define PyFrame_Check(op) (((PyObject*)op)->ob_type == &PyFrame_Type)
 
+PyAPI_FUNC(PyFrameObject *) PyFrame_New(PyThreadState *, PyCodeObject *,
+                                       PyObject *, PyObject *) PYSTON_NOEXCEPT;
+
 /* Return the line of code the frame is currently executing. */
 PyAPI_FUNC(int) PyFrame_GetLineNumber(PyFrameObject *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(void) PyFrame_SetLineNumber(PyFrameObject *, int line_number) PYSTON_NOEXCEPT;
 // Pyston changes: add a function to get globals
 PyAPI_FUNC(PyObject *) PyFrame_GetGlobals(PyFrameObject *) PYSTON_NOEXCEPT;
 // Pyston changes: add a function to get the code object

from_cpython/Modules/_ctypes/callbacks.c

@@ -5,9 +5,7 @@
 #include "Python.h"
 #include "compile.h" /* required only for 2.3, as it seems */
 
-// Pyston change: We don't have this file and commented out the function that needs it, but
-// we may want to support that function in the future.
-//#include "frameobject.h"
+#include "frameobject.h"
 
 #include <ffi.h>
 #ifdef MS_WIN32
@@ -152,9 +150,6 @@ failed:
 /* after code that pyrex generates */
 void _ctypes_add_traceback(char *funcname, char *filename, int lineno)
 {
-    // TODO: Pyston change:
-    // Supporting this will require frameobject.h
-#if 0
     PyObject *py_globals = 0;
     PyCodeObject *py_code = 0;
     PyFrameObject *py_frame = 0;
@@ -170,15 +165,16 @@ void _ctypes_add_traceback(char *funcname, char *filename, int lineno)
         0                    /*PyObject *locals*/
         );
     if (!py_frame) goto bad;
-    py_frame->f_lineno = lineno;
+
+    // Pyston change:
+    // py_frame->f_lineno = lineno;
+    PyFrame_SetLineNumber(py_frame, lineno);
+
     PyTraceBack_Here(py_frame);
   bad:
     Py_XDECREF(py_globals);
     Py_XDECREF(py_code);
     Py_XDECREF(py_frame);
-#else
-    assert(false);
-#endif
 }
 
 #ifdef MS_WIN32

src/runtime/capi.cpp

@@ -1644,7 +1644,10 @@ extern "C" void PyEval_ReleaseThread(PyThreadState* tstate) noexcept {
 }
 
 extern "C" PyThreadState* PyThreadState_Get(void) noexcept {
-    Py_FatalError("Unimplemented");
+    if (_PyThreadState_Current == NULL)
+        Py_FatalError("PyThreadState_Get: no current thread");
+
+    return _PyThreadState_Current;
 }
 
 extern "C" PyThreadState* PyEval_SaveThread(void) noexcept {

src/runtime/code.cpp

@@ -28,12 +28,18 @@ BoxedClass* code_cls;
 
 Box* BoxedCode::name(Box* b, void*) {
     RELEASE_ASSERT(b->cls == code_cls, "");
-    return incref(static_cast<BoxedCode*>(b)->f->source->getName());
+    BoxedCode* code = static_cast<BoxedCode*>(b);
+    if (code->_name)
+        return incref(code->_name);
+    return incref(code->f->source->getName());
 }
 
 Box* BoxedCode::filename(Box* b, void*) {
     RELEASE_ASSERT(b->cls == code_cls, "");
-    return incref(static_cast<BoxedCode*>(b)->f->source->getFn());
+    BoxedCode* code = static_cast<BoxedCode*>(b);
+    if (code->_filename)
+        return incref(code->_filename);
+    return incref(code->f->source->getFn());
 }
 
 Box* BoxedCode::firstlineno(Box* b, void*) {
@@ -41,11 +47,8 @@ Box* BoxedCode::firstlineno(Box* b, void*) {
     BoxedCode* code = static_cast<BoxedCode*>(b);
     FunctionMetadata* md = code->f;
 
-    if (!md->source) {
-        // I don't think it really matters what we return here;
-        // in CPython, builtin functions don't have code objects.
-        return boxInt(-1);
-    }
+    if (!md || !md->source)
+        return boxInt(code->_firstline);
 
     if (md->source->ast->lineno == (uint32_t)-1)
         return boxInt(-1);
@@ -55,7 +58,6 @@ Box* BoxedCode::firstlineno(Box* b, void*) {
 
 Box* BoxedCode::argcount(Box* b, void*) {
     RELEASE_ASSERT(b->cls == code_cls, "");
-
     return boxInt(static_cast<BoxedCode*>(b)->f->num_args);
 }
 
@@ -96,14 +98,91 @@ Box* BoxedCode::flags(Box* b, void*) {
     return boxInt(flags);
 }
 
+int BoxedCode::traverse(Box* self, visitproc visit, void *arg) noexcept {
+    Py_FatalError("untested");
+
+    BoxedCode* o = static_cast<BoxedCode*>(self);
+    Py_VISIT(o->_filename);
+    Py_VISIT(o->_name);
+    return 0;
+}
+
+void BoxedCode::dealloc(Box* b) noexcept {
+    Py_FatalError("untested");
+
+    BoxedCode* o = static_cast<BoxedCode*>(b);
+    Py_XDECREF(o->_filename);
+    Py_XDECREF(o->_name);
+    PyObject_DEL(o);
+}
+
 FunctionMetadata* metadataFromCode(Box* code) {
     assert(code->cls == code_cls);
     return static_cast<BoxedCode*>(code)->f;
 }
 
-extern "C" PyCodeObject* PyCode_New(int, int, int, int, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*,
-                                    PyObject*, PyObject*, PyObject*, int, PyObject*) noexcept {
-    RELEASE_ASSERT(0, "not implemented");
+extern "C" PyCodeObject* PyCode_New(int argcount, int nlocals, int stacksize, int flags, PyObject* code,
+                                    PyObject* consts, PyObject* names, PyObject* varnames, PyObject* freevars,
+                                    PyObject* cellvars, PyObject* filename, PyObject* name, int firstlineno,
+                                    PyObject* lnotab) noexcept {
+    // Check if this is a dummy code object like PyCode_NewEmpty generates.
+    // Because we currently support dummy ones only.
+    bool is_dummy = argcount == 0 && nlocals == 0 && stacksize == 0 && flags == 0;
+    is_dummy = is_dummy && code == EmptyString && lnotab == EmptyString;
+    for (auto&& var : { consts, names, varnames, freevars, cellvars })
+        is_dummy = is_dummy && var == EmptyTuple;
+    RELEASE_ASSERT(is_dummy, "not implemented");
+    // ok this is an empty/dummy code object
+
+    RELEASE_ASSERT(PyString_Check(filename), "");
+    RELEASE_ASSERT(PyString_Check(name), "");
+
+    return (PyCodeObject*)new BoxedCode(filename, name, firstlineno);
+}
+
+extern "C" PyCodeObject* PyCode_NewEmpty(const char* filename, const char* funcname, int firstlineno) noexcept {
+    static PyObject* emptystring = NULL;
+    static PyObject* nulltuple = NULL;
+    PyObject* filename_ob = NULL;
+    PyObject* funcname_ob = NULL;
+    PyCodeObject* result = NULL;
+    if (emptystring == NULL) {
+        emptystring = PyGC_RegisterStaticConstant(PyString_FromString(""));
+        if (emptystring == NULL)
+            goto failed;
+    }
+    if (nulltuple == NULL) {
+        nulltuple = PyGC_RegisterStaticConstant(PyTuple_New(0));
+        if (nulltuple == NULL)
+            goto failed;
+    }
+    funcname_ob = PyString_FromString(funcname);
+    if (funcname_ob == NULL)
+        goto failed;
+    filename_ob = PyString_FromString(filename);
+    if (filename_ob == NULL)
+        goto failed;
+
+    result = PyCode_New(0,           /* argcount */
+                        0,           /* nlocals */
+                        0,           /* stacksize */
+                        0,           /* flags */
+                        emptystring, /* code */
+                        nulltuple,   /* consts */
+                        nulltuple,   /* names */
+                        nulltuple,   /* varnames */
+                        nulltuple,   /* freevars */
+                        nulltuple,   /* cellvars */
+                        filename_ob, /* filename */
+                        funcname_ob, /* name */
+                        firstlineno, /* firstlineno */
+                        emptystring  /* lnotab */
+                        );
+
+failed:
+    Py_XDECREF(funcname_ob);
+    Py_XDECREF(filename_ob);
+    return result;
 }
 
 extern "C" int PyCode_GetArgCount(PyCodeObject* op) noexcept {
@@ -122,7 +201,7 @@ extern "C" PyObject* PyCode_GetName(PyCodeObject* op) noexcept {
 
 void setupCode() {
     code_cls
-        = BoxedClass::create(type_cls, object_cls, 0, 0, sizeof(BoxedCode), false, "code", true, NULL, NULL, false);
+        = BoxedClass::create(type_cls, object_cls, 0, 0, sizeof(BoxedCode), false, "code", false, (destructor)BoxedCode::dealloc, NULL, true, (traverseproc)BoxedCode::traverse, NOCLEAR);
 
     code_cls->giveAttrBorrowed("__new__", None); // Hacky way of preventing users from instantiating this
 

src/runtime/code.h

@@ -24,8 +24,16 @@ namespace pyston {
 class BoxedCode : public Box {
 public:
     FunctionMetadata* f;
+    Box* _filename;
+    Box* _name;
+    int _firstline;
 
-    BoxedCode(FunctionMetadata* f) : f(f) {}
+    BoxedCode(FunctionMetadata* f) : f(f), _filename(NULL), _name(NULL), _firstline(-1) {}
+    BoxedCode(Box* filename, Box* name, int firstline)
+        : f(NULL), _filename(filename), _name(name), _firstline(firstline) {
+        Py_XINCREF(filename);
+        Py_XINCREF(name);
+    }
 
     DEFAULT_CLASS(code_cls);
 
@@ -37,6 +45,9 @@ public:
     static Box* argcount(Box* b, void*);
     static Box* varnames(Box* b, void*);
     static Box* flags(Box* b, void*);
+
+    static int traverse(Box* self, visitproc visit, void *arg) noexcept;
+    static void dealloc(Box* b) noexcept;
 };
 }
 

src/runtime/frame.cpp

@@ -13,6 +13,8 @@
 // limitations under the License.
 
 #include "Python.h"
+
+#include "frameobject.h"
 #include "pythread.h"
 
 #include "codegen/unwinding.h"
@@ -34,9 +36,7 @@ class BoxedFrame : public Box {
 private:
     // Call boxFrame to get a BoxedFrame object.
     BoxedFrame(FrameInfo* frame_info) __attribute__((visibility("default")))
-    : frame_info(frame_info), _back(NULL), _code(NULL), _globals(NULL), _locals(NULL), _stmt(NULL) {
-        assert(frame_info);
-    }
+    : frame_info(frame_info), _back(NULL), _code(NULL), _globals(NULL), _locals(NULL), _linenumber(-1) {}
 
 public:
     FrameInfo* frame_info;
@@ -46,7 +46,7 @@ public:
     Box* _globals;
     Box* _locals;
 
-    AST_stmt* _stmt;
+    int _linenumber;
 
 
     bool hasExited() const { return frame_info == NULL; }
@@ -129,7 +129,7 @@ public:
         auto f = static_cast<BoxedFrame*>(obj);
 
         if (f->hasExited())
-            return boxInt(f->_stmt->lineno);
+            return boxInt(f->_linenumber);
 
         AST_stmt* stmt = f->frame_info->stmt;
         return boxInt(stmt->lineno);
@@ -145,7 +145,7 @@ public:
         autoDecref(globals(this, NULL));
         assert(!_locals);
         _locals = locals(this, NULL);
-        _stmt = frame_info->stmt;
+        _linenumber = frame_info->stmt->lineno;
 
         frame_info = NULL; // this means exited == true
         assert(hasExited());
@@ -184,6 +184,16 @@ public:
         Py_CLEAR(o->_locals);
         return 0;
     }
+
+    static Box* boxFrame(Box* back, BoxedCode* code, Box* globals, Box* locals) {
+        BoxedFrame* frame = new BoxedFrame(NULL);
+        frame->_back = back;
+        frame->_code = (Box*)code;
+        frame->_globals = globals;
+        frame->_locals = locals;
+        frame->_linenumber = -1;
+        return frame;
+    }
 };
 
 extern "C" int PyFrame_ClearFreeList(void) noexcept {
@@ -255,6 +265,23 @@ extern "C" int PyFrame_GetLineNumber(PyFrameObject* _f) noexcept {
     return autoDecref(lineno)->n;
 }
 
+extern "C" void PyFrame_SetLineNumber(PyFrameObject* _f, int linenumber) noexcept {
+    BoxedFrame* f = (BoxedFrame*)_f;
+    RELEASE_ASSERT(f->hasExited(),
+                   "if this frame did not exit yet the line number may get overwriten, may be a problem?");
+    f->_linenumber = linenumber;
+}
+
+extern "C" PyFrameObject* PyFrame_New(PyThreadState* tstate, PyCodeObject* code, PyObject* globals,
+                                      PyObject* locals) noexcept {
+    RELEASE_ASSERT(tstate == &cur_thread_state, "");
+
+    RELEASE_ASSERT(PyCode_Check((Box*)code), "");
+    RELEASE_ASSERT(!globals || PyDict_Check(globals) || globals->cls == attrwrapper_cls, "%s", globals->cls->tp_name);
+    RELEASE_ASSERT(!locals || PyDict_Check(locals), "%s", locals->cls->tp_name);
+    return (PyFrameObject*)BoxedFrame::boxFrame(getFrame(0), (BoxedCode*)code, globals, locals);
+}
+
 extern "C" PyObject* PyFrame_GetGlobals(PyFrameObject* f) noexcept {
     return BoxedFrame::globals((Box*)f, NULL);
 }

test/extra/lxml_test.py

@@ -16,7 +16,7 @@ def install_and_test_lxml():
     subprocess.check_call(["tar", "-zxf", "Cython-0.22.tar.gz"], cwd=SRC_DIR)
 
     CYTHON_DIR = os.path.abspath(os.path.join(SRC_DIR, "Cython-0.22"))
-    PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "integration", "Cython_0001-Pyston-change-we-don-t-support-custom-traceback-entr.patch"))
+    PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "integration", "Cython-0.22.patch"))
     subprocess.check_call(["patch", "-p1", "--input=" + PATCH_FILE], cwd=CYTHON_DIR)
     print "Applied Cython patch"
     subprocess.check_call([PYTHON_EXE, "setup.py", "install"], cwd=CYTHON_DIR)

test/integration/Cython_0001-Pyston-change-we-don-t-support-custom-traceback-entr.patch → test/integration/Cython-0.22.patch

@@ -74,27 +74,20 @@ index 9cc38f0..ab05ad1 100644
  // PyPy does not have this function
  static PyObject * __Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) {
 diff --git a/Cython/Utility/Exceptions.c b/Cython/Utility/Exceptions.c
-index 354a776..8af3cb7 100644
+index 354a776..8af3cb7 100644 static void __Pyx_AddTraceback(const char *funcname, int c_line,
 --- a/Cython/Utility/Exceptions.c
 +++ b/Cython/Utility/Exceptions.c
-@@ -450,7 +450,8 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line,
- /////////////// AddTraceback ///////////////
- //@requires: ModuleSetupCode.c::CodeObjectCache
- //@substitute: naming
--
-+// Pyston change: We don't support custom traceback entries currently
-+#if 0
- #include "compile.h"
- #include "frameobject.h"
- #include "traceback.h"
-@@ -534,3 +535,7 @@ bad:
+@@ -528,7 +528,9 @@
+         0                    /*PyObject *locals*/
+     );
+     if (!py_frame) goto bad;
+-    py_frame->f_lineno = py_line;
++    // Pyston change:
++    // py_frame->f_lineno = py_line;
++    PyFrame_SetLineNumber(py_frame, py_line);
+     PyTraceBack_Here(py_frame);
+ bad:
      Py_XDECREF(py_code);
-     Py_XDECREF(py_frame);
- }
-+#else
-+static void __Pyx_AddTraceback(const char *funcname, int c_line, int py_line, const char *filename) {
-+}
-+#endif
 diff --git a/Cython/Utility/Generator.c b/Cython/Utility/Generator.c
 index 0310570..70e550c 100644
 --- a/Cython/Utility/Generator.c
@@ -205,3 +198,4 @@ index 6477fb2..75dcdda 100644
 -- 
 1.9.1
 
+

test/integration/numpy_test.py

@@ -69,7 +69,7 @@ if not os.path.exists(CYTHON_DIR):
     subprocess.check_call(["wget", url], cwd=SRC_DIR)
     subprocess.check_call(["tar", "-zxf", "Cython-0.22.tar.gz"], cwd=SRC_DIR)
 
-    PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "Cython_0001-Pyston-change-we-don-t-support-custom-traceback-entr.patch"))
+    PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "Cython-0.22.patch"))
     subprocess.check_call(["patch", "-p1", "--input=" + PATCH_FILE], cwd=CYTHON_DIR)
     print ">>> Applied Cython patch"
 

test/tests/ctypes_traceback.py

+from ctypes import *
+libc = CDLL("libc.so.6")
+qsort = libc.qsort
+qsort.restype = None
+CMPFUNC = CFUNCTYPE(c_int, POINTER(c_int), POINTER(c_int))
+def py_cmp_func(a, b):
+    1/0
+cmp_func = CMPFUNC(py_cmp_func)
+
+IntArray3 = c_int * 3
+ia = IntArray3(1, 2, 3)
+qsort(ia, len(ia), sizeof(c_int), cmp_func)
+print "finished"