tracebacks: implement PyFrame_New and PyCode_New / custom traceback entries from the CAPI

it only allows creating empty code objects but this is enough to allow ctypes and cython to create custom traceback entries.

It's not enough for removing the pyexpat and all cython code object workarounds (because they need 'real' code object / frames)

from_cpython/Include/Python.h

@@ -74,7 +74,6 @@
 #include "classobject.h"
 #include "cobject.h"
 #include "fileobject.h"
-#include "frameobject.h"
 #include "pycapsule.h"
 #include "traceback.h"
 #include "sliceobject.h"

from_cpython/Include/frameobject.h

@@ -58,12 +58,11 @@ PyAPI_DATA(PyTypeObject) PyFrame_Type;
 
 #define PyFrame_Check(op) ((op)->ob_type == &PyFrame_Type)
 #define PyFrame_IsRestricted(f) \
-	((f)->f_builtins != (f)->f_tstate->interp->builtins)
+    ((f)->f_builtins != (f)->f_tstate->interp->builtins)
 
 PyAPI_FUNC(PyFrameObject *) PyFrame_New(PyThreadState *, PyCodeObject *,
                                        PyObject *, PyObject *);
 
-
 /* The rest of the interface is specific for frame objects */
 
 /* Block management functions */
@@ -89,8 +88,12 @@ PyAPI_DATA(PyTypeObject*) frame_cls;
 #define PyFrame_Type (*frame_cls)
 #define PyFrame_Check(op) (((PyObject*)op)->ob_type == &PyFrame_Type)
 
+PyAPI_FUNC(PyFrameObject *) PyFrame_New(PyThreadState *, PyCodeObject *,
+                                       PyObject *, PyObject *) PYSTON_NOEXCEPT;
+
 /* Return the line of code the frame is currently executing. */
 PyAPI_FUNC(int) PyFrame_GetLineNumber(PyFrameObject *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(void) PyFrame_SetLineNumber(PyFrameObject *, int line_number) PYSTON_NOEXCEPT;
 // Pyston changes: add a function to get globals
 PyAPI_FUNC(PyObject *) PyFrame_GetGlobals(PyFrameObject *) PYSTON_NOEXCEPT;
 // Pyston changes: add a function to get the code object

from_cpython/Modules/_ctypes/callbacks.c

@@ -5,9 +5,7 @@
 #include "Python.h"
 #include "compile.h" /* required only for 2.3, as it seems */
 
-// Pyston change: We don't have this file and commented out the function that needs it, but
-// we may want to support that function in the future.
-//#include "frameobject.h"
+#include "frameobject.h"
 
 #include <ffi.h>
 #ifdef MS_WIN32
@@ -152,9 +150,6 @@ failed:
 /* after code that pyrex generates */
 void _ctypes_add_traceback(char *funcname, char *filename, int lineno)
 {
-    // TODO: Pyston change:
-    // Supporting this will require frameobject.h
-#if 0
     PyObject *py_globals = 0;
     PyCodeObject *py_code = 0;
     PyFrameObject *py_frame = 0;
@@ -170,15 +165,16 @@ void _ctypes_add_traceback(char *funcname, char *filename, int lineno)
         0                    /*PyObject *locals*/
         );
     if (!py_frame) goto bad;
-    py_frame->f_lineno = lineno;
+
+    // Pyston change:
+    // py_frame->f_lineno = lineno;
+    PyFrame_SetLineNumber(py_frame, lineno);
+
     PyTraceBack_Here(py_frame);
   bad:
     Py_XDECREF(py_globals);
     Py_XDECREF(py_code);
     Py_XDECREF(py_frame);
-#else
-    assert(false);
-#endif
 }
 
 #ifdef MS_WIN32

src/runtime/capi.cpp

@@ -1656,7 +1656,10 @@ extern "C" void PyEval_ReleaseThread(PyThreadState* tstate) noexcept {
 }
 
 extern "C" PyThreadState* PyThreadState_Get(void) noexcept {
-    Py_FatalError("Unimplemented");
+    if (_PyThreadState_Current == NULL)
+        Py_FatalError("PyThreadState_Get: no current thread");
+
+    return _PyThreadState_Current;
 }
 
 extern "C" PyThreadState* PyEval_SaveThread(void) noexcept {

src/runtime/code.cpp

@@ -29,16 +29,26 @@ BoxedClass* code_cls;
 void BoxedCode::gcHandler(GCVisitor* v, Box* b) {
     assert(b->cls == code_cls);
     Box::gcHandler(v, b);
+
+    BoxedCode* code = (BoxedCode*)b;
+    v->visit(&code->_filename);
+    v->visit(&code->_name);
 }
 
 Box* BoxedCode::name(Box* b, void*) {
     RELEASE_ASSERT(b->cls == code_cls, "");
-    return static_cast<BoxedCode*>(b)->f->source->getName();
+    BoxedCode* code = static_cast<BoxedCode*>(b);
+    if (code->_name)
+        return code->_name;
+    return code->f->source->getName();
 }
 
 Box* BoxedCode::filename(Box* b, void*) {
     RELEASE_ASSERT(b->cls == code_cls, "");
-    return static_cast<BoxedCode*>(b)->f->source->getFn();
+    BoxedCode* code = static_cast<BoxedCode*>(b);
+    if (code->_filename)
+        return code->_filename;
+    return code->f->source->getFn();
 }
 
 Box* BoxedCode::firstlineno(Box* b, void*) {
@@ -46,11 +56,8 @@ Box* BoxedCode::firstlineno(Box* b, void*) {
     BoxedCode* code = static_cast<BoxedCode*>(b);
     FunctionMetadata* md = code->f;
 
-    if (!md->source) {
-        // I don't think it really matters what we return here;
-        // in CPython, builtin functions don't have code objects.
-        return boxInt(-1);
-    }
+    if (!md || !md->source)
+        return boxInt(code->_firstline);
 
     if (md->source->ast->lineno == (uint32_t)-1)
         return boxInt(-1);
@@ -105,9 +112,68 @@ FunctionMetadata* metadataFromCode(Box* code) {
     return static_cast<BoxedCode*>(code)->f;
 }
 
-extern "C" PyCodeObject* PyCode_New(int, int, int, int, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*,
-                                    PyObject*, PyObject*, PyObject*, int, PyObject*) noexcept {
-    RELEASE_ASSERT(0, "not implemented");
+extern "C" PyCodeObject* PyCode_New(int argcount, int nlocals, int stacksize, int flags, PyObject* code,
+                                    PyObject* consts, PyObject* names, PyObject* varnames, PyObject* freevars,
+                                    PyObject* cellvars, PyObject* filename, PyObject* name, int firstlineno,
+                                    PyObject* lnotab) noexcept {
+    // Check if this is a dummy code object like PyCode_NewEmpty generates.
+    // Because we currently support dummy ones only.
+    bool is_dummy = argcount == 0 && nlocals == 0 && stacksize == 0 && flags == 0;
+    is_dummy = is_dummy && code == EmptyString && lnotab == EmptyString;
+    for (auto&& var : { consts, names, varnames, freevars, cellvars })
+        is_dummy = is_dummy && var == EmptyTuple;
+    RELEASE_ASSERT(is_dummy, "not implemented");
+    // ok this is an empty/dummy code object
+
+    RELEASE_ASSERT(PyString_Check(filename), "");
+    RELEASE_ASSERT(PyString_Check(name), "");
+
+    return (PyCodeObject*)new BoxedCode(filename, name, firstlineno);
+}
+
+extern "C" PyCodeObject* PyCode_NewEmpty(const char* filename, const char* funcname, int firstlineno) noexcept {
+    static PyObject* emptystring = NULL;
+    static PyObject* nulltuple = NULL;
+    PyObject* filename_ob = NULL;
+    PyObject* funcname_ob = NULL;
+    PyCodeObject* result = NULL;
+    if (emptystring == NULL) {
+        emptystring = PyString_FromString("");
+        if (emptystring == NULL)
+            goto failed;
+    }
+    if (nulltuple == NULL) {
+        nulltuple = PyTuple_New(0);
+        if (nulltuple == NULL)
+            goto failed;
+    }
+    funcname_ob = PyString_FromString(funcname);
+    if (funcname_ob == NULL)
+        goto failed;
+    filename_ob = PyString_FromString(filename);
+    if (filename_ob == NULL)
+        goto failed;
+
+    result = PyCode_New(0,           /* argcount */
+                        0,           /* nlocals */
+                        0,           /* stacksize */
+                        0,           /* flags */
+                        emptystring, /* code */
+                        nulltuple,   /* consts */
+                        nulltuple,   /* names */
+                        nulltuple,   /* varnames */
+                        nulltuple,   /* freevars */
+                        nulltuple,   /* cellvars */
+                        filename_ob, /* filename */
+                        funcname_ob, /* name */
+                        firstlineno, /* firstlineno */
+                        emptystring  /* lnotab */
+                        );
+
+failed:
+    Py_XDECREF(funcname_ob);
+    Py_XDECREF(filename_ob);
+    return result;
 }
 
 extern "C" int PyCode_GetArgCount(PyCodeObject* op) noexcept {

src/runtime/code.h

@@ -24,8 +24,13 @@ namespace pyston {
 class BoxedCode : public Box {
 public:
     FunctionMetadata* f;
+    Box* _filename;
+    Box* _name;
+    int _firstline;
 
-    BoxedCode(FunctionMetadata* f) : f(f) {}
+    BoxedCode(FunctionMetadata* f) : f(f), _filename(NULL), _name(NULL), _firstline(-1) {}
+    BoxedCode(Box* filename, Box* name, int firstline)
+        : f(NULL), _filename(filename), _name(name), _firstline(firstline) {}
 
     DEFAULT_CLASS(code_cls);
 

src/runtime/frame.cpp

@@ -13,6 +13,8 @@
 // limitations under the License.
 
 #include "Python.h"
+
+#include "frameobject.h"
 #include "pythread.h"
 
 #include "codegen/unwinding.h"
@@ -33,9 +35,7 @@ class BoxedFrame : public Box {
 private:
     // Call boxFrame to get a BoxedFrame object.
     BoxedFrame(FrameInfo* frame_info) __attribute__((visibility("default")))
-    : frame_info(frame_info), _back(NULL), _code(NULL), _globals(NULL), _locals(NULL), _stmt(NULL) {
-        assert(frame_info);
-    }
+    : frame_info(frame_info), _back(NULL), _code(NULL), _globals(NULL), _locals(NULL), _linenumber(-1) {}
 
 public:
     FrameInfo* frame_info;
@@ -45,7 +45,7 @@ public:
     Box* _globals;
     Box* _locals;
 
-    AST_stmt* _stmt;
+    int _linenumber;
 
 
     bool hasExited() const { return frame_info == NULL; }
@@ -132,7 +132,7 @@ public:
         auto f = static_cast<BoxedFrame*>(obj);
 
         if (f->hasExited())
-            return boxInt(f->_stmt->lineno);
+            return boxInt(f->_linenumber);
 
         AST_stmt* stmt = f->frame_info->stmt;
         return boxInt(stmt->lineno);
@@ -146,7 +146,7 @@ public:
         _code = code(this, NULL);
         _globals = globals(this, NULL);
         _locals = locals(this, NULL);
-        _stmt = frame_info->stmt;
+        _linenumber = frame_info->stmt->lineno;
 
         frame_info = NULL; // this means exited == true
         assert(hasExited());
@@ -160,6 +160,16 @@ public:
         assert(fi->frame_obj->cls == frame_cls);
         return fi->frame_obj;
     }
+
+    static Box* boxFrame(Box* back, BoxedCode* code, Box* globals, Box* locals) {
+        BoxedFrame* frame = new BoxedFrame(NULL);
+        frame->_back = back;
+        frame->_code = (Box*)code;
+        frame->_globals = globals;
+        frame->_locals = locals;
+        frame->_linenumber = -1;
+        return frame;
+    }
 };
 
 
@@ -196,6 +206,23 @@ extern "C" int PyFrame_GetLineNumber(PyFrameObject* _f) noexcept {
     return lineno->n;
 }
 
+extern "C" void PyFrame_SetLineNumber(PyFrameObject* _f, int linenumber) noexcept {
+    BoxedFrame* f = (BoxedFrame*)_f;
+    RELEASE_ASSERT(f->hasExited(),
+                   "if this frame did not exit yet the line number may get overwriten, may be a problem?");
+    f->_linenumber = linenumber;
+}
+
+extern "C" PyFrameObject* PyFrame_New(PyThreadState* tstate, PyCodeObject* code, PyObject* globals,
+                                      PyObject* locals) noexcept {
+    RELEASE_ASSERT(tstate == &cur_thread_state, "");
+
+    RELEASE_ASSERT(PyCode_Check((Box*)code), "");
+    RELEASE_ASSERT(!globals || PyDict_Check(globals) || globals->cls == attrwrapper_cls, "%s", globals->cls->tp_name);
+    RELEASE_ASSERT(!locals || PyDict_Check(locals), "%s", locals->cls->tp_name);
+    return (PyFrameObject*)BoxedFrame::boxFrame(getFrame(0), (BoxedCode*)code, globals, locals);
+}
+
 extern "C" PyObject* PyFrame_GetGlobals(PyFrameObject* f) noexcept {
     return BoxedFrame::globals((Box*)f, NULL);
 }

test/extra/lxml_test.py

@@ -16,7 +16,7 @@ def install_and_test_lxml():
     subprocess.check_call(["tar", "-zxf", "Cython-0.22.tar.gz"], cwd=SRC_DIR)
 
     CYTHON_DIR = os.path.abspath(os.path.join(SRC_DIR, "Cython-0.22"))
-    PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "integration", "Cython_0001-Pyston-change-we-don-t-support-custom-traceback-entr.patch"))
+    PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "integration", "Cython-0.22.patch"))
     subprocess.check_call(["patch", "-p1", "--input=" + PATCH_FILE], cwd=CYTHON_DIR)
     print "Applied Cython patch"
     subprocess.check_call([PYTHON_EXE, "setup.py", "install"], cwd=CYTHON_DIR)

test/integration/Cython_0001-Pyston-change-we-don-t-support-custom-traceback-entr.patch → test/integration/Cython-0.22.patch

@@ -74,27 +74,20 @@ index 9cc38f0..ab05ad1 100644
  // PyPy does not have this function
  static PyObject * __Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) {
 diff --git a/Cython/Utility/Exceptions.c b/Cython/Utility/Exceptions.c
-index 354a776..8af3cb7 100644
+index 354a776..8af3cb7 100644 static void __Pyx_AddTraceback(const char *funcname, int c_line,
 --- a/Cython/Utility/Exceptions.c
 +++ b/Cython/Utility/Exceptions.c
-@@ -450,7 +450,8 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line,
- /////////////// AddTraceback ///////////////
- //@requires: ModuleSetupCode.c::CodeObjectCache
- //@substitute: naming
--
-+// Pyston change: We don't support custom traceback entries currently
-+#if 0
- #include "compile.h"
- #include "frameobject.h"
- #include "traceback.h"
-@@ -534,3 +535,7 @@ bad:
+@@ -528,7 +528,9 @@
+         0                    /*PyObject *locals*/
+     );
+     if (!py_frame) goto bad;
+-    py_frame->f_lineno = py_line;
++    // Pyston change:
++    // py_frame->f_lineno = py_line;
++    PyFrame_SetLineNumber(py_frame, py_line);
+     PyTraceBack_Here(py_frame);
+ bad:
      Py_XDECREF(py_code);
-     Py_XDECREF(py_frame);
- }
-+#else
-+static void __Pyx_AddTraceback(const char *funcname, int c_line, int py_line, const char *filename) {
-+}
-+#endif
 diff --git a/Cython/Utility/Generator.c b/Cython/Utility/Generator.c
 index 0310570..70e550c 100644
 --- a/Cython/Utility/Generator.c
@@ -205,3 +198,4 @@ index 6477fb2..75dcdda 100644
 -- 
 1.9.1
 
+

test/integration/numpy_test.py

@@ -69,7 +69,7 @@ if not os.path.exists(CYTHON_DIR):
     subprocess.check_call(["wget", url], cwd=SRC_DIR)
     subprocess.check_call(["tar", "-zxf", "Cython-0.22.tar.gz"], cwd=SRC_DIR)
 
-    PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "Cython_0001-Pyston-change-we-don-t-support-custom-traceback-entr.patch"))
+    PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "Cython-0.22.patch"))
     subprocess.check_call(["patch", "-p1", "--input=" + PATCH_FILE], cwd=CYTHON_DIR)
     print ">>> Applied Cython patch"
 

test/tests/ctypes_traceback.py

+from ctypes import *
+libc = CDLL("libc.so.6")
+qsort = libc.qsort
+qsort.restype = None
+CMPFUNC = CFUNCTYPE(c_int, POINTER(c_int), POINTER(c_int))
+def py_cmp_func(a, b):
+    1/0
+cmp_func = CMPFUNC(py_cmp_func)
+
+IntArray3 = c_int * 3
+ia = IntArray3(1, 2, 3)
+qsort(ia, len(ia), sizeof(c_int), cmp_func)
+print "finished"