Merge pull request #1117 from undingen/custom_tb

add support for custom traceback entries

from_cpython/Include/Python.h

@@ -74,7 +74,6 @@
 #include "classobject.h"
 #include "cobject.h"
 #include "fileobject.h"
-#include "frameobject.h"
 #include "pycapsule.h"
 #include "traceback.h"
 #include "sliceobject.h"

from_cpython/Include/frameobject.h

@@ -58,12 +58,11 @@ PyAPI_DATA(PyTypeObject) PyFrame_Type;
 
 #define PyFrame_Check(op) ((op)->ob_type == &PyFrame_Type)
 #define PyFrame_IsRestricted(f) \
-	((f)->f_builtins != (f)->f_tstate->interp->builtins)
+    ((f)->f_builtins != (f)->f_tstate->interp->builtins)
 
 PyAPI_FUNC(PyFrameObject *) PyFrame_New(PyThreadState *, PyCodeObject *,
                                        PyObject *, PyObject *);
 
-
 /* The rest of the interface is specific for frame objects */
 
 /* Block management functions */
@@ -89,8 +88,12 @@ PyAPI_DATA(PyTypeObject*) frame_cls;
 #define PyFrame_Type (*frame_cls)
 #define PyFrame_Check(op) (((PyObject*)op)->ob_type == &PyFrame_Type)
 
+PyAPI_FUNC(PyFrameObject *) PyFrame_New(PyThreadState *, PyCodeObject *,
+                                       PyObject *, PyObject *) PYSTON_NOEXCEPT;
+
 /* Return the line of code the frame is currently executing. */
 PyAPI_FUNC(int) PyFrame_GetLineNumber(PyFrameObject *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(void) PyFrame_SetLineNumber(PyFrameObject *, int line_number) PYSTON_NOEXCEPT;
 // Pyston changes: add a function to get globals
 PyAPI_FUNC(PyObject *) PyFrame_GetGlobals(PyFrameObject *) PYSTON_NOEXCEPT;
 // Pyston changes: add a function to get the code object

from_cpython/Modules/_ctypes/callbacks.c

@@ -5,9 +5,7 @@
 #include "Python.h"
 #include "compile.h" /* required only for 2.3, as it seems */
 
-// Pyston change: We don't have this file and commented out the function that needs it, but
-// we may want to support that function in the future.
-//#include "frameobject.h"
+#include "frameobject.h"
 
 #include <ffi.h>
 #ifdef MS_WIN32
@@ -152,9 +150,6 @@ failed:
 /* after code that pyrex generates */
 void _ctypes_add_traceback(char *funcname, char *filename, int lineno)
 {
-    // TODO: Pyston change:
-    // Supporting this will require frameobject.h
-#if 0
     PyObject *py_globals = 0;
     PyCodeObject *py_code = 0;
     PyFrameObject *py_frame = 0;
@@ -170,15 +165,16 @@ void _ctypes_add_traceback(char *funcname, char *filename, int lineno)
         0                    /*PyObject *locals*/
         );
     if (!py_frame) goto bad;
-    py_frame->f_lineno = lineno;
+
+    // Pyston change:
+    // py_frame->f_lineno = lineno;
+    PyFrame_SetLineNumber(py_frame, lineno);
+
     PyTraceBack_Here(py_frame);
   bad:
     Py_XDECREF(py_globals);
     Py_XDECREF(py_code);
     Py_XDECREF(py_frame);
-#else
-    assert(false);
-#endif
 }
 
 #ifdef MS_WIN32

from_cpython/Python/traceback.c

@@ -126,8 +126,6 @@ PyTraceBack_Here(PyFrameObject *frame)
 int
 PyTraceBack_Here_Tb(PyFrameObject *frame, PyTracebackObject** tb)
 {
-    if ((PyObject*)*tb == Py_None)
-        *tb = NULL;
     *tb = newtracebackobject(*tb, frame);
     if (*tb == NULL)
         return -1;

src/codegen/unwinding.cpp

@@ -719,12 +719,12 @@ ExcInfo* getFrameExcInfo() {
 
     if (!copy_from_exc->type) {
         // No exceptions found:
-        *copy_from_exc = ExcInfo(None, None, None);
+        *copy_from_exc = ExcInfo(None, None, NULL);
     }
 
     assert(gc::isValidGCObject(copy_from_exc->type));
     assert(gc::isValidGCObject(copy_from_exc->value));
-    assert(gc::isValidGCObject(copy_from_exc->traceback));
+    assert(!copy_from_exc->traceback || gc::isValidGCObject(copy_from_exc->traceback));
 
     for (auto* ex : to_update) {
         *ex = *copy_from_exc;

src/runtime/builtin_modules/sys.cpp

@@ -48,19 +48,18 @@ Box* sysExcInfo() {
     ExcInfo* exc = getFrameExcInfo();
     assert(exc->type);
     assert(exc->value);
-    assert(exc->traceback);
-    return BoxedTuple::create({ exc->type, exc->value, exc->traceback });
+    Box* tb = exc->traceback ? exc->traceback : None;
+    return BoxedTuple::create({ exc->type, exc->value, tb });
 }
 
 Box* sysExcClear() {
     ExcInfo* exc = getFrameExcInfo();
     assert(exc->type);
     assert(exc->value);
-    assert(exc->traceback);
 
     exc->type = None;
     exc->value = None;
-    exc->traceback = None;
+    exc->traceback = NULL;
 
     return None;
 }

src/runtime/capi.cpp

@@ -715,8 +715,6 @@ void checkAndThrowCAPIException() {
             value = None;
 
         Box* tb = cur_thread_state.curexc_traceback;
-        if (!tb)
-            tb = None;
 
         // Make sure to call PyErr_Clear() *before* normalizing the exception, since otherwise
         // the normalization can think that it had raised an exception, resulting to a call
@@ -736,7 +734,7 @@ void checkAndThrowCAPIException() {
 
         RELEASE_ASSERT(value->cls == type, "unsupported");
 
-        if (tb != None)
+        if (tb)
             throw ExcInfo(value->cls, value, tb);
         raiseExc(value);
     }
@@ -760,7 +758,7 @@ extern "C" void PyErr_SetExcInfo(PyObject* type, PyObject* value, PyObject* trac
     ExcInfo* exc = getFrameExcInfo();
     exc->type = type ? type : None;
     exc->value = value ? value : None;
-    exc->traceback = traceback ? traceback : None;
+    exc->traceback = traceback;
 }
 
 extern "C" const char* PyExceptionClass_Name(PyObject* o) noexcept {
@@ -1658,7 +1656,10 @@ extern "C" void PyEval_ReleaseThread(PyThreadState* tstate) noexcept {
 }
 
 extern "C" PyThreadState* PyThreadState_Get(void) noexcept {
-    Py_FatalError("Unimplemented");
+    if (_PyThreadState_Current == NULL)
+        Py_FatalError("PyThreadState_Get: no current thread");
+
+    return _PyThreadState_Current;
 }
 
 extern "C" PyThreadState* PyEval_SaveThread(void) noexcept {

src/runtime/code.cpp

@@ -29,16 +29,26 @@ BoxedClass* code_cls;
 void BoxedCode::gcHandler(GCVisitor* v, Box* b) {
     assert(b->cls == code_cls);
     Box::gcHandler(v, b);
+
+    BoxedCode* code = (BoxedCode*)b;
+    v->visit(&code->_filename);
+    v->visit(&code->_name);
 }
 
 Box* BoxedCode::name(Box* b, void*) {
     RELEASE_ASSERT(b->cls == code_cls, "");
-    return static_cast<BoxedCode*>(b)->f->source->getName();
+    BoxedCode* code = static_cast<BoxedCode*>(b);
+    if (code->_name)
+        return code->_name;
+    return code->f->source->getName();
 }
 
 Box* BoxedCode::filename(Box* b, void*) {
     RELEASE_ASSERT(b->cls == code_cls, "");
-    return static_cast<BoxedCode*>(b)->f->source->getFn();
+    BoxedCode* code = static_cast<BoxedCode*>(b);
+    if (code->_filename)
+        return code->_filename;
+    return code->f->source->getFn();
 }
 
 Box* BoxedCode::firstlineno(Box* b, void*) {
@@ -46,11 +56,8 @@ Box* BoxedCode::firstlineno(Box* b, void*) {
     BoxedCode* code = static_cast<BoxedCode*>(b);
     FunctionMetadata* md = code->f;
 
-    if (!md->source) {
-        // I don't think it really matters what we return here;
-        // in CPython, builtin functions don't have code objects.
-        return boxInt(-1);
-    }
+    if (!md || !md->source)
+        return boxInt(code->_firstline);
 
     if (md->source->ast->lineno == (uint32_t)-1)
         return boxInt(-1);
@@ -105,9 +112,68 @@ FunctionMetadata* metadataFromCode(Box* code) {
     return static_cast<BoxedCode*>(code)->f;
 }
 
-extern "C" PyCodeObject* PyCode_New(int, int, int, int, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*,
-                                    PyObject*, PyObject*, PyObject*, int, PyObject*) noexcept {
-    RELEASE_ASSERT(0, "not implemented");
+extern "C" PyCodeObject* PyCode_New(int argcount, int nlocals, int stacksize, int flags, PyObject* code,
+                                    PyObject* consts, PyObject* names, PyObject* varnames, PyObject* freevars,
+                                    PyObject* cellvars, PyObject* filename, PyObject* name, int firstlineno,
+                                    PyObject* lnotab) noexcept {
+    // Check if this is a dummy code object like PyCode_NewEmpty generates.
+    // Because we currently support dummy ones only.
+    bool is_dummy = argcount == 0 && nlocals == 0 && stacksize == 0 && flags == 0;
+    is_dummy = is_dummy && code == EmptyString && lnotab == EmptyString;
+    for (auto&& var : { consts, names, varnames, freevars, cellvars })
+        is_dummy = is_dummy && var == EmptyTuple;
+    RELEASE_ASSERT(is_dummy, "not implemented");
+    // ok this is an empty/dummy code object
+
+    RELEASE_ASSERT(PyString_Check(filename), "");
+    RELEASE_ASSERT(PyString_Check(name), "");
+
+    return (PyCodeObject*)new BoxedCode(filename, name, firstlineno);
+}
+
+extern "C" PyCodeObject* PyCode_NewEmpty(const char* filename, const char* funcname, int firstlineno) noexcept {
+    static PyObject* emptystring = NULL;
+    static PyObject* nulltuple = NULL;
+    PyObject* filename_ob = NULL;
+    PyObject* funcname_ob = NULL;
+    PyCodeObject* result = NULL;
+    if (emptystring == NULL) {
+        emptystring = PyString_FromString("");
+        if (emptystring == NULL)
+            goto failed;
+    }
+    if (nulltuple == NULL) {
+        nulltuple = PyTuple_New(0);
+        if (nulltuple == NULL)
+            goto failed;
+    }
+    funcname_ob = PyString_FromString(funcname);
+    if (funcname_ob == NULL)
+        goto failed;
+    filename_ob = PyString_FromString(filename);
+    if (filename_ob == NULL)
+        goto failed;
+
+    result = PyCode_New(0,           /* argcount */
+                        0,           /* nlocals */
+                        0,           /* stacksize */
+                        0,           /* flags */
+                        emptystring, /* code */
+                        nulltuple,   /* consts */
+                        nulltuple,   /* names */
+                        nulltuple,   /* varnames */
+                        nulltuple,   /* freevars */
+                        nulltuple,   /* cellvars */
+                        filename_ob, /* filename */
+                        funcname_ob, /* name */
+                        firstlineno, /* firstlineno */
+                        emptystring  /* lnotab */
+                        );
+
+failed:
+    Py_XDECREF(funcname_ob);
+    Py_XDECREF(filename_ob);
+    return result;
 }
 
 extern "C" int PyCode_GetArgCount(PyCodeObject* op) noexcept {

src/runtime/code.h

@@ -24,8 +24,13 @@ namespace pyston {
 class BoxedCode : public Box {
 public:
     FunctionMetadata* f;
+    Box* _filename;
+    Box* _name;
+    int _firstline;
 
-    BoxedCode(FunctionMetadata* f) : f(f) {}
+    BoxedCode(FunctionMetadata* f) : f(f), _filename(NULL), _name(NULL), _firstline(-1) {}
+    BoxedCode(Box* filename, Box* name, int firstline)
+        : f(NULL), _filename(filename), _name(name), _firstline(firstline) {}
 
     DEFAULT_CLASS(code_cls);
 

src/runtime/cxx_unwind.cpp

@@ -73,10 +73,10 @@ namespace pyston {
 
 void checkExcInfo(const ExcInfo* exc) {
     assert(exc);
-    assert(exc->type && exc->value && exc->traceback);
+    assert(exc->type && exc->value);
     ASSERT(gc::isValidGCObject(exc->type), "%p", exc->type);
     ASSERT(gc::isValidGCObject(exc->value), "%p", exc->value);
-    ASSERT(gc::isValidGCObject(exc->traceback), "%p", exc->traceback);
+    ASSERT(!exc->traceback || gc::isValidGCObject(exc->traceback), "%p", exc->traceback);
 }
 
 static StatCounter us_unwind_loop("us_unwind_loop");

src/runtime/exceptions.cpp

@@ -27,7 +27,7 @@ namespace pyston {
 
 void raiseExc(Box* exc_obj) {
     assert(!PyErr_Occurred());
-    throw ExcInfo(exc_obj->cls, exc_obj, None);
+    throw ExcInfo(exc_obj->cls, exc_obj, NULL);
 }
 
 // Have a special helper function for syntax errors, since we want to include the location
@@ -46,7 +46,7 @@ void raiseSyntaxError(const char* msg, int lineno, int col_offset, llvm::StringR
         auto args = BoxedTuple::create({ boxString(file), boxInt(lineno), None, loc });
         Box* exc = runtimeCall(SyntaxError, ArgPassSpec(2), boxString(msg), args, NULL, NULL, NULL);
         assert(!PyErr_Occurred());
-        throw ExcInfo(exc->cls, exc, None);
+        throw ExcInfo(exc->cls, exc, NULL);
     } else {
         PyErr_SetString(SyntaxError, msg);
         PyErr_SyntaxLocation(file.str().c_str(), lineno);
@@ -140,10 +140,6 @@ ExcInfo excInfoForRaise(Box* type, Box* value, Box* tb) {
 
     assert(PyExceptionClass_Check(type));
 
-    if (tb == NULL) {
-        tb = None;
-    }
-
     return ExcInfo(type, value, tb);
 }
 
@@ -169,7 +165,7 @@ extern "C" void raise0_capi(ExcInfo* frame_exc_info) noexcept {
         frame_exc_info->type = TypeError;
         frame_exc_info->value
             = boxString("exceptions must be old-style classes or derived from BaseException, not NoneType");
-        frame_exc_info->traceback = None;
+        frame_exc_info->traceback = NULL;
         PyErr_NormalizeException(&frame_exc_info->type, &frame_exc_info->value, &frame_exc_info->traceback);
     }
 

src/runtime/frame.cpp

@@ -13,6 +13,8 @@
 // limitations under the License.
 
 #include "Python.h"
+
+#include "frameobject.h"
 #include "pythread.h"
 
 #include "codegen/unwinding.h"
@@ -33,9 +35,7 @@ class BoxedFrame : public Box {
 private:
     // Call boxFrame to get a BoxedFrame object.
     BoxedFrame(FrameInfo* frame_info) __attribute__((visibility("default")))
-    : frame_info(frame_info), _back(NULL), _code(NULL), _globals(NULL), _locals(NULL), _stmt(NULL) {
-        assert(frame_info);
-    }
+    : frame_info(frame_info), _back(NULL), _code(NULL), _globals(NULL), _locals(NULL), _linenumber(-1) {}
 
 public:
     FrameInfo* frame_info;
@@ -45,7 +45,7 @@ public:
     Box* _globals;
     Box* _locals;
 
-    AST_stmt* _stmt;
+    int _linenumber;
 
 
     bool hasExited() const { return frame_info == NULL; }
@@ -132,7 +132,7 @@ public:
         auto f = static_cast<BoxedFrame*>(obj);
 
         if (f->hasExited())
-            return boxInt(f->_stmt->lineno);
+            return boxInt(f->_linenumber);
 
         AST_stmt* stmt = f->frame_info->stmt;
         return boxInt(stmt->lineno);
@@ -146,7 +146,7 @@ public:
         _code = code(this, NULL);
         _globals = globals(this, NULL);
         _locals = locals(this, NULL);
-        _stmt = frame_info->stmt;
+        _linenumber = frame_info->stmt->lineno;
 
         frame_info = NULL; // this means exited == true
         assert(hasExited());
@@ -160,6 +160,16 @@ public:
         assert(fi->frame_obj->cls == frame_cls);
         return fi->frame_obj;
     }
+
+    static Box* boxFrame(Box* back, BoxedCode* code, Box* globals, Box* locals) {
+        BoxedFrame* frame = new BoxedFrame(NULL);
+        frame->_back = back;
+        frame->_code = (Box*)code;
+        frame->_globals = globals;
+        frame->_locals = locals;
+        frame->_linenumber = -1;
+        return frame;
+    }
 };
 
 
@@ -196,6 +206,23 @@ extern "C" int PyFrame_GetLineNumber(PyFrameObject* _f) noexcept {
     return lineno->n;
 }
 
+extern "C" void PyFrame_SetLineNumber(PyFrameObject* _f, int linenumber) noexcept {
+    BoxedFrame* f = (BoxedFrame*)_f;
+    RELEASE_ASSERT(f->hasExited(),
+                   "if this frame did not exit yet the line number may get overwriten, may be a problem?");
+    f->_linenumber = linenumber;
+}
+
+extern "C" PyFrameObject* PyFrame_New(PyThreadState* tstate, PyCodeObject* code, PyObject* globals,
+                                      PyObject* locals) noexcept {
+    RELEASE_ASSERT(tstate == &cur_thread_state, "");
+
+    RELEASE_ASSERT(PyCode_Check((Box*)code), "");
+    RELEASE_ASSERT(!globals || PyDict_Check(globals) || globals->cls == attrwrapper_cls, "%s", globals->cls->tp_name);
+    RELEASE_ASSERT(!locals || PyDict_Check(locals), "%s", locals->cls->tp_name);
+    return (PyFrameObject*)BoxedFrame::boxFrame(getFrame(0), (BoxedCode*)code, globals, locals);
+}
+
 extern "C" PyObject* PyFrame_GetGlobals(PyFrameObject* f) noexcept {
     return BoxedFrame::globals((Box*)f, NULL);
 }

test/extra/lxml_test.py

@@ -16,7 +16,7 @@ def install_and_test_lxml():
     subprocess.check_call(["tar", "-zxf", "Cython-0.22.tar.gz"], cwd=SRC_DIR)
 
     CYTHON_DIR = os.path.abspath(os.path.join(SRC_DIR, "Cython-0.22"))
-    PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "integration", "Cython_0001-Pyston-change-we-don-t-support-custom-traceback-entr.patch"))
+    PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "integration", "Cython-0.22.patch"))
     subprocess.check_call(["patch", "-p1", "--input=" + PATCH_FILE], cwd=CYTHON_DIR)
     print "Applied Cython patch"
     subprocess.check_call([PYTHON_EXE, "setup.py", "install"], cwd=CYTHON_DIR)

test/integration/Cython_0001-Pyston-change-we-don-t-support-custom-traceback-entr.patch → test/integration/Cython-0.22.patch

@@ -74,27 +74,20 @@ index 9cc38f0..ab05ad1 100644
  // PyPy does not have this function
  static PyObject * __Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) {
 diff --git a/Cython/Utility/Exceptions.c b/Cython/Utility/Exceptions.c
-index 354a776..8af3cb7 100644
+index 354a776..8af3cb7 100644 static void __Pyx_AddTraceback(const char *funcname, int c_line,
 --- a/Cython/Utility/Exceptions.c
 +++ b/Cython/Utility/Exceptions.c
-@@ -450,7 +450,8 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line,
- /////////////// AddTraceback ///////////////
- //@requires: ModuleSetupCode.c::CodeObjectCache
- //@substitute: naming
--
-+// Pyston change: We don't support custom traceback entries currently
-+#if 0
- #include "compile.h"
- #include "frameobject.h"
- #include "traceback.h"
-@@ -534,3 +535,7 @@ bad:
+@@ -528,7 +528,9 @@
+         0                    /*PyObject *locals*/
+     );
+     if (!py_frame) goto bad;
+-    py_frame->f_lineno = py_line;
++    // Pyston change:
++    // py_frame->f_lineno = py_line;
++    PyFrame_SetLineNumber(py_frame, py_line);
+     PyTraceBack_Here(py_frame);
+ bad:
      Py_XDECREF(py_code);
-     Py_XDECREF(py_frame);
- }
-+#else
-+static void __Pyx_AddTraceback(const char *funcname, int c_line, int py_line, const char *filename) {
-+}
-+#endif
 diff --git a/Cython/Utility/Generator.c b/Cython/Utility/Generator.c
 index 0310570..70e550c 100644
 --- a/Cython/Utility/Generator.c
@@ -205,3 +198,4 @@ index 6477fb2..75dcdda 100644
 -- 
 1.9.1
 
+

test/integration/numpy_test.py

@@ -69,7 +69,7 @@ if not os.path.exists(CYTHON_DIR):
     subprocess.check_call(["wget", url], cwd=SRC_DIR)
     subprocess.check_call(["tar", "-zxf", "Cython-0.22.tar.gz"], cwd=SRC_DIR)
 
-    PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "Cython_0001-Pyston-change-we-don-t-support-custom-traceback-entr.patch"))
+    PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "Cython-0.22.patch"))
     subprocess.check_call(["patch", "-p1", "--input=" + PATCH_FILE], cwd=CYTHON_DIR)
     print ">>> Applied Cython patch"
 

test/tests/ctypes_traceback.py

+from ctypes import *
+libc = CDLL("libc.so.6")
+qsort = libc.qsort
+qsort.restype = None
+CMPFUNC = CFUNCTYPE(c_int, POINTER(c_int), POINTER(c_int))
+def py_cmp_func(a, b):
+    1/0
+cmp_func = CMPFUNC(py_cmp_func)
+
+IntArray3 = c_int * 3
+ia = IntArray3(1, 2, 3)
+qsort(ia, len(ia), sizeof(c_int), cmp_func)
+print "finished"