Add lxml test and add PyGILState_Ensure, PyGILState_Release

We pass most of the tests, some of the errors are becasue we don't use refcounting and lxml depends on it (aka pypy has the same issues)

.travis.yml

@@ -48,6 +48,8 @@ addons:
     - python-dev
     - texlive-extra-utils
     - libcurl4-openssl-dev
+    - libxml2-dev 
+    - libxslt1-dev
 
 before_install:
   - if [ "$CC" = "clang" ]; then export CC="clang-3.5" CXX="clang++-3.5"; fi

from_cpython/Include/object.h

@@ -812,10 +812,16 @@ PyAPI_FUNC(void) _Py_AddToAllObjects(PyObject *, int force) PYSTON_NOEXCEPT;
 /* Without Py_TRACE_REFS, there's little enough to do that we expand code
  * inline.
  */
+/* Pyston change: we don't have a refcount
 #define _Py_NewReference(op) (                          \
     _Py_INC_TPALLOCS(op) _Py_COUNT_ALLOCS_COMMA         \
     _Py_INC_REFTOTAL  _Py_REF_DEBUG_COMMA               \
     Py_REFCNT(op) = 1)
+*/
+#define _Py_NewReference(op) (                          \
+    _Py_INC_TPALLOCS(op) _Py_COUNT_ALLOCS_COMMA         \
+    _Py_INC_REFTOTAL  _Py_REF_DEBUG_COMMA               \
+    (void)(op))
 
 #define _Py_ForgetReference(op) _Py_INC_TPFREES(op)
 

from_cpython/Include/pystate.h

@@ -112,6 +112,7 @@ typedef struct _ts {
 #endif
 typedef struct _ts {
     int recursion_depth;
+    int gilstate_counter;
 
     PyObject *curexc_type;
     PyObject *curexc_value;

src/core/threading.cpp

@@ -38,7 +38,7 @@ std::unordered_set<PerThreadSetBase*> PerThreadSetBase::all_instances;
 
 extern "C" {
 __thread PyThreadState cur_thread_state
-    = { 0, NULL, NULL, NULL, NULL }; // not sure if we need to explicitly request zero-initialization
+    = { 0, 1, NULL, NULL, NULL, NULL }; // not sure if we need to explicitly request zero-initialization
 }
 
 PthreadFastMutex threading_lock;
@@ -99,6 +99,16 @@ public:
 
     bool isValid() { return saved; }
 
+    // This is a quick and dirty way to determine if the current thread holds the gil:
+    // the only way it can't (at least for now) is if it had saved its threadstate.
+    // This only works when looking at a thread that is not actively acquiring or releasing
+    // the GIL, so for now just guard on it only being called for the current thread.
+    // TODO It's pretty brittle to reuse the saved flag like this.
+    bool holdsGil() {
+        assert(pthread_self() == this->pthread_id);
+        return !saved;
+    }
+
     ucontext_t* getContext() { return &ucontext; }
 
     void pushGCObject(gc::GCVisitable* obj) { gc_objs_stacks.push_back(obj); }
@@ -227,6 +237,94 @@ static void visitLocalStack(gc::GCVisitor* v) {
     current_internal_thread_state->accept(v);
 }
 
+static void registerThread(bool is_starting_thread) {
+    pthread_t current_thread = pthread_self();
+
+    LOCK_REGION(&threading_lock);
+
+    pthread_attr_t thread_attrs;
+    int code = pthread_getattr_np(current_thread, &thread_attrs);
+    if (code)
+        err(1, NULL);
+
+    void* stack_start;
+    size_t stack_size;
+    code = pthread_attr_getstack(&thread_attrs, &stack_start, &stack_size);
+    RELEASE_ASSERT(code == 0, "");
+
+    pthread_attr_destroy(&thread_attrs);
+
+#if STACK_GROWS_DOWN
+    void* stack_bottom = static_cast<char*>(stack_start) + stack_size;
+#else
+    void* stack_bottom = stack_start;
+#endif
+    current_internal_thread_state = new ThreadStateInternal(stack_bottom, current_thread, &cur_thread_state);
+    current_threads[current_thread] = current_internal_thread_state;
+
+    if (is_starting_thread)
+        num_starting_threads--;
+
+    if (VERBOSITY() >= 2)
+        printf("child initialized; tid=%ld\n", current_thread);
+}
+
+static void unregisterThread() {
+    current_internal_thread_state->assertNoGenerators();
+    {
+        pthread_t current_thread = pthread_self();
+        LOCK_REGION(&threading_lock);
+
+        current_threads.erase(current_thread);
+        if (VERBOSITY() >= 2)
+            printf("thread tid=%ld exited\n", current_thread);
+    }
+    current_internal_thread_state = 0;
+}
+
+extern "C" PyGILState_STATE PyGILState_Ensure(void) noexcept {
+    if (!current_internal_thread_state) {
+        /* Create a new thread state for this thread */
+        registerThread(false);
+        if (current_internal_thread_state == NULL)
+            Py_FatalError("Couldn't create thread-state for new thread");
+
+        acquireGLRead();
+        return PyGILState_UNLOCKED;
+    } else {
+        ++cur_thread_state.gilstate_counter;
+        if (current_internal_thread_state->holdsGil()) {
+            return PyGILState_LOCKED;
+        } else {
+            endAllowThreads();
+            return PyGILState_UNLOCKED;
+        }
+    }
+}
+
+extern "C" void PyGILState_Release(PyGILState_STATE oldstate) noexcept {
+    if (!current_internal_thread_state)
+        Py_FatalError("auto-releasing thread-state, but no thread-state for this thread");
+
+    --cur_thread_state.gilstate_counter;
+    RELEASE_ASSERT(cur_thread_state.gilstate_counter >= 0, "");
+
+    if (oldstate == PyGILState_UNLOCKED) {
+        beginAllowThreads();
+    }
+
+    if (cur_thread_state.gilstate_counter == 0) {
+        assert(oldstate == PyGILState_UNLOCKED);
+        RELEASE_ASSERT(0, "this is currently untested");
+        unregisterThread();
+    }
+}
+
+extern "C" PyThreadState* PyGILState_GetThisThreadState(void) noexcept {
+    Py_FatalError("unimplemented");
+}
+
+
 void visitAllStacks(gc::GCVisitor* v) {
     visitLocalStack(v);
 
@@ -317,51 +415,14 @@ static void* _thread_start(void* _arg) {
     Box* arg3 = arg->arg3;
     delete arg;
 
-    pthread_t current_thread = pthread_self();
-
-    {
-        LOCK_REGION(&threading_lock);
-
-        pthread_attr_t thread_attrs;
-        int code = pthread_getattr_np(current_thread, &thread_attrs);
-        if (code)
-            err(1, NULL);
-
-        void* stack_start;
-        size_t stack_size;
-        code = pthread_attr_getstack(&thread_attrs, &stack_start, &stack_size);
-        RELEASE_ASSERT(code == 0, "");
-
-        pthread_attr_destroy(&thread_attrs);
-
-#if STACK_GROWS_DOWN
-        void* stack_bottom = static_cast<char*>(stack_start) + stack_size;
-#else
-        void* stack_bottom = stack_start;
-#endif
-        current_internal_thread_state = new ThreadStateInternal(stack_bottom, current_thread, &cur_thread_state);
-        current_threads[current_thread] = current_internal_thread_state;
-
-        num_starting_threads--;
-
-        if (VERBOSITY() >= 2)
-            printf("child initialized; tid=%ld\n", current_thread);
-    }
+    registerThread(true);
 
     threading::GLReadRegion _glock;
     assert(!PyErr_Occurred());
 
     void* rtn = start_func(arg1, arg2, arg3);
-    current_internal_thread_state->assertNoGenerators();
 
-    {
-        LOCK_REGION(&threading_lock);
-
-        current_threads.erase(current_thread);
-        if (VERBOSITY() >= 2)
-            printf("thread tid=%ld exited\n", current_thread);
-    }
-    current_internal_thread_state = 0;
+    unregisterThread();
 
     return rtn;
 }

src/runtime/capi.cpp

@@ -767,18 +767,6 @@ finally:
     --tstate->recursion_depth;
 }
 
-extern "C" PyGILState_STATE PyGILState_Ensure(void) noexcept {
-    Py_FatalError("unimplemented");
-}
-
-extern "C" void PyGILState_Release(PyGILState_STATE) noexcept {
-    Py_FatalError("unimplemented");
-}
-
-extern "C" PyThreadState* PyGILState_GetThisThreadState(void) noexcept {
-    Py_FatalError("unimplemented");
-}
-
 void setCAPIException(const ExcInfo& e) {
     cur_thread_state.curexc_type = e.type;
     cur_thread_state.curexc_value = e.value;
@@ -867,9 +855,9 @@ extern "C" void PyErr_GetExcInfo(PyObject** ptype, PyObject** pvalue, PyObject**
 
 extern "C" void PyErr_SetExcInfo(PyObject* type, PyObject* value, PyObject* traceback) noexcept {
     ExcInfo* exc = getFrameExcInfo();
-    exc->type = type;
-    exc->value = value;
-    exc->traceback = traceback;
+    exc->type = type ? type : None;
+    exc->value = value ? value : None;
+    exc->traceback = traceback ? traceback : None;
 }
 
 extern "C" void PyErr_SetString(PyObject* exception, const char* string) noexcept {
@@ -1432,11 +1420,13 @@ extern "C" PyThreadState* PyThreadState_Get(void) noexcept {
 }
 
 extern "C" PyThreadState* PyEval_SaveThread(void) noexcept {
-    Py_FatalError("Unimplemented");
+    beginAllowThreads();
+    return PyThreadState_GET();
 }
 
 extern "C" void PyEval_RestoreThread(PyThreadState* tstate) noexcept {
-    Py_FatalError("Unimplemented");
+    RELEASE_ASSERT(tstate == PyThreadState_GET(), "");
+    endAllowThreads();
 }
 
 extern "C" char* PyModule_GetName(PyObject* m) noexcept {

src/runtime/types.cpp

@@ -1694,8 +1694,7 @@ static void functionSetCode(Box* self, Box* v, void*) {
     BoxedFunction* func = static_cast<BoxedFunction*>(self);
     BoxedCode* code = static_cast<BoxedCode*>(v);
 
-    if (!func->f->source || !code->f->source)
-        raiseExcHelper(TypeError, "__code__ can only be set on python functions");
+    RELEASE_ASSERT(func->f->source && code->f->source, "__code__ can only be set on python functions");
 
     RELEASE_ASSERT(!func->f->internal_callable.get<CXX>() && !func->f->internal_callable.get<CAPI>(),
                    "this could cause invalidation issues");

test/extra/lxml_patch.patch

+From 630b78d79e4e63d221b57726a3bc3855cedf4e28 Mon Sep 17 00:00:00 2001
+From: Marius Wachtler <undingen@gmail.com>
+Date: Tue, 22 Sep 2015 21:21:22 +0100
+Subject: [PATCH] WIP: changes to make lxml work with pyston added a backport
+ of a 3.1.2 patch in order to support the libxml2 travis-ci version + ubuntu
+ 14.04
+ https://github.com/lxml/lxml/commit/3b661503609314af8ae1ba23501cca901e21cae3
+
+---
+ src/lxml/includes/etree_defs.h | 8 +++++++-
+ src/lxml/includes/tree.pxd     | 3 +++
+ src/lxml/serializer.pxi        | 5 +++--
+ 3 files changed, 13 insertions(+), 3 deletions(-)
+
+diff --git a/src/lxml/includes/etree_defs.h b/src/lxml/includes/etree_defs.h
+index ff596ce..d3baef9 100644
+--- a/src/lxml/includes/etree_defs.h
++++ b/src/lxml/includes/etree_defs.h
+@@ -20,11 +20,15 @@
+ #define va_int(ap)     va_arg(ap, int)
+ #define va_charptr(ap) va_arg(ap, char *)
+ 
++/*
++Pyston change:
+ #ifdef PYPY_VERSION
+ #    define IS_PYPY 1
+ #else
+ #    define IS_PYPY 0
+ #endif
++*/
++#define IS_PYPY 0
+ 
+ #if PY_VERSION_HEX >= 0x03000000
+ #  define IS_PYTHON3 1
+@@ -152,7 +156,9 @@
+ #ifndef LIBXML2_NEW_BUFFER
+    typedef xmlBuffer xmlBuf;
+ #  define xmlBufContent(buf) xmlBufferContent(buf)
+-#  define xmlBufLength(buf) xmlBufferLength(buf)
++// Pyston change: change xmlBufLength to xmlBufUse. backport of lxml 3.1.2 patch
++//#  define xmlBufLength(buf) xmlBufferLength(buf)
++#  define xmlBufUse(buf) xmlBufferLength(buf)
+ #endif
+ 
+ /* libexslt 1.1.25+ support EXSLT functions in XPath */
+diff --git a/src/lxml/includes/tree.pxd b/src/lxml/includes/tree.pxd
+index afd49ba..a086384 100644
+--- a/src/lxml/includes/tree.pxd
++++ b/src/lxml/includes/tree.pxd
+@@ -363,6 +363,9 @@ cdef extern from "libxml/tree.h":
+     cdef int xmlBufferLength(xmlBuffer* buf) nogil
+     cdef const_xmlChar* xmlBufContent(xmlBuf* buf) nogil # new in libxml2 2.9
+     cdef size_t xmlBufLength(xmlBuf* buf) nogil # new in libxml2 2.9
++    # Pyston change: change xmlBufLength to xmlBufUse. backport of lxml 3.1.2 patch
++    # cdef size_t xmlBufLength(xmlBuf* buf) nogil
++    cdef size_t xmlBufUse(xmlBuf* buf) nogil
+     cdef int xmlKeepBlanksDefault(int val) nogil
+     cdef xmlChar* xmlNodeGetBase(xmlDoc* doc, xmlNode* node) nogil
+     cdef void xmlNodeSetBase(xmlNode* node, const_xmlChar* uri) nogil
+diff --git a/src/lxml/serializer.pxi b/src/lxml/serializer.pxi
+index f62887d..8579ed5 100644
+--- a/src/lxml/serializer.pxi
++++ b/src/lxml/serializer.pxi
+@@ -132,12 +132,13 @@ cdef _tostring(_Element element, encoding, doctype, method,
+         _raiseSerialisationError(error_result)
+ 
+     try:
++        # Pyston change: change xmlBufLength to xmlBufUse. backport of lxml 3.1.2 patch
+         if encoding is _unicode:
+             result = (<unsigned char*>tree.xmlBufContent(
+-                c_result_buffer))[:tree.xmlBufLength(c_result_buffer)].decode('UTF-8')
++                c_result_buffer))[:tree.xmlBufUse(c_result_buffer)].decode('UTF-8')
+         else:
+             result = <bytes>(<unsigned char*>tree.xmlBufContent(
+-                c_result_buffer))[:tree.xmlBufLength(c_result_buffer)]
++                c_result_buffer))[:tree.xmlBufUse(c_result_buffer)]
+     finally:
+         error_result = tree.xmlOutputBufferClose(c_buffer)
+     if error_result < 0:
+-- 
+1.9.1
+

test/extra/lxml_test.py

+import os, sys, subprocess, shutil
+sys.path.append(os.path.dirname(__file__) + "/../lib")
+
+from test_helper import create_virtenv, run_test
+
+ENV_NAME = "lxml_test_env_" + os.path.basename(sys.executable)
+SRC_DIR = os.path.abspath(os.path.join(ENV_NAME, "src"))
+PYTHON_EXE = os.path.abspath(os.path.join(ENV_NAME, "bin", "python"))
+
+def install_and_test_lxml():
+    shutil.rmtree(SRC_DIR, ignore_errors=True)
+    os.makedirs(SRC_DIR)
+
+    url = "http://cython.org/release/Cython-0.22.tar.gz"
+    subprocess.check_call(["wget", url], cwd=SRC_DIR)
+    subprocess.check_call(["tar", "-zxf", "Cython-0.22.tar.gz"], cwd=SRC_DIR)
+
+    CYTHON_DIR = os.path.abspath(os.path.join(SRC_DIR, "Cython-0.22"))
+    PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "integration", "Cython_0001-Pyston-change-we-don-t-support-custom-traceback-entr.patch"))
+    subprocess.check_call(["patch", "-p1", "--input=" + PATCH_FILE], cwd=CYTHON_DIR)
+    print "Applied Cython patch"
+    subprocess.check_call([PYTHON_EXE, "setup.py", "install"], cwd=CYTHON_DIR)
+    subprocess.check_call([PYTHON_EXE, "-c", "import Cython"], cwd=CYTHON_DIR)
+
+    url = "https://pypi.python.org/packages/source/l/lxml/lxml-3.0.1.tar.gz"
+    subprocess.check_call(["wget", url], cwd=SRC_DIR)
+    subprocess.check_call(["tar", "-zxf", "lxml-3.0.1.tar.gz"], cwd=SRC_DIR)
+    LXML_DIR = os.path.abspath(os.path.join(SRC_DIR, "lxml-3.0.1"))
+
+    PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "lxml_patch.patch"))
+    subprocess.check_call(["patch", "-p1", "--input=" + PATCH_FILE], cwd=LXML_DIR)
+    print "Applied lxml patch"
+
+    subprocess.check_call([PYTHON_EXE, "setup.py", "build_ext", "-i", "--with-cython"], cwd=LXML_DIR)
+   
+    expected = [{'ran': 1381, 'failures': 28, 'errors': 5}]
+    run_test([PYTHON_EXE, "test.py"], cwd=LXML_DIR, expected=expected)
+    
+create_virtenv(ENV_NAME, None, force_create = True)
+install_and_test_lxml()