Merge pull request #937 from undingen/pyopenssl

misc fixes for pyopenssl

.travis.yml

@@ -50,6 +50,7 @@ addons:
     - libcurl4-openssl-dev
     - libxml2-dev 
     - libxslt1-dev
+    - libssl-dev
 
 before_install:
   - if [ "$CC" = "clang" ]; then export CC="clang-3.5" CXX="clang++-3.5"; fi

from_cpython/Include/Python.h

@@ -18,8 +18,7 @@
 // Cython depends on having this define set:
 #define Py_PYTHON_H
 
-// Cython has some "not targeting CPython" support that is triggered by having PYPY_VERSION defined:
-#define PYPY_VERSION "Pyston"
+#define PYSTON_VERSION "0.3"
 
 // These include orders come from CPython:
 #include "patchlevel.h"

from_cpython/Objects/exceptions.c

@@ -2246,11 +2246,11 @@ _PyExc_Init(void)
     POST_INIT(UnicodeWarning)
     POST_INIT(BytesWarning)
 
-    PyExc_MemoryErrorInst = BaseException_new(&_PyExc_MemoryError, NULL, NULL);
+    PyExc_MemoryErrorInst = PyGC_AddRoot(BaseException_new(&_PyExc_MemoryError, NULL, NULL));
     if (!PyExc_MemoryErrorInst)
         Py_FatalError("Cannot pre-allocate MemoryError instance");
 
-    PyExc_RecursionErrorInst = BaseException_new(&_PyExc_RuntimeError, NULL, NULL);
+    PyExc_RecursionErrorInst = PyGC_AddRoot(BaseException_new(&_PyExc_RuntimeError, NULL, NULL));
     if (!PyExc_RecursionErrorInst)
         Py_FatalError("Cannot pre-allocate RuntimeError instance for "
                         "recursion errors");

src/capi/abstract.cpp

@@ -605,6 +605,32 @@ extern "C" int PyObject_AsReadBuffer(PyObject* obj, const void** buffer, Py_ssiz
     return 0;
 }
 
+extern "C" int PyObject_AsWriteBuffer(PyObject* obj, void** buffer, Py_ssize_t* buffer_len) noexcept {
+    PyBufferProcs* pb;
+    void* pp;
+    Py_ssize_t len;
+
+    if (obj == NULL || buffer == NULL || buffer_len == NULL) {
+        null_error();
+        return -1;
+    }
+    pb = obj->cls->tp_as_buffer;
+    if (pb == NULL || pb->bf_getwritebuffer == NULL || pb->bf_getsegcount == NULL) {
+        PyErr_SetString(PyExc_TypeError, "expected a writeable buffer object");
+        return -1;
+    }
+    if ((*pb->bf_getsegcount)(obj, NULL) != 1) {
+        PyErr_SetString(PyExc_TypeError, "expected a single-segment buffer object");
+        return -1;
+    }
+    len = (*pb->bf_getwritebuffer)(obj, 0, &pp);
+    if (len < 0)
+        return -1;
+    *buffer = pp;
+    *buffer_len = len;
+    return 0;
+}
+
 static PyObject* call_function_tail(PyObject* callable, PyObject* args) {
     PyObject* retval;
 

src/capi/object.cpp

@@ -564,10 +564,6 @@ extern "C" int PyObject_HasAttrString(PyObject* v, const char* name) noexcept {
     return 0;
 }
 
-extern "C" int PyObject_AsWriteBuffer(PyObject* obj, void** buffer, Py_ssize_t* buffer_len) noexcept {
-    Py_FatalError("unimplemented");
-}
-
 // I'm not sure how we can support this one:
 extern "C" PyObject** _PyObject_GetDictPtr(PyObject* obj) noexcept {
     fatalOrError(PyExc_NotImplementedError, "unimplemented");

src/capi/types.h

@@ -68,6 +68,7 @@ public:
         BoxedCApiFunction* o = static_cast<BoxedCApiFunction*>(_o);
 
         Box::gcHandler(v, o);
+        v->visitPotential(o->method_def);
         v->visit(&o->passthrough);
         v->visit(&o->module);
     }

src/gc/gc_alloc.h

@@ -51,6 +51,10 @@ extern "C" inline void* gc_alloc(size_t bytes, GCKind kind_id) {
     // inline it, which is especially useful for this function.
     ScopedStatTimer gc_alloc_stattimer(gc_alloc_stattimer_counter, 15);
 #endif
+
+    if ((size_t)(bytes) > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+
     size_t alloc_bytes = bytes + sizeof(GCAllocation);
 
 #ifndef NVALGRIND
@@ -124,6 +128,9 @@ extern "C" inline void* gc_realloc(void* ptr, size_t bytes) {
     // Normal realloc() supports receiving a NULL pointer, but we need to know what the GCKind is:
     assert(ptr);
 
+    if ((size_t)(bytes) > (size_t)PY_SSIZE_T_MAX)
+        return NULL;
+
     size_t alloc_bytes = bytes + sizeof(GCAllocation);
 
     GCAllocation* alloc;

src/runtime/capi.cpp

@@ -892,8 +892,20 @@ extern "C" int PyErr_BadArgument() noexcept {
 }
 
 extern "C" PyObject* PyErr_NoMemory() noexcept {
-    fatalOrError(PyExc_NotImplementedError, "unimplemented");
-    return nullptr;
+    if (PyErr_ExceptionMatches(PyExc_MemoryError))
+        /* already current */
+        return NULL;
+
+    /* raise the pre-allocated instance if it still exists */
+    if (PyExc_MemoryErrorInst)
+        PyErr_SetObject(PyExc_MemoryError, PyExc_MemoryErrorInst);
+    else
+        /* this will probably fail since there's no memory and hee,
+           hee, we have to instantiate this class
+        */
+        PyErr_SetNone(PyExc_MemoryError);
+
+    return NULL;
 }
 
 extern "C" const char* PyExceptionClass_Name(PyObject* o) noexcept {

src/runtime/classobj.cpp

@@ -54,6 +54,34 @@ static Box* classLookup(BoxedClassobj* cls, BoxedString* attr, GetattrRewriteArg
     return NULL;
 }
 
+extern "C" PyObject* _PyInstance_Lookup(PyObject* pinst, PyObject* pname) noexcept {
+    RELEASE_ASSERT(PyInstance_Check(pinst), "");
+    BoxedInstance* inst = (BoxedInstance*)pinst;
+
+    RELEASE_ASSERT(PyString_Check(pname), "");
+    BoxedString* name = (BoxedString*)pname;
+
+    try {
+        internStringMortalInplace(name);
+        Box* v = inst->getattr(name, NULL);
+        if (v == NULL)
+            v = classLookup(inst->inst_cls, name);
+        return v;
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return NULL;
+    }
+}
+
+extern "C" PyObject* PyInstance_NewRaw(PyObject* klass, PyObject* dict) noexcept {
+    RELEASE_ASSERT(!dict, "not implemented");
+    if (!PyClass_Check(klass)) {
+        PyErr_BadInternalCall();
+        return NULL;
+    }
+    return new BoxedInstance((BoxedClassobj*)klass);
+}
+
 extern "C" int PyClass_IsSubclass(PyObject* klass, PyObject* base) noexcept {
     Py_ssize_t i, n;
     if (klass == base)
@@ -155,6 +183,15 @@ Box* classobjCall(Box* _cls, Box* _args, Box* _kwargs) {
     return made;
 }
 
+extern "C" PyObject* PyInstance_New(PyObject* klass, PyObject* arg, PyObject* kw) noexcept {
+    try {
+        return classobjCall(klass, arg, kw);
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return NULL;
+    }
+}
+
 static Box* classobjGetattribute(Box* _cls, Box* _attr) {
     RELEASE_ASSERT(_cls->cls == classobj_cls, "");
     BoxedClassobj* cls = static_cast<BoxedClassobj*>(_cls);

src/runtime/file.cpp

@@ -929,11 +929,17 @@ Box* fileNew(BoxedClass* cls, Box* s, Box* m, Box** args) {
 
     assert(cls == file_cls);
 
-    if (s->cls == unicode_cls)
+    if (s->cls == unicode_cls) {
         s = _PyUnicode_AsDefaultEncodedString(s, NULL);
+        if (!s)
+            throwCAPIException();
+    }
 
-    if (m->cls == unicode_cls)
+    if (m->cls == unicode_cls) {
         m = _PyUnicode_AsDefaultEncodedString(m, NULL);
+        if (!m)
+            throwCAPIException();
+    }
 
     if (s->cls != str_cls) {
         raiseExcHelper(TypeError, "coercing to Unicode: need string of buffer, %s found", getTypeName(s));

src/runtime/import.cpp

@@ -15,6 +15,7 @@
 #include "runtime/import.h"
 
 #include <dlfcn.h>
+#include <fstream>
 #include <limits.h>
 
 #include "llvm/Support/FileSystem.h"
@@ -783,6 +784,63 @@ Box* impLoadDynamic(Box* _name, Box* _pathname, Box* _file) {
     return importCExtension(name, shortname, pathname->s());
 }
 
+// Parses the memory map and registers all memory regions with "rwxp" permission and which contain the requested file
+// path with the GC. In addition adds the BSS segment.
+static void registerDataSegment(void* dl_handle, llvm::StringRef lib_path) {
+    std::ifstream mapmap("/proc/self/maps");
+    std::string line;
+    llvm::SmallVector<std::pair<uint64_t, uint64_t>, 4> mem_ranges;
+    while (std::getline(mapmap, line)) {
+        // format is:
+        // address perms offset dev inode pathname
+        llvm::SmallVector<llvm::StringRef, 6> line_split;
+        llvm::StringRef(line).split(line_split, " ", 5, false);
+        RELEASE_ASSERT(line_split.size() >= 5, "%zu", line_split.size());
+        if (line_split.size() < 6)
+            continue; // pathname is missing
+
+        llvm::StringRef permissions = line_split[1].trim();
+        llvm::StringRef pathname = line_split[5].trim();
+
+        if (permissions == "rwxp" && pathname.endswith(lib_path)) {
+            llvm::StringRef mem_range = line_split[0].trim();
+            auto mem_range_split = mem_range.split('-');
+            uint64_t lower_addr = 0;
+            bool error = mem_range_split.first.getAsInteger(16, lower_addr);
+            RELEASE_ASSERT(!error, "");
+            uint64_t upper_addr = 0;
+            error = mem_range_split.second.getAsInteger(16, upper_addr);
+            RELEASE_ASSERT(!error, "");
+
+            RELEASE_ASSERT(lower_addr < upper_addr, "");
+            RELEASE_ASSERT(upper_addr - lower_addr < 1000000,
+                           "Large data section detected - there maybe something wrong");
+
+            mem_ranges.emplace_back(lower_addr, upper_addr);
+        }
+    }
+    RELEASE_ASSERT(mem_ranges.size() >= 1, "");
+
+    uintptr_t bss_start = (uintptr_t)dlsym(dl_handle, "__bss_start");
+    uintptr_t bss_end = (uintptr_t)dlsym(dl_handle, "_end");
+    RELEASE_ASSERT(bss_end - bss_start < 1000000, "Large BSS section detected - there maybe something wrong");
+
+    // most of the time the BSS section is inside one of memory regions, in that case we don't have to register it.
+    bool should_add_bss = true;
+    for (auto r : mem_ranges) {
+        if (r.first <= bss_start && bss_end <= r.second)
+            should_add_bss = false;
+        gc::registerPotentialRootRange((void*)r.first, (void*)r.second);
+    }
+
+    if (should_add_bss) {
+        // only track void* aligned memory
+        bss_start = (bss_start + (sizeof(void*) - 1)) & ~(sizeof(void*) - 1);
+        bss_end -= bss_end % sizeof(void*);
+        gc::registerPotentialRootRange((void*)bss_start, (void*)bss_end);
+    }
+}
+
 BoxedModule* importCExtension(BoxedString* full_name, const std::string& last_name, const std::string& path) {
     void* handle = dlopen(path.c_str(), RTLD_NOW);
     if (!handle) {
@@ -805,13 +863,7 @@ BoxedModule* importCExtension(BoxedString* full_name, const std::string& last_na
     assert(init);
 
     // Let the GC know about the static variables.
-    uintptr_t bss_start = (uintptr_t)dlsym(handle, "__bss_start");
-    uintptr_t bss_end = (uintptr_t)dlsym(handle, "_end");
-    RELEASE_ASSERT(bss_end - bss_start < 1000000, "Large BSS section detected - there maybe something wrong");
-    // only track void* aligned memory
-    bss_start = (bss_start + (sizeof(void*) - 1)) & ~(sizeof(void*) - 1);
-    bss_end -= bss_end % sizeof(void*);
-    gc::registerPotentialRootRange((void*)bss_start, (void*)bss_end);
+    registerDataSegment(handle, path);
 
     char* packagecontext = strdup(full_name->c_str());
     char* oldcontext = _Py_PackageContext;

src/runtime/str.cpp

@@ -2477,6 +2477,10 @@ extern "C" int PyString_AsStringAndSize(register PyObject* obj, register char**
 }
 
 extern "C" PyObject* PyString_FromStringAndSize(const char* s, ssize_t n) noexcept {
+    if (n < 0) {
+        PyErr_SetString(PyExc_SystemError, "Negative size passed to PyString_FromStringAndSize");
+        return NULL;
+    }
     if (s == NULL)
         return BoxedString::createUninitializedString(n);
     return boxString(llvm::StringRef(s, n));

src/runtime/types.cpp

@@ -3274,7 +3274,9 @@ extern "C" PyVarObject* PyObject_InitVar(PyVarObject* op, PyTypeObject* tp, Py_s
 }
 
 extern "C" PyObject* PyObject_Init(PyObject* op, PyTypeObject* tp) noexcept {
-    assert(op);
+    if (op == NULL)
+        return PyErr_NoMemory();
+
     assert(tp);
 
     assert(gc::isValidGCMemory(op));

test/extra/pyopenssl_test.py

+import os, sys
+sys.path.append(os.path.dirname(__file__) + "/../lib")
+
+from test_helper import create_virtenv, run_test
+
+ENV_NAME = os.path.abspath("pyopenssl_test_env_" + os.path.basename(sys.executable))
+NOSETESTS_EXE = os.path.abspath(os.path.join(ENV_NAME, "bin", "nosetests"))
+PYOPENSSL_DIR = os.path.abspath(os.path.join(ENV_NAME, "site-packages", "OpenSSL"))
+
+packages = ["nose==1.3.7", "pycparser==2.13", "cryptography==1.0.1", "pyopenssl==0.15.1", "pyasn1==0.1.7", "idna==2.0", "six==1.9.0", "enum34==1.0.4", "ipaddress==1.0.14", "cffi==1.1.0"]
+create_virtenv(ENV_NAME, packages, force_create = True)
+
+expected = [{'ran': 247, 'errors': 2, 'failures': 0}]
+run_test([NOSETESTS_EXE], cwd=PYOPENSSL_DIR, expected=expected)

test/integration/Cython_0001-Pyston-change-we-don-t-support-custom-traceback-entr.patch

-From 27b69179b90851da187d698a0817b0b8190a0449 Mon Sep 17 00:00:00 2001
+From e0257ba4efd903da2a0f1909d21d51ff1799a3c6 Mon Sep 17 00:00:00 2001
 From: Marius Wachtler <undingen@gmail.com>
 Date: Tue, 9 Jun 2015 19:26:44 +0200
 Subject: [PATCH] Pyston change: we don't support custom traceback entries yet
@@ -8,8 +8,9 @@ Subject: [PATCH] Pyston change: we don't support custom traceback entries yet
  Cython/Compiler/ModuleNode.py    |  8 ++++++--
  Cython/Utility/CythonFunction.c  |  4 +++-
  Cython/Utility/Exceptions.c      |  7 ++++++-
- Cython/Utility/Generator.c      | 41 +++++++++++++++++++++++++++++------------
- 5 files changed, 53 insertions(+), 16 deletions(-)
+ Cython/Utility/Generator.c       | 41 ++++++++++++++++++++++++++++------------
+ Cython/Utility/ModuleSetupCode.c |  2 +-
+ 6 files changed, 54 insertions(+), 17 deletions(-)
 
 diff --git a/Cython/Compiler/ExprNodes.py b/Cython/Compiler/ExprNodes.py
 index f99ec6e..7ab41f3 100644
@@ -188,6 +189,19 @@ index 0310570..70e550c 100644
  #endif
      0,                                  /*tp_version_tag*/
  #if PY_VERSION_HEX >= 0x030400a1
+diff --git a/Cython/Utility/ModuleSetupCode.c b/Cython/Utility/ModuleSetupCode.c
+index 6477fb2..75dcdda 100644
+--- a/Cython/Utility/ModuleSetupCode.c
++++ b/Cython/Utility/ModuleSetupCode.c
+@@ -32,7 +32,7 @@
+   #define Py_HUGE_VAL HUGE_VAL
+ #endif
+ 
+-#ifdef PYPY_VERSION
++#if defined(PYPY_VERSION) || defined(PYSTON_VERSION)
+ #define CYTHON_COMPILING_IN_PYPY 1
+ #define CYTHON_COMPILING_IN_CPYTHON 0
+ #else
 -- 
 1.9.1
 

test/tests/multiprocessing_ctypes_test.py

-# expected: fail
-# - relies on ctypes
-
+# skip-if: True
 import multiprocessing
 
 # from https://docs.python.org/2/library/multiprocessing.html