Merge pull request #1114 from undingen/cffi_test

fixes for CFFI and add an integration test for it

CMakeLists.txt

@@ -337,7 +337,7 @@ endif()
 add_pyston_test(defaults cpython --exit-code-only --skip-failing -t100)
 add_pyston_test(defaults integration --exit-code-only --skip-failing -t600)
 if(ENABLE_EXTRA_TESTS)
-  add_pyston_test(defaults extra -t600 --exit-code-only)
+  add_pyston_test(defaults extra -t900 --exit-code-only)
 endif()
 
 

from_cpython/Lib/test/test_ascii_formatd.py

-# expected: fail
 # PyOS_ascii_formatd is deprecated and not called from anywhere in
 #  Python itself. So this module is the only place it gets tested.
 # Test that it works, and test that it's deprecated.

from_cpython/Lib/test/test_uuid.py

-# expected: fail
 import unittest
 from test import test_support
 import io
@@ -13,6 +12,9 @@ def importable(name):
         return False
 
 class TestUUID(unittest.TestCase):
+    last_node = None
+    source2node = {}
+
     def test_UUID(self):
         equal = self.assertEqual
         ascending = []
@@ -280,13 +282,101 @@ class TestUUID(unittest.TestCase):
         badtype(lambda: setattr(u, 'clock_seq_low', 0))
         badtype(lambda: setattr(u, 'node', 0))
 
+    def check_node(self, node, source):
+        message = "%012x is not an RFC 4122 node ID" % node
+        self.assertTrue(0 < node, message)
+        self.assertTrue(node < (1L << 48), message)
+
+        TestUUID.source2node[source] = node
+        if TestUUID.last_node:
+            if TestUUID.last_node != node:
+                msg = "different sources disagree on node:\n"
+                for s, n in TestUUID.source2node.iteritems():
+                    msg += "    from source %r, node was %012x\n" % (s, n)
+                # There's actually no reason to expect the MAC addresses
+                # to agree across various methods -- e.g., a box may have
+                # multiple network interfaces, and different ways of getting
+                # a MAC address may favor different HW.
+                ##self.fail(msg)
+        else:
+            TestUUID.last_node = node
+
+    @unittest.skipUnless(os.name == 'posix', 'requires Posix')
+    def test_ifconfig_getnode(self):
+        node = uuid._ifconfig_getnode()
+        if node is not None:
+            self.check_node(node, 'ifconfig')
+
+    @unittest.skipUnless(os.name == 'nt', 'requires Windows')
+    def test_ipconfig_getnode(self):
+        node = uuid._ipconfig_getnode()
+        if node is not None:
+            self.check_node(node, 'ipconfig')
+
+    @unittest.skipUnless(importable('win32wnet'), 'requires win32wnet')
+    @unittest.skipUnless(importable('netbios'), 'requires netbios')
+    def test_netbios_getnode(self):
+        self.check_node(uuid._netbios_getnode(), 'netbios')
+
+    def test_random_getnode(self):
+        node = uuid._random_getnode()
+        # Least significant bit of first octet must be set.
+        self.assertTrue(node & 0x010000000000)
+        self.assertTrue(node < (1L << 48))
+
+    @unittest.skipUnless(os.name == 'posix', 'requires Posix')
+    @unittest.skipUnless(importable('ctypes'), 'requires ctypes')
+    def test_unixdll_getnode(self):
+        try: # Issues 1481, 3581: _uuid_generate_time() might be None.
+            self.check_node(uuid._unixdll_getnode(), 'unixdll')
+        except TypeError:
+            pass
+
+    @unittest.skipUnless(os.name == 'nt', 'requires Windows')
+    @unittest.skipUnless(importable('ctypes'), 'requires ctypes')
+    def test_windll_getnode(self):
+        self.check_node(uuid._windll_getnode(), 'windll')
+
     def test_getnode(self):
         node1 = uuid.getnode()
-        self.assertTrue(0 < node1 < (1 << 48), '%012x' % node1)
+        self.check_node(node1, "getnode1")
 
         # Test it again to ensure consistency.
         node2 = uuid.getnode()
-        self.assertEqual(node1, node2, '%012x != %012x' % (node1, node2))
+        self.check_node(node2, "getnode2")
+
+        self.assertEqual(node1, node2)
+
+    @unittest.skipUnless(os.name == 'posix', 'requires Posix')
+    def test_find_mac(self):
+        data = '''\
+
+fake hwaddr
+cscotun0  Link encap:UNSPEC  HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00
+eth0      Link encap:Ethernet  HWaddr 12:34:56:78:90:ab
+'''
+        def mock_popen(cmd):
+            return io.BytesIO(data)
+
+        path = os.environ.get("PATH", os.defpath).split(os.pathsep)
+        path.extend(('/sbin', '/usr/sbin'))
+        for dir in path:
+            executable = os.path.join(dir, 'ifconfig')
+            if (os.path.exists(executable) and
+                os.access(executable, os.F_OK | os.X_OK) and
+                not os.path.isdir(executable)):
+                break
+        else:
+            self.skipTest('requires ifconfig')
+
+        with test_support.swap_attr(os, 'popen', mock_popen):
+            mac = uuid._find_mac(
+                command='ifconfig',
+                args='',
+                hw_identifiers=['hwaddr'],
+                get_index=lambda x: x + 1,
+            )
+            self.assertEqual(mac, 0x1234567890ab)
 
     @unittest.skipUnless(importable('ctypes'), 'requires ctypes')
     def test_uuid1(self):
@@ -398,106 +488,11 @@ class TestUUID(unittest.TestCase):
             self.assertNotEqual(parent_value, child_value)
 
 
-class TestInternals(unittest.TestCase):
-    @unittest.skipUnless(os.name == 'posix', 'requires Posix')
-    def test_find_mac(self):
-        data = '''\
-
-fake hwaddr
-cscotun0  Link encap:UNSPEC  HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00
-eth0      Link encap:Ethernet  HWaddr 12:34:56:78:90:ab
-'''
-        def mock_popen(cmd):
-            return io.BytesIO(data)
 
-        path = os.environ.get("PATH", os.defpath).split(os.pathsep)
-        path.extend(('/sbin', '/usr/sbin'))
-        for dir in path:
-            executable = os.path.join(dir, 'ifconfig')
-            if (os.path.exists(executable) and
-                os.access(executable, os.F_OK | os.X_OK) and
-                not os.path.isdir(executable)):
-                break
-        else:
-            self.skipTest('requires ifconfig')
-
-        with test_support.swap_attr(os, 'popen', mock_popen):
-            mac = uuid._find_mac(
-                command='ifconfig',
-                args='',
-                hw_identifiers=['hwaddr'],
-                get_index=lambda x: x + 1,
-            )
-            self.assertEqual(mac, 0x1234567890ab)
-
-    def check_node(self, node, requires=None, network=False):
-        if requires and node is None:
-            self.skipTest('requires ' + requires)
-        hex = '%012x' % node
-        if test_support.verbose >= 2:
-            print hex + ' ',
-        if network:
-            # 47 bit will never be set in IEEE 802 addresses obtained
-            # from network cards.
-            self.assertFalse(node & 0x010000000000, hex)
-        self.assertTrue(0 < node < (1L << 48),
-                        "%s is not an RFC 4122 node ID" % hex)
-
-    @unittest.skipUnless(os.name == 'posix', 'requires Posix')
-    def test_ifconfig_getnode(self):
-        node = uuid._ifconfig_getnode()
-        self.check_node(node, 'ifconfig', True)
-
-    @unittest.skipUnless(os.name == 'posix', 'requires Posix')
-    def test_arp_getnode(self):
-        node = uuid._arp_getnode()
-        self.check_node(node, 'arp', True)
-
-    @unittest.skipUnless(os.name == 'posix', 'requires Posix')
-    def test_lanscan_getnode(self):
-        node = uuid._lanscan_getnode()
-        self.check_node(node, 'lanscan', True)
-
-    @unittest.skipUnless(os.name == 'posix', 'requires Posix')
-    def test_netstat_getnode(self):
-        node = uuid._netstat_getnode()
-        self.check_node(node, 'netstat', True)
-
-    @unittest.skipUnless(os.name == 'nt', 'requires Windows')
-    def test_ipconfig_getnode(self):
-        node = uuid._ipconfig_getnode()
-        self.check_node(node, 'ipconfig', True)
-
-    @unittest.skipUnless(importable('win32wnet'), 'requires win32wnet')
-    @unittest.skipUnless(importable('netbios'), 'requires netbios')
-    def test_netbios_getnode(self):
-        node = uuid._netbios_getnode()
-        self.check_node(node, network=True)
-
-    def test_random_getnode(self):
-        node = uuid._random_getnode()
-        # Least significant bit of first octet must be set.
-        self.assertTrue(node & 0x010000000000, '%012x' % node)
-        self.check_node(node)
-
-    @unittest.skipUnless(os.name == 'posix', 'requires Posix')
-    @unittest.skipUnless(importable('ctypes'), 'requires ctypes')
-    def test_unixdll_getnode(self):
-        try: # Issues 1481, 3581: _uuid_generate_time() might be None.
-            node = uuid._unixdll_getnode()
-        except TypeError:
-            self.skipTest('requires uuid_generate_time')
-        self.check_node(node)
-
-    @unittest.skipUnless(os.name == 'nt', 'requires Windows')
-    @unittest.skipUnless(importable('ctypes'), 'requires ctypes')
-    def test_windll_getnode(self):
-        node = uuid._windll_getnode()
-        self.check_node(node)
 
 
 def test_main():
-    test_support.run_unittest(TestUUID, TestInternals)
+    test_support.run_unittest(TestUUID)
 
 if __name__ == '__main__':
     test_main()

from_cpython/Lib/types.py

@@ -60,6 +60,7 @@ BuiltinFunctionType = type(len)
 # Pyston change:
 # BuiltinMethodType = type([].append)     # Same as BuiltinFunctionType
 BuiltinMethodType = type((1.0).hex)     # Same as BuiltinFunctionType
+BuiltinCAPIFunctionType = type(reload)  # Pyston change: added this type
 
 ModuleType = type(sys)
 FileType = file

from_cpython/Modules/_ctypes/ctypes.h

@@ -343,7 +343,8 @@ struct tagPyCArgObject {
 #ifdef HAVE_LONG_LONG
         PY_LONG_LONG q;
 #endif
-        long double D;
+        // Pyston change: disable this 16byte type because our allocator doesn't align allocs to 16byte.
+        // long double D;
         double d;
         float f;
         void *p;

src/capi/modsupport.cpp

@@ -440,7 +440,8 @@ extern "C" PyObject* Py_InitModule4(const char* name, PyMethodDef* methods, cons
     Box* passthrough = static_cast<Box*>(self);
 
     while (methods && methods->ml_name) {
-        module->giveAttr(methods->ml_name, new BoxedCApiFunction(methods, passthrough, boxString(name)));
+        module->setattr(internStringMortal(methods->ml_name),
+                        new BoxedCApiFunction(methods, passthrough, boxString(name)), NULL);
 
         methods++;
     }

src/runtime/import.cpp

@@ -49,6 +49,10 @@ extern "C" PyObject* PyImport_GetModuleDict(void) noexcept {
 extern "C" PyObject* _PyImport_LoadDynamicModule(char* name, char* pathname, FILE* fp) noexcept {
     BoxedString* name_boxed = boxString(name);
     try {
+        PyObject* m = _PyImport_FindExtension(name, pathname);
+        if (m != NULL)
+            return m;
+
         const char* lastdot = strrchr(name, '.');
         const char* shortname;
         if (lastdot == NULL) {
@@ -57,7 +61,12 @@ extern "C" PyObject* _PyImport_LoadDynamicModule(char* name, char* pathname, FIL
             shortname = lastdot + 1;
         }
 
-        return importCExtension(name_boxed, shortname, pathname);
+        m = importCExtension(name_boxed, shortname, pathname);
+
+        if (_PyImport_FixupExtension(name, pathname) == NULL)
+            return NULL;
+
+        return m;
     } catch (ExcInfo e) {
         removeModule(name_boxed);
         setCAPIException(e);

src/runtime/long.cpp

@@ -115,14 +115,18 @@ extern "C" int _PyLong_AsInt(PyObject* obj) noexcept {
 extern "C" unsigned long PyLong_AsUnsignedLongMask(PyObject* vv) noexcept {
     if (PyLong_Check(vv)) {
         BoxedLong* l = static_cast<BoxedLong*>(vv);
-        return mpz_get_ui(l->n);
+        auto v = mpz_get_ui(l->n);
+        if (mpz_sgn(l->n) == -1)
+            return -v;
+        return v;
     }
 
     Py_FatalError("unimplemented");
 }
 
 extern "C" unsigned PY_LONG_LONG PyLong_AsUnsignedLongLongMask(PyObject* vv) noexcept {
-    Py_FatalError("unimplemented");
+    static_assert(sizeof(PY_LONG_LONG) == sizeof(unsigned long), "");
+    return PyLong_AsUnsignedLongMask(vv);
 }
 
 extern "C" PY_LONG_LONG PyLong_AsLongLong(PyObject* vv) noexcept {

test/extra/cffi-1.2.1.patch

+diff -ur cffi-1.2.1_orig/c/_cffi_backend.c cffi-1.2.1/c/_cffi_backend.c
+--- cffi-1.2.1_orig/c/_cffi_backend.c	2016-02-22 18:48:06.861772302 +0000
++++ cffi-1.2.1/c/_cffi_backend.c	2016-02-24 18:57:47.479889228 +0000
+@@ -5065,6 +5065,10 @@
+     cd->c_weakreflist = NULL;
+     PyObject_GC_Track(cd);
+ 
++    // Pyston change: HACK which leaks mem but seems necessary at the moment to prevent crashes with our GC
++    PyGC_AddRoot((PyObject*)cd);
++    PyGC_AddRoot((PyObject*)infotuple);
++
+     cif_descr = (cif_description_t *)ct->ct_extra;
+     if (cif_descr == NULL) {
+         PyErr_Format(PyExc_NotImplementedError,
+@@ -5572,7 +5576,9 @@
+         return NULL;
+     }
+     x = (PyObject *)(raw + 42);
+-    if (Py_REFCNT(x) <= 0) {
++    // Pyston change:
++    // if (Py_REFCNT(x) <= 0) {
++    if (0) {
+         Py_FatalError("ffi.from_handle() detected that the address passed "
+                       "points to garbage. If it is really the result of "
+                       "ffi.new_handle(), then the Python object has already "
+diff -ur cffi-1.2.1_orig/c/cglob.c cffi-1.2.1/c/cglob.c
+--- cffi-1.2.1_orig/c/cglob.c	2016-02-22 18:48:06.853772302 +0000
++++ cffi-1.2.1/c/cglob.c	2016-02-25 11:02:26.319738004 +0000
+@@ -46,7 +46,15 @@
+ static PyObject *make_global_var(PyObject *name, CTypeDescrObject *type,
+                                  char *addr, gs_fetch_addr_fn fetch_addr)
+ {
+-    GlobSupportObject *gs = PyObject_New(GlobSupportObject, &GlobSupport_Type);
++    // Pyston change:
++    static int has_registered = 0;    
++    GlobSupportObject *gs;
++    if (!has_registered) {
++        PyType_Ready(&GlobSupport_Type);
++        has_registered = 1;
++    }
++
++    gs = PyObject_New(GlobSupportObject, &GlobSupport_Type);
+     if (gs == NULL)
+         return NULL;
+ 

test/extra/cffi_test.py

+import os, sys, subprocess, shutil
+sys.path.append(os.path.dirname(__file__) + "/../lib")
+
+from test_helper import create_virtenv, run_test
+
+ENV_NAME = "cffi_test_env_" + os.path.basename(sys.executable)
+SRC_DIR = os.path.abspath(os.path.join(ENV_NAME, "src"))
+PYTHON_EXE = os.path.abspath(os.path.join(ENV_NAME, "bin", "python"))
+PYTEST_EXE = os.path.abspath(os.path.join(ENV_NAME, "bin", "py.test"))
+
+def install_and_test_cffi():
+    shutil.rmtree(SRC_DIR, ignore_errors=True)
+    os.makedirs(SRC_DIR)
+
+    url = "https://pypi.python.org/packages/source/c/cffi/cffi-1.2.1.tar.gz"
+    subprocess.check_call(["wget", url], cwd=SRC_DIR)
+    subprocess.check_call(["tar", "-zxf", "cffi-1.2.1.tar.gz"], cwd=SRC_DIR)
+    CFFI_DIR = os.path.abspath(os.path.join(SRC_DIR, "cffi-1.2.1"))
+
+    PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "cffi-1.2.1.patch"))
+    subprocess.check_call(["patch", "-p1", "--input=" + PATCH_FILE], cwd=CFFI_DIR)
+
+    subprocess.check_call([PYTHON_EXE, "setup.py", "install"], cwd=CFFI_DIR)
+
+    # looks like clang 3.5 causes more errors like: 214 != -42 doing casts
+    if os.environ.has_key("CC") and "clang" in os.environ["CC"]:
+        expected = [{ "failed": 58, "passed": 1619, "skipped": 70, "xfailed": 4, "error": 5 }]
+    else:
+        expected = [{ "failed": 49, "passed": 1628, "skipped": 70, "xfailed": 4, "error": 5 }]
+    run_test([PYTEST_EXE], cwd=CFFI_DIR, expected=expected)
+   
+create_virtenv(ENV_NAME, ["pytest==2.8.7", "py==1.4.31", "pycparser==2.14"], force_create = True)
+install_and_test_cffi()

test/lib/test_helper.py

@@ -37,15 +37,24 @@ def pip_install(name, package_list):
 def parse_output(output):
     result = []
     for l in output.split('\n'):
+        # nosetest
         m = re.match("Ran (\d+) tests in", l)
         if m:
             result.append({"ran": int(m.group(1))})
             continue
         for res_type in ("errors", "failures", "skipped"):
-	    m = re.match("FAILED \(.*%s=(\d+).*\)" % res_type, l)
-	    if m:
-	        result[-1][res_type] = int(m.group(1))
-	
+            m = re.match("FAILED \(.*%s=(\d+).*\)" % res_type, l)
+            if m:
+                result[-1][res_type] = int(m.group(1))
+
+        # py.test
+        m = re.match(".* in \d+[.]\d+ seconds [=]*", l)
+        if m:
+            result.append({})
+            for res_type in ("failed", "passed", "skipped", "xfailed", "error"):
+                m = re.match(".* (\d+) %s.*" % res_type, l)
+                if m:
+                    result[-1][res_type] = int(m.group(1))
     return result
 
 def run_test(cmd, cwd, expected, env = None):

test/tests/ctypes_test.py

-# skip-if: True
 from ctypes import *
 s = "tmp"
 ap = create_string_buffer(s)

test/tests/type_flags.py

@@ -10,7 +10,7 @@ def test(cls):
 
 def testall(module):
     for n in sorted(dir((module))):
-        if n in ("reversed", "AttrwrapperType", "BuiltinMethodType", "BufferType", "DictProxyType"):
+        if n in ("reversed", "AttrwrapperType", "BuiltinMethodType", "BufferType", "DictProxyType", "BuiltinCAPIFunctionType"):
             continue
 
         cls = getattr(module, n)