fixes for CFFI and add an integration test for it

We can now run most of the CFFI tests.
Some of the changes were required because CFFI tests loading the same shared library several times from different locations.

This also enables some ctypes tests (test_uuid was coied from the wrong cpython version)

CMakeLists.txt

@@ -337,7 +337,7 @@ endif()
 add_pyston_test(defaults cpython --exit-code-only --skip-failing -t100)
 add_pyston_test(defaults integration --exit-code-only --skip-failing -t600)
 if(ENABLE_EXTRA_TESTS)
-  add_pyston_test(defaults extra -t600 --exit-code-only)
+  add_pyston_test(defaults extra -t900 --exit-code-only)
 endif()
 
 

from_cpython/Lib/test/test_ascii_formatd.py

-# expected: fail
 # PyOS_ascii_formatd is deprecated and not called from anywhere in
 #  Python itself. So this module is the only place it gets tested.
 # Test that it works, and test that it's deprecated.

from_cpython/Lib/test/test_uuid.py

-# expected: fail
 import unittest
 from test import test_support
 import io
@@ -13,6 +12,9 @@ def importable(name):
         return False
 
 class TestUUID(unittest.TestCase):
+    last_node = None
+    source2node = {}
+
     def test_UUID(self):
         equal = self.assertEqual
         ascending = []
@@ -280,13 +282,101 @@ class TestUUID(unittest.TestCase):
         badtype(lambda: setattr(u, 'clock_seq_low', 0))
         badtype(lambda: setattr(u, 'node', 0))
 
+    def check_node(self, node, source):
+        message = "%012x is not an RFC 4122 node ID" % node
+        self.assertTrue(0 < node, message)
+        self.assertTrue(node < (1L << 48), message)
+
+        TestUUID.source2node[source] = node
+        if TestUUID.last_node:
+            if TestUUID.last_node != node:
+                msg = "different sources disagree on node:\n"
+                for s, n in TestUUID.source2node.iteritems():
+                    msg += "    from source %r, node was %012x\n" % (s, n)
+                # There's actually no reason to expect the MAC addresses
+                # to agree across various methods -- e.g., a box may have
+                # multiple network interfaces, and different ways of getting
+                # a MAC address may favor different HW.
+                ##self.fail(msg)
+        else:
+            TestUUID.last_node = node
+
+    @unittest.skipUnless(os.name == 'posix', 'requires Posix')
+    def test_ifconfig_getnode(self):
+        node = uuid._ifconfig_getnode()
+        if node is not None:
+            self.check_node(node, 'ifconfig')
+
+    @unittest.skipUnless(os.name == 'nt', 'requires Windows')
+    def test_ipconfig_getnode(self):
+        node = uuid._ipconfig_getnode()
+        if node is not None:
+            self.check_node(node, 'ipconfig')
+
+    @unittest.skipUnless(importable('win32wnet'), 'requires win32wnet')
+    @unittest.skipUnless(importable('netbios'), 'requires netbios')
+    def test_netbios_getnode(self):
+        self.check_node(uuid._netbios_getnode(), 'netbios')
+
+    def test_random_getnode(self):
+        node = uuid._random_getnode()
+        # Least significant bit of first octet must be set.
+        self.assertTrue(node & 0x010000000000)
+        self.assertTrue(node < (1L << 48))
+
+    @unittest.skipUnless(os.name == 'posix', 'requires Posix')
+    @unittest.skipUnless(importable('ctypes'), 'requires ctypes')
+    def test_unixdll_getnode(self):
+        try: # Issues 1481, 3581: _uuid_generate_time() might be None.
+            self.check_node(uuid._unixdll_getnode(), 'unixdll')
+        except TypeError:
+            pass
+
+    @unittest.skipUnless(os.name == 'nt', 'requires Windows')
+    @unittest.skipUnless(importable('ctypes'), 'requires ctypes')
+    def test_windll_getnode(self):
+        self.check_node(uuid._windll_getnode(), 'windll')
+
     def test_getnode(self):
         node1 = uuid.getnode()
-        self.assertTrue(0 < node1 < (1 << 48), '%012x' % node1)
+        self.check_node(node1, "getnode1")
 
         # Test it again to ensure consistency.
         node2 = uuid.getnode()
-        self.assertEqual(node1, node2, '%012x != %012x' % (node1, node2))
+        self.check_node(node2, "getnode2")
+
+        self.assertEqual(node1, node2)
+
+    @unittest.skipUnless(os.name == 'posix', 'requires Posix')
+    def test_find_mac(self):
+        data = '''\
+
+fake hwaddr
+cscotun0  Link encap:UNSPEC  HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00
+eth0      Link encap:Ethernet  HWaddr 12:34:56:78:90:ab
+'''
+        def mock_popen(cmd):
+            return io.BytesIO(data)
+
+        path = os.environ.get("PATH", os.defpath).split(os.pathsep)
+        path.extend(('/sbin', '/usr/sbin'))
+        for dir in path:
+            executable = os.path.join(dir, 'ifconfig')
+            if (os.path.exists(executable) and
+                os.access(executable, os.F_OK | os.X_OK) and
+                not os.path.isdir(executable)):
+                break
+        else:
+            self.skipTest('requires ifconfig')
+
+        with test_support.swap_attr(os, 'popen', mock_popen):
+            mac = uuid._find_mac(
+                command='ifconfig',
+                args='',
+                hw_identifiers=['hwaddr'],
+                get_index=lambda x: x + 1,
+            )
+            self.assertEqual(mac, 0x1234567890ab)
 
     @unittest.skipUnless(importable('ctypes'), 'requires ctypes')
     def test_uuid1(self):
@@ -398,106 +488,11 @@ class TestUUID(unittest.TestCase):
             self.assertNotEqual(parent_value, child_value)
 
 
-class TestInternals(unittest.TestCase):
-    @unittest.skipUnless(os.name == 'posix', 'requires Posix')
-    def test_find_mac(self):
-        data = '''\
-
-fake hwaddr
-cscotun0  Link encap:UNSPEC  HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00
-eth0      Link encap:Ethernet  HWaddr 12:34:56:78:90:ab
-'''
-        def mock_popen(cmd):
-            return io.BytesIO(data)
 
-        path = os.environ.get("PATH", os.defpath).split(os.pathsep)
-        path.extend(('/sbin', '/usr/sbin'))
-        for dir in path:
-            executable = os.path.join(dir, 'ifconfig')
-            if (os.path.exists(executable) and
-                os.access(executable, os.F_OK | os.X_OK) and
-                not os.path.isdir(executable)):
-                break
-        else:
-            self.skipTest('requires ifconfig')
-
-        with test_support.swap_attr(os, 'popen', mock_popen):
-            mac = uuid._find_mac(
-                command='ifconfig',
-                args='',
-                hw_identifiers=['hwaddr'],
-                get_index=lambda x: x + 1,
-            )
-            self.assertEqual(mac, 0x1234567890ab)
-
-    def check_node(self, node, requires=None, network=False):
-        if requires and node is None:
-            self.skipTest('requires ' + requires)
-        hex = '%012x' % node
-        if test_support.verbose >= 2:
-            print hex + ' ',
-        if network:
-            # 47 bit will never be set in IEEE 802 addresses obtained
-            # from network cards.
-            self.assertFalse(node & 0x010000000000, hex)
-        self.assertTrue(0 < node < (1L << 48),
-                        "%s is not an RFC 4122 node ID" % hex)
-
-    @unittest.skipUnless(os.name == 'posix', 'requires Posix')
-    def test_ifconfig_getnode(self):
-        node = uuid._ifconfig_getnode()
-        self.check_node(node, 'ifconfig', True)
-
-    @unittest.skipUnless(os.name == 'posix', 'requires Posix')
-    def test_arp_getnode(self):
-        node = uuid._arp_getnode()
-        self.check_node(node, 'arp', True)
-
-    @unittest.skipUnless(os.name == 'posix', 'requires Posix')
-    def test_lanscan_getnode(self):
-        node = uuid._lanscan_getnode()
-        self.check_node(node, 'lanscan', True)
-
-    @unittest.skipUnless(os.name == 'posix', 'requires Posix')
-    def test_netstat_getnode(self):
-        node = uuid._netstat_getnode()
-        self.check_node(node, 'netstat', True)
-
-    @unittest.skipUnless(os.name == 'nt', 'requires Windows')
-    def test_ipconfig_getnode(self):
-        node = uuid._ipconfig_getnode()
-        self.check_node(node, 'ipconfig', True)
-
-    @unittest.skipUnless(importable('win32wnet'), 'requires win32wnet')
-    @unittest.skipUnless(importable('netbios'), 'requires netbios')
-    def test_netbios_getnode(self):
-        node = uuid._netbios_getnode()
-        self.check_node(node, network=True)
-
-    def test_random_getnode(self):
-        node = uuid._random_getnode()
-        # Least significant bit of first octet must be set.
-        self.assertTrue(node & 0x010000000000, '%012x' % node)
-        self.check_node(node)
-
-    @unittest.skipUnless(os.name == 'posix', 'requires Posix')
-    @unittest.skipUnless(importable('ctypes'), 'requires ctypes')
-    def test_unixdll_getnode(self):
-        try: # Issues 1481, 3581: _uuid_generate_time() might be None.
-            node = uuid._unixdll_getnode()
-        except TypeError:
-            self.skipTest('requires uuid_generate_time')
-        self.check_node(node)
-
-    @unittest.skipUnless(os.name == 'nt', 'requires Windows')
-    @unittest.skipUnless(importable('ctypes'), 'requires ctypes')
-    def test_windll_getnode(self):
-        node = uuid._windll_getnode()
-        self.check_node(node)
 
 
 def test_main():
-    test_support.run_unittest(TestUUID, TestInternals)
+    test_support.run_unittest(TestUUID)
 
 if __name__ == '__main__':
     test_main()

from_cpython/Lib/types.py

@@ -60,6 +60,7 @@ BuiltinFunctionType = type(len)
 # Pyston change:
 # BuiltinMethodType = type([].append)     # Same as BuiltinFunctionType
 BuiltinMethodType = type((1.0).hex)     # Same as BuiltinFunctionType
+BuiltinCAPIFunctionType = type(reload)  # Pyston change: added this type
 
 ModuleType = type(sys)
 FileType = file

src/capi/modsupport.cpp

@@ -440,7 +440,8 @@ extern "C" PyObject* Py_InitModule4(const char* name, PyMethodDef* methods, cons
     Box* passthrough = static_cast<Box*>(self);
 
     while (methods && methods->ml_name) {
-        module->giveAttr(methods->ml_name, new BoxedCApiFunction(methods, passthrough, boxString(name)));
+        module->setattr(internStringMortal(methods->ml_name),
+                        new BoxedCApiFunction(methods, passthrough, boxString(name)), NULL);
 
         methods++;
     }

src/runtime/import.cpp

@@ -49,6 +49,10 @@ extern "C" PyObject* PyImport_GetModuleDict(void) noexcept {
 extern "C" PyObject* _PyImport_LoadDynamicModule(char* name, char* pathname, FILE* fp) noexcept {
     BoxedString* name_boxed = boxString(name);
     try {
+        PyObject* m = _PyImport_FindExtension(name, pathname);
+        if (m != NULL)
+            return m;
+
         const char* lastdot = strrchr(name, '.');
         const char* shortname;
         if (lastdot == NULL) {
@@ -57,7 +61,12 @@ extern "C" PyObject* _PyImport_LoadDynamicModule(char* name, char* pathname, FIL
             shortname = lastdot + 1;
         }
 
-        return importCExtension(name_boxed, shortname, pathname);
+        m = importCExtension(name_boxed, shortname, pathname);
+
+        if (_PyImport_FixupExtension(name, pathname) == NULL)
+            return NULL;
+
+        return m;
     } catch (ExcInfo e) {
         removeModule(name_boxed);
         setCAPIException(e);

src/runtime/long.cpp

@@ -115,14 +115,18 @@ extern "C" int _PyLong_AsInt(PyObject* obj) noexcept {
 extern "C" unsigned long PyLong_AsUnsignedLongMask(PyObject* vv) noexcept {
     if (PyLong_Check(vv)) {
         BoxedLong* l = static_cast<BoxedLong*>(vv);
-        return mpz_get_ui(l->n);
+        auto v = mpz_get_ui(l->n);
+        if (mpz_sgn(l->n) == -1)
+            return -v;
+        return v;
     }
 
     Py_FatalError("unimplemented");
 }
 
 extern "C" unsigned PY_LONG_LONG PyLong_AsUnsignedLongLongMask(PyObject* vv) noexcept {
-    Py_FatalError("unimplemented");
+    static_assert(sizeof(PY_LONG_LONG) == sizeof(unsigned long), "");
+    return PyLong_AsUnsignedLongMask(vv);
 }
 
 extern "C" PY_LONG_LONG PyLong_AsLongLong(PyObject* vv) noexcept {

test/extra/cffi-1.2.1.patch

+diff -ur cffi-1.2.1_orig/c/_cffi_backend.c cffi-1.2.1/c/_cffi_backend.c
+--- cffi-1.2.1_orig/c/_cffi_backend.c	2016-02-22 18:48:06.861772302 +0000
++++ cffi-1.2.1/c/_cffi_backend.c	2016-02-24 18:57:47.479889228 +0000
+@@ -5065,6 +5065,10 @@
+     cd->c_weakreflist = NULL;
+     PyObject_GC_Track(cd);
+ 
++    // Pyston change: HACK which leaks mem but seems necessary at the moment to prevent crashes with our GC
++    PyGC_AddRoot((PyObject*)cd);
++    PyGC_AddRoot((PyObject*)infotuple);
++
+     cif_descr = (cif_description_t *)ct->ct_extra;
+     if (cif_descr == NULL) {
+         PyErr_Format(PyExc_NotImplementedError,
+@@ -5572,7 +5576,9 @@
+         return NULL;
+     }
+     x = (PyObject *)(raw + 42);
+-    if (Py_REFCNT(x) <= 0) {
++    // Pyston change:
++    // if (Py_REFCNT(x) <= 0) {
++    if (0) {
+         Py_FatalError("ffi.from_handle() detected that the address passed "
+                       "points to garbage. If it is really the result of "
+                       "ffi.new_handle(), then the Python object has already "
+diff -ur cffi-1.2.1_orig/c/cglob.c cffi-1.2.1/c/cglob.c
+--- cffi-1.2.1_orig/c/cglob.c	2016-02-22 18:48:06.853772302 +0000
++++ cffi-1.2.1/c/cglob.c	2016-02-25 11:02:26.319738004 +0000
+@@ -46,7 +46,15 @@
+ static PyObject *make_global_var(PyObject *name, CTypeDescrObject *type,
+                                  char *addr, gs_fetch_addr_fn fetch_addr)
+ {
+-    GlobSupportObject *gs = PyObject_New(GlobSupportObject, &GlobSupport_Type);
++    // Pyston change:
++    static int has_registered = 0;    
++    GlobSupportObject *gs;
++    if (!has_registered) {
++        PyType_Ready(&GlobSupport_Type);
++        has_registered = 1;
++    }
++
++    gs = PyObject_New(GlobSupportObject, &GlobSupport_Type);
+     if (gs == NULL)
+         return NULL;
+ 

test/extra/cffi_test.py

+import os, sys, subprocess, shutil
+sys.path.append(os.path.dirname(__file__) + "/../lib")
+
+from test_helper import create_virtenv, run_test
+
+ENV_NAME = "cffi_test_env_" + os.path.basename(sys.executable)
+SRC_DIR = os.path.abspath(os.path.join(ENV_NAME, "src"))
+PYTHON_EXE = os.path.abspath(os.path.join(ENV_NAME, "bin", "python"))
+PYTEST_EXE = os.path.abspath(os.path.join(ENV_NAME, "bin", "py.test"))
+
+def install_and_test_cffi():
+    shutil.rmtree(SRC_DIR, ignore_errors=True)
+    os.makedirs(SRC_DIR)
+
+    url = "https://pypi.python.org/packages/source/c/cffi/cffi-1.2.1.tar.gz"
+    subprocess.check_call(["wget", url], cwd=SRC_DIR)
+    subprocess.check_call(["tar", "-zxf", "cffi-1.2.1.tar.gz"], cwd=SRC_DIR)
+    CFFI_DIR = os.path.abspath(os.path.join(SRC_DIR, "cffi-1.2.1"))
+
+    PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "cffi-1.2.1.patch"))
+    subprocess.check_call(["patch", "-p1", "--input=" + PATCH_FILE], cwd=CFFI_DIR)
+
+    subprocess.check_call([PYTHON_EXE, "setup.py", "install"], cwd=CFFI_DIR)
+
+    # looks like clang 3.5 causes more errors like: 214 != -42 doing casts
+    if os.environ.has_key("CC") and "clang" in os.environ["CC"]:
+        expected = [{ "failed": 58, "passed": 1619, "skipped": 70, "xfailed": 4, "error": 5 }]
+    else:
+        expected = [{ "failed": 49, "passed": 1628, "skipped": 70, "xfailed": 4, "error": 5 }]
+    run_test([PYTEST_EXE], cwd=CFFI_DIR, expected=expected)
+   
+create_virtenv(ENV_NAME, ["pytest==2.8.7", "py==1.4.31", "pycparser==2.14"], force_create = True)
+install_and_test_cffi()

test/lib/test_helper.py

@@ -37,15 +37,24 @@ def pip_install(name, package_list):
 def parse_output(output):
     result = []
     for l in output.split('\n'):
+        # nosetest
         m = re.match("Ran (\d+) tests in", l)
         if m:
             result.append({"ran": int(m.group(1))})
             continue
         for res_type in ("errors", "failures", "skipped"):
-	    m = re.match("FAILED \(.*%s=(\d+).*\)" % res_type, l)
-	    if m:
-	        result[-1][res_type] = int(m.group(1))
-	
+            m = re.match("FAILED \(.*%s=(\d+).*\)" % res_type, l)
+            if m:
+                result[-1][res_type] = int(m.group(1))
+
+        # py.test
+        m = re.match(".* in \d+[.]\d+ seconds [=]*", l)
+        if m:
+            result.append({})
+            for res_type in ("failed", "passed", "skipped", "xfailed", "error"):
+                m = re.match(".* (\d+) %s.*" % res_type, l)
+                if m:
+                    result[-1][res_type] = int(m.group(1))
     return result
 
 def run_test(cmd, cwd, expected, env = None):

test/tests/ctypes_test.py

-# skip-if: True
 from ctypes import *
 s = "tmp"
 ap = create_string_buffer(s)

test/tests/type_flags.py

@@ -10,7 +10,7 @@ def test(cls):
 
 def testall(module):
     for n in sorted(dir((module))):
-        if n in ("reversed", "AttrwrapperType", "BuiltinMethodType", "BufferType", "DictProxyType"):
+        if n in ("reversed", "AttrwrapperType", "BuiltinMethodType", "BufferType", "DictProxyType", "BuiltinCAPIFunctionType"):
             continue
 
         cls = getattr(module, n)