Add more external projects to our travis-ci test suite

.travis.yml

@@ -54,7 +54,8 @@ script:
   - ninja -j4 pyston
   - ccache -s
   - PYSTON_RUN_ARGS=G ninja check-pyston
-  - PYSTON_RUN_ARGS=G python $TRAVIS_BUILD_DIR/tools/tester.py -R ./pyston -a=-S -k -t180 --exit-code-only $TRAVIS_BUILD_DIR/test/extra
+  - mysql -e 'create database mysqldb_test charset utf8;'
+  - PYSTON_RUN_ARGS=G python $TRAVIS_BUILD_DIR/tools/tester.py -R ./pyston -a=-S -k -t600 --exit-code-only $TRAVIS_BUILD_DIR/test/extra
 
 os:
   - linux

test/extra/cheetah_test.py

+import os, sys
+from test_helper import create_virtenv, run_test
+
+ENV_NAME = os.path.abspath("cheetah_test_env_" + os.path.basename(sys.executable))
+create_virtenv(ENV_NAME, ["cheetah==2.4.4"], force_create = True)
+
+cheetah_exe = os.path.join(ENV_NAME, "bin", "cheetah")
+env = os.environ
+env["PATH"] = env["PATH"] + ":" + os.path.join(ENV_NAME, "bin")
+expected = [{'errors': 4, 'failures': 53}, {'errors': 232, 'failures': 53}]
+run_test([cheetah_exe, "test"], cwd=ENV_NAME, expected=expected, env=env)

test/extra/mysqldb_0001-Pyston-change-register-types.patch

+From e1e415a1738fd296f83d0b1ce7ad8e7a1682d97a Mon Sep 17 00:00:00 2001
+From: Marius Wachtler <undingen@gmail.com>
+Date: Mon, 8 Jun 2015 17:35:14 +0200
+Subject: [PATCH] Pyston change: register types
+
+---
+ _mysql.c | 5 +++++
+ 1 file changed, 5 insertions(+)
+
+diff --git a/_mysql.c b/_mysql.c
+index 5b81c79..718d3f8 100644
+--- a/_mysql.c
++++ b/_mysql.c
+@@ -3086,6 +3086,11 @@ init_mysql(void)
+ 	_mysql_ResultObject_Type.tp_free = _PyObject_GC_Del;
+ #endif
+ #endif
++	// Pyston change:
++	if (PyType_Ready(&_mysql_ConnectionObject_Type) < 0)
++		return;
++	if (PyType_Ready(&_mysql_ResultObject_Type) < 0)
++		return;
+ 
+ 	if (!(dict = PyModule_GetDict(module))) goto error;
+ 	if (PyDict_SetItemString(dict, "version_info",
+-- 
+2.1.4
+

test/extra/mysqldb_test.py

+# This tests needs mysql and a database:
+# mysql -e 'create database mysqldb_test charset utf8;'
+
+import os, sys, subprocess, shutil
+from test_helper import create_virtenv, run_test
+
+ENV_NAME = "mysqldb_test_env_" + os.path.basename(sys.executable)
+SRC_DIR = os.path.abspath(os.path.join(ENV_NAME, "src"))
+PYTHON_EXE = os.path.abspath(os.path.join(ENV_NAME, "bin", "python"))
+
+def install_and_test_mysqldb():
+    shutil.rmtree(SRC_DIR, ignore_errors=True)
+    os.makedirs(SRC_DIR)
+    subprocess.check_call(["git", "clone", "https://github.com/farcepest/MySQLdb1.git"], cwd=SRC_DIR)
+
+    MYSQLDB_DIR = os.path.abspath(os.path.join(SRC_DIR, "MySQLdb1"))
+    subprocess.check_call(["git", "checkout", "MySQLdb-1.2.5"], cwd=MYSQLDB_DIR)      
+    
+    nosetests_exe = os.path.abspath(ENV_NAME + "/bin/nosetests")
+
+    #apply patch
+    PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "mysqldb_0001-Pyston-change-register-types.patch"))
+    subprocess.check_call(["patch", "-p1", "--input=" + PATCH_FILE], cwd=MYSQLDB_DIR)
+    print "Applied mysqldb patch"
+
+    subprocess.check_call([PYTHON_EXE, "setup.py", "build"], cwd=MYSQLDB_DIR)
+    subprocess.check_call([PYTHON_EXE, "setup.py", "install"], cwd=MYSQLDB_DIR)
+
+    env = os.environ
+    env["TESTDB"] = "travis.cnf" 
+    expected = []
+    run_test([nosetests_exe], cwd=MYSQLDB_DIR, expected=expected, env=env)
+
+packages = ["nose==1.3.7"]
+create_virtenv(ENV_NAME, packages, force_create = True)
+install_and_test_mysqldb()

test/extra/protobuf_test.py

+import os, sys, subprocess, shutil
+from test_helper import create_virtenv, run_test
+
+ENV_NAME = "protobuf_test_env_" + os.path.basename(sys.executable)
+SRC_DIR = os.path.abspath(os.path.join(ENV_NAME, "src"))
+PYTHON_EXE = os.path.abspath(os.path.join(ENV_NAME, "bin", "python"))
+
+def install_and_test_protobuf():
+    shutil.rmtree(SRC_DIR, ignore_errors=True)
+    os.makedirs(SRC_DIR)
+
+    url = "http://archive.ubuntu.com/ubuntu/pool/main/p/protobuf/protobuf_2.5.0.orig.tar.gz"
+    subprocess.check_call(["wget", url], cwd=SRC_DIR)
+    subprocess.check_call(["tar", "-zxf", "protobuf_2.5.0.orig.tar.gz"], cwd=SRC_DIR)
+    PROTOBUF_DIR = os.path.abspath(os.path.join(SRC_DIR, "protobuf-2.5.0"))
+
+    INSTALL_DIR = os.path.join(SRC_DIR, "protobuf_install")
+    subprocess.check_call(["./configure", "--prefix="+INSTALL_DIR], cwd=PROTOBUF_DIR)
+    subprocess.check_call(["make", "-j4"], cwd=PROTOBUF_DIR)
+    subprocess.check_call(["make", "install"], cwd=PROTOBUF_DIR)
+    
+    PROTOBUF_PY_DIR = os.path.join(PROTOBUF_DIR, "python")
+    env = os.environ
+    env["PATH"] = env["PATH"] + ":" + os.path.join(INSTALL_DIR, "bin")
+    env["LD_LIBRARY_PATH"] = os.path.join(INSTALL_DIR, "lib")
+    subprocess.check_call([PYTHON_EXE, "setup.py", "build"], cwd=PROTOBUF_PY_DIR, env=env)
+    
+    expected = [{"failures": 0, "errors": 1}]
+    run_test([PYTHON_EXE, "setup.py", "test"], cwd=PROTOBUF_PY_DIR, expected=expected, env=env)
+    
+create_virtenv(ENV_NAME, None, force_create = True)
+install_and_test_protobuf()

test/extra/pyicu_101.patch

+diff -ur PyICU-1.0.1_o/common.h PyICU-1.0.1/common.h
+--- PyICU-1.0.1_o/common.h	2010-03-29 20:04:02.000000000 +0200
++++ PyICU-1.0.1/common.h	2015-05-18 22:38:10.625582065 +0200
+@@ -199,9 +199,15 @@
+ 
+ #else
+ 
++// Pyston change:
++/*
+ #define parseArgs(args, types, rest...) \
+     _parseArgs(((PyTupleObject *)(args))->ob_item, \
+                ((PyTupleObject *)(args))->ob_size, types, ##rest)
++*/
++#define parseArgs(args, types, rest...) \
++    _parseArgs(PyTuple_Items(args), PyTuple_Size(args), types, ##rest)
++
+ 
+ #define parseArg(arg, types, rest...) \
+     _parseArgs(&(arg), 1, types, ##rest)
+diff -ur PyICU-1.0.1_o/_icu.cpp PyICU-1.0.1/_icu.cpp
+--- PyICU-1.0.1_o/_icu.cpp	2010-04-02 00:12:54.000000000 +0200
++++ PyICU-1.0.1/_icu.cpp	2015-05-19 15:26:15.131375981 +0200
+@@ -193,6 +193,8 @@
+     if (!PyArg_ParseTuple(args, "Os", &object, &doc))
+         return NULL;
+ 
++// Pyston change:
++#if 0
+     /* constructors */
+     if (PyObject_TypeCheck(object, &PyWrapperDescr_Type))
+     {
+@@ -206,6 +208,14 @@
+         ((PyMethodDescrObject *) object)->d_method->ml_doc = strdup(doc);
+         Py_RETURN_NONE;
+     }
++#else
++    if (PyObject_TypeCheck(object, &PyWrapperDescr_Type) ||
++        PyObject_TypeCheck(object, &PyMethod_Type) ||
++        PyObject_TypeCheck(object, _method_type)) {
++        // Ignore the doc string for now.
++        Py_RETURN_NONE;
++    }
++#endif
+ 
+     /* class methods */
+     if (PyObject_TypeCheck(object, &PyCFunction_Type))

test/extra/pyicu_test.py

+import os, sys, subprocess, shutil
+from test_helper import create_virtenv, run_test
+
+ENV_NAME = "pyicu_test_env_" + os.path.basename(sys.executable)
+SRC_DIR = os.path.abspath(os.path.join(ENV_NAME, "src"))
+PYTHON_EXE = os.path.abspath(os.path.join(ENV_NAME, "bin", "python"))
+
+def install_and_test_pyicu():
+    shutil.rmtree(SRC_DIR, ignore_errors=True)
+    os.makedirs(SRC_DIR)
+
+    url = "http://download.icu-project.org/files/icu4c/4.2.1/icu4c-4_2_1-src.tgz"
+    subprocess.check_call(["wget", url], cwd=SRC_DIR)
+    subprocess.check_call(["tar", "-zxf", "icu4c-4_2_1-src.tgz"], cwd=SRC_DIR)
+    ICU_DIR = os.path.abspath(os.path.join(SRC_DIR, "icu", "source"))
+    
+    INSTALL_DIR = os.path.join(SRC_DIR, "icu_install")
+    subprocess.check_call(["./runConfigureICU", "Linux", "--prefix=" + INSTALL_DIR], cwd=ICU_DIR)
+    subprocess.check_call(["make", "-j4"], cwd=ICU_DIR)
+    subprocess.check_call(["make", "install"], cwd=ICU_DIR)
+    
+    url = "https://pypi.python.org/packages/source/P/PyICU/PyICU-1.0.1.tar.gz"
+    subprocess.check_call(["wget", url], cwd=SRC_DIR)
+    subprocess.check_call(["tar", "-zxf", "PyICU-1.0.1.tar.gz"], cwd=SRC_DIR)
+    PYICU_DIR = os.path.abspath(os.path.join(SRC_DIR, "PyICU-1.0.1"))
+
+    PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "pyicu_101.patch"))
+    subprocess.check_call(["patch", "-p1", "--input=" + PATCH_FILE], cwd=PYICU_DIR)
+    print "Applied PyICU patch"
+
+    env = os.environ
+    INC_DIR = os.path.abspath(os.path.join(INSTALL_DIR, "include"))
+    LIB_DIR = os.path.abspath(os.path.join(INSTALL_DIR, "lib"))
+    env["CFLAGS"] = env["CXXFLAGS"] = " ".join(["-I" + INC_DIR, "-L" + LIB_DIR])
+    env["LD_LIBRARY_PATH"] = LIB_DIR
+    subprocess.check_call([PYTHON_EXE, "setup.py", "build"], cwd=PYICU_DIR, env=env)
+    subprocess.check_call([PYTHON_EXE, "setup.py", "install"], cwd=PYICU_DIR, env=env)
+    
+    expected = []
+    run_test([PYTHON_EXE, "setup.py", "test"], cwd=PYICU_DIR, expected=expected)
+    
+create_virtenv(ENV_NAME, None, force_create = True)
+install_and_test_pyicu()

test/extra/test_helper.py

+# skip-if: True
+import os
+import sys
+import subprocess
+import re
+
+def create_virtenv(name, package_list = None, force_create = False):
+    if force_create or not os.path.exists(name) or os.stat(sys.executable).st_mtime > os.stat(name + "/bin/python").st_mtime:
+        if os.path.exists(name):
+            subprocess.check_call(["rm", "-rf", name])
+        
+        print "Creating virtualenv to install testing dependencies..."
+        
+        VIRTUALENV_SCRIPT = os.path.dirname(__file__) + "/../integration/virtualenv/virtualenv.py"
+
+        try:
+            args = [sys.executable, VIRTUALENV_SCRIPT, "-p", sys.executable, name]
+            print "Running", args
+            subprocess.check_call(args)
+
+            if package_list:
+                pip_install(name, package_list)
+        except:
+            print "Error occurred; trying to remove partially-created directory"
+            ei = sys.exc_info()
+            try:
+                subprocess.check_call(["rm", "-rf", name])
+            except Exception as e:
+                print e
+            raise ei[0], ei[1], ei[2]
+    else:
+        print "Reusing existing virtualenv"
+
+def pip_install(name, package_list):
+    subprocess.check_call([name + "/bin/pip", "install"] + package_list)
+
+def parse_output(output):
+    result = []
+    it = re.finditer("FAILED \(failures=(\d+), errors=(\d+)\)", output)
+    for m in it:
+        d = { "failures" : int(m.group(1)), "errors" : int(m.group(2)) }
+        result.append(d)
+    it = re.finditer("FAILED \(errors=(\d+), failures=(\d+)\)", output)
+    for m in it:
+        d = { "failures" : int(m.group(2)), "errors" : int(m.group(1)) }
+        result.append(d)
+    it = re.finditer("FAILED \(failures=(\d+)\)", output)
+    for m in it:
+        d = { "failures" : int(m.group(1)), "errors" : 0 }
+        result.append(d)
+    it = re.finditer("FAILED \(errors=(\d+)\)", output)
+    for m in it:
+        d = { "failures" : 0, "errors" : int(m.group(1)) }
+        result.append(d)
+    return result
+
+def run_test(cmd, cwd, expected, env = None):
+    process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd, env=env)
+    output, unused_err = process.communicate()
+    errcode = process.poll()
+    result = parse_output(output)
+    
+    print
+    print "Return code:", errcode
+    if expected == result:
+        print "Received expected output"
+    else:
+        print >> sys.stderr, output
+        print >> sys.stderr, "WRONG output"
+        print >> sys.stderr, "is:", result
+        print >> sys.stderr, "expected:", expected
+        assert result == expected
+