merge

Cython/Build/BuildExecutable.py

@@ -13,17 +13,27 @@ import sys
 import os
 from distutils import sysconfig
 
-INCDIR = sysconfig.get_python_inc()
-LIBDIR1 = sysconfig.get_config_var('LIBDIR')
-LIBDIR2 = sysconfig.get_config_var('LIBPL')
-PYLIB = sysconfig.get_config_var('LIBRARY')[3:-2]
+def get_config_var(name):
+    return sysconfig.get_config_var(name) or ''
 
-CC = sysconfig.get_config_var('CC')
-CFLAGS = sysconfig.get_config_var('CFLAGS') + ' ' + os.environ.get('CFLAGS', '')
-LINKCC = sysconfig.get_config_var('LINKCC')
-LINKFORSHARED = sysconfig.get_config_var('LINKFORSHARED')
-LIBS = sysconfig.get_config_var('LIBS')
-SYSLIBS = sysconfig.get_config_var('SYSLIBS')
+INCDIR = sysconfig.get_python_inc()
+LIBDIR1 = get_config_var('LIBDIR')
+LIBDIR2 = get_config_var('LIBPL')
+PYLIB = get_config_var('LIBRARY')
+PYLIB_DYN = get_config_var('LDLIBRARY')
+if PYLIB_DYN == PYLIB:
+    # no shared library
+    PYLIB_DYN = ''
+else:
+    PYLIB_DYN = os.path.splitext(PYLIB_DYN[3:])[0] # 'lib(XYZ).so' -> XYZ
+
+CC = get_config_var('CC')
+CFLAGS = get_config_var('CFLAGS') + ' ' + os.environ.get('CFLAGS', '')
+LINKCC = get_config_var('LINKCC')
+LINKFORSHARED = get_config_var('LINKFORSHARED')
+LIBS = get_config_var('LIBS')
+SYSLIBS = get_config_var('SYSLIBS')
+EXE_EXT = sysconfig.get_config_var('EXE')
 
 def _debug(msg, *args):
     if DEBUG:
@@ -36,12 +46,14 @@ def dump_config():
     _debug('LIBDIR1: %s', LIBDIR1)
     _debug('LIBDIR2: %s', LIBDIR2)
     _debug('PYLIB: %s', PYLIB)
+    _debug('PYLIB_DYN: %s', PYLIB_DYN)
     _debug('CC: %s', CC)
     _debug('CFLAGS: %s', CFLAGS)
     _debug('LINKCC: %s', LINKCC)
     _debug('LINKFORSHARED: %s', LINKFORSHARED)
     _debug('LIBS: %s', LIBS)
     _debug('SYSLIBS: %s', SYSLIBS)
+    _debug('EXE_EXT: %s', EXE_EXT)
 
 def runcmd(cmd, shell=True):
     if shell:
@@ -61,7 +73,8 @@ def runcmd(cmd, shell=True):
         sys.exit(returncode)
 
 def clink(basename):
-    runcmd([LINKCC, '-o', basename, basename+'.o', '-L'+LIBDIR1, '-L'+LIBDIR2, '-l'+PYLIB]
+    runcmd([LINKCC, '-o', basename + EXE_EXT, basename+'.o', '-L'+LIBDIR1, '-L'+LIBDIR2]
+           + [PYLIB_DYN and ('-l'+PYLIB_DYN) or os.path.join(LIBDIR1, PYLIB)]
            + LIBS.split() + SYSLIBS.split() + LINKFORSHARED.split())
 
 def ccompile(basename):
@@ -75,8 +88,8 @@ def cycompile(input_file, options=()):
     if result.num_errors > 0:
         sys.exit(1)
 
-def exec_file(basename, args=()):
-    runcmd([os.path.abspath(basename)] + list(args), shell=False)
+def exec_file(program_name, args=()):
+    runcmd([os.path.abspath(program_name)] + list(args), shell=False)
 
 def build(input_file, compiler_args=()):
     """
@@ -88,7 +101,7 @@ def build(input_file, compiler_args=()):
     cycompile(input_file, compiler_args)
     ccompile(basename)
     clink(basename)
-    return basename
+    return basename + EXE_EXT
 
 def build_and_run(args):
     """
@@ -114,3 +127,6 @@ def build_and_run(args):
 
     program_name = build(input_file, cy_args)
     exec_file(program_name, args)
+
+if __name__ == '__main__':
+    build_and_run(sys.argv[1:])

Cython/Compiler/Builtin.py

@@ -95,6 +95,54 @@ static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) {
 }
 """)
 
+globals_utility_code = UtilityCode(
+# This is a stub implementation until we have something more complete.
+# Currently, we only handle the most common case of a read-only dict
+# of Python names.  Supporting cdef names in the module and write
+# access requires a rewrite as a dedicated class.
+proto = """
+static PyObject* __Pyx_Globals(); /*proto*/
+""",
+impl = '''
+static PyObject* __Pyx_Globals() {
+    Py_ssize_t i;
+    /*PyObject *d;*/
+    PyObject *names = NULL;
+    PyObject *globals = PyObject_GetAttrString(%(MODULE)s, "__dict__");
+    if (!globals) {
+        PyErr_SetString(PyExc_TypeError,
+            "current module must have __dict__ attribute");
+        goto bad;
+    }
+    names = PyObject_Dir(%(MODULE)s);
+    if (!names)
+        goto bad;
+    for (i = 0; i < PyList_GET_SIZE(names); i++) {
+        PyObject* name = PyList_GET_ITEM(names, i);
+        if (!PyDict_Contains(globals, name)) {
+            PyObject* value = PyObject_GetAttr(%(MODULE)s, PyList_GET_ITEM(names, i));
+            if (!value)
+                goto bad;
+            if (PyDict_SetItem(globals, name, value) < 0) {
+                Py_DECREF(value);
+                goto bad;
+            }
+        }
+    }
+    Py_DECREF(names);
+    return globals;
+    /*
+    d = PyDictProxy_New(globals);
+    Py_DECREF(globals);
+    return d;
+    */
+bad:
+    Py_XDECREF(names);
+    Py_XDECREF(globals);
+    return NULL;
+}
+''' % {'MODULE' : Naming.module_cname})
+
 pyexec_utility_code = UtilityCode(
 proto = """
 #if PY_VERSION_HEX < 0x02040000
@@ -384,6 +432,8 @@ builtin_function_table = [
                     utility_code = getattr3_utility_code),
     BuiltinFunction('getattr3',   "OOO",  "O",     "__Pyx_GetAttr3",     "getattr",
                     utility_code = getattr3_utility_code), # Pyrex compatibility
+    BuiltinFunction('globals',    "",     "O",     "__Pyx_Globals",
+                    utility_code = globals_utility_code),
     BuiltinFunction('hasattr',    "OO",   "b",     "__Pyx_HasAttr",
                     utility_code = hasattr_utility_code),
     BuiltinFunction('hash',       "O",    "h",     "PyObject_Hash"),

Cython/Compiler/CmdLine.py

@@ -163,15 +163,7 @@ def parse_command_line(args):
                 sys.stderr.write("Unknown compiler flag: %s\n" % option)
                 sys.exit(1)
         else:
-            arg = pop_arg()
-            if arg.endswith(".pyx"):
-                sources.append(arg)
-            elif arg.endswith(".py"):
-                # maybe do some other stuff, but this should work for now
-                sources.append(arg)
-            else:
-                sys.stderr.write(
-                    "cython: %s: Unknown filename suffix\n" % arg)
+            sources.append(pop_arg())
     if options.use_listing_file and len(sources) > 1:
         sys.stderr.write(
             "cython: Only one source file allowed when using -o\n")

Cython/Compiler/Code.pxd

@@ -59,9 +59,10 @@ cdef class StringConst:
     cdef public object text
     cdef public object escaped_value
     cdef public dict py_strings
+    cdef public list py_versions
 
     @cython.locals(intern=bint, is_str=bint, is_unicode=bint)
-    cpdef get_py_string_const(self, encoding, identifier=*, is_str=*)
+    cpdef get_py_string_const(self, encoding, identifier=*, is_str=*, py3str_cstring=*)
 
 ## cdef class PyStringConst:
 ##     cdef public object cname

Cython/Compiler/Code.py

@@ -25,12 +25,19 @@ except ImportError:
 
 
 non_portable_builtins_map = {
+    # builtins that have different names in different Python versions
     'bytes'         : ('PY_MAJOR_VERSION < 3',  'str'),
     'unicode'       : ('PY_MAJOR_VERSION >= 3', 'str'),
     'xrange'        : ('PY_MAJOR_VERSION >= 3', 'range'),
     'BaseException' : ('PY_VERSION_HEX < 0x02050000', 'Exception'),
     }
 
+uncachable_builtins = [
+    # builtin names that cannot be cached because they may or may not
+    # be available at import time
+    'WindowsError',
+    ]
+
 class UtilityCode(object):
     # Stores utility code to add during code generation.
     #
@@ -324,8 +331,16 @@ class StringConst(object):
         self.text = text
         self.escaped_value = StringEncoding.escape_byte_string(byte_string)
         self.py_strings = None
+        self.py_versions = []
 
-    def get_py_string_const(self, encoding, identifier=None, is_str=False):
+    def add_py_version(self, version):
+        if not version:
+            self.py_versions = [2,3]
+        elif version not in self.py_versions:
+            self.py_versions.append(version)
+
+    def get_py_string_const(self, encoding, identifier=None,
+                            is_str=False, py3str_cstring=None):
         py_strings = self.py_strings
         text = self.text
 
@@ -344,12 +359,15 @@ class StringConst(object):
             else:
                 encoding_key = ''.join(find_alphanums(encoding))
 
-        key = (is_str, is_unicode, encoding_key)
-        if py_strings is not None and key in py_strings:
-            py_string = py_strings[key]
+        key = (is_str, is_unicode, encoding_key, py3str_cstring)
+        if py_strings is not None:
+            try:
+                return py_strings[key]
+            except KeyError:
+                pass
         else:
-            if py_strings is None:
             self.py_strings = {}
+
         if identifier:
             intern = True
         elif identifier is None:
@@ -369,22 +387,24 @@ class StringConst(object):
             self.cname[len(Naming.const_prefix):])
 
         py_string = PyStringConst(
-                pystring_cname, encoding, is_unicode, is_str, intern)
+            pystring_cname, encoding, is_unicode, is_str, py3str_cstring, intern)
         self.py_strings[key] = py_string
-
         return py_string
 
 class PyStringConst(object):
     """Global info about a Python string constant held by GlobalState.
     """
     # cname       string
+    # py3str_cstring string
     # encoding    string
     # intern      boolean
     # is_unicode  boolean
     # is_str      boolean
 
-    def __init__(self, cname, encoding, is_unicode, is_str=False, intern=False):
+    def __init__(self, cname, encoding, is_unicode, is_str=False,
+                 py3str_cstring=None, intern=False):
         self.cname = cname
+        self.py3str_cstring = py3str_cstring
         self.encoding = encoding
         self.is_str = is_str
         self.is_unicode = is_unicode
@@ -595,7 +615,7 @@ class GlobalState(object):
             cleanup_writer.put_xdecref_clear(const.cname, type, nanny=False)
         return const
 
-    def get_string_const(self, text):
+    def get_string_const(self, text, py_version=None):
         # return a C string constant, creating a new one if necessary
         if text.is_unicode:
             byte_string = text.utf8encode()
@@ -605,12 +625,21 @@ class GlobalState(object):
             c = self.string_const_index[byte_string]
         except KeyError:
             c = self.new_string_const(text, byte_string)
+        c.add_py_version(py_version)
         return c
 
-    def get_py_string_const(self, text, identifier=None, is_str=False):
+    def get_py_string_const(self, text, identifier=None,
+                            is_str=False, unicode_value=None):
         # return a Python string constant, creating a new one if necessary
+        py3str_cstring = None
+        if is_str and unicode_value is not None \
+               and unicode_value.utf8encode() != text.byteencode():
+            py3str_cstring = self.get_string_const(unicode_value, py_version=3)
+            c_string = self.get_string_const(text, py_version=2)
+        else:
             c_string = self.get_string_const(text)
-        py_string = c_string.get_py_string_const(text.encoding, identifier, is_str)
+        py_string = c_string.get_py_string_const(
+            text.encoding, identifier, is_str, py3str_cstring)
         return py_string
 
     def get_interned_identifier(self, text):
@@ -711,8 +740,15 @@ class GlobalState(object):
 
         decls_writer = self.parts['decls']
         for _, cname, c in c_consts:
+            conditional = False
+            if c.py_versions and (2 not in c.py_versions or 3 not in c.py_versions):
+                conditional = True
+                decls_writer.putln("#if PY_MAJOR_VERSION %s 3" % (
+                    (2 in c.py_versions) and '<' or '>='))
             decls_writer.putln('static char %s[] = "%s";' % (
                 cname, StringEncoding.split_string_literal(c.escaped_value)))
+            if conditional:
+                decls_writer.putln("#endif")
             if c.py_strings is not None:
                 for py_string in c.py_strings.values():
                     py_strings.append((c.cname, len(py_string.cname), py_string))
@@ -736,6 +772,17 @@ class GlobalState(object):
 
                 decls_writer.putln(
                     "static PyObject *%s;" % py_string.cname)
+                if py_string.py3str_cstring:
+                    w.putln("#if PY_MAJOR_VERSION >= 3")
+                    w.putln(
+                        "{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % (
+                        py_string.cname,
+                        py_string.py3str_cstring.cname,
+                        py_string.py3str_cstring.cname,
+                        '0', 1, 0,
+                        py_string.intern
+                        ))
+                    w.putln("#else")
                 w.putln(
                     "{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % (
                     py_string.cname,
@@ -746,6 +793,8 @@ class GlobalState(object):
                     py_string.is_str,
                     py_string.intern
                     ))
+                if py_string.py3str_cstring:
+                    w.putln("#endif")
             w.putln("{0, 0, 0, 0, 0, 0, 0}")
             w.putln("};")
 
@@ -1003,8 +1052,10 @@ class CCodeWriter(object):
     def get_string_const(self, text):
         return self.globalstate.get_string_const(text).cname
 
-    def get_py_string_const(self, text, identifier=None, is_str=False):
-        return self.globalstate.get_py_string_const(text, identifier, is_str).cname
+    def get_py_string_const(self, text, identifier=None,
+                            is_str=False, unicode_value=None):
+        return self.globalstate.get_py_string_const(
+            text, identifier, is_str, unicode_value).cname
 
     def get_argument_default_const(self, type):
         return self.globalstate.get_py_const(type).cname

Cython/Compiler/ExprNodes.py

@@ -1122,16 +1122,6 @@ class StringNode(PyConstNode):
             if not dst_type.is_pyobject:
                 return BytesNode(self.pos, value=self.value).coerce_to(dst_type, env)
             self.check_for_coercion_error(dst_type, fail=True)
-
-        # this will be a unicode string in Py3, so make sure we can decode it
-        if self.value.encoding and isinstance(self.value, StringEncoding.BytesLiteral):
-            try:
-                self.value.decode(self.value.encoding)
-            except UnicodeDecodeError:
-                error(self.pos, ("Decoding unprefixed string literal from '%s' failed. Consider using"
-                                 "a byte string or unicode string explicitly, "
-                                 "or adjust the source code encoding.") % self.value.encoding)
-
         return self
 
     def can_coerce_to_char_literal(self):
@@ -1139,7 +1129,8 @@ class StringNode(PyConstNode):
 
     def generate_evaluation_code(self, code):
         self.result_code = code.get_py_string_const(
-            self.value, identifier=self.is_identifier, is_str=True)
+            self.value, identifier=self.is_identifier, is_str=True,
+            unicode_value=self.unicode_value)
 
     def get_constant_c_result_code(self):
         return None
@@ -1926,6 +1917,40 @@ class NextNode(AtomicExprNode):
         code.putln("}")
 
 
+class WithExitCallNode(ExprNode):
+    # The __exit__() call of a 'with' statement.  Used in both the
+    # except and finally clauses.
+
+    # with_stat  WithStatNode                the surrounding 'with' statement
+    # args       TupleNode or ResultStatNode the exception info tuple
+
+    subexprs = ['args']
+
+    def analyse_types(self, env):
+        self.args.analyse_types(env)
+        self.type = PyrexTypes.c_bint_type
+        self.is_temp = True
+
+    def generate_result_code(self, code):
+        if isinstance(self.args, TupleNode):
+            # call only if it was not already called (and decref-cleared)
+            code.putln("if (%s) {" % self.with_stat.exit_var)
+        result_var = code.funcstate.allocate_temp(py_object_type, manage_ref=False)
+        code.putln("%s = PyObject_Call(%s, %s, NULL);" % (
+            result_var,
+            self.with_stat.exit_var,
+            self.args.result()))
+        code.put_decref_clear(self.with_stat.exit_var, type=py_object_type)
+        code.putln(code.error_goto_if_null(result_var, self.pos))
+        code.put_gotref(result_var)
+        code.putln("%s = __Pyx_PyObject_IsTrue(%s);" % (self.result(), result_var))
+        code.put_decref_clear(result_var, type=py_object_type)
+        code.putln(code.error_goto_if_neg(self.result(), self.pos))
+        code.funcstate.release_temp(result_var)
+        if isinstance(self.args, TupleNode):
+            code.putln("}")
+
+
 class ExcValueNode(AtomicExprNode):
     #  Node created during analyse_types phase
     #  of an ExceptClauseNode to fetch the current
@@ -1960,7 +1985,7 @@ class TempNode(ExprNode):
 
     subexprs = []
 
-    def __init__(self, pos, type, env):
+    def __init__(self, pos, type, env=None):
         ExprNode.__init__(self, pos)
         self.type = type
         if type.is_pyobject:
@@ -1970,6 +1995,9 @@ class TempNode(ExprNode):
     def analyse_types(self, env):
         return self.type
 
+    def analyse_target_declaration(self, env):
+        pass
+
     def generate_result_code(self, code):
         pass
 
@@ -6665,6 +6693,14 @@ class CmpNode(object):
                     env.use_utility_code(pyunicode_equals_utility_code)
                     self.special_bool_cmp_function = "__Pyx_PyUnicode_Equals"
                     return True
+                elif type1 is Builtin.bytes_type or type2 is Builtin.bytes_type:
+                    env.use_utility_code(pybytes_equals_utility_code)
+                    self.special_bool_cmp_function = "__Pyx_PyBytes_Equals"
+                    return True
+                elif type1 is Builtin.str_type or type2 is Builtin.str_type:
+                    env.use_utility_code(pystr_equals_utility_code)
+                    self.special_bool_cmp_function = "__Pyx_PyString_Equals"
+                    return True
         return False
 
     def generate_operation_code(self, code, result_code,
@@ -6861,8 +6897,6 @@ static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int
                 return -1;
             return (equals == Py_EQ) ? (result == 0) : (result != 0);
         }
-    } else if ((s1 == Py_None) & (s2 == Py_None)) {
-        return (equals == Py_EQ);
     } else if ((s1 == Py_None) & PyUnicode_CheckExact(s2)) {
         return (equals == Py_NE);
     } else if ((s2 == Py_None) & PyUnicode_CheckExact(s1)) {
@@ -6879,6 +6913,53 @@ static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int
 }
 """)
 
+pybytes_equals_utility_code = UtilityCode(
+proto="""
+static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals); /*proto*/
+""",
+impl="""
+static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) {
+    if (s1 == s2) {   /* as done by PyObject_RichCompareBool(); also catches the (interned) empty string */
+        return (equals == Py_EQ);
+    } else if (PyBytes_CheckExact(s1) & PyBytes_CheckExact(s2)) {
+        if (PyBytes_GET_SIZE(s1) != PyBytes_GET_SIZE(s2)) {
+            return (equals == Py_NE);
+        } else if (PyBytes_GET_SIZE(s1) == 1) {
+            if (equals == Py_EQ)
+                return (PyBytes_AS_STRING(s1)[0] == PyBytes_AS_STRING(s2)[0]);
+            else
+                return (PyBytes_AS_STRING(s1)[0] != PyBytes_AS_STRING(s2)[0]);
+        } else {
+            int result = memcmp(PyBytes_AS_STRING(s1), PyBytes_AS_STRING(s2), PyBytes_GET_SIZE(s1));
+            return (equals == Py_EQ) ? (result == 0) : (result != 0);
+        }
+    } else if ((s1 == Py_None) & PyBytes_CheckExact(s2)) {
+        return (equals == Py_NE);
+    } else if ((s2 == Py_None) & PyBytes_CheckExact(s1)) {
+        return (equals == Py_NE);
+    } else {
+        int result;
+        PyObject* py_result = PyObject_RichCompare(s1, s2, equals);
+        if (!py_result)
+            return -1;
+        result = __Pyx_PyObject_IsTrue(py_result);
+        Py_DECREF(py_result);
+        return result;
+    }
+}
+""",
+requires=[Builtin.include_string_h_utility_code])
+
+pystr_equals_utility_code = UtilityCode(
+proto="""
+#if PY_MAJOR_VERSION >= 3
+#define __Pyx_PyString_Equals __Pyx_PyUnicode_Equals
+#else
+#define __Pyx_PyString_Equals __Pyx_PyBytes_Equals
+#endif
+""",
+requires=[pybytes_equals_utility_code, pyunicode_equals_utility_code])
+
 
 class PrimaryCmpNode(ExprNode, CmpNode):
     #  Non-cascaded comparison or first comparison of
@@ -7684,11 +7765,18 @@ impl = """
 static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name) {
     PyObject *result;
     result = PyObject_GetAttr(dict, name);
-    if (!result)
+    if (!result) {
+        if (dict != %(BUILTINS)s) {
+            PyErr_Clear();
+            result = PyObject_GetAttr(%(BUILTINS)s, name);
+        }
+        if (!result) {
             PyErr_SetObject(PyExc_NameError, name);
+        }
+    }
     return result;
 }
-""")
+""" % {'BUILTINS' : Naming.builtins_cname})
 
 #------------------------------------------------------------------------------------
 
@@ -8532,6 +8620,17 @@ static PyObject *__Pyx_Generator_Throw(PyObject *gen, PyObject *args, CYTHON_UNU
 typedef PyObject *(*__pyx_generator_body_t)(PyObject *, PyObject *);
 """,
 impl="""
+static CYTHON_INLINE void __Pyx_Generator_ExceptionClear(struct __pyx_Generator_object *self)
+{
+    Py_XDECREF(self->exc_type);
+    Py_XDECREF(self->exc_value);
+    Py_XDECREF(self->exc_traceback);
+
+    self->exc_type = NULL;
+    self->exc_value = NULL;
+    self->exc_traceback = NULL;
+}
+
 static CYTHON_INLINE PyObject *__Pyx_Generator_SendEx(struct __pyx_Generator_object *self, PyObject *value)
 {
     PyObject *retval;
@@ -8556,10 +8655,21 @@ static CYTHON_INLINE PyObject *__Pyx_Generator_SendEx(struct __pyx_Generator_obj
         return NULL;
     }
 
+
+    if (value)
+        __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, &self->exc_traceback);
+    else
+        __Pyx_Generator_ExceptionClear(self);
+
     self->is_running = 1;
     retval = self->body((PyObject *) self, value);
     self->is_running = 0;
 
+    if (retval)
+        __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, &self->exc_traceback);
+    else
+        __Pyx_Generator_ExceptionClear(self);
+
     return retval;
 }
 
@@ -8612,10 +8722,10 @@ static PyObject *__Pyx_Generator_Throw(PyObject *self, PyObject *args, CYTHON_UN
 
     if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb))
         return NULL;
-    __Pyx_Raise(typ, val, tb);
+    __Pyx_Raise(typ, val, tb, NULL);
     return __Pyx_Generator_SendEx(generator, NULL);
 }
 """,
 proto_block='utility_code_proto_before_types',
-requires=[Nodes.raise_utility_code],
+requires=[Nodes.raise_utility_code, Nodes.swap_exception_utility_code],
 )

Cython/Compiler/Main.py

@@ -619,7 +619,7 @@ def run_pipeline(source, options, full_module_name = None):
                 options.annotate = True
 
     # Get pipeline
-    if source_ext.lower() == '.py':
+    if source_ext.lower() == '.py' or not source_ext:
         pipeline = context.create_py_pipeline(options, result)
     else:
         pipeline = context.create_pyx_pipeline(options, result)

Cython/Compiler/Optimize.py

@@ -930,7 +930,15 @@ class FlattenInListTransform(Visitor.VisitorTransform, SkipDeclarations):
         conds = []
         temps = []
         for arg in args:
-            if not arg.is_simple():
+            try:
+                # Trial optimisation to avoid redundant temp
+                # assignments.  However, since is_simple() is meant to
+                # be called after type analysis, we ignore any errors
+                # and just play safe in that case.
+                is_simple_arg = arg.is_simple()
+            except Exception:
+                is_simple_arg = False
+            if not is_simple_arg:
                 # must evaluate all non-simple RHS before doing the comparisons
                 arg = UtilNodes.LetRefNode(arg)
                 temps.append(arg)
@@ -3134,6 +3142,15 @@ class ConstantFolding(Visitor.VisitorTransform, SkipDeclarations):
         return ExprNodes.BoolNode(node.pos, value=bool_result,
                                   constant_result=bool_result)
 
+    def visit_CondExprNode(self, node):
+        self._calculate_const(node)
+        if node.test.constant_result is ExprNodes.not_a_constant:
+            return node
+        if node.test.constant_result:
+            return node.true_val
+        else:
+            return node.false_val
+
     def visit_IfStatNode(self, node):
         self.visitchildren(node)
         # eliminate dead code based on constant condition results

Cython/Compiler/ParseTreeTransforms.py

@@ -11,11 +11,12 @@ import Naming
 import ExprNodes
 import Nodes
 import Options
+import Builtin
 
 from Cython.Compiler.Visitor import VisitorTransform, TreeVisitor
 from Cython.Compiler.Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
 from Cython.Compiler.ModuleNode import ModuleNode
-from Cython.Compiler.UtilNodes import LetNode, LetRefNode
+from Cython.Compiler.UtilNodes import LetNode, LetRefNode, ResultRefNode
 from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
 from Cython.Compiler.StringEncoding import EncodedString
 from Cython.Compiler.Errors import error, warning, CompileError, InternalError
@@ -928,81 +929,55 @@ class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
 
 
 class WithTransform(CythonTransform, SkipDeclarations):
-
-    # EXCINFO is manually set to a variable that contains
-    # the exc_info() tuple that can be generated by the enclosing except
-    # statement.
-    template_without_target = TreeFragment(u"""
-        MGR = EXPR
-        EXIT = MGR.__exit__
-        MGR.__enter__()
-        EXC = True
-        try:
-            try:
-                EXCINFO = None
-                BODY
-            except:
-                EXC = False
-                if not EXIT(*EXCINFO):
-                    raise
-        finally:
-            if EXC:
-                EXIT(None, None, None)
-    """, temps=[u'MGR', u'EXC', u"EXIT"],
-    pipeline=[NormalizeTree(None)])
-
-    template_with_target = TreeFragment(u"""
-        MGR = EXPR
-        EXIT = MGR.__exit__
-        VALUE = MGR.__enter__()
-        EXC = True
-        try:
-            try:
-                EXCINFO = None
-                TARGET = VALUE
-                BODY
-            except:
-                EXC = False
-                if not EXIT(*EXCINFO):
-                    raise
-        finally:
-            if EXC:
-                EXIT(None, None, None)
-            MGR = EXIT = VALUE = EXC = None
-
-    """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE"],
-    pipeline=[NormalizeTree(None)])
-
     def visit_WithStatNode(self, node):
-        # TODO: Cleanup badly needed
-        TemplateTransform.temp_name_counter += 1
-        handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter
-
-        self.visitchildren(node, ['body'])
-        excinfo_temp = ExprNodes.NameNode(node.pos, name=handle)#TempHandle(Builtin.tuple_type)
-        if node.target is not None:
-            result = self.template_with_target.substitute({
-                u'EXPR' : node.manager,
-                u'BODY' : node.body,
-                u'TARGET' : node.target,
-                u'EXCINFO' : excinfo_temp
-                }, pos=node.pos)
-        else:
-            result = self.template_without_target.substitute({
-                u'EXPR' : node.manager,
-                u'BODY' : node.body,
-                u'EXCINFO' : excinfo_temp
-                }, pos=node.pos)
-
-        # Set except excinfo target to EXCINFO
-        try_except = result.stats[-1].body.stats[-1]
-        try_except.except_clauses[0].excinfo_target = ExprNodes.NameNode(node.pos, name=handle)
-#            excinfo_temp.ref(node.pos))
-
-#        result.stats[-1].body.stats[-1] = TempsBlockNode(
-#            node.pos, temps=[excinfo_temp], body=try_except)
+        self.visitchildren(node, 'body')
+        pos = node.pos
+        body, target, manager = node.body, node.target, node.manager
+        node.target_temp = ExprNodes.TempNode(pos, type=PyrexTypes.py_object_type)
+        if target is not None:
+            node.has_target = True
+            body = Nodes.StatListNode(
+                pos, stats = [
+                    Nodes.WithTargetAssignmentStatNode(
+                        pos, lhs = target, rhs = node.target_temp),
+                    body
+                    ])
+            node.target = None
+
+        excinfo_target = ResultRefNode(
+            pos=pos, type=Builtin.tuple_type, may_hold_none=False)
+        except_clause = Nodes.ExceptClauseNode(
+            pos, body = Nodes.IfStatNode(
+                pos, if_clauses = [
+                    Nodes.IfClauseNode(
+                        pos, condition = ExprNodes.NotNode(
+                            pos, operand = ExprNodes.WithExitCallNode(
+                                pos, with_stat = node,
+                                args = excinfo_target)),
+                        body = Nodes.ReraiseStatNode(pos),
+                        ),
+                    ],
+                else_clause = None),
+            pattern = None,
+            target = None,
+            excinfo_target = excinfo_target,
+            )
 
-        return result
+        node.body = Nodes.TryFinallyStatNode(
+            pos, body = Nodes.TryExceptStatNode(
+                pos, body = body,
+                except_clauses = [except_clause],
+                else_clause = None,
+                ),
+            finally_clause = Nodes.ExprStatNode(
+                pos, expr = ExprNodes.WithExitCallNode(
+                    pos, with_stat = node,
+                    args = ExprNodes.TupleNode(
+                        pos, args = [ExprNodes.NoneNode(pos) for _ in range(3)]
+                        ))),
+            handle_error_case = False,
+            )
+        return node
 
     def visit_ExprNode(self, node):
         # With statements are never inside expressions.
@@ -1602,6 +1577,12 @@ class CreateClosureClasses(CythonTransform):
                           is_cdef=True)
         klass.declare_var(pos=pos, name='resume_label', cname='resume_label', type=PyrexTypes.c_int_type,
                           is_cdef=True)
+        klass.declare_var(pos=pos, name='exc_type', cname='exc_type',
+                          type=PyrexTypes.py_object_type, is_cdef=True)
+        klass.declare_var(pos=pos, name='exc_value', cname='exc_value',
+                          type=PyrexTypes.py_object_type, is_cdef=True)
+        klass.declare_var(pos=pos, name='exc_traceback', cname='exc_traceback',
+                          type=PyrexTypes.py_object_type, is_cdef=True)
 
         import TypeSlots
         e = klass.declare_pyfunction('send', pos)

Cython/Compiler/Parsing.py

@@ -1172,6 +1172,7 @@ def p_raise_statement(s):
     exc_type = None
     exc_value = None
     exc_tb = None
+    cause = None
     if s.sy not in statement_terminators:
         exc_type = p_test(s)
         if s.sy == ',':
@@ -1180,11 +1181,15 @@ def p_raise_statement(s):
             if s.sy == ',':
                 s.next()
                 exc_tb = p_test(s)
+        elif s.sy == 'from':
+            s.next()
+            cause = p_test(s)
     if exc_type or exc_value or exc_tb:
         return Nodes.RaiseStatNode(pos,
             exc_type = exc_type,
             exc_value = exc_value,
-            exc_tb = exc_tb)
+            exc_tb = exc_tb,
+            cause = cause)
     else:
         return Nodes.ReraiseStatNode(pos)
 
@@ -1660,15 +1665,27 @@ def p_simple_statement_list(s, ctx, first_statement = 0):
     # Parse a series of simple statements on one line
     # separated by semicolons.
     stat = p_simple_statement(s, first_statement = first_statement)
-    if s.sy == ';':
-        stats = [stat]
+    pos = stat.pos
+    stats = []
+    if not isinstance(stat, Nodes.PassStatNode):
+        stats.append(stat)
     while s.sy == ';':
         #print "p_simple_statement_list: maybe more to follow" ###
         s.next()
         if s.sy in ('NEWLINE', 'EOF'):
             break
-            stats.append(p_simple_statement(s))
-        stat = Nodes.StatListNode(stats[0].pos, stats = stats)
+        stat = p_simple_statement(s, first_statement = first_statement)
+        if isinstance(stat, Nodes.PassStatNode):
+            continue
+        stats.append(stat)
+        first_statement = False
+
+    if not stats:
+        stat = Nodes.PassStatNode(pos)
+    elif len(stats) == 1:
+        stat = stats[0]
+    else:
+        stat = Nodes.StatListNode(pos, stats = stats)
     s.expect_newline("Syntax error in simple statement list")
     return stat
 
@@ -1805,9 +1822,14 @@ def p_statement_list(s, ctx, first_statement = 0):
     pos = s.position()
     stats = []
     while s.sy not in ('DEDENT', 'EOF'):
-        stats.append(p_statement(s, ctx, first_statement = first_statement))
-        first_statement = 0
-    if len(stats) == 1:
+        stat = p_statement(s, ctx, first_statement = first_statement)
+        if isinstance(stat, Nodes.PassStatNode):
+            continue
+        stats.append(stat)
+        first_statement = False
+    if not stats:
+        return Nodes.PassStatNode(pos)
+    elif len(stats) == 1:
         return stats[0]
     else:
         return Nodes.StatListNode(pos, stats = stats)

Cython/Compiler/PyrexTypes.py

@@ -2739,7 +2739,8 @@ def best_match(args, functions, pos=None, env=None):
     possibilities = []
     bad_types = []
     needed_coercions = {}
-    for func, func_type in candidates:
+
+    for index, (func, func_type) in enumerate(candidates):
         score = [0,0,0,0]
         for i in range(min(len(args), len(func_type.args))):
             src_type = args[i].type
@@ -2782,7 +2783,7 @@ def best_match(args, functions, pos=None, env=None):
                 bad_types.append((func, error_mesg))
                 break
         else:
-            possibilities.append((score, func)) # so we can sort it
+            possibilities.append((score, index, func)) # so we can sort it
 
     if possibilities:
         possibilities.sort()
@@ -2791,7 +2792,7 @@ def best_match(args, functions, pos=None, env=None):
                 error(pos, "ambiguous overloaded method")
             return None
 
-        function = possibilities[0][1]
+        function = possibilities[0][-1]
 
         if function in needed_coercions and env:
             arg_i, coerce_to_type = needed_coercions[function]

Cython/Compiler/Symtab.py

@@ -564,11 +564,16 @@ class Scope(object):
         entry.is_anonymous = True
         return entry
 
-    def declare_lambda_function(self, func_cname, pos):
+    def declare_lambda_function(self, lambda_name, pos):
         # Add an entry for an anonymous Python function.
-        entry = self.declare_var(None, py_object_type, pos,
-                                 cname=func_cname, visibility='private')
-        entry.name = EncodedString(func_cname)
+        func_cname = self.mangle(Naming.lambda_func_prefix + u'funcdef_', lambda_name)
+        pymethdef_cname = self.mangle(Naming.lambda_func_prefix + u'methdef_', lambda_name)
+        qualified_name = self.qualify_name(lambda_name)
+
+        entry = self.declare(None, func_cname, py_object_type, pos, 'private')
+        entry.name = lambda_name
+        entry.qualified_name = qualified_name
+        entry.pymethdef_cname = pymethdef_cname
         entry.func_cname = func_cname
         entry.signature = pyfunction_signature
         entry.is_anonymous = True
@@ -927,21 +932,18 @@ class ModuleScope(Scope):
         return self.outer_scope.lookup(name, language_level = self.context.language_level)
 
     def declare_builtin(self, name, pos):
-        if not hasattr(builtins, name) and name not in Code.non_portable_builtins_map:
-            # 'xrange' and 'BaseException' are special cased in Code.py
+        if not hasattr(builtins, name) \
+               and name not in Code.non_portable_builtins_map \
+               and name not in Code.uncachable_builtins:
             if self.has_import_star:
                 entry = self.declare_var(name, py_object_type, pos)
                 return entry
-            ## elif self.outer_scope is not None:
-            ##     entry = self.outer_scope.declare_builtin(name, pos)
-            ##     print entry
-            ##     return entry
             else:
-                # unknown - assume it's builtin and look it up at runtime
                 if Options.error_on_unknown_names:
                     error(pos, "undeclared name not builtin: %s" % name)
                 else:
                     warning(pos, "undeclared name not builtin: %s" % name, 2)
+                # unknown - assume it's builtin and look it up at runtime
                 entry = self.declare(name, None, py_object_type, pos, 'private')
                 entry.is_builtin = 1
                 return entry
@@ -950,7 +952,7 @@ class ModuleScope(Scope):
                 if entry.name == name:
                     return entry
         entry = self.declare(None, None, py_object_type, pos, 'private')
-        if Options.cache_builtins:
+        if Options.cache_builtins and name not in Code.uncachable_builtins:
             entry.is_builtin = 1
             entry.is_const = 1 # cached
             entry.name = name
@@ -959,6 +961,7 @@ class ModuleScope(Scope):
             self.undeclared_cached_builtins.append(entry)
         else:
             entry.is_builtin = 1
+            entry.name = name
         return entry
 
     def find_module(self, module_name, pos):
@@ -1414,6 +1417,12 @@ class GeneratorExpressionScope(Scope):
         self.entries[name] = entry
         return entry
 
+    def declare_lambda_function(self, func_cname, pos):
+        return self.outer_scope.declare_lambda_function(func_cname, pos)
+
+    def add_lambda_def(self, def_node):
+        return self.outer_scope.add_lambda_def(def_node)
+
 
 class ClosureScope(LocalScope):
 

Cython/Compiler/Tests/TestTreeFragment.py

@@ -54,11 +54,10 @@ class TestTreeFragments(CythonTest):
             x = TMP
         """)
         T = F.substitute(temps=[u"TMP"])
-        s = T.stats
-        self.assert_(s[0].expr.name == "__tmpvar_1")
-#        self.assert_(isinstance(s[0].expr, TempRefNode))
-#        self.assert_(isinstance(s[1].rhs, TempRefNode))
-#        self.assert_(s[0].expr.handle is s[1].rhs.handle)
+        s = T.body.stats
+        self.assert_(isinstance(s[0].expr, TempRefNode))
+        self.assert_(isinstance(s[1].rhs, TempRefNode))
+        self.assert_(s[0].expr.handle is s[1].rhs.handle)
 
 if __name__ == "__main__":
     import unittest

Cython/Compiler/TreeFragment.py

@@ -121,16 +121,15 @@ class TemplateTransform(VisitorTransform):
         temphandles = []
         for temp in temps:
             TemplateTransform.temp_name_counter += 1
-            handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter
-#            handle = UtilNodes.TempHandle(PyrexTypes.py_object_type)
+            handle = UtilNodes.TempHandle(PyrexTypes.py_object_type)
             tempmap[temp] = handle
-#            temphandles.append(handle)
+            temphandles.append(handle)
         self.tempmap = tempmap
         result = super(TemplateTransform, self).__call__(node)
-#        if temps:
-#            result = UtilNodes.TempsBlockNode(self.get_pos(node),
-#                                              temps=temphandles,
-#                                              body=result)
+        if temps:
+            result = UtilNodes.TempsBlockNode(self.get_pos(node),
+                                              temps=temphandles,
+                                              body=result)
         return result
 
     def get_pos(self, node):
@@ -161,9 +160,8 @@ class TemplateTransform(VisitorTransform):
     def visit_NameNode(self, node):
         temphandle = self.tempmap.get(node.name)
         if temphandle:
-            return NameNode(pos=node.pos, name=temphandle)
             # Replace name with temporary
-            #return temphandle.ref(self.get_pos(node))
+            return temphandle.ref(self.get_pos(node))
         else:
             return self.try_substitution(node, node.name)
 

Cython/Compiler/UtilNodes.py

@@ -134,6 +134,10 @@ class ResultRefNode(AtomicExprNode):
             self.type = type
         assert self.pos is not None
 
+    def clone_node(self):
+        # nothing to do here
+        return self
+
     def analyse_types(self, env):
         if self.expression is not None:
             self.type = self.expression.type

Cython/Includes/libcpp/string.pxd

+
+cdef extern from "<string>" namespace "std":
+
+    size_t npos = -1
+
+    cdef cppclass string:
+        string()
+        string(char *)
+        string(char *, size_t)
+        string(string&)
+        # as a string formed by a repetition of character c, n times.
+        string(size_t, char)
+
+        char* c_str()
+        size_t size()
+        size_t max_size()
+        size_t length()
+        void resize(size_t)
+        void resize(size_t, char c)
+        size_t capacity()
+        void reserve(size_t)
+        void clear()
+        bint empty()
+
+        char at(size_t)
+        char operator[](size_t)
+        int compare(string&)
+
+        string& append(string&)
+        string& append(string&, size_t, size_t)
+        string& append(char *)
+        string& append(char *, size_t)
+        string& append(size_t, char)
+
+        void push_back(char c)
+
+        string& assign (string&)
+        string& assign (string&, size_t, size_t)
+        string& assign (char *, size_t)
+        string& assign (char *)
+        string& assign (size_t n, char c)
+
+        string& insert(size_t, string&)
+        string& insert(size_t, string&, size_t, size_t)
+        string& insert(size_t, char* s, size_t)
+
+
+        string& insert(size_t, char* s)
+        string& insert(size_t, size_t, char c)
+
+        size_t copy(char *, size_t, size_t)
+
+        size_t find(string&)
+        size_t find(string&, size_t)
+        size_t find(char*, size_t pos, size_t)
+        size_t find(char*, size_t pos)
+        size_t find(char, size_t pos)
+
+        size_t rfind(string&, size_t)
+        size_t rfind(char* s, size_t, size_t)
+        size_t rfind(char*, size_t pos)
+        size_t rfind(char c, size_t)
+        size_t rfind(char c)
+
+        size_t find_first_of(string&, size_t)
+        size_t find_first_of(char* s, size_t, size_t)
+        size_t find_first_of(char*, size_t pos)
+        size_t find_first_of(char c, size_t)
+        size_t find_first_of(char c)
+
+        size_t find_first_not_of(string&, size_t)
+        size_t find_first_not_of(char* s, size_t, size_t)
+        size_t find_first_not_of(char*, size_t pos)
+        size_t find_first_not_of(char c, size_t)
+        size_t find_first_not_of(char c)
+
+        size_t find_last_of(string&, size_t)
+        size_t find_last_of(char* s, size_t, size_t)
+        size_t find_last_of(char*, size_t pos)
+        size_t find_last_of(char c, size_t)
+        size_t find_last_of(char c)
+
+        size_t find_last_not_of(string&, size_t)
+        size_t find_last_not_of(char* s, size_t, size_t)
+        size_t find_last_not_of(char*, size_t pos)
+
+        string substr(size_t, size_t)
+        string substr()
+        string substr(size_t)
+
+        size_t find_last_not_of(char c, size_t)
+        size_t find_last_not_of(char c)
+
+        #string& operator= (string&)
+        #string& operator= (char*)
+        #string& operator= (char)
+
+        bint operator==(string&)
+        bint operator==(char*)
+
+        bint operator!= (string& rhs )
+        bint operator!= (char* )
+
+        bint operator< (string&)
+        bint operator< (char*)
+
+        bint operator> (string&)
+        bint operator> (char*)
+
+        bint operator<= (string&)
+        bint operator<= (char*)
+
+        bint operator>= (string&)
+        bint operator>= (char*)

Cython/Plex/Errors.py

@@ -39,7 +39,7 @@ class UnrecognizedInput(PlexError):
 
   def __init__(self, scanner, state_name):
     self.scanner = scanner
-    self.position = scanner.position()
+    self.position = scanner.get_position()
     self.state_name = state_name
 
   def __str__(self):

Cython/Plex/Scanners.py

@@ -299,6 +299,11 @@ class Scanner(object):
     """
     return (self.name, self.start_line, self.start_col)
 
+  def get_position(self):
+    """Python accessible wrapper around position(), only for error reporting.
+    """
+    return self.position()
+
   def begin(self, state_name):
     """Set the current state of the scanner to the named state."""
     self.initial_state = (

pyximport/pyximport.py

@@ -261,7 +261,8 @@ class PyImporter(PyxImporter):
         self.super = super(PyImporter, self)
         self.super.__init__(extension='.py', pyxbuild_dir=pyxbuild_dir)
         self.uncompilable_modules = {}
-        self.blocked_modules = ['Cython', 'distutils.extension']
+        self.blocked_modules = ['Cython', 'distutils.extension',
+                                'distutils.sysconfig']
 
     def find_module(self, fullname, package_path=None):
         if fullname in sys.modules:

setup.py

@@ -249,7 +249,11 @@ except ValueError:
 
 try:
     sys.argv.remove("--no-cython-compile")
+    compile_cython_itself = False
 except ValueError:
+    compile_cython_itself = True
+
+if compile_cython_itself:
     compile_cython_modules(cython_profile, cython_compile_more, cython_with_refnanny)
 
 setup_args.update(setuptools_extra_args)

tests/bugs.txt

@@ -10,7 +10,6 @@ cfunc_call_tuple_args_T408
 compile.cpp_operators
 cpp_templated_ctypedef
 cpp_structs
-with_statement_module_level_T536
 function_as_method_T494
 closure_inside_cdef_T554
 pure_mode_cmethod_inheritance_T583
@@ -22,12 +21,20 @@ class_scope_T671
 
 # CPython regression tests that don't current work:
 pyregr.test_threadsignals
-pyregr.test_module
 pyregr.test_capi
 pyregr.test_socket
 pyregr.test_threading
 pyregr.test_sys
+pyregr.test_pep3131
 
 # CPython regression tests that don't make sense
 pyregr.test_gdb
 pyregr.test_support
+
+# Inlined generators
+all
+any
+builtin_sorted
+dictcomp
+inlined_generator_expressions
+setcomp

tests/compile/future_imports.pyx

+# mode: compile
+
+from __future__ import nested_scopes
+
+from __future__ import with_statement
+
+pass
+
+from __future__ import nested_scopes ; from __future__ import nested_scopes

tests/errors/w_cdef_override.pyx

 # mode: error
-# tags: werror
+# tag: werror
 
 cdef foo():
     pass

tests/run/arithmetic_analyse_types.pyx

@@ -18,40 +18,40 @@ def short_binop(short val):
     """
     Arithmetic in C is always done with at least int precision.
     
-    >>> short_binop(3)
-    'int called'
+    >>> print(short_binop(3))
+    int called
     """
     assert typeof(val + val) == "int", typeof(val + val)
     assert typeof(val - val) == "int", typeof(val - val)
     assert typeof(val & val) == "int", typeof(val & val)
     cdef int_return x = f(val + val)
-    return x.msg
+    return x.msg.decode('ASCII')
 
 def short_unnop(short val):
     """
     Arithmetic in C is always done with at least int precision.
     
-    >>> short_unnop(3)
-    'int called'
+    >>> print(short_unnop(3))
+    int called
     """
     cdef int_return x = f(-val)
-    return x.msg
+    return x.msg.decode('ASCII')
 
 def longlong_binop(long long val):
     """
-    >>> longlong_binop(3)
-    'long long called'
+    >>> print(longlong_binop(3))
+    long long called
     """
     cdef longlong_return x = f(val * val)
-    return x.msg
+    return x.msg.decode('ASCII')
 
 def longlong_unnop(long long val):
     """
-    >>> longlong_unnop(3)
-    'long long called'
+    >>> print(longlong_unnop(3))
+    long long called
     """
     cdef longlong_return x = f(~val)
-    return x.msg
+    return x.msg.decode('ASCII')
 
 
 def test_bint(bint a):

tests/run/arithmetic_analyse_types_helper.h

 /* A set of mutually incompatable return types. */
 
-struct short_return { const char *msg; };
-struct int_return { const char *msg; };
-struct longlong_return { const char *msg; };
+struct short_return { char *msg; };
+struct int_return { char *msg; };
+struct longlong_return { char *msg; };
 
 /* A set of overloaded methods. */
 
 short_return f(short arg) {
     short_return val;
-    val.msg = "short called";
+    val.msg = (char*)"short called";
     return val;
 }
 
 int_return f(int arg) {
     int_return val;
-    val.msg = "int called";
+    val.msg = (char*)"int called";
     return val;
 }
 
 longlong_return f(long long arg) {
     longlong_return val;
-    val.msg = "long long called";
+    val.msg = (char*)"long long called";
     return val;
 }
  

tests/run/cpp_stl_string.pyx

+# tag: cpp
+
+from libcpp.string cimport string
+
+b_asdf = b'asdf'
+b_asdg = b'asdg'
+b_s = b's'
+
+def test_indexing(char *py_str):
+    """
+    >>> test_indexing(b_asdf)
+    ('s', 's')
+    """
+    cdef string s
+    s = string(py_str)
+    return chr(s[1]), chr(s.at(1))
+
+def test_size(char *py_str):
+    """
+    >>> test_size(b_asdf)
+    (4, 4)
+    """
+    cdef string s
+    s = string(py_str)
+    return s.size(), s.length()
+
+def test_compare(char *a, char *b):
+    """
+    >>> test_compare(b_asdf, b_asdf)
+    0
+
+    >>> test_compare(b_asdf, b_asdg) < 0
+    True
+    """
+    cdef string s = string(a)
+    cdef string t = string(b)
+    return s.compare(t)
+
+def test_empty():
+    """
+    >>> test_empty()
+    (True, False)
+    """
+    cdef string a = string(<char *>b"")
+    cdef string b = string(<char *>b"aa")
+    return a.empty(), b.empty()
+
+def test_push_back(char *a):
+    """
+    >>> test_push_back(b_asdf) == b_asdf + b_s
+    True
+    """
+    cdef string s = string(a)
+    s.push_back(<char>ord('s'))
+    return s.c_str()
+
+def test_insert(char *a, char *b, int i):
+    """
+    >>> test_insert('AAAA'.encode('ASCII'), 'BBBB'.encode('ASCII'), 2) == 'AABBBBAA'.encode('ASCII')
+    True
+    """
+    cdef string s = string(a)
+    cdef string t = string(b)
+    cdef string u = s.insert(i, t)
+    return u.c_str()
+
+def test_copy(char *a):
+    """
+    >>> test_copy(b_asdf) == b_asdf[1:]
+    True
+    """
+    cdef string t = string(a)
+    cdef char buffer[6]
+    cdef size_t length = t.copy(buffer, 4, 1)
+    buffer[length] = c'\0'
+    return buffer
+
+def test_find(char *a, char *b):
+    """
+    >>> test_find(b_asdf, 'df'.encode('ASCII'))
+    2
+    """
+    cdef string s = string(a)
+    cdef string t = string(b)
+    cdef size_t i = s.find(t)
+    return i
+
+def test_clear():
+    """
+    >>> test_clear() == ''.encode('ASCII')
+    True
+    """
+    cdef string s = string(<char *>"asdf")
+    s.clear()
+    return s.c_str()
+
+def test_assign(char *a):
+    """
+    >>> test_assign(b_asdf) == 'ggg'.encode('ASCII')
+    True
+    """
+    cdef string s = string(a)
+    s.assign(<char *>"ggg")
+    return s.c_str()
+
+
+def test_substr(char *a):
+    """
+    >>> test_substr('ABCDEFGH'.encode('ASCII')) == ('BCDEFGH'.encode('ASCII'), 'BCDE'.encode('ASCII'), 'ABCDEFGH'.encode('ASCII'))
+    True
+    """
+    cdef string s = string(a)
+    cdef string x, y, z
+    x = s.substr(1)
+    y = s.substr(1, 4)
+    z = s.substr()
+    return x.c_str(), y.c_str(), z.c_str()
+
+def test_append(char *a, char *b):
+    """
+    >>> test_append(b_asdf, '1234'.encode('ASCII')) == b_asdf + '1234'.encode('ASCII')
+    True
+    """
+    cdef string s = string(a)
+    cdef string t = string(b)
+    cdef string j = s.append(t)
+    return j.c_str()
+
+def test_char_compare(py_str):
+    """
+    >>> test_char_compare(b_asdf)
+    True
+    """
+    cdef char *a = py_str
+    cdef string b = string(a)
+    return b.compare(b) == 0
+
+def test_cstr(char *a):
+    """
+    >>> test_cstr(b_asdf) == b_asdf
+    True
+    """
+    cdef string b = string(a)
+    return b.c_str()
+
+def test_equals_operator(char *a, char *b):
+    """
+    >>> test_equals_operator(b_asdf, b_asdf)
+    (True, False)
+    """
+    cdef string s = string(a)
+    cdef string t = string(b)
+    return t == s, t != <char *>"asdf"
+
+def test_less_than(char *a, char *b):
+    """
+    >>> test_less_than(b_asdf[:-1], b_asdf)
+    (True, True, True)
+
+    >>> test_less_than(b_asdf[:-1], b_asdf[:-1])
+    (False, False, True)
+    """
+    cdef string s = string(a)
+    cdef string t = string(b)
+    return (s < t, s < b, s <= b)
+
+def test_greater_than(char *a, char *b):
+    """
+    >>> test_greater_than(b_asdf[:-1], b_asdf)
+    (False, False, False)
+
+    >>> test_greater_than(b_asdf[:-1], b_asdf[:-1])
+    (False, False, True)
+    """
+    cdef string s = string(a)
+    cdef string t = string(b)
+    return (s > t, s > b, s >= b)

tests/run/cython3.pyx

 # cython: language_level=3
+# mode: run
+# tag: generators, python3
 
 cimport cython
 
@@ -89,6 +91,16 @@ def list_comp():
     assert x == 'abc' # don't leak in Py3 code
     return result
 
+def list_comp_with_lambda():
+    """
+    >>> list_comp_with_lambda()
+    [0, 4, 8]
+    """
+    x = 'abc'
+    result = [x*2 for x in range(5) if (lambda x:x % 2)(x) == 0]
+    assert x == 'abc' # don't leak in Py3 code
+    return result
+
 module_level_lc = [ module_level_loopvar*2 for module_level_loopvar in range(4) ]
 def list_comp_module_level():
     """

tests/run/flatin.pyx

@@ -13,7 +13,7 @@ def test_in(s):
     >>> test_in('')
     5
     """
-    if s in (u'ABC', u'BCD'):
+    if s in (u'ABC', u'BCD', u'ABC'[:3], u'ABC'[::-1], u'ABC'[-1]):
         return 1
     elif s.upper() in (u'ABC', u'BCD'):
         return 2

tests/run/funcexceptraisefrom.pyx

+__doc__ = u"""
+>>> def bar():
+...     try:
+...         foo()
+...     except ValueError:
+...         if IS_PY3:
+...             print(isinstance(sys.exc_info()[1].__cause__, TypeError))
+...         else:
+...             print(True)
+
+>>> bar()
+True
+
+>>> print(sys.exc_info())
+(None, None, None)
+
+>>> def bar2():
+...     try:
+...         foo2()
+...     except ValueError:
+...         if IS_PY3:
+...             cause = sys.exc_info()[1].__cause__
+...             print(isinstance(cause, TypeError))
+...             print(cause.args==('value',))
+...             pass
+...         else:
+...             print(True)
+...             print(True)
+
+>>> bar2()
+True
+True
+"""
+
+import sys
+IS_PY3 = sys.version_info[0] >= 3
+if not IS_PY3:
+    sys.exc_clear()
+
+def foo():
+    try:
+        raise TypeError
+    except TypeError:
+        raise ValueError from TypeError
+
+def foo2():
+    try:
+        raise TypeError
+    except TypeError:
+        raise ValueError() from TypeError('value')
+

tests/run/generator_expressions.pyx

+# mode: run
+# tag: generators, lambda
+
+def genexpr():
+    """
+    >>> genexpr()
+    [0, 2, 4, 6, 8]
+    """
+    x = 'abc'
+    result = list( x*2 for x in range(5) )
+    assert x == 'abc' # don't leak
+    return result
+
+def genexpr_if():
+    """
+    >>> genexpr_if()
+    [0, 4, 8]
+    """
+    x = 'abc'
+    result = list( x*2 for x in range(5) if x % 2 == 0 )
+    assert x == 'abc' # don't leak
+    return result
+
+def genexpr_with_lambda():
+    """
+    >>> genexpr_with_lambda()
+    [0, 4, 8]
+    """
+    x = 'abc'
+    result = list( x*2 for x in range(5) if (lambda x:x % 2)(x) == 0 )
+    assert x == 'abc' # don't leak
+    return result
+
+def genexpr_of_lambdas(int N):
+    """
+    >>> [ (f(), g()) for f,g in genexpr_of_lambdas(5) ]
+    [(0, 0), (1, 2), (2, 4), (3, 6), (4, 8)]
+    """
+    return ( ((lambda : x), (lambda : x*2)) for x in range(N) )

tests/run/generators_py.py

@@ -167,6 +167,40 @@ def check_yield_in_except():
     except ValueError:
         yield
 
+def yield_in_except_throw_exc_type():
+    """
+    >>> import sys
+    >>> g = yield_in_except_throw_exc_type()
+    >>> next(g)
+    >>> g.throw(TypeError)
+    Traceback (most recent call last):
+    TypeError
+    >>> next(g)
+    Traceback (most recent call last):
+    StopIteration
+    """
+    try:
+        raise ValueError
+    except ValueError:
+        yield
+
+def yield_in_except_throw_instance():
+    """
+    >>> import sys
+    >>> g = yield_in_except_throw_instance()
+    >>> next(g)
+    >>> g.throw(TypeError())
+    Traceback (most recent call last):
+    TypeError
+    >>> next(g)
+    Traceback (most recent call last):
+    StopIteration
+    """
+    try:
+        raise ValueError
+    except ValueError:
+        yield
+
 def test_swap_assignment():
     """
     >>> gen = test_swap_assignment()

tests/run/ifelseexpr_T267.pyx

+# mode: run
+# tag: condexpr
 # ticket: 267
 
-"""
->>> constants(4)
-1
->>> constants(5)
-10
->>> temps(4)
-1
->>> temps(5)
-10
->>> nested(1)
-1
->>> nested(2)
-2
->>> nested(3)
-3
-"""
+cimport cython
 
 def ident(x): return x
 
 def constants(x):
+    """
+    >>> constants(4)
+    1
+    >>> constants(5)
+    10
+    """
     a = 1 if x < 5 else 10
     return a
 
 def temps(x):
+    """
+    >>> temps(4)
+    1
+    >>> temps(5)
+    10
+    """
     return ident(1) if ident(x) < ident(5) else ident(10)
 
 def nested(x):
+    """
+    >>> nested(1)
+    1
+    >>> nested(2)
+    2
+    >>> nested(3)
+    3
+    """
     return 1 if x == 1 else (2 if x == 2 else 3)
+
+@cython.test_fail_if_path_exists('//CondExprNode')
+def const_true(a,b):
+    """
+    >>> const_true(1,2)
+    1
+    """
+    return a if 1 == 1 else b
+
+@cython.test_fail_if_path_exists('//CondExprNode')
+def const_false(a,b):
+    """
+    >>> const_false(1,2)
+    2
+    """
+    return a if 1 != 1 else b

tests/run/relativeimport_T542.pyx

@@ -20,10 +20,9 @@ def test_relative():
 
 def test_absolute():
     """
-    >>> test_absolute()
+    >>> test_absolute()   # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    ...
-    ImportError: No module named debug
+    ImportError: No module named ...debug...
     """
     import debug
     return

tests/run/strliterals.pyx

@@ -132,6 +132,18 @@ __doc__ = ur"""
     >>> len(bytes_uescape)
     28
 
+    >>> (sys.version_info[0] >= 3 and sys.maxunicode == 1114111 and len(str_uescape) == 3 or
+    ...  sys.version_info[0] >= 3 and sys.maxunicode == 65535   and len(str_uescape) == 4 or
+    ...  sys.version_info[0] <  3 and len(str_uescape) == 17 or
+    ...  len(str_uescape))
+    True
+    >>> (sys.version_info[0] >= 3 and str_uescape[0] == 'c' or
+    ...  sys.version_info[0] <  3 and str_uescape[0] == '\\' or
+    ...  str_uescape[0])
+    True
+    >>> print(str_uescape[-1])
+    B
+
     >>> newlines == "Aaa\n"
     True
     
@@ -173,6 +185,7 @@ bresc = br'\12\'\"\\'
 uresc = ur'\12\'\"\\'
 
 bytes_uescape = b'\u1234\U12345678\u\u1\u12\uX'
+str_uescape = '\u0063\U00012345\x42'
 
 newlines = "Aaa\n"
 

tests/run/test_raisefrom.pyx

+import unittest
+# adapted from pyregr
+class TestCause(unittest.TestCase):
+    def test_invalid_cause(self):
+        try:
+            raise IndexError from 5
+        except TypeError as e:
+            self.assertTrue("exception cause" in str(e))
+        else:
+            self.fail("No exception raised")
+
+    def test_class_cause(self):
+        try:
+            raise IndexError from KeyError
+        except IndexError as e:
+            self.assertTrue(isinstance(e.__cause__, KeyError))
+        else:
+            self.fail("No exception raised")
+
+    def test_instance_cause(self):
+        cause = KeyError()
+        try:
+            raise IndexError from cause
+        except IndexError as e:
+            self.assertTrue(e.__cause__ is cause)
+        else:
+            self.fail("No exception raised")
+
+    def test_erroneous_cause(self):
+        class MyException(Exception):
+            def __init__(self):
+                raise RuntimeError()
+
+        try:
+            raise IndexError from MyException
+        except RuntimeError:
+            pass
+        else:
+            self.fail("No exception raised")

tests/run/trybreak.pyx

-__doc__ = u"""
->>> print(foo())
-a
-"""
-
 # Indirectly makes sure the cleanup happens correctly on breaking.
-def foo():
-    for x in "abc":
+
+def try_except_break():
+    """
+    >>> print(try_except_break())
+    a
+    """
+    for x in list("abc"):
         try:
             x()
         except:
             break
-    for x in "abc":
+    return x
+
+def try_break_except():
+    """
+    >>> print(try_break_except())
+    a
+    """
+    for x in list("abc"):
+        try:
+            break
+        except:
+            pass
+    return x
+
+def try_no_break_except_return():
+    """
+    >>> print(try_no_break_except_return())
+    a
+    """
+    for x in list("abc"):
         try:
             x()
+            break
         except:
             return x
+    return x

tests/run/with_statement_module_level_T536.pyx

@@ -3,17 +3,32 @@
 __doc__ = """
 >>> inner_result
 ['ENTER']
->>> result
+>>> result  # doctest: +ELLIPSIS
+['ENTER', ...EXIT (<...ValueError...>,...ValueError..., <traceback object at ...)...]
+
+>>> inner_result_no_exc
+['ENTER']
+>>> result_no_exc
 ['ENTER', 'EXIT (None, None, None)']
 """
 
-result = []
-
 class ContextManager(object):
+    def __init__(self, result):
+        self.result = result
     def __enter__(self):
-        result.append("ENTER")
+        self.result.append("ENTER")
     def __exit__(self, *values):
-        result.append("EXIT %r" % (values,))
+        self.result.append("EXIT %r" % (values,))
+        return True
+
+result_no_exc = []
 
-with ContextManager() as c:
+with ContextManager(result_no_exc) as c:
+    inner_result_no_exc = result_no_exc[:]
+
+result = []
+
+with ContextManager(result) as c:
     inner_result = result[:]
+    raise ValueError('TEST')
+    

tests/run/withstat.pyx

@@ -56,17 +56,6 @@ def with_pass():
     with ContextManager(u"value") as x:
         pass
 
-def with_return():
-    """
-    >>> with_return()
-    enter
-    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
-    """
-    with ContextManager(u"value") as x:
-        # FIXME: DISABLED - currently crashes!!
-        # return x
-        pass
-
 def with_exception(exit_ret):
     """
     >>> with_exception(None)

tests/run/withstat_py.py

+import sys
+
+def typename(t):
+    name = type(t).__name__
+    if sys.version_info < (2,5):
+        if name == 'classobj' and issubclass(t, MyException):
+            name = 'type'
+        elif name == 'instance' and isinstance(t, MyException):
+            name = 'MyException'
+    return "<type '%s'>" % name
+
+class MyException(Exception):
+    pass
+
+class ContextManager(object):
+    def __init__(self, value, exit_ret = None):
+        self.value = value
+        self.exit_ret = exit_ret
+
+    def __exit__(self, a, b, tb):
+        print("exit %s %s %s" % (typename(a), typename(b), typename(tb)))
+        return self.exit_ret
+
+    def __enter__(self):
+        print("enter")
+        return self.value
+
+def no_as():
+    """
+    >>> no_as()
+    enter
+    hello
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    """
+    with ContextManager("value"):
+        print("hello")
+
+def basic():
+    """
+    >>> basic()
+    enter
+    value
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    """
+    with ContextManager("value") as x:
+        print(x)
+
+def with_pass():
+    """
+    >>> with_pass()
+    enter
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    """
+    with ContextManager("value") as x:
+        pass
+
+def with_return():
+    """
+    >>> print(with_return())
+    enter
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    value
+    """
+    with ContextManager("value") as x:
+        return x
+
+def with_break():
+    """
+    >>> print(with_break())
+    enter
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    a
+    """
+    for c in list("abc"):
+        with ContextManager("value") as x:
+            break
+        print("FAILED")
+    return c
+
+def with_continue():
+    """
+    >>> print(with_continue())
+    enter
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    enter
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    enter
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    c
+    """
+    for c in list("abc"):
+        with ContextManager("value") as x:
+            continue
+        print("FAILED")
+    return c
+
+def with_exception(exit_ret):
+    """
+    >>> with_exception(None)
+    enter
+    value
+    exit <type 'type'> <type 'MyException'> <type 'traceback'>
+    outer except
+    >>> with_exception(True)
+    enter
+    value
+    exit <type 'type'> <type 'MyException'> <type 'traceback'>
+    """
+    try:
+        with ContextManager("value", exit_ret=exit_ret) as value:
+            print(value)
+            raise MyException()
+    except:
+        print("outer except")
+
+def functions_in_with():
+    """
+    >>> f = functions_in_with()
+    enter
+    exit <type 'type'> <type 'MyException'> <type 'traceback'>
+    outer except
+    >>> f(1)[0]
+    1
+    >>> print(f(1)[1])
+    value
+    """
+    try:
+        with ContextManager("value") as value:
+            def f(x): return x, value
+            make = lambda x:x()
+            raise make(MyException)
+    except:
+        print("outer except")
+    return f
+
+def multitarget():
+    """
+    >>> multitarget()
+    enter
+    1 2 3 4 5
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    """
+    with ContextManager((1, 2, (3, (4, 5)))) as (a, b, (c, (d, e))):
+        print('%s %s %s %s %s' % (a, b, c, d, e))
+
+def tupletarget():
+    """
+    >>> tupletarget()
+    enter
+    (1, 2, (3, (4, 5)))
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    """
+    with ContextManager((1, 2, (3, (4, 5)))) as t:
+        print(t)
+
+def multimanager():
+    """
+    >>> multimanager()
+    enter
+    enter
+    enter
+    enter
+    enter
+    enter
+    2
+    value
+    1 2 3 4 5
+    nested
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    """
+    with ContextManager(1), ContextManager(2) as x, ContextManager('value') as y,\
+            ContextManager(3), ContextManager((1, 2, (3, (4, 5)))) as (a, b, (c, (d, e))):
+        with ContextManager('nested') as nested:
+            print(x)
+            print(y)
+            print('%s %s %s %s %s' % (a, b, c, d, e))
+            print(nested)
+
+# Tests borrowed from pyregr test_with.py,
+# modified to follow the constraints of Cython.
+import unittest
+
+class Dummy(object):
+    def __init__(self, value=None, gobble=False):
+        if value is None:
+            value = self
+        self.value = value
+        self.gobble = gobble
+        self.enter_called = False
+        self.exit_called = False
+
+    def __enter__(self):
+        self.enter_called = True
+        return self.value
+
+    def __exit__(self, *exc_info):
+        self.exit_called = True
+        self.exc_info = exc_info
+        if self.gobble:
+            return True
+
+class InitRaises(object):
+    def __init__(self): raise RuntimeError()
+
+class EnterRaises(object):
+    def __enter__(self): raise RuntimeError()
+    def __exit__(self, *exc_info): pass
+
+class ExitRaises(object):
+    def __enter__(self): pass
+    def __exit__(self, *exc_info): raise RuntimeError()
+
+class NestedWith(unittest.TestCase):
+    """
+    >>> NestedWith().runTest()
+    """
+
+    def runTest(self):
+        self.testNoExceptions()
+        self.testExceptionInExprList()
+        self.testExceptionInEnter()
+        self.testExceptionInExit()
+        self.testEnterReturnsTuple()
+
+    def testNoExceptions(self):
+        with Dummy() as a, Dummy() as b:
+            self.assertTrue(a.enter_called)
+            self.assertTrue(b.enter_called)
+        self.assertTrue(a.exit_called)
+        self.assertTrue(b.exit_called)
+
+    def testExceptionInExprList(self):
+        try:
+            with Dummy() as a, InitRaises():
+                pass
+        except:
+            pass
+        self.assertTrue(a.enter_called)
+        self.assertTrue(a.exit_called)
+
+    def testExceptionInEnter(self):
+        try:
+            with Dummy() as a, EnterRaises():
+                self.fail('body of bad with executed')
+        except RuntimeError:
+            pass
+        else:
+            self.fail('RuntimeError not reraised')
+        self.assertTrue(a.enter_called)
+        self.assertTrue(a.exit_called)
+
+    def testExceptionInExit(self):
+        body_executed = False
+        with Dummy(gobble=True) as a, ExitRaises():
+            body_executed = True
+        self.assertTrue(a.enter_called)
+        self.assertTrue(a.exit_called)
+        self.assertTrue(body_executed)
+        self.assertNotEqual(a.exc_info[0], None)
+
+    def testEnterReturnsTuple(self):
+        with Dummy(value=(1,2)) as (a1, a2), \
+             Dummy(value=(10, 20)) as (b1, b2):
+            self.assertEquals(1, a1)
+            self.assertEquals(2, a2)
+            self.assertEquals(10, b1)
+            self.assertEquals(20, b2)