merge

Cython/Build/BuildExecutable.py

@@ -13,17 +13,27 @@ import sys
 import os
 from distutils import sysconfig
 
-INCDIR = sysconfig.get_python_inc()
-LIBDIR1 = sysconfig.get_config_var('LIBDIR')
-LIBDIR2 = sysconfig.get_config_var('LIBPL')
-PYLIB = sysconfig.get_config_var('LIBRARY')[3:-2]
+def get_config_var(name):
+    return sysconfig.get_config_var(name) or ''
 
-CC = sysconfig.get_config_var('CC')
-CFLAGS = sysconfig.get_config_var('CFLAGS') + ' ' + os.environ.get('CFLAGS', '')
-LINKCC = sysconfig.get_config_var('LINKCC')
-LINKFORSHARED = sysconfig.get_config_var('LINKFORSHARED')
-LIBS = sysconfig.get_config_var('LIBS')
-SYSLIBS = sysconfig.get_config_var('SYSLIBS')
+INCDIR = sysconfig.get_python_inc()
+LIBDIR1 = get_config_var('LIBDIR')
+LIBDIR2 = get_config_var('LIBPL')
+PYLIB = get_config_var('LIBRARY')
+PYLIB_DYN = get_config_var('LDLIBRARY')
+if PYLIB_DYN == PYLIB:
+    # no shared library
+    PYLIB_DYN = ''
+else:
+    PYLIB_DYN = os.path.splitext(PYLIB_DYN[3:])[0] # 'lib(XYZ).so' -> XYZ
+
+CC = get_config_var('CC')
+CFLAGS = get_config_var('CFLAGS') + ' ' + os.environ.get('CFLAGS', '')
+LINKCC = get_config_var('LINKCC')
+LINKFORSHARED = get_config_var('LINKFORSHARED')
+LIBS = get_config_var('LIBS')
+SYSLIBS = get_config_var('SYSLIBS')
+EXE_EXT = sysconfig.get_config_var('EXE')
 
 def _debug(msg, *args):
     if DEBUG:
@@ -36,12 +46,14 @@ def dump_config():
     _debug('LIBDIR1: %s', LIBDIR1)
     _debug('LIBDIR2: %s', LIBDIR2)
     _debug('PYLIB: %s', PYLIB)
+    _debug('PYLIB_DYN: %s', PYLIB_DYN)
     _debug('CC: %s', CC)
     _debug('CFLAGS: %s', CFLAGS)
     _debug('LINKCC: %s', LINKCC)
     _debug('LINKFORSHARED: %s', LINKFORSHARED)
     _debug('LIBS: %s', LIBS)
     _debug('SYSLIBS: %s', SYSLIBS)
+    _debug('EXE_EXT: %s', EXE_EXT)
 
 def runcmd(cmd, shell=True):
     if shell:
@@ -61,7 +73,8 @@ def runcmd(cmd, shell=True):
         sys.exit(returncode)
 
 def clink(basename):
-    runcmd([LINKCC, '-o', basename, basename+'.o', '-L'+LIBDIR1, '-L'+LIBDIR2, '-l'+PYLIB]
+    runcmd([LINKCC, '-o', basename + EXE_EXT, basename+'.o', '-L'+LIBDIR1, '-L'+LIBDIR2]
+           + [PYLIB_DYN and ('-l'+PYLIB_DYN) or os.path.join(LIBDIR1, PYLIB)]
            + LIBS.split() + SYSLIBS.split() + LINKFORSHARED.split())
 
 def ccompile(basename):
@@ -75,8 +88,8 @@ def cycompile(input_file, options=()):
     if result.num_errors > 0:
         sys.exit(1)
 
-def exec_file(basename, args=()):
-    runcmd([os.path.abspath(basename)] + list(args), shell=False)
+def exec_file(program_name, args=()):
+    runcmd([os.path.abspath(program_name)] + list(args), shell=False)
 
 def build(input_file, compiler_args=()):
     """
@@ -88,7 +101,7 @@ def build(input_file, compiler_args=()):
     cycompile(input_file, compiler_args)
     ccompile(basename)
     clink(basename)
-    return basename
+    return basename + EXE_EXT
 
 def build_and_run(args):
     """
@@ -114,3 +127,6 @@ def build_and_run(args):
 
     program_name = build(input_file, cy_args)
     exec_file(program_name, args)
+
+if __name__ == '__main__':
+    build_and_run(sys.argv[1:])

Cython/Compiler/Builtin.py

@@ -95,6 +95,54 @@ static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) {
 }
 """)
 
+globals_utility_code = UtilityCode(
+# This is a stub implementation until we have something more complete.
+# Currently, we only handle the most common case of a read-only dict
+# of Python names.  Supporting cdef names in the module and write
+# access requires a rewrite as a dedicated class.
+proto = """
+static PyObject* __Pyx_Globals(); /*proto*/
+""",
+impl = '''
+static PyObject* __Pyx_Globals() {
+    Py_ssize_t i;
+    /*PyObject *d;*/
+    PyObject *names = NULL;
+    PyObject *globals = PyObject_GetAttrString(%(MODULE)s, "__dict__");
+    if (!globals) {
+        PyErr_SetString(PyExc_TypeError,
+            "current module must have __dict__ attribute");
+        goto bad;
+    }
+    names = PyObject_Dir(%(MODULE)s);
+    if (!names)
+        goto bad;
+    for (i = 0; i < PyList_GET_SIZE(names); i++) {
+        PyObject* name = PyList_GET_ITEM(names, i);
+        if (!PyDict_Contains(globals, name)) {
+            PyObject* value = PyObject_GetAttr(%(MODULE)s, PyList_GET_ITEM(names, i));
+            if (!value)
+                goto bad;
+            if (PyDict_SetItem(globals, name, value) < 0) {
+                Py_DECREF(value);
+                goto bad;
+            }
+        }
+    }
+    Py_DECREF(names);
+    return globals;
+    /*
+    d = PyDictProxy_New(globals);
+    Py_DECREF(globals);
+    return d;
+    */
+bad:
+    Py_XDECREF(names);
+    Py_XDECREF(globals);
+    return NULL;
+}
+''' % {'MODULE' : Naming.module_cname})
+
 pyexec_utility_code = UtilityCode(
 proto = """
 #if PY_VERSION_HEX < 0x02040000
@@ -384,6 +432,8 @@ builtin_function_table = [
                     utility_code = getattr3_utility_code),
     BuiltinFunction('getattr3',   "OOO",  "O",     "__Pyx_GetAttr3",     "getattr",
                     utility_code = getattr3_utility_code), # Pyrex compatibility
+    BuiltinFunction('globals',    "",     "O",     "__Pyx_Globals",
+                    utility_code = globals_utility_code),
     BuiltinFunction('hasattr',    "OO",   "b",     "__Pyx_HasAttr",
                     utility_code = hasattr_utility_code),
     BuiltinFunction('hash',       "O",    "h",     "PyObject_Hash"),

Cython/Compiler/CmdLine.py

@@ -163,15 +163,7 @@ def parse_command_line(args):
                 sys.stderr.write("Unknown compiler flag: %s\n" % option)
                 sys.exit(1)
         else:
-            arg = pop_arg()
-            if arg.endswith(".pyx"):
-                sources.append(arg)
-            elif arg.endswith(".py"):
-                # maybe do some other stuff, but this should work for now
-                sources.append(arg)
-            else:
-                sys.stderr.write(
-                    "cython: %s: Unknown filename suffix\n" % arg)
+            sources.append(pop_arg())
     if options.use_listing_file and len(sources) > 1:
         sys.stderr.write(
             "cython: Only one source file allowed when using -o\n")

Cython/Compiler/Code.pxd

@@ -59,9 +59,10 @@ cdef class StringConst:
     cdef public object text
     cdef public object escaped_value
     cdef public dict py_strings
+    cdef public list py_versions
 
     @cython.locals(intern=bint, is_str=bint, is_unicode=bint)
-    cpdef get_py_string_const(self, encoding, identifier=*, is_str=*)
+    cpdef get_py_string_const(self, encoding, identifier=*, is_str=*, py3str_cstring=*)
 
 ## cdef class PyStringConst:
 ##     cdef public object cname

Cython/Compiler/Code.py

@@ -25,12 +25,19 @@ except ImportError:
 
 
 non_portable_builtins_map = {
+    # builtins that have different names in different Python versions
     'bytes'         : ('PY_MAJOR_VERSION < 3',  'str'),
     'unicode'       : ('PY_MAJOR_VERSION >= 3', 'str'),
     'xrange'        : ('PY_MAJOR_VERSION >= 3', 'range'),
     'BaseException' : ('PY_VERSION_HEX < 0x02050000', 'Exception'),
     }
 
+uncachable_builtins = [
+    # builtin names that cannot be cached because they may or may not
+    # be available at import time
+    'WindowsError',
+    ]
+
 class UtilityCode(object):
     # Stores utility code to add during code generation.
     #
@@ -324,8 +331,16 @@ class StringConst(object):
         self.text = text
         self.escaped_value = StringEncoding.escape_byte_string(byte_string)
         self.py_strings = None
+        self.py_versions = []
+
+    def add_py_version(self, version):
+        if not version:
+            self.py_versions = [2,3]
+        elif version not in self.py_versions:
+            self.py_versions.append(version)
 
-    def get_py_string_const(self, encoding, identifier=None, is_str=False):
+    def get_py_string_const(self, encoding, identifier=None,
+                            is_str=False, py3str_cstring=None):
         py_strings = self.py_strings
         text = self.text
 
@@ -344,47 +359,52 @@ class StringConst(object):
             else:
                 encoding_key = ''.join(find_alphanums(encoding))
 
-        key = (is_str, is_unicode, encoding_key)
-        if py_strings is not None and key in py_strings:
-            py_string = py_strings[key]
+        key = (is_str, is_unicode, encoding_key, py3str_cstring)
+        if py_strings is not None:
+            try:
+                return py_strings[key]
+            except KeyError:
+                pass
         else:
-            if py_strings is None:
-                self.py_strings = {}
-            if identifier:
-                intern = True
-            elif identifier is None:
-                if isinstance(text, unicode):
-                    intern = bool(possible_unicode_identifier(text))
-                else:
-                    intern = bool(possible_bytes_identifier(text))
-            else:
-                intern = False
-            if intern:
-                prefix = Naming.interned_str_prefix
-            else:
-                prefix = Naming.py_const_prefix
-            pystring_cname = "%s%s_%s" % (
-                prefix,
-                (is_str and 's') or (is_unicode and 'u') or 'b',
-                self.cname[len(Naming.const_prefix):])
-
-            py_string = PyStringConst(
-                pystring_cname, encoding, is_unicode, is_str, intern)
-            self.py_strings[key] = py_string
+            self.py_strings = {}
 
+        if identifier:
+            intern = True
+        elif identifier is None:
+            if isinstance(text, unicode):
+                intern = bool(possible_unicode_identifier(text))
+            else:
+                intern = bool(possible_bytes_identifier(text))
+        else:
+            intern = False
+        if intern:
+            prefix = Naming.interned_str_prefix
+        else:
+            prefix = Naming.py_const_prefix
+        pystring_cname = "%s%s_%s" % (
+            prefix,
+            (is_str and 's') or (is_unicode and 'u') or 'b',
+            self.cname[len(Naming.const_prefix):])
+
+        py_string = PyStringConst(
+            pystring_cname, encoding, is_unicode, is_str, py3str_cstring, intern)
+        self.py_strings[key] = py_string
         return py_string
 
 class PyStringConst(object):
     """Global info about a Python string constant held by GlobalState.
     """
     # cname       string
+    # py3str_cstring string
     # encoding    string
     # intern      boolean
     # is_unicode  boolean
     # is_str      boolean
 
-    def __init__(self, cname, encoding, is_unicode, is_str=False, intern=False):
+    def __init__(self, cname, encoding, is_unicode, is_str=False,
+                 py3str_cstring=None, intern=False):
         self.cname = cname
+        self.py3str_cstring = py3str_cstring
         self.encoding = encoding
         self.is_str = is_str
         self.is_unicode = is_unicode
@@ -595,7 +615,7 @@ class GlobalState(object):
             cleanup_writer.put_xdecref_clear(const.cname, type, nanny=False)
         return const
 
-    def get_string_const(self, text):
+    def get_string_const(self, text, py_version=None):
         # return a C string constant, creating a new one if necessary
         if text.is_unicode:
             byte_string = text.utf8encode()
@@ -605,12 +625,21 @@ class GlobalState(object):
             c = self.string_const_index[byte_string]
         except KeyError:
             c = self.new_string_const(text, byte_string)
+        c.add_py_version(py_version)
         return c
 
-    def get_py_string_const(self, text, identifier=None, is_str=False):
+    def get_py_string_const(self, text, identifier=None,
+                            is_str=False, unicode_value=None):
         # return a Python string constant, creating a new one if necessary
-        c_string = self.get_string_const(text)
-        py_string = c_string.get_py_string_const(text.encoding, identifier, is_str)
+        py3str_cstring = None
+        if is_str and unicode_value is not None \
+               and unicode_value.utf8encode() != text.byteencode():
+            py3str_cstring = self.get_string_const(unicode_value, py_version=3)
+            c_string = self.get_string_const(text, py_version=2)
+        else:
+            c_string = self.get_string_const(text)
+        py_string = c_string.get_py_string_const(
+            text.encoding, identifier, is_str, py3str_cstring)
         return py_string
 
     def get_interned_identifier(self, text):
@@ -711,8 +740,15 @@ class GlobalState(object):
 
         decls_writer = self.parts['decls']
         for _, cname, c in c_consts:
+            conditional = False
+            if c.py_versions and (2 not in c.py_versions or 3 not in c.py_versions):
+                conditional = True
+                decls_writer.putln("#if PY_MAJOR_VERSION %s 3" % (
+                    (2 in c.py_versions) and '<' or '>='))
             decls_writer.putln('static char %s[] = "%s";' % (
                 cname, StringEncoding.split_string_literal(c.escaped_value)))
+            if conditional:
+                decls_writer.putln("#endif")
             if c.py_strings is not None:
                 for py_string in c.py_strings.values():
                     py_strings.append((c.cname, len(py_string.cname), py_string))
@@ -736,6 +772,17 @@ class GlobalState(object):
 
                 decls_writer.putln(
                     "static PyObject *%s;" % py_string.cname)
+                if py_string.py3str_cstring:
+                    w.putln("#if PY_MAJOR_VERSION >= 3")
+                    w.putln(
+                        "{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % (
+                        py_string.cname,
+                        py_string.py3str_cstring.cname,
+                        py_string.py3str_cstring.cname,
+                        '0', 1, 0,
+                        py_string.intern
+                        ))
+                    w.putln("#else")
                 w.putln(
                     "{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % (
                     py_string.cname,
@@ -746,6 +793,8 @@ class GlobalState(object):
                     py_string.is_str,
                     py_string.intern
                     ))
+                if py_string.py3str_cstring:
+                    w.putln("#endif")
             w.putln("{0, 0, 0, 0, 0, 0, 0}")
             w.putln("};")
 
@@ -1003,8 +1052,10 @@ class CCodeWriter(object):
     def get_string_const(self, text):
         return self.globalstate.get_string_const(text).cname
 
-    def get_py_string_const(self, text, identifier=None, is_str=False):
-        return self.globalstate.get_py_string_const(text, identifier, is_str).cname
+    def get_py_string_const(self, text, identifier=None,
+                            is_str=False, unicode_value=None):
+        return self.globalstate.get_py_string_const(
+            text, identifier, is_str, unicode_value).cname
 
     def get_argument_default_const(self, type):
         return self.globalstate.get_py_const(type).cname

Cython/Compiler/Main.py

@@ -616,10 +616,10 @@ def run_pipeline(source, options, full_module_name = None):
         if os.path.exists(html_filename):
             line = codecs.open(html_filename, "r", encoding="UTF-8").readline()
             if line.startswith(u'<!-- Generated by Cython'):
-                options.annotate = True            
+                options.annotate = True
 
     # Get pipeline
-    if source_ext.lower() == '.py':
+    if source_ext.lower() == '.py' or not source_ext:
         pipeline = context.create_py_pipeline(options, result)
     else:
         pipeline = context.create_pyx_pipeline(options, result)

Cython/Compiler/ModuleNode.py

@@ -73,7 +73,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         self.generate_c_code(env, options, result)
         self.generate_h_code(env, options, result)
         self.generate_api_code(env, result)
-    
+
     def has_imported_c_functions(self):
         for module in self.referenced_modules:
             for entry in module.cfunc_entries:
@@ -172,7 +172,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
 
     def api_name(self, env):
         return env.qualified_name.replace(".", "__")
-    
+
     def generate_api_code(self, env, result):
         def api_entries(entries, pxd=0):
             return [entry for entry in entries
@@ -255,7 +255,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
 
     def generate_cclass_header_code(self, type, h_code):
         h_code.putln("%s %s %s;" % (
-            Naming.extern_c_macro, 
+            Naming.extern_c_macro,
             PyrexTypes.public_decl("PyTypeObject", "DL_IMPORT"),
             type.typeobj_cname))
 
@@ -1785,7 +1785,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("#endif")
         code.putln("{")
         tempdecl_code = code.insertion_point()
-        
+
         code.put_declare_refcount_context()
         code.putln("#if CYTHON_REFNANNY")
         code.putln("__Pyx_RefNanny = __Pyx_RefNannyImportAPI(\"refnanny\");")
@@ -2900,104 +2900,104 @@ static int __Pyx_main(int argc, wchar_t **argv) {
 static wchar_t*
 __Pyx_char2wchar(char* arg)
 {
-	wchar_t *res;
+    wchar_t *res;
 #ifdef HAVE_BROKEN_MBSTOWCS
-	/* Some platforms have a broken implementation of
-	 * mbstowcs which does not count the characters that
-	 * would result from conversion.  Use an upper bound.
-	 */
-	size_t argsize = strlen(arg);
+    /* Some platforms have a broken implementation of
+     * mbstowcs which does not count the characters that
+     * would result from conversion.  Use an upper bound.
+     */
+    size_t argsize = strlen(arg);
 #else
-	size_t argsize = mbstowcs(NULL, arg, 0);
+    size_t argsize = mbstowcs(NULL, arg, 0);
 #endif
-	size_t count;
-	unsigned char *in;
-	wchar_t *out;
+    size_t count;
+    unsigned char *in;
+    wchar_t *out;
 #ifdef HAVE_MBRTOWC
-	mbstate_t mbs;
+    mbstate_t mbs;
 #endif
-	if (argsize != (size_t)-1) {
-		res = (wchar_t *)malloc((argsize+1)*sizeof(wchar_t));
-		if (!res)
-			goto oom;
-		count = mbstowcs(res, arg, argsize+1);
-		if (count != (size_t)-1) {
-			wchar_t *tmp;
-			/* Only use the result if it contains no
-			   surrogate characters. */
-			for (tmp = res; *tmp != 0 &&
-				     (*tmp < 0xd800 || *tmp > 0xdfff); tmp++)
-				;
-			if (*tmp == 0)
-				return res;
-		}
-		free(res);
-	}
-	/* Conversion failed. Fall back to escaping with surrogateescape. */
+    if (argsize != (size_t)-1) {
+        res = (wchar_t *)malloc((argsize+1)*sizeof(wchar_t));
+        if (!res)
+            goto oom;
+        count = mbstowcs(res, arg, argsize+1);
+        if (count != (size_t)-1) {
+            wchar_t *tmp;
+            /* Only use the result if it contains no
+               surrogate characters. */
+            for (tmp = res; *tmp != 0 &&
+                     (*tmp < 0xd800 || *tmp > 0xdfff); tmp++)
+                ;
+            if (*tmp == 0)
+                return res;
+        }
+        free(res);
+    }
+    /* Conversion failed. Fall back to escaping with surrogateescape. */
 #ifdef HAVE_MBRTOWC
-	/* Try conversion with mbrtwoc (C99), and escape non-decodable bytes. */
-
-	/* Overallocate; as multi-byte characters are in the argument, the
-	   actual output could use less memory. */
-	argsize = strlen(arg) + 1;
-	res = malloc(argsize*sizeof(wchar_t));
-	if (!res) goto oom;
-	in = (unsigned char*)arg;
-	out = res;
-	memset(&mbs, 0, sizeof mbs);
-	while (argsize) {
-		size_t converted = mbrtowc(out, (char*)in, argsize, &mbs);
-		if (converted == 0)
-			/* Reached end of string; null char stored. */
-			break;
-		if (converted == (size_t)-2) {
-			/* Incomplete character. This should never happen,
-			   since we provide everything that we have -
-			   unless there is a bug in the C library, or I
-			   misunderstood how mbrtowc works. */
-			fprintf(stderr, "unexpected mbrtowc result -2\\n");
-			return NULL;
-		}
-		if (converted == (size_t)-1) {
-			/* Conversion error. Escape as UTF-8b, and start over
-			   in the initial shift state. */
-			*out++ = 0xdc00 + *in++;
-			argsize--;
-			memset(&mbs, 0, sizeof mbs);
-			continue;
-		}
-		if (*out >= 0xd800 && *out <= 0xdfff) {
-			/* Surrogate character.  Escape the original
-			   byte sequence with surrogateescape. */
-			argsize -= converted;
-			while (converted--)
-				*out++ = 0xdc00 + *in++;
-			continue;
-		}
-		/* successfully converted some bytes */
-		in += converted;
-		argsize -= converted;
-		out++;
-	}
+    /* Try conversion with mbrtwoc (C99), and escape non-decodable bytes. */
+
+    /* Overallocate; as multi-byte characters are in the argument, the
+       actual output could use less memory. */
+    argsize = strlen(arg) + 1;
+    res = malloc(argsize*sizeof(wchar_t));
+    if (!res) goto oom;
+    in = (unsigned char*)arg;
+    out = res;
+    memset(&mbs, 0, sizeof mbs);
+    while (argsize) {
+        size_t converted = mbrtowc(out, (char*)in, argsize, &mbs);
+        if (converted == 0)
+            /* Reached end of string; null char stored. */
+            break;
+        if (converted == (size_t)-2) {
+            /* Incomplete character. This should never happen,
+               since we provide everything that we have -
+               unless there is a bug in the C library, or I
+               misunderstood how mbrtowc works. */
+            fprintf(stderr, "unexpected mbrtowc result -2\\n");
+            return NULL;
+        }
+        if (converted == (size_t)-1) {
+            /* Conversion error. Escape as UTF-8b, and start over
+               in the initial shift state. */
+            *out++ = 0xdc00 + *in++;
+            argsize--;
+            memset(&mbs, 0, sizeof mbs);
+            continue;
+        }
+        if (*out >= 0xd800 && *out <= 0xdfff) {
+            /* Surrogate character.  Escape the original
+               byte sequence with surrogateescape. */
+            argsize -= converted;
+            while (converted--)
+                *out++ = 0xdc00 + *in++;
+            continue;
+        }
+        /* successfully converted some bytes */
+        in += converted;
+        argsize -= converted;
+        out++;
+    }
 #else
-	/* Cannot use C locale for escaping; manually escape as if charset
-	   is ASCII (i.e. escape all bytes > 128. This will still roundtrip
-	   correctly in the locale's charset, which must be an ASCII superset. */
-	res = malloc((strlen(arg)+1)*sizeof(wchar_t));
-	if (!res) goto oom;
-	in = (unsigned char*)arg;
-	out = res;
-	while(*in)
-		if(*in < 128)
-			*out++ = *in++;
-		else
-			*out++ = 0xdc00 + *in++;
-	*out = 0;
+    /* Cannot use C locale for escaping; manually escape as if charset
+       is ASCII (i.e. escape all bytes > 128. This will still roundtrip
+       correctly in the locale's charset, which must be an ASCII superset. */
+    res = malloc((strlen(arg)+1)*sizeof(wchar_t));
+    if (!res) goto oom;
+    in = (unsigned char*)arg;
+    out = res;
+    while(*in)
+        if(*in < 128)
+            *out++ = *in++;
+        else
+            *out++ = 0xdc00 + *in++;
+    *out = 0;
 #endif
-	return res;
+    return res;
 oom:
-	fprintf(stderr, "out of memory\\n");
-	return NULL;
+    fprintf(stderr, "out of memory\\n");
+    return NULL;
 }
 
 int

Cython/Compiler/Optimize.py

@@ -930,7 +930,15 @@ class FlattenInListTransform(Visitor.VisitorTransform, SkipDeclarations):
         conds = []
         temps = []
         for arg in args:
-            if not arg.is_simple():
+            try:
+                # Trial optimisation to avoid redundant temp
+                # assignments.  However, since is_simple() is meant to
+                # be called after type analysis, we ignore any errors
+                # and just play safe in that case.
+                is_simple_arg = arg.is_simple()
+            except Exception:
+                is_simple_arg = False
+            if not is_simple_arg:
                 # must evaluate all non-simple RHS before doing the comparisons
                 arg = UtilNodes.LetRefNode(arg)
                 temps.append(arg)
@@ -3134,6 +3142,15 @@ class ConstantFolding(Visitor.VisitorTransform, SkipDeclarations):
         return ExprNodes.BoolNode(node.pos, value=bool_result,
                                   constant_result=bool_result)
 
+    def visit_CondExprNode(self, node):
+        self._calculate_const(node)
+        if node.test.constant_result is ExprNodes.not_a_constant:
+            return node
+        if node.test.constant_result:
+            return node.true_val
+        else:
+            return node.false_val
+
     def visit_IfStatNode(self, node):
         self.visitchildren(node)
         # eliminate dead code based on constant condition results

Cython/Compiler/ParseTreeTransforms.py

@@ -11,11 +11,12 @@ import Naming
 import ExprNodes
 import Nodes
 import Options
+import Builtin
 
 from Cython.Compiler.Visitor import VisitorTransform, TreeVisitor
 from Cython.Compiler.Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
 from Cython.Compiler.ModuleNode import ModuleNode
-from Cython.Compiler.UtilNodes import LetNode, LetRefNode
+from Cython.Compiler.UtilNodes import LetNode, LetRefNode, ResultRefNode
 from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
 from Cython.Compiler.StringEncoding import EncodedString
 from Cython.Compiler.Errors import error, warning, CompileError, InternalError
@@ -928,81 +929,55 @@ class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
 
 
 class WithTransform(CythonTransform, SkipDeclarations):
-
-    # EXCINFO is manually set to a variable that contains
-    # the exc_info() tuple that can be generated by the enclosing except
-    # statement.
-    template_without_target = TreeFragment(u"""
-        MGR = EXPR
-        EXIT = MGR.__exit__
-        MGR.__enter__()
-        EXC = True
-        try:
-            try:
-                EXCINFO = None
-                BODY
-            except:
-                EXC = False
-                if not EXIT(*EXCINFO):
-                    raise
-        finally:
-            if EXC:
-                EXIT(None, None, None)
-    """, temps=[u'MGR', u'EXC', u"EXIT"],
-    pipeline=[NormalizeTree(None)])
-
-    template_with_target = TreeFragment(u"""
-        MGR = EXPR
-        EXIT = MGR.__exit__
-        VALUE = MGR.__enter__()
-        EXC = True
-        try:
-            try:
-                EXCINFO = None
-                TARGET = VALUE
-                BODY
-            except:
-                EXC = False
-                if not EXIT(*EXCINFO):
-                    raise
-        finally:
-            if EXC:
-                EXIT(None, None, None)
-            MGR = EXIT = VALUE = EXC = None
-
-    """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE"],
-    pipeline=[NormalizeTree(None)])
-
     def visit_WithStatNode(self, node):
-        # TODO: Cleanup badly needed
-        TemplateTransform.temp_name_counter += 1
-        handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter
-
-        self.visitchildren(node, ['body'])
-        excinfo_temp = ExprNodes.NameNode(node.pos, name=handle)#TempHandle(Builtin.tuple_type)
-        if node.target is not None:
-            result = self.template_with_target.substitute({
-                u'EXPR' : node.manager,
-                u'BODY' : node.body,
-                u'TARGET' : node.target,
-                u'EXCINFO' : excinfo_temp
-                }, pos=node.pos)
-        else:
-            result = self.template_without_target.substitute({
-                u'EXPR' : node.manager,
-                u'BODY' : node.body,
-                u'EXCINFO' : excinfo_temp
-                }, pos=node.pos)
-
-        # Set except excinfo target to EXCINFO
-        try_except = result.stats[-1].body.stats[-1]
-        try_except.except_clauses[0].excinfo_target = ExprNodes.NameNode(node.pos, name=handle)
-#            excinfo_temp.ref(node.pos))
-
-#        result.stats[-1].body.stats[-1] = TempsBlockNode(
-#            node.pos, temps=[excinfo_temp], body=try_except)
-
-        return result
+        self.visitchildren(node, 'body')
+        pos = node.pos
+        body, target, manager = node.body, node.target, node.manager
+        node.target_temp = ExprNodes.TempNode(pos, type=PyrexTypes.py_object_type)
+        if target is not None:
+            node.has_target = True
+            body = Nodes.StatListNode(
+                pos, stats = [
+                    Nodes.WithTargetAssignmentStatNode(
+                        pos, lhs = target, rhs = node.target_temp),
+                    body
+                    ])
+            node.target = None
+
+        excinfo_target = ResultRefNode(
+            pos=pos, type=Builtin.tuple_type, may_hold_none=False)
+        except_clause = Nodes.ExceptClauseNode(
+            pos, body = Nodes.IfStatNode(
+                pos, if_clauses = [
+                    Nodes.IfClauseNode(
+                        pos, condition = ExprNodes.NotNode(
+                            pos, operand = ExprNodes.WithExitCallNode(
+                                pos, with_stat = node,
+                                args = excinfo_target)),
+                        body = Nodes.ReraiseStatNode(pos),
+                        ),
+                    ],
+                else_clause = None),
+            pattern = None,
+            target = None,
+            excinfo_target = excinfo_target,
+            )
+
+        node.body = Nodes.TryFinallyStatNode(
+            pos, body = Nodes.TryExceptStatNode(
+                pos, body = body,
+                except_clauses = [except_clause],
+                else_clause = None,
+                ),
+            finally_clause = Nodes.ExprStatNode(
+                pos, expr = ExprNodes.WithExitCallNode(
+                    pos, with_stat = node,
+                    args = ExprNodes.TupleNode(
+                        pos, args = [ExprNodes.NoneNode(pos) for _ in range(3)]
+                        ))),
+            handle_error_case = False,
+            )
+        return node
 
     def visit_ExprNode(self, node):
         # With statements are never inside expressions.
@@ -1256,7 +1231,7 @@ if VALUE is not None:
             arg = copy.deepcopy(arg_template)
             arg.declarator.name = entry.name
             init_method.args.append(arg)
-            
+
         # setters/getters
         for entry, attr in zip(var_entries, attributes):
             # TODO: branch on visibility
@@ -1269,7 +1244,7 @@ if VALUE is not None:
                 }, pos = entry.pos).stats[0]
             property.name = entry.name
             wrapper_class.body.stats.append(property)
-            
+
         wrapper_class.analyse_declarations(self.env_stack[-1])
         return self.visit_CClassDefNode(wrapper_class)
 
@@ -1602,6 +1577,12 @@ class CreateClosureClasses(CythonTransform):
                           is_cdef=True)
         klass.declare_var(pos=pos, name='resume_label', cname='resume_label', type=PyrexTypes.c_int_type,
                           is_cdef=True)
+        klass.declare_var(pos=pos, name='exc_type', cname='exc_type',
+                          type=PyrexTypes.py_object_type, is_cdef=True)
+        klass.declare_var(pos=pos, name='exc_value', cname='exc_value',
+                          type=PyrexTypes.py_object_type, is_cdef=True)
+        klass.declare_var(pos=pos, name='exc_traceback', cname='exc_traceback',
+                          type=PyrexTypes.py_object_type, is_cdef=True)
 
         import TypeSlots
         e = klass.declare_pyfunction('send', pos)

Cython/Compiler/Parsing.py

@@ -1172,6 +1172,7 @@ def p_raise_statement(s):
     exc_type = None
     exc_value = None
     exc_tb = None
+    cause = None
     if s.sy not in statement_terminators:
         exc_type = p_test(s)
         if s.sy == ',':
@@ -1180,11 +1181,15 @@ def p_raise_statement(s):
             if s.sy == ',':
                 s.next()
                 exc_tb = p_test(s)
+        elif s.sy == 'from':
+            s.next()
+            cause = p_test(s)
     if exc_type or exc_value or exc_tb:
         return Nodes.RaiseStatNode(pos,
             exc_type = exc_type,
             exc_value = exc_value,
-            exc_tb = exc_tb)
+            exc_tb = exc_tb,
+            cause = cause)
     else:
         return Nodes.ReraiseStatNode(pos)
 
@@ -1660,15 +1665,27 @@ def p_simple_statement_list(s, ctx, first_statement = 0):
     # Parse a series of simple statements on one line
     # separated by semicolons.
     stat = p_simple_statement(s, first_statement = first_statement)
-    if s.sy == ';':
-        stats = [stat]
-        while s.sy == ';':
-            #print "p_simple_statement_list: maybe more to follow" ###
-            s.next()
-            if s.sy in ('NEWLINE', 'EOF'):
-                break
-            stats.append(p_simple_statement(s))
-        stat = Nodes.StatListNode(stats[0].pos, stats = stats)
+    pos = stat.pos
+    stats = []
+    if not isinstance(stat, Nodes.PassStatNode):
+        stats.append(stat)
+    while s.sy == ';':
+        #print "p_simple_statement_list: maybe more to follow" ###
+        s.next()
+        if s.sy in ('NEWLINE', 'EOF'):
+            break
+        stat = p_simple_statement(s, first_statement = first_statement)
+        if isinstance(stat, Nodes.PassStatNode):
+            continue
+        stats.append(stat)
+        first_statement = False
+
+    if not stats:
+        stat = Nodes.PassStatNode(pos)
+    elif len(stats) == 1:
+        stat = stats[0]
+    else:
+        stat = Nodes.StatListNode(pos, stats = stats)
     s.expect_newline("Syntax error in simple statement list")
     return stat
 
@@ -1805,9 +1822,14 @@ def p_statement_list(s, ctx, first_statement = 0):
     pos = s.position()
     stats = []
     while s.sy not in ('DEDENT', 'EOF'):
-        stats.append(p_statement(s, ctx, first_statement = first_statement))
-        first_statement = 0
-    if len(stats) == 1:
+        stat = p_statement(s, ctx, first_statement = first_statement)
+        if isinstance(stat, Nodes.PassStatNode):
+            continue
+        stats.append(stat)
+        first_statement = False
+    if not stats:
+        return Nodes.PassStatNode(pos)
+    elif len(stats) == 1:
         return stats[0]
     else:
         return Nodes.StatListNode(pos, stats = stats)
@@ -2523,7 +2545,7 @@ def p_c_struct_or_union_definition(s, pos, ctx):
         s.expect_dedent()
     else:
         s.expect_newline("Syntax error in struct or union definition")
-    return Nodes.CStructOrUnionDefNode(pos, 
+    return Nodes.CStructOrUnionDefNode(pos,
         name = name, cname = cname, kind = kind, attributes = attributes,
         typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,
         api = ctx.api, in_pxd = ctx.level == 'module_pxd', packed = packed)

Cython/Compiler/Symtab.py

@@ -409,13 +409,13 @@ class Scope(object):
         except ValueError, e:
             error(pos, e.args[0])
             type = PyrexTypes.error_type
-        entry = self.declare_type(name, type, pos, cname, 
+        entry = self.declare_type(name, type, pos, cname,
                                   visibility = visibility, api = api)
         type.qualified_name = entry.qualified_name
         return entry
-        
-    def declare_struct_or_union(self, name, kind, scope, 
-                                typedef_flag, pos, cname = None, 
+
+    def declare_struct_or_union(self, name, kind, scope,
+                                typedef_flag, pos, cname = None,
                                 visibility = 'private', api = 0,
                                 packed = False):
         # Add an entry for a struct or union definition.
@@ -496,7 +496,7 @@ class Scope(object):
         if entry.visibility != visibility:
             error(pos, "'%s' previously declared as '%s'" % (
                 entry.name, entry.visibility))
-    
+
     def declare_enum(self, name, pos, cname, typedef_flag,
             visibility = 'private', api = 0):
         if name:
@@ -512,7 +512,7 @@ class Scope(object):
             visibility = visibility, api = api)
         entry.enum_values = []
         self.sue_entries.append(entry)
-        return entry    
+        return entry
 
     def declare_var(self, name, type, pos,
                     cname = None, visibility = 'private', api = 0, is_cdef = 0):
@@ -564,11 +564,16 @@ class Scope(object):
         entry.is_anonymous = True
         return entry
 
-    def declare_lambda_function(self, func_cname, pos):
+    def declare_lambda_function(self, lambda_name, pos):
         # Add an entry for an anonymous Python function.
-        entry = self.declare_var(None, py_object_type, pos,
-                                 cname=func_cname, visibility='private')
-        entry.name = EncodedString(func_cname)
+        func_cname = self.mangle(Naming.lambda_func_prefix + u'funcdef_', lambda_name)
+        pymethdef_cname = self.mangle(Naming.lambda_func_prefix + u'methdef_', lambda_name)
+        qualified_name = self.qualify_name(lambda_name)
+
+        entry = self.declare(None, func_cname, py_object_type, pos, 'private')
+        entry.name = lambda_name
+        entry.qualified_name = qualified_name
+        entry.pymethdef_cname = pymethdef_cname
         entry.func_cname = func_cname
         entry.signature = pyfunction_signature
         entry.is_anonymous = True
@@ -927,21 +932,18 @@ class ModuleScope(Scope):
         return self.outer_scope.lookup(name, language_level = self.context.language_level)
 
     def declare_builtin(self, name, pos):
-        if not hasattr(builtins, name) and name not in Code.non_portable_builtins_map:
-            # 'xrange' and 'BaseException' are special cased in Code.py
+        if not hasattr(builtins, name) \
+               and name not in Code.non_portable_builtins_map \
+               and name not in Code.uncachable_builtins:
             if self.has_import_star:
                 entry = self.declare_var(name, py_object_type, pos)
                 return entry
-            ## elif self.outer_scope is not None:
-            ##     entry = self.outer_scope.declare_builtin(name, pos)
-            ##     print entry
-            ##     return entry
             else:
-                # unknown - assume it's builtin and look it up at runtime
                 if Options.error_on_unknown_names:
                     error(pos, "undeclared name not builtin: %s" % name)
                 else:
                     warning(pos, "undeclared name not builtin: %s" % name, 2)
+                # unknown - assume it's builtin and look it up at runtime
                 entry = self.declare(name, None, py_object_type, pos, 'private')
                 entry.is_builtin = 1
                 return entry
@@ -950,7 +952,7 @@ class ModuleScope(Scope):
                 if entry.name == name:
                     return entry
         entry = self.declare(None, None, py_object_type, pos, 'private')
-        if Options.cache_builtins:
+        if Options.cache_builtins and name not in Code.uncachable_builtins:
             entry.is_builtin = 1
             entry.is_const = 1 # cached
             entry.name = name
@@ -959,6 +961,7 @@ class ModuleScope(Scope):
             self.undeclared_cached_builtins.append(entry)
         else:
             entry.is_builtin = 1
+            entry.name = name
         return entry
 
     def find_module(self, module_name, pos):
@@ -1067,7 +1070,7 @@ class ModuleScope(Scope):
         buffer_defaults = None, shadow = 0):
         # If this is a non-extern typedef class, expose the typedef, but use
         # the non-typedef struct internally to avoid needing forward
-        # declarations for anonymous structs. 
+        # declarations for anonymous structs.
         if typedef_flag and visibility != 'extern':
             if not (visibility == 'public' or api):
                 warning(pos, "ctypedef only valid for 'extern' , 'public', and 'api'", 2)
@@ -1414,6 +1417,12 @@ class GeneratorExpressionScope(Scope):
         self.entries[name] = entry
         return entry
 
+    def declare_lambda_function(self, func_cname, pos):
+        return self.outer_scope.declare_lambda_function(func_cname, pos)
+
+    def add_lambda_def(self, def_node):
+        return self.outer_scope.add_lambda_def(def_node)
+
 
 class ClosureScope(LocalScope):
 
@@ -1464,7 +1473,7 @@ class StructOrUnionScope(Scope):
     def declare_cfunction(self, name, type, pos,
                           cname = None, visibility = 'private', defining = 0,
                           api = 0, in_pxd = 0, modifiers = ()): # currently no utility code ...
-        return self.declare_var(name, type, pos, 
+        return self.declare_var(name, type, pos,
                                 cname=cname, visibility=visibility)
 
 class ClassScope(Scope):
@@ -1629,7 +1638,7 @@ class CClassScope(ClassScope):
         if name == "__new__":
             error(pos, "__new__ method of extension type will change semantics "
                 "in a future version of Pyrex and Cython. Use __cinit__ instead.")
-        entry = self.declare_var(name, py_object_type, pos, 
+        entry = self.declare_var(name, py_object_type, pos,
                                  visibility='extern')
         special_sig = get_special_method_signature(name)
         if special_sig:
@@ -1755,7 +1764,7 @@ class CppClassScope(Scope):
         self.inherited_var_entries = []
 
     def declare_var(self, name, type, pos,
-                    cname = None, visibility = 'extern', api = 0, 
+                    cname = None, visibility = 'extern', api = 0,
                     is_cdef = 0, allow_pyobject = 0):
         # Add an entry for an attribute.
         if not cname:
@@ -1797,7 +1806,7 @@ class CppClassScope(Scope):
             error(pos, "no matching function for call to %s::%s()" %
                   (self.default_constructor, self.default_constructor))
 
-    def declare_cfunction(self, name, type, pos, cname = None, 
+    def declare_cfunction(self, name, type, pos, cname = None,
                           visibility = 'extern', api = 0, defining = 0,
                           in_pxd = 0, modifiers = (), utility_code = None):
         if name == self.name.split('::')[-1] and cname is None:
@@ -1805,7 +1814,7 @@ class CppClassScope(Scope):
             name = '<init>'
             type.return_type = self.lookup(self.name).type
         prev_entry = self.lookup_here(name)
-        entry = self.declare_var(name, type, pos, 
+        entry = self.declare_var(name, type, pos,
                                  cname=cname, visibility=visibility)
         if prev_entry:
             entry.overloaded_alternatives = prev_entry.all_alternatives()

Cython/Compiler/Tests/TestTreeFragment.py

@@ -54,11 +54,10 @@ class TestTreeFragments(CythonTest):
             x = TMP
         """)
         T = F.substitute(temps=[u"TMP"])
-        s = T.stats
-        self.assert_(s[0].expr.name == "__tmpvar_1")
-#        self.assert_(isinstance(s[0].expr, TempRefNode))
-#        self.assert_(isinstance(s[1].rhs, TempRefNode))
-#        self.assert_(s[0].expr.handle is s[1].rhs.handle)
+        s = T.body.stats
+        self.assert_(isinstance(s[0].expr, TempRefNode))
+        self.assert_(isinstance(s[1].rhs, TempRefNode))
+        self.assert_(s[0].expr.handle is s[1].rhs.handle)
 
 if __name__ == "__main__":
     import unittest

Cython/Compiler/TreeFragment.py

@@ -121,16 +121,15 @@ class TemplateTransform(VisitorTransform):
         temphandles = []
         for temp in temps:
             TemplateTransform.temp_name_counter += 1
-            handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter
-#            handle = UtilNodes.TempHandle(PyrexTypes.py_object_type)
+            handle = UtilNodes.TempHandle(PyrexTypes.py_object_type)
             tempmap[temp] = handle
-#            temphandles.append(handle)
+            temphandles.append(handle)
         self.tempmap = tempmap
         result = super(TemplateTransform, self).__call__(node)
-#        if temps:
-#            result = UtilNodes.TempsBlockNode(self.get_pos(node),
-#                                              temps=temphandles,
-#                                              body=result)
+        if temps:
+            result = UtilNodes.TempsBlockNode(self.get_pos(node),
+                                              temps=temphandles,
+                                              body=result)
         return result
 
     def get_pos(self, node):
@@ -161,9 +160,8 @@ class TemplateTransform(VisitorTransform):
     def visit_NameNode(self, node):
         temphandle = self.tempmap.get(node.name)
         if temphandle:
-            return NameNode(pos=node.pos, name=temphandle)
             # Replace name with temporary
-            #return temphandle.ref(self.get_pos(node))
+            return temphandle.ref(self.get_pos(node))
         else:
             return self.try_substitution(node, node.name)
 

Cython/Compiler/UtilNodes.py

@@ -134,6 +134,10 @@ class ResultRefNode(AtomicExprNode):
             self.type = type
         assert self.pos is not None
 
+    def clone_node(self):
+        # nothing to do here
+        return self
+
     def analyse_types(self, env):
         if self.expression is not None:
             self.type = self.expression.type

Cython/Includes/libcpp/string.pxd

+
+cdef extern from "<string>" namespace "std":
+
+    size_t npos = -1
+
+    cdef cppclass string:
+        string()
+        string(char *)
+        string(char *, size_t)
+        string(string&)
+        # as a string formed by a repetition of character c, n times.
+        string(size_t, char)
+
+        char* c_str()
+        size_t size()
+        size_t max_size()
+        size_t length()
+        void resize(size_t)
+        void resize(size_t, char c)
+        size_t capacity()
+        void reserve(size_t)
+        void clear()
+        bint empty()
+
+        char at(size_t)
+        char operator[](size_t)
+        int compare(string&)
+
+        string& append(string&)
+        string& append(string&, size_t, size_t)
+        string& append(char *)
+        string& append(char *, size_t)
+        string& append(size_t, char)
+
+        void push_back(char c)
+
+        string& assign (string&)
+        string& assign (string&, size_t, size_t)
+        string& assign (char *, size_t)
+        string& assign (char *)
+        string& assign (size_t n, char c)
+
+        string& insert(size_t, string&)
+        string& insert(size_t, string&, size_t, size_t)
+        string& insert(size_t, char* s, size_t)
+
+
+        string& insert(size_t, char* s)
+        string& insert(size_t, size_t, char c)
+
+        size_t copy(char *, size_t, size_t)
+
+        size_t find(string&)
+        size_t find(string&, size_t)
+        size_t find(char*, size_t pos, size_t)
+        size_t find(char*, size_t pos)
+        size_t find(char, size_t pos)
+
+        size_t rfind(string&, size_t)
+        size_t rfind(char* s, size_t, size_t)
+        size_t rfind(char*, size_t pos)
+        size_t rfind(char c, size_t)
+        size_t rfind(char c)
+
+        size_t find_first_of(string&, size_t)
+        size_t find_first_of(char* s, size_t, size_t)
+        size_t find_first_of(char*, size_t pos)
+        size_t find_first_of(char c, size_t)
+        size_t find_first_of(char c)
+
+        size_t find_first_not_of(string&, size_t)
+        size_t find_first_not_of(char* s, size_t, size_t)
+        size_t find_first_not_of(char*, size_t pos)
+        size_t find_first_not_of(char c, size_t)
+        size_t find_first_not_of(char c)
+
+        size_t find_last_of(string&, size_t)
+        size_t find_last_of(char* s, size_t, size_t)
+        size_t find_last_of(char*, size_t pos)
+        size_t find_last_of(char c, size_t)
+        size_t find_last_of(char c)
+
+        size_t find_last_not_of(string&, size_t)
+        size_t find_last_not_of(char* s, size_t, size_t)
+        size_t find_last_not_of(char*, size_t pos)
+
+        string substr(size_t, size_t)
+        string substr()
+        string substr(size_t)
+
+        size_t find_last_not_of(char c, size_t)
+        size_t find_last_not_of(char c)
+
+        #string& operator= (string&)
+        #string& operator= (char*)
+        #string& operator= (char)
+
+        bint operator==(string&)
+        bint operator==(char*)
+
+        bint operator!= (string& rhs )
+        bint operator!= (char* )
+
+        bint operator< (string&)
+        bint operator< (char*)
+
+        bint operator> (string&)
+        bint operator> (char*)
+
+        bint operator<= (string&)
+        bint operator<= (char*)
+
+        bint operator>= (string&)
+        bint operator>= (char*)

Cython/Plex/Errors.py

@@ -39,7 +39,7 @@ class UnrecognizedInput(PlexError):
 
   def __init__(self, scanner, state_name):
     self.scanner = scanner
-    self.position = scanner.position()
+    self.position = scanner.get_position()
     self.state_name = state_name
 
   def __str__(self):

Cython/Plex/Scanners.py

@@ -299,6 +299,11 @@ class Scanner(object):
     """
     return (self.name, self.start_line, self.start_col)
 
+  def get_position(self):
+    """Python accessible wrapper around position(), only for error reporting.
+    """
+    return self.position()
+
   def begin(self, state_name):
     """Set the current state of the scanner to the named state."""
     self.initial_state = (

pyximport/pyximport.py

@@ -261,7 +261,8 @@ class PyImporter(PyxImporter):
         self.super = super(PyImporter, self)
         self.super.__init__(extension='.py', pyxbuild_dir=pyxbuild_dir)
         self.uncompilable_modules = {}
-        self.blocked_modules = ['Cython', 'distutils.extension']
+        self.blocked_modules = ['Cython', 'distutils.extension',
+                                'distutils.sysconfig']
 
     def find_module(self, fullname, package_path=None):
         if fullname in sys.modules:

setup.py

@@ -249,7 +249,11 @@ except ValueError:
 
 try:
     sys.argv.remove("--no-cython-compile")
+    compile_cython_itself = False
 except ValueError:
+    compile_cython_itself = True
+
+if compile_cython_itself:
     compile_cython_modules(cython_profile, cython_compile_more, cython_with_refnanny)
 
 setup_args.update(setuptools_extra_args)

tests/bugs.txt

@@ -10,7 +10,6 @@ cfunc_call_tuple_args_T408
 compile.cpp_operators
 cpp_templated_ctypedef
 cpp_structs
-with_statement_module_level_T536
 function_as_method_T494
 closure_inside_cdef_T554
 pure_mode_cmethod_inheritance_T583
@@ -22,12 +21,20 @@ class_scope_T671
 
 # CPython regression tests that don't current work:
 pyregr.test_threadsignals
-pyregr.test_module
 pyregr.test_capi
 pyregr.test_socket
 pyregr.test_threading
 pyregr.test_sys
+pyregr.test_pep3131
 
 # CPython regression tests that don't make sense
 pyregr.test_gdb
 pyregr.test_support
+
+# Inlined generators
+all
+any
+builtin_sorted
+dictcomp
+inlined_generator_expressions
+setcomp

tests/compile/future_imports.pyx

+# mode: compile
+
+from __future__ import nested_scopes
+
+from __future__ import with_statement
+
+pass
+
+from __future__ import nested_scopes ; from __future__ import nested_scopes

tests/errors/w_cdef_override.pyx

 # mode: error
-# tags: werror
+# tag: werror
 
 cdef foo():
     pass

tests/run/arithmetic_analyse_types.pyx

@@ -18,40 +18,40 @@ def short_binop(short val):
     """
     Arithmetic in C is always done with at least int precision.
     
-    >>> short_binop(3)
-    'int called'
+    >>> print(short_binop(3))
+    int called
     """
     assert typeof(val + val) == "int", typeof(val + val)
     assert typeof(val - val) == "int", typeof(val - val)
     assert typeof(val & val) == "int", typeof(val & val)
     cdef int_return x = f(val + val)
-    return x.msg
+    return x.msg.decode('ASCII')
 
 def short_unnop(short val):
     """
     Arithmetic in C is always done with at least int precision.
     
-    >>> short_unnop(3)
-    'int called'
+    >>> print(short_unnop(3))
+    int called
     """
     cdef int_return x = f(-val)
-    return x.msg
+    return x.msg.decode('ASCII')
 
 def longlong_binop(long long val):
     """
-    >>> longlong_binop(3)
-    'long long called'
+    >>> print(longlong_binop(3))
+    long long called
     """
     cdef longlong_return x = f(val * val)
-    return x.msg
+    return x.msg.decode('ASCII')
 
 def longlong_unnop(long long val):
     """
-    >>> longlong_unnop(3)
-    'long long called'
+    >>> print(longlong_unnop(3))
+    long long called
     """
     cdef longlong_return x = f(~val)
-    return x.msg
+    return x.msg.decode('ASCII')
 
 
 def test_bint(bint a):

tests/run/arithmetic_analyse_types_helper.h

 /* A set of mutually incompatable return types. */
 
-struct short_return { const char *msg; };
-struct int_return { const char *msg; };
-struct longlong_return { const char *msg; };
+struct short_return { char *msg; };
+struct int_return { char *msg; };
+struct longlong_return { char *msg; };
 
 /* A set of overloaded methods. */
 
 short_return f(short arg) {
     short_return val;
-    val.msg = "short called";
+    val.msg = (char*)"short called";
     return val;
 }
 
 int_return f(int arg) {
     int_return val;
-    val.msg = "int called";
+    val.msg = (char*)"int called";
     return val;
 }
 
 longlong_return f(long long arg) {
     longlong_return val;
-    val.msg = "long long called";
+    val.msg = (char*)"long long called";
     return val;
 }
  

tests/run/cpp_stl_string.pyx

+# tag: cpp
+
+from libcpp.string cimport string
+
+b_asdf = b'asdf'
+b_asdg = b'asdg'
+b_s = b's'
+
+def test_indexing(char *py_str):
+    """
+    >>> test_indexing(b_asdf)
+    ('s', 's')
+    """
+    cdef string s
+    s = string(py_str)
+    return chr(s[1]), chr(s.at(1))
+
+def test_size(char *py_str):
+    """
+    >>> test_size(b_asdf)
+    (4, 4)
+    """
+    cdef string s
+    s = string(py_str)
+    return s.size(), s.length()
+
+def test_compare(char *a, char *b):
+    """
+    >>> test_compare(b_asdf, b_asdf)
+    0
+
+    >>> test_compare(b_asdf, b_asdg) < 0
+    True
+    """
+    cdef string s = string(a)
+    cdef string t = string(b)
+    return s.compare(t)
+
+def test_empty():
+    """
+    >>> test_empty()
+    (True, False)
+    """
+    cdef string a = string(<char *>b"")
+    cdef string b = string(<char *>b"aa")
+    return a.empty(), b.empty()
+
+def test_push_back(char *a):
+    """
+    >>> test_push_back(b_asdf) == b_asdf + b_s
+    True
+    """
+    cdef string s = string(a)
+    s.push_back(<char>ord('s'))
+    return s.c_str()
+
+def test_insert(char *a, char *b, int i):
+    """
+    >>> test_insert('AAAA'.encode('ASCII'), 'BBBB'.encode('ASCII'), 2) == 'AABBBBAA'.encode('ASCII')
+    True
+    """
+    cdef string s = string(a)
+    cdef string t = string(b)
+    cdef string u = s.insert(i, t)
+    return u.c_str()
+
+def test_copy(char *a):
+    """
+    >>> test_copy(b_asdf) == b_asdf[1:]
+    True
+    """
+    cdef string t = string(a)
+    cdef char buffer[6]
+    cdef size_t length = t.copy(buffer, 4, 1)
+    buffer[length] = c'\0'
+    return buffer
+
+def test_find(char *a, char *b):
+    """
+    >>> test_find(b_asdf, 'df'.encode('ASCII'))
+    2
+    """
+    cdef string s = string(a)
+    cdef string t = string(b)
+    cdef size_t i = s.find(t)
+    return i
+
+def test_clear():
+    """
+    >>> test_clear() == ''.encode('ASCII')
+    True
+    """
+    cdef string s = string(<char *>"asdf")
+    s.clear()
+    return s.c_str()
+
+def test_assign(char *a):
+    """
+    >>> test_assign(b_asdf) == 'ggg'.encode('ASCII')
+    True
+    """
+    cdef string s = string(a)
+    s.assign(<char *>"ggg")
+    return s.c_str()
+
+
+def test_substr(char *a):
+    """
+    >>> test_substr('ABCDEFGH'.encode('ASCII')) == ('BCDEFGH'.encode('ASCII'), 'BCDE'.encode('ASCII'), 'ABCDEFGH'.encode('ASCII'))
+    True
+    """
+    cdef string s = string(a)
+    cdef string x, y, z
+    x = s.substr(1)
+    y = s.substr(1, 4)
+    z = s.substr()
+    return x.c_str(), y.c_str(), z.c_str()
+
+def test_append(char *a, char *b):
+    """
+    >>> test_append(b_asdf, '1234'.encode('ASCII')) == b_asdf + '1234'.encode('ASCII')
+    True
+    """
+    cdef string s = string(a)
+    cdef string t = string(b)
+    cdef string j = s.append(t)
+    return j.c_str()
+
+def test_char_compare(py_str):
+    """
+    >>> test_char_compare(b_asdf)
+    True
+    """
+    cdef char *a = py_str
+    cdef string b = string(a)
+    return b.compare(b) == 0
+
+def test_cstr(char *a):
+    """
+    >>> test_cstr(b_asdf) == b_asdf
+    True
+    """
+    cdef string b = string(a)
+    return b.c_str()
+
+def test_equals_operator(char *a, char *b):
+    """
+    >>> test_equals_operator(b_asdf, b_asdf)
+    (True, False)
+    """
+    cdef string s = string(a)
+    cdef string t = string(b)
+    return t == s, t != <char *>"asdf"
+
+def test_less_than(char *a, char *b):
+    """
+    >>> test_less_than(b_asdf[:-1], b_asdf)
+    (True, True, True)
+
+    >>> test_less_than(b_asdf[:-1], b_asdf[:-1])
+    (False, False, True)
+    """
+    cdef string s = string(a)
+    cdef string t = string(b)
+    return (s < t, s < b, s <= b)
+
+def test_greater_than(char *a, char *b):
+    """
+    >>> test_greater_than(b_asdf[:-1], b_asdf)
+    (False, False, False)
+
+    >>> test_greater_than(b_asdf[:-1], b_asdf[:-1])
+    (False, False, True)
+    """
+    cdef string s = string(a)
+    cdef string t = string(b)
+    return (s > t, s > b, s >= b)

tests/run/cython3.pyx

 # cython: language_level=3
+# mode: run
+# tag: generators, python3
 
 cimport cython
 
@@ -89,6 +91,16 @@ def list_comp():
     assert x == 'abc' # don't leak in Py3 code
     return result
 
+def list_comp_with_lambda():
+    """
+    >>> list_comp_with_lambda()
+    [0, 4, 8]
+    """
+    x = 'abc'
+    result = [x*2 for x in range(5) if (lambda x:x % 2)(x) == 0]
+    assert x == 'abc' # don't leak in Py3 code
+    return result
+
 module_level_lc = [ module_level_loopvar*2 for module_level_loopvar in range(4) ]
 def list_comp_module_level():
     """

tests/run/flatin.pyx

@@ -13,7 +13,7 @@ def test_in(s):
     >>> test_in('')
     5
     """
-    if s in (u'ABC', u'BCD'):
+    if s in (u'ABC', u'BCD', u'ABC'[:3], u'ABC'[::-1], u'ABC'[-1]):
         return 1
     elif s.upper() in (u'ABC', u'BCD'):
         return 2

tests/run/funcexceptraisefrom.pyx

+__doc__ = u"""
+>>> def bar():
+...     try:
+...         foo()
+...     except ValueError:
+...         if IS_PY3:
+...             print(isinstance(sys.exc_info()[1].__cause__, TypeError))
+...         else:
+...             print(True)
+
+>>> bar()
+True
+
+>>> print(sys.exc_info())
+(None, None, None)
+
+>>> def bar2():
+...     try:
+...         foo2()
+...     except ValueError:
+...         if IS_PY3:
+...             cause = sys.exc_info()[1].__cause__
+...             print(isinstance(cause, TypeError))
+...             print(cause.args==('value',))
+...             pass
+...         else:
+...             print(True)
+...             print(True)
+
+>>> bar2()
+True
+True
+"""
+
+import sys
+IS_PY3 = sys.version_info[0] >= 3
+if not IS_PY3:
+    sys.exc_clear()
+
+def foo():
+    try:
+        raise TypeError
+    except TypeError:
+        raise ValueError from TypeError
+
+def foo2():
+    try:
+        raise TypeError
+    except TypeError:
+        raise ValueError() from TypeError('value')
+

tests/run/generator_expressions.pyx

+# mode: run
+# tag: generators, lambda
+
+def genexpr():
+    """
+    >>> genexpr()
+    [0, 2, 4, 6, 8]
+    """
+    x = 'abc'
+    result = list( x*2 for x in range(5) )
+    assert x == 'abc' # don't leak
+    return result
+
+def genexpr_if():
+    """
+    >>> genexpr_if()
+    [0, 4, 8]
+    """
+    x = 'abc'
+    result = list( x*2 for x in range(5) if x % 2 == 0 )
+    assert x == 'abc' # don't leak
+    return result
+
+def genexpr_with_lambda():
+    """
+    >>> genexpr_with_lambda()
+    [0, 4, 8]
+    """
+    x = 'abc'
+    result = list( x*2 for x in range(5) if (lambda x:x % 2)(x) == 0 )
+    assert x == 'abc' # don't leak
+    return result
+
+def genexpr_of_lambdas(int N):
+    """
+    >>> [ (f(), g()) for f,g in genexpr_of_lambdas(5) ]
+    [(0, 0), (1, 2), (2, 4), (3, 6), (4, 8)]
+    """
+    return ( ((lambda : x), (lambda : x*2)) for x in range(N) )

tests/run/generators_py.py

@@ -167,6 +167,40 @@ def check_yield_in_except():
     except ValueError:
         yield
 
+def yield_in_except_throw_exc_type():
+    """
+    >>> import sys
+    >>> g = yield_in_except_throw_exc_type()
+    >>> next(g)
+    >>> g.throw(TypeError)
+    Traceback (most recent call last):
+    TypeError
+    >>> next(g)
+    Traceback (most recent call last):
+    StopIteration
+    """
+    try:
+        raise ValueError
+    except ValueError:
+        yield
+
+def yield_in_except_throw_instance():
+    """
+    >>> import sys
+    >>> g = yield_in_except_throw_instance()
+    >>> next(g)
+    >>> g.throw(TypeError())
+    Traceback (most recent call last):
+    TypeError
+    >>> next(g)
+    Traceback (most recent call last):
+    StopIteration
+    """
+    try:
+        raise ValueError
+    except ValueError:
+        yield
+
 def test_swap_assignment():
     """
     >>> gen = test_swap_assignment()

tests/run/ifelseexpr_T267.pyx

+# mode: run
+# tag: condexpr
 # ticket: 267
 
-"""
->>> constants(4)
-1
->>> constants(5)
-10
->>> temps(4)
-1
->>> temps(5)
-10
->>> nested(1)
-1
->>> nested(2)
-2
->>> nested(3)
-3
-"""
+cimport cython
 
 def ident(x): return x
 
 def constants(x):
+    """
+    >>> constants(4)
+    1
+    >>> constants(5)
+    10
+    """
     a = 1 if x < 5 else 10
     return a
 
 def temps(x):
+    """
+    >>> temps(4)
+    1
+    >>> temps(5)
+    10
+    """
     return ident(1) if ident(x) < ident(5) else ident(10)
 
 def nested(x):
+    """
+    >>> nested(1)
+    1
+    >>> nested(2)
+    2
+    >>> nested(3)
+    3
+    """
     return 1 if x == 1 else (2 if x == 2 else 3)
+
+@cython.test_fail_if_path_exists('//CondExprNode')
+def const_true(a,b):
+    """
+    >>> const_true(1,2)
+    1
+    """
+    return a if 1 == 1 else b
+
+@cython.test_fail_if_path_exists('//CondExprNode')
+def const_false(a,b):
+    """
+    >>> const_false(1,2)
+    2
+    """
+    return a if 1 != 1 else b

tests/run/relativeimport_T542.pyx

@@ -20,10 +20,9 @@ def test_relative():
 
 def test_absolute():
     """
-    >>> test_absolute()
+    >>> test_absolute()   # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    ...
-    ImportError: No module named debug
+    ImportError: No module named ...debug...
     """
     import debug
     return

tests/run/strliterals.pyx

@@ -132,6 +132,18 @@ __doc__ = ur"""
     >>> len(bytes_uescape)
     28
 
+    >>> (sys.version_info[0] >= 3 and sys.maxunicode == 1114111 and len(str_uescape) == 3 or
+    ...  sys.version_info[0] >= 3 and sys.maxunicode == 65535   and len(str_uescape) == 4 or
+    ...  sys.version_info[0] <  3 and len(str_uescape) == 17 or
+    ...  len(str_uescape))
+    True
+    >>> (sys.version_info[0] >= 3 and str_uescape[0] == 'c' or
+    ...  sys.version_info[0] <  3 and str_uescape[0] == '\\' or
+    ...  str_uescape[0])
+    True
+    >>> print(str_uescape[-1])
+    B
+
     >>> newlines == "Aaa\n"
     True
     
@@ -173,6 +185,7 @@ bresc = br'\12\'\"\\'
 uresc = ur'\12\'\"\\'
 
 bytes_uescape = b'\u1234\U12345678\u\u1\u12\uX'
+str_uescape = '\u0063\U00012345\x42'
 
 newlines = "Aaa\n"
 

tests/run/test_raisefrom.pyx

+import unittest
+# adapted from pyregr
+class TestCause(unittest.TestCase):
+    def test_invalid_cause(self):
+        try:
+            raise IndexError from 5
+        except TypeError as e:
+            self.assertTrue("exception cause" in str(e))
+        else:
+            self.fail("No exception raised")
+
+    def test_class_cause(self):
+        try:
+            raise IndexError from KeyError
+        except IndexError as e:
+            self.assertTrue(isinstance(e.__cause__, KeyError))
+        else:
+            self.fail("No exception raised")
+
+    def test_instance_cause(self):
+        cause = KeyError()
+        try:
+            raise IndexError from cause
+        except IndexError as e:
+            self.assertTrue(e.__cause__ is cause)
+        else:
+            self.fail("No exception raised")
+
+    def test_erroneous_cause(self):
+        class MyException(Exception):
+            def __init__(self):
+                raise RuntimeError()
+
+        try:
+            raise IndexError from MyException
+        except RuntimeError:
+            pass
+        else:
+            self.fail("No exception raised")

tests/run/trybreak.pyx

-__doc__ = u"""
->>> print(foo())
-a
-"""
-
 # Indirectly makes sure the cleanup happens correctly on breaking.
-def foo():
-    for x in "abc":
+
+def try_except_break():
+    """
+    >>> print(try_except_break())
+    a
+    """
+    for x in list("abc"):
         try:
             x()
         except:
             break
-    for x in "abc":
+    return x
+
+def try_break_except():
+    """
+    >>> print(try_break_except())
+    a
+    """
+    for x in list("abc"):
+        try:
+            break
+        except:
+            pass
+    return x
+
+def try_no_break_except_return():
+    """
+    >>> print(try_no_break_except_return())
+    a
+    """
+    for x in list("abc"):
         try:
             x()
+            break
         except:
             return x
+    return x

tests/run/with_statement_module_level_T536.pyx

@@ -3,17 +3,32 @@
 __doc__ = """
 >>> inner_result
 ['ENTER']
->>> result
+>>> result  # doctest: +ELLIPSIS
+['ENTER', ...EXIT (<...ValueError...>,...ValueError..., <traceback object at ...)...]
+
+>>> inner_result_no_exc
+['ENTER']
+>>> result_no_exc
 ['ENTER', 'EXIT (None, None, None)']
 """
 
-result = []
-
 class ContextManager(object):
+    def __init__(self, result):
+        self.result = result
     def __enter__(self):
-        result.append("ENTER")
+        self.result.append("ENTER")
     def __exit__(self, *values):
-        result.append("EXIT %r" % (values,))
+        self.result.append("EXIT %r" % (values,))
+        return True
+
+result_no_exc = []
+
+with ContextManager(result_no_exc) as c:
+    inner_result_no_exc = result_no_exc[:]
+
+result = []
 
-with ContextManager() as c:
+with ContextManager(result) as c:
     inner_result = result[:]
+    raise ValueError('TEST')
+    

tests/run/withstat.pyx

@@ -56,17 +56,6 @@ def with_pass():
     with ContextManager(u"value") as x:
         pass
 
-def with_return():
-    """
-    >>> with_return()
-    enter
-    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
-    """
-    with ContextManager(u"value") as x:
-        # FIXME: DISABLED - currently crashes!!
-        # return x
-        pass
-
 def with_exception(exit_ret):
     """
     >>> with_exception(None)

tests/run/withstat_py.py

+import sys
+
+def typename(t):
+    name = type(t).__name__
+    if sys.version_info < (2,5):
+        if name == 'classobj' and issubclass(t, MyException):
+            name = 'type'
+        elif name == 'instance' and isinstance(t, MyException):
+            name = 'MyException'
+    return "<type '%s'>" % name
+
+class MyException(Exception):
+    pass
+
+class ContextManager(object):
+    def __init__(self, value, exit_ret = None):
+        self.value = value
+        self.exit_ret = exit_ret
+
+    def __exit__(self, a, b, tb):
+        print("exit %s %s %s" % (typename(a), typename(b), typename(tb)))
+        return self.exit_ret
+
+    def __enter__(self):
+        print("enter")
+        return self.value
+
+def no_as():
+    """
+    >>> no_as()
+    enter
+    hello
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    """
+    with ContextManager("value"):
+        print("hello")
+
+def basic():
+    """
+    >>> basic()
+    enter
+    value
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    """
+    with ContextManager("value") as x:
+        print(x)
+
+def with_pass():
+    """
+    >>> with_pass()
+    enter
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    """
+    with ContextManager("value") as x:
+        pass
+
+def with_return():
+    """
+    >>> print(with_return())
+    enter
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    value
+    """
+    with ContextManager("value") as x:
+        return x
+
+def with_break():
+    """
+    >>> print(with_break())
+    enter
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    a
+    """
+    for c in list("abc"):
+        with ContextManager("value") as x:
+            break
+        print("FAILED")
+    return c
+
+def with_continue():
+    """
+    >>> print(with_continue())
+    enter
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    enter
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    enter
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    c
+    """
+    for c in list("abc"):
+        with ContextManager("value") as x:
+            continue
+        print("FAILED")
+    return c
+
+def with_exception(exit_ret):
+    """
+    >>> with_exception(None)
+    enter
+    value
+    exit <type 'type'> <type 'MyException'> <type 'traceback'>
+    outer except
+    >>> with_exception(True)
+    enter
+    value
+    exit <type 'type'> <type 'MyException'> <type 'traceback'>
+    """
+    try:
+        with ContextManager("value", exit_ret=exit_ret) as value:
+            print(value)
+            raise MyException()
+    except:
+        print("outer except")
+
+def functions_in_with():
+    """
+    >>> f = functions_in_with()
+    enter
+    exit <type 'type'> <type 'MyException'> <type 'traceback'>
+    outer except
+    >>> f(1)[0]
+    1
+    >>> print(f(1)[1])
+    value
+    """
+    try:
+        with ContextManager("value") as value:
+            def f(x): return x, value
+            make = lambda x:x()
+            raise make(MyException)
+    except:
+        print("outer except")
+    return f
+
+def multitarget():
+    """
+    >>> multitarget()
+    enter
+    1 2 3 4 5
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    """
+    with ContextManager((1, 2, (3, (4, 5)))) as (a, b, (c, (d, e))):
+        print('%s %s %s %s %s' % (a, b, c, d, e))
+
+def tupletarget():
+    """
+    >>> tupletarget()
+    enter
+    (1, 2, (3, (4, 5)))
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    """
+    with ContextManager((1, 2, (3, (4, 5)))) as t:
+        print(t)
+
+def multimanager():
+    """
+    >>> multimanager()
+    enter
+    enter
+    enter
+    enter
+    enter
+    enter
+    2
+    value
+    1 2 3 4 5
+    nested
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
+    """
+    with ContextManager(1), ContextManager(2) as x, ContextManager('value') as y,\
+            ContextManager(3), ContextManager((1, 2, (3, (4, 5)))) as (a, b, (c, (d, e))):
+        with ContextManager('nested') as nested:
+            print(x)
+            print(y)
+            print('%s %s %s %s %s' % (a, b, c, d, e))
+            print(nested)
+
+# Tests borrowed from pyregr test_with.py,
+# modified to follow the constraints of Cython.
+import unittest
+
+class Dummy(object):
+    def __init__(self, value=None, gobble=False):
+        if value is None:
+            value = self
+        self.value = value
+        self.gobble = gobble
+        self.enter_called = False
+        self.exit_called = False
+
+    def __enter__(self):
+        self.enter_called = True
+        return self.value
+
+    def __exit__(self, *exc_info):
+        self.exit_called = True
+        self.exc_info = exc_info
+        if self.gobble:
+            return True
+
+class InitRaises(object):
+    def __init__(self): raise RuntimeError()
+
+class EnterRaises(object):
+    def __enter__(self): raise RuntimeError()
+    def __exit__(self, *exc_info): pass
+
+class ExitRaises(object):
+    def __enter__(self): pass
+    def __exit__(self, *exc_info): raise RuntimeError()
+
+class NestedWith(unittest.TestCase):
+    """
+    >>> NestedWith().runTest()
+    """
+
+    def runTest(self):
+        self.testNoExceptions()
+        self.testExceptionInExprList()
+        self.testExceptionInEnter()
+        self.testExceptionInExit()
+        self.testEnterReturnsTuple()
+
+    def testNoExceptions(self):
+        with Dummy() as a, Dummy() as b:
+            self.assertTrue(a.enter_called)
+            self.assertTrue(b.enter_called)
+        self.assertTrue(a.exit_called)
+        self.assertTrue(b.exit_called)
+
+    def testExceptionInExprList(self):
+        try:
+            with Dummy() as a, InitRaises():
+                pass
+        except:
+            pass
+        self.assertTrue(a.enter_called)
+        self.assertTrue(a.exit_called)
+
+    def testExceptionInEnter(self):
+        try:
+            with Dummy() as a, EnterRaises():
+                self.fail('body of bad with executed')
+        except RuntimeError:
+            pass
+        else:
+            self.fail('RuntimeError not reraised')
+        self.assertTrue(a.enter_called)
+        self.assertTrue(a.exit_called)
+
+    def testExceptionInExit(self):
+        body_executed = False
+        with Dummy(gobble=True) as a, ExitRaises():
+            body_executed = True
+        self.assertTrue(a.enter_called)
+        self.assertTrue(a.exit_called)
+        self.assertTrue(body_executed)
+        self.assertNotEqual(a.exc_info[0], None)
+
+    def testEnterReturnsTuple(self):
+        with Dummy(value=(1,2)) as (a1, a2), \
+             Dummy(value=(10, 20)) as (b1, b2):
+            self.assertEquals(1, a1)
+            self.assertEquals(2, a2)
+            self.assertEquals(10, b1)
+            self.assertEquals(20, b2)