merge

Cython/Compiler/AutoDocTransforms.py

@@ -3,9 +3,7 @@ from Cython.Compiler.Nodes import DefNode, CFuncDefNode
 from Cython.Compiler.Errors import CompileError
 from Cython.Compiler.StringEncoding import EncodedString
 from Cython.Compiler import Options
-from Cython.Compiler import PyrexTypes
-
-
+from Cython.Compiler import PyrexTypes, ExprNodes
 
 class EmbedSignature(CythonTransform):
 
@@ -16,15 +14,26 @@ class EmbedSignature(CythonTransform):
         self.class_node = None
 
     def _fmt_arg_defv(self, arg):
-        if not arg.default:
+        default_val = arg.default
+        if not default_val:
             return None
         try:
             denv = self.denv  # XXX
-            ctval = arg.default.compile_time_value(self.denv)
-            return '%r' % ctval
+            ctval = default_val.compile_time_value(self.denv)
+            repr_val = '%r' % ctval
+            if isinstance(default_val, ExprNodes.UnicodeNode):
+                if repr_val[:1] != 'u':
+                    return u'u%s' % repr_val
+            elif isinstance(default_val, ExprNodes.BytesNode):
+                if repr_val[:1] != 'b':
+                    return u'b%s' % repr_val
+            elif isinstance(default_val, ExprNodes.StringNode):
+                if repr_val[:1] in ('u', 'b'):
+                    repr_val[1:]
+            return repr_val
         except Exception:
             try:
-                return arg.default.name # XXX
+                return default_val.name # XXX
             except AttributeError:
                 return '<???>'
 

Cython/Compiler/Code.py

@@ -747,8 +747,9 @@ class GlobalState(object):
                     u'*/', u'*[inserted by cython to avoid comment closer]/'
                     ).replace(
                     u'/*', u'/[inserted by cython to avoid comment start]*'
-                    ).encode('ASCII', 'replace').decode('ASCII')
-                 for line in source_desc.get_lines()]
+                    )
+                 for line in source_desc.get_lines(encoding='ASCII',
+                                                   error_handling='ignore')]
             if len(F) == 0: F.append(u'')
             self.input_file_contents[source_desc] = F
             return F

Cython/Compiler/ExprNodes.py

@@ -13,7 +13,8 @@ import Nodes
 from Nodes import Node
 import PyrexTypes
 from PyrexTypes import py_object_type, c_long_type, typecast, error_type, unspecified_type
-from Builtin import list_type, tuple_type, set_type, dict_type, unicode_type, bytes_type, type_type
+from Builtin import list_type, tuple_type, set_type, dict_type, \
+     unicode_type, str_type, bytes_type, type_type
 import Builtin
 import Symtab
 import Options
@@ -821,6 +822,9 @@ class BytesNode(ConstNode):
         if isinstance(sizeof_node, SizeofTypeNode):
             return sizeof_node.arg_type
 
+    def can_coerce_to_char_literal(self):
+        return len(self.value) == 1
+
     def coerce_to(self, dst_type, env):
         if dst_type == PyrexTypes.c_char_ptr_type:
             self.type = PyrexTypes.c_char_ptr_type
@@ -830,7 +834,7 @@ class BytesNode(ConstNode):
             return CastNode(self, PyrexTypes.c_uchar_ptr_type)
 
         if dst_type.is_int:
-            if len(self.value) > 1:
+            if not self.can_coerce_to_char_literal():
                 error(self.pos, "Only single-character strings can be coerced into ints.")
                 return self
             return CharNode(self.pos, value=self.value)
@@ -905,11 +909,11 @@ class StringNode(PyConstNode):
     # value          BytesLiteral or EncodedString
     # is_identifier  boolean
 
-    type = Builtin.str_type
+    type = str_type
     is_identifier = False
 
     def coerce_to(self, dst_type, env):
-        if dst_type is not py_object_type and dst_type is not Builtin.str_type:
+        if dst_type is not py_object_type and dst_type is not str_type:
 #            if dst_type is Builtin.bytes_type:
 #                # special case: bytes = 'str literal'
 #                return BytesNode(self.pos, value=self.value)
@@ -927,6 +931,9 @@ class StringNode(PyConstNode):
 
         return self
 
+    def can_coerce_to_char_literal(self):
+        return not self.is_identifier and len(self.value) == 1
+
     def generate_evaluation_code(self, code):
         self.result_code = code.get_py_string_const(
             self.value, identifier=self.is_identifier, is_str=True)
@@ -2668,7 +2675,7 @@ class GeneralCallNode(CallNode):
                 self.type = error_type
                 return error_type
             if hasattr(self.function, 'entry') and not self.function.entry.as_variable:
-                error(self.pos, "Keyword arguments not allowed in cdef functions.")
+                error(self.pos, "Keyword and starred arguments not allowed in cdef functions.")
             else:
                 self.function = self.function.coerce_to_pyobject(env)
         self.positional_args = \
@@ -5065,6 +5072,80 @@ class CmpNode(object):
                 result = result and cascade.compile_time_value(operand2, denv)
         return result
 
+    def find_common_int_type(self, env, op, operand1, operand2):
+        # type1 != type2 and at least one of the types is not a C int
+        type1 = operand1.type
+        type2 = operand2.type
+        type1_can_be_int = False
+        type2_can_be_int = False
+
+        if isinstance(operand1, (StringNode, BytesNode)) \
+               and operand1.can_coerce_to_char_literal():
+            type1_can_be_int = True
+        if isinstance(operand2, (StringNode, BytesNode)) \
+                 and operand2.can_coerce_to_char_literal():
+            type2_can_be_int = True
+
+        if type1.is_int:
+            if type2_can_be_int:
+                return type1
+        elif type2.is_int:
+            if type1_can_be_int:
+                return type2
+        elif type1_can_be_int:
+            if type2_can_be_int:
+                return PyrexTypes.c_uchar_type
+
+        return None
+
+    def find_common_type(self, env, op, operand1, common_type=None):
+        operand2 = self.operand2
+        type1 = operand1.type
+        type2 = operand2.type
+
+        new_common_type = None
+
+        if type1 == str_type and (type2.is_string or type2 in (bytes_type, unicode_type)) or \
+               type2 == str_type and (type1.is_string or type1 in (bytes_type, unicode_type)):
+            error(self.pos, "Comparisons between bytes/unicode and str are not portable to Python 3")
+
+        elif operand1.type.is_complex or operand2.type.is_complex:
+            if op not in ('==', '!='):
+                error(self.pos, "complex types unordered")
+            if operand1.type.is_pyobject:
+                new_common_type = operand1.type
+            elif operand2.type.is_pyobject:
+                new_common_type = operand2.type
+            else:
+                new_common_type = PyrexTypes.widest_numeric_type(type1, type2)
+
+        elif common_type is None or not common_type.is_pyobject:
+            if not type1.is_int or not type2.is_int:
+                new_common_type = self.find_common_int_type(env, op, operand1, operand2)
+
+        if new_common_type is None:
+            new_common_type = PyrexTypes.spanning_type(operand1.type, operand2.type)
+
+        if common_type is None:
+            common_type = new_common_type
+        else:
+            # we could do a lot better by splitting the comparison
+            # into a non-Python part and a Python part, but this is
+            # safer for now
+            common_type = PyrexTypes.spanning_type(common_type, new_common_type)
+
+        if self.cascade:
+            common_type = self.cascade.find_common_type(env, self.operator, operand2, common_type)
+
+        return common_type
+
+    def coerce_operands_to(self, dst_type, env):
+        operand2 = self.operand2
+        if operand2.type != dst_type:
+            self.operand2 = operand2.coerce_to(dst_type, env)
+        if self.cascade:
+            self.cascade.coerce_operands_to(dst_type, env)
+
     def is_python_comparison(self):
         return (self.has_python_operands()
             or (self.cascade and self.cascade.is_python_comparison())
@@ -5075,13 +5156,7 @@ class CmpNode(object):
             or (self.cascade and self.cascade.is_python_result()))
 
     def check_types(self, env, operand1, op, operand2):
-        if operand1.type.is_complex or operand2.type.is_complex:
-            if op not in ('==', '!='):
-                error(self.pos, "complex types unordered")
-            common_type = PyrexTypes.widest_numeric_type(operand1.type, operand2.type)
-            self.operand1 = operand1.coerce_to(common_type, env)
-            self.operand2 = operand2.coerce_to(common_type, env)
-        elif not self.types_okay(operand1, op, operand2):
+        if not self.types_okay(operand1, op, operand2):
             error(self.pos, "Invalid types for '%s' (%s, %s)" %
                 (self.operator, operand1.type, operand2.type))
     
@@ -5224,12 +5299,14 @@ class PrimaryCmpNode(ExprNode, CmpNode):
         self.operand1.analyse_types(env)
         self.operand2.analyse_types(env)
         if self.cascade:
-            self.cascade.analyse_types(env, self.operand2)
-        self.is_pycmp = self.is_python_comparison()
-        if self.is_pycmp:
-            self.coerce_operands_to_pyobjects(env)
-        if self.has_int_operands():
-            self.coerce_chars_to_ints(env)
+            self.cascade.analyse_types(env)
+
+        common_type = self.find_common_type(env, self.operator, self.operand1)
+        self.is_pycmp = common_type.is_pyobject
+        if self.operand1.type != common_type:
+            self.operand1 = self.operand1.coerce_to(common_type, env)
+        self.coerce_operands_to(common_type, env)
+
         if self.cascade:
             self.operand2 = self.operand2.coerce_to_simple(env)
             self.cascade.coerce_cascaded_operands_to_temp(env)
@@ -5261,19 +5338,6 @@ class PrimaryCmpNode(ExprNode, CmpNode):
         if self.cascade:
             self.cascade.coerce_operands_to_pyobjects(env)
     
-    def has_int_operands(self):
-        return (self.operand1.type.is_int or self.operand2.type.is_int) \
-           or (self.cascade and self.cascade.has_int_operands())
-    
-    def coerce_chars_to_ints(self, env):
-        # coerce literal single-char strings to c chars
-        if self.operand1.type.is_string and isinstance(self.operand1, BytesNode):
-            self.operand1 = self.operand1.coerce_to(PyrexTypes.c_uchar_type, env)
-        if self.operand2.type.is_string and isinstance(self.operand2, BytesNode):
-            self.operand2 = self.operand2.coerce_to(PyrexTypes.c_uchar_type, env)
-        if self.cascade:
-            self.cascade.coerce_chars_to_ints(env)
-    
     def check_const(self):
         self.operand1.check_const()
         self.operand2.check_const()
@@ -5353,10 +5417,10 @@ class CascadedCmpNode(Node, CmpNode):
     def type_dependencies(self, env):
         return ()
 
-    def analyse_types(self, env, operand1):
+    def analyse_types(self, env):
         self.operand2.analyse_types(env)
         if self.cascade:
-            self.cascade.analyse_types(env, self.operand2)
+            self.cascade.analyse_types(env)
     
     def check_operand_types(self, env, operand1):
         self.check_types(env, 
@@ -5372,13 +5436,6 @@ class CascadedCmpNode(Node, CmpNode):
         if self.cascade:
             self.cascade.coerce_operands_to_pyobjects(env)
 
-    def has_int_operands(self):
-        return self.operand2.type.is_int
-        
-    def coerce_chars_to_ints(self, env):
-        if self.operand2.type.is_string and isinstance(self.operand2, BytesNode):
-            self.operand2 = self.operand2.coerce_to(PyrexTypes.c_uchar_type, env)
-
     def coerce_cascaded_operands_to_temp(self, env):
         if self.cascade:
             #self.operand2 = self.operand2.coerce_to_temp(env) #CTT

Cython/Compiler/Nodes.py

@@ -3459,7 +3459,6 @@ class RaiseStatNode(StatNode):
             self.exc_tb.analyse_types(env)
             self.exc_tb = self.exc_tb.coerce_to_pyobject(env)
         env.use_utility_code(raise_utility_code)
-        env.use_utility_code(restore_exception_utility_code)
 
     nogil_check = Node.gil_error
     gil_message = "Raising exception"
@@ -3506,7 +3505,6 @@ class ReraiseStatNode(StatNode):
     child_attrs = []
 
     def analyse_expressions(self, env):
-        env.use_utility_code(raise_utility_code)
         env.use_utility_code(restore_exception_utility_code)
 
     nogil_check = Node.gil_error
@@ -3515,7 +3513,12 @@ class ReraiseStatNode(StatNode):
     def generate_execution_code(self, code):
         vars = code.funcstate.exc_vars
         if vars:
-            code.putln("__Pyx_Raise(%s, %s, %s);" % tuple(vars))
+            for varname in vars:
+                code.put_giveref(varname)
+            code.putln("__Pyx_ErrRestore(%s, %s, %s);" % tuple(vars))
+            for varname in vars:
+                code.put("%s = 0; " % varname)
+            code.putln()
             code.putln(code.error_goto(self.pos))
         else:
             error(self.pos, "Reraise not inside except clause")
@@ -4125,8 +4128,9 @@ class TryExceptStatNode(StatNode):
         code.put_goto(try_end_label)
         if code.label_used(try_return_label):
             code.put_label(try_return_label)
-            for var in Naming.exc_save_vars:
-                code.put_xdecref_clear(var, py_object_type)
+            for var in Naming.exc_save_vars: code.put_xgiveref(var)
+            code.putln("__Pyx_ExceptionReset(%s);" %
+                       ', '.join(Naming.exc_save_vars))
             code.put_goto(old_return_label)
         code.put_label(our_error_label)
         for temp_name, type in temps_to_clean_up:
@@ -4920,13 +4924,57 @@ requires=[printing_utility_code])
 
 #------------------------------------------------------------------------------------
 
-# The following function is based on do_raise() from ceval.c.
+# Exception raising code
+#
+# Exceptions are raised by __Pyx_Raise() and stored as plain
+# type/value/tb in PyThreadState->curexc_*.  When being caught by an
+# 'except' statement, curexc_* is moved over to exc_* by
+# __Pyx_GetException()
+
+restore_exception_utility_code = UtilityCode(
+proto = """
+static INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb); /*proto*/
+static INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb); /*proto*/
+""",
+impl = """
+static INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb) {
+    PyObject *tmp_type, *tmp_value, *tmp_tb;
+    PyThreadState *tstate = PyThreadState_GET();
+
+    tmp_type = tstate->curexc_type;
+    tmp_value = tstate->curexc_value;
+    tmp_tb = tstate->curexc_traceback;
+    tstate->curexc_type = type;
+    tstate->curexc_value = value;
+    tstate->curexc_traceback = tb;
+    Py_XDECREF(tmp_type);
+    Py_XDECREF(tmp_value);
+    Py_XDECREF(tmp_tb);
+}
+
+static INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb) {
+    PyThreadState *tstate = PyThreadState_GET();
+    *type = tstate->curexc_type;
+    *value = tstate->curexc_value;
+    *tb = tstate->curexc_traceback;
+
+    tstate->curexc_type = 0;
+    tstate->curexc_value = 0;
+    tstate->curexc_traceback = 0;
+}
+
+""")
+
+# The following function is based on do_raise() from ceval.c. There
+# are separate versions for Python2 and Python3 as exception handling
+# has changed quite a lot between the two versions.
 
 raise_utility_code = UtilityCode(
 proto = """
 static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb); /*proto*/
 """,
 impl = """
+#if PY_MAJOR_VERSION < 3
 static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb) {
     Py_XINCREF(type);
     Py_XINCREF(value);
@@ -4982,6 +5030,7 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb) {
             }
         #endif
     }
+
     __Pyx_ErrRestore(type, value, tb);
     return;
 raise_error:
@@ -4990,6 +5039,166 @@ raise_error:
     Py_XDECREF(tb);
     return;
 }
+
+#else // Python 3+
+
+static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb) {
+    if (tb == Py_None) {
+        tb = 0;
+    } else if (tb && !PyTraceBack_Check(tb)) {
+        PyErr_SetString(PyExc_TypeError,
+            "raise: arg 3 must be a traceback or None");
+        goto bad;
+    }
+    if (value == Py_None)
+        value = 0;
+
+    if (PyExceptionInstance_Check(type)) {
+        if (value) {
+            PyErr_SetString(PyExc_TypeError,
+                "instance exception may not have a separate value");
+            goto bad;
+        }
+        value = type;
+        type = (PyObject*) Py_TYPE(value);
+    } else if (!PyExceptionClass_Check(type)) {
+        PyErr_SetString(PyExc_TypeError,
+            "raise: exception class must be a subclass of BaseException");
+        goto bad;
+    }
+
+    PyErr_SetObject(type, value);
+
+    if (tb) {
+        PyThreadState *tstate = PyThreadState_GET();
+        PyObject* tmp_tb = tstate->curexc_traceback;
+        if (tb != tmp_tb) {
+            Py_INCREF(tb);
+            tstate->curexc_traceback = tb;
+            Py_XDECREF(tmp_tb);
+        }
+    }
+
+bad:
+    return;
+}
+#endif
+""",
+requires=[restore_exception_utility_code])
+
+#------------------------------------------------------------------------------------
+
+get_exception_utility_code = UtilityCode(
+proto = """
+static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); /*proto*/
+""",
+impl = """
+static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) {
+    PyObject *local_type, *local_value, *local_tb;
+    PyObject *tmp_type, *tmp_value, *tmp_tb;
+    PyThreadState *tstate = PyThreadState_GET();
+    local_type = tstate->curexc_type;
+    local_value = tstate->curexc_value;
+    local_tb = tstate->curexc_traceback;
+    tstate->curexc_type = 0;
+    tstate->curexc_value = 0;
+    tstate->curexc_traceback = 0;
+    PyErr_NormalizeException(&local_type, &local_value, &local_tb);
+    if (unlikely(tstate->curexc_type))
+        goto bad;
+    #if PY_MAJOR_VERSION >= 3
+    if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0))
+        goto bad;
+    #endif
+    *type = local_type;
+    *value = local_value;
+    *tb = local_tb;
+    Py_INCREF(local_type);
+    Py_INCREF(local_value);
+    Py_INCREF(local_tb);
+    tmp_type = tstate->exc_type;
+    tmp_value = tstate->exc_value;
+    tmp_tb = tstate->exc_traceback;
+    tstate->exc_type = local_type;
+    tstate->exc_value = local_value;
+    tstate->exc_traceback = local_tb;
+    /* Make sure tstate is in a consistent state when we XDECREF
+       these objects (XDECREF may run arbitrary code). */
+    Py_XDECREF(tmp_type);
+    Py_XDECREF(tmp_value);
+    Py_XDECREF(tmp_tb);
+    return 0;
+bad:
+    *type = 0;
+    *value = 0;
+    *tb = 0;
+    Py_XDECREF(local_type);
+    Py_XDECREF(local_value);
+    Py_XDECREF(local_tb);
+    return -1;
+}
+
+""")
+
+#------------------------------------------------------------------------------------
+
+get_exception_tuple_utility_code = UtilityCode(proto="""
+static PyObject *__Pyx_GetExceptionTuple(void); /*proto*/
+""",
+# I doubt that calling __Pyx_GetException() here is correct as it moves
+# the exception from tstate->curexc_* to tstate->exc_*, which prevents
+# exception handlers later on from receiving it.
+impl = """
+static PyObject *__Pyx_GetExceptionTuple(void) {
+    PyObject *type = NULL, *value = NULL, *tb = NULL;
+    if (__Pyx_GetException(&type, &value, &tb) == 0) {
+        PyObject* exc_info = PyTuple_New(3);
+        if (exc_info) {
+            Py_INCREF(type);
+            Py_INCREF(value);
+            Py_INCREF(tb);
+            PyTuple_SET_ITEM(exc_info, 0, type);
+            PyTuple_SET_ITEM(exc_info, 1, value);
+            PyTuple_SET_ITEM(exc_info, 2, tb);
+            return exc_info;
+        }
+    }
+    return NULL;
+}
+""",
+requires=[get_exception_utility_code])
+
+#------------------------------------------------------------------------------------
+
+reset_exception_utility_code = UtilityCode(
+proto = """
+static INLINE void __Pyx_ExceptionSave(PyObject **type, PyObject **value, PyObject **tb); /*proto*/
+static void __Pyx_ExceptionReset(PyObject *type, PyObject *value, PyObject *tb); /*proto*/
+""",
+impl = """
+static INLINE void __Pyx_ExceptionSave(PyObject **type, PyObject **value, PyObject **tb) {
+    PyThreadState *tstate = PyThreadState_GET();
+    *type = tstate->exc_type;
+    *value = tstate->exc_value;
+    *tb = tstate->exc_traceback;
+    Py_XINCREF(*type);
+    Py_XINCREF(*value);
+    Py_XINCREF(*tb);
+}
+
+static void __Pyx_ExceptionReset(PyObject *type, PyObject *value, PyObject *tb) {
+    PyObject *tmp_type, *tmp_value, *tmp_tb;
+    PyThreadState *tstate = PyThreadState_GET();
+    tmp_type = tstate->exc_type;
+    tmp_value = tstate->exc_value;
+    tmp_tb = tstate->exc_traceback;
+    tstate->exc_type = type;
+    tstate->exc_value = value;
+    tstate->exc_traceback = tb;
+    Py_XDECREF(tmp_type);
+    Py_XDECREF(tmp_value);
+    Py_XDECREF(tmp_tb);
+}
 """)
 
 #------------------------------------------------------------------------------------
@@ -5339,57 +5548,6 @@ bad:
     'EMPTY_BYTES' : Naming.empty_bytes,
 })
 
-restore_exception_utility_code = UtilityCode(
-proto = """
-static INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb); /*proto*/
-static INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb); /*proto*/
-""",
-impl = """
-static INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb) {
-    PyObject *tmp_type, *tmp_value, *tmp_tb;
-    PyThreadState *tstate = PyThreadState_GET();
-
-#if PY_MAJOR_VERSION >= 3
-    /* Note: this is a temporary work-around to prevent crashes in Python 3.0 */
-    if ((tstate->exc_type != NULL) & (tstate->exc_type != Py_None)) {
-        tmp_type = tstate->exc_type;
-        tmp_value = tstate->exc_value;
-        tmp_tb = tstate->exc_traceback;
-        PyErr_NormalizeException(&type, &value, &tb);
-        PyErr_NormalizeException(&tmp_type, &tmp_value, &tmp_tb);
-        tstate->exc_type = 0;
-        tstate->exc_value = 0;
-        tstate->exc_traceback = 0;
-        PyException_SetContext(value, tmp_value);
-        Py_DECREF(tmp_type);
-        Py_XDECREF(tmp_tb);
-    }
-#endif
-
-    tmp_type = tstate->curexc_type;
-    tmp_value = tstate->curexc_value;
-    tmp_tb = tstate->curexc_traceback;
-    tstate->curexc_type = type;
-    tstate->curexc_value = value;
-    tstate->curexc_traceback = tb;
-    Py_XDECREF(tmp_type);
-    Py_XDECREF(tmp_value);
-    Py_XDECREF(tmp_tb);
-}
-
-static INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb) {
-    PyThreadState *tstate = PyThreadState_GET();
-    *type = tstate->curexc_type;
-    *value = tstate->curexc_value;
-    *tb = tstate->curexc_traceback;
-
-    tstate->curexc_type = 0;
-    tstate->curexc_value = 0;
-    tstate->curexc_traceback = 0;
-}
-
-""")
-
 #------------------------------------------------------------------------------------
 
 unraisable_exception_utility_code = UtilityCode(
@@ -5508,107 +5666,6 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) {
 
 #------------------------------------------------------------------------------------
 
-get_exception_utility_code = UtilityCode(
-proto = """
-static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); /*proto*/
-""",
-impl = """
-static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) {
-    PyObject *tmp_type, *tmp_value, *tmp_tb;
-    PyThreadState *tstate = PyThreadState_GET();
-    *type = tstate->curexc_type;
-    *value = tstate->curexc_value;
-    *tb = tstate->curexc_traceback;
-    tstate->curexc_type = 0;
-    tstate->curexc_value = 0;
-    tstate->curexc_traceback = 0;
-    PyErr_NormalizeException(type, value, tb);
-    if (PyErr_Occurred())
-        goto bad;
-    Py_INCREF(*type);
-    Py_INCREF(*value);
-    Py_INCREF(*tb);
-    tmp_type = tstate->exc_type;
-    tmp_value = tstate->exc_value;
-    tmp_tb = tstate->exc_traceback;
-    tstate->exc_type = *type;
-    tstate->exc_value = *value;
-    tstate->exc_traceback = *tb;
-    /* Make sure tstate is in a consistent state when we XDECREF
-    these objects (XDECREF may run arbitrary code). */
-    Py_XDECREF(tmp_type);
-    Py_XDECREF(tmp_value);
-    Py_XDECREF(tmp_tb);
-    return 0;
-bad:
-    Py_XDECREF(*type);
-    Py_XDECREF(*value);
-    Py_XDECREF(*tb);
-    return -1;
-}
-
-""")
-
-#------------------------------------------------------------------------------------
-
-get_exception_tuple_utility_code = UtilityCode(proto="""
-static PyObject *__Pyx_GetExceptionTuple(void); /*proto*/
-""",
-impl = """
-static PyObject *__Pyx_GetExceptionTuple(void) {
-    PyObject *type = NULL, *value = NULL, *tb = NULL;
-    if (__Pyx_GetException(&type, &value, &tb) == 0) {
-        PyObject* exc_info = PyTuple_New(3);
-        if (exc_info) {
-            Py_INCREF(type);
-            Py_INCREF(value);
-            Py_INCREF(tb);
-            PyTuple_SET_ITEM(exc_info, 0, type);
-            PyTuple_SET_ITEM(exc_info, 1, value);
-            PyTuple_SET_ITEM(exc_info, 2, tb);
-            return exc_info;
-        }
-    }
-    return NULL;
-}
-""",
-requires=[get_exception_utility_code])
-
-#------------------------------------------------------------------------------------
-
-reset_exception_utility_code = UtilityCode(
-proto = """
-static INLINE void __Pyx_ExceptionSave(PyObject **type, PyObject **value, PyObject **tb); /*proto*/
-static void __Pyx_ExceptionReset(PyObject *type, PyObject *value, PyObject *tb); /*proto*/
-""",
-impl = """
-static INLINE void __Pyx_ExceptionSave(PyObject **type, PyObject **value, PyObject **tb) {
-    PyThreadState *tstate = PyThreadState_GET();
-    *type = tstate->exc_type;
-    *value = tstate->exc_value;
-    *tb = tstate->exc_traceback;
-    Py_XINCREF(*type);
-    Py_XINCREF(*value);
-    Py_XINCREF(*tb);
-}
-
-static void __Pyx_ExceptionReset(PyObject *type, PyObject *value, PyObject *tb) {
-    PyObject *tmp_type, *tmp_value, *tmp_tb;
-    PyThreadState *tstate = PyThreadState_GET();
-    tmp_type = tstate->exc_type;
-    tmp_value = tstate->exc_value;
-    tmp_tb = tstate->exc_traceback;
-    tstate->exc_type = type;
-    tstate->exc_value = value;
-    tstate->exc_traceback = tb;
-    Py_XDECREF(tmp_type);
-    Py_XDECREF(tmp_value);
-    Py_XDECREF(tmp_tb);
-}
-""")
-
-#------------------------------------------------------------------------------------
-
 force_init_threads_utility_code = UtilityCode(
 proto="""
 #ifndef __PYX_FORCE_INIT_THREADS

Cython/Compiler/Scanning.py

@@ -9,6 +9,7 @@ import os
 import platform
 import stat
 import sys
+import codecs
 from time import time
 
 import cython
@@ -279,8 +280,12 @@ class FileSourceDescriptor(SourceDescriptor):
         self.filename = filename
         self._cmp_name = filename
     
-    def get_lines(self):
+    def get_lines(self, encoding=None, error_handling=None):
+        if not encoding:
             return Utils.open_source_file(self.filename)
+        else:
+            return codecs.open(self.filename, "rU", encoding=encoding,
+                               errors=error_handling)
     
     def get_description(self):
         return self.filename
@@ -307,8 +312,12 @@ class StringSourceDescriptor(SourceDescriptor):
         self.codelines = [x + "\n" for x in code.split("\n")]
         self._cmp_name = name
     
-    def get_lines(self):
+    def get_lines(self, encoding=None, error_handling=None):
+        if not encoding:
             return self.codelines
+        else:
+            return [ line.encode(encoding, error_handling).decode(encoding)
+                     for line in self.codelines ]
 
     def get_description(self):
         return self.name

Cython/Includes/python_buffer.pxd

@@ -25,23 +25,85 @@ cdef extern from "Python.h":
         PyBUF_WRITE,
         PyBUF_SHADOW
 
-    int PyObject_CheckBuffer(object obj)
-    int PyObject_GetBuffer(object obj, Py_buffer *view, int flags)
-    void PyObject_ReleaseBuffer(object obj, Py_buffer *view)
+    bint PyObject_CheckBuffer(object obj)
+    # Return 1 if obj supports the buffer interface otherwise 0.
+
+    int PyObject_GetBuffer(object obj, Py_buffer *view, int flags) except -1
+    # Export obj into a Py_buffer, view. These arguments must never be
+    # NULL. The flags argument is a bit field indicating what kind of
+    # buffer the caller is prepared to deal with and therefore what
+    # kind of buffer the exporter is allowed to return. The buffer
+    # interface allows for complicated memory sharing possibilities,
+    # but some caller may not be able to handle all the complexity but
+    # may want to see if the exporter will let them take a simpler
+    # view to its memory.
+
+    # Some exporters may not be able to share memory in every possible
+    # way and may need to raise errors to signal to some consumers
+    # that something is just not possible. These errors should be a
+    # BufferError unless there is another error that is actually
+    # causing the problem. The exporter can use flags information to
+    # simplify how much of the Py_buffer structure is filled in with
+    # non-default values and/or raise an error if the object can’t
+    # support a simpler view of its memory.
+
+    # 0 is returned on success and -1 on error.
+
+    void PyBuffer_Release(object obj, object view)
+    # Release the buffer view over obj. This should be called when the
+    # buffer is no longer being used as it may free memory from it.
+
     void* PyBuffer_GetPointer(Py_buffer *view, Py_ssize_t *indices)
+    # ??
+
     int PyBuffer_SizeFromFormat(char *) # actually const char
+    # Return the implied ~Py_buffer.itemsize from the struct-stype
+    # ~Py_buffer.format
+
     int PyBuffer_ToContiguous(void *buf, Py_buffer *view, Py_ssize_t len, char fort)
+    # ??
+
     int PyBuffer_FromContiguous(Py_buffer *view, void *buf, Py_ssize_t len, char fort)
+    # ??
+
+    int PyObject_CopyToObject(object obj, void *buf, Py_ssize_t len, char fortran) except -1
+    # Copy len bytes of data pointed to by the contiguous chunk of
+    # memory pointed to by buf into the buffer exported by obj. The
+    # buffer must of course be writable. Return 0 on success and
+    # return -1 and raise an error on failure. If the object does not
+    # have a writable buffer, then an error is raised. If fortran is
+    # 'F', then if the object is multi-dimensional, then the data will
+    # be copied into the array in Fortran-style (first dimension
+    # varies the fastest). If fortran is 'C', then the data will be
+    # copied into the array in C-style (last dimension varies the
+    # fastest). If fortran is 'A', then it does not matter and the
+    # copy will be made in whatever way is more efficient.
+
     int PyObject_CopyData(object dest, object src)
-    int PyBuffer_IsContiguous(Py_buffer *view, char fort)
+    # Copy the data from the src buffer to the buffer of destination
+
+    bint PyBuffer_IsContiguous(Py_buffer *view, char fort)
+    # Return 1 if the memory defined by the view is C-style (fortran
+    # is 'C') or Fortran-style (fortran is 'F') contiguous or either
+    # one (fortran is 'A'). Return 0 otherwise.
+
     void PyBuffer_FillContiguousStrides(int ndims, 
                                         Py_ssize_t *shape, 
                                         Py_ssize_t *strides,
                                         int itemsize,
                                         char fort)
+    # Fill the strides array with byte-strides of a contiguous
+    # (Fortran-style if fort is 'F' or C-style otherwise) array of the
+    # given shape with the given number of bytes per element.
+
     int PyBuffer_FillInfo(Py_buffer *view, void *buf,
                           Py_ssize_t len, int readonly,
-                          int flags)
+                          int flags) except -1
+    # Fill in a buffer-info structure, view, correctly for an exporter
+    # that can only share a contiguous chunk of memory of “unsigned
+    # bytes” of the given length. Return 0 on success and -1 (with
+    # raising an error) on error.
 
-    object PyObject_Format(object obj,
-                           object format_spec)
+    object PyObject_Format(object obj, object format_spec)
+    # Takes an arbitrary object and returns the result of calling
+    # obj.__format__(format_spec).

tests/bugs.txt

@@ -5,7 +5,6 @@ methodmangling_T5
 class_attribute_init_values_T18
 numpy_ValueError_T172
 unsignedbehaviour_T184
-funcexc_iter_T228
 bad_c_struct_T252
 missing_baseclass_in_predecl_T262
 extended_unpacking_T409

tests/errors/e_cdef_keywords_T241.pyx

@@ -14,6 +14,6 @@ a.some_method(1, 2)
 a.some_method(1, y=2)
 
 _ERRORS = u"""
-9:13: Keyword arguments not allowed in cdef functions.
-14:13: Keyword arguments not allowed in cdef functions.
+9:13: Keyword and starred arguments not allowed in cdef functions.
+14:13: Keyword and starred arguments not allowed in cdef functions.
 """

tests/errors/nogil.pyx

@@ -116,9 +116,7 @@ _ERRORS = u"""
 39: 9: Constructing Python tuple not allowed without gil
 40: 8: Constructing Python list not allowed without gil
 41: 8: Constructing Python dict not allowed without gil
-42:12: Creating temporary Python reference not allowed without gil
 42:12: Truth-testing Python object not allowed without gil
-42:17: Creating temporary Python reference not allowed without gil
 43:13: Python type test not allowed without gil
 45:10: Operation not allowed without gil
 46:8: Operation not allowed without gil

tests/run/__getattribute__.pyx

 __doc__ = u"""
 __getattribute__ and __getattr__ special methods for a single class.
+"""
 
+cdef class just_getattribute:
+    """
     >>> a = just_getattribute()
     >>> a.bar
     'bar'
     >>> a.invalid
     Traceback (most recent call last):
     AttributeError
+    """
+    def __getattribute__(self,n):
+        if n == 'bar':
+            return n
+        else:
+            raise AttributeError
 
+cdef class just_getattr:
+    """
     >>> a = just_getattr()
     >>> a.foo
     10
@@ -16,25 +27,7 @@ __getattribute__ and __getattr__ special methods for a single class.
     >>> a.invalid
     Traceback (most recent call last):
     AttributeError
-
-    >>> a = both()
-    >>> a.foo
-    10
-    >>> a.bar
-    'bar'
-    >>> a.invalid
-    Traceback (most recent call last):
-    AttributeError
-"""
-
-cdef class just_getattribute:
-    def __getattribute__(self,n):
-        if n == 'bar':
-            return n
-        else:
-            raise AttributeError
-
-cdef class just_getattr:
+    """
     cdef readonly int foo
     def __init__(self):
         self.foo = 10
@@ -45,6 +38,16 @@ cdef class just_getattr:
             raise AttributeError
 
 cdef class both:
+    """
+    >>> a = both()
+    >>> a.foo
+    10
+    >>> a.bar
+    'bar'
+    >>> a.invalid
+    Traceback (most recent call last):
+    AttributeError
+    """
     cdef readonly int foo
     def __init__(self):
         self.foo = 10

tests/run/ctruthtests.pyx

+__doc__ = u"""
+>>> test_int(0)
+False
+>>> test_int(1)
+True
+
+>>> test_short(0)
+False
+>>> test_short(1)
+True
+
+>>> test_Py_ssize_t(0)
+False
+>>> test_Py_ssize_t(1)
+True
+
+>>> test_ptr()
+False
+>>> test_ptr2()
+2
+
+>>> test_attr_int(TestExtInt(0))
+False
+>>> test_attr_int(TestExtInt(1))
+True
+
+>>> test_attr_ptr(TestExtPtr(0))
+False
+>>> test_attr_ptr(TestExtPtr(1))
+True
+"""
+
+def test_ptr():
+    cdef void* p = NULL
+    if p:
+        return True
+    else:
+        return False
+
+def test_ptr2():
+    cdef char* p1 = NULL
+    cdef char* p2 = NULL
+    p1 += 1
+
+    if p1 and p2:
+        return 1
+    elif p1 or p2:
+        return 2
+    else:
+        return 3
+
+def test_int(int i):
+    if i:
+        return True
+    else:
+        return False
+
+def test_short(short i):
+    if i:
+        return True
+    else:
+        return False
+
+def test_Py_ssize_t(Py_ssize_t i):
+    if i:
+        return True
+    else:
+        return False
+
+cdef class TestExtInt:
+    cdef int i
+    def __init__(self, i): self.i = i
+
+def test_attr_int(TestExtInt e):
+    if e.i:
+        return True
+    else:
+        return False
+
+cdef class TestExtPtr:
+    cdef void* p
+    def __init__(self, int i): self.p = <void*>i
+
+def test_attr_ptr(TestExtPtr e):
+    if e.p:
+        return True
+    else:
+        return False

tests/run/embedsignatures.pyx

@@ -137,10 +137,6 @@ __doc__ = ur"""
 
 """
 
-import sys
-if sys.version_info[0] >= 3:
-    __doc__ = __doc__.replace(u"u'spam'", u"'spam'")
-
 cdef class Ext:
 
     def __init__(self, a, b, c=None):

tests/run/for_in_range_T372.pyx

 __doc__ = u"""
 >>> test_modify()
-0 1 2 3 4
+0
+1
+2
+3
+4
+<BLANKLINE>
 (4, 0)
 >>> test_fix()
-0 1 2 3 4
+0
+1
+2
+3
+4
+<BLANKLINE>
 4
 >>> test_break()
-0 1 2
+0
+1
+2
+<BLANKLINE>
 (2, 0)
 >>> test_return()
-0 1 2
+0
+1
+2
 (2, 0)
 """
 
@@ -20,7 +35,7 @@ cimport cython
 def test_modify():
     cdef int i, n = 5
     for i in range(n):
-        print i,
+        print i
         n = 0
     print
     return i,n
@@ -30,7 +45,7 @@ def test_modify():
 def test_fix():
     cdef int i
     for i in range(5):
-        print i,
+        print i
     print
     return i
 
@@ -39,10 +54,12 @@ def test_fix():
 def test_break():
     cdef int i, n = 5
     for i in range(n):
-        print i,
+        print i
         n = 0
         if i == 2:
             break
+    else:
+        print "FAILED!"
     print
     return i,n
 
@@ -51,7 +68,7 @@ def test_break():
 def test_return():
     cdef int i, n = 5
     for i in range(n):
-        print i,
+        print i
         n = 0
         if i == 2:
             return i,n

tests/run/funcexc_iter_T228.pyx

@@ -45,6 +45,8 @@ True
 """
 
 import sys
+if sys.version_info[0] < 3:
+    sys.exc_clear()
 
 cdef class cy_iterator(object):
     def __iter__(self):

tests/run/funcexceptraise.pyx

@@ -11,6 +11,8 @@ __doc__ = u"""
 """
 
 import sys
+if sys.version_info[0] < 3:
+    sys.exc_clear()
 
 def foo():
     try:

tests/run/funcexceptreraise.pyx

+
+import sys
+
+def reraise(f, exc):
+    """
+    >>> def f(exc): raise exc
+    >>> reraise(f, TypeError)
+    Traceback (most recent call last):
+    TypeError
+
+    >>> def f(exc): raise exc('hiho')
+    >>> reraise(f, TypeError)
+    Traceback (most recent call last):
+    TypeError: hiho
+    """
+    try:
+        f(exc)
+    except:
+        assert sys.exc_info()[0] is exc, str(sys.exc_info()[1])
+        raise
+
+def reraise_original(f, exc, raise_catch):
+    """
+    >>> def f(exc): raise exc
+    >>> def raise_catch_py():
+    ...     try: raise ValueError
+    ...     except: pass
+
+    >>> reraise_original(f, TypeError, raise_catch_py)
+    Traceback (most recent call last):
+    TypeError
+
+    >>> reraise_original(f, TypeError, raise_catch_cy)
+    Traceback (most recent call last):
+    TypeError
+
+    >>> reraise_original(f, TypeError, raise_catch_cy_non_empty)
+    Traceback (most recent call last):
+    TypeError
+    """
+    try:
+        f(exc)
+    except:
+        raise_catch()
+        assert sys.exc_info()[0] is exc, str(sys.exc_info()[1])
+        raise
+
+
+def raise_catch_cy():
+    try: raise ValueError
+    except: pass
+
+def raise_catch_cy_non_empty():
+    try: raise ValueError
+    except:
+        a = 1+1

tests/run/jarausch1.pyx

 __doc__ = u"""
-   >>> py_x = br'\\\\'
-   >>> assert x == py_x
+   >>> b == br'\\\\'
+   True
+   >>> s ==  r'\\\\'
+   True
+   >>> u == ur'\\\\'
+   True
 """
 
 import sys
 if sys.version_info[0] < 3:
     __doc__ = __doc__.replace(u" br'", u" r'")
+else:
+    __doc__ = __doc__.replace(u" ur'", u" r'")
 
-x = r'\\'
+b = br'\\'
+s =  r'\\'
+u = ur'\\'

tests/run/kostyrka2.pyx

 __doc__ = u"""
     >>> x = X()
     >>> x.slots
-    [b'']
+    ['']
 """
 
-import sys
-if sys.version_info[0] < 3:
-    __doc__ = __doc__.replace(u"b'", u"'")
-
 class X:
         slots = ["", ]

tests/run/literalslice.pyx

 __doc__ = u"""
   >>> test_str(1)
-  b'b'
+  'b'
 
   >>> test_unicode_ascii(2)
   u'c'
@@ -10,14 +10,14 @@ __doc__ = u"""
   >>> test_int_list(2)
   3
   >>> test_str_list(1)
-  b'bcd'
+  'bcd'
 
   >>> test_int_tuple(2)
   3
   >>> test_str_tuple(0)
-  b'a'
+  'a'
   >>> test_mix_tuple(1)
-  b'abc'
+  'abc'
   >>> test_mix_tuple(0)
   1
 """
@@ -30,9 +30,6 @@ else:
     __doc__ = __doc__.replace(u" b'", u" '")
 
 def test_str(n):
-    if IS_PY3:
-        return bytes(["abcd"[n]])
-    else:
     return "abcd"[n]
 
 def test_unicode_ascii(n):

tests/run/r_vree_1.pyx

-__doc__ = """# disabled in Py3
+
+import sys
+if sys.version_info[0] < 3:
+    __doc__ = u"""
+
     >>> test(0)
     0L
     >>> test(1)
     1L
 
-    >>> import sys
     >>> sys.maxint + 1 > sys.maxint
     True
     >>> type(sys.maxint * 2 + 1) is long
@@ -19,7 +22,16 @@ __doc__ = """# disabled in Py3
     True
     >>> test(256 ** unsigned_long_size() - 1) > sys.maxint
     True
-"""
+    """
+else:
+    __doc__ = u"""
+    >>> test(0)
+    0
+    >>> test(1)
+    1
+    >>> test(256 ** unsigned_long_size() - 1) > 0
+    True
+    """
 
 def test(k):
     cdef unsigned long m

tests/run/str_char_coercion_T412.pyx

+__doc__ = u"""
+>>> test_eq()
+True
+True
+True
+True
+
+>>> test_cascaded_eq()
+True
+True
+True
+True
+True
+True
+True
+True
+
+>>> test_cascaded_ineq()
+True
+True
+True
+True
+True
+True
+True
+True
+
+>>> test_long_ineq()
+True
+
+>>> test_long_ineq_py()
+True
+True
+"""
+
+cdef int   i = 'x'
+cdef char  c = 'x'
+cdef char* s = 'x'
+
+def test_eq():
+    print i ==  'x'
+    print i == c'x'
+    print c ==  'x'
+    print c == c'x'
+#    print s ==  'x' # error
+#    print s == c'x' # error
+
+def test_cascaded_eq():
+    print  'x' == i ==  'x'
+    print  'x' == i == c'x'
+    print c'x' == i ==  'x'
+    print c'x' == i == c'x'
+
+    print  'x' == c ==  'x'
+    print  'x' == c == c'x'
+    print c'x' == c ==  'x'
+    print c'x' == c == c'x'
+
+def test_cascaded_ineq():
+    print  'a' <= i <=  'z'
+    print  'a' <= i <= c'z'
+    print c'a' <= i <=  'z'
+    print c'a' <= i <= c'z'
+
+    print  'a' <= c <=  'z'
+    print  'a' <= c <= c'z'
+    print c'a' <= c <=  'z'
+    print c'a' <= c <= c'z'
+
+def test_long_ineq():
+    print 'a' < 'b' < 'c' < 'd' < c < 'y' < 'z'
+
+def test_long_ineq_py():
+    print 'abcdef' < 'b' < 'c' < 'd' < 'y' < 'z'
+    print 'a' < 'b' < 'cde' < 'd' < 'y' < 'z'

tests/run/unicodeliterals.pyx

 # -*- coding: utf-8 -*-
 
-__doc__ = r"""
+__doc__ = br"""
     >>> sa
-    b'abc'
+    'abc'
     >>> ua
     u'abc'
     >>> b
@@ -19,7 +19,7 @@ __doc__ = r"""
     u'S\xf8k ik\xfc\xd6\xe4abc'
     >>> null
     u'\x00'
-""".decode(u"ASCII") + """
+""".decode("ASCII") + b"""
     >>> len(sa)
     3
     >>> len(ua)
@@ -38,7 +38,7 @@ __doc__ = r"""
     12
     >>> len(null)
     1
-""".decode(u"ASCII") + u"""
+""".decode("ASCII") + u"""
     >>> ua == u'abc'
     True
     >>> b == u'123'

tests/run/unicodeliteralsdefault.pyx

@@ -6,9 +6,9 @@
 # This file is written in UTF-8, but it has no encoding declaration,
 # so it just defaults to UTF-8 (PEP 3120).
 
-__doc__ = r"""
+__doc__ = br"""
     >>> sa
-    b'abc'
+    'abc'
     >>> ua
     u'abc'
     >>> b
@@ -25,7 +25,7 @@ __doc__ = r"""
     u'S\xf8k ik\xfc\xd6\xe4abc'
     >>> null
     u'\x00'
-""".decode(u"ASCII") + """
+""".decode("ASCII") + b"""
     >>> len(sa)
     3
     >>> len(ua)
@@ -44,7 +44,7 @@ __doc__ = r"""
     12
     >>> len(null)
     1
-""".decode(u"ASCII") + u"""
+""".decode("ASCII") + u"""
     >>> ua == u'abc'
     True
     >>> b == u'123'

tests/run/unicodeliteralslatin1.pyx

 # -*- coding: latin-1 -*-
 
-__doc__ = r"""
+__doc__ = br"""
     >>> sa
-    b'abc'
+    'abc'
     >>> ua
     u'abc'
     >>> b
@@ -19,7 +19,7 @@ __doc__ = r"""
     u'S\xf8k ik\xfc\xd6\xe4abc'
     >>> null
     u'\x00'
-""".decode(u"ASCII") + """
+""".decode("ASCII") + b"""
     >>> len(sa)
     3
     >>> len(ua)
@@ -38,7 +38,7 @@ __doc__ = r"""
     12
     >>> len(null)
     1
-""".decode(u"ASCII") + u"""
+""".decode("ASCII") + u"""
     >>> ua == u'abc'
     True
     >>> b == u'123'