merge in latest cython-devel

Cython/Compiler/Builtin.py

@@ -91,7 +91,10 @@ builtin_types_table = [
     ("int",     "PyInt_Type",      []),
     ("long",    "PyLong_Type",     []),
     ("float",   "PyFloat_Type",    []),
-    ("complex", "PyComplex_Type",  []),
+    
+# Until we have a way to access attributes of a type, 
+# we don't want to make this one builtin.    
+#    ("complex", "PyComplex_Type",  []),
 
     ("bytes",   "PyBytes_Type",    []),
     ("str",     "PyString_Type",   []),
@@ -340,10 +343,14 @@ def init_builtin_funcs():
     for desc in builtin_function_table:
         declare_builtin_func(*desc)
 
+builtin_types = {}
+
 def init_builtin_types():
+    global builtin_types
     for name, cname, funcs in builtin_types_table:
         utility = builtin_utility_code.get(name)
         the_type = builtin_scope.declare_builtin_type(name, cname, utility)
+        builtin_types[name] = the_type
         for name, args, ret, cname in funcs:
             sig = Signature(args, ret)
             the_type.scope.declare_cfunction(name, sig.function_type(), None, cname)

Cython/Compiler/ExprNodes.py

@@ -4272,33 +4272,70 @@ class MulNode(NumBinopNode):
             return NumBinopNode.is_py_operation(self)
 
 
-class FloorDivNode(NumBinopNode):
-    #  '//' operator.
+class DivNode(NumBinopNode):
+    #  '/' or '//' operator.
+    
+    cdivision = None
+    
+    def generate_evaluation_code(self, code):
+        if not self.type.is_pyobject:
+            if self.cdivision is None:
+                self.cdivision = (code.globalstate.directives['cdivision'] 
+                                    or not self.type.signed
+                                    or self.type.is_float)
+            if not self.cdivision:
+                code.globalstate.use_utility_code(div_utility_code.specialize(self.type))
+        NumBinopNode.generate_evaluation_code(self, code)
     
     def calculate_result_code(self):
-        return "(%s %s %s)" % (
-            self.operand1.result(), 
-            "/",  # c division is by default floor-div 
-            self.operand2.result())
+        if self.cdivision:
+            return "(%s / %s)" % (
+                self.operand1.result(), 
+                self.operand2.result())
+        else:
+            return "__Pyx_div_%s(%s, %s)" % (
+                    self.type.specalization_name(),
+                    self.operand1.result(), 
+                    self.operand2.result())
 
 
 class ModNode(NumBinopNode):
     #  '%' operator.
+
+    cdivision = None
     
     def is_py_operation(self):
         return (self.operand1.type.is_string
             or self.operand2.type.is_string
             or NumBinopNode.is_py_operation(self))
 
+    def generate_evaluation_code(self, code):
+        if not self.type.is_pyobject:
+            if self.cdivision is None:
+                self.cdivision = code.globalstate.directives['cdivision'] or not self.type.signed
+            if not self.cdivision:
+                math_h_modifier = getattr(self.type, 'math_h_modifier', '__Pyx_INT')
+                if self.type.is_int:
+                    code.globalstate.use_utility_code(mod_int_helper_macro)
+                code.globalstate.use_utility_code(mod_utility_code.specialize(self.type, math_h_modifier=math_h_modifier))
+        NumBinopNode.generate_evaluation_code(self, code)
+    
     def calculate_result_code(self):
-        if self.operand1.type.is_float or self.operand2.type.is_float:
-            return "fmod(%s, %s)" % (
-                self.operand1.result(), 
-                self.operand2.result())
+        if self.cdivision:
+            if self.type.is_float:
+                return "fmod%s(%s, %s)" % (
+                    self.type.math_h_modifier,
+                    self.operand1.result(), 
+                    self.operand2.result())
+            else:
+                return "(%s %% %s)" % (
+                    self.operand1.result(), 
+                    self.operand2.result())
         else:
-            return "(%s %% %s)" % (
-                self.operand1.result(), 
-                self.operand2.result())
+            return "__Pyx_mod_%s(%s, %s)" % (
+                    self.type.specalization_name(),
+                    self.operand1.result(), 
+                    self.operand2.result())
 
 class PowNode(NumBinopNode):
     #  '**' operator.
@@ -4894,20 +4931,20 @@ class CascadedCmpNode(Node, CmpNode):
 
 
 binop_node_classes = {
-    "or":        BoolBinopNode,
-    "and":    BoolBinopNode,
+    "or":       BoolBinopNode,
+    "and":      BoolBinopNode,
     "|":        IntBinopNode,
     "^":        IntBinopNode,
     "&":        IntBinopNode,
-    "<<":        IntBinopNode,
-    ">>":        IntBinopNode,
+    "<<":       IntBinopNode,
+    ">>":       IntBinopNode,
     "+":        AddNode,
     "-":        SubNode,
     "*":        MulNode,
-    "/":        NumBinopNode,
-    "//":        FloorDivNode,
+    "/":        DivNode,
+    "//":       DivNode,
     "%":        ModNode,
-    "**":        PowNode
+    "**":       PowNode
 }
 
 def binop_node(pos, operator, operand1, operand2):
@@ -5687,3 +5724,34 @@ static INLINE %(type)s %(func_name)s(%(type)s b, %(type)s e) {
     return t;
 }
 """)
+
+# ------------------------------ Division ------------------------------------
+
+# This is so we can treat floating point and integer mod simultaneously. 
+mod_int_helper_macro = UtilityCode(proto="""
+#define fmod__Pyx_INT(a, b) ((a) % (b))
+""")
+
+mod_utility_code = UtilityCode(
+proto="""
+static INLINE %(type)s __Pyx_mod_%(type_name)s(%(type)s, %(type)s); /* proto */
+""",
+impl="""
+static INLINE %(type)s __Pyx_mod_%(type_name)s(%(type)s a, %(type)s b) {
+    %(type)s res = fmod%(math_h_modifier)s(a, b);
+    res += (res * b < 0) * b;
+    return res;
+}
+""")
+
+div_utility_code = UtilityCode(
+proto="""
+static INLINE %(type)s __Pyx_div_%(type_name)s(%(type)s, %(type)s); /* proto */
+""",
+impl="""
+static INLINE %(type)s __Pyx_div_%(type_name)s(%(type)s a, %(type)s b) {
+    %(type)s res = a / b;
+    res -= (res < 0);
+    return res;
+}
+""")

Cython/Compiler/Nodes.py

@@ -19,8 +19,6 @@ import Options
 import ControlFlow
 import DebugFlags
 
-from DebugFlags import debug_disposal_code
-
 absolute_path_length = 0
 
 def relative_position(pos):
@@ -1295,7 +1293,12 @@ class CFuncDefNode(FuncDefNode):
         self.entry.inline_func_in_pxd = self.inline_in_pxd
         self.return_type = type.return_type
         
-        if self.overridable and len(self.args) > 0:
+        if self.overridable and not env.is_module_scope:
+            if len(self.args) < 1 or not self.args[0].type.is_pyobject:
+                # An error will be produced in the cdef function
+                self.overridable = False
+            
+        if self.overridable:
             import ExprNodes
             py_func_body = self.call_self_node(is_module_scope = env.is_module_scope)
             self.py_func = DefNode(pos = self.pos, 
@@ -2635,6 +2638,11 @@ class CClassDefNode(ClassDefNode):
                 return
         else:
             home_scope = env
+
+        if self.visibility == 'extern':
+            if self.module_name == '__builtin__' and self.class_name in Builtin.builtin_types:
+                warning(self.pos, "%s already a builtin Cython type" % self.class_name, 1)
+
         self.entry = home_scope.declare_c_class(
             name = self.class_name, 
             pos = self.pos,
@@ -4207,7 +4215,12 @@ class ExceptClauseNode(Node):
             self.match_flag = env.allocate_temp(PyrexTypes.c_int_type)
             self.pattern.release_temp(env)
             env.release_temp(self.match_flag)
-        self.exc_vars = [env.allocate_temp(py_object_type) for i in xrange(3)]
+
+        if self.target or self.excinfo_target:
+            self.exc_vars = [env.allocate_temp(py_object_type) for i in xrange(3)]
+        else:
+            self.exc_vars = None
+
         if self.target:
             self.exc_value = ExprNodes.ExcValueNode(self.pos, env, self.exc_vars[1])
             self.exc_value.allocate_temps(env)
@@ -4224,10 +4237,10 @@ class ExceptClauseNode(Node):
             self.excinfo_target.analyse_target_expression(env, self.excinfo_tuple)
 
         self.body.analyse_expressions(env)
-        for var in self.exc_vars:
-            env.release_temp(var)
-        env.use_utility_code(get_exception_utility_code)
-        env.use_utility_code(restore_exception_utility_code)
+
+        if self.exc_vars:
+            for var in self.exc_vars:
+                env.release_temp(var)
     
     def generate_handling_code(self, code, end_label):
         code.mark_pos(self.pos)
@@ -4244,14 +4257,32 @@ class ExceptClauseNode(Node):
                     self.match_flag)
         else:
             code.putln("/*except:*/ {")
+
+        if self.exc_vars:
+            exc_vars = self.exc_vars
+        elif not getattr(self.body, 'stats', True):
+            # most simple case: no exception variable, empty body (pass)
+            # => reset the exception state, done
+            code.putln("PyErr_Restore(0,0,0);")
+            code.put_goto(end_label)
+            code.putln("}")
+            return
+        else:
+            # during type analysis, we didn't know if we need the
+            # exception value, but apparently, we do
+            exc_vars = [code.funcstate.allocate_temp(py_object_type,
+                                                     manage_ref=True)
+                        for i in xrange(3)]
+
         code.putln('__Pyx_AddTraceback("%s");' % self.function_name)
         # We always have to fetch the exception value even if
         # there is no target, because this also normalises the 
         # exception and stores it in the thread state.
-        exc_args = "&%s, &%s, &%s" % tuple(self.exc_vars)
+        code.globalstate.use_utility_code(get_exception_utility_code)
+        exc_args = "&%s, &%s, &%s" % tuple(exc_vars)
         code.putln("if (__Pyx_GetException(%s) < 0) %s" % (exc_args,
             code.error_goto(self.pos)))
-        for x in self.exc_vars:
+        for x in exc_vars:
             code.put_gotref(x)
         if self.target:
             self.exc_value.generate_evaluation_code(code)
@@ -4266,27 +4297,32 @@ class ExceptClauseNode(Node):
         code.continue_label = code.new_label('except_continue')
 
         old_exc_vars = code.funcstate.exc_vars
-        code.funcstate.exc_vars = self.exc_vars
+        code.funcstate.exc_vars = exc_vars
         self.body.generate_execution_code(code)
         code.funcstate.exc_vars = old_exc_vars
-        for var in self.exc_vars:
+        for var in exc_vars:
             code.putln("__Pyx_DECREF(%s); %s = 0;" % (var, var))
         code.put_goto(end_label)
         
         if code.label_used(code.break_label):
             code.put_label(code.break_label)
-            for var in self.exc_vars:
+            for var in exc_vars:
                 code.putln("__Pyx_DECREF(%s); %s = 0;" % (var, var))
             code.put_goto(old_break_label)
         code.break_label = old_break_label
 
         if code.label_used(code.continue_label):
             code.put_label(code.continue_label)
-            for var in self.exc_vars:
+            for var in exc_vars:
                 code.putln("__Pyx_DECREF(%s); %s = 0;" % (var, var))
             code.put_goto(old_continue_label)
         code.continue_label = old_continue_label
-        
+
+        if not self.exc_vars:
+            # clean up locally allocated temps
+            for temp in exc_vars:
+                code.funcstate.release_temp(temp)
+
         code.putln(
             "}")
 
@@ -5538,7 +5574,12 @@ impl = """
 static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) {
     PyObject *tmp_type, *tmp_value, *tmp_tb;
     PyThreadState *tstate = PyThreadState_GET();
-    __Pyx_ErrFetch(type, value, tb);
+    *type = tstate->curexc_type;
+    *value = tstate->curexc_value;
+    *tb = tstate->curexc_traceback;
+    tstate->curexc_type = 0;
+    tstate->curexc_value = 0;
+    tstate->curexc_traceback = 0;
     PyErr_NormalizeException(type, value, tb);
     if (PyErr_Occurred())
         goto bad;

Cython/Compiler/Options.py

@@ -55,22 +55,22 @@ c_line_in_traceback = 1
 
 
 # Declare pragmas
-option_types = {
-    'boundscheck' : bool,
-    'nonecheck' : bool,
-    'embedsignature' : bool,
-    'locals' : dict,
-    'auto_cpdef': bool,
-}
-
 option_defaults = {
     'boundscheck' : True,
     'nonecheck' : False,
     'embedsignature' : False,
     'locals' : {},
     'auto_cpdef': False,
+    'cdivision': True,  # Will be False in 0.12
 }
 
+# Override types possibilities above, if needed
+option_types = { }
+
+for key, val in option_defaults.items():
+    if key not in option_types:
+        option_types[key] = type(val)
+
 def parse_option_value(name, value):
     """
     Parses value as an option value for the given name and returns

Cython/Compiler/ParseTreeTransforms.py

@@ -987,6 +987,18 @@ class TransformBuiltinMethods(EnvTransform):
                     error(node.function.pos, u"sizeof takes exactly one argument" % function)
                 else:
                     node = AmpersandNode(node.function.pos, operand=node.args[0])
+            elif function == 'cmod':
+                if len(node.args) != 2:
+                    error(node.function.pos, u"cmod takes exactly one argument" % function)
+                else:
+                    node = binop_node(node.function.pos, '%', node.args[0], node.args[1])
+                    node.cdivision = True
+            elif function == 'cdiv':
+                if len(node.args) != 2:
+                    error(node.function.pos, u"cmod takes exactly one argument" % function)
+                else:
+                    node = binop_node(node.function.pos, '/', node.args[0], node.args[1])
+                    node.cdivision = True
             else:
                 error(node.function.pos, u"'%s' not a valid cython language construct" % function)
         

Cython/Compiler/PyrexTypes.py

@@ -14,6 +14,9 @@ class BaseType(object):
     def cast_code(self, expr_code):
         return "((%s)%s)" % (self.declaration_code(""), expr_code)
     
+    def specalization_name(self):
+        return self.declaration_code("").replace(" ", "__")
+    
     def base_declaration_code(self, base_code, entity_code):
         if entity_code:
             return "%s %s" % (base_code, entity_code)
@@ -601,8 +604,9 @@ class CFloatType(CNumericType):
     to_py_function = "PyFloat_FromDouble"
     from_py_function = "__pyx_PyFloat_AsDouble"
     
-    def __init__(self, rank, pymemberdef_typecode = None):
+    def __init__(self, rank, pymemberdef_typecode = None, math_h_modifier = ''):
         CNumericType.__init__(self, rank, 1, pymemberdef_typecode)
+        self.math_h_modifier = math_h_modifier
     
     def assignable_from_resolved_type(self, src_type):
         return src_type.is_numeric or src_type is error_type
@@ -1193,9 +1197,9 @@ c_slonglong_type =   CLongLongType(6, 2, "T_LONGLONG")
 c_py_ssize_t_type =  CPySSizeTType(4, 2, "T_PYSSIZET")
 c_size_t_type =      CSizeTType(5, 0, "T_SIZET")
 
-c_float_type =       CFloatType(7, "T_FLOAT")
+c_float_type =       CFloatType(7, "T_FLOAT", math_h_modifier='f')
 c_double_type =      CFloatType(8, "T_DOUBLE")
-c_longdouble_type =  CFloatType(9)
+c_longdouble_type =  CFloatType(9, math_h_modifier='l')
 
 c_null_ptr_type =     CNullPtrType(c_void_type)
 c_char_array_type =   CCharArrayType(None)

Cython/Utils.py

@@ -122,8 +122,11 @@ class UtilityCode(object):
         else:
             writer.put(self.cleanup)
     
-    def specialize(self, **data):
+    def specialize(self, pyrex_type=None, **data):
         # Dicts aren't hashable...
+        if pyrex_type is not None:
+            data['type'] = pyrex_type.declaration_code('')
+            data['type_name'] = pyrex_type.specalization_name()
         key = data.items(); key.sort(); key = tuple(key)
         try:
             return self._cache[key]

runtests.py

@@ -561,6 +561,8 @@ if __name__ == '__main__':
     if WITH_CYTHON:
         from Cython.Compiler.Version import version
         sys.stderr.write("Running tests against Cython %s\n" % version)
+        from Cython.Compiler import DebugFlags
+        DebugFlags.debug_temp_code_comments = 1
     else:
         sys.stderr.write("Running tests without Cython.\n")
     sys.stderr.write("Python %s\n" % sys.version)

tests/bugs/return_outside_function_T135.pyx

+
+return 'bar'
+
+class A:
+    return None
+
+cdef class B:
+    return None
+
+try: return None
+except: pass
+
+try: return None
+finally: pass
+
+for i in (1,2):
+    return None
+
+while True:
+    return None
+
+if True:
+    return None
+else:
+    return None
+
+
+_ERRORS = u'''
+ 2:0: Return not inside a function body
+ 5:4: Return not inside a function body
+ 8:4: Return not inside a function body
+10:5: Return not inside a function body
+13:5: Return not inside a function body
+17:4: Return not inside a function body
+20:4: Return not inside a function body
+23:4: Return not inside a function body
+25:4: Return not inside a function body
+'''

tests/errors/break_outside_loop.pyx

+
+break
+
+class A:
+    break
+
+cdef class B:
+    break
+
+def test():
+    break
+
+try: break
+except: pass
+
+try: break
+finally: pass
+
+if True:
+    break
+else:
+    break
+
+
+_ERRORS = u'''
+ 2:0: break statement not inside loop
+ 5:4: break statement not inside loop
+ 8:4: break statement not inside loop
+11:4: break statement not inside loop
+16:5: break statement not inside loop
+20:4: break statement not inside loop
+22:4: break statement not inside loop
+'''

tests/errors/builtin_type_conflict_T170.pyx

+cdef extern from *:
+    ctypedef class __builtin__.list [object PyListObject]:
+        pass
+
+cdef list foo = []
+
+# This is too invasive for Python 0.11.x, re-enable in 0.12
+NEW_ERRORS = u"""
+:2:4: list already a builtin Cython type
+"""
+
+_ERRORS = u"""
+:5:16: Cannot coerce list to type 'list'
+"""
+

tests/errors/continue_outside_loop.pyx

+
+continue
+
+class A:
+    continue
+
+cdef class B:
+    continue
+
+def test():
+    continue
+
+try: continue
+except: pass
+
+try: continue
+finally: pass
+
+if True:
+    continue
+else:
+    continue
+
+
+_ERRORS = u'''
+ 2:0: continue statement not inside loop
+ 5:4: continue statement not inside loop
+ 8:4: continue statement not inside loop
+11:4: continue statement not inside loop
+16:5: continue statement not inside loop
+20:4: continue statement not inside loop
+22:4: continue statement not inside loop
+'''

tests/bugs/missing_self_in_cpdef_method_T165.pyx → tests/errors/missing_self_in_cpdef_method_T165.pyx

@@ -3,3 +3,6 @@ cdef class A:
     cpdef a(int not_self):
         pass
 
+_ERRORS = u"""
+3:10: Self argument (int) of C method 'a' does not match parent type (A)
+"""

tests/run/cdivision_CEP_516.pyx

+__doc__ = u"""
+>>> v = [(17, 10), (-17, 10), (-17, -10), (17, -10)]
+>>> standard = [(a % b) for a, b in v]
+>>> standard
+[7, 3, -7, -3]
+>>> [mod_int_py(a, b) for a, b in v] == standard
+True
+>>> [mod_short_py(a, b) for a, b in v] == standard
+True
+>>> [mod_float_py(a, b) for a, b in v] == standard
+True
+>>> [mod_double_py(a, b) for a, b in v] == standard
+True
+
+>>> [mod_int_c(a, b) for a, b in v]
+[7, -7, -7, 7]
+>>> [mod_float_c(a, b) for a, b in v]
+[7.0, -7.0, -7.0, 7.0]
+>>> [mod_double_c(a, b) for a, b in v]
+[7.0, -7.0, -7.0, 7.0]
+
+>>> [div_int_py(a, b) for a, b in v]
+[1, -2, 1, -2]
+>>> [div_int_c(a, b) for a, b in v]
+[1, -1, 1, -1]
+
+>>> [test_cdiv_cmod(a, b) for a, b in v]
+[(1, 7), (-1, -7), (1, -7), (-1, 7)]
+"""
+
+cimport cython
+
+@cython.cdivision(False)
+def mod_int_py(int a, int b):
+    return a % b
+
+@cython.cdivision(False)
+def mod_short_py(short a, short b):
+    return a % b
+
+@cython.cdivision(False)
+def mod_double_py(double a, double b):
+    return a % b
+
+@cython.cdivision(False)
+def mod_float_py(float a, float b):
+    return a % b
+
+@cython.cdivision(True)
+def mod_int_c(int a, int b):
+    return a % b
+
+@cython.cdivision(True)
+def mod_float_c(float a, float b):
+    return a % b
+
+@cython.cdivision(True)
+def mod_double_c(double a, double b):
+    return a % b
+
+
+@cython.cdivision(False)
+def div_int_py(int a, int b):
+    return a // b
+
+@cython.cdivision(True)
+def div_int_c(int a, int b):
+    return a // b
+
+
+@cython.cdivision(False)
+def test_cdiv_cmod(short a, short b):
+    cdef short q = cython.cdiv(a, b)
+    cdef short r = cython.cmod(a, b)
+    return q, r

tests/run/funcexceptchained.pyx

@@ -8,7 +8,13 @@ __doc__ = u"""
 ...   except AttributeError:
 ...     print(sys.exc_info()[0] is AttributeError or sys.exc_info()[0])
 ...     try: raise KeyError
-...     except: print(sys.exc_info()[0] is KeyError or sys.exc_info()[0])
+...     except:
+...       print(sys.exc_info()[0] is KeyError or sys.exc_info()[0])
+...       if IS_PY3:
+...         print(isinstance(sys.exc_info()[1].__context__, AttributeError)
+...               or sys.exc_info()[1].__context__)
+...       else:
+...         print(True)
 ...     print((IS_PY3 and sys.exc_info()[0] is AttributeError) or
 ...           (not IS_PY3 and sys.exc_info()[0] is KeyError) or
 ...           sys.exc_info()[0])
@@ -24,6 +30,7 @@ True
 True
 True
 True
+True
 >>> print(sys.exc_info()[0]) # test_py()
 None
 
@@ -32,6 +39,7 @@ True
 True
 True
 True
+True
 >>> print(sys.exc_info()[0]) # test_c()
 None
 
@@ -52,6 +60,7 @@ True
 True
 True
 True
+True
 >>> print(sys.exc_info()[0]) # test_py2()
 None
 
@@ -62,6 +71,7 @@ True
 True
 True
 True
+True
 >>> print(sys.exc_info()[0]) # test_c2()
 None
 """
@@ -76,7 +86,13 @@ def test_c(outer_exc):
     except AttributeError:
         print(sys.exc_info()[0] is AttributeError or sys.exc_info()[0])
         try: raise KeyError
-        except: print(sys.exc_info()[0] is KeyError or sys.exc_info()[0])
+        except:
+            print(sys.exc_info()[0] is KeyError or sys.exc_info()[0])
+            if IS_PY3:
+                print(isinstance(sys.exc_info()[1].__context__, AttributeError)
+                      or sys.exc_info()[1].__context__)
+            else:
+                print(True)
         print(sys.exc_info()[0] is AttributeError or sys.exc_info()[0])
     print(sys.exc_info()[0] is outer_exc or sys.exc_info()[0])
 

tests/run/funcexceptreplace.pyx

+__doc__ = u"""
+>>> try: exc()
+... except IndexError:
+...     if IS_PY3:
+...         print(isinstance(sys.exc_info()[1].__context__, ValueError))
+...     else:
+...         print(True)
+True
+"""
+
+import sys
+IS_PY3 = sys.version_info[0] >= 3
+
+def exc():
+    try:
+        raise ValueError
+    except ValueError:
+        raise IndexError