merged in latest cython-devel

Cython/Compiler/Builtin.py

@@ -32,7 +32,7 @@ builtin_function_table = [
     ('isinstance', "OO",   "b",     "PyObject_IsInstance"),
     ('issubclass', "OO",   "b",     "PyObject_IsSubclass"),
     #('iter',       "O",    "O",     "PyObject_GetIter"),   # optimised later on
-    ('len',        "O",    "Z",     "PyObject_Length"),
+    ('len',        "O",    "z",     "PyObject_Length"),
     ('locals',     "",     "O",     "__pyx_locals"),
     #('map',       "",     "",      ""),
     #('max',       "",     "",      ""),
@@ -103,7 +103,7 @@ builtin_types_table = [
 
     ("tuple",   "PyTuple_Type",    []),
 
-    ("list",    "PyList_Type",     [("insert", "OZO",  "i", "PyList_Insert")]),
+    ("list",    "PyList_Type",     [("insert", "OzO",  "i", "PyList_Insert")]),
 
     ("dict",    "PyDict_Type",     [("items", "O",   "O", "PyDict_Items"),
                                     ("keys",  "O",   "O", "PyDict_Keys"),

Cython/Compiler/ExprNodes.py

@@ -19,6 +19,7 @@ from Builtin import list_type, tuple_type, set_type, dict_type, \
 import Builtin
 import Symtab
 import Options
+from Cython import Utils
 from Annotate import AnnotationItem
 from Cython import Utils
 
@@ -1958,6 +1959,9 @@ class IndexNode(ExprNode):
             self.type = PyrexTypes.error_type
             return
         
+        if isinstance(self.index, IntNode) and Utils.long_literal(self.index.value):
+            self.index = self.index.coerce_to_pyobject(env)
+        
         # Handle the case where base is a literal char* (and we expect a string, not an int)
         if isinstance(self.base, BytesNode):
             self.base = self.base.coerce_to_pyobject(env)

Cython/Compiler/Main.py

@@ -19,7 +19,7 @@ import Errors
 import Parsing
 import Version
 from Scanning import PyrexScanner, FileSourceDescriptor
-from Errors import PyrexError, CompileError, InternalError, error
+from Errors import PyrexError, CompileError, InternalError, error, warning
 from Symtab import BuiltinScope, ModuleScope
 from Cython import Utils
 from Cython.Utils import open_new_file, replace_suffix
@@ -323,7 +323,27 @@ class Context(object):
         # the directory containing the source file is searched first
         # for a dotted filename, and its containing package root
         # directory is searched first for a non-dotted filename.
-        return self.search_include_directories(qualified_name, ".pxd", pos)
+        pxd = self.search_include_directories(qualified_name, ".pxd", pos)
+        if pxd is None: # XXX Keep this until Includes/Deprecated is removed
+            if (qualified_name.startswith('python') or
+                qualified_name in ('stdlib', 'stdio', 'stl')):
+                standard_include_path = os.path.abspath(os.path.normpath(
+                        os.path.join(os.path.dirname(__file__), os.path.pardir, 'Includes')))
+                deprecated_include_path = os.path.join(standard_include_path, 'Deprecated')
+                self.include_directories.append(deprecated_include_path)
+                try:
+                    pxd = self.search_include_directories(qualified_name, ".pxd", pos)
+                finally:
+                    self.include_directories.pop()
+                if pxd:
+                    name = qualified_name
+                    if name.startswith('python'):
+                        warning(pos, "'%s' is deprecated, use 'cpython'" % name, 1)
+                    elif name in ('stdlib', 'stdio'):
+                        warning(pos, "'%s' is deprecated, use 'libc.%s'" % (name, name), 1)
+                    elif name in ('stl'):
+                        warning(pos, "'%s' is deprecated, use 'libcpp.*.*'" % name, 1)
+        return pxd
 
     def find_pyx_file(self, qualified_name, pos):
         # Search include path for the .pyx file corresponding to the

Cython/Compiler/Optimize.py

@@ -803,7 +803,12 @@ class FlattenInListTransform(Visitor.VisitorTransform, SkipDeclarations):
         lhs = UtilNodes.ResultRefNode(node.operand1)
 
         conds = []
+        temps = []
         for arg in args:
+            if not arg.is_simple():
+                # must evaluate all non-simple RHS before doing the comparisons
+                arg = UtilNodes.LetRefNode(arg)
+                temps.append(arg)
             cond = ExprNodes.PrimaryCmpNode(
                                 pos = node.pos,
                                 operand1 = lhs,
@@ -822,7 +827,10 @@ class FlattenInListTransform(Visitor.VisitorTransform, SkipDeclarations):
                                 operand2 = right)
 
         condition = reduce(concat, conds)
-        return UtilNodes.EvalWithTempExprNode(lhs, condition)
+        new_node = UtilNodes.EvalWithTempExprNode(lhs, condition)
+        for temp in temps[::-1]:
+            new_node = UtilNodes.EvalWithTempExprNode(temp, new_node)
+        return new_node
 
     visit_Node = Visitor.VisitorTransform.recurse_to_children
 

Cython/Compiler/TreeFragment.py

@@ -14,7 +14,6 @@ from ExprNodes import NameNode
 import Parsing
 import Main
 import UtilNodes
-import StringEncoding
 
 """
 Support for parsing strings into code trees.
@@ -121,7 +120,7 @@ class TemplateTransform(VisitorTransform):
         temphandles = []
         for temp in temps:
             TemplateTransform.temp_name_counter += 1
-            handle = StringEncoding.EncodedString("__tmpvar_%d" % TemplateTransform.temp_name_counter)
+            handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter
 #            handle = UtilNodes.TempHandle(PyrexTypes.py_object_type)
             tempmap[temp] = handle
 #            temphandles.append(handle)

Cython/Compiler/TypeInference.py

@@ -3,6 +3,7 @@ import ExprNodes
 import Nodes
 import Builtin
 import PyrexTypes
+from Cython import Utils
 from PyrexTypes import py_object_type, unspecified_type
 from Visitor import CythonTransform
 
@@ -176,6 +177,24 @@ class MarkOverflowingArithmetic(CythonTransform):
     
     visit_Node = visit_safe_node
     
+    def visit_assignment(self, lhs, rhs):
+        if (isinstance(rhs, ExprNodes.IntNode) 
+                and isinstance(lhs, ExprNodes.NameNode)
+                and Utils.long_literal(rhs.value)):
+            entry = lhs.entry or self.env.lookup(lhs.name)
+            if entry:
+                entry.might_overflow = True
+    
+    def visit_SingleAssignmentNode(self, node):
+        self.visit_assignment(node.lhs, node.rhs)
+        self.visitchildren(node)
+        return node
+    
+    def visit_CascadedAssignmentNode(self, node):
+        for lhs in node.lhs_list:
+            self.visit_assignment(lhs, node.rhs)
+        self.visitchildren(node)
+        return node
 
 class PyObjectTypeInferer(object):
     """

Cython/Compiler/TypeSlots.py

@@ -29,7 +29,8 @@ class Signature(object):
     #    'b'  bint
     #    'I'  int *
     #    'l'  long
-    #    'Z'  Py_ssize_t
+    #    'z'  Py_ssize_t
+    #    'Z'  Py_ssize_t *
     #    's'  char *
     #    'S'  char **
     #    'r'  int used only to signal exception
@@ -48,7 +49,8 @@ class Signature(object):
         'b': PyrexTypes.c_bint_type,
         'I': PyrexTypes.c_int_ptr_type,
         'l': PyrexTypes.c_long_type,
-        'Z': PyrexTypes.c_py_ssize_t_type,
+        'z': PyrexTypes.c_py_ssize_t_type,
+        'Z': PyrexTypes.c_py_ssize_t_ptr_type,
         's': PyrexTypes.c_char_ptr_type,
         'S': PyrexTypes.c_char_ptr_ptr_type,
         'r': PyrexTypes.c_returncode_type,
@@ -63,7 +65,7 @@ class Signature(object):
         'b': "-1",
         'l': "-1",
         'r': "-1",
-        'Z': "-1",
+        'z': "-1",
     }
     
     def __init__(self, arg_format, ret_format):
@@ -484,30 +486,26 @@ ternaryfunc = Signature("OOO", "O")        # typedef PyObject * (*ternaryfunc)(P
 iternaryfunc = Signature("TOO", "O")       # typedef PyObject * (*ternaryfunc)(PyObject *, PyObject *, PyObject *);
 callfunc = Signature("T*", "O")            # typedef PyObject * (*ternaryfunc)(PyObject *, PyObject *, PyObject *);
 inquiry = Signature("T", "i")              # typedef int (*inquiry)(PyObject *);
-lenfunc = Signature("T", "Z")              # typedef Py_ssize_t (*lenfunc)(PyObject *);
+lenfunc = Signature("T", "z")              # typedef Py_ssize_t (*lenfunc)(PyObject *);
 
                                            # typedef int (*coercion)(PyObject **, PyObject **);
 intargfunc = Signature("Ti", "O")          # typedef PyObject *(*intargfunc)(PyObject *, int);
-ssizeargfunc = Signature("TZ", "O")        # typedef PyObject *(*ssizeargfunc)(PyObject *, Py_ssize_t);
+ssizeargfunc = Signature("Tz", "O")        # typedef PyObject *(*ssizeargfunc)(PyObject *, Py_ssize_t);
 intintargfunc = Signature("Tii", "O")      # typedef PyObject *(*intintargfunc)(PyObject *, int, int);
-ssizessizeargfunc = Signature("TZZ", "O")  # typedef PyObject *(*ssizessizeargfunc)(PyObject *, Py_ssize_t, Py_ssize_t);
+ssizessizeargfunc = Signature("Tzz", "O")  # typedef PyObject *(*ssizessizeargfunc)(PyObject *, Py_ssize_t, Py_ssize_t);
 intobjargproc = Signature("TiO", 'r')      # typedef int(*intobjargproc)(PyObject *, int, PyObject *);
-ssizeobjargproc = Signature("TZO", 'r')    # typedef int(*ssizeobjargproc)(PyObject *, Py_ssize_t, PyObject *);
+ssizeobjargproc = Signature("TzO", 'r')    # typedef int(*ssizeobjargproc)(PyObject *, Py_ssize_t, PyObject *);
 intintobjargproc = Signature("TiiO", 'r')  # typedef int(*intintobjargproc)(PyObject *, int, int, PyObject *);
-ssizessizeobjargproc = Signature("TZZO", 'r') # typedef int(*ssizessizeobjargproc)(PyObject *, Py_ssize_t, Py_ssize_t, PyObject *);
+ssizessizeobjargproc = Signature("TzzO", 'r') # typedef int(*ssizessizeobjargproc)(PyObject *, Py_ssize_t, Py_ssize_t, PyObject *);
 
 intintargproc = Signature("Tii", 'r')
-ssizessizeargproc = Signature("TZZ", 'r')
+ssizessizeargproc = Signature("Tzz", 'r')
 objargfunc = Signature("TO", "O")
 objobjargproc = Signature("TOO", 'r')      # typedef int (*objobjargproc)(PyObject *, PyObject *, PyObject *);
-getreadbufferproc = Signature("TiP", 'i')  # typedef int (*getreadbufferproc)(PyObject *, int, void **);
-getwritebufferproc = Signature("TiP", 'i') # typedef int (*getwritebufferproc)(PyObject *, int, void **);
-getsegcountproc = Signature("TI", 'i')     # typedef int (*getsegcountproc)(PyObject *, int *);
-getcharbufferproc = Signature("TiS", 'i')  # typedef int (*getcharbufferproc)(PyObject *, int, const char **);
-readbufferproc = Signature("TZP", "Z")     # typedef Py_ssize_t (*readbufferproc)(PyObject *, Py_ssize_t, void **);
-writebufferproc = Signature("TZP", "Z")    # typedef Py_ssize_t (*writebufferproc)(PyObject *, Py_ssize_t, void **);
-segcountproc = Signature("TZ", "Z")        # typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *);
-charbufferproc = Signature("TZS", "Z")     # typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **);
+readbufferproc = Signature("TzP", "z")     # typedef Py_ssize_t (*readbufferproc)(PyObject *, Py_ssize_t, void **);
+writebufferproc = Signature("TzP", "z")    # typedef Py_ssize_t (*writebufferproc)(PyObject *, Py_ssize_t, void **);
+segcountproc = Signature("TZ", "z")        # typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *);
+charbufferproc = Signature("TzS", "z")     # typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **);
 objargproc = Signature("TO", 'r')          # typedef int (*objobjproc)(PyObject *, PyObject *);
                                            # typedef int (*visitproc)(PyObject *, void *);
                                            # typedef int (*traverseproc)(PyObject *, visitproc, void *);
@@ -625,10 +623,10 @@ PyMappingMethods = (
 )
 
 PyBufferProcs = (
-    MethodSlot(getreadbufferproc, "bf_getreadbuffer", "__getreadbuffer__", py3 = False),
-    MethodSlot(getwritebufferproc, "bf_getwritebuffer", "__getwritebuffer__", py3 = False),
-    MethodSlot(getsegcountproc, "bf_getsegcount", "__getsegcount__", py3 = False),
-    MethodSlot(getcharbufferproc, "bf_getcharbuffer", "__getcharbuffer__", py3 = False),
+    MethodSlot(readbufferproc, "bf_getreadbuffer", "__getreadbuffer__", py3 = False),
+    MethodSlot(writebufferproc, "bf_getwritebuffer", "__getwritebuffer__", py3 = False),
+    MethodSlot(segcountproc, "bf_getsegcount", "__getsegcount__", py3 = False),
+    MethodSlot(charbufferproc, "bf_getcharbuffer", "__getcharbuffer__", py3 = False),
 
     MethodSlot(getbufferproc, "bf_getbuffer", "__getbuffer__", ifdef = "PY_VERSION_HEX >= 0x02060000"),
     MethodSlot(releasebufferproc, "bf_releasebuffer", "__releasebuffer__", ifdef = "PY_VERSION_HEX >= 0x02060000")

Cython/Distutils/build_ext.py

@@ -36,6 +36,8 @@ class build_ext(_build_ext.build_ext):
          "generate C++ source files"),
         ('pyrex-create-listing', None,
          "write errors to a listing file"),
+        ('pyrex-line-directives', None,
+         "emit source line directives"),
         ('pyrex-include-dirs=', None,
          "path to the Cython include files" + sep_by),
         ('pyrex-c-in-temp', None,
@@ -45,13 +47,14 @@ class build_ext(_build_ext.build_ext):
         ])
 
     boolean_options.extend([
-        'pyrex-cplus', 'pyrex-create-listing', 'pyrex-c-in-temp'
+        'pyrex-cplus', 'pyrex-create-listing', 'pyrex-line-directives', 'pyrex-c-in-temp'
     ])
 
     def initialize_options(self):
         _build_ext.build_ext.initialize_options(self)
         self.pyrex_cplus = 0
         self.pyrex_create_listing = 0
+        self.pyrex_line_directives = 0
         self.pyrex_include_dirs = None
         self.pyrex_c_in_temp = 0
         self.pyrex_gen_pxi = 0
@@ -114,6 +117,8 @@ class build_ext(_build_ext.build_ext):
         
         create_listing = self.pyrex_create_listing or \
             getattr(extension, 'pyrex_create_listing', 0)
+        line_directives = self.pyrex_line_directives or \
+            getattr(extension, 'pyrex_line_directives', 0)
         cplus = self.pyrex_cplus or getattr(extension, 'pyrex_cplus', 0) or \
                 (extension.language and extension.language.lower() == 'c++')
         pyrex_gen_pxi = self.pyrex_gen_pxi or getattr(extension, 'pyrex_gen_pxi', 0)
@@ -186,6 +191,7 @@ class build_ext(_build_ext.build_ext):
                     include_path = includes,
                     output_file = target,
                     cplus = cplus,
+                    emit_linenums = line_directives,
                     generate_pxi = pyrex_gen_pxi)
                 result = cython_compile(source, options=options,
                                         full_module_name=module_name)

Cython/Distutils/extension.py

@@ -21,6 +21,8 @@ class Extension(_Extension.Extension):
         Unix form for portability)
     pyrex_create_listing_file : boolean
         write pyrex error messages to a listing (.lis) file.
+    pyrex_line_directivess : boolean
+        emit pyx line numbers for debugging/profiling
     pyrex_cplus : boolean
         use the C++ compiler for compiling and linking.
     pyrex_c_in_temp : boolean
@@ -47,6 +49,7 @@ class Extension(_Extension.Extension):
             language = None,
             pyrex_include_dirs = None,
             pyrex_create_listing = 0,
+            pyrex_line_directives = 0,
             pyrex_cplus = 0,
             pyrex_c_in_temp = 0,
             pyrex_gen_pxi = 0,
@@ -70,6 +73,7 @@ class Extension(_Extension.Extension):
 
         self.pyrex_include_dirs = pyrex_include_dirs or []
         self.pyrex_create_listing = pyrex_create_listing
+        self.pyrex_line_directives = pyrex_line_directives
         self.pyrex_cplus = pyrex_cplus
         self.pyrex_c_in_temp = pyrex_c_in_temp
         self.pyrex_gen_pxi = pyrex_gen_pxi

Cython/Includes/cpython/object.pxd

 from cpython.ref cimport PyObject, PyTypeObject
-from stdio cimport FILE
+from libc.stdio cimport FILE
 
 cdef extern from "Python.h":
     

Cython/Includes/numpy.pxd

@@ -16,10 +16,10 @@
 
 DEF _buffer_format_string_len = 255
 
-cimport python_buffer as pybuf
-from python_ref cimport PyObject, Py_INCREF, Py_XDECREF
-cimport stdlib
-cimport stdio
+cimport cpython.buffer as pybuf
+from cpython cimport PyObject, Py_INCREF, Py_XDECREF
+from libc cimport stdlib
+from libc cimport stdio
 
 cdef extern from "Python.h":
     ctypedef int Py_intptr_t

tests/compile/c_directives.pyx

@@ -6,7 +6,7 @@
 
 print 3
 
-cimport python_dict as asadf, python_exc, cython as cy
+cimport cython as cy
 
 def e(object[int, ndim=2] buf):
     print buf[3, 2] # no bc

tests/compile/shipped_pxds.pyx

-cimport python_bool
-cimport python_buffer
-cimport python_bytes
-cimport python_cobject
-cimport python_complex
-cimport python_dict
-cimport python_exc
-cimport python_float
-cimport python_function
-cimport python_getargs
-cimport python_instance
-cimport python_int
-cimport python_iterator
-cimport python_list
-cimport python_long
-cimport python_mapping
-cimport python_mem
-cimport python_method
-cimport python_module
-cimport python_number
-cimport python_object
-cimport python
-cimport python_pycapsule
-cimport python_ref
-cimport python_sequence
-cimport python_set
-cimport python_string
-cimport python_tuple
-cimport python_type
-cimport python_unicode
-cimport python_version
-cimport python_weakref
-cimport stdio
-cimport stdlib

tests/run/bufaccess.pyx

@@ -9,12 +9,12 @@
 
 from __future__ import unicode_literals
 
-cimport stdlib
-cimport python_buffer
-cimport stdio
+from libc cimport stdlib
+from libc cimport stdio
+cimport cpython.buffer
 cimport cython
 
-from python_ref cimport PyObject
+from cpython cimport PyObject, Py_INCREF, Py_DECREF
 
 __test__ = {}
 
@@ -871,7 +871,6 @@ def printbuf_td_h_double(object[td_h_double] buf, shape):
 #
 # Object access
 #
-from python_ref cimport Py_INCREF, Py_DECREF
 def addref(*args):
     for item in args: Py_INCREF(item)
 def decref(*args):
@@ -983,13 +982,13 @@ def buffer_cast_fails(object[char, cast=True] buf):
 
 
 available_flags = (
-    ('FORMAT', python_buffer.PyBUF_FORMAT),
-    ('INDIRECT', python_buffer.PyBUF_INDIRECT),
-    ('ND', python_buffer.PyBUF_ND),
-    ('STRIDES', python_buffer.PyBUF_STRIDES),
-    ('C_CONTIGUOUS', python_buffer.PyBUF_C_CONTIGUOUS),
-    ('F_CONTIGUOUS', python_buffer.PyBUF_F_CONTIGUOUS),
-    ('WRITABLE', python_buffer.PyBUF_WRITABLE)
+    ('FORMAT', cpython.buffer.PyBUF_FORMAT),
+    ('INDIRECT', cpython.buffer.PyBUF_INDIRECT),
+    ('ND', cpython.buffer.PyBUF_ND),
+    ('STRIDES', cpython.buffer.PyBUF_STRIDES),
+    ('C_CONTIGUOUS', cpython.buffer.PyBUF_C_CONTIGUOUS),
+    ('F_CONTIGUOUS', cpython.buffer.PyBUF_F_CONTIGUOUS),
+    ('WRITABLE', cpython.buffer.PyBUF_WRITABLE)
 )
 
 cdef class MockBuffer:

tests/run/buffmt.pyx

@@ -7,7 +7,7 @@ def testcase(func):
     __test__[func.__name__] = func.__doc__
     return func
 
-cimport stdlib
+from libc cimport stdlib
 
 def little_endian():
     cdef unsigned int n = 1

tests/run/cython_includes.pyx

 
 from libc.stdio cimport sprintf
-from python cimport PyType_Check
-from python_type cimport PyType_Check as PyType_Check2
+from cpython cimport PyType_Check
+from cpython cimport PyType_Check as PyType_Check2
 from cpython.type cimport PyType_Check as PyType_Check3
 
 def libc_cimports():

tests/run/in_list_with_side_effects_T544.pyx

+
+def count(i=[0]):
+    i[0] += 1
+    return i[0]
+
+def test(x):
+    """
+    >>> def py_count(i=[0]):
+    ...     i[0] += 1
+    ...     return i[0]
+    >>> 1 in (py_count(), py_count(), py_count(), py_count())
+    True
+    >>> 4 in (py_count(), py_count(), py_count(), py_count())
+    False
+    >>> 12 in (py_count(), py_count(), py_count(), py_count())
+    True
+
+    >>> test(1)
+    True
+    >>> test(4)
+    False
+    >>> test(12)
+    True
+    """
+    return x in (count(), count(), count(), count())

tests/run/index.pyx

@@ -126,3 +126,10 @@ def test_boundscheck(list L, tuple t, object o, unsigned long ix):
     IndexError: string index out of range
     """
     return L[ix], t[ix], o[ix]
+
+def large_literal_index(object o):
+    """
+    >>> large_literal_index({1000000000000000000000000000000: True})
+    True
+    """
+    return o[1000000000000000000000000000000]

tests/run/inplace.pyx

@@ -30,7 +30,7 @@ def h(double a, double b):
     a *= b
     return a
 
-cimport stdlib
+from libc cimport stdlib
 
 def arrays():
     """

tests/run/r_pythonapi.pyx

@@ -11,7 +11,7 @@ if sys.version_info[0] >= 3:
 cdef extern from "string.h":
     void memcpy(char *d, char *s, int n)
 
-from python_unicode cimport PyUnicode_DecodeUTF8
+from cpython cimport PyUnicode_DecodeUTF8
     
 def spam():
     cdef char buf[12]

tests/run/type_inference.pyx

@@ -417,3 +417,18 @@ def common_extension_type_base():
     w = A()
     w = CC()
     assert typeof(w) == "Python object", typeof(w)
+
+
+@infer_types(None)
+def large_literals():
+    """
+    >>> large_literals()
+    """
+    # It's only safe to infer small integer literals.
+    a = 10
+    b = 100000000000000000000000000000000
+    assert typeof(a) == "long", typeof(a)
+    assert typeof(b) == "Python object", typeof(b)
+    c, d = 10, 100000000000000000000000000000000
+    assert typeof(c) == "long", typeof(c)
+    assert typeof(d) == "Python object", typeof(d)