Merge branch 'master' into grammar

CHANGES.rst

@@ -9,6 +9,8 @@ Latest
 Features added
 --------------
 
+* C (cdef) functions allow inner Python functions.
+
 * Enums can now be declared as cpdef to export their values to
   the module's Python namespace.  Cpdef enums in pxd files export
   their values to their own module, iff it exists.

Cython/Compiler/ExprNodes.py

@@ -7126,7 +7126,7 @@ class SortedDictKeysNode(ExprNode):
         else:
             # originally used PyMapping_Keys() here, but that may return a tuple
             code.globalstate.use_utility_code(UtilityCode.load_cached(
-                'PyObjectCallMethod', 'ObjectHandling.c'))
+                'PyObjectCallMethod0', 'ObjectHandling.c'))
             keys_cname = code.intern_identifier(StringEncoding.EncodedString("keys"))
             code.putln('%s = __Pyx_PyObject_CallMethod0(%s, %s); %s' % (
                 self.result(), dict_result, keys_cname,

Cython/Compiler/PyrexTypes.py

@@ -3153,10 +3153,8 @@ class CppClassType(CType):
             if for_display:
                 brackets = "[%s]"
             else:
-                brackets = "<%s>"
+                brackets = "<%s> "
             templates = brackets % ",".join(template_strings)
-            if templates[-2:] == ">>":
-                templates = templates[:-2] + "> >"
         else:
             templates = ""
         if pyrex or for_display:

Cython/Utility/Builtins.c

@@ -250,7 +250,7 @@ static CYTHON_INLINE unsigned PY_LONG_LONG __Pyx_abs_longlong(PY_LONG_LONG x) {
 static CYTHON_INLINE PyObject* __Pyx_PyDict_Keys(PyObject* d); /*proto*/
 
 //////////////////// py_dict_keys ////////////////////
-//@requires: ObjectHandling.c::PyObjectCallMethod
+//@requires: ObjectHandling.c::PyObjectCallMethod1
 
 static CYTHON_INLINE PyObject* __Pyx_PyDict_Keys(PyObject* d) {
     if (PY_MAJOR_VERSION >= 3)
@@ -264,7 +264,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyDict_Keys(PyObject* d) {
 static CYTHON_INLINE PyObject* __Pyx_PyDict_Values(PyObject* d); /*proto*/
 
 //////////////////// py_dict_values ////////////////////
-//@requires: ObjectHandling.c::PyObjectCallMethod
+//@requires: ObjectHandling.c::PyObjectCallMethod1
 
 static CYTHON_INLINE PyObject* __Pyx_PyDict_Values(PyObject* d) {
     if (PY_MAJOR_VERSION >= 3)
@@ -278,7 +278,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyDict_Values(PyObject* d) {
 static CYTHON_INLINE PyObject* __Pyx_PyDict_Items(PyObject* d); /*proto*/
 
 //////////////////// py_dict_items ////////////////////
-//@requires: ObjectHandling.c::PyObjectCallMethod
+//@requires: ObjectHandling.c::PyObjectCallMethod1
 
 static CYTHON_INLINE PyObject* __Pyx_PyDict_Items(PyObject* d) {
     if (PY_MAJOR_VERSION >= 3)
@@ -292,7 +292,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyDict_Items(PyObject* d) {
 static CYTHON_INLINE PyObject* __Pyx_PyDict_IterKeys(PyObject* d); /*proto*/
 
 //////////////////// py_dict_iterkeys ////////////////////
-//@requires: ObjectHandling.c::PyObjectCallMethod
+//@requires: ObjectHandling.c::PyObjectCallMethod0
 
 static CYTHON_INLINE PyObject* __Pyx_PyDict_IterKeys(PyObject* d) {
     return __Pyx_PyObject_CallMethod0(d, (PY_MAJOR_VERSION >= 3) ? PYIDENT("keys") : PYIDENT("iterkeys"));
@@ -303,7 +303,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyDict_IterKeys(PyObject* d) {
 static CYTHON_INLINE PyObject* __Pyx_PyDict_IterValues(PyObject* d); /*proto*/
 
 //////////////////// py_dict_itervalues ////////////////////
-//@requires: ObjectHandling.c::PyObjectCallMethod
+//@requires: ObjectHandling.c::PyObjectCallMethod0
 
 static CYTHON_INLINE PyObject* __Pyx_PyDict_IterValues(PyObject* d) {
     return __Pyx_PyObject_CallMethod0(d, (PY_MAJOR_VERSION >= 3) ? PYIDENT("values") : PYIDENT("itervalues"));
@@ -314,7 +314,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyDict_IterValues(PyObject* d) {
 static CYTHON_INLINE PyObject* __Pyx_PyDict_IterItems(PyObject* d); /*proto*/
 
 //////////////////// py_dict_iteritems ////////////////////
-//@requires: ObjectHandling.c::PyObjectCallMethod
+//@requires: ObjectHandling.c::PyObjectCallMethod0
 
 static CYTHON_INLINE PyObject* __Pyx_PyDict_IterItems(PyObject* d) {
     return __Pyx_PyObject_CallMethod0(d, (PY_MAJOR_VERSION >= 3) ? PYIDENT("items") : PYIDENT("iteritems"));
@@ -328,7 +328,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyDict_IterItems(PyObject* d) {
 static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewKeys(PyObject* d); /*proto*/
 
 //////////////////// py_dict_viewkeys ////////////////////
-//@requires: ObjectHandling.c::PyObjectCallMethod
+//@requires: ObjectHandling.c::PyObjectCallMethod0
 
 static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewKeys(PyObject* d) {
     return __Pyx_PyObject_CallMethod0(d, (PY_MAJOR_VERSION >= 3) ? PYIDENT("keys") : PYIDENT("viewkeys"));
@@ -342,7 +342,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewKeys(PyObject* d) {
 static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewValues(PyObject* d); /*proto*/
 
 //////////////////// py_dict_viewvalues ////////////////////
-//@requires: ObjectHandling.c::PyObjectCallMethod
+//@requires: ObjectHandling.c::PyObjectCallMethod0
 
 static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewValues(PyObject* d) {
     return __Pyx_PyObject_CallMethod0(d, (PY_MAJOR_VERSION >= 3) ? PYIDENT("values") : PYIDENT("viewvalues"));
@@ -356,7 +356,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewValues(PyObject* d) {
 static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewItems(PyObject* d); /*proto*/
 
 //////////////////// py_dict_viewitems ////////////////////
-//@requires: ObjectHandling.c::PyObjectCallMethod
+//@requires: ObjectHandling.c::PyObjectCallMethod0
 
 static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewItems(PyObject* d) {
     return __Pyx_PyObject_CallMethod0(d, (PY_MAJOR_VERSION >= 3) ? PYIDENT("items") : PYIDENT("viewitems"));

Cython/Utility/Generator.c

@@ -59,7 +59,7 @@ static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue);
 //@requires: Exceptions.c::PyErrFetchRestore
 //@requires: Exceptions.c::SwapException
 //@requires: Exceptions.c::RaiseException
-//@requires: ObjectHandling.c::PyObjectCallMethod
+//@requires: ObjectHandling.c::PyObjectCallMethod1
 //@requires: CommonTypes.c::FetchCommonType
 
 static PyObject *__Pyx_Generator_Next(PyObject *self);

Cython/Utility/ObjectHandling.c

@@ -1092,31 +1092,123 @@ static CYTHON_INLINE int __Pyx_PyObject_SetAttrStr(PyObject* obj, PyObject* attr
 #define __Pyx_PyObject_SetAttrStr(o,n,v) PyObject_SetAttr(o,n,v)
 #endif
 
-/////////////// PyObjectCallMethod.proto ///////////////
+
+/////////////// PyObjectCallMethod0.proto ///////////////
+
+static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name); /*proto*/
+
+/////////////// PyObjectCallMethod0 ///////////////
 //@requires: PyObjectGetAttrStr
-//@requires: PyObjectCall
-//@substitute: naming
+//@requires: PyObjectCallOneArg
+//@requires: PyObjectCallNoArg
 
-static PyObject* __Pyx_PyObject_CallMethodTuple(PyObject* obj, PyObject* method_name, PyObject* args) {
+static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name) {
     PyObject *method, *result = NULL;
-    if (unlikely(!args)) return NULL;
     method = __Pyx_PyObject_GetAttrStr(obj, method_name);
     if (unlikely(!method)) goto bad;
-    result = __Pyx_PyObject_Call(method, args, NULL);
+#if CYTHON_COMPILING_IN_CPYTHON
+    if (likely(PyMethod_Check(method))) {
+        PyObject *self = PyMethod_GET_SELF(method);
+        if (likely(self)) {
+            PyObject *function = PyMethod_GET_FUNCTION(method);
+            result = __Pyx_PyObject_CallOneArg(function, self);
+            Py_DECREF(method);
+            return result;
+        }
+    }
+#endif
+    result = __Pyx_PyObject_CallNoArg(method);
     Py_DECREF(method);
 bad:
-    Py_DECREF(args);
     return result;
 }
 
-#define __Pyx_PyObject_CallMethod3(obj, name, arg1, arg2, arg3) \
-    __Pyx_PyObject_CallMethodTuple(obj, name, PyTuple_Pack(3, arg1, arg2, arg3))
-#define __Pyx_PyObject_CallMethod2(obj, name, arg1, arg2) \
-    __Pyx_PyObject_CallMethodTuple(obj, name, PyTuple_Pack(2, arg1, arg2))
-#define __Pyx_PyObject_CallMethod1(obj, name, arg1) \
-    __Pyx_PyObject_CallMethodTuple(obj, name, PyTuple_Pack(1, arg1))
-#define __Pyx_PyObject_CallMethod0(obj, name) \
-    __Pyx_PyObject_CallMethodTuple(obj, name, (Py_INCREF($empty_tuple), $empty_tuple))
+
+/////////////// PyObjectCallMethod1.proto ///////////////
+
+static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg); /*proto*/
+
+/////////////// PyObjectCallMethod1 ///////////////
+//@requires: PyObjectGetAttrStr
+//@requires: PyObjectCallOneArg
+
+static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg) {
+    PyObject *method, *result = NULL;
+    method = __Pyx_PyObject_GetAttrStr(obj, method_name);
+    if (unlikely(!method)) goto bad;
+#if CYTHON_COMPILING_IN_CPYTHON
+    if (likely(PyMethod_Check(method))) {
+        PyObject *self = PyMethod_GET_SELF(method);
+        if (likely(self)) {
+            PyObject *args;
+            PyObject *function = PyMethod_GET_FUNCTION(method);
+            args = PyTuple_New(2);
+            if (unlikely(!args)) goto bad;
+            Py_INCREF(self);
+            PyTuple_SET_ITEM(args, 0, self);
+            Py_INCREF(arg);
+            PyTuple_SET_ITEM(args, 1, arg);
+            Py_INCREF(function);
+            Py_DECREF(method); method = NULL;
+            result = __Pyx_PyObject_Call(function, args, NULL);
+            Py_DECREF(args);
+            Py_DECREF(function);
+            return result;
+        }
+    }
+#endif
+    result = __Pyx_PyObject_CallOneArg(method, arg);
+bad:
+    Py_XDECREF(method);
+    return result;
+}
+
+
+/////////////// PyObjectCallMethod2.proto ///////////////
+
+static PyObject* __Pyx_PyObject_CallMethod2(PyObject* obj, PyObject* method_name, PyObject* arg1, PyObject* arg2); /*proto*/
+
+/////////////// PyObjectCallMethod2 ///////////////
+//@requires: PyObjectGetAttrStr
+//@requires: PyObjectCall
+
+static PyObject* __Pyx_PyObject_CallMethod2(PyObject* obj, PyObject* method_name, PyObject* arg1, PyObject* arg2) {
+    PyObject *args, *method, *result = NULL;
+    method = __Pyx_PyObject_GetAttrStr(obj, method_name);
+#if CYTHON_COMPILING_IN_CPYTHON
+    if (likely(PyMethod_Check(method)) && likely(PyMethod_GET_SELF(method))) {
+        PyObject *self, *function;
+        self = PyMethod_GET_SELF(method);
+        function = PyMethod_GET_FUNCTION(method);
+        args = PyTuple_New(3);
+        if (unlikely(!args)) goto bad;
+        Py_INCREF(self);
+        PyTuple_SET_ITEM(args, 0, self);
+        Py_INCREF(arg1);
+        PyTuple_SET_ITEM(args, 1, arg1);
+        Py_INCREF(arg2);
+        PyTuple_SET_ITEM(args, 2, arg2);
+        Py_INCREF(function);
+        Py_DECREF(method);
+        method = function;
+    } else
+#endif
+    {
+        args = PyTuple_New(2);
+        if (unlikely(!args)) goto bad;
+        Py_INCREF(arg1);
+        PyTuple_SET_ITEM(args, 0, arg1);
+        Py_INCREF(arg2);
+        PyTuple_SET_ITEM(args, 1, arg2);
+    }
+    result = __Pyx_PyObject_Call(method, args, NULL);
+    Py_DECREF(args);
+    Py_DECREF(method);
+    return result;
+bad:
+    Py_XDECREF(method);
+    return result;
+}
 
 
 /////////////// tp_new.proto ///////////////
@@ -1223,6 +1315,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObjec
 }
 #endif
 
+
 /////////////// PyObjectCallNoArg.proto ///////////////
 //@requires: PyObjectCall
 //@substitute: naming

Cython/Utility/Optimize.c

@@ -12,7 +12,7 @@ static CYTHON_INLINE int __Pyx_PyObject_Append(PyObject* L, PyObject* x); /*prot
 
 /////////////// append ///////////////
 //@requires: ListAppend
-//@requires: ObjectHandling.c::PyObjectCallMethod
+//@requires: ObjectHandling.c::PyObjectCallMethod1
 
 static CYTHON_INLINE int __Pyx_PyObject_Append(PyObject* L, PyObject* x) {
     if (likely(PyList_CheckExact(L))) {
@@ -85,7 +85,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyList_Pop(PyObject* L); /*proto*/
 static CYTHON_INLINE PyObject* __Pyx__PyObject_Pop(PyObject* L); /*proto*/
 
 /////////////// pop ///////////////
-//@requires: ObjectHandling.c::PyObjectCallMethod
+//@requires: ObjectHandling.c::PyObjectCallMethod0
 
 static CYTHON_INLINE PyObject* __Pyx__PyObject_Pop(PyObject* L) {
 #if CYTHON_COMPILING_IN_CPYTHON
@@ -122,7 +122,7 @@ static PyObject* __Pyx__PyList_PopIndex(PyObject* L, Py_ssize_t ix); /*proto*/
 static PyObject* __Pyx__PyObject_PopIndex(PyObject* L, PyObject* py_ix); /*proto*/
 
 /////////////// pop_index ///////////////
-//@requires: ObjectHandling.c::PyObjectCallMethod
+//@requires: ObjectHandling.c::PyObjectCallMethod1
 
 static PyObject* __Pyx__PyObject_PopIndex(PyObject* L, PyObject* py_ix) {
     PyObject *r;
@@ -193,7 +193,7 @@ static PyObject* __Pyx_PyDict_GetItemDefault(PyObject* d, PyObject* key, PyObjec
 static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value, int is_safe_type); /*proto*/
 
 /////////////// dict_setdefault ///////////////
-//@requires: ObjectHandling.c::PyObjectCallMethod
+//@requires: ObjectHandling.c::PyObjectCallMethod2
 
 static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value,
                                                        CYTHON_UNUSED int is_safe_type) {
@@ -250,7 +250,7 @@ static CYTHON_INLINE int __Pyx_dict_iter_next(PyObject* dict_or_iter, Py_ssize_t
 /////////////// dict_iter ///////////////
 //@requires: ObjectHandling.c::UnpackTuple2
 //@requires: ObjectHandling.c::IterFinish
-//@requires: ObjectHandling.c::PyObjectCallMethod
+//@requires: ObjectHandling.c::PyObjectCallMethod0
 
 static CYTHON_INLINE PyObject* __Pyx_dict_iterator(PyObject* iterable, int is_dict, PyObject* method_name,
                                                    Py_ssize_t* p_orig_length, int* p_source_is_dict) {

Cython/Utility/StringTools.c

@@ -747,7 +747,7 @@ static CYTHON_INLINE int __Pyx_PyByteArray_AppendObject(PyObject* bytearray, PyO
 static CYTHON_INLINE int __Pyx_PyByteArray_Append(PyObject* bytearray, int value);
 
 //////////////////// ByteArrayAppend ////////////////////
-//@requires: ObjectHandling.c::PyObjectCallMethod
+//@requires: ObjectHandling.c::PyObjectCallMethod1
 
 static CYTHON_INLINE int __Pyx_PyByteArray_Append(PyObject* bytearray, int value) {
     PyObject *pyval, *retval;

Cython/Utility/arrayarray.h

@@ -27,7 +27,7 @@ typedef struct arraydescr {
     int itemsize;
     PyObject * (*getitem)(struct arrayobject *, Py_ssize_t);
     int (*setitem)(struct arrayobject *, Py_ssize_t, PyObject *);
-#if PY_VERSION_HEX >= 0x03000000
+#if PY_MAJOR_VERSION >= 3
     char *formats;
 #endif    
 } arraydescr;
@@ -55,7 +55,7 @@ struct arrayobject {
     Py_ssize_t allocated;
     struct arraydescr *ob_descr;
     PyObject *weakreflist; /* List of weak references */
-#if PY_VERSION_HEX >= 0x03000000
+#if PY_MAJOR_VERSION >= 3
         int ob_exports;  /* Number of exported buffers */
 #endif
 };
@@ -118,7 +118,6 @@ static CYTHON_INLINE int resize(arrayobject *self, Py_ssize_t n) {
 }
 
 // suitable for small increments; over allocation 50% ;
-// Remains non-smart in Python 2.3- ; but exists for compatibility
 static CYTHON_INLINE int resize_smart(arrayobject *self, Py_ssize_t n) {
     void *items = (void*) self->data.ob_item;
     Py_ssize_t newsize;

docs/src/tutorial/memory_allocation.rst

@@ -85,15 +85,15 @@ e.g.::
 
       cdef double* data
 
-      def __cinit__(self, number):
-          # allocate some memory (filled with random data)
+      def __cinit__(self, size_t number):
+          # allocate some memory (uninitialised, may contain arbitrary data)
           self.data = <double*> PyMem_Malloc(number * sizeof(double))
           if not self.data:
               raise MemoryError()
 
-      def resize(self, new_number):
+      def resize(self, size_t new_number):
           # Allocates new_number * sizeof(double) bytes,
-          # preserving the contents and making a best-effort to
+          # preserving the current content and making a best-effort to
           # re-use the original data location.
           mem = <double*> PyMem_Realloc(self.data, new_number * sizeof(double))
           if not mem:

tests/compile/cpp_templates.pyx

@@ -16,6 +16,8 @@ cdef extern from "templates.h":
         T getValue1()
         U getValue2()
 
+    void template_function[T](TemplateTest1[T] &)
+
 cdef TemplateTest1[int] a
 cdef TemplateTest1[int]* b = new TemplateTest1[int]()
 
@@ -39,3 +41,11 @@ cdef TemplateTest1_int aa
 # Verify that T767 is fixed.
 cdef public int func(int arg):
     return arg
+
+# Regression test: the function call used to produce
+#   template_function<TemplateTest1<int>>(__pyx_v_t);
+# which is valid C++11, but not valid C++98 because the ">>" would be
+# parsed as a single token.
+cdef public void use_nested_templates():
+    cdef TemplateTest1[TemplateTest1[int]] t
+    template_function(t)

tests/compile/templates.h

@@ -22,4 +22,9 @@ public:
     U getValue2() { return value2; }
 };
 
+template <typename T>
+void template_function(TemplateTest1<T> &)
+{
+}
+
 #endif

tests/memoryview/memslice.pyx

@@ -1665,8 +1665,8 @@ cdef test_structs_with_arr(FusedStruct array[10]):
         for j in range(3):
             myslice1[i].chars[j] = 97 + j
 
-    if sys.version_info[:2] >= (2, 7) and sys.version_info[:2] < (3, 3):
-        size1 = sizeof(FusedStruct)
+    if (2, 7) <= sys.version_info[:2] < (3, 3):
+        size1 = <Py_ssize_t>sizeof(FusedStruct)
         size2 = len(builtins.memoryview(myslice1)[0])
         assert size1 == size2, (size1, size2, builtins.memoryview(myslice1).format)
 

tests/run/cpp_template_subclasses_helper.h

@@ -3,6 +3,7 @@ using namespace std;
 class Base {
 public:
     virtual const char* name() { return "Base"; }
+    virtual ~Base() {}
 };
 
 template <class A1>

tests/run/hash_T326.pyx

+# mode: run
 # ticket: 326
+# tag: hash
 
-__doc__ = u"""
 
+cdef class A:
+    """
     >>> hash(A(5))
     5
     >>> hash(A(-1))
@@ -12,10 +15,7 @@ __doc__ = u"""
     Traceback (most recent call last):
     ...
     TypeError: That's kind of a round number...
-
-"""
-
-cdef class A:
+    """
     cdef long a
     def __init__(self, a):
         self.a = a
@@ -25,6 +25,7 @@ cdef class A:
         else:
             return self.a
 
+
 cpdef long __hash__(long x):
     """
     >>> __hash__(-1)