Merge pull request #76 from markflorisson88/arrayinstruct

Buffer format arrays, strings and alignment on first member

Cython/Compiler/Buffer.py

@@ -649,7 +649,6 @@ class GetAndReleaseBufferUtilityCode(object):
         """))
 
 
-
 def mangle_dtype_name(dtype):
     # Use prefixes to seperate user defined types from builtins
     # (consider "typedef float unsigned_int")
@@ -662,16 +661,20 @@ def mangle_dtype_name(dtype):
             prefix = "nn_"
         else:
             prefix = ""
-        return prefix + dtype.declaration_code("").replace(" ", "_")
+        type_decl = dtype.declaration_code("")
+        type_decl = type_decl.replace(" ", "_")
+        return prefix + type_decl.replace("[", "_").replace("]", "_")
 
 def get_type_information_cname(code, dtype, maxdepth=None):
-    # Output the run-time type information (__Pyx_TypeInfo) for given dtype,
-    # and return the name of the type info struct.
-    #
-    # Structs with two floats of the same size are encoded as complex numbers.
-    # One can seperate between complex numbers declared as struct or with native
-    # encoding by inspecting to see if the fields field of the type is
-    # filled in.
+    """
+    Output the run-time type information (__Pyx_TypeInfo) for given dtype,
+    and return the name of the type info struct.
+
+    Structs with two floats of the same size are encoded as complex numbers.
+    One can seperate between complex numbers declared as struct or with native
+    encoding by inspecting to see if the fields field of the type is
+    filled in.
+    """
     namesuffix = mangle_dtype_name(dtype)
     name = "__Pyx_TypeInfo_%s" % namesuffix
     structinfo_name = "__Pyx_StructFields_%s" % namesuffix
@@ -688,6 +691,12 @@ def get_type_information_cname(code, dtype, maxdepth=None):
         code.globalstate.utility_codes.add(name)
         typecode = code.globalstate['typeinfo']
 
+        arraysizes = []
+        if dtype.is_array:
+            while dtype.is_array:
+                arraysizes.append(dtype.size)
+                dtype = dtype.base_type
+
         complex_possible = dtype.is_struct_or_union and dtype.can_be_complex()
 
         declcode = dtype.declaration_code("")
@@ -729,7 +738,6 @@ def get_type_information_cname(code, dtype, maxdepth=None):
         elif dtype.is_pyobject:
             typegroup = 'O'
         else:
-            print dtype
             assert False
 
         if dtype.is_int:
@@ -737,15 +745,13 @@ def get_type_information_cname(code, dtype, maxdepth=None):
         else:
             is_unsigned = "0"
 
-        typecode.putln(('static __Pyx_TypeInfo %s = { "%s", %s, sizeof(%s), \'%s\', %s, %s };'
-                        ) % (name,
-                             rep,
-                             structinfo_name,
-                             declcode,
-                             typegroup,
-                             is_unsigned,
-                             flags,
-                        ), safe=True)
+        typeinfo = ('static __Pyx_TypeInfo %s = '
+                        '{ "%s", %s, sizeof(%s), { %s }, %s, \'%s\', %s, %s };')
+        tup = (name, rep, structinfo_name, declcode,
+               ', '.join([str(x) for x in arraysizes]), len(arraysizes),
+               typegroup, is_unsigned, flags)
+        typecode.putln(typeinfo % tup, safe=True)
+
     return name
 
 def load_buffer_utility(util_code_name, context=None, **kwargs):

Cython/Compiler/ExprNodes.py

@@ -2548,6 +2548,7 @@ class IndexNode(ExprNode):
                         warning(index.pos, "Index should be typed for more "
                                            "efficient access", level=2)
                         IndexNode.warned_untyped_idx = True
+
                     self.memslice_index = True
                     index = index.coerce_to(index_type, env)
                     indices[i] = index
@@ -6935,8 +6936,9 @@ class CythonArrayNode(ExprNode):
                                                         "n" * len(shapes),
                                                         ", ".join(shapes)))
 
-        err = "!%s || !%s || !PyBytes_Check(%s)" % (format_temp, shapes_temp,
-                                                    format_temp)
+        err = "!%s || !%s || !PyBytes_AsString(%s)" % (format_temp,
+                                                       shapes_temp,
+                                                       format_temp)
         code.putln(code.error_goto_if(err, self.pos))
         code.put_gotref(format_temp)
         code.put_gotref(shapes_temp)

Cython/Compiler/MemoryView.py

@@ -157,7 +157,7 @@ def src_conforms_to_dst(src, dst):
 
     return True
 
-def valid_memslice_dtype(dtype):
+def valid_memslice_dtype(dtype, i=0):
     """
     Return whether type dtype can be used as the base type of a
     memoryview slice.
@@ -178,6 +178,8 @@ def valid_memslice_dtype(dtype):
         dtype.is_error or
         # Pointers are not valid (yet)
         # (dtype.is_ptr and valid_memslice_dtype(dtype.base_type)) or
+        (dtype.is_array and i < 8 and
+         valid_memslice_dtype(dtype.base_type, i + 1)) or
         dtype.is_numeric or
         dtype.is_pyobject or
         dtype.is_fused or # accept this as it will be replaced by specializations later

Cython/Utility/MemoryView.pyx

@@ -394,7 +394,6 @@ cdef class memoryview(object):
         info.readonly = 0
         info.obj = self
 
-
     # Some properties that have the same sematics as in NumPy
     property T:
         @cname('__pyx_memoryview_transpose')
@@ -920,6 +919,8 @@ cdef extern from *:
       char* name
       __Pyx_StructField* fields
       size_t size
+      size_t arraysize[8]
+      int ndim
       char typegroup
       char is_unsigned
       int flags
@@ -945,6 +946,11 @@ cdef extern from *:
 
 @cname('__pyx_format_from_typeinfo')
 cdef format_from_typeinfo(__Pyx_TypeInfo *type):
+    """
+    We want to return bytes, but python 3 doesn't allow you to do anything
+    useful with bytes. So use str and convert back and forth to/from unicode.
+    Thank you python 3 for making bytes the most useless thing ever!
+    """
     cdef __Pyx_StructField *field
     cdef __pyx_typeinfo_string fmt
 
@@ -960,12 +966,16 @@ cdef format_from_typeinfo(__Pyx_TypeInfo *type):
         field = type.fields
 
         while field.type:
-            parts.append(format_from_typeinfo(field.type))
+            parts.append(format_from_typeinfo(field.type).decode('ascii'))
             field += 1
 
         result = alignment.join(parts) + '}'
     else:
         fmt = __Pyx_TypeInfoToFormat(type)
-        result = fmt.string
+        if type.arraysize[0]:
+            extents = [str(type.arraysize[i]) for i in range(type.ndim)]
+            result = "(%s)%s" % (','.join(extents), fmt.string.decode('ascii'))
+        else:
+            result = fmt.string.decode('ascii')
 
-    return result
+    return result.encode('ascii')

tests/errors/memview_declarations.pyx

@@ -46,6 +46,14 @@ cdef intp[:, :] myarray
 cdef int[:] a10 = <int[:10]> object()
 cdef int[:] a11 = <int[:5.4]> <int *> 1
 
+cdef struct Valid:
+    int array[1][2][3][4][5][6][7][8]
+cdef struct Invalid:
+    int array[1][2][3][4][5][6][7][8][9]
+
+cdef Valid[:] validslice
+cdef Invalid[:] invalidslice
+
 # These are VALID
 cdef int[::view.indirect_contiguous, ::view.contiguous] a9
 
@@ -70,4 +78,5 @@ _ERRORS = u'''
 44:10: Invalid base type for memoryview slice: intp
 46:35: Can only create cython.array from pointer or array
 47:24: Cannot assign type 'double' to 'Py_ssize_t'
+55:13: Invalid base type for memoryview slice: Invalid
 '''

tests/run/memslice.pyx

@@ -1456,6 +1456,151 @@ def test_memslice_prange(arg):
             for k in range(src.shape[2]):
                 assert src[i, j, k] == dst[i, j, k], (src[i, j, k] == dst[i, j, k])
 
+# Test arrays in structs
+cdef struct ArrayStruct:
+    int ints[10]
+    char chars[3]
+
+cdef packed struct PackedArrayStruct:
+    int ints[10]
+    char chars[3]
+
+cdef fused FusedStruct:
+    ArrayStruct
+    PackedArrayStruct
+
+@testcase
+def test_memslice_struct_with_arrays():
+    """
+    >>> test_memslice_struct_with_arrays()
+    abc
+    abc
+    """
+    cdef ArrayStruct a1[10]
+    cdef PackedArrayStruct a2[10]
+
+    test_structs_with_arr(a1)
+    test_structs_with_arr(a2)
+
+cdef test_structs_with_arr(FusedStruct array[10]):
+    cdef FusedStruct[:] myslice1, myslice2, myslice3, myslice4
+    cdef int i, j
+
+    myslice1 = <FusedStruct[:10]> array
+
+    for i in range(10):
+        for j in range(10):
+            myslice1[i].ints[j] = i
+        for j in range(3):
+            myslice1[i].chars[j] = 97 + j
+
+    if sys.version_info[:2] >= (2, 7):
+        if sys.version_info[0] < 3:
+            import __builtin__ as builtins
+        else:
+            import builtins
+
+        size1 = sizeof(FusedStruct)
+        size2 = len(builtins.memoryview(myslice1)[0])
+        assert size1 == size2, (size1, size2, builtins.memoryview(myslice1).format)
+
+        myslice2 = builtins.memoryview(myslice1)
+        for i in range(10):
+            assert myslice2[i].ints[i] == myslice1[i].ints[i]
+            assert myslice2[i].chars[i] == myslice1[i].chars[i]
+
+    myslice3 = <object> myslice1
+    myslice4 = myslice1
+    for i in range(10):
+        for j in range(10):
+            assert myslice3[i].ints[j] == myslice4[i].ints[j] == myslice1[i].ints[j]
+        for j in range(3):
+            assert myslice3[i].chars[j] == myslice4[i].chars[j] == myslice1[i].chars[j]
+
+    print myslice1[0].chars[:3].decode('ascii')
+
+# Test padding at the end of structs in the buffer support
+cdef struct PaddedAtEnd:
+    int a[3]
+    char b[3]
+
+cdef struct AlignedNested:
+    PaddedAtEnd a
+    char chars[1]
+
+cdef struct PaddedAtEndNormal:
+    int a
+    char b
+    char c
+    char d
+
+cdef struct AlignedNestedNormal:
+    PaddedAtEndNormal a
+    char chars
+
+# Test nested structs in a struct, make sure we compute padding each time
+# accordingly. If the first struct member is a struct, align on the first
+# member of that struct (recursively)
+cdef struct A:
+    double d
+    char c
+
+cdef struct B:
+    char c1
+    A a
+    char c2
+
+cdef struct C:
+    A a
+    char c1
+
+cdef struct D:
+    B b
+    C cstruct
+    int a[2]
+    char c
+
+cdef fused FusedPadded:
+    ArrayStruct
+    PackedArrayStruct
+    AlignedNested
+    AlignedNestedNormal
+    A
+    B
+    C
+    D
+
+@testcase
+def test_padded_structs():
+    """
+    >>> test_padded_structs()
+    """
+    cdef ArrayStruct a1[10]
+    cdef PackedArrayStruct a2[10]
+    cdef AlignedNested a3[10]
+    cdef AlignedNestedNormal a4[10]
+    cdef A a5[10]
+    cdef B a6[10]
+    cdef C a7[10]
+    cdef D a8[10]
+
+    _test_padded(a1)
+    _test_padded(a2)
+    _test_padded(a3)
+    _test_padded(a4)
+    _test_padded(a5)
+    _test_padded(a6)
+    _test_padded(a7)
+    # There is a pre-existing bug that doesn't parse the format for this
+    # struct properly -- fix this
+    #_test_padded(a8)
+
+cdef _test_padded(FusedPadded myarray[10]):
+    # test that the buffer format parser accepts our format string...
+    cdef FusedPadded[:] myslice = <FusedPadded[:10]> myarray
+    obj = myslice
+    cdef FusedPadded[:] myotherslice = obj
+
 @testcase
 def test_object_indices():
     """

tests/run/numpy_memoryview.pyx

@@ -391,3 +391,120 @@ def acquire_release_cycle(obj):
     del buf
 
     gc.collect()
+
+cdef packed struct StructArray:
+    int a[4]
+    char b[5]
+
+@testcase_numpy_1_5
+def test_memslice_structarray(data, dtype):
+    """
+    >>> data = [(range(4), 'spam\\0'), (range(4, 8), 'ham\\0\\0'), (range(8, 12), 'eggs\\0')]
+    >>> dtype = np.dtype([('a', '4i'), ('b', '5b')])
+    >>> test_memslice_structarray([(L, map(ord, s)) for L, s in data], dtype)
+    0
+    1
+    2
+    3
+    spam
+    4
+    5
+    6
+    7
+    ham
+    8
+    9
+    10
+    11
+    eggs
+
+    Test the same thing with the string format specifier
+
+    >>> dtype = np.dtype([('a', '4i'), ('b', 'S5')])
+    >>> test_memslice_structarray(data, dtype)
+    0
+    1
+    2
+    3
+    spam
+    4
+    5
+    6
+    7
+    ham
+    8
+    9
+    10
+    11
+    eggs
+    """
+    a = np.empty((3,), dtype=dtype)
+    a[:] = data
+    cdef StructArray[:] myslice = a
+    cdef int i, j
+    for i in range(3):
+        for j in range(4):
+            print myslice[i].a[j]
+        print myslice[i].b
+
+@testcase_numpy_1_5
+def test_structarray_errors(StructArray[:] a):
+    """
+    >>> dtype = np.dtype([('a', '4i'), ('b', '5b')])
+    >>> test_structarray_errors(np.empty((5,), dtype=dtype))
+
+    >>> dtype = np.dtype([('a', '6i'), ('b', '5b')])
+    >>> test_structarray_errors(np.empty((5,), dtype=dtype))
+    Traceback (most recent call last):
+       ...
+    ValueError: Expected a dimension of size 4, got 6
+
+    >>> dtype = np.dtype([('a', '(4,4)i'), ('b', '5b')])
+    >>> test_structarray_errors(np.empty((5,), dtype=dtype))
+    Traceback (most recent call last):
+       ...
+    ValueError: Expected 1 dimension(s), got 2
+
+    Test the same thing with the string format specifier
+
+    >>> dtype = np.dtype([('a', '4i'), ('b', 'S5')])
+    >>> test_structarray_errors(np.empty((5,), dtype=dtype))
+
+    >>> dtype = np.dtype([('a', '6i'), ('b', 'S5')])
+    >>> test_structarray_errors(np.empty((5,), dtype=dtype))
+    Traceback (most recent call last):
+       ...
+    ValueError: Expected a dimension of size 4, got 6
+
+    >>> dtype = np.dtype([('a', '(4,4)i'), ('b', 'S5')])
+    >>> test_structarray_errors(np.empty((5,), dtype=dtype))
+    Traceback (most recent call last):
+       ...
+    ValueError: Expected 1 dimension(s), got 2
+    """
+
+cdef struct StringStruct:
+    char c[4][4]
+
+ctypedef char String[4][4]
+
+def stringstructtest(StringStruct[:] view):
+    pass
+
+def stringtest(String[:] view):
+    pass
+
+@testcase_numpy_1_5
+def test_string_invalid_dims():
+    """
+    >>> dtype = np.dtype([('a', 'S4')])
+    >>> data = ['spam', 'eggs']
+    >>> stringstructtest(np.array(data, dtype=dtype))
+    Traceback (most recent call last):
+       ...
+    ValueError: Expected 2 dimensions, got 1
+    >>> stringtest(np.array(data, dtype='S4'))
+    Traceback (most recent call last):
+       ...
+    ValueError: Expected 2 dimensions, got 1
+    """