Commit a04c0f4e authored by Mark's avatar Mark

Merge pull request #76 from markflorisson88/arrayinstruct

Buffer format arrays, strings and alignment on first member
parents a8d81e52 e52e8771
...@@ -649,7 +649,6 @@ class GetAndReleaseBufferUtilityCode(object): ...@@ -649,7 +649,6 @@ class GetAndReleaseBufferUtilityCode(object):
""")) """))
def mangle_dtype_name(dtype): def mangle_dtype_name(dtype):
# Use prefixes to seperate user defined types from builtins # Use prefixes to seperate user defined types from builtins
# (consider "typedef float unsigned_int") # (consider "typedef float unsigned_int")
...@@ -662,16 +661,20 @@ def mangle_dtype_name(dtype): ...@@ -662,16 +661,20 @@ def mangle_dtype_name(dtype):
prefix = "nn_" prefix = "nn_"
else: else:
prefix = "" prefix = ""
return prefix + dtype.declaration_code("").replace(" ", "_") type_decl = dtype.declaration_code("")
type_decl = type_decl.replace(" ", "_")
return prefix + type_decl.replace("[", "_").replace("]", "_")
def get_type_information_cname(code, dtype, maxdepth=None): def get_type_information_cname(code, dtype, maxdepth=None):
# Output the run-time type information (__Pyx_TypeInfo) for given dtype, """
# and return the name of the type info struct. Output the run-time type information (__Pyx_TypeInfo) for given dtype,
# and return the name of the type info struct.
# Structs with two floats of the same size are encoded as complex numbers.
# One can seperate between complex numbers declared as struct or with native Structs with two floats of the same size are encoded as complex numbers.
# encoding by inspecting to see if the fields field of the type is One can seperate between complex numbers declared as struct or with native
# filled in. encoding by inspecting to see if the fields field of the type is
filled in.
"""
namesuffix = mangle_dtype_name(dtype) namesuffix = mangle_dtype_name(dtype)
name = "__Pyx_TypeInfo_%s" % namesuffix name = "__Pyx_TypeInfo_%s" % namesuffix
structinfo_name = "__Pyx_StructFields_%s" % namesuffix structinfo_name = "__Pyx_StructFields_%s" % namesuffix
...@@ -688,6 +691,12 @@ def get_type_information_cname(code, dtype, maxdepth=None): ...@@ -688,6 +691,12 @@ def get_type_information_cname(code, dtype, maxdepth=None):
code.globalstate.utility_codes.add(name) code.globalstate.utility_codes.add(name)
typecode = code.globalstate['typeinfo'] typecode = code.globalstate['typeinfo']
arraysizes = []
if dtype.is_array:
while dtype.is_array:
arraysizes.append(dtype.size)
dtype = dtype.base_type
complex_possible = dtype.is_struct_or_union and dtype.can_be_complex() complex_possible = dtype.is_struct_or_union and dtype.can_be_complex()
declcode = dtype.declaration_code("") declcode = dtype.declaration_code("")
...@@ -729,7 +738,6 @@ def get_type_information_cname(code, dtype, maxdepth=None): ...@@ -729,7 +738,6 @@ def get_type_information_cname(code, dtype, maxdepth=None):
elif dtype.is_pyobject: elif dtype.is_pyobject:
typegroup = 'O' typegroup = 'O'
else: else:
print dtype
assert False assert False
if dtype.is_int: if dtype.is_int:
...@@ -737,15 +745,13 @@ def get_type_information_cname(code, dtype, maxdepth=None): ...@@ -737,15 +745,13 @@ def get_type_information_cname(code, dtype, maxdepth=None):
else: else:
is_unsigned = "0" is_unsigned = "0"
typecode.putln(('static __Pyx_TypeInfo %s = { "%s", %s, sizeof(%s), \'%s\', %s, %s };' typeinfo = ('static __Pyx_TypeInfo %s = '
) % (name, '{ "%s", %s, sizeof(%s), { %s }, %s, \'%s\', %s, %s };')
rep, tup = (name, rep, structinfo_name, declcode,
structinfo_name, ', '.join([str(x) for x in arraysizes]), len(arraysizes),
declcode, typegroup, is_unsigned, flags)
typegroup, typecode.putln(typeinfo % tup, safe=True)
is_unsigned,
flags,
), safe=True)
return name return name
def load_buffer_utility(util_code_name, context=None, **kwargs): def load_buffer_utility(util_code_name, context=None, **kwargs):
......
...@@ -2548,6 +2548,7 @@ class IndexNode(ExprNode): ...@@ -2548,6 +2548,7 @@ class IndexNode(ExprNode):
warning(index.pos, "Index should be typed for more " warning(index.pos, "Index should be typed for more "
"efficient access", level=2) "efficient access", level=2)
IndexNode.warned_untyped_idx = True IndexNode.warned_untyped_idx = True
self.memslice_index = True self.memslice_index = True
index = index.coerce_to(index_type, env) index = index.coerce_to(index_type, env)
indices[i] = index indices[i] = index
...@@ -6935,7 +6936,8 @@ class CythonArrayNode(ExprNode): ...@@ -6935,7 +6936,8 @@ class CythonArrayNode(ExprNode):
"n" * len(shapes), "n" * len(shapes),
", ".join(shapes))) ", ".join(shapes)))
err = "!%s || !%s || !PyBytes_Check(%s)" % (format_temp, shapes_temp, err = "!%s || !%s || !PyBytes_AsString(%s)" % (format_temp,
shapes_temp,
format_temp) format_temp)
code.putln(code.error_goto_if(err, self.pos)) code.putln(code.error_goto_if(err, self.pos))
code.put_gotref(format_temp) code.put_gotref(format_temp)
......
...@@ -157,7 +157,7 @@ def src_conforms_to_dst(src, dst): ...@@ -157,7 +157,7 @@ def src_conforms_to_dst(src, dst):
return True return True
def valid_memslice_dtype(dtype): def valid_memslice_dtype(dtype, i=0):
""" """
Return whether type dtype can be used as the base type of a Return whether type dtype can be used as the base type of a
memoryview slice. memoryview slice.
...@@ -178,6 +178,8 @@ def valid_memslice_dtype(dtype): ...@@ -178,6 +178,8 @@ def valid_memslice_dtype(dtype):
dtype.is_error or dtype.is_error or
# Pointers are not valid (yet) # Pointers are not valid (yet)
# (dtype.is_ptr and valid_memslice_dtype(dtype.base_type)) or # (dtype.is_ptr and valid_memslice_dtype(dtype.base_type)) or
(dtype.is_array and i < 8 and
valid_memslice_dtype(dtype.base_type, i + 1)) or
dtype.is_numeric or dtype.is_numeric or
dtype.is_pyobject or dtype.is_pyobject or
dtype.is_fused or # accept this as it will be replaced by specializations later dtype.is_fused or # accept this as it will be replaced by specializations later
......
...@@ -48,6 +48,8 @@ typedef struct { ...@@ -48,6 +48,8 @@ typedef struct {
const char* name; /* for error messages only */ const char* name; /* for error messages only */
struct __Pyx_StructField_* fields; struct __Pyx_StructField_* fields;
size_t size; /* sizeof(type) */ size_t size; /* sizeof(type) */
size_t arraysize[8]; /* length of array in each dimension */
int ndim;
char typegroup; /* _R_eal, _C_omplex, Signed _I_nt, _U_nsigned int, _S_truct, _P_ointer, _O_bject */ char typegroup; /* _R_eal, _C_omplex, Signed _I_nt, _U_nsigned int, _S_truct, _P_ointer, _O_bject */
char is_unsigned; char is_unsigned;
int flags; int flags;
...@@ -69,10 +71,12 @@ typedef struct { ...@@ -69,10 +71,12 @@ typedef struct {
__Pyx_BufFmt_StackElem* head; __Pyx_BufFmt_StackElem* head;
size_t fmt_offset; size_t fmt_offset;
size_t new_count, enc_count; size_t new_count, enc_count;
size_t struct_alignment;
int is_complex; int is_complex;
char enc_type; char enc_type;
char new_packmode; char new_packmode;
char enc_packmode; char enc_packmode;
char is_valid_array;
} __Pyx_BufFmt_Context; } __Pyx_BufFmt_Context;
...@@ -118,6 +122,8 @@ static void __Pyx_BufFmt_Init(__Pyx_BufFmt_Context* ctx, ...@@ -118,6 +122,8 @@ static void __Pyx_BufFmt_Init(__Pyx_BufFmt_Context* ctx,
ctx->enc_count = 0; ctx->enc_count = 0;
ctx->enc_type = 0; ctx->enc_type = 0;
ctx->is_complex = 0; ctx->is_complex = 0;
ctx->is_valid_array = 0;
ctx->struct_alignment = 0;
while (type->typegroup == 'S') { while (type->typegroup == 'S') {
++ctx->head; ++ctx->head;
ctx->head->field = type->fields; ctx->head->field = type->fields;
...@@ -142,6 +148,15 @@ static int __Pyx_BufFmt_ParseNumber(const char** ts) { ...@@ -142,6 +148,15 @@ static int __Pyx_BufFmt_ParseNumber(const char** ts) {
return count; return count;
} }
static int __Pyx_BufFmt_ExpectNumber(const char **ts) {
int number = __Pyx_BufFmt_ParseNumber(ts);
if (number == -1) /* First char was not a digit */
PyErr_Format(PyExc_ValueError,\
"Does not understand character buffer dtype format string ('%c')", **ts);
return number;
}
static void __Pyx_BufFmt_RaiseUnexpectedChar(char ch) { static void __Pyx_BufFmt_RaiseUnexpectedChar(char ch) {
PyErr_Format(PyExc_ValueError, PyErr_Format(PyExc_ValueError,
"Unexpected format string character: '%c'", ch); "Unexpected format string character: '%c'", ch);
...@@ -165,6 +180,7 @@ static const char* __Pyx_BufFmt_DescribeTypeChar(char ch, int is_complex) { ...@@ -165,6 +180,7 @@ static const char* __Pyx_BufFmt_DescribeTypeChar(char ch, int is_complex) {
case 'T': return "a struct"; case 'T': return "a struct";
case 'O': return "Python object"; case 'O': return "Python object";
case 'P': return "a pointer"; case 'P': return "a pointer";
case 's': case 'p': return "a string";
case 0: return "end"; case 0: return "end";
default: return "unparseable format string"; default: return "unparseable format string";
} }
...@@ -172,7 +188,7 @@ static const char* __Pyx_BufFmt_DescribeTypeChar(char ch, int is_complex) { ...@@ -172,7 +188,7 @@ static const char* __Pyx_BufFmt_DescribeTypeChar(char ch, int is_complex) {
static size_t __Pyx_BufFmt_TypeCharToStandardSize(char ch, int is_complex) { static size_t __Pyx_BufFmt_TypeCharToStandardSize(char ch, int is_complex) {
switch (ch) { switch (ch) {
case '?': case 'c': case 'b': case 'B': return 1; case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1;
case 'h': case 'H': return 2; case 'h': case 'H': return 2;
case 'i': case 'I': case 'l': case 'L': return 4; case 'i': case 'I': case 'l': case 'L': return 4;
case 'q': case 'Q': return 8; case 'q': case 'Q': return 8;
...@@ -191,7 +207,7 @@ static size_t __Pyx_BufFmt_TypeCharToStandardSize(char ch, int is_complex) { ...@@ -191,7 +207,7 @@ static size_t __Pyx_BufFmt_TypeCharToStandardSize(char ch, int is_complex) {
static size_t __Pyx_BufFmt_TypeCharToNativeSize(char ch, int is_complex) { static size_t __Pyx_BufFmt_TypeCharToNativeSize(char ch, int is_complex) {
switch (ch) { switch (ch) {
case 'c': case 'b': case 'B': return 1; case 'c': case 'b': case 'B': case 's': case 'p': return 1;
case 'h': case 'H': return sizeof(short); case 'h': case 'H': return sizeof(short);
case 'i': case 'I': return sizeof(int); case 'i': case 'I': return sizeof(int);
case 'l': case 'L': return sizeof(long); case 'l': case 'L': return sizeof(long);
...@@ -222,7 +238,7 @@ typedef struct { char c; PY_LONG_LONG x; } __Pyx_st_longlong; ...@@ -222,7 +238,7 @@ typedef struct { char c; PY_LONG_LONG x; } __Pyx_st_longlong;
static size_t __Pyx_BufFmt_TypeCharToAlignment(char ch, int is_complex) { static size_t __Pyx_BufFmt_TypeCharToAlignment(char ch, int is_complex) {
switch (ch) { switch (ch) {
case '?': case 'c': case 'b': case 'B': return 1; case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1;
case 'h': case 'H': return sizeof(__Pyx_st_short) - sizeof(short); case 'h': case 'H': return sizeof(__Pyx_st_short) - sizeof(short);
case 'i': case 'I': return sizeof(__Pyx_st_int) - sizeof(int); case 'i': case 'I': return sizeof(__Pyx_st_int) - sizeof(int);
case 'l': case 'L': return sizeof(__Pyx_st_long) - sizeof(long); case 'l': case 'L': return sizeof(__Pyx_st_long) - sizeof(long);
...@@ -239,13 +255,53 @@ static size_t __Pyx_BufFmt_TypeCharToAlignment(char ch, int is_complex) { ...@@ -239,13 +255,53 @@ static size_t __Pyx_BufFmt_TypeCharToAlignment(char ch, int is_complex) {
} }
} }
/* These are for computing the padding at the end of the struct to align
on the first member of the struct. This will probably the same as above,
but we don't have any guarantees.
*/
typedef struct { short x; char c; } __Pyx_pad_short;
typedef struct { int x; char c; } __Pyx_pad_int;
typedef struct { long x; char c; } __Pyx_pad_long;
typedef struct { float x; char c; } __Pyx_pad_float;
typedef struct { double x; char c; } __Pyx_pad_double;
typedef struct { long double x; char c; } __Pyx_pad_longdouble;
typedef struct { void *x; char c; } __Pyx_pad_void_p;
#ifdef HAVE_LONG_LONG
typedef struct { PY_LONG_LONG x; char c; } __Pyx_pad_longlong;
#endif
static size_t __Pyx_BufFmt_TypeCharToPadding(char ch, int is_complex) {
switch (ch) {
case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1;
case 'h': case 'H': return sizeof(__Pyx_pad_short) - sizeof(short);
case 'i': case 'I': return sizeof(__Pyx_pad_int) - sizeof(int);
case 'l': case 'L': return sizeof(__Pyx_pad_long) - sizeof(long);
#ifdef HAVE_LONG_LONG
case 'q': case 'Q': return sizeof(__Pyx_pad_longlong) - sizeof(PY_LONG_LONG);
#endif
case 'f': return sizeof(__Pyx_pad_float) - sizeof(float);
case 'd': return sizeof(__Pyx_pad_double) - sizeof(double);
case 'g': return sizeof(__Pyx_pad_longdouble) - sizeof(long double);
case 'P': case 'O': return sizeof(__Pyx_pad_void_p) - sizeof(void*);
default:
__Pyx_BufFmt_RaiseUnexpectedChar(ch);
return 0;
}
}
static char __Pyx_BufFmt_TypeCharToGroup(char ch, int is_complex) { static char __Pyx_BufFmt_TypeCharToGroup(char ch, int is_complex) {
switch (ch) { switch (ch) {
case 'c': case 'b': case 'h': case 'i': case 'l': case 'q': return 'I'; case 'c': case 'b': case 'h': case 'i':
case 'B': case 'H': case 'I': case 'L': case 'Q': return 'U'; case 'l': case 'q': case 's': case 'p':
case 'f': case 'd': case 'g': return (is_complex ? 'C' : 'R'); return 'I';
case 'O': return 'O'; case 'B': case 'H': case 'I': case 'L': case 'Q':
case 'P': return 'P'; return 'U';
case 'f': case 'd': case 'g':
return (is_complex ? 'C' : 'R');
case 'O':
return 'O';
case 'P':
return 'P';
default: { default: {
__Pyx_BufFmt_RaiseUnexpectedChar(ch); __Pyx_BufFmt_RaiseUnexpectedChar(ch);
return 0; return 0;
...@@ -281,8 +337,40 @@ static void __Pyx_BufFmt_RaiseExpected(__Pyx_BufFmt_Context* ctx) { ...@@ -281,8 +337,40 @@ static void __Pyx_BufFmt_RaiseExpected(__Pyx_BufFmt_Context* ctx) {
static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) { static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) {
char group; char group;
size_t size, offset; size_t size, offset, arraysize = 1;
/* printf("processing... %s\n", ctx->head->field->type->name); */
if (ctx->enc_type == 0) return 0; if (ctx->enc_type == 0) return 0;
/* Validate array size */
if (ctx->head->field->type->arraysize[0]) {
int i, ndim = 0;
/* handle strings ('s' and 'p') */
if (ctx->enc_type == 's' || ctx->enc_type == 'p') {
ctx->is_valid_array = ctx->head->field->type->ndim == 1;
ndim = 1;
if (ctx->enc_count != ctx->head->field->type->arraysize[0]) {
PyErr_Format(PyExc_ValueError,
"Expected a dimension of size %zu, got %zu",
ctx->head->field->type->arraysize[0], ctx->enc_count);
return -1;
}
}
if (!ctx->is_valid_array) {
PyErr_Format(PyExc_ValueError, "Expected %d dimensions, got %d",
ctx->head->field->type->ndim, ndim);
return -1;
}
for (i = 0; i < ctx->head->field->type->ndim; i++) {
arraysize *= ctx->head->field->type->arraysize[i];
}
ctx->is_valid_array = 0;
ctx->enc_count = 1;
}
group = __Pyx_BufFmt_TypeCharToGroup(ctx->enc_type, ctx->is_complex); group = __Pyx_BufFmt_TypeCharToGroup(ctx->enc_type, ctx->is_complex);
do { do {
__Pyx_StructField* field = ctx->head->field; __Pyx_StructField* field = ctx->head->field;
...@@ -293,12 +381,17 @@ static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) { ...@@ -293,12 +381,17 @@ static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) {
} else { } else {
size = __Pyx_BufFmt_TypeCharToStandardSize(ctx->enc_type, ctx->is_complex); size = __Pyx_BufFmt_TypeCharToStandardSize(ctx->enc_type, ctx->is_complex);
} }
if (ctx->enc_packmode == '@') { if (ctx->enc_packmode == '@') {
size_t align_at = __Pyx_BufFmt_TypeCharToAlignment(ctx->enc_type, ctx->is_complex); size_t align_at = __Pyx_BufFmt_TypeCharToAlignment(ctx->enc_type, ctx->is_complex);
size_t align_mod_offset; size_t align_mod_offset;
if (align_at == 0) return -1; if (align_at == 0) return -1;
align_mod_offset = ctx->fmt_offset % align_at; align_mod_offset = ctx->fmt_offset % align_at;
if (align_mod_offset > 0) ctx->fmt_offset += align_at - align_mod_offset; if (align_mod_offset > 0) ctx->fmt_offset += align_at - align_mod_offset;
if (ctx->struct_alignment == 0)
ctx->struct_alignment = __Pyx_BufFmt_TypeCharToPadding(ctx->enc_type,
ctx->is_complex);
} }
if (type->size != size || type->typegroup != group) { if (type->size != size || type->typegroup != group) {
...@@ -324,6 +417,8 @@ static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) { ...@@ -324,6 +417,8 @@ static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) {
} }
ctx->fmt_offset += size; ctx->fmt_offset += size;
if (arraysize)
ctx->fmt_offset += (arraysize - 1) * size;
--ctx->enc_count; /* Consume from buffer string */ --ctx->enc_count; /* Consume from buffer string */
...@@ -360,10 +455,67 @@ static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) { ...@@ -360,10 +455,67 @@ static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) {
return 0; return 0;
} }
/* Parse an array in the format string (e.g. (1,2,3)) */
static CYTHON_INLINE PyObject *
__pyx_buffmt_parse_array(__Pyx_BufFmt_Context* ctx, const char** tsp)
{
const char *ts = *tsp;
int i = 0, number;
int ndim = ctx->head->field->type->ndim;
;
++ts;
if (ctx->new_count != 1) {
PyErr_SetString(PyExc_ValueError,
"Cannot handle repeated arrays in format string");
return NULL;
}
/* Process the previous element */
if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL;
/* Parse all numbers in the format string */
while (*ts && *ts != ')') {
if (isspace(*ts))
continue;
number = __Pyx_BufFmt_ExpectNumber(&ts);
if (number == -1) return NULL;
if (i < ndim && (size_t) number != ctx->head->field->type->arraysize[i])
return PyErr_Format(PyExc_ValueError,
"Expected a dimension of size %zu, got %d",
ctx->head->field->type->arraysize[i], number);
if (*ts != ',' && *ts != ')')
return PyErr_Format(PyExc_ValueError,
"Expected a comma in format string, got '%c'", *ts);
if (*ts == ',') ts++;
i++;
}
if (i != ndim)
return PyErr_Format(PyExc_ValueError, "Expected %d dimension(s), got %d",
ctx->head->field->type->ndim, i);
if (!*ts) {
PyErr_SetString(PyExc_ValueError,
"Unexpected end of format string, expected ')'");
return NULL;
}
ctx->is_valid_array = 1;
ctx->new_count = 1;
*tsp = ++ts;
return Py_None;
}
static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const char* ts) { static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const char* ts) {
int got_Z = 0; int got_Z = 0;
while (1) { while (1) {
switch(*ts) { switch(*ts) {
/* puts(ts); */
case 0: case 0:
if (ctx->enc_type != 0 && ctx->head == NULL) { if (ctx->enc_type != 0 && ctx->head == NULL) {
__Pyx_BufFmt_RaiseExpected(ctx); __Pyx_BufFmt_RaiseExpected(ctx);
...@@ -406,12 +558,17 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha ...@@ -406,12 +558,17 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha
{ {
const char* ts_after_sub; const char* ts_after_sub;
size_t i, struct_count = ctx->new_count; size_t i, struct_count = ctx->new_count;
size_t struct_alignment = ctx->struct_alignment;
ctx->new_count = 1; ctx->new_count = 1;
++ts; ++ts;
if (*ts != '{') { if (*ts != '{') {
PyErr_SetString(PyExc_ValueError, "Buffer acquisition: Expected '{' after 'T'"); PyErr_SetString(PyExc_ValueError, "Buffer acquisition: Expected '{' after 'T'");
return NULL; return NULL;
} }
if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL;
ctx->enc_type = 0; /* Erase processed last struct element */
ctx->enc_count = 0;
ctx->struct_alignment = 0;
++ts; ++ts;
ts_after_sub = ts; ts_after_sub = ts;
for (i = 0; i != struct_count; ++i) { for (i = 0; i != struct_count; ++i) {
...@@ -419,10 +576,20 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha ...@@ -419,10 +576,20 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha
if (!ts_after_sub) return NULL; if (!ts_after_sub) return NULL;
} }
ts = ts_after_sub; ts = ts_after_sub;
if (struct_alignment) ctx->struct_alignment = struct_alignment;
} }
break; break;
case '}': /* end of substruct; either repeat or move on */ case '}': /* end of substruct; either repeat or move on */
{
size_t alignment = ctx->struct_alignment;
++ts; ++ts;
if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL;
ctx->enc_type = 0; /* Erase processed last struct element */
if (alignment && ctx->fmt_offset % alignment) {
/* Pad struct on size of the first member */
ctx->fmt_offset += alignment - (ctx->fmt_offset % alignment);
}
}
return ts; return ts;
case 'x': case 'x':
if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL;
...@@ -443,7 +610,7 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha ...@@ -443,7 +610,7 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha
case 'c': case 'b': case 'B': case 'h': case 'H': case 'i': case 'I': case 'c': case 'b': case 'B': case 'h': case 'H': case 'i': case 'I':
case 'l': case 'L': case 'q': case 'Q': case 'l': case 'L': case 'q': case 'Q':
case 'f': case 'd': case 'g': case 'f': case 'd': case 'g':
case 'O': case 'O': case 's': case 'p':
if (ctx->enc_type == *ts && got_Z == ctx->is_complex && if (ctx->enc_type == *ts && got_Z == ctx->is_complex &&
ctx->enc_packmode == ctx->new_packmode) { ctx->enc_packmode == ctx->new_packmode) {
/* Continue pooling same type */ /* Continue pooling same type */
...@@ -465,14 +632,13 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha ...@@ -465,14 +632,13 @@ static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const cha
while(*ts != ':') ++ts; while(*ts != ':') ++ts;
++ts; ++ts;
break; break;
case '(':
if (!__pyx_buffmt_parse_array(ctx, &ts)) return NULL;
break;
default: default:
{ {
int number = __Pyx_BufFmt_ParseNumber(&ts); int number = __Pyx_BufFmt_ExpectNumber(&ts);
if (number == -1) { /* First char was not a digit */ if (number == -1) return NULL;
PyErr_Format(PyExc_ValueError,
"Does not understand character buffer dtype format string ('%c')", *ts);
return NULL;
}
ctx->new_count = (size_t)number; ctx->new_count = (size_t)number;
} }
} }
......
...@@ -394,7 +394,6 @@ cdef class memoryview(object): ...@@ -394,7 +394,6 @@ cdef class memoryview(object):
info.readonly = 0 info.readonly = 0
info.obj = self info.obj = self
# Some properties that have the same sematics as in NumPy # Some properties that have the same sematics as in NumPy
property T: property T:
@cname('__pyx_memoryview_transpose') @cname('__pyx_memoryview_transpose')
...@@ -920,6 +919,8 @@ cdef extern from *: ...@@ -920,6 +919,8 @@ cdef extern from *:
char* name char* name
__Pyx_StructField* fields __Pyx_StructField* fields
size_t size size_t size
size_t arraysize[8]
int ndim
char typegroup char typegroup
char is_unsigned char is_unsigned
int flags int flags
...@@ -945,6 +946,11 @@ cdef extern from *: ...@@ -945,6 +946,11 @@ cdef extern from *:
@cname('__pyx_format_from_typeinfo') @cname('__pyx_format_from_typeinfo')
cdef format_from_typeinfo(__Pyx_TypeInfo *type): cdef format_from_typeinfo(__Pyx_TypeInfo *type):
"""
We want to return bytes, but python 3 doesn't allow you to do anything
useful with bytes. So use str and convert back and forth to/from unicode.
Thank you python 3 for making bytes the most useless thing ever!
"""
cdef __Pyx_StructField *field cdef __Pyx_StructField *field
cdef __pyx_typeinfo_string fmt cdef __pyx_typeinfo_string fmt
...@@ -960,12 +966,16 @@ cdef format_from_typeinfo(__Pyx_TypeInfo *type): ...@@ -960,12 +966,16 @@ cdef format_from_typeinfo(__Pyx_TypeInfo *type):
field = type.fields field = type.fields
while field.type: while field.type:
parts.append(format_from_typeinfo(field.type)) parts.append(format_from_typeinfo(field.type).decode('ascii'))
field += 1 field += 1
result = alignment.join(parts) + '}' result = alignment.join(parts) + '}'
else: else:
fmt = __Pyx_TypeInfoToFormat(type) fmt = __Pyx_TypeInfoToFormat(type)
result = fmt.string if type.arraysize[0]:
extents = [str(type.arraysize[i]) for i in range(type.ndim)]
result = "(%s)%s" % (','.join(extents), fmt.string.decode('ascii'))
else:
result = fmt.string.decode('ascii')
return result return result.encode('ascii')
...@@ -46,6 +46,14 @@ cdef intp[:, :] myarray ...@@ -46,6 +46,14 @@ cdef intp[:, :] myarray
cdef int[:] a10 = <int[:10]> object() cdef int[:] a10 = <int[:10]> object()
cdef int[:] a11 = <int[:5.4]> <int *> 1 cdef int[:] a11 = <int[:5.4]> <int *> 1
cdef struct Valid:
int array[1][2][3][4][5][6][7][8]
cdef struct Invalid:
int array[1][2][3][4][5][6][7][8][9]
cdef Valid[:] validslice
cdef Invalid[:] invalidslice
# These are VALID # These are VALID
cdef int[::view.indirect_contiguous, ::view.contiguous] a9 cdef int[::view.indirect_contiguous, ::view.contiguous] a9
...@@ -70,4 +78,5 @@ _ERRORS = u''' ...@@ -70,4 +78,5 @@ _ERRORS = u'''
44:10: Invalid base type for memoryview slice: intp 44:10: Invalid base type for memoryview slice: intp
46:35: Can only create cython.array from pointer or array 46:35: Can only create cython.array from pointer or array
47:24: Cannot assign type 'double' to 'Py_ssize_t' 47:24: Cannot assign type 'double' to 'Py_ssize_t'
55:13: Invalid base type for memoryview slice: Invalid
''' '''
...@@ -1456,6 +1456,151 @@ def test_memslice_prange(arg): ...@@ -1456,6 +1456,151 @@ def test_memslice_prange(arg):
for k in range(src.shape[2]): for k in range(src.shape[2]):
assert src[i, j, k] == dst[i, j, k], (src[i, j, k] == dst[i, j, k]) assert src[i, j, k] == dst[i, j, k], (src[i, j, k] == dst[i, j, k])
# Test arrays in structs
cdef struct ArrayStruct:
int ints[10]
char chars[3]
cdef packed struct PackedArrayStruct:
int ints[10]
char chars[3]
cdef fused FusedStruct:
ArrayStruct
PackedArrayStruct
@testcase
def test_memslice_struct_with_arrays():
"""
>>> test_memslice_struct_with_arrays()
abc
abc
"""
cdef ArrayStruct a1[10]
cdef PackedArrayStruct a2[10]
test_structs_with_arr(a1)
test_structs_with_arr(a2)
cdef test_structs_with_arr(FusedStruct array[10]):
cdef FusedStruct[:] myslice1, myslice2, myslice3, myslice4
cdef int i, j
myslice1 = <FusedStruct[:10]> array
for i in range(10):
for j in range(10):
myslice1[i].ints[j] = i
for j in range(3):
myslice1[i].chars[j] = 97 + j
if sys.version_info[:2] >= (2, 7):
if sys.version_info[0] < 3:
import __builtin__ as builtins
else:
import builtins
size1 = sizeof(FusedStruct)
size2 = len(builtins.memoryview(myslice1)[0])
assert size1 == size2, (size1, size2, builtins.memoryview(myslice1).format)
myslice2 = builtins.memoryview(myslice1)
for i in range(10):
assert myslice2[i].ints[i] == myslice1[i].ints[i]
assert myslice2[i].chars[i] == myslice1[i].chars[i]
myslice3 = <object> myslice1
myslice4 = myslice1
for i in range(10):
for j in range(10):
assert myslice3[i].ints[j] == myslice4[i].ints[j] == myslice1[i].ints[j]
for j in range(3):
assert myslice3[i].chars[j] == myslice4[i].chars[j] == myslice1[i].chars[j]
print myslice1[0].chars[:3].decode('ascii')
# Test padding at the end of structs in the buffer support
cdef struct PaddedAtEnd:
int a[3]
char b[3]
cdef struct AlignedNested:
PaddedAtEnd a
char chars[1]
cdef struct PaddedAtEndNormal:
int a
char b
char c
char d
cdef struct AlignedNestedNormal:
PaddedAtEndNormal a
char chars
# Test nested structs in a struct, make sure we compute padding each time
# accordingly. If the first struct member is a struct, align on the first
# member of that struct (recursively)
cdef struct A:
double d
char c
cdef struct B:
char c1
A a
char c2
cdef struct C:
A a
char c1
cdef struct D:
B b
C cstruct
int a[2]
char c
cdef fused FusedPadded:
ArrayStruct
PackedArrayStruct
AlignedNested
AlignedNestedNormal
A
B
C
D
@testcase
def test_padded_structs():
"""
>>> test_padded_structs()
"""
cdef ArrayStruct a1[10]
cdef PackedArrayStruct a2[10]
cdef AlignedNested a3[10]
cdef AlignedNestedNormal a4[10]
cdef A a5[10]
cdef B a6[10]
cdef C a7[10]
cdef D a8[10]
_test_padded(a1)
_test_padded(a2)
_test_padded(a3)
_test_padded(a4)
_test_padded(a5)
_test_padded(a6)
_test_padded(a7)
# There is a pre-existing bug that doesn't parse the format for this
# struct properly -- fix this
#_test_padded(a8)
cdef _test_padded(FusedPadded myarray[10]):
# test that the buffer format parser accepts our format string...
cdef FusedPadded[:] myslice = <FusedPadded[:10]> myarray
obj = myslice
cdef FusedPadded[:] myotherslice = obj
@testcase @testcase
def test_object_indices(): def test_object_indices():
""" """
......
...@@ -391,3 +391,120 @@ def acquire_release_cycle(obj): ...@@ -391,3 +391,120 @@ def acquire_release_cycle(obj):
del buf del buf
gc.collect() gc.collect()
cdef packed struct StructArray:
int a[4]
char b[5]
@testcase_numpy_1_5
def test_memslice_structarray(data, dtype):
"""
>>> data = [(range(4), 'spam\\0'), (range(4, 8), 'ham\\0\\0'), (range(8, 12), 'eggs\\0')]
>>> dtype = np.dtype([('a', '4i'), ('b', '5b')])
>>> test_memslice_structarray([(L, map(ord, s)) for L, s in data], dtype)
0
1
2
3
spam
4
5
6
7
ham
8
9
10
11
eggs
Test the same thing with the string format specifier
>>> dtype = np.dtype([('a', '4i'), ('b', 'S5')])
>>> test_memslice_structarray(data, dtype)
0
1
2
3
spam
4
5
6
7
ham
8
9
10
11
eggs
"""
a = np.empty((3,), dtype=dtype)
a[:] = data
cdef StructArray[:] myslice = a
cdef int i, j
for i in range(3):
for j in range(4):
print myslice[i].a[j]
print myslice[i].b
@testcase_numpy_1_5
def test_structarray_errors(StructArray[:] a):
"""
>>> dtype = np.dtype([('a', '4i'), ('b', '5b')])
>>> test_structarray_errors(np.empty((5,), dtype=dtype))
>>> dtype = np.dtype([('a', '6i'), ('b', '5b')])
>>> test_structarray_errors(np.empty((5,), dtype=dtype))
Traceback (most recent call last):
...
ValueError: Expected a dimension of size 4, got 6
>>> dtype = np.dtype([('a', '(4,4)i'), ('b', '5b')])
>>> test_structarray_errors(np.empty((5,), dtype=dtype))
Traceback (most recent call last):
...
ValueError: Expected 1 dimension(s), got 2
Test the same thing with the string format specifier
>>> dtype = np.dtype([('a', '4i'), ('b', 'S5')])
>>> test_structarray_errors(np.empty((5,), dtype=dtype))
>>> dtype = np.dtype([('a', '6i'), ('b', 'S5')])
>>> test_structarray_errors(np.empty((5,), dtype=dtype))
Traceback (most recent call last):
...
ValueError: Expected a dimension of size 4, got 6
>>> dtype = np.dtype([('a', '(4,4)i'), ('b', 'S5')])
>>> test_structarray_errors(np.empty((5,), dtype=dtype))
Traceback (most recent call last):
...
ValueError: Expected 1 dimension(s), got 2
"""
cdef struct StringStruct:
char c[4][4]
ctypedef char String[4][4]
def stringstructtest(StringStruct[:] view):
pass
def stringtest(String[:] view):
pass
@testcase_numpy_1_5
def test_string_invalid_dims():
"""
>>> dtype = np.dtype([('a', 'S4')])
>>> data = ['spam', 'eggs']
>>> stringstructtest(np.array(data, dtype=dtype))
Traceback (most recent call last):
...
ValueError: Expected 2 dimensions, got 1
>>> stringtest(np.array(data, dtype='S4'))
Traceback (most recent call last):
...
ValueError: Expected 2 dimensions, got 1
"""
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment