Commit efc2e33d authored by Mark Florisson's avatar Mark Florisson

initializedcheck directive + uninitialized memslice checking + memoryview...

initializedcheck directive + uninitialized memslice checking + memoryview indexing object<->dtype conversion
parent 71e92014
......@@ -1422,6 +1422,7 @@ class NameNode(AtomicExprNode):
self.result_ctype = py_object_type
def analyse_types(self, env):
self.initialized_check = env.directives['initializedcheck']
if self.entry is None:
self.entry = env.lookup(self.name)
if not self.entry:
......@@ -1455,15 +1456,21 @@ class NameNode(AtomicExprNode):
#print "Entry:", self.entry.__dict__ ###
self.analyse_entry(env)
entry = self.entry
if entry.is_declared_generic:
self.result_ctype = py_object_type
if entry.is_pyglobal or entry.is_builtin:
if entry.is_builtin and entry.is_const:
self.is_temp = 0
else:
self.is_temp = 1
env.use_utility_code(get_name_interned_utility_code)
self.is_used_as_rvalue = 1
elif entry.type.is_memoryviewslice:
self.is_temp = False
self.is_used_as_rvalue = True
def nogil_check(self, env):
self.nogil = True
......@@ -1474,6 +1481,10 @@ class NameNode(AtomicExprNode):
self.gil_error()
elif entry.is_pyglobal:
self.gil_error()
elif self.entry.type.is_memoryviewslice:
if self.cf_is_null or self.cf_maybe_null:
import MemoryView
MemoryView.err_if_nogil_initialized_check(self.pos, env)
gil_message = "Accessing Python global or builtin"
......@@ -1614,10 +1625,14 @@ class NameNode(AtomicExprNode):
code.put_gotref(self.py_result())
elif entry.is_local or entry.in_closure or entry.from_closure:
if entry.type.check_for_null_code(entry.cname):
if (self.cf_maybe_null or self.cf_is_null) \
and not self.allow_null:
code.put_error_if_unbound(self.pos, entry)
# Raise UnboundLocalError for objects and memoryviewslices
raise_unbound = (
(self.cf_maybe_null or self.cf_is_null) and not self.allow_null)
null_code = entry.type.check_for_null_code(entry.cname)
memslice_check = entry.type.is_memoryviewslice and self.initialized_check
if null_code and raise_unbound and (entry.type.is_pyobject or memslice_check):
code.put_error_if_unbound(self.pos, entry)
def generate_assignment_code(self, rhs, code):
#print "NameNode.generate_assignment_code:", self.name ###
......@@ -2401,18 +2416,23 @@ class IndexNode(ExprNode):
if not x.type.is_int:
buffer_access = False
if buffer_access:
if buffer_access and not self.base.type.is_memoryviewslice:
assert hasattr(self.base, "entry") # Must be a NameNode-like node
# On cloning, indices is cloned. Otherwise, unpack index into indices
assert not (buffer_access and isinstance(self.index, CloneNode))
self.nogil = env.nogil
if buffer_access:
if self.base.type.is_memoryviewslice and not self.base.is_name:
self.base = self.base.coerce_to_temp(env)
self.indices = indices
self.index = None
self.type = self.base.type.dtype
self.is_buffer_access = True
self.buffer_type = self.base.entry.type
self.buffer_type = self.base.type #self.base.entry.type
if getting and self.type.is_pyobject:
self.is_temp = True
......@@ -2734,9 +2754,14 @@ class IndexNode(ExprNode):
# Generate buffer access code using these temps
import Buffer, MemoryView
# The above could happen because child_attrs is wrong somewhere so that
# options are not propagated.
entry = self.base.entry
if self.base.is_name:
entry = self.base.entry
else:
assert self.base.is_temp
cname = self.base.result()
entry = Symtab.Entry(cname, cname, self.base.type, self.base.pos)
if entry.type.is_buffer:
buffer_entry = Buffer.BufferEntry(entry)
negative_indices = entry.type.negative_indices
......@@ -3345,20 +3370,27 @@ class SimpleCallNode(CallNode):
if i > 0 or i == 1 and self.self is not None: # skip first arg
warning(arg.pos, "Argument evaluation order in C function call is undefined and may not be as expected", 0)
break
# Calc result type and code fragment
if isinstance(self.function, NewExprNode):
self.type = PyrexTypes.CPtrType(self.function.class_type)
else:
self.type = func_type.return_type
if self.function.is_name or self.function.is_attribute:
if self.function.entry and self.function.entry.utility_code:
self.is_temp = 1 # currently doesn't work for self.calculate_result_code()
if self.type.is_pyobject:
self.result_ctype = py_object_type
self.is_temp = 1
elif func_type.exception_value is not None \
or func_type.exception_check:
self.is_temp = 1
elif self.type.is_memoryviewslice:
self.is_temp = 1
# func_type.exception_check = True
# Called in 'nogil' context?
self.nogil = env.nogil
if (self.nogil and
......@@ -3436,6 +3468,9 @@ class SimpleCallNode(CallNode):
exc_checks = []
if self.type.is_pyobject and self.is_temp:
exc_checks.append("!%s" % self.result())
elif self.type.is_memoryviewslice:
assert self.is_temp
exc_checks.append(self.type.error_condition(self.result()))
else:
exc_val = func_type.exception_value
exc_check = func_type.exception_check
......@@ -3758,6 +3793,7 @@ class AttributeNode(ExprNode):
self.analyse_types(env, target = 1)
def analyse_types(self, env, target = 0):
self.initialized_check = env.directives['initializedcheck']
if self.analyse_as_cimported_attribute(env, target):
self.entry.used = True
elif not target and self.analyse_as_unbound_cmethod(env):
......@@ -3862,6 +3898,8 @@ class AttributeNode(ExprNode):
self.result_ctype = py_object_type
elif target and self.obj.type.is_builtin_type:
error(self.pos, "Assignment to an immutable object field")
#elif self.type.is_memoryviewslice and not target:
# self.is_temp = True
def analyse_attribute(self, env, obj_type = None):
# Look up attribute and set self.type and self.member.
......@@ -3933,6 +3971,8 @@ class AttributeNode(ExprNode):
def nogil_check(self, env):
if self.is_py_attr:
self.gil_error()
import MemoryView
MemoryView.err_if_nogil_initialized_check(self.pos, env, 'attribute')
gil_message = "Accessing Python attribute"
......@@ -3987,6 +4027,16 @@ class AttributeNode(ExprNode):
code.intern_identifier(self.attribute),
code.error_goto_if_null(self.result(), self.pos)))
code.put_gotref(self.py_result())
elif self.type.is_memoryviewslice:
if self.initialized_check:
code.putln(textwrap.dedent('''
if (unlikely(!%s.memview)) {
PyErr_SetString(PyExc_AttributeError,
"Memoryview is not initialized");
%s
}''' % (self.result(), code.error_goto(self.pos))))
#code.putln("%s = %s;" % (self.result(),
# self.calculate_result_code()))
else:
# result_code contains what is needed, but we may need to insert
# a check and raise an exception
......@@ -7860,6 +7910,8 @@ class CoerceToPyTypeNode(CoercionNode):
# FIXME: check that the target type and the resulting type are compatible
pass
self.env = env
gil_message = "Converting to Python object"
def may_be_none(self):
......@@ -7887,7 +7939,7 @@ class CoerceToPyTypeNode(CoercionNode):
def generate_result_code(self, code):
if self.arg.type.is_memoryviewslice:
funccall = self.arg.type.get_to_py_function(self.arg)
funccall = self.arg.type.get_to_py_function(self.env, self.arg)
else:
funccall = "%s(%s)" % (self.arg.type.to_py_function,
self.arg.result())
......
from Errors import CompileError
from Errors import CompileError, error
import ExprNodes
from ExprNodes import IntNode, NoneNode, IntBinopNode, NameNode, AttributeNode
from Visitor import CythonTransform
......@@ -17,6 +17,12 @@ BOTH_CF_ERR = "Cannot specify an array that is both C and Fortran contiguous."
INVALID_ERR = "Invalid axis specification."
EXPR_ERR = "no expressions allowed in axis spec, only names and literals."
CF_ERR = "Invalid axis specification for a C/Fortran contiguous array."
ERR_UNINITIALIZED = ("Cannot check if memoryview %s is initialized without the "
"GIL, consider using initializedcheck(False)")
def err_if_nogil_initialized_check(pos, env, name='variable'):
if env.nogil and env.directives['initializedcheck']:
error(pos, ERR_UNINITIALIZED % name)
def concat_flags(*flags):
return "(%s)" % "|".join(flags)
......@@ -85,7 +91,8 @@ def put_acquire_memoryviewslice(lhs_cname, lhs_type, lhs_pos, rhs, code,
rhstmp = code.funcstate.allocate_temp(lhs_type, manage_ref=False)
code.putln("%s = %s;" % (rhstmp, rhs.result_as(lhs_type)))
code.putln(code.error_goto_if_null("%s.memview" % rhstmp, lhs_pos))
# Allow uninitialized assignment
#code.putln(code.put_error_if_unbound(lhs_pos, rhs.entry))
put_assign_to_memviewslice(lhs_cname, rhstmp, lhs_type, code, incref_rhs)
if not pretty_rhs:
......@@ -739,9 +746,15 @@ def _resolve_AttributeNode(env, node):
modnames = path[:-1]
# must be at least 1 module name, o/w not an AttributeNode.
assert modnames
scope = env.lookup(modnames[0]).as_module
for modname in modnames[1:]:
scope = scope.lookup(modname).as_module
scope = env
for modname in modnames:
mod = scope.lookup(modname)
if not mod:
raise CompileError(
node.pos, "undeclared name not builtin: %s" % modname)
scope = mod.as_module
return scope.lookup(path[-1])
def load_memview_cy_utility(util_code_name, context=None, **kwargs):
......
......@@ -10,7 +10,7 @@ cython.declare(sys=object, os=object, time=object, copy=object,
CppClassScope=object, UtilityCode=object, EncodedString=object,
absolute_path_length=cython.Py_ssize_t)
import sys, os, time, copy
import sys, os, time, copy, textwrap
import Builtin
from Errors import error, warning, InternalError, CompileError
......@@ -1501,7 +1501,12 @@ class FuncDefNode(StatNode, BlockNode):
# code.put_xdecref_memoryviewslice(entry.cname)
code.putln("__Pyx_ErrRestore(__pyx_type, __pyx_value, __pyx_tb);}")
err_val = self.error_value()
if self.return_type.is_memoryviewslice:
MemoryView.put_init_entry(Naming.retval_cname, code)
err_val = Naming.retval_cname
else:
err_val = self.error_value()
exc_check = self.caller_will_check_exceptions()
if err_val is not None or exc_check:
# TODO: Fix exception tracing (though currently unused by cProfile).
......@@ -1552,6 +1557,26 @@ class FuncDefNode(StatNode, BlockNode):
Buffer.put_release_buffer_code(code, entry)
if is_getbuffer_slot:
self.getbuffer_normal_cleanup(code)
if self.return_type.is_memoryviewslice:
# See if our return value is uninitialized on non-error return
# import MemoryView
# MemoryView.err_if_nogil_initialized_check(self.pos, env)
cond = code.unlikely(self.return_type.error_condition(
Naming.retval_cname))
code.putln(
'if (%s) {' % cond)
if env.nogil:
code.put_ensure_gil()
code.putln(
'PyErr_SetString('
'PyExc_TypeError,'
'"Memoryview return value is not initialized");')
if env.nogil:
code.put_release_ensured_gil()
code.putln(
'}')
# ----- Return cleanup for both error and no-error return
code.put_label(code.return_from_error_cleanup_label)
......@@ -4312,8 +4337,6 @@ class ReturnStatNode(StatNode):
self.return_type)
elif self.return_type.is_memoryviewslice:
code.put_xdecref_memoryviewslice(Naming.retval_cname)
#code.put_xdecref("%s.memview" % Naming.retval_cname,
# self.return_type)
if self.value:
self.value.generate_evaluation_code(code)
......@@ -7042,7 +7065,12 @@ class CnameDecoratorNode(StatNode):
for name, entry in scope.entries.iteritems():
if entry.func_cname:
entry.func_cname = '%s_%s' % (self.cname, entry.cname)
cname = entry.cname
if '.' in cname:
# remove __pyx_base from func_cname
cname = cname.split('.')[-1]
entry.func_cname = '%s_%s' % (self.cname, cname)
def analyse_expressions(self, env):
self.node.analyse_expressions(env)
......
......@@ -75,6 +75,7 @@ buffer_max_dims = 32
directive_defaults = {
'boundscheck' : True,
'nonecheck' : False,
'initializedcheck' : True,
'embedsignature' : False,
'locals' : {},
'auto_cpdef': False,
......
......@@ -387,6 +387,8 @@ class MemoryViewSliceType(PyrexType):
self.mode = MemoryView.get_mode(axes)
self.writable_needed = False
self.dtype_name = MemoryView.mangle_dtype_name(self.dtype)
def same_as_resolved_type(self, other_type):
return ((other_type.is_memoryviewslice and
self.dtype.same_as(other_type.dtype) and
......@@ -500,9 +502,7 @@ class MemoryViewSliceType(PyrexType):
return True
def specialization_suffix(self):
import MemoryView
dtype_name = MemoryView.mangle_dtype_name(self.dtype)
return "%s_%s" % (self.axes_to_name(), dtype_name)
return "%s_%s" % (self.axes_to_name(), self.dtype_name)
#def global_init_code(self, entry, code):
# code.putln("%s.data = NULL;" % entry.cname)
......@@ -552,9 +552,49 @@ class MemoryViewSliceType(PyrexType):
def create_to_py_utility_code(self, env):
return True
def get_to_py_function(self, obj):
return "__pyx_memoryview_fromslice(&%s, %s.memview->obj, %s, %s);" % (
obj.result(), obj.result(), self.flags, self.ndim)
def get_to_py_function(self, env, obj):
to_py_func, from_py_func = self.dtype_object_conversion_funcs(env)
to_py_func = "(PyObject *(*)(char *)) " + to_py_func
from_py_func = "(int (*)(char *, PyObject *)) " + from_py_func
tup = (obj.result(), obj.result(), self.flags, self.ndim,
to_py_func, from_py_func)
return ("__pyx_memoryview_fromslice(&%s, %s.memview->obj, "
"%s, %s, %s, %s);" % tup)
def dtype_object_conversion_funcs(self, env):
import MemoryView, Code
get_function = "__pyx_memview_get_%s" % self.dtype_name
set_function = "__pyx_memview_set_%s" % self.dtype_name
context = dict(
get_function = get_function,
set_function = set_function,
)
if self.dtype.is_pyobject:
utility_name = "MemviewObjectToObject"
else:
if not (self.dtype.create_to_py_utility_code(env) and
self.dtype.create_from_py_utility_code(env)):
print "cannot convert %s" % self.dtype
return "NULL", "NULL"
utility_name = "MemviewDtypeToObject"
error_condition = (self.dtype.error_condition('value') or
'PyErr_Occurred()')
context.update(
to_py_function = self.dtype.to_py_function,
from_py_function = self.dtype.from_py_function,
dtype = self.dtype.declaration_code(""),
error_condition = error_condition,
)
utility = Code.ContentHashingUtilityCode.load(
utility_name, "MemoryView_C.c", context=context)
env.use_utility_code(utility)
return get_function, set_function
def axes_to_code(self):
"Return a list of code constants for each axis"
......
......@@ -64,7 +64,7 @@ class CythonUtilityCode(Code.UtilityCodeBase):
is_cython_utility = True
def __init__(self, impl, name="__pyxutil", prefix="", requires=None,
file=None):
file=None, from_scope=None):
# 1) We need to delay the parsing/processing, so that all modules can be
# imported without import loops
# 2) The same utility code object can be used for multiple source files;
......@@ -76,6 +76,7 @@ class CythonUtilityCode(Code.UtilityCodeBase):
self.file = file
self.prefix = prefix
self.requires = requires or []
self.from_scope = from_scope
def get_tree(self, entries_only=False):
from AnalysedTreeTransforms import AutoTestDictTransform
......@@ -108,6 +109,15 @@ class CythonUtilityCode(Code.UtilityCodeBase):
pipeline = Pipeline.insert_into_pipeline(pipeline, transform,
before=before)
if self.from_scope:
def scope_transform(module_node):
module_node.scope.merge_in(self.from_scope)
return module_node
transform = ParseTreeTransforms.AnalyseDeclarationsTransform
pipeline = Pipeline.insert_into_pipeline(pipeline, transform,
before=scope_transform)
(err, tree) = Pipeline.run_pipeline(pipeline, tree)
assert not err, err
return tree
......
......@@ -217,18 +217,10 @@ cdef class memoryview(object):
if self.lock != NULL:
PyThread_free_lock(self.lock)
@cname('__pyx_memoryview_getitem')
def __getitem__(memoryview self, object index):
# cdef Py_ssize_t idx
cdef char *itemp = <char *> self.view.buf
cdef bytes bytesitem
cdef str fmt = self.view.format
import struct
try:
itemsize = struct.calcsize(fmt)
except struct.error:
raise TypeError("Unsupported format: %r" % fmt)
cdef char *get_item_pointer(memoryview self, object index) except NULL:
cdef Py_ssize_t dim
cdef Py_buffer view = self.view
cdef char *itemp = <char *> view.buf
if index is Ellipsis:
return self
......@@ -252,17 +244,50 @@ cdef class memoryview(object):
for dim, idx in enumerate(tup):
_check_index(idx)
itemp = pybuffer_index(&self.view, itemp, idx, dim + 1)
itemp = pybuffer_index(&self.view, itemp, idx, dim)
return itemp
@cname('__pyx_memoryview_getitem')
def __getitem__(memoryview self, object index):
cdef char *itemp = self.get_item_pointer(index)
return self.convert_item_to_object(itemp)
@cname('__pyx_memoryview_setitem')
def __setitem__(memoryview self, object index, object value):
cdef char *itemp = self.get_item_pointer(index)
self.assign_item_from_object(itemp, value)
cdef convert_item_to_object(self, char *itemp):
"""Only used if instantiated manually by the user, or if Cython doesn't
know how to convert the type"""
import struct
cdef bytes bytesitem
# Do a manual and complete check here instead of this easy hack
bytesitem = itemp[:self.view.itemsize]
return struct.unpack(fmt, bytesitem)
return struct.unpack(self.view.format, bytesitem)
cdef assign_item_from_object(self, char *itemp, object value):
"""Only used if instantiated manually by the user, or if Cython doesn't
know how to convert the type"""
import struct
cdef char c
cdef bytes bytesvalue
cdef Py_ssize_t i
if isinstance(value, tuple):
bytesvalue = struct.pack(self.view.format, *value)
else:
bytesvalue = struct.pack(self.view.format, value)
for i, c in enumerate(bytesvalue):
itemp[i] = c
def __repr__(self):
return "<MemoryView of %s at 0x%x>" % (self.obj.__class__.__name__, id(self))
return "<MemoryView of %r at 0x%x>" % (self.obj.__class__.__name__, id(self))
def __str__(self):
return "<MemoryView of %r at 0x%x>" % (self.obj, id(self))
return "<MemoryView of %r object>" % (self.obj.__class__.__name__,)
@cname('__pyx_memoryviewslice')
......@@ -274,28 +299,47 @@ cdef class _memoryviewslice(memoryview):
# Restore the original Py_buffer before releasing
cdef Py_buffer orig_view
cdef object (*to_object_func)(char *)
cdef int (*to_dtype_func)(char *, object) except 0
def __cinit__(self, object obj, int flags):
self.orig_view = self.view
def __dealloc__(self):
self.view = self.orig_view
cdef convert_item_to_object(self, char *itemp):
if self.to_object_func != NULL:
self.to_object_func(itemp)
else:
memoryview.convert_item_to_object(self, itemp)
cdef assign_item_from_object(self, char *itemp, object value):
if self.to_dtype_func != NULL:
self.to_dtype_func(itemp, value)
else:
memoryview.assign_item_from_object(self, itemp, value)
@cname('__pyx_memoryview_new')
cdef memoryview_cwrapper(object o, int flags):
return memoryview(o, flags)
@cname('__pyx_memoryview_fromslice')
cdef memoryview_from_memview_cwrapper({{memviewslice_name}} *memviewslice,
object orig_obj, int flags, int new_ndim):
cdef memoryview_from_memslice_cwrapper(
{{memviewslice_name}} *memviewslice, object orig_obj, int flags, int cur_ndim,
object (*to_object_func)(char *), int (*to_dtype_func)(char *, object)):
cdef _memoryviewslice result = _memoryviewslice(orig_obj, flags)
cdef int new_ndim = result.view.ndim - cur_ndim
result.from_slice = memviewslice[0]
result.view.shape = <Py_ssize_t *> (&result.from_slice.shape + new_ndim)
result.view.strides = <Py_ssize_t *> (&result.from_slice.strides + new_ndim)
result.view.suboffsets = <Py_ssize_t *> (&result.from_slice.suboffsets + new_ndim)
result.view.ndim = new_ndim
result.view.ndim = cur_ndim
result.to_object_func = to_object_func
return result
......@@ -336,14 +380,14 @@ cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, int dim
if index < 0:
index += view.shape[dim]
if index < 0:
raise IndexError("Out of bounds in dimension %d" % dim)
raise IndexError("Out of bounds on buffer access (axis %d)" % dim)
if index > shape:
raise IndexError("Out of bounds in dimension %d" % dim)
raise IndexError("Out of bounds on buffer access (axis %d)" % dim)
resultp = bufp + index * stride
if suboffset >= 0:
resultp = (<char **> resultp)[0] + suboffset
return resultp
......@@ -49,10 +49,7 @@ static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *, int, int);
static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *, int, int);
/////////////// MemviewSliceIndex.proto ///////////////
static CYTHON_INLINE char *__pyx_memviewslice_index_full(char *bufp, Py_ssize_t idx, Py_ssize_t stride, Py_ssize_t suboffset);
static CYTHON_INLINE char *__pyx_memviewslice_index_full_contig(char *bufp, Py_ssize_t suboffset);
static CYTHON_INLINE char *__pyx_memviewslice_index_full(const char *bufp, Py_ssize_t idx, Py_ssize_t stride, Py_ssize_t suboffset);
/////////////// ObjectToMemviewSlice ///////////////
......@@ -173,10 +170,6 @@ static int __Pyx_ValidateAndInit_memviewslice(
}
}
if (spec & (__Pyx_MEMVIEW_PTR|__Pyx_MEMVIEW_FULL) && !buf->suboffsets) {
memviewslice->suboffsets[i] = -1;
}
if (spec & __Pyx_MEMVIEW_PTR) {
if (buf->suboffsets && buf->suboffsets[i] < 0) {
PyErr_Format(PyExc_ValueError,
......@@ -238,16 +231,16 @@ static int __Pyx_init_memviewslice(
PyErr_SetString(PyExc_ValueError,
"buf is NULL.");
goto fail;
} else if(memviewslice->memview || memviewslice->data) {
} else if (memviewslice->memview || memviewslice->data) {
PyErr_SetString(PyExc_ValueError,
"memviewslice is already initialized!");
goto fail;
}
for(i=0; i<ndim; i++) {
for (i = 0; i < ndim; i++) {
memviewslice->strides[i] = buf->strides[i];
memviewslice->shape[i] = buf->shape[i];
if(buf->suboffsets) {
if (buf->suboffsets) {
memviewslice->suboffsets[i] = buf->suboffsets[i];
} else {
memviewslice->suboffsets[i] = -1;
......@@ -294,15 +287,16 @@ static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *memslice,
int first_time;
struct {{memview_struct_name}} *memview = memslice->memview;
if (!memview)
__pyx_fatalerror("memoryslice is not initialized (line %d)", lineno);
return; /* allow uninitialized memoryview assignment */
/* __pyx_fatalerror("memoryslice is not initialized (line %d)", lineno); */
if (memview->acquisition_count <= 0)
__pyx_fatalerror("Acquisition count is %d (line %d)",
memview->acquisition_count, lineno);
PyThread_acquire_lock(memview->lock, 1);
//PyThread_acquire_lock(memview->lock, 1);
first_time = (memview->acquisition_count++ == 0);
PyThread_release_lock(memview->lock);
//PyThread_release_lock(memview->lock);
if (first_time) {
if (have_gil) {
......@@ -327,9 +321,9 @@ static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *memslice,
__pyx_fatalerror("Acquisition count is %d (line %d)",
memview->acquisition_count, lineno);
PyThread_acquire_lock(memview->lock, 1);
//PyThread_acquire_lock(memview->lock, 1);
last_time = (memview->acquisition_count-- == 1);
PyThread_release_lock(memview->lock);
//PyThread_release_lock(memview->lock);
if (last_time) {
if (have_gil) {
......@@ -418,10 +412,46 @@ no_fail:
/////////////// MemviewSliceIndex ///////////////
static CYTHON_INLINE char *__pyx_memviewslice_index_full(char *bufp, Py_ssize_t idx, Py_ssize_t stride, Py_ssize_t suboffset) {
static CYTHON_INLINE char *__pyx_memviewslice_index_full(const char *bufp, Py_ssize_t idx, Py_ssize_t stride, Py_ssize_t suboffset) {
bufp = bufp + idx * stride;
if (suboffset >= 0) {
bufp = *((char **) bufp) + suboffset;
}
return bufp;
}
/////////////// MemviewDtypeToObject.proto ///////////////
PyObject *{{get_function}}(const char *itemp); /* proto */
int {{set_function}}(const char *itemp, PyObject *obj); /* proto */
/////////////// MemviewDtypeToObject ///////////////
{{#__pyx_memview_<dtype_name>_to_object}}
PyObject *{{get_function}}(const char *itemp) {
return (PyObject *) {{to_py_function}}(*({{dtype}} *) itemp);
}
int {{set_function}}(const char *itemp, PyObject *obj) {
{{dtype}} value = {{from_py_function}}(obj);
if ({{error_condition}})
return 0;
*({{dtype}} *) itemp = value;
return 1;
}
/////////////// MemviewObjectToObject.proto ///////////////
PyObject *{{get_function}}(const char *itemp); /* proto */
int {{set_function}}(const char *itemp, PyObject *obj); /* proto */
/////////////// MemviewObjectToObject ///////////////
PyObject *{{get_function}}(const char *itemp) {
PyObject *result = *(PyObject **) itemp;
Py_INCREF(result);
return result;
}
int {{set_function}}(const char *itemp, PyObject *obj) {
Py_INCREF(obj);
*(PyObject **) itemp = obj;
return 1;
}
......@@ -9,7 +9,13 @@ u'''
from cython.view cimport memoryview
from cython cimport array, PyBUF_C_CONTIGUOUS
from cython cimport view
include "mockbuffers.pxi"
#
### Test for some coercions
#
def init_obj():
return 3
......@@ -38,14 +44,14 @@ def g():
obj = init_obj()
mview = array((10,), itemsize=sizeof(int), format='i')
cdef class Foo:
cdef class ExtClass(object):
cdef int[::1] mview
def __init__(self):
self.mview = array((10,), itemsize=sizeof(int), format='i')
self.mview = array((10,), itemsize=sizeof(int), format='i')
class pyfoo:
class PyClass(object):
def __init__(self):
self.mview = array((10,), itemsize=sizeof(long), format='l')
......@@ -74,6 +80,540 @@ def call():
returnmvs()
cdef object obj = returnobj()
cdg()
f = Foo()
pf = pyfoo()
f = ExtClass()
pf = PyClass()
cdef int[:] func():
pass
cdef ExtClass get_ext_obj():
print 'get_ext_obj called'
return ExtClass.__new__(ExtClass)
def test_cdef_attribute():
"""
>>> test_cdef_attribute()
Memoryview is not initialized
local variable 'myview' referenced before assignment
get_ext_obj called
Memoryview is not initialized
<MemoryView of 'array' object>
"""
cdef ExtClass extobj = ExtClass.__new__(ExtClass)
try:
print extobj.mview
except AttributeError, e:
print e.args[0]
else:
print "No AttributeError was raised"
cdef int[:] myview
try:
print myview
except UnboundLocalError, e:
print e.args[0]
else:
print "No UnboundLocalError was raised"
# uninitialized assignment is valid
cdef int[:] otherview = myview
try:
print get_ext_obj().mview
except AttributeError, e:
print e.args[0]
else:
print "No AttributeError was raised"
print ExtClass().mview
def basic_struct(MyStruct[:] mslice):
"""
See also buffmt.pyx
>>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)]))
1 2 3 4 5
>>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="bbqii"))
1 2 3 4 5
"""
buf = mslice
print buf[0].a, buf[0].b, buf[0].c, buf[0].d, buf[0].e
def nested_struct(NestedStruct[:] mslice):
"""
See also buffmt.pyx
>>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)]))
1 2 3 4 5
>>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="T{ii}T{2i}i"))
1 2 3 4 5
"""
buf = mslice
print buf[0].x.a, buf[0].x.b, buf[0].y.a, buf[0].y.b, buf[0].z
def packed_struct(PackedStruct[:] mslice):
"""
See also buffmt.pyx
>>> packed_struct(PackedStructMockBuffer(None, [(1, 2)]))
1 2
>>> packed_struct(PackedStructMockBuffer(None, [(1, 2)], format="T{c^i}"))
1 2
>>> packed_struct(PackedStructMockBuffer(None, [(1, 2)], format="T{c=i}"))
1 2
"""
buf = mslice
print buf[0].a, buf[0].b
def nested_packed_struct(NestedPackedStruct[:] mslice):
"""
See also buffmt.pyx
>>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)]))
1 2 3 4 5
>>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="ci^ci@i"))
1 2 3 4 5
>>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="^c@i^ci@i"))
1 2 3 4 5
"""
buf = mslice
print buf[0].a, buf[0].b, buf[0].sub.a, buf[0].sub.b, buf[0].c
def complex_dtype(long double complex[:] mslice):
"""
>>> complex_dtype(LongComplexMockBuffer(None, [(0, -1)]))
-1j
"""
buf = mslice
print buf[0]
def complex_inplace(long double complex[:] mslice):
"""
>>> complex_inplace(LongComplexMockBuffer(None, [(0, -1)]))
(1+1j)
"""
buf = mslice
buf[0] = buf[0] + 1 + 2j
print buf[0]
def complex_struct_dtype(LongComplex[:] mslice):
"""
Note that the format string is "Zg" rather than "2g", yet a struct
is accessed.
>>> complex_struct_dtype(LongComplexMockBuffer(None, [(0, -1)]))
0.0 -1.0
"""
buf = mslice
print buf[0].real, buf[0].imag
def complex_struct_inplace(LongComplex[:] mslice):
"""
>>> complex_struct_inplace(LongComplexMockBuffer(None, [(0, -1)]))
1.0 1.0
"""
buf = mslice
buf[0].real += 1
buf[0].imag += 2
print buf[0].real, buf[0].imag
#
# Getting items and index bounds checking
#
def get_int_2d(int[:, :] mslice, int i, int j):
"""
>>> C = IntMockBuffer("C", range(6), (2,3))
>>> get_int_2d(C, 1, 1)
acquired C
released C
4
Check negative indexing:
>>> get_int_2d(C, -1, 0)
acquired C
acquired C
released C
released C
3
>>> get_int_2d(C, -1, -2)
acquired C
acquired C
released C
released C
4
>>> get_int_2d(C, -2, -3)
acquired C
acquired C
released C
released C
0
Out-of-bounds errors:
>>> get_int_2d(C, 2, 0)
Traceback (most recent call last):
...
IndexError: Out of bounds on buffer access (axis 0)
>>> get_int_2d(C, 0, -4)
Traceback (most recent call last):
...
IndexError: Out of bounds on buffer access (axis 1)
"""
buf = mslice
return buf[i, j]
def set_int_2d(int[:, :] mslice, int i, int j, int value):
"""
Uses get_int_2d to read back the value afterwards. For pure
unit test, one should support reading in MockBuffer instead.
>>> C = IntMockBuffer("C", range(6), (2,3))
>>> set_int_2d(C, 1, 1, 10)
acquired C
acquired C
released C
released C
>>> get_int_2d(C, 1, 1)
acquired C
acquired C
released C
released C
10
Check negative indexing:
>>> set_int_2d(C, -1, 0, 3)
acquired C
acquired C
released C
released C
>>> get_int_2d(C, -1, 0)
acquired C
acquired C
released C
released C
3
>>> set_int_2d(C, -1, -2, 8)
acquired C
acquired C
released C
released C
>>> get_int_2d(C, -1, -2)
acquired C
acquired C
released C
released C
8
>>> set_int_2d(C, -2, -3, 9)
acquired C
acquired C
released C
released C
>>> get_int_2d(C, -2, -3)
acquired C
acquired C
released C
released C
9
Out-of-bounds errors:
>>> set_int_2d(C, 2, 0, 19)
Traceback (most recent call last):
...
IndexError: Out of bounds on buffer access (axis 0)
>>> set_int_2d(C, 0, -4, 19)
Traceback (most recent call last):
...
IndexError: Out of bounds on buffer access (axis 1)
"""
buf = mslice
buf[i, j] = value
#
# Test all kinds of indexing and flags
#
def writable(unsigned short int[:, :, :] mslice):
"""
>>> R = UnsignedShortMockBuffer("R", range(27), shape=(3, 3, 3))
>>> writable(R)
acquired R
released R
>>> [str(x) for x in R.recieved_flags] # Py2/3
['FORMAT', 'ND', 'STRIDES', 'WRITABLE']
"""
buf = mslice
buf[2, 2, 1] = 23
def strided(int[:] mslice):
"""
>>> A = IntMockBuffer("A", range(4))
>>> strided(A)
acquired A
released A
2
Check that the suboffsets were patched back prior to release.
>>> A.release_ok
True
"""
buf = mslice
return buf[2]
def c_contig(int[::1] mslice):
"""
>>> A = IntMockBuffer(None, range(4))
>>> c_contig(A)
2
"""
buf = mslice
return buf[2]
def c_contig_2d(int[:, ::1] mslice):
"""
Multi-dim has seperate implementation
>>> A = IntMockBuffer(None, range(12), shape=(3,4))
>>> c_contig_2d(A)
7
"""
buf = mslice
return buf[1, 3]
def f_contig(int[::1, :] mslice):
"""
>>> A = IntMockBuffer(None, range(4), shape=(2, 2), strides=(1, 2))
>>> f_contig(A)
2
"""
buf = mslice
return buf[0, 1]
def f_contig_2d(int[::1, :] mslice):
"""
Must set up strides manually to ensure Fortran ordering.
>>> A = IntMockBuffer(None, range(12), shape=(4,3), strides=(1, 4))
>>> f_contig_2d(A)
7
"""
buf = mslice
return buf[3, 1]
def generic(int[::view.generic, ::view.generic] mslice1,
int[::view.generic, ::view.generic] mslice2):
"""
>>> A = IntMockBuffer("A", [[0,1,2], [3,4,5], [6,7,8]])
>>> B = IntMockBuffer("B", [[0,1,2], [3,4,5], [6,7,8]], shape=(3, 3), strides=(1, 3))
>>> generic(A, B)
acquired A
acquired B
4
4
10
11
released A
released B
"""
buf1, buf2 = mslice1, mslice2
print buf1[1, 1]
print buf2[1, 1]
buf1[2, -1] = 10
buf2[2, -1] = 11
print buf1[2, 2]
print buf2[2, 2]
def generic_contig(int[::view.generic_contiguous, :] mslice1,
int[::view.generic_contiguous, :] mslice2):
"""
>>> A = IntMockBuffer("A", [[0,1,2], [3,4,5], [6,7,8]])
>>> B = IntMockBuffer("B", [[0,1,2], [3,4,5], [6,7,8]], shape=(3, 3), strides=(1, 3))
>>> generic_contig(A, B)
acquired A
acquired B
4
4
10
11
released A
released B
"""
buf1, buf2 = mslice1, mslice2
print buf1[1, 1]
print buf2[1, 1]
buf1[2, -1] = 10
buf2[2, -1] = 11
print buf1[2, 2]
print buf2[2, 2]
ctypedef int td_cy_int
cdef extern from "bufaccess.h":
ctypedef td_cy_int td_h_short # Defined as short, but Cython doesn't know this!
ctypedef float td_h_double # Defined as double
ctypedef unsigned int td_h_ushort # Defined as unsigned short
ctypedef td_h_short td_h_cy_short
def printbuf_td_cy_int(td_cy_int[:] mslice, shape):
"""
>>> printbuf_td_cy_int(IntMockBuffer(None, range(3)), (3,))
0 1 2 END
>>> printbuf_td_cy_int(ShortMockBuffer(None, range(3)), (3,))
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch, expected 'td_cy_int' but got 'short'
"""
buf = mslice
cdef int i
for i in range(shape[0]):
print buf[i],
print 'END'
def printbuf_td_h_short(td_h_short[:] mslice, shape):
"""
>>> printbuf_td_h_short(ShortMockBuffer(None, range(3)), (3,))
0 1 2 END
>>> printbuf_td_h_short(IntMockBuffer(None, range(3)), (3,))
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch, expected 'td_h_short' but got 'int'
"""
buf = mslice
cdef int i
for i in range(shape[0]):
print buf[i],
print 'END'
def printbuf_td_h_cy_short(td_h_cy_short[:] mslice, shape):
"""
>>> printbuf_td_h_cy_short(ShortMockBuffer(None, range(3)), (3,))
0 1 2 END
>>> printbuf_td_h_cy_short(IntMockBuffer(None, range(3)), (3,))
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch, expected 'td_h_cy_short' but got 'int'
"""
buf = mslice
cdef int i
for i in range(shape[0]):
print buf[i],
print 'END'
def printbuf_td_h_ushort(td_h_ushort[:] mslice, shape):
"""
>>> printbuf_td_h_ushort(UnsignedShortMockBuffer(None, range(3)), (3,))
0 1 2 END
>>> printbuf_td_h_ushort(ShortMockBuffer(None, range(3)), (3,))
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch, expected 'td_h_ushort' but got 'short'
"""
buf = mslice
cdef int i
for i in range(shape[0]):
print buf[i],
print 'END'
def printbuf_td_h_double(td_h_double[:] mslice, shape):
"""
>>> printbuf_td_h_double(DoubleMockBuffer(None, [0.25, 1, 3.125]), (3,))
0.25 1.0 3.125 END
>>> printbuf_td_h_double(FloatMockBuffer(None, [0.25, 1, 3.125]), (3,))
Traceback (most recent call last):
...
ValueError: Buffer dtype mismatch, expected 'td_h_double' but got 'float'
"""
buf = mslice
cdef int i
for i in range(shape[0]):
print buf[i],
print 'END'
#
# Object access
#
def addref(*args):
for item in args: Py_INCREF(item)
def decref(*args):
for item in args: Py_DECREF(item)
def get_refcount(x):
return (<PyObject*>x).ob_refcnt
def printbuf_object(object[:] mslice, shape):
"""
Only play with unique objects, interned numbers etc. will have
unpredictable refcounts.
ObjectMockBuffer doesn't do anything about increfing/decrefing,
we to the "buffer implementor" refcounting directly in the
testcase.
>>> a, b, c = "globally_unique_string_23234123", {4:23}, [34,3]
>>> get_refcount(a), get_refcount(b), get_refcount(c)
(2, 2, 2)
>>> A = ObjectMockBuffer(None, [a, b, c])
>>> printbuf_object(A, (3,))
'globally_unique_string_23234123' 2
{4: 23} 2
[34, 3] 2
"""
buf = mslice
cdef int i
for i in range(shape[0]):
print repr(buf[i]), (<PyObject*>buf[i]).ob_refcnt
def assign_to_object(object[:] mslice, int idx, obj):
"""
See comments on printbuf_object above.
>>> a, b = [1, 2, 3], [4, 5, 6]
>>> get_refcount(a), get_refcount(b)
(2, 2)
>>> addref(a)
>>> A = ObjectMockBuffer(None, [1, a]) # 1, ...,otherwise it thinks nested lists...
>>> get_refcount(a), get_refcount(b)
(3, 2)
>>> assign_to_object(A, 1, b)
>>> get_refcount(a), get_refcount(b)
(2, 3)
>>> decref(b)
"""
buf = mslice
buf[idx] = obj
def assign_temporary_to_object(object[:] mslice):
"""
See comments on printbuf_object above.
>>> a, b = [1, 2, 3], {4:23}
>>> get_refcount(a)
2
>>> addref(a)
>>> A = ObjectMockBuffer(None, [b, a])
>>> get_refcount(a)
3
>>> assign_temporary_to_object(A)
>>> get_refcount(a)
2
>>> printbuf_object(A, (2,))
{4: 23} 2
{1: 8} 2
To avoid leaking a reference in our testcase we need to
replace the temporary with something we can manually decref :-)
>>> assign_to_object(A, 1, a)
>>> decref(a)
"""
buf = mslice
buf[1] = {3-2: 2+(2*4)-2}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment