MemoryViewSlice indexing and object coercion + MemoryView indexing

Cython/Compiler/Buffer.py

@@ -215,6 +215,50 @@ class BufferEntry(object):
     def _for_all_ndim(self, s):
         return [s % (self.cname, i) for i in range(self.type.ndim)]
 
+    def generate_buffer_lookup_code(self, code, index_cnames):
+        # Create buffer lookup and return it
+        # This is done via utility macros/inline functions, which vary
+        # according to the access mode used.
+        params = []
+        nd = self.type.ndim
+        mode = self.type.mode
+        if mode == 'full':
+            for i, s, o in zip(index_cnames,
+                               self.get_buf_stridevars(),
+                               self.get_buf_suboffsetvars()):
+                params.append(i)
+                params.append(s)
+                params.append(o)
+            funcname = "__Pyx_BufPtrFull%dd" % nd
+            funcgen = buf_lookup_full_code
+        else:
+            if mode == 'strided':
+                funcname = "__Pyx_BufPtrStrided%dd" % nd
+                funcgen = buf_lookup_strided_code
+            elif mode == 'c':
+                funcname = "__Pyx_BufPtrCContig%dd" % nd
+                funcgen = buf_lookup_c_code
+            elif mode == 'fortran':
+                funcname = "__Pyx_BufPtrFortranContig%dd" % nd
+                funcgen = buf_lookup_fortran_code
+            else:
+                assert False
+            for i, s in zip(index_cnames, self.get_buf_stridevars()):
+                params.append(i)
+                params.append(s)
+
+        # Make sure the utility code is available
+        if funcname not in code.globalstate.utility_codes:
+            code.globalstate.utility_codes.add(funcname)
+            protocode = code.globalstate['utility_code_proto']
+            defcode = code.globalstate['utility_code_def']
+            funcgen(protocode, defcode, name=funcname, nd=nd)
+
+        buf_ptr_type_code = self.buf_ptr_type.declaration_code("")
+        ptrcode = "%s(%s, %s, %s)" % (funcname, buf_ptr_type_code, self.buf_ptr,
+                                      ", ".join(params))
+        return ptrcode
+
 
 def get_flags(buffer_aux, buffer_type):
     flags = 'PyBUF_FORMAT'
@@ -412,7 +456,7 @@ def put_buffer_lookup_code(entry, index_signeds, index_cnames, directives,
                 cast = "(size_t)"
             code.putln("if (%s) %s = %d;" % (
                 code.unlikely("%s >= %s%s" % (cname, cast, shape)),
-                tmp_cname, dim))
+                              tmp_cname, dim))
         code.globalstate.use_utility_code(raise_indexerror_code)
         code.putln("if (%s) {" % code.unlikely("%s != -1" % tmp_cname))
         code.putln('__Pyx_RaiseBufferIndexError(%s);' % tmp_cname)
@@ -426,48 +470,7 @@ def put_buffer_lookup_code(entry, index_signeds, index_cnames, directives,
             if signed != 0:
                 code.putln("if (%s < 0) %s += %s;" % (cname, cname, shape))
 
-    # Create buffer lookup and return it
-    # This is done via utility macros/inline functions, which vary
-    # according to the access mode used.
-    params = []
-    nd = entry.type.ndim
-    mode = entry.type.mode
-    if mode == 'full':
-        for i, s, o in zip(index_cnames,
-                           entry.get_buf_stridevars(),
-                           entry.get_buf_suboffsetvars()):
-            params.append(i)
-            params.append(s)
-            params.append(o)
-        funcname = "__Pyx_BufPtrFull%dd" % nd
-        funcgen = buf_lookup_full_code
-    else:
-        if mode == 'strided':
-            funcname = "__Pyx_BufPtrStrided%dd" % nd
-            funcgen = buf_lookup_strided_code
-        elif mode == 'c':
-            funcname = "__Pyx_BufPtrCContig%dd" % nd
-            funcgen = buf_lookup_c_code
-        elif mode == 'fortran':
-            funcname = "__Pyx_BufPtrFortranContig%dd" % nd
-            funcgen = buf_lookup_fortran_code
-        else:
-            assert False
-        for i, s in zip(index_cnames, entry.get_buf_stridevars()):
-            params.append(i)
-            params.append(s)
-
-    # Make sure the utility code is available
-    if funcname not in code.globalstate.utility_codes:
-        code.globalstate.utility_codes.add(funcname)
-        protocode = code.globalstate['utility_code_proto']
-        defcode = code.globalstate['utility_code_def']
-        funcgen(protocode, defcode, name=funcname, nd=nd)
-
-    buf_ptr_type_code = entry.buf_ptr_type.declaration_code("")
-    ptrcode = "%s(%s, %s, %s)" % (funcname, buf_ptr_type_code, entry.buf_ptr,
-                                  ", ".join(params))
-    return ptrcode
+    return entry.generate_buffer_lookup_code(code, index_cnames)
 
 
 def use_bufstruct_declare_code(env):

Cython/Compiler/Code.py

@@ -23,7 +23,6 @@ import DebugFlags
 import Errors
 from Cython import Tempita as tempita
 
-from Cython.Utils import none_or_sub
 try:
     from __builtin__ import basestring
 except ImportError:
@@ -162,10 +161,10 @@ class UtilityCodeBase(object):
             kwargs['impl'] = impl
 
         if 'name' not in kwargs:
-            if from_file:
-                kwargs['name'] = os.path.splitext(from_file)[0]
-            else:
-                kwargs['name'] = util_code_name
+            kwargs['name'] = util_code_name
+
+        if 'file' not in kwargs and from_file:
+            kwargs['file'] = from_file
 
         return cls(**kwargs)
 
@@ -188,13 +187,8 @@ class UtilityCodeBase(object):
 
         proto, impl = utilities[util_code_name]
         if context is not None:
-            if '__name' not in context:
-                context['__name'] = util_code_name
-
-            if proto:
-                proto = tempita.sub(proto, **context)
-            if impl:
-                impl = tempita.sub(impl, **context)
+            proto = sub_tempita(proto, context, from_file, util_code_name)
+            impl = sub_tempita(impl, context, from_file, util_code_name)
 
         if cls.is_cython_utility:
             # Remember line numbers
@@ -204,19 +198,56 @@ class UtilityCodeBase(object):
 
     load_as_string = classmethod(load_as_string)
 
+    def none_or_sub(self, s, context, tempita):
+        """
+        Format a string in this utility code with context. If None, do nothing.
+        """
+        if s is None:
+            return None
+
+        if tempita:
+            return sub_tempita(s, context, self.file, self.name)
+
+        return s % context
+
     def __str__(self):
         return "<%s(%s)" % (type(self).__name__, self.name)
 
 
+def sub_tempita(s, context, file, name):
+    "Run tempita on string s with context context."
+    if not s:
+        return None
+
+    if file:
+        context['__name'] = "%s:%s" % (file, name)
+    elif name:
+        context['__name'] = name
+
+    return tempita.sub(s, **context)
+
+
 class UtilityCode(UtilityCodeBase):
-    # Stores utility code to add during code generation.
-    #
-    # See GlobalState.put_utility_code.
-    #
-    # hashes/equals by instance
+    """
+    Stores utility code to add during code generation.
+
+    See GlobalState.put_utility_code.
+
+    hashes/equals by instance
+
+    proto           C prototypes
+    impl            implemenation code
+    init            code to call on module initialization
+    requires        utility code dependencies
+    proto_block     the place in the resulting file where the prototype should
+                    end up
+    name            name of the utility code (or None)
+    file            filename of the utility code file this utility was loaded
+                    from (or None)
+    """
 
     def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None,
-                 proto_block='utility_code_proto', name=None):
+                 proto_block='utility_code_proto', name=None, file=None):
         # proto_block: Which code block to dump prototype in. See GlobalState.
         self.proto = proto
         self.impl = impl
@@ -227,11 +258,13 @@ class UtilityCode(UtilityCodeBase):
         self.specialize_list = []
         self.proto_block = proto_block
         self.name = name
+        self.file = file
 
     def get_tree(self):
         pass
 
-    def specialize(self, pyrex_type=None, **data):
+
+    def specialize(self, pyrex_type=None, tempita=False, **data):
         # Dicts aren't hashable...
         if pyrex_type is not None:
             data['type'] = pyrex_type.declaration_code('')
@@ -244,12 +277,15 @@ class UtilityCode(UtilityCodeBase):
                 requires = None
             else:
                 requires = [r.specialize(data) for r in self.requires]
+
             s = self._cache[key] = UtilityCode(
-                                        none_or_sub(self.proto, data),
-                                        none_or_sub(self.impl, data),
-                                        none_or_sub(self.init, data),
-                                        none_or_sub(self.cleanup, data),
-                                        requires, self.proto_block)
+                    self.none_or_sub(self.proto, data, tempita),
+                    self.none_or_sub(self.impl, data, tempita),
+                    self.none_or_sub(self.init, data, tempita),
+                    self.none_or_sub(self.cleanup, data, tempita),
+                    requires,
+                    self.proto_block)
+
             self.specialize_list.append(s)
             return s
 
@@ -275,6 +311,29 @@ class UtilityCode(UtilityCodeBase):
                 self.cleanup(writer, output.module_pos)
 
 
+class ContentHashingUtilityCode(UtilityCode):
+    "UtilityCode that hashes and compares based on self.proto and self.impl"
+
+    def __hash__(self):
+        return hash((self.proto, self.impl))
+
+    def __eq__(self, other):
+        return (self.proto, self.impl) == (other.proto, other.impl)
+
+
+class LazyUtilityCode(UtilityCodeBase):
+    """
+    Utility code that calls a callback with the root code writer when
+    available. Useful when you only have 'env' but not 'code'.
+    """
+
+    def __init__(self, callback):
+        self.callback = callback
+
+    def put_code(self, globalstate):
+        utility = self.callback(globalstate.rootwriter)
+        globalstate.use_utility_code(utility)
+
 
 class FunctionState(object):
     # return_label     string          function return point label
@@ -1538,6 +1597,15 @@ class CCodeWriter(object):
         for entry in entries:
             self.put_var_xdecref_clear(entry)
 
+    def put_incref_memoryviewslice(self, slice_cname, have_gil=False):
+        self.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil)))
+
+    def put_xdecref_memoryviewslice(self, slice_cname, have_gil=False):
+        self.putln("__PYX_XDEC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil)))
+
+    def put_xgiveref_memoryviewslice(self, slice_cname):
+        self.put_xgiveref("%s.memview" % slice_cname)
+
     def put_init_to_py_none(self, cname, type, nanny=True):
         from PyrexTypes import py_object_type, typecast
         py_none = typecast(type, py_object_type, "Py_None")

Cython/Compiler/CythonScope.py

@@ -10,10 +10,12 @@ import MemoryView
 class CythonScope(ModuleScope):
     is_cython_builtin = 1
 
-    def __init__(self):
+    def __init__(self, context):
         ModuleScope.__init__(self, u'cython', None, None)
         self.pxd_file_loaded = True
         self.populate_cython_scope()
+        # The Main.Context object
+        self.context = context
 
     def lookup_type(self, name):
         # This function should go away when types are all first-level objects.
@@ -80,6 +82,10 @@ class CythonScope(ModuleScope):
         # The view sub-scope
         #
         self.viewscope = viewscope = ModuleScope(u'cython.view', self, None)
+
+        # Hacky monkey patch
+        self.viewscope.global_scope = self.global_scope
+
         self.declare_module('view', viewscope, None)
         viewscope.is_cython_builtin = True
         viewscope.pxd_file_loaded = True
@@ -93,7 +99,7 @@ def create_cython_scope(context, create_testscope):
     # One could in fact probably make it a singleton,
     # but not sure yet whether any code mutates it (which would kill reusing
     # it across different contexts)
-    scope = CythonScope()
+    scope = CythonScope(context)
 
     if create_testscope:
         scope.test_cythonscope()
@@ -129,6 +135,12 @@ cython_test_extclass_utility_code = \
 cythonview_testscope_utility_code = load_testscope_utility("View.TestScope")
 
 view_utility_code = MemoryView.load_memview_cy_utility(
-        "View.MemoryView", requires=(Buffer.GetAndReleaseBufferUtilityCode(),))
-
-cython_array_utility_code = MemoryView.load_memview_cy_utility("CythonArray")
+        "View.MemoryView", context=MemoryView.context,
+        requires=[Buffer.GetAndReleaseBufferUtilityCode(),
+                  MemoryView.memviewslice_declare_code],
+)
+
+cython_array_utility_code = MemoryView.load_memview_cy_utility(
+        "CythonArray",
+        context=MemoryView.context,
+        requires=[view_utility_code])

Cython/Compiler/ExprNodes.py

@@ -587,7 +587,12 @@ class ExprNode(Node):
         if dst_type.is_memoryviewslice:
             import MemoryView
             if not src.type.is_memoryviewslice:
-                src = CoerceToMemViewSliceNode(src, dst_type, env)
+                if src.type.is_pyobject:
+                    src = CoerceToMemViewSliceNode(src, dst_type, env)
+                else:
+                    error(self.pos,
+                          "Cannot convert '%s' to memoryviewslice" %
+                                                                (src_type,))
             elif not MemoryView.src_conforms_to_dst(src.type, dst_type):
                 error(self.pos, "Memoryview '%s' not conformable to memoryview '%s'." %
                         (src.type, dst_type))
@@ -1282,6 +1287,7 @@ class NameNode(AtomicExprNode):
     #  cf_is_null      boolean   Is uninitialized before this node
     #  cf_maybe_null   boolean   Maybe uninitialized before this node
     #  allow_null      boolean   Don't raise UnboundLocalError
+    #  nogil           boolean   Whether it is used in a nogil context
 
     is_name = True
     is_cython_module = False
@@ -1293,6 +1299,7 @@ class NameNode(AtomicExprNode):
     cf_maybe_null = True
     cf_is_null = False
     allow_null = False
+    nogil = False
 
     def create_analysed_rvalue(pos, env, entry):
         node = NameNode(pos)
@@ -1452,6 +1459,7 @@ class NameNode(AtomicExprNode):
             self.is_used_as_rvalue = 1
 
     def nogil_check(self, env):
+        self.nogil = True
         if self.is_used_as_rvalue:
             entry = self.entry
             if entry.is_builtin:
@@ -1655,9 +1663,16 @@ class NameNode(AtomicExprNode):
         else:
             if self.type.is_memoryviewslice:
                 import MemoryView
-                MemoryView.gen_acquire_memoryviewslice(rhs, self.type,
+                MemoryView.put_acquire_memoryviewslice(rhs, self.type,
                         self.entry.is_cglobal, self.result(), self.pos, code)
-                # self.generate_acquire_memoryviewslice(rhs, code)
+
+                if isinstance(rhs, CoerceToMemViewSliceNode):
+                    # We had a new reference, the lhs now has another,
+                    # dispose of ours.
+                    # code.put_xdecref_memoryviewslice(rhs.result())
+                    code.put_decref("%s.memview" % rhs.result(),
+                                    cython_memoryview_ptr_type,
+                                    nanny=False)
 
             elif self.type.is_buffer:
                 # Generate code for doing the buffer release/acquisition.
@@ -1736,12 +1751,17 @@ class NameNode(AtomicExprNode):
                 '__Pyx_DelAttrString(%s, "%s")' % (
                     Naming.module_cname,
                     self.entry.name))
-        elif self.entry.type.is_pyobject:
+        elif self.entry.type.is_pyobject or self.entry.type.is_memoryviewslice:
             if not self.cf_is_null:
                 if self.cf_maybe_null:
                     code.put_error_if_unbound(self.pos, self.entry)
-                code.put_decref(self.result(), self.ctype())
-                code.putln('%s = NULL;' % self.result())
+
+                if self.entry.type.is_pyobject:
+                    code.put_decref(self.result(), self.ctype())
+                    code.putln('%s = NULL;' % self.result())
+                else:
+                    code.put_xdecref_memoryviewslice(self.entry.cname,
+                                                     have_gil=not self.nogil)
         else:
             error(self.pos, "Deletion of C names not supported")
 
@@ -2347,10 +2367,12 @@ class IndexNode(ExprNode):
         buffer_access = False
         memoryviewslice_access = False
 
-        if (self.base.type.is_memoryviewslice and not self.indices and
+        is_memslice = self.base.type.is_memoryviewslice
+
+        if (is_memslice and not self.indices and
                 isinstance(self.index, EllipsisNode)):
             memoryviewslice_access = True
-        elif self.base.type.is_buffer or self.base.type.is_memoryviewslice:
+        elif self.base.type.is_buffer or is_memslice:
             if self.indices:
                 indices = self.indices
             else:
@@ -2358,6 +2380,7 @@ class IndexNode(ExprNode):
                     indices = self.index.args
                 else:
                     indices = [self.index]
+
             if len(indices) == self.base.type.ndim:
                 buffer_access = True
                 skip_child_analysis = True
@@ -2365,14 +2388,9 @@ class IndexNode(ExprNode):
                     x.analyse_types(env)
                     if not x.type.is_int:
                         buffer_access = False
+
             if buffer_access:
                 assert hasattr(self.base, "entry") # Must be a NameNode-like node
-        
-#        if self.base.type.is_memoryviewslice:
-#            assert hasattr(self.base, "entry")
-#            if self.indices or not isinstance(self.index, EllipsisNode):
-#                error(self.pos, "Memoryviews currently support ellipsis indexing only.")
-#            else: memoryviewslice_access = True
 
         # On cloning, indices is cloned. Otherwise, unpack index into indices
         assert not (buffer_access and isinstance(self.index, CloneNode))
@@ -2386,7 +2404,10 @@ class IndexNode(ExprNode):
 
             if getting and self.type.is_pyobject:
                 self.is_temp = True
-            if setting:
+
+            if setting and self.base.type.is_memoryviewslice:
+                self.type.writable_needed = True
+            elif setting:
                 if not self.base.entry.type.writable:
                     error(self.pos, "Writing to readonly buffer")
                 else:
@@ -3990,8 +4011,8 @@ class AttributeNode(ExprNode):
                 import MemoryView
                 MemoryView.put_assign_to_memviewslice(select_code, rhs.result(), self.type,
                         pos=self.pos, code=code)
-                if rhs.is_temp:
-                    code.put_xdecref_clear("%s.memview" % rhs.result(), cython_memoryview_ptr_type)
+                #if rhs.is_temp:
+                #    code.put_xdecref_clear("%s.memview" % rhs.result(), cython_memoryview_ptr_type)
             if not self.type.is_memoryviewslice:
                 code.putln(
                     "%s = %s;" % (
@@ -7647,46 +7668,14 @@ class CoerceToMemViewSliceNode(CoercionNode):
         self.type = dst_type
         self.env = env
         self.is_temp = 1
+        self.arg = arg
 
     def generate_result_code(self, code):
-        import MemoryView, Buffer
-        memviewobj = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True)
-        buf_flag = MemoryView.get_buf_flag(self.type.axes)
-        code.putln("%s = (PyObject *) __pyx_memoryview_new(%s, %s);" %
-                                (memviewobj, self.arg.py_result(), buf_flag))
-        code.putln(code.error_goto_if_PyErr(self.pos))
-        ndim = len(self.type.axes)
-        spec_int_arr = code.funcstate.allocate_temp(
-                PyrexTypes.c_array_type(PyrexTypes.c_int_type, ndim),
-                manage_ref=False)
-        specs_code = MemoryView.specs_to_code(self.type.axes)
-        for idx, cspec in enumerate(specs_code):
-            code.putln("%s[%d] = %s;" % (spec_int_arr, idx, cspec))
-
-        code.globalstate.use_utility_code(Buffer.acquire_utility_code)
-        code.globalstate.use_utility_code(MemoryView.memviewslice_init_code)
-        dtype_typeinfo = Buffer.get_type_information_cname(code, self.type.dtype)
-
-        MemoryView.put_init_entry(self.result(), code)
-        code.putln("{")
-        code.putln("__Pyx_BufFmt_StackElem __pyx_stack[%d];" %
-                self.type.dtype.struct_nesting_depth())
-        result = self.result()
-        if self.type.is_c_contig:
-            c_or_f_flag = "__Pyx_IS_C_CONTIG"
-        elif self.type.is_f_contig:
-            c_or_f_flag = "__Pyx_IS_F_CONTIG"
-        else:
-            c_or_f_flag = "0"
-        code.putln(code.error_goto_if("-1 == __Pyx_ValidateAndInit_memviewslice("
-                        "(struct __pyx_memoryview_obj *) %(memviewobj)s,"
-                        " %(spec_int_arr)s, %(c_or_f_flag)s, %(ndim)d,"
-                        " &%(dtype_typeinfo)s, __pyx_stack, &%(result)s)" % locals(), self.pos))
-        code.putln("}")
-        code.put_gotref(
-                code.as_pyobject("%s.memview" % self.result(), cython_memoryview_ptr_type))
-        code.funcstate.release_temp(memviewobj)
-        code.funcstate.release_temp(spec_int_arr)
+        self.type.create_from_py_utility_code(self.env)
+        code.putln("%s = %s(%s);" % (self.result(),
+                                     self.type.from_py_function,
+                                     self.arg.py_result()))
+
 
 class CastNode(CoercionNode):
     #  Wrap a node in a C type cast.

Cython/Compiler/Main.py

@@ -430,7 +430,7 @@ def create_default_resultobj(compilation_source, options):
 
 def run_pipeline(source, options, full_module_name = None):
     import Pipeline
-    # Set up context
+
     context = options.create_context()
 
     # Set up source object
@@ -438,6 +438,7 @@ def run_pipeline(source, options, full_module_name = None):
     abs_path = os.path.abspath(source)
     source_ext = os.path.splitext(source)[1]
     full_module_name = full_module_name or context.extract_module_name(source, options)
+
     if options.relative_path_in_code_position_comments:
         rel_path = full_module_name.replace('.', os.sep) + source_ext
         if not abs_path.endswith(rel_path):
@@ -520,7 +521,7 @@ class CompilationOptions(object):
 
     def create_context(self):
         return Context(self.include_path, self.compiler_directives,
-                      self.cplus, self.language_level, options=self)
+                       self.cplus, self.language_level, options=self)
 
 
 class CompilationResult(object):
@@ -584,7 +585,8 @@ def compile_multiple(sources, options):
     a CompilationResultSet. Performs timestamp checking and/or recursion
     if these are specified in the options.
     """
-    context = options.create_context()
+    # run_pipeline creates the context
+    # context = options.create_context()
     sources = [os.path.abspath(source) for source in sources]
     processed = set()
     results = CompilationResultSet()

Cython/Compiler/MemoryView.py

@@ -46,17 +46,21 @@ _spec_to_const = {
         'follow' : MEMVIEW_FOLLOW,
         }
 
+_spec_to_abbrev = {
+    'direct'  : 'd',
+    'ptr'     : 'p',
+    'full'    : 'f',
+    'contig'  : 'c',
+    'strided' : 's',
+    'follow'  : '_',
+}
+
 memview_name = u'memoryview'
 memview_typeptr_cname = '__pyx_memoryview_type'
 memview_objstruct_cname = '__pyx_memoryview_obj'
 memviewslice_cname = u'__Pyx_memviewslice'
 
-def specs_to_code(specs):
-    arr = []
-    for access, packing in specs:
-        arr.append("(%s | %s)" % (_spec_to_const[access],
-                                  _spec_to_const[packing]))
-    return arr
+
 
 def put_init_entry(mv_cname, code):
     code.putln("%s.data = NULL;" % mv_cname)
@@ -70,7 +74,7 @@ def mangle_dtype_name(dtype):
 def axes_to_str(axes):
     return "".join([access[0].upper()+packing[0] for (access, packing) in axes])
 
-def gen_acquire_memoryviewslice(rhs, lhs_type, lhs_is_cglobal, lhs_result, lhs_pos, code):
+def put_acquire_memoryviewslice(rhs, lhs_type, lhs_is_cglobal, lhs_result, lhs_pos, code):
     # import MemoryView
     assert rhs.type.is_memoryviewslice
 
@@ -80,32 +84,17 @@ def gen_acquire_memoryviewslice(rhs, lhs_type, lhs_is_cglobal, lhs_result, lhs_p
     else:
         rhstmp = code.funcstate.allocate_temp(lhs_type, manage_ref=False)
         code.putln("%s = %s;" % (rhstmp, rhs.result_as(lhs_type)))
-        code.putln(code.error_goto_if_null("%s.memview" % rhstmp, lhs_pos))
-
-    if not rhs.result_in_temp():
-        code.put_incref("%s.memview" % rhstmp, cython_memoryview_ptr_type)
-
-    if lhs_is_cglobal:
-        code.put_gotref("%s.memview" % lhs_result)
-
-    #XXX: this is here because self.lhs_of_first_assignment is not set correctly,
-    #     once that is working this should take that flag into account.
-    #     See NameNode.generate_assignment_code
-    code.put_xdecref("%s.memview" % lhs_result, cython_memoryview_ptr_type)
-
-    if lhs_is_cglobal:
-        code.put_giveref("%s.memview" % rhstmp)
 
+    code.putln(code.error_goto_if_null("%s.memview" % rhstmp, lhs_pos))
     put_assign_to_memviewslice(lhs_result, rhstmp, lhs_type,
                                      lhs_pos, code=code)
 
-    if rhs.result_in_temp() or not pretty_rhs:
-        code.putln("%s.memview = 0;" % rhstmp)
-
     if not pretty_rhs:
         code.funcstate.release_temp(rhstmp)
 
 def put_assign_to_memviewslice(lhs_cname, rhs_cname, memviewslicetype, pos, code):
+    code.put_xdecref_memoryviewslice(lhs_cname)
+    code.put_incref_memoryviewslice(rhs_cname)
 
     code.putln("%s.memview = %s.memview;" % (lhs_cname, rhs_cname))
     code.putln("%s.data = %s.data;" % (lhs_cname, rhs_cname))
@@ -195,6 +184,45 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry):
     def get_buf_shapevars(self):
         return self._for_all_ndim("%s.shape[%d]")
 
+    def generate_buffer_lookup_code(self, code, index_cnames):
+        bufp = self.buf_ptr
+        type_decl = self.type.dtype.declaration_code("")
+
+        for dim, (access, packing) in enumerate(self.type.axes):
+            shape = "%s.shape[%d]" % (self.cname, dim)
+            stride = "%s.strides[%d]" % (self.cname, dim)
+            suboffset = "%s.suboffsets[%d]" % (self.cname, dim)
+            index = index_cnames[dim]
+
+            if access == 'full' and packing in ('strided', 'follow'):
+                code.globalstate.use_utility_code(memviewslice_index_helpers)
+                bufp = ('__pyx_memviewslice_index_full(%s, %s, %s)' %
+                                            (bufp, index, stride, suboffset))
+
+            elif access == 'full' and packing == 'contig':
+                # We can skip stride multiplication with the cast
+                code.globalstate.use_utility_code(memviewslice_index_helpers)
+                bufp = '((char *) ((%s *) %s) + %s)' % (type_decl, bufp, index)
+                bufp = ('__pyx_memviewslice_index_full_contig(%s, %s)' %
+                                                            (bufp, suboffset))
+
+            elif access == 'ptr' and packing in ('strided', 'follow'):
+                bufp = ("(*((char **) %s + %s * %s) + %s)" %
+                                             (bufp, index, stride, suboffset))
+
+            elif access == 'ptr' and packing == 'contig':
+                bufp = "(*((char **) %s) + %s)" % (bufp, suboffset)
+
+            elif access == 'direct' and packing in ('strided', 'follow'):
+                bufp = "(%s + %s * %s)" % (bufp, index, stride)
+
+            else:
+                assert (access, packing) == ('direct', 'contig'), (access, packing)
+                bufp = '((char *) (((%s *) %s) + %s))' % (type_decl, bufp, index)
+
+            bufp = '( /* dim=%d */ %s )' % (dim, bufp)
+
+        return "((%s *) %s)" % (type_decl, bufp)
 
 def get_copy_func_name(to_memview):
     base = "__Pyx_BufferNew_%s_From_%s_%s"
@@ -370,10 +398,10 @@ class CopyFuncUtilCode(object):
 
         if self.to_memview.is_c_contig:
             mode = 'c'
-            contig_flag = 'PyBUF_C_CONTIGUOUS'
+            contig_flag = memview_c_contiguous
         elif self.to_memview.is_f_contig:
             mode = 'fortran'
-            contig_flag = "PyBUF_F_CONTIGUOUS"
+            contig_flag = memview_f_contiguous
 
         context = dict(
             copy_name=self.copy_func_name,
@@ -735,11 +763,13 @@ class MemoryViewSliceTransform(CythonTransform):
         return node
 
 
-def load_memview_cy_utility(util_code_name, **kwargs):
-    return CythonUtilityCode.load(util_code_name, "MemoryView.pyx", **kwargs)
+def load_memview_cy_utility(util_code_name, context=None, **kwargs):
+    return CythonUtilityCode.load(util_code_name, "MemoryView.pyx",
+                                  context=context, **kwargs)
 
-def load_memview_c_utility(util_code_name, **kwargs):
-    return UtilityCode.load(util_code_name, "MemoryView_C.c", **kwargs)
+def load_memview_c_utility(util_code_name, context=None, **kwargs):
+    return UtilityCode.load(util_code_name, "MemoryView_C.c",
+                            context=context, **kwargs)
 
 context = {
     'memview_struct_name': memview_objstruct_cname,
@@ -753,6 +783,8 @@ memviewslice_declare_code = load_memview_c_utility(
 
 memviewslice_init_code = load_memview_c_utility(
     "MemviewSliceInit",
-    context={'BUF_MAX_NDIMS': Options.buffer_max_dims},
+    context=dict(context, BUF_MAX_NDIMS=Options.buffer_max_dims),
     requires=[memviewslice_declare_code],
 )
+
+memviewslice_index_helpers = load_memview_c_utility("MemviewSliceIndex")
\ No newline at end of file

Cython/Compiler/ModuleNode.py

@@ -79,6 +79,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                          scope.python_include_files)
 
         if merge_scope:
+            # Ensure that we don't generate import code for these entries!
+            for entry in scope.c_class_entries:
+                entry.type.module_name = self.full_module_name
+
             self.scope.merge_in(scope)
 
     def analyse_declarations(self, env):
@@ -345,7 +349,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         self.generate_declarations_for_modules(env, modules, globalstate)
         h_code.write('\n')
 
-        for utilcode in env.utility_code_list:
+        for utilcode in env.utility_code_list[:]:
             globalstate.use_utility_code(utilcode)
         globalstate.finalize_main_c_code()
 
@@ -2255,7 +2259,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         # Generate type import code for extern extension types
         # and type ready code for non-extern ones.
         for entry in env.c_class_entries:
-            if entry.visibility == 'extern':
+            if entry.visibility == 'extern' and not entry.utility_code_definition:
                 self.generate_type_import_code(env, entry.type, entry.pos, code)
             else:
                 self.generate_base_type_import_code(env, entry, code)
@@ -2265,8 +2269,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
 
     def generate_base_type_import_code(self, env, entry, code):
         base_type = entry.type.base_type
-        if base_type and base_type.module_name != env.qualified_name \
-               and not base_type.is_builtin_type:
+        if (base_type and base_type.module_name != env.qualified_name and not
+               base_type.is_builtin_type and not entry.utility_code_definition):
             self.generate_type_import_code(env, base_type, self.pos, code)
 
     def use_type_import_utility_code(self, env):

Cython/Compiler/Nodes.py

@@ -1344,17 +1344,19 @@ class FuncDefNode(StatNode, BlockNode):
             if not entry.in_closure:
                 code.put_var_declaration(entry)
 
+        # Initialize the return variable __pyx_r
         init = ""
         if not self.return_type.is_void:
             if self.return_type.is_pyobject:
                 init = " = NULL"
+            elif self.return_type.is_memoryviewslice:
+                init = "= {0, 0}"
+
             code.putln(
                 "%s%s;" %
                     (self.return_type.declaration_code(Naming.retval_cname),
                      init))
-            if self.return_type.is_memoryviewslice:
-                import MemoryView
-                MemoryView.put_init_entry(Naming.retval_cname, code)
+
         tempvardecl_code = code.insertion_point()
         self.generate_keyword_list(code)
 
@@ -1431,15 +1433,18 @@ class FuncDefNode(StatNode, BlockNode):
                 if (acquire_gil or entry.assignments) and not entry.in_closure:
                     code.put_var_incref(entry)
             if entry.type.is_memoryviewslice:
-                code.put_incref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
+                code.put_incref_memoryviewslice(entry.cname,
+                                                have_gil=not lenv.nogil)
+                #code.put_incref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
         # ----- Initialise local buffer auxiliary variables
         for entry in lenv.var_entries + lenv.arg_entries:
             if entry.type.is_buffer and entry.buffer_aux.buflocal_nd_var.used:
                 Buffer.put_init_vars(entry, code)
         # ----- Initialise local memoryviewslices
         for entry in lenv.var_entries:
-            if entry.type.is_memoryviewslice:
-                MemoryView.put_init_entry(entry.cname, code)
+            if entry.visibility == "private" and not entry.used:
+                continue
+
         # ----- Check and convert arguments
         self.generate_argument_type_tests(code)
         # ----- Acquire buffer arguments
@@ -1544,7 +1549,8 @@ class FuncDefNode(StatNode, BlockNode):
             if not entry.used or entry.in_closure:
                 continue
             if entry.type.is_memoryviewslice:
-                code.put_xdecref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
+                #code.put_xdecref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
+                code.put_xdecref_memoryviewslice(entry.cname)
             if entry.type.is_pyobject:
                 code.put_var_decref(entry)
 
@@ -1554,7 +1560,8 @@ class FuncDefNode(StatNode, BlockNode):
                 if (acquire_gil or entry.assignments) and not entry.in_closure:
                     code.put_var_decref(entry)
             if entry.type.is_memoryviewslice:
-                code.put_decref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
+                code.put_xdecref_memoryviewslice(entry.cname)
+                #code.put_decref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
         if self.needs_closure:
             code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type)
 
@@ -1567,8 +1574,9 @@ class FuncDefNode(StatNode, BlockNode):
                 err_val = default_retval
             if self.return_type.is_pyobject:
                 code.put_xgiveref(self.return_type.as_pyobject(Naming.retval_cname))
-            elif self.return_type.is_memoryviewslice:
-                code.put_xgiveref(code.as_pyobject("%s.memview" % Naming.retval_cname,cython_memoryview_ptr_type))
+            #elif self.return_type.is_memoryviewslice:
+            #    code.put_xgiveref(code.as_pyobject("%s.memview" % Naming.retval_cname,cython_memoryview_ptr_type))
+            #    code.put_xgiveref_memoryviewslice(Naming.retval_cname)
 
         if self.entry.is_special and self.entry.name == "__hash__":
             # Returning -1 for __hash__ is supposed to signal an error
@@ -4153,7 +4161,8 @@ class DelStatNode(StatNode):
     def analyse_expressions(self, env):
         for arg in self.args:
             arg.analyse_target_expression(env, None)
-            if arg.type.is_pyobject:
+            if arg.type.is_pyobject or (arg.is_name and
+                                        arg.type.is_memoryviewslice):
                 pass
             elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
                 self.cpp_check(env)
@@ -4172,7 +4181,7 @@ class DelStatNode(StatNode):
 
     def generate_execution_code(self, code):
         for arg in self.args:
-            if arg.type.is_pyobject:
+            if arg.type.is_pyobject or arg.type.is_memoryviewslice:
                 arg.generate_deletion_code(code)
             elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
                 arg.generate_result_code(code)
@@ -4274,14 +4283,15 @@ class ReturnStatNode(StatNode):
             code.put_xdecref(Naming.retval_cname,
                              self.return_type)
         elif self.return_type.is_memoryviewslice:
-            code.put_xdecref("%s.memview" % Naming.retval_cname,
-                    self.return_type)
+            code.put_xdecref_memoryviewslice(Naming.retval_cname)
+            #code.put_xdecref("%s.memview" % Naming.retval_cname,
+            #        self.return_type)
 
         if self.value:
             self.value.generate_evaluation_code(code)
             if self.return_type.is_memoryviewslice:
                 import MemoryView
-                MemoryView.gen_acquire_memoryviewslice(self.value, self.return_type,
+                MemoryView.put_acquire_memoryviewslice(self.value, self.return_type,
                         False, Naming.retval_cname, None, code)
             else:
                 self.value.make_owned_reference(code)

Cython/Compiler/PyrexTypes.py

@@ -2,12 +2,14 @@
 #   Cython/Python language types
 #
 
-from Code import UtilityCode
+from Code import UtilityCode, LazyUtilityCode
 import StringEncoding
 import Naming
 import copy
 from Errors import error
 
+import cython
+
 class BaseType(object):
     #
     #  Base class for all Cython types including pseudo-types.
@@ -331,6 +333,15 @@ class MemoryViewSliceType(PyrexType):
     has_attributes = 1
     scope = None
 
+    # These are specialcased in Defnode
+    from_py_function = None
+    to_py_function = None
+
+    exception_value = None
+    exception_check = None
+
+    utility_counter = 0
+
     def __init__(self, base_dtype, axes):
         '''
         MemoryViewSliceType(base, axes)
@@ -375,6 +386,7 @@ class MemoryViewSliceType(PyrexType):
         assert not (self.is_c_contig and self.is_f_contig)
 
         self.mode = MemoryView.get_mode(axes)
+        self.writable_needed = False
 
     def same_as_resolved_type(self, other_type):
         return ((other_type.is_memoryviewslice and
@@ -495,11 +507,68 @@ class MemoryViewSliceType(PyrexType):
 
     def global_init_code(self, entry, code):
         code.putln("%s.data = NULL;" % entry.cname)
-        code.put_init_to_py_none("%s.memview" % entry.cname, cython_memoryview_ptr_type, nanny=False)
+        code.putln("%s.memview = NULL;" % entry.cname)
+        #code.put_init_to_py_none("%s.memview" % entry.cname, cython_memoryview_ptr_type, nanny=False)
 
     def check_for_null_code(self, cname):
         return cname + '.memview'
 
+    def create_from_py_utility_code(self, env):
+        import MemoryView, Buffer, Code
+
+        # We don't have 'code', so use a LazyUtilityCode with a callback.
+        def lazy_utility_callback(code):
+            context['dtype_typeinfo'] = Buffer.get_type_information_cname(
+                                                          code, self.dtype)
+            return Code.ContentHashingUtilityCode.load(
+                        "ObjectToMemviewSlice", "MemoryView_C.c", context)
+
+        env.use_utility_code(Buffer.acquire_utility_code)
+        env.use_utility_code(MemoryView.memviewslice_init_code)
+        env.use_utility_code(LazyUtilityCode(lazy_utility_callback))
+
+        if self.is_c_contig:
+            c_or_f_flag = "__Pyx_IS_C_CONTIG"
+        elif self.is_f_contig:
+            c_or_f_flag = "__Pyx_IS_F_CONTIG"
+        else:
+            c_or_f_flag = "0"
+
+        # specializing through UtilityCode.specialize is not so useful as
+        # specialize on too many things to include in the function name
+        funcname = "__Pyx_PyObject_to_MemoryviewSlice_%d" % self.utility_counter
+
+        context = dict(
+            MemoryView.context,
+            buf_flag = MemoryView.get_buf_flag(self.axes),
+            ndim = self.ndim,
+            axes_specs = ', '.join(self.axes_specs_to_code()),
+            dtype_typedecl = self.dtype.declaration_code(""),
+            struct_nesting_depth = self.dtype.struct_nesting_depth(),
+            c_or_f_flag = c_or_f_flag,
+            funcname = funcname,
+        )
+
+        self.from_py_function = funcname
+        MemoryViewSliceType.utility_counter += 1
+
+        return True
+
+    def axes_specs_to_code(self):
+        "Return a list of code constants for each axis"
+        import MemoryView
+        d = MemoryView._spec_to_const
+        return ["(%s | %s)" % (d[a], d[p]) for a, p in self.axes]
+
+    def axes_specs_to_name(self):
+        "Return an abbreviated name for our axes"
+        import MemoryView
+        d = MemoryView._spec_to_abbrev
+        return "".join(["%s%s" % (d[a], d[p]) for a, p in self.axes])
+
+    def error_condition(self, result_code):
+        return "!%s.memview" % result_code
+
 
 class BufferType(BaseType):
     #
@@ -2698,8 +2767,8 @@ cython_memoryview_type = CStructOrUnionType("__pyx_memoryview_obj", "struct",
 
 cython_memoryview_ptr_type = CPtrType(cython_memoryview_type)
 
-memoryviewslice_type = CStructOrUnionType("__Pyx_memviewslice", "struct",
-                                    None, 1, "__Pyx_memviewslice")
+memoryviewslice_type = CStructOrUnionType("memoryviewslice", "struct",
+                                          None, 1, "__Pyx_memviewslice")
 
 error_type =    ErrorType()
 unspecified_type = UnspecifiedType()

Cython/Compiler/Symtab.py

@@ -382,6 +382,10 @@ class Scope(object):
 #                entries[name] = entry
             if not shadow:
                 entries[name] = entry
+
+        if type.is_memoryviewslice:
+            entry.init = "{ 0, 0 }"
+
         entry.scope = self
         entry.visibility = visibility
         return entry

Cython/Compiler/UtilityCode.py

@@ -15,7 +15,7 @@ class NonManglingModuleScope(Symtab.ModuleScope):
         entry.used = True
         return super(NonManglingModuleScope, self).add_imported_entry(
                                                         name, entry, pos)
-    
+
     def mangle(self, prefix, name=None):
         if name:
             if prefix in (Naming.typeobj_prefix, Naming.func_prefix, Naming.var_prefix, Naming.pyfunc_prefix):
@@ -63,7 +63,8 @@ class CythonUtilityCode(Code.UtilityCodeBase):
 
     is_cython_utility = True
 
-    def __init__(self, impl, name="__pyxutil", prefix="", requires=None):
+    def __init__(self, impl, name="__pyxutil", prefix="", requires=None,
+                 file=None):
         # 1) We need to delay the parsing/processing, so that all modules can be
         #    imported without import loops
         # 2) The same utility code object can be used for multiple source files;
@@ -72,6 +73,7 @@ class CythonUtilityCode(Code.UtilityCodeBase):
         # Hence, delay any processing until later.
         self.impl = impl
         self.name = name
+        self.file = file
         self.prefix = prefix
         self.requires = requires or []
 
@@ -113,10 +115,11 @@ class CythonUtilityCode(Code.UtilityCodeBase):
     def put_code(self, output):
         pass
 
-    def declare_in_scope(self, dest_scope, used=False):
+    def declare_in_scope(self, dest_scope, used=False, modname=None):
         """
         Declare all entries from the utility code in dest_scope. Code will only
-        be included for used entries.
+        be included for used entries. If module_name is given, declare the
+        type entries with that name.
         """
         tree = self.get_tree(entries_only=True)
 
@@ -130,6 +133,10 @@ class CythonUtilityCode(Code.UtilityCodeBase):
             entry.utility_code_definition = self
             entry.used = used
 
+            if modname and entry.type.is_extension_type:
+                entry.qualified_name = modname
+                entry.type.module_name = modname
+
         dest_scope.merge_in(tree.scope, merge_unused=True)
         tree.scope = dest_scope
 

Cython/Utility/MemoryView.pyx

@@ -10,7 +10,10 @@ cdef extern from "Python.h":
         PyBUF_C_CONTIGUOUS,
         PyBUF_F_CONTIGUOUS,
         PyBUF_ANY_CONTIGUOUS
+        PyBUF_FORMAT
 
+cdef extern from *:
+    object __pyx_memoryview_new(object obj, int flags)
 
 @cname("__pyx_array")
 cdef class array:
@@ -105,7 +108,12 @@ cdef class array:
         info.strides = self.strides
         info.suboffsets = NULL
         info.itemsize = self.itemsize
-        info.format = self.format
+
+        if flags & PyBUF_FORMAT:
+            info.format = self.format
+        else:
+            info.format = NULL
+
         # we do not need to call releasebuffer
         info.obj = None
 
@@ -130,6 +138,15 @@ cdef class array:
         self.format = NULL
         self.itemsize = 0
 
+    def __getitem__(self, index):
+        view = __pyx_memoryview_new(self, PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT)
+        return view[index]
+
+    def __setitem__(self, index, value):
+        view = __pyx_memoryview_new(self, PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT)
+        view[index] = value
+
+
 @cname("__pyx_array_new")
 cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *mode):
     return array(shape, itemsize, format, mode.decode('ASCII'))
@@ -137,8 +154,10 @@ cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *
 ########## View.MemoryView ##########
 
 # from cpython cimport ...
-cdef extern from "pythread.h":
+cdef extern from "Python.h":
+    int PyIndex_Check(object)
 
+cdef extern from "pythread.h":
     ctypedef void *PyThread_type_lock
 
     PyThread_type_lock PyThread_allocate_lock()
@@ -150,6 +169,12 @@ cdef extern from *:
     int __Pyx_GetBuffer(object, Py_buffer *, int)
     void __Pyx_ReleaseBuffer(Py_buffer *)
 
+    ctypedef struct {{memviewslice_name}}:
+        char *data
+        Py_ssize_t shape[{{max_dims}}]
+        Py_ssize_t strides[{{max_dims}}]
+        Py_ssize_t suboffsets[{{max_dims}}]
+
 
 @cname('__pyx_MemviewEnum')
 cdef class Enum(object):
@@ -173,23 +198,143 @@ cdef class memoryview(object):
 
     cdef object obj
     cdef Py_buffer view
-    cdef PyThread_type_lock acqcnt_lock
+    cdef PyThread_type_lock lock
     cdef int acquisition_count
 
+
     def __cinit__(memoryview self, object obj, int flags):
         self.obj = obj
-        #self.acqcnt_lock = PyThread_allocate_lock()
-        #if self.acqcnt_lock == NULL:
-        #    raise MemoryError
-
         __Pyx_GetBuffer(obj, &self.view, flags)
 
+        self.lock = PyThread_allocate_lock()
+        if self.lock == NULL:
+            raise MemoryError
+
     def __dealloc__(memoryview self):
-        #PyThread_free_lock(self.acqcnt_lock)
-        self.obj = None
         __Pyx_ReleaseBuffer(&self.view)
+        PyThread_free_lock(self.lock)
+
+    @cname('__pyx_memoryview_getitem')
+    def __getitem__(memoryview self, object index):
+        # cdef Py_ssize_t idx
+        cdef char *itemp = <char *> self.view.buf
+        cdef bytes bytesitem
+        cdef str fmt = self.view.format
+
+        import struct
+        try:
+            itemsize = struct.calcsize(fmt)
+        except struct.error:
+            raise TypeError("Unsupported format: %r" % fmt)
+
+        if index is Ellipsis:
+            return self
+
+        elif isinstance(index, slice):
+            if index == slice(None):
+                return self
+
+            raise NotImplementedError
+
+        else:
+            if not isinstance(index, tuple):
+                index = (index,)
+
+            tup = _unellipsify(index, self.view.ndim)
+
+            if len(tup) != self.view.ndim:
+                raise NotImplementedError(
+                        "Expected %d indices (got %d)" %
+                             (self.view.ndim, len(tup)))
+
+            for dim, idx in enumerate(tup):
+                _check_index(idx)
+                itemp = pybuffer_index(&self.view, itemp, idx, dim + 1)
+
+        # Do a manual and complete check here instead of this easy hack
+        bytesitem = itemp[:self.view.itemsize]
+        return struct.unpack(fmt, bytesitem)
+
+
+@cname('__pyx_memoryviewslice')
+cdef class _memoryviewslice(memoryview):
+    "Internal class for passing memory view slices to Python"
+
+    # We need this to keep our shape/strides/suboffset pointers valid
+    cdef {{memviewslice_name}} from_slice
+    # Restore the original Py_buffer before releasing
+    cdef Py_buffer orig_view
+
+    def __cinit__(self, object obj, int flags):
+        self.orig_view = self.view
+
+    def __dealloc__(self):
+        self.view = self.orig_view
 
 
 @cname('__pyx_memoryview_new')
-cdef memoryview memoryview_cwrapper(object o, int flags):
+cdef memoryview_cwrapper(object o, int flags):
     return memoryview(o, flags)
+
+@cname('__pyx_memoryview_fromslice')
+cdef memoryview memoryview_from_memview_cwrapper(memoryview m,  int flags,
+                        int new_ndim, {{memviewslice_name}} *memviewslice):
+    cdef _memoryviewslice result = _memoryviewslice(m.obj, flags)
+
+    result.from_slice = memviewslice[0]
+
+    result.view.shape = <Py_ssize_t *> (&result.from_slice.shape + new_ndim)
+    result.view.strides = <Py_ssize_t *> (&result.from_slice.strides + new_ndim)
+    result.view.suboffsets = <Py_ssize_t *> (&result.from_slice.suboffsets + new_ndim)
+    result.view.ndim = new_ndim
+
+    return result
+
+cdef _check_index(index):
+    if not PyIndex_Check(index):
+        raise TypeError("Cannot index with %s" % type(index))
+
+cdef tuple _unellipsify(tuple tup, int ndim):
+    if Ellipsis in tup:
+        result = []
+        for idx, item in enumerate(tup):
+            if item is Ellipsis:
+                result.extend([slice(None)] * (ndim - len(tup) + 1))
+                result.extend(tup[idx + 1:])
+                break
+
+            result.append(item)
+
+        return tuple(result)
+
+    return tup
+
+@cname('__pyx_pybuffer_index')
+cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, int dim) except NULL:
+    cdef Py_ssize_t shape, stride, suboffset = -1
+    cdef Py_ssize_t itemsize = view.itemsize
+    cdef char *resultp
+
+    if view.ndim == 0:
+        shape = view.len / itemsize
+        stride = itemsize
+    else:
+        shape = view.shape[dim]
+        stride = view.strides[dim]
+        if view.suboffsets != NULL:
+            suboffset = view.suboffsets[dim]
+
+    if index < 0:
+        index += view.shape[dim]
+        if index < 0:
+            raise IndexError("Out of bounds in dimension %d" % dim)
+
+    if index > shape:
+        raise IndexError("Out of bounds in dimension %d" % dim)
+
+    resultp = bufp + index * stride
+
+    if suboffset >= 0:
+        resultp = (<char **> resultp)[0] + suboffset
+
+    return resultp

Cython/Utility/MemoryView_C.c

@@ -12,6 +12,9 @@ typedef struct {
   Py_ssize_t suboffsets[{{max_dims}}];
 } {{memviewslice_name}};
 
+/////////////// ObjectToMemviewSlice.proto ///////////////
+{{# __Pyx_PyObject_to_MemoryviewSlice_<count> }}
+static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *);
 
 ////////// MemviewSliceInit.proto //////////
 
@@ -27,6 +30,7 @@ typedef struct {
 #define __Pyx_IS_C_CONTIG 1
 #define __Pyx_IS_F_CONTIG 2
 
+
 /* #define __PYX_MEMSLICE_GETDATA(SLICE) ((char *) SLICE->memview->view->buf) */
 
 static int __Pyx_ValidateAndInit_memviewslice(struct __pyx_memoryview_obj *memview,
@@ -38,6 +42,49 @@ static int __Pyx_init_memviewslice(
                 int ndim,
                 __Pyx_memviewslice *memviewslice);
 
+#define __PYX_INC_MEMVIEW(slice, have_gil) __Pyx_INC_MEMVIEW(slice, have_gil, __LINE__)
+#define __PYX_XDEC_MEMVIEW(slice, have_gil) __Pyx_XDEC_MEMVIEW(slice, have_gil, __LINE__)
+static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *, int, int);
+static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *, int, int);
+
+/////////////// MemviewSliceIndex.proto ///////////////
+
+static CYTHON_INLINE char *__pyx_memviewslice_index_full(char *bufp, Py_ssize_t idx, Py_ssize_t stride, Py_ssize_t suboffset);
+static CYTHON_INLINE char *__pyx_memviewslice_index_full_contig(char *bufp, Py_ssize_t suboffset);
+
+
+/////////////// ObjectToMemviewSlice ///////////////
+
+{{#__Pyx_PyObject_to_MemoryviewSlice_<count>}}
+
+static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *obj) {
+    {{memviewslice_name}} result;
+    result.memview = NULL;
+    result.data = NULL;
+    struct __pyx_memoryview_obj *memview =  \
+        (struct __pyx_memoryview_obj *) __pyx_memoryview_new(obj, {{buf_flag}});
+    __Pyx_BufFmt_StackElem stack[{{struct_nesting_depth}}];
+    int axes_specs[] = { {{axes_specs}} };
+    int retcode;
+
+    if (unlikely(!memview))
+        goto __pyx_fail;
+
+    retcode = __Pyx_ValidateAndInit_memviewslice(memview, axes_specs,
+            {{c_or_f_flag}}, {{ndim}}, &{{dtype_typeinfo}}, stack, &result);
+
+    if (unlikely(retcode == -1))
+        goto __pyx_fail;
+
+    memview->acquisition_count = 1;
+    return result;
+__pyx_fail:
+    Py_XDECREF(memview);
+    result.memview = NULL;
+    result.data = NULL;
+    return result;
+}
+
 ////////// MemviewSliceInit //////////
 
 static int __Pyx_ValidateAndInit_memviewslice(
@@ -203,8 +250,6 @@ static int __Pyx_init_memviewslice(
         }
     }
 
-    __Pyx_INCREF((PyObject *)memview);
-    __Pyx_GIVEREF((PyObject *)memview);
     memviewslice->memview = memview;
     memviewslice->data = (char *)buf->buf;
     retval = 0;
@@ -220,6 +265,62 @@ no_fail:
     return retval;
 }
 
+static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *memslice,
+                                            int have_gil, int lineno) {
+    int first_time;
+    struct {{memview_struct_name}} *memview = memslice->memview;
+    if (!memview) {
+        char msg[50];
+        snprintf(msg, 50, "memoryslice is not initialized (line %d)", lineno);
+        Py_FatalError(msg);
+    }
+
+    PyThread_acquire_lock(memview->lock, 1);
+    first_time = (memview->acquisition_count++ == 0);
+    PyThread_release_lock(memview->lock);
+
+    /* printf("INCREF %d: acquisition_count=%d, refcount=%d\n", lineno,
+           memview->acquisition_count, memview->ob_refcnt); */
+
+    if (first_time) {
+        if (have_gil) {
+            Py_INCREF((PyObject *) memview);
+        } else {
+            PyGILState_STATE _gilstate = PyGILState_Ensure();
+            Py_INCREF((PyObject *) memview);
+            PyGILState_Release(_gilstate);
+        }
+    }
+}
+
+static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *memslice,
+                                             int have_gil, int lineno) {
+    int last_time;
+    struct {{memview_struct_name}} *memview = memslice->memview;
+
+    if (!memview) {
+        return;
+    }
+
+    PyThread_acquire_lock(memview->lock, 1);
+    last_time = (memview->acquisition_count-- == 1);
+    PyThread_release_lock(memview->lock);
+
+    /* printf("DECREF %d: acquisition_count=%d, refcount=%d\n", lineno,
+           memview->acquisition_count, memview->ob_refcnt); */
+
+    if (last_time) {
+        if (have_gil) {
+            Py_CLEAR(memview);
+        } else {
+            PyGILState_STATE _gilstate = PyGILState_Ensure();
+            Py_CLEAR(memview);
+            PyGILState_Release(_gilstate);
+        }
+        memslice->data = NULL;
+    }
+}
+
 ////////// MemviewSliceCopyTemplate //////////
 
 static __Pyx_memviewslice {{copy_name}}(const __Pyx_memviewslice from_mvs) {
@@ -259,7 +360,8 @@ static __Pyx_memviewslice {{copy_name}}(const __Pyx_memviewslice from_mvs) {
     }
     __Pyx_GOTREF(array_obj);
 
-    memview_obj = __pyx_memoryview_new((PyObject *) array_obj, {{contig_flag}});
+    memview_obj = (struct __pyx_memoryview_obj *) __pyx_memoryview_new(
+                                (PyObject *) array_obj, {{contig_flag}});
     if (unlikely(!memview_obj)) {
         goto fail;
     }
@@ -292,3 +394,21 @@ no_fail:
 
 }
 
+/////////////// MemviewSliceIndex ///////////////
+
+static CYTHON_INLINE char *__pyx_memviewslice_index_full(char *bufp, Py_ssize_t idx, Py_ssize_t stride, Py_ssize_t suboffset) {
+    bufp = bufp + idx * stride;
+    if (suboffset >= 0) {
+        bufp = *((char **) bufp) + suboffset;
+    }
+    return bufp;
+}
+
+/* The call has already done the indexing */
+static CYTHON_INLINE char *__pyx_memviewslice_index_full_contig(char *bufp, Py_ssize_t suboffset) {
+    if (suboffset >= 0) {
+        bufp = *((char **) bufp) + suboffset;
+    }
+    return bufp;
+}
+

Cython/Utils.py

@@ -4,6 +4,7 @@
 #
 
 import os, sys, re, codecs
+from Cython import Tempita
 
 def replace_suffix(path, newsuf):
     base, _ = os.path.splitext(path)
@@ -215,10 +216,3 @@ def long_literal(value):
     if isinstance(value, basestring):
         value = str_to_number(value)
     return not -2**31 <= value < 2**31
-
-def none_or_sub(s, data):
-    if s is None:
-        return s
-    else:
-        return s % data
-

tests/run/bufaccess.pyx

@@ -9,13 +9,6 @@
 
 from __future__ import unicode_literals
 
-from libc cimport stdlib
-from libc cimport stdio
-cimport cpython.buffer
-cimport cython
-
-from cpython cimport PyObject, Py_INCREF, Py_DECREF
-
 __test__ = {}
 
 import sys
@@ -33,9 +26,8 @@ def testcase(func):
     __test__[func.__name__] = doctest
     return func
 
-def testcas(a):
-    pass
 
+include "mockbuffers.pxi"
 
 #
 # Buffer acquire and release tests
@@ -966,246 +958,6 @@ def buffer_cast_fails(object[char, cast=True] buf):
     """
     return buf[0]
 
-
-#
-# Testcase support code (more tests below!, because of scope rules)
-#
-
-
-available_flags = (
-    ('FORMAT', cpython.buffer.PyBUF_FORMAT),
-    ('INDIRECT', cpython.buffer.PyBUF_INDIRECT),
-    ('ND', cpython.buffer.PyBUF_ND),
-    ('STRIDES', cpython.buffer.PyBUF_STRIDES),
-    ('C_CONTIGUOUS', cpython.buffer.PyBUF_C_CONTIGUOUS),
-    ('F_CONTIGUOUS', cpython.buffer.PyBUF_F_CONTIGUOUS),
-    ('WRITABLE', cpython.buffer.PyBUF_WRITABLE)
-)
-
-cdef class MockBuffer:
-    cdef object format, offset
-    cdef void* buffer
-    cdef int len, itemsize, ndim
-    cdef Py_ssize_t* strides
-    cdef Py_ssize_t* shape
-    cdef Py_ssize_t* suboffsets
-    cdef object label, log
-
-    cdef readonly object recieved_flags, release_ok
-    cdef public object fail
-
-    def __init__(self, label, data, shape=None, strides=None, format=None, offset=0):
-        # It is important not to store references to data after the constructor
-        # as refcounting is checked on object buffers.
-        self.label = label
-        self.release_ok = True
-        self.log = ""
-        self.offset = offset
-        self.itemsize = self.get_itemsize()
-        if format is None: format = self.get_default_format()
-        if shape is None: shape = (len(data),)
-        if strides is None:
-            strides = []
-            cumprod = 1
-            rshape = list(shape)
-            rshape.reverse()
-            for s in rshape:
-                strides.append(cumprod)
-                cumprod *= s
-            strides.reverse()
-        strides = [x * self.itemsize for x in strides]
-        suboffsets = [-1] * len(shape)
-        datashape = [len(data)]
-        p = data
-        while True:
-            p = p[0]
-            if isinstance(p, list): datashape.append(len(p))
-            else: break
-        if len(datashape) > 1:
-            # indirect access
-            self.ndim = len(datashape)
-            shape = datashape
-            self.buffer = self.create_indirect_buffer(data, shape)
-            suboffsets = [0] * (self.ndim-1) + [-1]
-            strides = [sizeof(void*)] * (self.ndim-1) + [self.itemsize]
-            self.suboffsets = self.list_to_sizebuf(suboffsets)
-        else:
-            # strided and/or simple access
-            self.buffer = self.create_buffer(data)
-            self.ndim = len(shape)
-            self.suboffsets = NULL
-
-        try:
-            format = format.encode('ASCII')
-        except AttributeError:
-            pass
-        self.format = format
-        self.len = len(data) * self.itemsize
-
-        self.strides = self.list_to_sizebuf(strides)
-        self.shape = self.list_to_sizebuf(shape)
-
-    def __dealloc__(self):
-        stdlib.free(self.strides)
-        stdlib.free(self.shape)
-        if self.suboffsets != NULL:
-            stdlib.free(self.suboffsets)
-            # must recursively free indirect...
-        else:
-            stdlib.free(self.buffer)
-
-    cdef void* create_buffer(self, data):
-        cdef size_t n = <size_t>(len(data) * self.itemsize)
-        cdef char* buf = <char*>stdlib.malloc(n)
-        cdef char* it = buf
-        for value in data:
-            self.write(it, value)
-            it += self.itemsize
-        return buf
-
-    cdef void* create_indirect_buffer(self, data, shape):
-        cdef size_t n = 0
-        cdef void** buf
-        assert shape[0] == len(data)
-        if len(shape) == 1:
-            return self.create_buffer(data)
-        else:
-            shape = shape[1:]
-            n = <size_t>len(data) * sizeof(void*)
-            buf = <void**>stdlib.malloc(n)
-            for idx, subdata in enumerate(data):
-                buf[idx] = self.create_indirect_buffer(subdata, shape)
-            return buf
-
-    cdef Py_ssize_t* list_to_sizebuf(self, l):
-        cdef size_t n = <size_t>len(l) * sizeof(Py_ssize_t)
-        cdef Py_ssize_t* buf = <Py_ssize_t*>stdlib.malloc(n)
-        for i, x in enumerate(l):
-            buf[i] = x
-        return buf
-
-    def __getbuffer__(MockBuffer self, Py_buffer* buffer, int flags):
-        if self.fail:
-            raise ValueError("Failing on purpose")
-
-        self.recieved_flags = []
-        cdef int value
-        for name, value in available_flags:
-            if (value & flags) == value:
-                self.recieved_flags.append(name)
-
-        buffer.buf = <void*>(<char*>self.buffer + (<int>self.offset * self.itemsize))
-        buffer.obj = self
-        buffer.len = self.len
-        buffer.readonly = 0
-        buffer.format = <char*>self.format
-        buffer.ndim = self.ndim
-        buffer.shape = self.shape
-        buffer.strides = self.strides
-        buffer.suboffsets = self.suboffsets
-        buffer.itemsize = self.itemsize
-        buffer.internal = NULL
-        if self.label:
-            msg = "acquired %s" % self.label
-            print msg
-            self.log += msg + "\n"
-
-    def __releasebuffer__(MockBuffer self, Py_buffer* buffer):
-        if buffer.suboffsets != self.suboffsets:
-            self.release_ok = False
-        if self.label:
-            msg = "released %s" % self.label
-            print msg
-            self.log += msg + "\n"
-
-    def printlog(self):
-        print self.log[:-1]
-
-    def resetlog(self):
-        self.log = ""
-
-    cdef int write(self, char* buf, object value) except -1: raise Exception()
-    cdef get_itemsize(self):
-        print "ERROR, not subclassed", self.__class__
-    cdef get_default_format(self):
-        print "ERROR, not subclassed", self.__class__
-
-cdef class CharMockBuffer(MockBuffer):
-    cdef int write(self, char* buf, object value) except -1:
-        (<char*>buf)[0] = <char>value
-        return 0
-    cdef get_itemsize(self): return sizeof(char)
-    cdef get_default_format(self): return b"@b"
-
-cdef class IntMockBuffer(MockBuffer):
-    cdef int write(self, char* buf, object value) except -1:
-        (<int*>buf)[0] = <int>value
-        return 0
-    cdef get_itemsize(self): return sizeof(int)
-    cdef get_default_format(self): return b"@i"
-
-cdef class UnsignedIntMockBuffer(MockBuffer):
-    cdef int write(self, char* buf, object value) except -1:
-        (<unsigned int*>buf)[0] = <unsigned int>value
-        return 0
-    cdef get_itemsize(self): return sizeof(unsigned int)
-    cdef get_default_format(self): return b"@I"
-
-cdef class ShortMockBuffer(MockBuffer):
-    cdef int write(self, char* buf, object value) except -1:
-        (<short*>buf)[0] = <short>value
-        return 0
-    cdef get_itemsize(self): return sizeof(short)
-    cdef get_default_format(self): return b"h" # Try without endian specifier
-
-cdef class UnsignedShortMockBuffer(MockBuffer):
-    cdef int write(self, char* buf, object value) except -1:
-        (<unsigned short*>buf)[0] = <unsigned short>value
-        return 0
-    cdef get_itemsize(self): return sizeof(unsigned short)
-    cdef get_default_format(self): return b"@1H" # Try with repeat count
-
-cdef class FloatMockBuffer(MockBuffer):
-    cdef int write(self, char* buf, object value) except -1:
-        (<float*>buf)[0] = <float>(<double>value)
-        return 0
-    cdef get_itemsize(self): return sizeof(float)
-    cdef get_default_format(self): return b"f"
-
-cdef class DoubleMockBuffer(MockBuffer):
-    cdef int write(self, char* buf, object value) except -1:
-        (<double*>buf)[0] = <double>value
-        return 0
-    cdef get_itemsize(self): return sizeof(double)
-    cdef get_default_format(self): return b"d"
-
-cdef extern from *:
-    void* addr_of_pyobject "(void*)"(object)
-
-cdef class ObjectMockBuffer(MockBuffer):
-    cdef int write(self, char* buf, object value) except -1:
-        (<void**>buf)[0] = addr_of_pyobject(value)
-        return 0
-
-    cdef get_itemsize(self): return sizeof(void*)
-    cdef get_default_format(self): return b"@O"
-
-
-cdef class IntStridedMockBuffer(IntMockBuffer):
-    cdef __cythonbufferdefaults__ = {"mode" : "strided"}
-
-cdef class ErrorBuffer:
-    cdef object label
-
-    def __init__(self, label):
-        self.label = label
-
-    def __getbuffer__(ErrorBuffer self, Py_buffer* buffer, int flags):
-        raise Exception("acquiring %s" % self.label)
-
-    def __releasebuffer__(ErrorBuffer self, Py_buffer* buffer):
-        raise Exception("releasing %s" % self.label)
-
 #
 # Typed buffers
 #
@@ -1257,75 +1009,6 @@ def bufdefaults1(IntStridedMockBuffer[int, ndim=1] buf):
     pass
 
 
-#
-# Structs
-#
-cdef struct MyStruct:
-    char a
-    char b
-    long long int c
-    int d
-    int e
-
-cdef struct SmallStruct:
-    int a
-    int b
-
-cdef struct NestedStruct:
-    SmallStruct x
-    SmallStruct y
-    int z
-
-cdef packed struct PackedStruct:
-    char a
-    int b
-
-cdef struct NestedPackedStruct:
-    char a
-    int b
-    PackedStruct sub
-    int c
-
-cdef class MyStructMockBuffer(MockBuffer):
-    cdef int write(self, char* buf, object value) except -1:
-        cdef MyStruct* s
-        s = <MyStruct*>buf;
-        s.a, s.b, s.c, s.d, s.e = value
-        return 0
-
-    cdef get_itemsize(self): return sizeof(MyStruct)
-    cdef get_default_format(self): return b"2bq2i"
-
-cdef class NestedStructMockBuffer(MockBuffer):
-    cdef int write(self, char* buf, object value) except -1:
-        cdef NestedStruct* s
-        s = <NestedStruct*>buf;
-        s.x.a, s.x.b, s.y.a, s.y.b, s.z = value
-        return 0
-
-    cdef get_itemsize(self): return sizeof(NestedStruct)
-    cdef get_default_format(self): return b"2T{ii}i"
-
-cdef class PackedStructMockBuffer(MockBuffer):
-    cdef int write(self, char* buf, object value) except -1:
-        cdef PackedStruct* s
-        s = <PackedStruct*>buf;
-        s.a, s.b = value
-        return 0
-
-    cdef get_itemsize(self): return sizeof(PackedStruct)
-    cdef get_default_format(self): return b"^ci"
-
-cdef class NestedPackedStructMockBuffer(MockBuffer):
-    cdef int write(self, char* buf, object value) except -1:
-        cdef NestedPackedStruct* s
-        s = <NestedPackedStruct*>buf;
-        s.a, s.b, s.sub.a, s.sub.b, s.c = value
-        return 0
-
-    cdef get_itemsize(self): return sizeof(NestedPackedStruct)
-    cdef get_default_format(self): return b"ci^ci@i"
-
 @testcase
 def basic_struct(object[MyStruct] buf):
     """

tests/run/cythonarray.pyx

@@ -2,10 +2,8 @@
 
 from __future__ import unicode_literals
 
-# from cython cimport array
-# cimport cython.array as array
+from cython cimport array
 cimport cython as cy
-# array = cython.array
 
 def contiguity():
     '''
@@ -69,3 +67,37 @@ def dont_allocate_buffer():
 
 cdef void callback(char *data):
     print "callback called %d" % <long> data
+
+cdef create_array(shape, mode):
+    cdef array result = array(shape, itemsize=sizeof(int), format='i', mode=mode)
+    cdef int *data = <int *> result.data
+    cdef int i, j, cidx, fidx
+
+    for i in range(shape[0]):
+        for j in range(shape[1]):
+            cidx = i * shape[1] + j
+            fidx = i + j * shape[0]
+
+            if mode == 'fortran':
+                data[fidx] = cidx
+            else:
+                data[cidx] = cidx
+
+    return result
+
+def test_cython_array():
+    """
+    >>> test_cython_array()
+    98
+    61
+    98
+    61
+    """
+    cdef int[:, ::1] carr = create_array((14, 10), 'c')
+    cdef int[::1, :] farr = create_array((14, 10), 'fortran')
+
+    print carr[9, 8]
+    print carr[6, 1]
+
+    print farr[9, 8]
+    print farr[6, 1]

tests/run/memslice.pyx

+# Tests the buffer access syntax functionality by constructing
+# mock buffer objects.
+#
+# Note that the buffers are mock objects created for testing
+# the buffer access behaviour -- for instance there is no flag
+# checking in the buffer objects (why test our test case?), rather
+# what we want to test is what is passed into the flags argument.
+#
+
+from __future__ import unicode_literals
+
+from cython cimport view
+
+__test__ = {}
+
+import sys
+import re
+exclude = []#re.compile('object').search]
+
+def testcase(func):
+    for e in exclude:
+        if e(func.__name__):
+            return func
+    doctest = func.__doc__
+    if sys.version_info >= (3,1,1):
+        doctest = doctest.replace('does not have the buffer interface',
+                                  'does not support the buffer interface')
+    __test__[func.__name__] = doctest
+    return func
+
+
+include "mockbuffers.pxi"
+
+#
+# Buffer acquire and release tests
+#
+
+def nousage():
+    """
+    The challenge here is just compilation.
+    """
+    cdef int[:, :] buf
+
+@testcase
+def acquire_release(o1, o2):
+    """
+    >>> A = IntMockBuffer("A", range(6))
+    >>> B = IntMockBuffer("B", range(6))
+    >>> acquire_release(A, B)
+    acquired A
+    released A
+    acquired B
+    released B
+    >>> acquire_release(None, None)
+    >>> acquire_release(None, B)
+    acquired B
+    released B
+    """
+    cdef int[:] buf
+    buf = o1
+    buf = o2
+
+@testcase
+def acquire_raise(o):
+    """
+    Apparently, doctest won't handle mixed exceptions and print
+    stats, so need to circumvent this.
+
+    >>> A = IntMockBuffer("A", range(6))
+    >>> A.resetlog()
+    >>> acquire_raise(A)
+    Traceback (most recent call last):
+        ...
+    Exception: on purpose
+    >>> A.printlog()
+    acquired A
+    released A
+
+    """
+    cdef int[:] buf
+    buf = o
+    raise Exception("on purpose")
+
+@testcase
+def acquire_failure1():
+    """
+    >>> acquire_failure1()
+    acquired working
+    0 3
+    0 3
+    released working
+    """
+    cdef int[:] buf
+    buf = IntMockBuffer("working", range(4))
+    print buf[0], buf[3]
+    try:
+        buf = ErrorBuffer()
+        assert False
+    except Exception:
+        print buf[0], buf[3]
+
+@testcase
+def acquire_failure2():
+    """
+    >>> acquire_failure2()
+    acquired working
+    0 3
+    0 3
+    released working
+    """
+    cdef int[:] buf = IntMockBuffer("working", range(4))
+    print buf[0], buf[3]
+    try:
+        buf = ErrorBuffer()
+        assert False
+    except Exception:
+        print buf[0], buf[3]
+
+@testcase
+def acquire_failure3():
+    """
+    >>> acquire_failure3()
+    acquired working
+    0 3
+    released working
+    acquired working
+    0 3
+    released working
+    """
+    cdef int[:] buf
+    buf = IntMockBuffer("working", range(4))
+    print buf[0], buf[3]
+    try:
+        buf = object()
+        assert False
+    except Exception:
+        print buf[0], buf[3]
+
+
+@testcase
+def acquire_nonbuffer1(first, second=None):
+    """
+    >>> acquire_nonbuffer1(3)
+    Traceback (most recent call last):
+      ...
+    TypeError: 'int' does not have the buffer interface
+    >>> acquire_nonbuffer1(type)
+    Traceback (most recent call last):
+      ...
+    TypeError: 'type' does not have the buffer interface
+    >>> acquire_nonbuffer1(None, 2)
+    Traceback (most recent call last):
+      ...
+    TypeError: 'int' does not have the buffer interface
+    """
+    cdef int[:] buf
+    buf = first
+    buf = second
+
+@testcase
+def acquire_nonbuffer2():
+    """
+    >>> acquire_nonbuffer2()
+    acquired working
+    0 3
+    released working
+    acquired working
+    0 3
+    released working
+    """
+    cdef int[:] buf = IntMockBuffer("working", range(4))
+    print buf[0], buf[3]
+    try:
+        buf = ErrorBuffer
+        assert False
+    except Exception:
+        print buf[0], buf[3]
+
+
+@testcase
+def as_argument(int[:] bufarg, int n):
+    """
+    >>> A = IntMockBuffer("A", range(6))
+    >>> as_argument(A, 6)
+    acquired A
+    0 1 2 3 4 5 END
+    released A
+    """
+    cdef int i
+    for i in range(n):
+        print bufarg[i],
+    print 'END'
+
+@testcase
+def as_argument_defval(int[:] bufarg=IntMockBuffer('default', range(6)), int n=6):
+    """
+    >>> as_argument_defval()
+    acquired default
+    0 1 2 3 4 5 END
+    released default
+    >>> A = IntMockBuffer("A", range(6))
+    >>> as_argument_defval(A, 6)
+    acquired A
+    0 1 2 3 4 5 END
+    released A
+    """
+    cdef int i
+    for i in range(n):
+        print bufarg[i],
+    print 'END'
+
+@testcase
+def cdef_assignment(obj, n):
+    """
+    >>> A = IntMockBuffer("A", range(6))
+    >>> cdef_assignment(A, 6)
+    acquired A
+    0 1 2 3 4 5 END
+    released A
+
+    """
+    cdef int[:] buf = obj
+    cdef int i
+    for i in range(n):
+        print buf[i],
+    print 'END'
+
+@testcase
+def forin_assignment(objs, int pick):
+    """
+    >>> A = IntMockBuffer("A", range(6))
+    >>> B = IntMockBuffer("B", range(6))
+    >>> forin_assignment([A, B, A, A], 2)
+    acquired A
+    2
+    released A
+    acquired B
+    2
+    released B
+    acquired A
+    2
+    released A
+    acquired A
+    2
+    released A
+    """
+    cdef int[:] buf
+    for buf in objs:
+        print buf[pick]
+
+@testcase
+def cascaded_buffer_assignment(obj):
+    """
+    >>> A = IntMockBuffer("A", range(6))
+    >>> cascaded_buffer_assignment(A)
+    acquired A
+    acquired A
+    released A
+    released A
+    """
+    cdef int[:] a, b
+    a = b = obj
+
+@testcase
+def tuple_buffer_assignment1(a, b):
+    """
+    >>> A = IntMockBuffer("A", range(6))
+    >>> B = IntMockBuffer("B", range(6))
+    >>> tuple_buffer_assignment1(A, B)
+    acquired A
+    acquired B
+    released A
+    released B
+    """
+    cdef int[:] x, y
+    x, y = a, b
+
+@testcase
+def tuple_buffer_assignment2(tup):
+    """
+    >>> A = IntMockBuffer("A", range(6))
+    >>> B = IntMockBuffer("B", range(6))
+    >>> tuple_buffer_assignment2((A, B))
+    acquired A
+    acquired B
+    released A
+    released B
+    """
+    cdef int[:] x, y
+    x, y = tup
+
+@testcase
+def explicitly_release_buffer():
+    """
+    >>> explicitly_release_buffer()
+    acquired A
+    released A
+    After release
+    """
+    cdef int[:] x = IntMockBuffer("A", range(10))
+    del x
+    print "After release"
+
+#
+# Getting items and index bounds checking
+#
+@testcase
+def get_int_2d(int[:, :] buf, int i, int j):
+    """
+    >>> C = IntMockBuffer("C", range(6), (2,3))
+    >>> get_int_2d(C, 1, 1)
+    acquired C
+    released C
+    4
+
+    Check negative indexing:
+    >>> get_int_2d(C, -1, 0)
+    acquired C
+    released C
+    3
+    >>> get_int_2d(C, -1, -2)
+    acquired C
+    released C
+    4
+    >>> get_int_2d(C, -2, -3)
+    acquired C
+    released C
+    0
+
+    Out-of-bounds errors:
+    >>> get_int_2d(C, 2, 0)
+    Traceback (most recent call last):
+        ...
+    IndexError: Out of bounds on buffer access (axis 0)
+    >>> get_int_2d(C, 0, -4)
+    Traceback (most recent call last):
+        ...
+    IndexError: Out of bounds on buffer access (axis 1)
+    """
+    return buf[i, j]
+
+@testcase
+def get_int_2d_uintindex(int[:, :] buf, unsigned int i, unsigned int j):
+    """
+    Unsigned indexing:
+    >>> C = IntMockBuffer("C", range(6), (2,3))
+    >>> get_int_2d_uintindex(C, 0, 0)
+    acquired C
+    released C
+    0
+    >>> get_int_2d_uintindex(C, 1, 2)
+    acquired C
+    released C
+    5
+    """
+    # This is most interesting with regards to the C code
+    # generated.
+    return buf[i, j]
+
+@testcase
+def set_int_2d(int[:, :] buf, int i, int j, int value):
+    """
+    Uses get_int_2d to read back the value afterwards. For pure
+    unit test, one should support reading in MockBuffer instead.
+
+    >>> C = IntMockBuffer("C", range(6), (2,3))
+    >>> set_int_2d(C, 1, 1, 10)
+    acquired C
+    released C
+    >>> get_int_2d(C, 1, 1)
+    acquired C
+    released C
+    10
+
+    Check negative indexing:
+    >>> set_int_2d(C, -1, 0, 3)
+    acquired C
+    released C
+    >>> get_int_2d(C, -1, 0)
+    acquired C
+    released C
+    3
+
+    >>> set_int_2d(C, -1, -2, 8)
+    acquired C
+    released C
+    >>> get_int_2d(C, -1, -2)
+    acquired C
+    released C
+    8
+
+    >>> set_int_2d(C, -2, -3, 9)
+    acquired C
+    released C
+    >>> get_int_2d(C, -2, -3)
+    acquired C
+    released C
+    9
+
+    Out-of-bounds errors:
+    >>> set_int_2d(C, 2, 0, 19)
+    Traceback (most recent call last):
+        ...
+    IndexError: Out of bounds on buffer access (axis 0)
+    >>> set_int_2d(C, 0, -4, 19)
+    Traceback (most recent call last):
+        ...
+    IndexError: Out of bounds on buffer access (axis 1)
+
+    """
+    buf[i, j] = value
+
+@testcase
+def list_comprehension(int[:] buf, len):
+    """
+    >>> list_comprehension(IntMockBuffer(None, [1,2,3]), 3)
+    1|2|3
+    """
+    cdef int i
+    print u"|".join([unicode(buf[i]) for i in range(len)])
+
+#
+# The negative_indices buffer option
+#
+@testcase
+def no_negative_indices(object[int, negative_indices=False] buf, int idx):
+    """
+    The most interesting thing here is to inspect the C source and
+    make sure optimal code is produced.
+
+    >>> A = IntMockBuffer(None, range(6))
+    >>> no_negative_indices(A, 3)
+    3
+    >>> no_negative_indices(A, -1)
+    Traceback (most recent call last):
+        ...
+    IndexError: Out of bounds on buffer access (axis 0)
+    """
+    return buf[idx]
+
+@testcase
+@cython.wraparound(False)
+def wraparound_directive(int[:] buf, int pos_idx, int neg_idx):
+    """
+    Again, the most interesting thing here is to inspect the C source.
+
+    >>> A = IntMockBuffer(None, range(4))
+    >>> wraparound_directive(A, 2, -1)
+    5
+    >>> wraparound_directive(A, -1, 2)
+    Traceback (most recent call last):
+        ...
+    IndexError: Out of bounds on buffer access (axis 0)
+    """
+    cdef int byneg
+    with cython.wraparound(True):
+        byneg = buf[neg_idx]
+    return buf[pos_idx] + byneg
+
+
+#
+# Test which flags are passed.
+#
+# @testcase
+# def readonly(obj):
+#     """
+#     >>> R = UnsignedShortMockBuffer("R", range(27), shape=(3, 3, 3))
+#     >>> readonly(R)
+#     acquired R
+#     25
+#     released R
+#     >>> [str(x) for x in R.recieved_flags]  # Works in both py2 and py3
+#     ['FORMAT', 'INDIRECT', 'ND', 'STRIDES']
+#     """
+#     cdef unsigned short int[:, :, :] buf = obj
+#     print buf[2, 2, 1]
+
+@testcase
+def writable(obj):
+    """
+    >>> R = UnsignedShortMockBuffer("R", range(27), shape=(3, 3, 3))
+    >>> writable(R)
+    acquired R
+    released R
+    >>> [str(x) for x in R.recieved_flags] # Py2/3
+    ['FORMAT', 'ND', 'STRIDES', 'WRITABLE']
+    """
+    cdef unsigned short int[:, :, :] buf = obj
+    buf[2, 2, 1] = 23
+
+@testcase
+def strided(int[:] buf):
+    """
+    >>> A = IntMockBuffer("A", range(4))
+    >>> strided(A)
+    acquired A
+    released A
+    2
+    >>> [str(x) for x in A.recieved_flags] # Py2/3
+    ['FORMAT', 'ND', 'STRIDES', 'WRITABLE']
+
+    Check that the suboffsets were patched back prior to release.
+    >>> A.release_ok
+    True
+    """
+    return buf[2]
+
+@testcase
+def c_contig(int[::1] buf):
+    """
+    >>> A = IntMockBuffer(None, range(4))
+    >>> c_contig(A)
+    2
+    >>> [str(x) for x in A.recieved_flags]
+    ['FORMAT', 'ND', 'STRIDES', 'C_CONTIGUOUS']
+    """
+    return buf[2]
+
+@testcase
+def c_contig_2d(int[:, ::1] buf):
+    """
+    Multi-dim has seperate implementation
+
+    >>> A = IntMockBuffer(None, range(12), shape=(3,4))
+    >>> c_contig_2d(A)
+    7
+    >>> [str(x) for x in A.recieved_flags]
+    ['FORMAT', 'ND', 'STRIDES', 'C_CONTIGUOUS']
+    """
+    return buf[1, 3]
+
+@testcase
+def f_contig(int[::1, :] buf):
+    """
+    >>> A = IntMockBuffer(None, range(4), shape=(2, 2))
+    >>> f_contig(A)
+    2
+    >>> [str(x) for x in A.recieved_flags]
+    ['FORMAT', 'ND', 'STRIDES', 'F_CONTIGUOUS']
+    """
+    return buf[0, 1]
+
+@testcase
+def f_contig_2d(object[int, ndim=2, mode='fortran'] buf):
+    """
+    Must set up strides manually to ensure Fortran ordering.
+
+    >>> A = IntMockBuffer(None, range(12), shape=(4,3), strides=(1, 4))
+    >>> f_contig_2d(A)
+    7
+    >>> [str(x) for x in A.recieved_flags]
+    ['FORMAT', 'ND', 'STRIDES', 'F_CONTIGUOUS']
+    """
+    return buf[3, 1]
+
+#
+# Test compiler options for bounds checking. We create an array with a
+# safe "boundary" (memory
+# allocated outside of what it published) and then check whether we get back
+# what we stored in the memory or an error.
+
+@testcase
+def safe_get(int[:] buf, int idx):
+    """
+    >>> A = IntMockBuffer(None, range(10), shape=(3,), offset=5)
+
+    Validate our testing buffer...
+    >>> safe_get(A, 0)
+    5
+    >>> safe_get(A, 2)
+    7
+    >>> safe_get(A, -3)
+    5
+
+    Access outside it. This is already done above for bounds check
+    testing but we include it to tell the story right.
+
+    >>> safe_get(A, -4)
+    Traceback (most recent call last):
+        ...
+    IndexError: Out of bounds on buffer access (axis 0)
+    >>> safe_get(A, 3)
+    Traceback (most recent call last):
+        ...
+    IndexError: Out of bounds on buffer access (axis 0)
+    """
+    return buf[idx]
+
+@testcase
+@cython.boundscheck(False) # outer decorators should take precedence
+@cython.boundscheck(True)
+def unsafe_get(int[:] buf, int idx):
+    """
+    Access outside of the area the buffer publishes.
+    >>> A = IntMockBuffer(None, range(10), shape=(3,), offset=5)
+    >>> unsafe_get(A, -4)
+    4
+    >>> unsafe_get(A, -5)
+    3
+    >>> unsafe_get(A, 3)
+    8
+    """
+    return buf[idx]
+
+# @testcase
+# @cython.boundscheck(False)
+# def unsafe_get_nonegative(object[int, negative_indices=False] buf, int idx):
+#     """
+#     Also inspect the C source to see that it is optimal...
+#
+#     >>> A = IntMockBuffer(None, range(10), shape=(3,), offset=5)
+#     >>> unsafe_get_nonegative(A, -2)
+#     3
+#     """
+#     return buf[idx]
+
+@testcase
+def mixed_get(int[:] buf, int unsafe_idx, int safe_idx):
+    """
+    >>> A = IntMockBuffer(None, range(10), shape=(3,), offset=5)
+    >>> mixed_get(A, -4, 0)
+    (4, 5)
+    >>> mixed_get(A, 0, -4)
+    Traceback (most recent call last):
+        ...
+    IndexError: Out of bounds on buffer access (axis 0)
+    """
+    with cython.boundscheck(False):
+        one = buf[unsafe_idx]
+    with cython.boundscheck(True):
+        two = buf[safe_idx]
+    return (one, two)
+
+#
+# Coercions
+#
+## @testcase
+## def coercions(object[unsigned char] uc):
+##     """
+## TODO
+##     """
+##     print type(uc[0])
+##     uc[0] = -1
+##     print uc[0]
+##     uc[0] = <int>3.14
+##     print uc[0]
+
+##     cdef char* ch = b"asfd"
+##     cdef object[object] objbuf
+##     objbuf[3] = ch
+
+
+#
+# Testing that accessing data using various types of buffer access
+# all works.
+#
+
+def printbuf_int(int[:] buf, shape):
+    # Utility func
+    cdef int i
+    for i in range(shape[0]):
+        print buf[i],
+    print 'END'
+
+
+@testcase
+def printbuf_int_2d(o, shape):
+    """
+    Strided:
+
+    >>> printbuf_int_2d(IntMockBuffer("A", range(6), (2,3)), (2,3))
+    acquired A
+    0 1 2 END
+    3 4 5 END
+    released A
+    >>> printbuf_int_2d(IntMockBuffer("A", range(100), (3,3), strides=(20,5)), (3,3))
+    acquired A
+    0 5 10 END
+    20 25 30 END
+    40 45 50 END
+    released A
+
+    Indirect:
+    >>> printbuf_int_2d(IntMockBuffer("A", [[1,2],[3,4]]), (2,2))
+    acquired A
+    1 2 END
+    3 4 END
+    released A
+    """
+    # should make shape builtin
+    cdef int[:, :] buf
+    buf = o
+    cdef int i, j
+    for i in range(shape[0]):
+        for j in range(shape[1]):
+            print buf[i, j],
+        print 'END'
+
+@testcase
+def printbuf_float(o, shape):
+    """
+    >>> printbuf_float(FloatMockBuffer("F", [1.0, 1.25, 0.75, 1.0]), (4,))
+    acquired F
+    1.0 1.25 0.75 1.0 END
+    released F
+    """
+
+    # should make shape builtin
+    cdef float[:] buf
+    buf = o
+    cdef int i, j
+    for i in range(shape[0]):
+        print buf[i],
+    print "END"
+
+
+#
+# Test assignments
+#
+@testcase
+def inplace_operators(int[:] buf):
+    """
+    >>> buf = IntMockBuffer(None, [2, 2])
+    >>> inplace_operators(buf)
+    >>> printbuf_int(buf, (2,))
+    0 3 END
+    """
+    cdef int j = 0
+    buf[1] += 1
+    buf[j] *= 2
+    buf[0] -= 4
+
+
+
+#
+# Typedefs
+#
+# Test three layers of typedefs going through a h file for plain int, and
+# simply a header file typedef for floats and unsigned.
+
+ctypedef int td_cy_int
+cdef extern from "bufaccess.h":
+    ctypedef td_cy_int td_h_short # Defined as short, but Cython doesn't know this!
+    ctypedef float td_h_double # Defined as double
+    ctypedef unsigned int td_h_ushort # Defined as unsigned short
+ctypedef td_h_short td_h_cy_short
+
+@testcase
+def printbuf_td_cy_int(td_cy_int[:] buf, shape):
+    """
+    >>> printbuf_td_cy_int(IntMockBuffer(None, range(3)), (3,))
+    0 1 2 END
+    >>> printbuf_td_cy_int(ShortMockBuffer(None, range(3)), (3,))
+    Traceback (most recent call last):
+       ...
+    ValueError: Buffer dtype mismatch, expected 'td_cy_int' but got 'short'
+    """
+    cdef int i
+    for i in range(shape[0]):
+        print buf[i],
+    print 'END'
+
+@testcase
+def printbuf_td_h_short(object[td_h_short] buf, shape):
+    """
+    >>> printbuf_td_h_short(ShortMockBuffer(None, range(3)), (3,))
+    0 1 2 END
+    >>> printbuf_td_h_short(IntMockBuffer(None, range(3)), (3,))
+    Traceback (most recent call last):
+       ...
+    ValueError: Buffer dtype mismatch, expected 'td_h_short' but got 'int'
+    """
+    cdef int i
+    for i in range(shape[0]):
+        print buf[i],
+    print 'END'
+
+@testcase
+def printbuf_td_h_cy_short(object[td_h_cy_short] buf, shape):
+    """
+    >>> printbuf_td_h_cy_short(ShortMockBuffer(None, range(3)), (3,))
+    0 1 2 END
+    >>> printbuf_td_h_cy_short(IntMockBuffer(None, range(3)), (3,))
+    Traceback (most recent call last):
+       ...
+    ValueError: Buffer dtype mismatch, expected 'td_h_cy_short' but got 'int'
+    """
+    cdef int i
+    for i in range(shape[0]):
+        print buf[i],
+    print 'END'
+
+@testcase
+def printbuf_td_h_ushort(object[td_h_ushort] buf, shape):
+    """
+    >>> printbuf_td_h_ushort(UnsignedShortMockBuffer(None, range(3)), (3,))
+    0 1 2 END
+    >>> printbuf_td_h_ushort(ShortMockBuffer(None, range(3)), (3,))
+    Traceback (most recent call last):
+       ...
+    ValueError: Buffer dtype mismatch, expected 'td_h_ushort' but got 'short'
+    """
+    cdef int i
+    for i in range(shape[0]):
+        print buf[i],
+    print 'END'
+
+@testcase
+def printbuf_td_h_double(object[td_h_double] buf, shape):
+    """
+    >>> printbuf_td_h_double(DoubleMockBuffer(None, [0.25, 1, 3.125]), (3,))
+    0.25 1.0 3.125 END
+    >>> printbuf_td_h_double(FloatMockBuffer(None, [0.25, 1, 3.125]), (3,))
+    Traceback (most recent call last):
+       ...
+    ValueError: Buffer dtype mismatch, expected 'td_h_double' but got 'float'
+    """
+    cdef int i
+    for i in range(shape[0]):
+        print buf[i],
+    print 'END'
+
+
+#
+# Object access
+#
+def addref(*args):
+    for item in args: Py_INCREF(item)
+def decref(*args):
+    for item in args: Py_DECREF(item)
+
+def get_refcount(x):
+    return (<PyObject*>x).ob_refcnt
+
+@testcase
+def printbuf_object(object[object] buf, shape):
+    """
+    Only play with unique objects, interned numbers etc. will have
+    unpredictable refcounts.
+
+    ObjectMockBuffer doesn't do anything about increfing/decrefing,
+    we to the "buffer implementor" refcounting directly in the
+    testcase.
+
+    >>> a, b, c = "globally_unique_string_23234123", {4:23}, [34,3]
+    >>> get_refcount(a), get_refcount(b), get_refcount(c)
+    (2, 2, 2)
+    >>> A = ObjectMockBuffer(None, [a, b, c])
+    >>> printbuf_object(A, (3,))
+    'globally_unique_string_23234123' 2
+    {4: 23} 2
+    [34, 3] 2
+    """
+    cdef int i
+    for i in range(shape[0]):
+        print repr(buf[i]), (<PyObject*>buf[i]).ob_refcnt
+
+@testcase
+def assign_to_object(object[object] buf, int idx, obj):
+    """
+    See comments on printbuf_object above.
+
+    >>> a, b = [1, 2, 3], [4, 5, 6]
+    >>> get_refcount(a), get_refcount(b)
+    (2, 2)
+    >>> addref(a)
+    >>> A = ObjectMockBuffer(None, [1, a]) # 1, ...,otherwise it thinks nested lists...
+    >>> get_refcount(a), get_refcount(b)
+    (3, 2)
+    >>> assign_to_object(A, 1, b)
+    >>> get_refcount(a), get_refcount(b)
+    (2, 3)
+    >>> decref(b)
+    """
+    buf[idx] = obj
+
+@testcase
+def assign_temporary_to_object(object[object] buf):
+    """
+    See comments on printbuf_object above.
+
+    >>> a, b = [1, 2, 3], {4:23}
+    >>> get_refcount(a)
+    2
+    >>> addref(a)
+    >>> A = ObjectMockBuffer(None, [b, a])
+    >>> get_refcount(a)
+    3
+    >>> assign_temporary_to_object(A)
+    >>> get_refcount(a)
+    2
+
+    >>> printbuf_object(A, (2,))
+    {4: 23} 2
+    {1: 8} 2
+
+    To avoid leaking a reference in our testcase we need to
+    replace the temporary with something we can manually decref :-)
+    >>> assign_to_object(A, 1, a)
+    >>> decref(a)
+    """
+    buf[1] = {3-2: 2+(2*4)-2}
+
+#
+# cast option
+#
+@testcase
+def buffer_cast(object[unsigned int, cast=True] buf, int idx):
+    """
+    Round-trip a signed int through unsigned int buffer access.
+
+    >>> A = IntMockBuffer(None, [-100])
+    >>> buffer_cast(A, 0)
+    -100
+    """
+    cdef unsigned int data = buf[idx]
+    return <int>data
+
+@testcase
+def buffer_cast_fails(object[char, cast=True] buf):
+    """
+    Cannot cast between datatype of different sizes.
+
+    >>> buffer_cast_fails(IntMockBuffer(None, [0]))
+    Traceback (most recent call last):
+        ...
+    ValueError: Item size of buffer (4 bytes) does not match size of 'char' (1 byte)
+    """
+    return buf[0]
+
+#
+# Typed buffers
+#
+@testcase
+def typedbuffer1(obj):
+    """
+    >>> typedbuffer1(IntMockBuffer("A", range(10)))
+    acquired A
+    released A
+    >>> typedbuffer1(None)
+    >>> typedbuffer1(4)
+    Traceback (most recent call last):
+       ...
+    TypeError: Cannot convert int to memslice.IntMockBuffer
+    """
+    cdef IntMockBuffer[int, ndim=1] buf = obj
+
+@testcase
+def typedbuffer2(IntMockBuffer[int, ndim=1] obj):
+    """
+    >>> typedbuffer2(IntMockBuffer("A", range(10)))
+    acquired A
+    released A
+    >>> typedbuffer2(None)
+    >>> typedbuffer2(4)
+    Traceback (most recent call last):
+       ...
+    TypeError: Argument 'obj' has incorrect type (expected memslice.IntMockBuffer, got int)
+    """
+    pass
+
+#
+# Test __cythonbufferdefaults__
+#
+@testcase
+def bufdefaults1(IntStridedMockBuffer[int, ndim=1] buf):
+    """
+    For IntStridedMockBuffer, mode should be
+    "strided" by defaults which should show
+    up in the flags.
+
+    >>> A = IntStridedMockBuffer("A", range(10))
+    >>> bufdefaults1(A)
+    acquired A
+    released A
+    >>> [str(x) for x in A.recieved_flags]
+    ['FORMAT', 'ND', 'STRIDES']
+    """
+    pass
+
+
+@testcase
+def basic_struct(object[MyStruct] buf):
+    """
+    See also buffmt.pyx
+
+    >>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)]))
+    1 2 3 4 5
+    >>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="bbqii"))
+    1 2 3 4 5
+    """
+    print buf[0].a, buf[0].b, buf[0].c, buf[0].d, buf[0].e
+
+@testcase
+def nested_struct(object[NestedStruct] buf):
+    """
+    See also buffmt.pyx
+
+    >>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)]))
+    1 2 3 4 5
+    >>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="T{ii}T{2i}i"))
+    1 2 3 4 5
+    """
+    print buf[0].x.a, buf[0].x.b, buf[0].y.a, buf[0].y.b, buf[0].z
+
+@testcase
+def packed_struct(object[PackedStruct] buf):
+    """
+    See also buffmt.pyx
+
+    >>> packed_struct(PackedStructMockBuffer(None, [(1, 2)]))
+    1 2
+    >>> packed_struct(PackedStructMockBuffer(None, [(1, 2)], format="T{c^i}"))
+    1 2
+    >>> packed_struct(PackedStructMockBuffer(None, [(1, 2)], format="T{c=i}"))
+    1 2
+
+    """
+    print buf[0].a, buf[0].b
+
+@testcase
+def nested_packed_struct(object[NestedPackedStruct] buf):
+    """
+    See also buffmt.pyx
+
+    >>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)]))
+    1 2 3 4 5
+    >>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="ci^ci@i"))
+    1 2 3 4 5
+    >>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="^c@i^ci@i"))
+    1 2 3 4 5
+    """
+    print buf[0].a, buf[0].b, buf[0].sub.a, buf[0].sub.b, buf[0].c
+
+cdef struct LongComplex:
+    long double real
+    long double imag
+
+cdef class LongComplexMockBuffer(MockBuffer):
+    cdef int write(self, char* buf, object value) except -1:
+        cdef LongComplex* s
+        s = <LongComplex*>buf;
+        s.real, s.imag = value
+        return 0
+
+    cdef get_itemsize(self): return sizeof(LongComplex)
+    cdef get_default_format(self): return b"Zg"
+
+#cdef extern from "complex.h":
+#    pass
+
+@testcase
+def complex_dtype(object[long double complex] buf):
+    """
+    >>> complex_dtype(LongComplexMockBuffer(None, [(0, -1)]))
+    -1j
+    """
+    print buf[0]
+
+@testcase
+def complex_inplace(object[long double complex] buf):
+    """
+    >>> complex_inplace(LongComplexMockBuffer(None, [(0, -1)]))
+    (1+1j)
+    """
+    buf[0] = buf[0] + 1 + 2j
+    print buf[0]
+
+@testcase
+def complex_struct_dtype(object[LongComplex] buf):
+    """
+    Note that the format string is "Zg" rather than "2g", yet a struct
+    is accessed.
+    >>> complex_struct_dtype(LongComplexMockBuffer(None, [(0, -1)]))
+    0.0 -1.0
+    """
+    print buf[0].real, buf[0].imag
+
+@testcase
+def complex_struct_inplace(object[LongComplex] buf):
+    """
+    >>> complex_struct_inplace(LongComplexMockBuffer(None, [(0, -1)]))
+    1.0 1.0
+    """
+    buf[0].real += 1
+    buf[0].imag += 2
+    print buf[0].real, buf[0].imag
+
+#
+# Nogil
+#
+@testcase
+@cython.boundscheck(False)
+def buffer_nogil():
+    """
+    >>> buffer_nogil()
+    10
+    """
+    cdef int[:] buf = IntMockBuffer(None, [1,2,3])
+    with nogil:
+        buf[1] = 10
+    return buf[1]

tests/run/memslice_indexing.pyx

-cimport cython
-from cython cimport array
-
-from libc.stdlib cimport malloc, free
-
-def create_array(shape, mode='c'):
-    cdef array result = array(shape, itemsize=sizeof(int), format='i', mode=mode)
-    cdef int *data = <int *> result.data
-    cdef int i, j, value
-
-    for i in range(shape[0]):
-        for j in range(shape[1]):
-            value = i * shape[0] + j
-            if mode == 'fortran':
-                data[i + j * 10] = value
-            else:
-                data[value] = value
-
-    return result
-
-def slice_contig_indexing():
-    """
-    >>> print("disabled")
-    disabled
-
-    slice_contig_indexing()
-    98
-    61
-    98
-    61
-    """
-    cdef int[:, ::1] carr = create_array((14, 10))
-    cdef int[::1, :] farr = create_array((10, 14), mode='fortran')
-
-    print carr[9, 8]
-    print carr[6, 1]
-
-    print farr[9, 8]
-    print farr[6, 1]
-

tests/run/mockbuffers.pxi

+from libc cimport stdlib
+from libc cimport stdio
+cimport cpython.buffer
+cimport cython
+
+from cpython cimport PyObject, Py_INCREF, Py_DECREF
+
+
+available_flags = (
+    ('FORMAT', cpython.buffer.PyBUF_FORMAT),
+    ('INDIRECT', cpython.buffer.PyBUF_INDIRECT),
+    ('ND', cpython.buffer.PyBUF_ND),
+    ('STRIDES', cpython.buffer.PyBUF_STRIDES),
+    ('C_CONTIGUOUS', cpython.buffer.PyBUF_C_CONTIGUOUS),
+    ('F_CONTIGUOUS', cpython.buffer.PyBUF_F_CONTIGUOUS),
+    ('WRITABLE', cpython.buffer.PyBUF_WRITABLE)
+)
+
+cdef class MockBuffer:
+    cdef object format, offset
+    cdef void* buffer
+    cdef int len, itemsize, ndim
+    cdef Py_ssize_t* strides
+    cdef Py_ssize_t* shape
+    cdef Py_ssize_t* suboffsets
+    cdef object label, log
+
+    cdef readonly object recieved_flags, release_ok
+    cdef public object fail
+
+    def __init__(self, label, data, shape=None, strides=None, format=None, offset=0):
+        # It is important not to store references to data after the constructor
+        # as refcounting is checked on object buffers.
+        self.label = label
+        self.release_ok = True
+        self.log = ""
+        self.offset = offset
+        self.itemsize = self.get_itemsize()
+        if format is None: format = self.get_default_format()
+        if shape is None: shape = (len(data),)
+        if strides is None:
+            strides = []
+            cumprod = 1
+            rshape = list(shape)
+            rshape.reverse()
+            for s in rshape:
+                strides.append(cumprod)
+                cumprod *= s
+            strides.reverse()
+        strides = [x * self.itemsize for x in strides]
+        suboffsets = [-1] * len(shape)
+        datashape = [len(data)]
+        p = data
+        while True:
+            p = p[0]
+            if isinstance(p, list): datashape.append(len(p))
+            else: break
+        if len(datashape) > 1:
+            # indirect access
+            self.ndim = len(datashape)
+            shape = datashape
+            self.buffer = self.create_indirect_buffer(data, shape)
+            suboffsets = [0] * (self.ndim-1) + [-1]
+            strides = [sizeof(void*)] * (self.ndim-1) + [self.itemsize]
+            self.suboffsets = self.list_to_sizebuf(suboffsets)
+        else:
+            # strided and/or simple access
+            self.buffer = self.create_buffer(data)
+            self.ndim = len(shape)
+            self.suboffsets = NULL
+
+        try:
+            format = format.encode('ASCII')
+        except AttributeError:
+            pass
+        self.format = format
+        self.len = len(data) * self.itemsize
+
+        self.strides = self.list_to_sizebuf(strides)
+        self.shape = self.list_to_sizebuf(shape)
+
+    def __dealloc__(self):
+        stdlib.free(self.strides)
+        stdlib.free(self.shape)
+        if self.suboffsets != NULL:
+            stdlib.free(self.suboffsets)
+            # must recursively free indirect...
+        else:
+            stdlib.free(self.buffer)
+
+    cdef void* create_buffer(self, data) except NULL:
+        cdef size_t n = <size_t>(len(data) * self.itemsize)
+        cdef char* buf = <char*>stdlib.malloc(n)
+        if buf == NULL:
+            raise MemoryError
+        cdef char* it = buf
+        for value in data:
+            self.write(it, value)
+            it += self.itemsize
+        return buf
+
+    cdef void* create_indirect_buffer(self, data, shape):
+        cdef size_t n = 0
+        cdef void** buf
+        assert shape[0] == len(data)
+        if len(shape) == 1:
+            return self.create_buffer(data)
+        else:
+            shape = shape[1:]
+            n = <size_t>len(data) * sizeof(void*)
+            buf = <void**>stdlib.malloc(n)
+            for idx, subdata in enumerate(data):
+                buf[idx] = self.create_indirect_buffer(subdata, shape)
+            return buf
+
+    cdef Py_ssize_t* list_to_sizebuf(self, l):
+        cdef size_t n = <size_t>len(l) * sizeof(Py_ssize_t)
+        cdef Py_ssize_t* buf = <Py_ssize_t*>stdlib.malloc(n)
+        for i, x in enumerate(l):
+            buf[i] = x
+        return buf
+
+    def __getbuffer__(MockBuffer self, Py_buffer* buffer, int flags):
+        if self.fail:
+            raise ValueError("Failing on purpose")
+
+        self.recieved_flags = []
+        cdef int value
+        for name, value in available_flags:
+            if (value & flags) == value:
+                self.recieved_flags.append(name)
+
+        buffer.buf = <void*>(<char*>self.buffer + (<int>self.offset * self.itemsize))
+        buffer.obj = self
+        buffer.len = self.len
+        buffer.readonly = 0
+        buffer.format = <char*>self.format
+        buffer.ndim = self.ndim
+        buffer.shape = self.shape
+        buffer.strides = self.strides
+        buffer.suboffsets = self.suboffsets
+        buffer.itemsize = self.itemsize
+        buffer.internal = NULL
+        if self.label:
+            msg = "acquired %s" % self.label
+            print msg
+            self.log += msg + "\n"
+
+    def __releasebuffer__(MockBuffer self, Py_buffer* buffer):
+        if buffer.suboffsets != self.suboffsets:
+            self.release_ok = False
+        if self.label:
+            msg = "released %s" % self.label
+            print msg
+            self.log += msg + "\n"
+
+    def printlog(self):
+        print self.log[:-1]
+
+    def resetlog(self):
+        self.log = ""
+
+    cdef int write(self, char* buf, object value) except -1: raise Exception()
+    cdef get_itemsize(self):
+        print "ERROR, not subclassed", self.__class__
+    cdef get_default_format(self):
+        print "ERROR, not subclassed", self.__class__
+
+cdef class CharMockBuffer(MockBuffer):
+    cdef int write(self, char* buf, object value) except -1:
+        (<char*>buf)[0] = <char>value
+        return 0
+    cdef get_itemsize(self): return sizeof(char)
+    cdef get_default_format(self): return b"@b"
+
+cdef class IntMockBuffer(MockBuffer):
+    cdef int write(self, char* buf, object value) except -1:
+        (<int*>buf)[0] = <int>value
+        return 0
+    cdef get_itemsize(self): return sizeof(int)
+    cdef get_default_format(self): return b"@i"
+
+cdef class UnsignedIntMockBuffer(MockBuffer):
+    cdef int write(self, char* buf, object value) except -1:
+        (<unsigned int*>buf)[0] = <unsigned int>value
+        return 0
+    cdef get_itemsize(self): return sizeof(unsigned int)
+    cdef get_default_format(self): return b"@I"
+
+cdef class ShortMockBuffer(MockBuffer):
+    cdef int write(self, char* buf, object value) except -1:
+        (<short*>buf)[0] = <short>value
+        return 0
+    cdef get_itemsize(self): return sizeof(short)
+    cdef get_default_format(self): return b"h" # Try without endian specifier
+
+cdef class UnsignedShortMockBuffer(MockBuffer):
+    cdef int write(self, char* buf, object value) except -1:
+        (<unsigned short*>buf)[0] = <unsigned short>value
+        return 0
+    cdef get_itemsize(self): return sizeof(unsigned short)
+    cdef get_default_format(self): return b"@1H" # Try with repeat count
+
+cdef class FloatMockBuffer(MockBuffer):
+    cdef int write(self, char* buf, object value) except -1:
+        (<float*>buf)[0] = <float>(<double>value)
+        return 0
+    cdef get_itemsize(self): return sizeof(float)
+    cdef get_default_format(self): return b"f"
+
+cdef class DoubleMockBuffer(MockBuffer):
+    cdef int write(self, char* buf, object value) except -1:
+        (<double*>buf)[0] = <double>value
+        return 0
+    cdef get_itemsize(self): return sizeof(double)
+    cdef get_default_format(self): return b"d"
+
+cdef extern from *:
+    void* addr_of_pyobject "(void*)"(object)
+
+cdef class ObjectMockBuffer(MockBuffer):
+    cdef int write(self, char* buf, object value) except -1:
+        (<void**>buf)[0] = addr_of_pyobject(value)
+        return 0
+
+    cdef get_itemsize(self): return sizeof(void*)
+    cdef get_default_format(self): return b"@O"
+
+
+cdef class IntStridedMockBuffer(IntMockBuffer):
+    cdef __cythonbufferdefaults__ = {"mode" : "strided"}
+
+cdef class ErrorBuffer:
+    cdef object label
+
+    def __init__(self, label):
+        self.label = label
+
+    def __getbuffer__(ErrorBuffer self, Py_buffer* buffer, int flags):
+        raise Exception("acquiring %s" % self.label)
+
+    def __releasebuffer__(ErrorBuffer self, Py_buffer* buffer):
+        raise Exception("releasing %s" % self.label)
+
+#
+# Structs
+#
+cdef struct MyStruct:
+    char a
+    char b
+    long long int c
+    int d
+    int e
+
+cdef struct SmallStruct:
+    int a
+    int b
+
+cdef struct NestedStruct:
+    SmallStruct x
+    SmallStruct y
+    int z
+
+cdef packed struct PackedStruct:
+    char a
+    int b
+
+cdef struct NestedPackedStruct:
+    char a
+    int b
+    PackedStruct sub
+    int c
+
+cdef class MyStructMockBuffer(MockBuffer):
+    cdef int write(self, char* buf, object value) except -1:
+        cdef MyStruct* s
+        s = <MyStruct*>buf;
+        s.a, s.b, s.c, s.d, s.e = value
+        return 0
+
+    cdef get_itemsize(self): return sizeof(MyStruct)
+    cdef get_default_format(self): return b"2bq2i"
+
+cdef class NestedStructMockBuffer(MockBuffer):
+    cdef int write(self, char* buf, object value) except -1:
+        cdef NestedStruct* s
+        s = <NestedStruct*>buf;
+        s.x.a, s.x.b, s.y.a, s.y.b, s.z = value
+        return 0
+
+    cdef get_itemsize(self): return sizeof(NestedStruct)
+    cdef get_default_format(self): return b"2T{ii}i"
+
+cdef class PackedStructMockBuffer(MockBuffer):
+    cdef int write(self, char* buf, object value) except -1:
+        cdef PackedStruct* s
+        s = <PackedStruct*>buf;
+        s.a, s.b = value
+        return 0
+
+    cdef get_itemsize(self): return sizeof(PackedStruct)
+    cdef get_default_format(self): return b"^ci"
+
+cdef class NestedPackedStructMockBuffer(MockBuffer):
+    cdef int write(self, char* buf, object value) except -1:
+        cdef NestedPackedStruct* s
+        s = <NestedPackedStruct*>buf;
+        s.a, s.b, s.sub.a, s.sub.b, s.c = value
+        return 0
+
+    cdef get_itemsize(self): return sizeof(NestedPackedStruct)
+    cdef get_default_format(self): return b"ci^ci@i"
+