MemoryViewSlice indexing and object coercion + MemoryView indexing

Cython/Compiler/Buffer.py

@@ -215,6 +215,50 @@ class BufferEntry(object):
     def _for_all_ndim(self, s):
         return [s % (self.cname, i) for i in range(self.type.ndim)]
 
+    def generate_buffer_lookup_code(self, code, index_cnames):
+        # Create buffer lookup and return it
+        # This is done via utility macros/inline functions, which vary
+        # according to the access mode used.
+        params = []
+        nd = self.type.ndim
+        mode = self.type.mode
+        if mode == 'full':
+            for i, s, o in zip(index_cnames,
+                               self.get_buf_stridevars(),
+                               self.get_buf_suboffsetvars()):
+                params.append(i)
+                params.append(s)
+                params.append(o)
+            funcname = "__Pyx_BufPtrFull%dd" % nd
+            funcgen = buf_lookup_full_code
+        else:
+            if mode == 'strided':
+                funcname = "__Pyx_BufPtrStrided%dd" % nd
+                funcgen = buf_lookup_strided_code
+            elif mode == 'c':
+                funcname = "__Pyx_BufPtrCContig%dd" % nd
+                funcgen = buf_lookup_c_code
+            elif mode == 'fortran':
+                funcname = "__Pyx_BufPtrFortranContig%dd" % nd
+                funcgen = buf_lookup_fortran_code
+            else:
+                assert False
+            for i, s in zip(index_cnames, self.get_buf_stridevars()):
+                params.append(i)
+                params.append(s)
+
+        # Make sure the utility code is available
+        if funcname not in code.globalstate.utility_codes:
+            code.globalstate.utility_codes.add(funcname)
+            protocode = code.globalstate['utility_code_proto']
+            defcode = code.globalstate['utility_code_def']
+            funcgen(protocode, defcode, name=funcname, nd=nd)
+
+        buf_ptr_type_code = self.buf_ptr_type.declaration_code("")
+        ptrcode = "%s(%s, %s, %s)" % (funcname, buf_ptr_type_code, self.buf_ptr,
+                                      ", ".join(params))
+        return ptrcode
+
 
 def get_flags(buffer_aux, buffer_type):
     flags = 'PyBUF_FORMAT'
@@ -412,7 +456,7 @@ def put_buffer_lookup_code(entry, index_signeds, index_cnames, directives,
                 cast = "(size_t)"
             code.putln("if (%s) %s = %d;" % (
                 code.unlikely("%s >= %s%s" % (cname, cast, shape)),
-                tmp_cname, dim))
+                              tmp_cname, dim))
         code.globalstate.use_utility_code(raise_indexerror_code)
         code.putln("if (%s) {" % code.unlikely("%s != -1" % tmp_cname))
         code.putln('__Pyx_RaiseBufferIndexError(%s);' % tmp_cname)
@@ -426,48 +470,7 @@ def put_buffer_lookup_code(entry, index_signeds, index_cnames, directives,
             if signed != 0:
                 code.putln("if (%s < 0) %s += %s;" % (cname, cname, shape))
 
-    # Create buffer lookup and return it
-    # This is done via utility macros/inline functions, which vary
-    # according to the access mode used.
-    params = []
-    nd = entry.type.ndim
-    mode = entry.type.mode
-    if mode == 'full':
-        for i, s, o in zip(index_cnames,
-                           entry.get_buf_stridevars(),
-                           entry.get_buf_suboffsetvars()):
-            params.append(i)
-            params.append(s)
-            params.append(o)
-        funcname = "__Pyx_BufPtrFull%dd" % nd
-        funcgen = buf_lookup_full_code
-    else:
-        if mode == 'strided':
-            funcname = "__Pyx_BufPtrStrided%dd" % nd
-            funcgen = buf_lookup_strided_code
-        elif mode == 'c':
-            funcname = "__Pyx_BufPtrCContig%dd" % nd
-            funcgen = buf_lookup_c_code
-        elif mode == 'fortran':
-            funcname = "__Pyx_BufPtrFortranContig%dd" % nd
-            funcgen = buf_lookup_fortran_code
-        else:
-            assert False
-        for i, s in zip(index_cnames, entry.get_buf_stridevars()):
-            params.append(i)
-            params.append(s)
-
-    # Make sure the utility code is available
-    if funcname not in code.globalstate.utility_codes:
-        code.globalstate.utility_codes.add(funcname)
-        protocode = code.globalstate['utility_code_proto']
-        defcode = code.globalstate['utility_code_def']
-        funcgen(protocode, defcode, name=funcname, nd=nd)
-
-    buf_ptr_type_code = entry.buf_ptr_type.declaration_code("")
-    ptrcode = "%s(%s, %s, %s)" % (funcname, buf_ptr_type_code, entry.buf_ptr,
-                                  ", ".join(params))
-    return ptrcode
+    return entry.generate_buffer_lookup_code(code, index_cnames)
 
 
 def use_bufstruct_declare_code(env):

Cython/Compiler/Code.py

@@ -23,7 +23,6 @@ import DebugFlags
 import Errors
 from Cython import Tempita as tempita
 
-from Cython.Utils import none_or_sub
 try:
     from __builtin__ import basestring
 except ImportError:
@@ -162,10 +161,10 @@ class UtilityCodeBase(object):
             kwargs['impl'] = impl
 
         if 'name' not in kwargs:
-            if from_file:
-                kwargs['name'] = os.path.splitext(from_file)[0]
-            else:
-                kwargs['name'] = util_code_name
+            kwargs['name'] = util_code_name
+
+        if 'file' not in kwargs and from_file:
+            kwargs['file'] = from_file
 
         return cls(**kwargs)
 
@@ -188,13 +187,8 @@ class UtilityCodeBase(object):
 
         proto, impl = utilities[util_code_name]
         if context is not None:
-            if '__name' not in context:
-                context['__name'] = util_code_name
-
-            if proto:
-                proto = tempita.sub(proto, **context)
-            if impl:
-                impl = tempita.sub(impl, **context)
+            proto = sub_tempita(proto, context, from_file, util_code_name)
+            impl = sub_tempita(impl, context, from_file, util_code_name)
 
         if cls.is_cython_utility:
             # Remember line numbers
@@ -204,19 +198,56 @@ class UtilityCodeBase(object):
 
     load_as_string = classmethod(load_as_string)
 
+    def none_or_sub(self, s, context, tempita):
+        """
+        Format a string in this utility code with context. If None, do nothing.
+        """
+        if s is None:
+            return None
+
+        if tempita:
+            return sub_tempita(s, context, self.file, self.name)
+
+        return s % context
+
     def __str__(self):
         return "<%s(%s)" % (type(self).__name__, self.name)
 
 
+def sub_tempita(s, context, file, name):
+    "Run tempita on string s with context context."
+    if not s:
+        return None
+
+    if file:
+        context['__name'] = "%s:%s" % (file, name)
+    elif name:
+        context['__name'] = name
+
+    return tempita.sub(s, **context)
+
+
 class UtilityCode(UtilityCodeBase):
-    # Stores utility code to add during code generation.
-    #
-    # See GlobalState.put_utility_code.
-    #
-    # hashes/equals by instance
+    """
+    Stores utility code to add during code generation.
+
+    See GlobalState.put_utility_code.
+
+    hashes/equals by instance
+
+    proto           C prototypes
+    impl            implemenation code
+    init            code to call on module initialization
+    requires        utility code dependencies
+    proto_block     the place in the resulting file where the prototype should
+                    end up
+    name            name of the utility code (or None)
+    file            filename of the utility code file this utility was loaded
+                    from (or None)
+    """
 
     def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None,
-                 proto_block='utility_code_proto', name=None):
+                 proto_block='utility_code_proto', name=None, file=None):
         # proto_block: Which code block to dump prototype in. See GlobalState.
         self.proto = proto
         self.impl = impl
@@ -227,11 +258,13 @@ class UtilityCode(UtilityCodeBase):
         self.specialize_list = []
         self.proto_block = proto_block
         self.name = name
+        self.file = file
 
     def get_tree(self):
         pass
 
-    def specialize(self, pyrex_type=None, **data):
+
+    def specialize(self, pyrex_type=None, tempita=False, **data):
         # Dicts aren't hashable...
         if pyrex_type is not None:
             data['type'] = pyrex_type.declaration_code('')
@@ -244,12 +277,15 @@ class UtilityCode(UtilityCodeBase):
                 requires = None
             else:
                 requires = [r.specialize(data) for r in self.requires]
+
             s = self._cache[key] = UtilityCode(
-                                        none_or_sub(self.proto, data),
-                                        none_or_sub(self.impl, data),
-                                        none_or_sub(self.init, data),
-                                        none_or_sub(self.cleanup, data),
-                                        requires, self.proto_block)
+                    self.none_or_sub(self.proto, data, tempita),
+                    self.none_or_sub(self.impl, data, tempita),
+                    self.none_or_sub(self.init, data, tempita),
+                    self.none_or_sub(self.cleanup, data, tempita),
+                    requires,
+                    self.proto_block)
+
             self.specialize_list.append(s)
             return s
 
@@ -275,6 +311,29 @@ class UtilityCode(UtilityCodeBase):
                 self.cleanup(writer, output.module_pos)
 
 
+class ContentHashingUtilityCode(UtilityCode):
+    "UtilityCode that hashes and compares based on self.proto and self.impl"
+
+    def __hash__(self):
+        return hash((self.proto, self.impl))
+
+    def __eq__(self, other):
+        return (self.proto, self.impl) == (other.proto, other.impl)
+
+
+class LazyUtilityCode(UtilityCodeBase):
+    """
+    Utility code that calls a callback with the root code writer when
+    available. Useful when you only have 'env' but not 'code'.
+    """
+
+    def __init__(self, callback):
+        self.callback = callback
+
+    def put_code(self, globalstate):
+        utility = self.callback(globalstate.rootwriter)
+        globalstate.use_utility_code(utility)
+
 
 class FunctionState(object):
     # return_label     string          function return point label
@@ -1538,6 +1597,15 @@ class CCodeWriter(object):
         for entry in entries:
             self.put_var_xdecref_clear(entry)
 
+    def put_incref_memoryviewslice(self, slice_cname, have_gil=False):
+        self.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil)))
+
+    def put_xdecref_memoryviewslice(self, slice_cname, have_gil=False):
+        self.putln("__PYX_XDEC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil)))
+
+    def put_xgiveref_memoryviewslice(self, slice_cname):
+        self.put_xgiveref("%s.memview" % slice_cname)
+
     def put_init_to_py_none(self, cname, type, nanny=True):
         from PyrexTypes import py_object_type, typecast
         py_none = typecast(type, py_object_type, "Py_None")

Cython/Compiler/CythonScope.py

@@ -10,10 +10,12 @@ import MemoryView
 class CythonScope(ModuleScope):
     is_cython_builtin = 1
 
-    def __init__(self):
+    def __init__(self, context):
         ModuleScope.__init__(self, u'cython', None, None)
         self.pxd_file_loaded = True
         self.populate_cython_scope()
+        # The Main.Context object
+        self.context = context
 
     def lookup_type(self, name):
         # This function should go away when types are all first-level objects.
@@ -80,6 +82,10 @@ class CythonScope(ModuleScope):
         # The view sub-scope
         #
         self.viewscope = viewscope = ModuleScope(u'cython.view', self, None)
+
+        # Hacky monkey patch
+        self.viewscope.global_scope = self.global_scope
+
         self.declare_module('view', viewscope, None)
         viewscope.is_cython_builtin = True
         viewscope.pxd_file_loaded = True
@@ -93,7 +99,7 @@ def create_cython_scope(context, create_testscope):
     # One could in fact probably make it a singleton,
     # but not sure yet whether any code mutates it (which would kill reusing
     # it across different contexts)
-    scope = CythonScope()
+    scope = CythonScope(context)
 
     if create_testscope:
         scope.test_cythonscope()
@@ -129,6 +135,12 @@ cython_test_extclass_utility_code = \
 cythonview_testscope_utility_code = load_testscope_utility("View.TestScope")
 
 view_utility_code = MemoryView.load_memview_cy_utility(
-        "View.MemoryView", requires=(Buffer.GetAndReleaseBufferUtilityCode(),))
-
-cython_array_utility_code = MemoryView.load_memview_cy_utility("CythonArray")
+        "View.MemoryView", context=MemoryView.context,
+        requires=[Buffer.GetAndReleaseBufferUtilityCode(),
+                  MemoryView.memviewslice_declare_code],
+)
+
+cython_array_utility_code = MemoryView.load_memview_cy_utility(
+        "CythonArray",
+        context=MemoryView.context,
+        requires=[view_utility_code])

Cython/Compiler/ExprNodes.py

@@ -587,7 +587,12 @@ class ExprNode(Node):
         if dst_type.is_memoryviewslice:
             import MemoryView
             if not src.type.is_memoryviewslice:
-                src = CoerceToMemViewSliceNode(src, dst_type, env)
+                if src.type.is_pyobject:
+                    src = CoerceToMemViewSliceNode(src, dst_type, env)
+                else:
+                    error(self.pos,
+                          "Cannot convert '%s' to memoryviewslice" %
+                                                                (src_type,))
             elif not MemoryView.src_conforms_to_dst(src.type, dst_type):
                 error(self.pos, "Memoryview '%s' not conformable to memoryview '%s'." %
                         (src.type, dst_type))
@@ -1282,6 +1287,7 @@ class NameNode(AtomicExprNode):
     #  cf_is_null      boolean   Is uninitialized before this node
     #  cf_maybe_null   boolean   Maybe uninitialized before this node
     #  allow_null      boolean   Don't raise UnboundLocalError
+    #  nogil           boolean   Whether it is used in a nogil context
 
     is_name = True
     is_cython_module = False
@@ -1293,6 +1299,7 @@ class NameNode(AtomicExprNode):
     cf_maybe_null = True
     cf_is_null = False
     allow_null = False
+    nogil = False
 
     def create_analysed_rvalue(pos, env, entry):
         node = NameNode(pos)
@@ -1452,6 +1459,7 @@ class NameNode(AtomicExprNode):
             self.is_used_as_rvalue = 1
 
     def nogil_check(self, env):
+        self.nogil = True
         if self.is_used_as_rvalue:
             entry = self.entry
             if entry.is_builtin:
@@ -1655,9 +1663,16 @@ class NameNode(AtomicExprNode):
         else:
             if self.type.is_memoryviewslice:
                 import MemoryView
-                MemoryView.gen_acquire_memoryviewslice(rhs, self.type,
+                MemoryView.put_acquire_memoryviewslice(rhs, self.type,
                         self.entry.is_cglobal, self.result(), self.pos, code)
-                # self.generate_acquire_memoryviewslice(rhs, code)
+
+                if isinstance(rhs, CoerceToMemViewSliceNode):
+                    # We had a new reference, the lhs now has another,
+                    # dispose of ours.
+                    # code.put_xdecref_memoryviewslice(rhs.result())
+                    code.put_decref("%s.memview" % rhs.result(),
+                                    cython_memoryview_ptr_type,
+                                    nanny=False)
 
             elif self.type.is_buffer:
                 # Generate code for doing the buffer release/acquisition.
@@ -1736,12 +1751,17 @@ class NameNode(AtomicExprNode):
                 '__Pyx_DelAttrString(%s, "%s")' % (
                     Naming.module_cname,
                     self.entry.name))
-        elif self.entry.type.is_pyobject:
+        elif self.entry.type.is_pyobject or self.entry.type.is_memoryviewslice:
             if not self.cf_is_null:
                 if self.cf_maybe_null:
                     code.put_error_if_unbound(self.pos, self.entry)
-                code.put_decref(self.result(), self.ctype())
-                code.putln('%s = NULL;' % self.result())
+
+                if self.entry.type.is_pyobject:
+                    code.put_decref(self.result(), self.ctype())
+                    code.putln('%s = NULL;' % self.result())
+                else:
+                    code.put_xdecref_memoryviewslice(self.entry.cname,
+                                                     have_gil=not self.nogil)
         else:
             error(self.pos, "Deletion of C names not supported")
 
@@ -2347,10 +2367,12 @@ class IndexNode(ExprNode):
         buffer_access = False
         memoryviewslice_access = False
 
-        if (self.base.type.is_memoryviewslice and not self.indices and
+        is_memslice = self.base.type.is_memoryviewslice
+
+        if (is_memslice and not self.indices and
                 isinstance(self.index, EllipsisNode)):
             memoryviewslice_access = True
-        elif self.base.type.is_buffer or self.base.type.is_memoryviewslice:
+        elif self.base.type.is_buffer or is_memslice:
             if self.indices:
                 indices = self.indices
             else:
@@ -2358,6 +2380,7 @@ class IndexNode(ExprNode):
                     indices = self.index.args
                 else:
                     indices = [self.index]
+
             if len(indices) == self.base.type.ndim:
                 buffer_access = True
                 skip_child_analysis = True
@@ -2365,14 +2388,9 @@ class IndexNode(ExprNode):
                     x.analyse_types(env)
                     if not x.type.is_int:
                         buffer_access = False
+
             if buffer_access:
                 assert hasattr(self.base, "entry") # Must be a NameNode-like node
-        
-#        if self.base.type.is_memoryviewslice:
-#            assert hasattr(self.base, "entry")
-#            if self.indices or not isinstance(self.index, EllipsisNode):
-#                error(self.pos, "Memoryviews currently support ellipsis indexing only.")
-#            else: memoryviewslice_access = True
 
         # On cloning, indices is cloned. Otherwise, unpack index into indices
         assert not (buffer_access and isinstance(self.index, CloneNode))
@@ -2386,7 +2404,10 @@ class IndexNode(ExprNode):
 
             if getting and self.type.is_pyobject:
                 self.is_temp = True
-            if setting:
+
+            if setting and self.base.type.is_memoryviewslice:
+                self.type.writable_needed = True
+            elif setting:
                 if not self.base.entry.type.writable:
                     error(self.pos, "Writing to readonly buffer")
                 else:
@@ -3990,8 +4011,8 @@ class AttributeNode(ExprNode):
                 import MemoryView
                 MemoryView.put_assign_to_memviewslice(select_code, rhs.result(), self.type,
                         pos=self.pos, code=code)
-                if rhs.is_temp:
-                    code.put_xdecref_clear("%s.memview" % rhs.result(), cython_memoryview_ptr_type)
+                #if rhs.is_temp:
+                #    code.put_xdecref_clear("%s.memview" % rhs.result(), cython_memoryview_ptr_type)
             if not self.type.is_memoryviewslice:
                 code.putln(
                     "%s = %s;" % (
@@ -7647,46 +7668,14 @@ class CoerceToMemViewSliceNode(CoercionNode):
         self.type = dst_type
         self.env = env
         self.is_temp = 1
+        self.arg = arg
 
     def generate_result_code(self, code):
-        import MemoryView, Buffer
-        memviewobj = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True)
-        buf_flag = MemoryView.get_buf_flag(self.type.axes)
-        code.putln("%s = (PyObject *) __pyx_memoryview_new(%s, %s);" %
-                                (memviewobj, self.arg.py_result(), buf_flag))
-        code.putln(code.error_goto_if_PyErr(self.pos))
-        ndim = len(self.type.axes)
-        spec_int_arr = code.funcstate.allocate_temp(
-                PyrexTypes.c_array_type(PyrexTypes.c_int_type, ndim),
-                manage_ref=False)
-        specs_code = MemoryView.specs_to_code(self.type.axes)
-        for idx, cspec in enumerate(specs_code):
-            code.putln("%s[%d] = %s;" % (spec_int_arr, idx, cspec))
-
-        code.globalstate.use_utility_code(Buffer.acquire_utility_code)
-        code.globalstate.use_utility_code(MemoryView.memviewslice_init_code)
-        dtype_typeinfo = Buffer.get_type_information_cname(code, self.type.dtype)
-
-        MemoryView.put_init_entry(self.result(), code)
-        code.putln("{")
-        code.putln("__Pyx_BufFmt_StackElem __pyx_stack[%d];" %
-                self.type.dtype.struct_nesting_depth())
-        result = self.result()
-        if self.type.is_c_contig:
-            c_or_f_flag = "__Pyx_IS_C_CONTIG"
-        elif self.type.is_f_contig:
-            c_or_f_flag = "__Pyx_IS_F_CONTIG"
-        else:
-            c_or_f_flag = "0"
-        code.putln(code.error_goto_if("-1 == __Pyx_ValidateAndInit_memviewslice("
-                        "(struct __pyx_memoryview_obj *) %(memviewobj)s,"
-                        " %(spec_int_arr)s, %(c_or_f_flag)s, %(ndim)d,"
-                        " &%(dtype_typeinfo)s, __pyx_stack, &%(result)s)" % locals(), self.pos))
-        code.putln("}")
-        code.put_gotref(
-                code.as_pyobject("%s.memview" % self.result(), cython_memoryview_ptr_type))
-        code.funcstate.release_temp(memviewobj)
-        code.funcstate.release_temp(spec_int_arr)
+        self.type.create_from_py_utility_code(self.env)
+        code.putln("%s = %s(%s);" % (self.result(),
+                                     self.type.from_py_function,
+                                     self.arg.py_result()))
+
 
 class CastNode(CoercionNode):
     #  Wrap a node in a C type cast.

Cython/Compiler/Main.py

@@ -430,7 +430,7 @@ def create_default_resultobj(compilation_source, options):
 
 def run_pipeline(source, options, full_module_name = None):
     import Pipeline
-    # Set up context
+
     context = options.create_context()
 
     # Set up source object
@@ -438,6 +438,7 @@ def run_pipeline(source, options, full_module_name = None):
     abs_path = os.path.abspath(source)
     source_ext = os.path.splitext(source)[1]
     full_module_name = full_module_name or context.extract_module_name(source, options)
+
     if options.relative_path_in_code_position_comments:
         rel_path = full_module_name.replace('.', os.sep) + source_ext
         if not abs_path.endswith(rel_path):
@@ -520,7 +521,7 @@ class CompilationOptions(object):
 
     def create_context(self):
         return Context(self.include_path, self.compiler_directives,
-                      self.cplus, self.language_level, options=self)
+                       self.cplus, self.language_level, options=self)
 
 
 class CompilationResult(object):
@@ -584,7 +585,8 @@ def compile_multiple(sources, options):
     a CompilationResultSet. Performs timestamp checking and/or recursion
     if these are specified in the options.
     """
-    context = options.create_context()
+    # run_pipeline creates the context
+    # context = options.create_context()
     sources = [os.path.abspath(source) for source in sources]
     processed = set()
     results = CompilationResultSet()

Cython/Compiler/MemoryView.py

@@ -46,17 +46,21 @@ _spec_to_const = {
         'follow' : MEMVIEW_FOLLOW,
         }
 
+_spec_to_abbrev = {
+    'direct'  : 'd',
+    'ptr'     : 'p',
+    'full'    : 'f',
+    'contig'  : 'c',
+    'strided' : 's',
+    'follow'  : '_',
+}
+
 memview_name = u'memoryview'
 memview_typeptr_cname = '__pyx_memoryview_type'
 memview_objstruct_cname = '__pyx_memoryview_obj'
 memviewslice_cname = u'__Pyx_memviewslice'
 
-def specs_to_code(specs):
-    arr = []
-    for access, packing in specs:
-        arr.append("(%s | %s)" % (_spec_to_const[access],
-                                  _spec_to_const[packing]))
-    return arr
+
 
 def put_init_entry(mv_cname, code):
     code.putln("%s.data = NULL;" % mv_cname)
@@ -70,7 +74,7 @@ def mangle_dtype_name(dtype):
 def axes_to_str(axes):
     return "".join([access[0].upper()+packing[0] for (access, packing) in axes])
 
-def gen_acquire_memoryviewslice(rhs, lhs_type, lhs_is_cglobal, lhs_result, lhs_pos, code):
+def put_acquire_memoryviewslice(rhs, lhs_type, lhs_is_cglobal, lhs_result, lhs_pos, code):
     # import MemoryView
     assert rhs.type.is_memoryviewslice
 
@@ -80,32 +84,17 @@ def gen_acquire_memoryviewslice(rhs, lhs_type, lhs_is_cglobal, lhs_result, lhs_p
     else:
         rhstmp = code.funcstate.allocate_temp(lhs_type, manage_ref=False)
         code.putln("%s = %s;" % (rhstmp, rhs.result_as(lhs_type)))
-        code.putln(code.error_goto_if_null("%s.memview" % rhstmp, lhs_pos))
-
-    if not rhs.result_in_temp():
-        code.put_incref("%s.memview" % rhstmp, cython_memoryview_ptr_type)
-
-    if lhs_is_cglobal:
-        code.put_gotref("%s.memview" % lhs_result)
-
-    #XXX: this is here because self.lhs_of_first_assignment is not set correctly,
-    #     once that is working this should take that flag into account.
-    #     See NameNode.generate_assignment_code
-    code.put_xdecref("%s.memview" % lhs_result, cython_memoryview_ptr_type)
-
-    if lhs_is_cglobal:
-        code.put_giveref("%s.memview" % rhstmp)
 
+    code.putln(code.error_goto_if_null("%s.memview" % rhstmp, lhs_pos))
     put_assign_to_memviewslice(lhs_result, rhstmp, lhs_type,
                                      lhs_pos, code=code)
 
-    if rhs.result_in_temp() or not pretty_rhs:
-        code.putln("%s.memview = 0;" % rhstmp)
-
     if not pretty_rhs:
         code.funcstate.release_temp(rhstmp)
 
 def put_assign_to_memviewslice(lhs_cname, rhs_cname, memviewslicetype, pos, code):
+    code.put_xdecref_memoryviewslice(lhs_cname)
+    code.put_incref_memoryviewslice(rhs_cname)
 
     code.putln("%s.memview = %s.memview;" % (lhs_cname, rhs_cname))
     code.putln("%s.data = %s.data;" % (lhs_cname, rhs_cname))
@@ -195,6 +184,45 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry):
     def get_buf_shapevars(self):
         return self._for_all_ndim("%s.shape[%d]")
 
+    def generate_buffer_lookup_code(self, code, index_cnames):
+        bufp = self.buf_ptr
+        type_decl = self.type.dtype.declaration_code("")
+
+        for dim, (access, packing) in enumerate(self.type.axes):
+            shape = "%s.shape[%d]" % (self.cname, dim)
+            stride = "%s.strides[%d]" % (self.cname, dim)
+            suboffset = "%s.suboffsets[%d]" % (self.cname, dim)
+            index = index_cnames[dim]
+
+            if access == 'full' and packing in ('strided', 'follow'):
+                code.globalstate.use_utility_code(memviewslice_index_helpers)
+                bufp = ('__pyx_memviewslice_index_full(%s, %s, %s)' %
+                                            (bufp, index, stride, suboffset))
+
+            elif access == 'full' and packing == 'contig':
+                # We can skip stride multiplication with the cast
+                code.globalstate.use_utility_code(memviewslice_index_helpers)
+                bufp = '((char *) ((%s *) %s) + %s)' % (type_decl, bufp, index)
+                bufp = ('__pyx_memviewslice_index_full_contig(%s, %s)' %
+                                                            (bufp, suboffset))
+
+            elif access == 'ptr' and packing in ('strided', 'follow'):
+                bufp = ("(*((char **) %s + %s * %s) + %s)" %
+                                             (bufp, index, stride, suboffset))
+
+            elif access == 'ptr' and packing == 'contig':
+                bufp = "(*((char **) %s) + %s)" % (bufp, suboffset)
+
+            elif access == 'direct' and packing in ('strided', 'follow'):
+                bufp = "(%s + %s * %s)" % (bufp, index, stride)
+
+            else:
+                assert (access, packing) == ('direct', 'contig'), (access, packing)
+                bufp = '((char *) (((%s *) %s) + %s))' % (type_decl, bufp, index)
+
+            bufp = '( /* dim=%d */ %s )' % (dim, bufp)
+
+        return "((%s *) %s)" % (type_decl, bufp)
 
 def get_copy_func_name(to_memview):
     base = "__Pyx_BufferNew_%s_From_%s_%s"
@@ -370,10 +398,10 @@ class CopyFuncUtilCode(object):
 
         if self.to_memview.is_c_contig:
             mode = 'c'
-            contig_flag = 'PyBUF_C_CONTIGUOUS'
+            contig_flag = memview_c_contiguous
         elif self.to_memview.is_f_contig:
             mode = 'fortran'
-            contig_flag = "PyBUF_F_CONTIGUOUS"
+            contig_flag = memview_f_contiguous
 
         context = dict(
             copy_name=self.copy_func_name,
@@ -735,11 +763,13 @@ class MemoryViewSliceTransform(CythonTransform):
         return node
 
 
-def load_memview_cy_utility(util_code_name, **kwargs):
-    return CythonUtilityCode.load(util_code_name, "MemoryView.pyx", **kwargs)
+def load_memview_cy_utility(util_code_name, context=None, **kwargs):
+    return CythonUtilityCode.load(util_code_name, "MemoryView.pyx",
+                                  context=context, **kwargs)
 
-def load_memview_c_utility(util_code_name, **kwargs):
-    return UtilityCode.load(util_code_name, "MemoryView_C.c", **kwargs)
+def load_memview_c_utility(util_code_name, context=None, **kwargs):
+    return UtilityCode.load(util_code_name, "MemoryView_C.c",
+                            context=context, **kwargs)
 
 context = {
     'memview_struct_name': memview_objstruct_cname,
@@ -753,6 +783,8 @@ memviewslice_declare_code = load_memview_c_utility(
 
 memviewslice_init_code = load_memview_c_utility(
     "MemviewSliceInit",
-    context={'BUF_MAX_NDIMS': Options.buffer_max_dims},
+    context=dict(context, BUF_MAX_NDIMS=Options.buffer_max_dims),
     requires=[memviewslice_declare_code],
 )
+
+memviewslice_index_helpers = load_memview_c_utility("MemviewSliceIndex")
\ No newline at end of file

Cython/Compiler/ModuleNode.py

@@ -79,6 +79,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                          scope.python_include_files)
 
         if merge_scope:
+            # Ensure that we don't generate import code for these entries!
+            for entry in scope.c_class_entries:
+                entry.type.module_name = self.full_module_name
+
             self.scope.merge_in(scope)
 
     def analyse_declarations(self, env):
@@ -345,7 +349,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         self.generate_declarations_for_modules(env, modules, globalstate)
         h_code.write('\n')
 
-        for utilcode in env.utility_code_list:
+        for utilcode in env.utility_code_list[:]:
             globalstate.use_utility_code(utilcode)
         globalstate.finalize_main_c_code()
 
@@ -2255,7 +2259,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         # Generate type import code for extern extension types
         # and type ready code for non-extern ones.
         for entry in env.c_class_entries:
-            if entry.visibility == 'extern':
+            if entry.visibility == 'extern' and not entry.utility_code_definition:
                 self.generate_type_import_code(env, entry.type, entry.pos, code)
             else:
                 self.generate_base_type_import_code(env, entry, code)
@@ -2265,8 +2269,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
 
     def generate_base_type_import_code(self, env, entry, code):
         base_type = entry.type.base_type
-        if base_type and base_type.module_name != env.qualified_name \
-               and not base_type.is_builtin_type:
+        if (base_type and base_type.module_name != env.qualified_name and not
+               base_type.is_builtin_type and not entry.utility_code_definition):
             self.generate_type_import_code(env, base_type, self.pos, code)
 
     def use_type_import_utility_code(self, env):

Cython/Compiler/Nodes.py

@@ -1344,17 +1344,19 @@ class FuncDefNode(StatNode, BlockNode):
             if not entry.in_closure:
                 code.put_var_declaration(entry)
 
+        # Initialize the return variable __pyx_r
         init = ""
         if not self.return_type.is_void:
             if self.return_type.is_pyobject:
                 init = " = NULL"
+            elif self.return_type.is_memoryviewslice:
+                init = "= {0, 0}"
+
             code.putln(
                 "%s%s;" %
                     (self.return_type.declaration_code(Naming.retval_cname),
                      init))
-            if self.return_type.is_memoryviewslice:
-                import MemoryView
-                MemoryView.put_init_entry(Naming.retval_cname, code)
+
         tempvardecl_code = code.insertion_point()
         self.generate_keyword_list(code)
 
@@ -1431,15 +1433,18 @@ class FuncDefNode(StatNode, BlockNode):
                 if (acquire_gil or entry.assignments) and not entry.in_closure:
                     code.put_var_incref(entry)
             if entry.type.is_memoryviewslice:
-                code.put_incref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
+                code.put_incref_memoryviewslice(entry.cname,
+                                                have_gil=not lenv.nogil)
+                #code.put_incref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
         # ----- Initialise local buffer auxiliary variables
         for entry in lenv.var_entries + lenv.arg_entries:
             if entry.type.is_buffer and entry.buffer_aux.buflocal_nd_var.used:
                 Buffer.put_init_vars(entry, code)
         # ----- Initialise local memoryviewslices
         for entry in lenv.var_entries:
-            if entry.type.is_memoryviewslice:
-                MemoryView.put_init_entry(entry.cname, code)
+            if entry.visibility == "private" and not entry.used:
+                continue
+
         # ----- Check and convert arguments
         self.generate_argument_type_tests(code)
         # ----- Acquire buffer arguments
@@ -1544,7 +1549,8 @@ class FuncDefNode(StatNode, BlockNode):
             if not entry.used or entry.in_closure:
                 continue
             if entry.type.is_memoryviewslice:
-                code.put_xdecref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
+                #code.put_xdecref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
+                code.put_xdecref_memoryviewslice(entry.cname)
             if entry.type.is_pyobject:
                 code.put_var_decref(entry)
 
@@ -1554,7 +1560,8 @@ class FuncDefNode(StatNode, BlockNode):
                 if (acquire_gil or entry.assignments) and not entry.in_closure:
                     code.put_var_decref(entry)
             if entry.type.is_memoryviewslice:
-                code.put_decref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
+                code.put_xdecref_memoryviewslice(entry.cname)
+                #code.put_decref("%s.memview" % entry.cname, cython_memoryview_ptr_type)
         if self.needs_closure:
             code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type)
 
@@ -1567,8 +1574,9 @@ class FuncDefNode(StatNode, BlockNode):
                 err_val = default_retval
             if self.return_type.is_pyobject:
                 code.put_xgiveref(self.return_type.as_pyobject(Naming.retval_cname))
-            elif self.return_type.is_memoryviewslice:
-                code.put_xgiveref(code.as_pyobject("%s.memview" % Naming.retval_cname,cython_memoryview_ptr_type))
+            #elif self.return_type.is_memoryviewslice:
+            #    code.put_xgiveref(code.as_pyobject("%s.memview" % Naming.retval_cname,cython_memoryview_ptr_type))
+            #    code.put_xgiveref_memoryviewslice(Naming.retval_cname)
 
         if self.entry.is_special and self.entry.name == "__hash__":
             # Returning -1 for __hash__ is supposed to signal an error
@@ -4153,7 +4161,8 @@ class DelStatNode(StatNode):
     def analyse_expressions(self, env):
         for arg in self.args:
             arg.analyse_target_expression(env, None)
-            if arg.type.is_pyobject:
+            if arg.type.is_pyobject or (arg.is_name and
+                                        arg.type.is_memoryviewslice):
                 pass
             elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
                 self.cpp_check(env)
@@ -4172,7 +4181,7 @@ class DelStatNode(StatNode):
 
     def generate_execution_code(self, code):
         for arg in self.args:
-            if arg.type.is_pyobject:
+            if arg.type.is_pyobject or arg.type.is_memoryviewslice:
                 arg.generate_deletion_code(code)
             elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
                 arg.generate_result_code(code)
@@ -4274,14 +4283,15 @@ class ReturnStatNode(StatNode):
             code.put_xdecref(Naming.retval_cname,
                              self.return_type)
         elif self.return_type.is_memoryviewslice:
-            code.put_xdecref("%s.memview" % Naming.retval_cname,
-                    self.return_type)
+            code.put_xdecref_memoryviewslice(Naming.retval_cname)
+            #code.put_xdecref("%s.memview" % Naming.retval_cname,
+            #        self.return_type)
 
         if self.value:
             self.value.generate_evaluation_code(code)
             if self.return_type.is_memoryviewslice:
                 import MemoryView
-                MemoryView.gen_acquire_memoryviewslice(self.value, self.return_type,
+                MemoryView.put_acquire_memoryviewslice(self.value, self.return_type,
                         False, Naming.retval_cname, None, code)
             else:
                 self.value.make_owned_reference(code)

Cython/Compiler/PyrexTypes.py

@@ -2,12 +2,14 @@
 #   Cython/Python language types
 #
 
-from Code import UtilityCode
+from Code import UtilityCode, LazyUtilityCode
 import StringEncoding
 import Naming
 import copy
 from Errors import error
 
+import cython
+
 class BaseType(object):
     #
     #  Base class for all Cython types including pseudo-types.
@@ -331,6 +333,15 @@ class MemoryViewSliceType(PyrexType):
     has_attributes = 1
     scope = None
 
+    # These are specialcased in Defnode
+    from_py_function = None
+    to_py_function = None
+
+    exception_value = None
+    exception_check = None
+
+    utility_counter = 0
+
     def __init__(self, base_dtype, axes):
         '''
         MemoryViewSliceType(base, axes)
@@ -375,6 +386,7 @@ class MemoryViewSliceType(PyrexType):
         assert not (self.is_c_contig and self.is_f_contig)
 
         self.mode = MemoryView.get_mode(axes)
+        self.writable_needed = False
 
     def same_as_resolved_type(self, other_type):
         return ((other_type.is_memoryviewslice and
@@ -495,11 +507,68 @@ class MemoryViewSliceType(PyrexType):
 
     def global_init_code(self, entry, code):
         code.putln("%s.data = NULL;" % entry.cname)
-        code.put_init_to_py_none("%s.memview" % entry.cname, cython_memoryview_ptr_type, nanny=False)
+        code.putln("%s.memview = NULL;" % entry.cname)
+        #code.put_init_to_py_none("%s.memview" % entry.cname, cython_memoryview_ptr_type, nanny=False)
 
     def check_for_null_code(self, cname):
         return cname + '.memview'
 
+    def create_from_py_utility_code(self, env):
+        import MemoryView, Buffer, Code
+
+        # We don't have 'code', so use a LazyUtilityCode with a callback.
+        def lazy_utility_callback(code):
+            context['dtype_typeinfo'] = Buffer.get_type_information_cname(
+                                                          code, self.dtype)
+            return Code.ContentHashingUtilityCode.load(
+                        "ObjectToMemviewSlice", "MemoryView_C.c", context)
+
+        env.use_utility_code(Buffer.acquire_utility_code)
+        env.use_utility_code(MemoryView.memviewslice_init_code)
+        env.use_utility_code(LazyUtilityCode(lazy_utility_callback))
+
+        if self.is_c_contig:
+            c_or_f_flag = "__Pyx_IS_C_CONTIG"
+        elif self.is_f_contig:
+            c_or_f_flag = "__Pyx_IS_F_CONTIG"
+        else:
+            c_or_f_flag = "0"
+
+        # specializing through UtilityCode.specialize is not so useful as
+        # specialize on too many things to include in the function name
+        funcname = "__Pyx_PyObject_to_MemoryviewSlice_%d" % self.utility_counter
+
+        context = dict(
+            MemoryView.context,
+            buf_flag = MemoryView.get_buf_flag(self.axes),
+            ndim = self.ndim,
+            axes_specs = ', '.join(self.axes_specs_to_code()),
+            dtype_typedecl = self.dtype.declaration_code(""),
+            struct_nesting_depth = self.dtype.struct_nesting_depth(),
+            c_or_f_flag = c_or_f_flag,
+            funcname = funcname,
+        )
+
+        self.from_py_function = funcname
+        MemoryViewSliceType.utility_counter += 1
+
+        return True
+
+    def axes_specs_to_code(self):
+        "Return a list of code constants for each axis"
+        import MemoryView
+        d = MemoryView._spec_to_const
+        return ["(%s | %s)" % (d[a], d[p]) for a, p in self.axes]
+
+    def axes_specs_to_name(self):
+        "Return an abbreviated name for our axes"
+        import MemoryView
+        d = MemoryView._spec_to_abbrev
+        return "".join(["%s%s" % (d[a], d[p]) for a, p in self.axes])
+
+    def error_condition(self, result_code):
+        return "!%s.memview" % result_code
+
 
 class BufferType(BaseType):
     #
@@ -2698,8 +2767,8 @@ cython_memoryview_type = CStructOrUnionType("__pyx_memoryview_obj", "struct",
 
 cython_memoryview_ptr_type = CPtrType(cython_memoryview_type)
 
-memoryviewslice_type = CStructOrUnionType("__Pyx_memviewslice", "struct",
-                                    None, 1, "__Pyx_memviewslice")
+memoryviewslice_type = CStructOrUnionType("memoryviewslice", "struct",
+                                          None, 1, "__Pyx_memviewslice")
 
 error_type =    ErrorType()
 unspecified_type = UnspecifiedType()

Cython/Compiler/Symtab.py

@@ -382,6 +382,10 @@ class Scope(object):
 #                entries[name] = entry
             if not shadow:
                 entries[name] = entry
+
+        if type.is_memoryviewslice:
+            entry.init = "{ 0, 0 }"
+
         entry.scope = self
         entry.visibility = visibility
         return entry

Cython/Compiler/UtilityCode.py

@@ -15,7 +15,7 @@ class NonManglingModuleScope(Symtab.ModuleScope):
         entry.used = True
         return super(NonManglingModuleScope, self).add_imported_entry(
                                                         name, entry, pos)
-    
+
     def mangle(self, prefix, name=None):
         if name:
             if prefix in (Naming.typeobj_prefix, Naming.func_prefix, Naming.var_prefix, Naming.pyfunc_prefix):
@@ -63,7 +63,8 @@ class CythonUtilityCode(Code.UtilityCodeBase):
 
     is_cython_utility = True
 
-    def __init__(self, impl, name="__pyxutil", prefix="", requires=None):
+    def __init__(self, impl, name="__pyxutil", prefix="", requires=None,
+                 file=None):
         # 1) We need to delay the parsing/processing, so that all modules can be
         #    imported without import loops
         # 2) The same utility code object can be used for multiple source files;
@@ -72,6 +73,7 @@ class CythonUtilityCode(Code.UtilityCodeBase):
         # Hence, delay any processing until later.
         self.impl = impl
         self.name = name
+        self.file = file
         self.prefix = prefix
         self.requires = requires or []
 
@@ -113,10 +115,11 @@ class CythonUtilityCode(Code.UtilityCodeBase):
     def put_code(self, output):
         pass
 
-    def declare_in_scope(self, dest_scope, used=False):
+    def declare_in_scope(self, dest_scope, used=False, modname=None):
         """
         Declare all entries from the utility code in dest_scope. Code will only
-        be included for used entries.
+        be included for used entries. If module_name is given, declare the
+        type entries with that name.
         """
         tree = self.get_tree(entries_only=True)
 
@@ -130,6 +133,10 @@ class CythonUtilityCode(Code.UtilityCodeBase):
             entry.utility_code_definition = self
             entry.used = used
 
+            if modname and entry.type.is_extension_type:
+                entry.qualified_name = modname
+                entry.type.module_name = modname
+
         dest_scope.merge_in(tree.scope, merge_unused=True)
         tree.scope = dest_scope
 

Cython/Utility/MemoryView.pyx

@@ -10,7 +10,10 @@ cdef extern from "Python.h":
         PyBUF_C_CONTIGUOUS,
         PyBUF_F_CONTIGUOUS,
         PyBUF_ANY_CONTIGUOUS
+        PyBUF_FORMAT
 
+cdef extern from *:
+    object __pyx_memoryview_new(object obj, int flags)
 
 @cname("__pyx_array")
 cdef class array:
@@ -105,7 +108,12 @@ cdef class array:
         info.strides = self.strides
         info.suboffsets = NULL
         info.itemsize = self.itemsize
-        info.format = self.format
+
+        if flags & PyBUF_FORMAT:
+            info.format = self.format
+        else:
+            info.format = NULL
+
         # we do not need to call releasebuffer
         info.obj = None
 
@@ -130,6 +138,15 @@ cdef class array:
         self.format = NULL
         self.itemsize = 0
 
+    def __getitem__(self, index):
+        view = __pyx_memoryview_new(self, PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT)
+        return view[index]
+
+    def __setitem__(self, index, value):
+        view = __pyx_memoryview_new(self, PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT)
+        view[index] = value
+
+
 @cname("__pyx_array_new")
 cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *mode):
     return array(shape, itemsize, format, mode.decode('ASCII'))
@@ -137,8 +154,10 @@ cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *
 ########## View.MemoryView ##########
 
 # from cpython cimport ...
-cdef extern from "pythread.h":
+cdef extern from "Python.h":
+    int PyIndex_Check(object)
 
+cdef extern from "pythread.h":
     ctypedef void *PyThread_type_lock
 
     PyThread_type_lock PyThread_allocate_lock()
@@ -150,6 +169,12 @@ cdef extern from *:
     int __Pyx_GetBuffer(object, Py_buffer *, int)
     void __Pyx_ReleaseBuffer(Py_buffer *)
 
+    ctypedef struct {{memviewslice_name}}:
+        char *data
+        Py_ssize_t shape[{{max_dims}}]
+        Py_ssize_t strides[{{max_dims}}]
+        Py_ssize_t suboffsets[{{max_dims}}]
+
 
 @cname('__pyx_MemviewEnum')
 cdef class Enum(object):
@@ -173,23 +198,143 @@ cdef class memoryview(object):
 
     cdef object obj
     cdef Py_buffer view
-    cdef PyThread_type_lock acqcnt_lock
+    cdef PyThread_type_lock lock
     cdef int acquisition_count
 
+
     def __cinit__(memoryview self, object obj, int flags):
         self.obj = obj
-        #self.acqcnt_lock = PyThread_allocate_lock()
-        #if self.acqcnt_lock == NULL:
-        #    raise MemoryError
-
         __Pyx_GetBuffer(obj, &self.view, flags)
 
+        self.lock = PyThread_allocate_lock()
+        if self.lock == NULL:
+            raise MemoryError
+
     def __dealloc__(memoryview self):
-        #PyThread_free_lock(self.acqcnt_lock)
-        self.obj = None
         __Pyx_ReleaseBuffer(&self.view)
+        PyThread_free_lock(self.lock)
+
+    @cname('__pyx_memoryview_getitem')
+    def __getitem__(memoryview self, object index):
+        # cdef Py_ssize_t idx
+        cdef char *itemp = <char *> self.view.buf
+        cdef bytes bytesitem
+        cdef str fmt = self.view.format
+
+        import struct
+        try:
+            itemsize = struct.calcsize(fmt)
+        except struct.error:
+            raise TypeError("Unsupported format: %r" % fmt)
+
+        if index is Ellipsis:
+            return self
+
+        elif isinstance(index, slice):
+            if index == slice(None):
+                return self
+
+            raise NotImplementedError
+
+        else:
+            if not isinstance(index, tuple):
+                index = (index,)
+
+            tup = _unellipsify(index, self.view.ndim)
+
+            if len(tup) != self.view.ndim:
+                raise NotImplementedError(
+                        "Expected %d indices (got %d)" %
+                             (self.view.ndim, len(tup)))
+
+            for dim, idx in enumerate(tup):
+                _check_index(idx)
+                itemp = pybuffer_index(&self.view, itemp, idx, dim + 1)
+
+        # Do a manual and complete check here instead of this easy hack
+        bytesitem = itemp[:self.view.itemsize]
+        return struct.unpack(fmt, bytesitem)
+
+
+@cname('__pyx_memoryviewslice')
+cdef class _memoryviewslice(memoryview):
+    "Internal class for passing memory view slices to Python"
+
+    # We need this to keep our shape/strides/suboffset pointers valid
+    cdef {{memviewslice_name}} from_slice
+    # Restore the original Py_buffer before releasing
+    cdef Py_buffer orig_view
+
+    def __cinit__(self, object obj, int flags):
+        self.orig_view = self.view
+
+    def __dealloc__(self):
+        self.view = self.orig_view
 
 
 @cname('__pyx_memoryview_new')
-cdef memoryview memoryview_cwrapper(object o, int flags):
+cdef memoryview_cwrapper(object o, int flags):
     return memoryview(o, flags)
+
+@cname('__pyx_memoryview_fromslice')
+cdef memoryview memoryview_from_memview_cwrapper(memoryview m,  int flags,
+                        int new_ndim, {{memviewslice_name}} *memviewslice):
+    cdef _memoryviewslice result = _memoryviewslice(m.obj, flags)
+
+    result.from_slice = memviewslice[0]
+
+    result.view.shape = <Py_ssize_t *> (&result.from_slice.shape + new_ndim)
+    result.view.strides = <Py_ssize_t *> (&result.from_slice.strides + new_ndim)
+    result.view.suboffsets = <Py_ssize_t *> (&result.from_slice.suboffsets + new_ndim)
+    result.view.ndim = new_ndim
+
+    return result
+
+cdef _check_index(index):
+    if not PyIndex_Check(index):
+        raise TypeError("Cannot index with %s" % type(index))
+
+cdef tuple _unellipsify(tuple tup, int ndim):
+    if Ellipsis in tup:
+        result = []
+        for idx, item in enumerate(tup):
+            if item is Ellipsis:
+                result.extend([slice(None)] * (ndim - len(tup) + 1))
+                result.extend(tup[idx + 1:])
+                break
+
+            result.append(item)
+
+        return tuple(result)
+
+    return tup
+
+@cname('__pyx_pybuffer_index')
+cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, int dim) except NULL:
+    cdef Py_ssize_t shape, stride, suboffset = -1
+    cdef Py_ssize_t itemsize = view.itemsize
+    cdef char *resultp
+
+    if view.ndim == 0:
+        shape = view.len / itemsize
+        stride = itemsize
+    else:
+        shape = view.shape[dim]
+        stride = view.strides[dim]
+        if view.suboffsets != NULL:
+            suboffset = view.suboffsets[dim]
+
+    if index < 0:
+        index += view.shape[dim]
+        if index < 0:
+            raise IndexError("Out of bounds in dimension %d" % dim)
+
+    if index > shape:
+        raise IndexError("Out of bounds in dimension %d" % dim)
+
+    resultp = bufp + index * stride
+
+    if suboffset >= 0:
+        resultp = (<char **> resultp)[0] + suboffset
+
+    return resultp

Cython/Utility/MemoryView_C.c

@@ -12,6 +12,9 @@ typedef struct {
   Py_ssize_t suboffsets[{{max_dims}}];
 } {{memviewslice_name}};
 
+/////////////// ObjectToMemviewSlice.proto ///////////////
+{{# __Pyx_PyObject_to_MemoryviewSlice_<count> }}
+static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *);
 
 ////////// MemviewSliceInit.proto //////////
 
@@ -27,6 +30,7 @@ typedef struct {
 #define __Pyx_IS_C_CONTIG 1
 #define __Pyx_IS_F_CONTIG 2
 
+
 /* #define __PYX_MEMSLICE_GETDATA(SLICE) ((char *) SLICE->memview->view->buf) */
 
 static int __Pyx_ValidateAndInit_memviewslice(struct __pyx_memoryview_obj *memview,
@@ -38,6 +42,49 @@ static int __Pyx_init_memviewslice(
                 int ndim,
                 __Pyx_memviewslice *memviewslice);
 
+#define __PYX_INC_MEMVIEW(slice, have_gil) __Pyx_INC_MEMVIEW(slice, have_gil, __LINE__)
+#define __PYX_XDEC_MEMVIEW(slice, have_gil) __Pyx_XDEC_MEMVIEW(slice, have_gil, __LINE__)
+static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *, int, int);
+static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *, int, int);
+
+/////////////// MemviewSliceIndex.proto ///////////////
+
+static CYTHON_INLINE char *__pyx_memviewslice_index_full(char *bufp, Py_ssize_t idx, Py_ssize_t stride, Py_ssize_t suboffset);
+static CYTHON_INLINE char *__pyx_memviewslice_index_full_contig(char *bufp, Py_ssize_t suboffset);
+
+
+/////////////// ObjectToMemviewSlice ///////////////
+
+{{#__Pyx_PyObject_to_MemoryviewSlice_<count>}}
+
+static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *obj) {
+    {{memviewslice_name}} result;
+    result.memview = NULL;
+    result.data = NULL;
+    struct __pyx_memoryview_obj *memview =  \
+        (struct __pyx_memoryview_obj *) __pyx_memoryview_new(obj, {{buf_flag}});
+    __Pyx_BufFmt_StackElem stack[{{struct_nesting_depth}}];
+    int axes_specs[] = { {{axes_specs}} };
+    int retcode;
+
+    if (unlikely(!memview))
+        goto __pyx_fail;
+
+    retcode = __Pyx_ValidateAndInit_memviewslice(memview, axes_specs,
+            {{c_or_f_flag}}, {{ndim}}, &{{dtype_typeinfo}}, stack, &result);
+
+    if (unlikely(retcode == -1))
+        goto __pyx_fail;
+
+    memview->acquisition_count = 1;
+    return result;
+__pyx_fail:
+    Py_XDECREF(memview);
+    result.memview = NULL;
+    result.data = NULL;
+    return result;
+}
+
 ////////// MemviewSliceInit //////////
 
 static int __Pyx_ValidateAndInit_memviewslice(
@@ -203,8 +250,6 @@ static int __Pyx_init_memviewslice(
         }
     }
 
-    __Pyx_INCREF((PyObject *)memview);
-    __Pyx_GIVEREF((PyObject *)memview);
     memviewslice->memview = memview;
     memviewslice->data = (char *)buf->buf;
     retval = 0;
@@ -220,6 +265,62 @@ no_fail:
     return retval;
 }
 
+static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *memslice,
+                                            int have_gil, int lineno) {
+    int first_time;
+    struct {{memview_struct_name}} *memview = memslice->memview;
+    if (!memview) {
+        char msg[50];
+        snprintf(msg, 50, "memoryslice is not initialized (line %d)", lineno);
+        Py_FatalError(msg);
+    }
+
+    PyThread_acquire_lock(memview->lock, 1);
+    first_time = (memview->acquisition_count++ == 0);
+    PyThread_release_lock(memview->lock);
+
+    /* printf("INCREF %d: acquisition_count=%d, refcount=%d\n", lineno,
+           memview->acquisition_count, memview->ob_refcnt); */
+
+    if (first_time) {
+        if (have_gil) {
+            Py_INCREF((PyObject *) memview);
+        } else {
+            PyGILState_STATE _gilstate = PyGILState_Ensure();
+            Py_INCREF((PyObject *) memview);
+            PyGILState_Release(_gilstate);
+        }
+    }
+}
+
+static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *memslice,
+                                             int have_gil, int lineno) {
+    int last_time;
+    struct {{memview_struct_name}} *memview = memslice->memview;
+
+    if (!memview) {
+        return;
+    }
+
+    PyThread_acquire_lock(memview->lock, 1);
+    last_time = (memview->acquisition_count-- == 1);
+    PyThread_release_lock(memview->lock);
+
+    /* printf("DECREF %d: acquisition_count=%d, refcount=%d\n", lineno,
+           memview->acquisition_count, memview->ob_refcnt); */
+
+    if (last_time) {
+        if (have_gil) {
+            Py_CLEAR(memview);
+        } else {
+            PyGILState_STATE _gilstate = PyGILState_Ensure();
+            Py_CLEAR(memview);
+            PyGILState_Release(_gilstate);
+        }
+        memslice->data = NULL;
+    }
+}
+
 ////////// MemviewSliceCopyTemplate //////////
 
 static __Pyx_memviewslice {{copy_name}}(const __Pyx_memviewslice from_mvs) {
@@ -259,7 +360,8 @@ static __Pyx_memviewslice {{copy_name}}(const __Pyx_memviewslice from_mvs) {
     }
     __Pyx_GOTREF(array_obj);
 
-    memview_obj = __pyx_memoryview_new((PyObject *) array_obj, {{contig_flag}});
+    memview_obj = (struct __pyx_memoryview_obj *) __pyx_memoryview_new(
+                                (PyObject *) array_obj, {{contig_flag}});
     if (unlikely(!memview_obj)) {
         goto fail;
     }
@@ -292,3 +394,21 @@ no_fail:
 
 }
 
+/////////////// MemviewSliceIndex ///////////////
+
+static CYTHON_INLINE char *__pyx_memviewslice_index_full(char *bufp, Py_ssize_t idx, Py_ssize_t stride, Py_ssize_t suboffset) {
+    bufp = bufp + idx * stride;
+    if (suboffset >= 0) {
+        bufp = *((char **) bufp) + suboffset;
+    }
+    return bufp;
+}
+
+/* The call has already done the indexing */
+static CYTHON_INLINE char *__pyx_memviewslice_index_full_contig(char *bufp, Py_ssize_t suboffset) {
+    if (suboffset >= 0) {
+        bufp = *((char **) bufp) + suboffset;
+    }
+    return bufp;
+}
+

Cython/Utils.py

@@ -4,6 +4,7 @@
 #
 
 import os, sys, re, codecs
+from Cython import Tempita
 
 def replace_suffix(path, newsuf):
     base, _ = os.path.splitext(path)
@@ -215,10 +216,3 @@ def long_literal(value):
     if isinstance(value, basestring):
         value = str_to_number(value)
     return not -2**31 <= value < 2**31
-
-def none_or_sub(s, data):
-    if s is None:
-        return s
-    else:
-        return s % data
-

tests/run/cythonarray.pyx

@@ -2,10 +2,8 @@
 
 from __future__ import unicode_literals
 
-# from cython cimport array
-# cimport cython.array as array
+from cython cimport array
 cimport cython as cy
-# array = cython.array
 
 def contiguity():
     '''
@@ -69,3 +67,37 @@ def dont_allocate_buffer():
 
 cdef void callback(char *data):
     print "callback called %d" % <long> data
+
+cdef create_array(shape, mode):
+    cdef array result = array(shape, itemsize=sizeof(int), format='i', mode=mode)
+    cdef int *data = <int *> result.data
+    cdef int i, j, cidx, fidx
+
+    for i in range(shape[0]):
+        for j in range(shape[1]):
+            cidx = i * shape[1] + j
+            fidx = i + j * shape[0]
+
+            if mode == 'fortran':
+                data[fidx] = cidx
+            else:
+                data[cidx] = cidx
+
+    return result
+
+def test_cython_array():
+    """
+    >>> test_cython_array()
+    98
+    61
+    98
+    61
+    """
+    cdef int[:, ::1] carr = create_array((14, 10), 'c')
+    cdef int[::1, :] farr = create_array((14, 10), 'fortran')
+
+    print carr[9, 8]
+    print carr[6, 1]
+
+    print farr[9, 8]
+    print farr[6, 1]

tests/run/memslice_indexing.pyx

-cimport cython
-from cython cimport array
-
-from libc.stdlib cimport malloc, free
-
-def create_array(shape, mode='c'):
-    cdef array result = array(shape, itemsize=sizeof(int), format='i', mode=mode)
-    cdef int *data = <int *> result.data
-    cdef int i, j, value
-
-    for i in range(shape[0]):
-        for j in range(shape[1]):
-            value = i * shape[0] + j
-            if mode == 'fortran':
-                data[i + j * 10] = value
-            else:
-                data[value] = value
-
-    return result
-
-def slice_contig_indexing():
-    """
-    >>> print("disabled")
-    disabled
-
-    slice_contig_indexing()
-    98
-    61
-    98
-    61
-    """
-    cdef int[:, ::1] carr = create_array((14, 10))
-    cdef int[::1, :] farr = create_array((10, 14), mode='fortran')
-
-    print carr[9, 8]
-    print carr[6, 1]
-
-    print farr[9, 8]
-    print farr[6, 1]
-