erp5_wendelin: Data Array View Line implementation using unstructured_to_structured method

bt5/erp5_wendelin/DocumentTemplateItem/portal_components/document.erp5.DataArrayViewLine.py

@@ -4,26 +4,21 @@
 # Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
 #                    Klaus Wölfel <klaus@nexedi.com>
 #
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsability of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# garantees and support are strongly adviced to contract a Free Software
-# Service Company
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
 #
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
 #
 ##############################################################################
 import numpy as np
@@ -32,6 +27,142 @@ from Products.ERP5Type.Base import TempBase
 from Products.ERP5Type.Utils import createExpressionContext, \
   evaluateExpressionFromString
 
+def _get_fields_and_offsets(dt, offset=0):
+  """
+  Returns a flat list of (dtype, count, offset) tuples of all the
+  scalar fields in the dtype "dt", including nested fields, in left
+  to right order.
+  """
+
+  # counts up elements in subarrays, including nested subarrays, and returns
+  # base dtype and count
+  def count_elem(dt):
+    count = 1
+    while dt.shape != ():
+      for size in dt.shape:
+        count *= size
+      dt = dt.base
+    return dt, count
+
+  fields = []
+  for name in dt.names:
+    field = dt.fields[name]
+    f_dt, f_offset = field[0], field[1]
+    f_dt, n = count_elem(f_dt)
+
+    if f_dt.names is None:
+      fields.append((np.dtype((f_dt, (n,))), n, f_offset + offset))
+    else:
+      subfields = _get_fields_and_offsets(f_dt, f_offset + offset)
+      size = f_dt.itemsize
+
+      for i in range(n):
+        if i == 0:
+          # optimization: avoid list comprehension if no subarray
+          fields.extend(subfields)
+        else:
+          fields.extend([(d, c, o + i*size) for d, c, o in subfields])
+  return fields
+
+def unstructured_to_structured(arr, dtype=None, names=None, align=False,
+                               copy=False, casting='unsafe'):
+  """
+  Converts an n-D unstructured array into an (n-1)-D structured array.
+  The last dimension of the input array is converted into a structure, with
+  number of field-elements equal to the size of the last dimension of the
+  input array. By default all output fields have the input array's dtype, but
+  an output structured dtype with an equal number of fields-elements can be
+  supplied instead.
+  Nested fields, as well as each element of any subarray fields, all count
+  towards the number of field-elements.
+  Parameters
+  ----------
+  arr : ndarray
+     Unstructured array or dtype to convert.
+  dtype : dtype, optional
+     The structured dtype of the output array
+  names : list of strings, optional
+     If dtype is not supplied, this specifies the field names for the output
+     dtype, in order. The field dtypes will be the same as the input array.
+  align : boolean, optional
+     Whether to create an aligned memory layout.
+  copy : bool, optional
+      See copy argument to `numpy.ndarray.astype`. If true, always return a
+      copy. If false, and `dtype` requirements are satisfied, a view is
+      returned.
+  casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional
+      See casting argument of `numpy.ndarray.astype`. Controls what kind of
+      data casting may occur.
+  Returns
+  -------
+  structured : ndarray
+     Structured array with fewer dimensions.
+  Examples
+  --------
+  >>> from numpy.lib import recfunctions as rfn
+  >>> dt = np.dtype([('a', 'i4'), ('b', 'f4,u2'), ('c', 'f4', 2)])
+  >>> a = np.arange(20).reshape((4,5))
+  >>> a
+  array([[ 0,  1,  2,  3,  4],
+         [ 5,  6,  7,  8,  9],
+         [10, 11, 12, 13, 14],
+         [15, 16, 17, 18, 19]])
+  >>> rfn.unstructured_to_structured(a, dt)
+  array([( 0, ( 1.,  2), [ 3.,  4.]), ( 5, ( 6.,  7), [ 8.,  9.]),
+         (10, (11., 12), [13., 14.]), (15, (16., 17), [18., 19.])],
+        dtype=[('a', '<i4'), ('b', [('f0', '<f4'), ('f1', '<u2')]), ('c', '<f4', (2,))])
+  """
+  if arr.shape == ():
+    raise ValueError('arr must have at least one dimension')
+  n_elem = arr.shape[-1]
+  if n_elem == 0:
+    # too many bugs elsewhere for this to work now
+    raise NotImplementedError("last axis with size 0 is not supported")
+
+  if dtype is None:
+    if names is None:
+      names = ['f{}'.format(n) for n in range(n_elem)]
+    out_dtype = np.dtype([(n, arr.dtype) for n in names], align=align)
+    fields = _get_fields_and_offsets(out_dtype)
+    dts, counts, offsets = zip(*fields)
+  else:
+    if names is not None:
+      raise ValueError("don't supply both dtype and names")
+    # if dtype is the args of np.dtype, construct it
+    dtype = np.dtype(dtype)
+    # sanity check of the input dtype
+    fields = _get_fields_and_offsets(dtype)
+    if len(fields) == 0:
+      dts, counts, offsets = [], [], []
+    else:
+      dts, counts, offsets = zip(*fields)
+
+    if n_elem != sum(counts):
+      raise ValueError('The length of the last dimension of arr must '
+                       'be equal to the number of fields in dtype')
+    out_dtype = dtype
+    if align and not out_dtype.isalignedstruct:
+      raise ValueError("align was True but dtype is not aligned")
+
+  names = ['f{}'.format(n) for n in range(len(fields))]
+
+  # Use a series of views and casts to convert to a structured array:
+
+  # first view as a packed structured array of one dtype
+  packed_fields = np.dtype({'names': names,
+                            'formats': [(arr.dtype, dt.shape) for dt in dts]})
+  arr = np.ascontiguousarray(arr).view(packed_fields)
+
+  # next cast to an unpacked but flattened format with varied dtypes
+  flattened_fields = np.dtype({'names': names,
+                               'formats': dts,
+                               'offsets': offsets,
+                               'itemsize': out_dtype.itemsize})
+  arr = arr.astype(flattened_fields, copy=copy, casting=casting)
+
+  # finally view as the final nested dtype and remove the last axis
+  return arr.view(out_dtype)[..., 0]
+
 class GetIndex(TempBase):
   def __getitem__(self, idx):
     return idx
@@ -51,27 +182,33 @@ class DataArrayViewLine(DataArray):
     """
     Get numpy view of Parent Data Array according to index.
     """
-    zbigarray = self.getPredecessorValue().getArray()
     getindex = GetIndex("getindex")
-    index_expression = self.getIndexExpression()
-    dtype_expression = self.getDtypeExpression()
+    zbigarray = self.getPredecessorValue().getArray()
+    array_view =zbigarray[:]
     name_list = self.getNameList()
+    dtype_expression = self.getDtypeExpression()
+    if dtype_expression is not None or name_list:
+      if dtype_expression is None:
+        dtype = zbigarray.dtype
+      else:
+        dtype = evaluateExpressionFromString(
+        createExpressionContext(None, portal=getindex),
+        dtype_expression)
+      dtype = np.dtype(dtype).newbyteorder('|') # copy=True did not return a copy
+      if name_list:
+        dtype.names = name_list
+      if zbigarray.dtype.names is None and dtype.names is not None:
+        array_view = unstructured_to_structured(array_view, dtype)
+      else:
+        array_view = array_view.view(dtype)
     index = evaluateExpressionFromString(
       createExpressionContext(None, portal=getindex),
-      "python: portal[%s]" %index_expression
+      "python: portal[%s]" %self.getIndexExpression()
     )
-    try:
-      array_view = zbigarray[index]
-    except TypeError:
-      array = zbigarray[:]
+    if isinstance(index, list) and array_view.dtype.names:
+      array = array_view
       new_dtype = np.dtype({name:array.dtype.fields[name] for name in index})
       array_view = np.ndarray(array.shape, new_dtype, array, 0, array.strides)
-
-    if dtype_expression is not None:
-      dtype = np.dtype(evaluateExpressionFromString(
-        createExpressionContext(None, portal=getindex), dtype_expression))
-      if name_list:
-        dtype.names = name_list
-      return np.ndarray((array_view.shape[0],), dtype, zbigarray[:], 0, array_view.strides[0])
     else:
+      array_view = array_view[index]
     return array_view