inventory_api: Interpolation method + group by time sequence backported

jerome/erp5!6
(cherry picked from commit 7a4b9c454c0f87ac19adb7f1d455af44aeaef6f8)

product/ERP5/Tool/SimulationTool.py

@@ -627,6 +627,8 @@ class SimulationTool(BaseTool):
         omit_output=0,
         omit_asset_increase=0,
         omit_asset_decrease=0,
+        # interpolation_method
+        interpolation_method='default',
         # group by
         group_by_node=0,
         group_by_node_category=0,
@@ -660,6 +662,7 @@ class SimulationTool(BaseTool):
         group_by_function_category=0,
         group_by_function_category_strict_membership=0,
         group_by_date=0,
+        group_by_time_sequence_list=None,
         # sort_on
         sort_on=None,
         group_by=None,
@@ -713,6 +716,43 @@ class SimulationTool(BaseTool):
           date_dict['range'] = 'ngt'
         if date_dict:
           column_value_dict['date'] = date_dict
+        if interpolation_method != 'default':
+          if not group_by_time_sequence_list:
+            if not (from_date and (to_date or at_date)):
+              raise ValueError("date_range is required to use interpolation_method")
+            # if we consider flow, we also select movement whose mirror date is
+            # in the from_date/to_date range and movement whose
+            # start_date/stop_date contains the report range.
+            # The selected range is wider, but the selected movements will have an
+            # "interpolation_ratio" applied to their quantity and prices.
+            if to_date:
+              column_value_dict['date'] = ComplexQuery(
+                    Query(date=(from_date, to_date), range='minmax'),
+                    Query(mirror_date=(from_date, to_date), range='minmax'),
+                    ComplexQuery(
+                        Query(mirror_date=from_date, range='min'),
+                        Query(date=to_date, range='max'),
+                        operator="AND"),
+                    ComplexQuery(
+                        Query(date=from_date, range='min'),
+                        Query(mirror_date=to_date, range='max'),
+                        operator="AND"),
+                    operator="OR"
+                )
+            else:
+              column_value_dict['date'] = ComplexQuery(
+                    Query(date=(from_date, at_date), range='minngt'),
+                    Query(mirror_date=(from_date, at_date), range='minngt'),
+                    ComplexQuery(
+                        Query(mirror_date=from_date, range='min'),
+                        Query(date=at_date, range='ngt'),
+                        operator="AND"),
+                    ComplexQuery(
+                        Query(date=from_date, range='min'),
+                        Query(mirror_date=at_date, range='ngt'),
+                        operator="AND"),
+                    operator="OR"
+                )
       else:
         column_value_dict['date'] = {'query': [to_date], 'range': 'ngt'}
         column_value_dict['mirror_date'] = {'query': [from_date], 'range': 'nlt'}
@@ -998,6 +1038,8 @@ class SimulationTool(BaseTool):
         new_kw['related_key_select_expression_list'] =\
                 related_key_select_expression_list
 
+# XXX
+      sql_kw['group_by_time_sequence_list'] = group_by_time_sequence_list
       return sql_kw, new_kw
 
     #######################################################
@@ -1082,6 +1124,17 @@ class SimulationTool(BaseTool):
       output_simulation_state - only take rows with specified simulation_state
                         and quantity < 0
 
+      interpolation_method - Method to consider movements when calculating flows.
+        * (default): Consider the movement decreases 100% of source stock on
+          start date and increase 100% of the destination node stock on stop
+          date.
+        * linear: consider the movement decreases source stock and increase
+          destination stock linearly between start date and stop date.
+        * all_or_nothing: consider only movement who starts after the beginning of
+          the query period and finishes after the end of the query period.
+        * one_for_all: consider the movement fully as long as it is partially
+          contained in the query period.
+
       ignore_variation -  do not take into account variation in inventory
                         calculation (useless on getInventory, but useful on
                         getInventoryList)
@@ -1208,6 +1261,7 @@ class SimulationTool(BaseTool):
         group_by_section_category=0,
         group_by_section_category_strict_membership=0,
         group_by_resource=None,
+        group_by_time_sequence_list=(),
         movement_list_mode=0,
         group_by=None,
         **ignored):
@@ -1233,7 +1287,8 @@ class SimulationTool(BaseTool):
            group_by_mirror_section or group_by_payment or \
            group_by_sub_variation or group_by_variation or \
            group_by_movement or group_by_date or group_by_section_category or\
-           group_by_section_category_strict_membership:
+           group_by_section_category_strict_membership or \
+           group_by_time_sequence_list:
           if group_by_resource is None:
             group_by_resource = 1
           new_group_by_dict['group_by_resource'] = group_by_resource
@@ -1256,6 +1311,7 @@ class SimulationTool(BaseTool):
                          omit_simulation=0,
                          only_accountable=True,
                          default_stock_table='stock',
+                         interpolation_method='default',
                          selection_domain=None, selection_report=None,
                          statistic=0, inventory_list=1,
                          precision=None, connection_id=None,
@@ -1334,6 +1390,7 @@ class SimulationTool(BaseTool):
         'stock_table_id': default_stock_table,
         'src__': src__,
         'ignore_variation': ignore_variation,
+        'interpolation_method': interpolation_method,
         'standardise': standardise,
         'omit_simulation': omit_simulation,
         'only_accountable': only_accountable,
@@ -1349,7 +1406,9 @@ class SimulationTool(BaseTool):
       # Get cached data
       if getattr(self, "Resource_zGetInventoryCacheResult", None) is not None and \
               optimisation__ and (not kw.get('from_date')) and \
-              'transformed_resource' not in kw:
+              'transformed_resource' not in kw \
+              and "category" not in str(kw) \
+              and "group_by_time_sequence_list" not in kw:
         # Here is the different kind of date
         # from_date : >=
         # to_date   : <
@@ -1379,7 +1438,7 @@ class SimulationTool(BaseTool):
           kw['from_date'] = cached_date
       else:
         cached_result = []
-      sql_kw, new_kw = self._generateKeywordDict(**kw)
+      sql_kw, new_kw = self._generateKeywordDict(interpolation_method=interpolation_method, **kw)
       # Copy kw content as _generateSQLKeywordDictFromKeywordDict
       # remove some values from it
       try:
@@ -1392,6 +1451,25 @@ class SimulationTool(BaseTool):
       stock_sql_kw = self._generateSQLKeywordDictFromKeywordDict(
           table=default_stock_table, sql_kw=sql_kw, new_kw=new_kw_copy)
       stock_sql_kw.update(base_inventory_kw)
+
+        # TODO: move in _generateSQLKeywordDictFromKeywordDict
+      if interpolation_method in ('linear', 'all_or_nothing', 'one_for_all'):
+          # XXX only DateTime instance are supported
+        from_date = kw.get('from_date')
+        if from_date:
+          from_date = from_date.toZone("UTC")
+        to_date = kw.get('to_date')
+        if to_date:
+          to_date = to_date.toZone("UTC")
+        at_date = kw.get('at_date')
+        if at_date:
+          at_date = at_date.toZone("UTC")
+        stock_sql_kw['interpolation_method_from_date'] = from_date
+        stock_sql_kw['interpolation_method_to_date'] = to_date
+        stock_sql_kw['interpolation_method_at_date'] = at_date
+      elif interpolation_method != 'default':
+        raise ValueError("Unsupported interpolation_method %r" % (interpolation_method,))
+
       delta_result = self.Resource_zGetInventoryList(
           **stock_sql_kw)
       if src__:

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Resource_zGetInventoryList.xml

@@ -45,7 +45,12 @@ convert_quantity_result\r\n
 quantity_unit_uid\r\n
 stock_table_id=stock\r\n
 transformed_uid\r\n
-transformed_variation_text</string> </value>
+transformed_variation_text\r\n
+group_by_time_sequence_list:list\r\n
+interpolation_method\r\n
+interpolation_method_from_date=not_applicable\r\n
+interpolation_method_to_date=not_applicable\r\n
+interpolation_method_at_date=not_applicable</string> </value>
         </item>
         <item>
             <key> <string>cache_time_</string> </key>
@@ -83,29 +88,50 @@ transformed_variation_text</string> </value>
             <key> <string>src</string> </key>
             <value> <string encoding="cdata"><![CDATA[
 
+<dtml-let interpolation_ratio="SimulationTool_zGetInterpolationMethod(\n
+  stock_table_id=stock_table_id,\n
+  interpolation_method=interpolation_method,\n
+  interpolation_method_from_date=interpolation_method_from_date,\n
+  interpolation_method_to_date=interpolation_method_to_date,\n
+  interpolation_method_at_date=interpolation_method_at_date,\n
+  group_by_time_sequence_list=group_by_time_sequence_list,\n
+  src__=1)">\n
+\n
 SELECT\n
 <dtml-if expr="precision is not None">\n
-  SUM(ROUND(<dtml-var stock_table_id>.quantity\n
-    <dtml-if transformed_uid> * transformation.quantity</dtml-if>, <dtml-var precision>)) AS inventory,\n
-  SUM(ROUND(<dtml-var stock_table_id>.quantity\n
-    <dtml-if transformed_uid> * transformation.quantity</dtml-if>, <dtml-var precision>)) AS total_quantity,\n
+  SUM(ROUND(\n
+    <dtml-var stock_table_id>.quantity\n
+    <dtml-if transformed_uid> * transformation.quantity</dtml-if>\n
+    * <dtml-var interpolation_ratio>, <dtml-var precision>)) AS inventory,\n
+  SUM(ROUND(\n
+    <dtml-var stock_table_id>.quantity\n
+    <dtml-if transformed_uid> * transformation.quantity</dtml-if>\n
+    * <dtml-var interpolation_ratio>, <dtml-var precision>)) AS total_quantity,\n
   <dtml-if convert_quantity_result>\n
     SUM(ROUND(<dtml-var stock_table_id>.quantity * measure.quantity\n
       <dtml-if quantity_unit_uid> / quantity_unit_conversion.quantity</dtml-if>\n
-      <dtml-if transformed_uid> * transformation.quantity</dtml-if>, <dtml-var precision>))\n
+    * <dtml-var interpolation_ratio>, <dtml-var precision>))\n
     AS converted_quantity,\n
   </dtml-if>\n
-  IFNULL(SUM(ROUND(<dtml-var stock_table_id>.total_price, <dtml-var precision>)), 0) AS total_price\n
+\n
+  IFNULL(SUM(ROUND(\n
+    <dtml-var stock_table_id>.total_price * <dtml-var interpolation_ratio>, <dtml-var precision>)), 0) AS total_price\n
 <dtml-else>\n
-  SUM(<dtml-var stock_table_id>.quantity <dtml-if transformed_uid> * transformation.quantity</dtml-if>) AS inventory,\n
-  SUM(<dtml-var stock_table_id>.quantity <dtml-if transformed_uid> * transformation.quantity</dtml-if>) AS total_quantity,\n
+  SUM(<dtml-var stock_table_id>.quantity\n
+      <dtml-if transformed_uid> * transformation.quantity</dtml-if>\n
+      * <dtml-var interpolation_ratio>\n
+     ) AS inventory,\n
+  SUM(<dtml-var stock_table_id>.quantity\n
+      <dtml-if transformed_uid> * transformation.quantity</dtml-if>\n
+      * <dtml-var interpolation_ratio>\n
+     ) AS total_quantity,\n
   <dtml-if convert_quantity_result>\n
     ROUND(SUM(<dtml-var stock_table_id>.quantity * measure.quantity\n
       <dtml-if quantity_unit_uid> / quantity_unit_conversion.quantity</dtml-if>\n
-      <dtml-if transformed_uid> * transformation.quantity</dtml-if>), 12)\n
+      <dtml-if transformed_uid> * transformation.quantity</dtml-if> * <dtml-var interpolation_ratio>), 12)\n
     AS converted_quantity,\n
   </dtml-if>\n
-  IFNULL(SUM(<dtml-var stock_table_id>.total_price), 0) AS total_price\n
+  IFNULL(SUM(<dtml-var stock_table_id>.total_price * <dtml-var interpolation_ratio>), 0) AS total_price\n
 </dtml-if>\n
 <dtml-if inventory_list>\n
   ,\n
@@ -140,6 +166,8 @@ SELECT\n
   COUNT(DISTINCT <dtml-var stock_table_id>.uid) AS stock_uid,\n
   MAX(<dtml-var stock_table_id>.date) AS date\n
 </dtml-if>\n
+<dtml-if group_by_time_sequence_list>, slot_index </dtml-if> <dtml-comment>XXX is this really needed?</dtml-comment>\n
+\n
 <dtml-if select_expression>, <dtml-var select_expression></dtml-if>\n
 \n
 FROM\n
@@ -153,6 +181,55 @@ FROM\n
   </dtml-if>\n
 </dtml-in>\n
 , <dtml-var stock_table_id>\n
+\n
+   <dtml-if group_by_time_sequence_list>\n
+     RIGHT JOIN\n
+       ( <dtml-in prefix="time_slot" expr="_.list(_.enumerate(group_by_time_sequence_list))">\n
+         SELECT\n
+           <dtml-sqlvar expr="time_slot_key" type="int"> slot_index,\n
+           <dtml-sqlvar expr="time_slot_item.get(\'from_date\')" type="datetime" optional> slot_from_date,\n
+           <dtml-sqlvar expr="time_slot_item.get(\'at_date\')" type="datetime" optional> slot_at_date,\n
+           <dtml-sqlvar expr="time_slot_item.get(\'to_date\')" type="datetime" optional> slot_to_date\n
+\n
+         <dtml-unless time_slot_end>UNION ALL</dtml-unless>\n
+       </dtml-in> ) slots\n
+     ON\n
+     <dtml-if group_by_time_sequence_list>\n
+     (\n
+       ( slot_from_date is not null AND\n
+        ( slot_at_date is not null AND\n
+         GREATEST(`stock`.`date`, `stock`.`mirror_date`) >= slot_from_date AND\n
+         LEAST(`stock`.`date`, `stock`.`mirror_date`) <= slot_at_date\n
+        ) OR (\n
+          (\n
+            slot_to_date is not null AND\n
+            GREATEST(`stock`.`date`, `stock`.`mirror_date`) >= slot_from_date AND\n
+            LEAST(`stock`.`date`, `stock`.`mirror_date`) < slot_to_date\n
+          ) OR (\n
+            GREATEST(`stock`.`date`, `stock`.`mirror_date`) >= slot_from_date AND\n
+            slot_at_date is null AND slot_to_date is null\n
+          )\n
+        )\n
+       ) OR (\n
+         slot_from_date is null AND (\n
+           ( slot_at_date is not null AND\n
+            ( LEAST(`stock`.`date`, `stock`.`mirror_date`) <= slot_at_date )\n
+           ) OR  LEAST(`stock`.`date`, `stock`.`mirror_date`) < slot_to_date\n
+         )\n
+       )\n
+     )\n
+     <dtml-else>\n
+     (\n
+       ( slot_from_date is null OR stock.date >= slot_from_date )\n
+       AND ( slot_at_date is null OR stock.date <= slot_at_date )\n
+       AND ( slot_to_date is null OR stock.date < slot_to_date )\n
+     )\n
+     </dtml-if>\n
+   </dtml-if>\n
+\n
+\n
+\n
+\n
 </dtml-if>\n
 <dtml-if quantity_unit_uid> <dtml-comment>XXX quantity unit conversion will not work when using implict_join=False</dtml-comment>\n
   LEFT JOIN quantity_unit_conversion ON \n
@@ -200,13 +277,19 @@ WHERE\n
 <dtml-if group_by_expression>\n
 GROUP BY\n
     <dtml-if transformed_uid>transformation.transformed_uid,</dtml-if>\n
+    <dtml-if group_by_time_sequence_list>slot_index,</dtml-if>\n
     <dtml-var group_by_expression>\n
+\n
 </dtml-if>\n
 <dtml-if order_by_expression>\n
 ORDER BY\n
   <dtml-var order_by_expression>\n
+<dtml-else>\n
+  <dtml-if group_by_time_sequence_list>\n
+    ORDER BY slot_index\n
+  </dtml-if>\n
 </dtml-if>\n
-
+</dtml-let>
 
 ]]></string> </value>
         </item>

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/SimulationTool_zGetInterpolationMethod.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="SQL" module="Products.ZSQLMethods.SQL"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_Use_Database_Methods_Permission</string> </key>
+            <value>
+              <list>
+                <string>Anonymous</string>
+              </list>
+            </value>
+        </item>
+        <item>
+            <key> <string>arguments_src</string> </key>
+            <value> <string>stock_table_id\r\n
+interpolation_method\r\n
+interpolation_method_from_date\r\n
+interpolation_method_to_date\r\n
+interpolation_method_at_date\r\n
+group_by_time_sequence_list:list</string> </value>
+        </item>
+        <item>
+            <key> <string>connection_id</string> </key>
+            <value> <string>cmf_activity_sql_connection</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>SimulationTool_zGetInterpolationMethod</string> </value>
+        </item>
+        <item>
+            <key> <string>src</string> </key>
+            <value> <string encoding="cdata"><![CDATA[
+
+<dtml-if expr="interpolation_method == \'linear\'">\n
+  <dtml-if group_by_time_sequence_list>\n
+    CASE\n
+     WHEN <dtml-var stock_table_id>.mirror_date = <dtml-var stock_table_id>.date THEN 1\n
+     ELSE (\n
+       UNIX_TIMESTAMP(\n
+         IFNULL(\n
+           LEAST(\n
+             IFNULL(slot_at_date, slot_to_date),\n
+             GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)\n
+           ),\n
+           GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)\n
+         )\n
+       )\n
+       - UNIX_TIMESTAMP(\n
+         GREATEST(\n
+           IFNULL(\n
+             slot_from_date, TIMESTAMP(0)\n
+           ),\n
+           LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)\n
+         )\n
+       )\n
+     )\n
+     / (\n
+       UNIX_TIMESTAMP(GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)) -\n
+       UNIX_TIMESTAMP(LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)) ) END\n
+  <dtml-else>\n
+  CASE\n
+     WHEN <dtml-var stock_table_id>.mirror_date = <dtml-var stock_table_id>.date THEN 1\n
+     ELSE (\n
+      UNIX_TIMESTAMP(LEAST(\n
+        <dtml-if interpolation_method_at_date>\n
+          <dtml-sqlvar interpolation_method_at_date type="datetime">\n
+        <dtml-else>\n
+          <dtml-sqlvar interpolation_method_to_date type="datetime">\n
+        </dtml-if>,\n
+            GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date) ))\n
+       - UNIX_TIMESTAMP(GREATEST(<dtml-sqlvar interpolation_method_from_date type="datetime">,\n
+                  LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date))))\n
+      / ( UNIX_TIMESTAMP(GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)) -\n
+         UNIX_TIMESTAMP(LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)) ) END\n
+  </dtml-if>\n
+<dtml-elif expr="interpolation_method == \'all_or_nothing\'">\n
+  CASE\n
+    WHEN (\n
+      -- movement contained in time frame\n
+      LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)\n
+        >= <dtml-sqlvar interpolation_method_from_date type="datetime"> AND\n
+      GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)\n
+      <dtml-if interpolation_method_at_date>\n
+        <= <dtml-sqlvar interpolation_method_at_date type="datetime">\n
+      <dtml-else>\n
+        < <dtml-sqlvar interpolation_method_to_date type="datetime">\n
+      </dtml-if>\n
+    ) THEN 1\n
+    ELSE 0\n
+  END\n
+<dtml-elif expr="interpolation_method == \'one_for_all\'">\n
+  CASE\n
+    WHEN (\n
+      -- movement overlaps with time frame\n
+      GREATEST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)\n
+        <= <dtml-sqlvar interpolation_method_from_date type="datetime"> OR\n
+      LEAST(<dtml-var stock_table_id>.date, <dtml-var stock_table_id>.mirror_date)\n
+      <dtml-if interpolation_method_at_date>\n
+        >= <dtml-sqlvar interpolation_method_at_date type="datetime">\n
+      <dtml-else>\n
+        > <dtml-sqlvar interpolation_method_to_date type="datetime">\n
+      </dtml-if>\n
+    ) THEN 0\n
+    ELSE 1\n
+  END\n
+<dtml-else>\n
+  1\n
+</dtml-if>\n
+
+
+]]></string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string></string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>