Uses pandas to process inventory

- creates an external method to call the pandas version of the inventory report
- adds a zsql method to get all the rows in the stocks table

bt5/erp5_r_order_prediction_tutorial/ExtensionTemplateItem/portal_components/extension.erp5.Rarima.py

 from math import cos
+from contextlib import contextmanager
 import rpy2.robjects as robjects
 from rpy2.robjects.packages import importr
 
@@ -19,4 +20,99 @@ def predict(self, data, n=5):
   pred = robjects.r.predict(fit, **kw)
   
   return list(pred[0])
+  
+def get_data(self):
+  import csv
+  import StringIO
+  
+  log_output = StringIO.StringIO()
+  
+  product = self.portal_catalog(title='Third Product', portal_type="Product")[0]
+  nexedi = self.portal_catalog(title='Nexedi', portal_type='Organisation')[1]
+  with Benchmark('Normal Inventory Report', log_output):
+    result = self.portal_simulation.getInventoryList(
+      resource_uid=product.getUid(),
+      node_uid=nexedi.getUid(),
+      simulation_state='planned',
+      group_by='date'
+    )
+  log_output.write(list(result))
+  return log_output.getvalue()
+    
+  quantity_list = self.SaleOrderModule_zGetQuantityList()
+  columns = quantity_list._names
+  
+  csv_file = StringIO.StringIO()
+  
+  writer = csv.writer(csv_file, delimiter=';')
+  writer.writerow(columns)
+  for row in quantity_list:
+    writer.writerow(row)
+      
+  return csv_file.getvalue()
+    
+
+def inventory_with_pandas(self, source__=False):
+  import pandas as pd
+  from StringIO import StringIO
+  
+  product = self.portal_catalog(title='Third Product', portal_type="Product")[0]
+  nexedi = self.portal_catalog(title='Nexedi', portal_type='Organisation')[1]
+  
+  output = StringIO()
+  with Benchmark('Inventory Report Pandas', output):
+    with Benchmark('Calling getInventoryList', output):
+      # raw_results = self.portal_simulation.getInventoryList()
+      raw_results = self.SaleOrderModule_zGetStock()
+    if source__ != 0:
+      return raw_results
+    
+    output.write('Columns: {}\n'.format(raw_results._names))
+    with Benchmark('Getting attributes', output):
+      # Using generator list comprehension here to avoid looping over the array
+      # 2 times. Can save a significant amount of time as the number of stock
+      # object is very big.
+      #
+      results = (
+        [
+          item['resource_uid'],
+          item['node_uid'],
+          item['mirror_node_uid'],
+          item['date'].strftime('%Y/%m/%d %H:%M:%S UTC'), 
+          item['total_quantity'],
+          item['simulation_state']
+        ] 
+        for item in raw_results
+      )
+      
+    with Benchmark('Creating pandas.DataFrame', output):
+      df = pd.DataFrame(results, columns=['resource_uid', 'node_uid', 'mirror_section_uid', 'date', 'quantity', 'simulation_state'])
+    
+    with Benchmark('Filtering, grouping and summing within the DataFrame', output):
+      result = df[
+        df['resource_uid'] == product.getUid()
+      ][
+        df['simulation_state'] == 'planned'
+      ][
+        df['node_uid'] == nexedi.getUid()
+      ].groupby(['date'])['quantity'].sum() 
+  
+  output.write(str(result))
+  return output.getvalue()
+  
+
+def get_data_inventory(self, profile=False, source__=0):
+  import cProfile
+  if profile:
+    results = cProfile.runctx('inventory_with_pandas(self, source__)', locals(), globals(), '/tmp/stats')
+    return 'ok'
+  return inventory_with_pandas(self, source__)
 
+@contextmanager
+def Benchmark(test_type, output):
+  from time import time
+  start = time()
+  yield
+  delta = time() - start
+  delta_msg = "[{test}] took {delta} seconds to run.\n".format(delta=delta,  test=test_type)
+  output.write(delta_msg)

bt5/erp5_r_order_prediction_tutorial/ExtensionTemplateItem/portal_components/extension.erp5.Rarima.xml

@@ -46,7 +46,9 @@
             <key> <string>text_content_warning_message</string> </key>
             <value>
               <tuple>
-                <string>W: 10,  2: Unused variable \'base\' (unused-variable)</string>
+                <string>W: 11,  2: Unused variable \'base\' (unused-variable)</string>
+                <string>W: 42,  2: Unreachable code (unreachable)</string>
+                <string>W:107,  4: Unused variable \'results\' (unused-variable)</string>
                 <string>W:  1,  0: Unused cos imported from math (unused-import)</string>
               </tuple>
             </value>

bt5/erp5_r_order_prediction_tutorial/SkinTemplateItem/portal_skins/erp5_r_order_prediction_tutorial/SaleOrderModule_getArimaPredictionList.xml

@@ -8,7 +8,7 @@
       <dictionary>
         <item>
             <key> <string>_function</string> </key>
-            <value> <string>predict</string> </value>
+            <value> <string>get_data</string> </value>
         </item>
         <item>
             <key> <string>_module</string> </key>

bt5/erp5_r_order_prediction_tutorial/SkinTemplateItem/portal_skins/erp5_r_order_prediction_tutorial/SaleOrderModule_getArimaPredictionListInventory.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_function</string> </key>
+            <value> <string>get_data_inventory</string> </value>
+        </item>
+        <item>
+            <key> <string>_module</string> </key>
+            <value> <string>Rarima</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>SaleOrderModule_getArimaPredictionListInventory</string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string></string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_r_order_prediction_tutorial/SkinTemplateItem/portal_skins/erp5_r_order_prediction_tutorial/SaleOrderModule_predictFutureOrderList.xml

@@ -50,7 +50,20 @@
         </item>
         <item>
             <key> <string>_body</string> </key>
-            <value> <string>quantity_list = context.SaleOrderModule_zGetQuantityList()\n
+            <value> <string>import csv\n
+import StringIO\n
+\n
+quantity_list = context.SaleOrderModule_zGetQuantityList()\n
+\n
+csv_file = StringIO.StringIO()\n
+\n
+with csv_file as csv_file:\n
+  writer = csv.writer(csv_file, delimiter=\';\')\n
+  for row in quantity_list:\n
+    writer.writerow(row)\n
+    \n
+return csv_file.read()\n
+\n
 current_order_list = map(lambda x: x[\'quantity\'], quantity_list)\n
 last_date = quantity_list[-1][\'date\']\n
 \n

bt5/erp5_r_order_prediction_tutorial/SkinTemplateItem/portal_skins/erp5_r_order_prediction_tutorial/SaleOrderModule_zGetQuantityList.xml

@@ -49,10 +49,32 @@
               </list>
             </value>
         </item>
+        <item>
+            <key> <string>allow_simple_one_argument_traversal</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
         <item>
             <key> <string>arguments_src</string> </key>
             <value> <string></string> </value>
         </item>
+        <item>
+            <key> <string>cache_time_</string> </key>
+            <value> <int>0</int> </value>
+        </item>
+        <item>
+            <key> <string>class_file_</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>class_name_</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>connection_hook</string> </key>
+            <value> <string></string> </value>
+        </item>
         <item>
             <key> <string>connection_id</string> </key>
             <value> <string>erp5_sql_connection</string> </value>
@@ -61,12 +83,20 @@
             <key> <string>id</string> </key>
             <value> <string>SaleOrderModule_zGetQuantityList</string> </value>
         </item>
+        <item>
+            <key> <string>max_cache_</string> </key>
+            <value> <int>0</int> </value>
+        </item>
+        <item>
+            <key> <string>max_rows_</string> </key>
+            <value> <int>0</int> </value>
+        </item>
         <item>
             <key> <string>src</string> </key>
             <value> <string encoding="cdata"><![CDATA[
 
 SELECT\n
-  stock.date as date, sum(stock.quantity) as quantity\n
+  stock.uid, stock.is_accountable, stock.date as date, sum(stock.quantity) as quantity\n
 FROM\n
   stock, catalog as product\n
 WHERE\n
@@ -80,7 +110,7 @@ AND\n
 AND\n
   stock.portal_type = "Simulation Movement"\n
 GROUP BY\n
-  DAY(stock.date), MONTH(stock.date), YEAR(stock.date)\n
+  stock.date\n
 ORDER BY\n
   stock.date
 

bt5/erp5_r_order_prediction_tutorial/SkinTemplateItem/portal_skins/erp5_r_order_prediction_tutorial/SaleOrderModule_zGetStock.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="SQL" module="Products.ZSQLMethods.SQL"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>allow_simple_one_argument_traversal</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>arguments_src</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>cache_time_</string> </key>
+            <value> <int>0</int> </value>
+        </item>
+        <item>
+            <key> <string>class_file_</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>class_name_</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>connection_hook</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>connection_id</string> </key>
+            <value> <string>erp5_sql_connection</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>SaleOrderModule_zGetStock</string> </value>
+        </item>
+        <item>
+            <key> <string>max_cache_</string> </key>
+            <value> <int>0</int> </value>
+        </item>
+        <item>
+            <key> <string>max_rows_</string> </key>
+            <value> <int>0</int> </value>
+        </item>
+        <item>
+            <key> <string>src</string> </key>
+            <value> <string>select * from stock where portal_type = "Simulation Movement";</string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string></string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

[Jérome Perrin]

Instead of passing simulation_state=planned, it is better to use getFutureInventoryList.

Also it may be better for the tutprial use getMovementHistoryList and then use pandas function to group by date (not sure)