erp5_wendelin_telecom_ingestion: optimization of queries

bt5/erp5_wendelin_telecom_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_telecom/Alarm_handleInvalids.py

@@ -31,7 +31,7 @@ for data_stream in portal_catalog(**kw_dict):
 
 
 
-# invalidate old (more than 10min) pending ingestions (e.g. split ingestions that were canceled/interrumped)
+# invalidate old (more than 10 min) pending ingestions (e.g. split ingestions that were canceled/interrumped)
 from DateTime import DateTime
 now = DateTime()
 now_minus_max = now - 1.0/24/60*9999
@@ -40,21 +40,20 @@ now_minus_10 = now - 1.0/24/60*10
 catalog_kw = {'creation_date': {'query': (now_minus_max, now_minus_10), 'range': 'minmax'}, 'simulation_state': 'planned', 'portal_type': 'Data Ingestion'}
 
 for data_ingestion in portal_catalog(**catalog_kw):
-  related_split_ingestions = portal_catalog(portal_type = "Data Ingestion",
-                                            simulation_state = "planned",
-                                            reference = data_ingestion.getReference())
+  # search related data ingestions that are not old yet (less than 10 min)
+  catalog_kw = {'creation_date': {'query': (now_minus_10, DateTime()), 'range': 'minmax'},
+                'simulation_state': 'planned',
+                'portal_type': 'Data Ingestion',
+                'reference': data_ingestion.getReference()}
   invalidate = True
-  for related_ingestion in related_split_ingestions:
-    # only invalidate if all related ingestion are old
-    if len(portal_catalog(id=related_ingestion.getId(), **catalog_kw)) == 0:
-      context.logEntry("Data Ingestion %s is old, but it has related ingestion that are not. So it won't be invalidated yet." % data_ingestion.getId())
-      invalidate = False
-      break
+  if len(portal_catalog(**catalog_kw)) > 0:
+    context.logEntry("Data Ingestion %s is old, but it has related ingestion that are not. So it won't be invalidated yet." % data_ingestion.getId())
+    invalidate = False
 
   if invalidate:
     # invalidate related Data Stream
     kw_dict = {"portal_type": "Data Stream",
-               "reference": data_ingestion.getReference()}
+               "id": data_ingestion.getId()}
     for data_stream in portal_catalog(**kw_dict):
       if not data_stream.getReference().endswith("_invalid"):
         data_stream.setReference(data_stream.getReference() + "_invalid")

bt5/erp5_wendelin_telecom_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_telecom/ERP5Site_startIngestionList.py

 from Products.ERP5Type.Log import log
 from Products.ZSQLCatalog.SQLCatalog import Query, SimpleQuery
 
-# sort manually beacause query doesn't sort
-def sorted_by_date(results):
-  A = [ x for x in results]
-  for j in range(1,len(A)):
-    key = A[j]
-    i = j-1
-    while (i > -1) and key.getId()[-3:] < A[i].getId()[-3:]:
-      A[i+1]=A[i]
-      i=i-1
-    A[i+1] = key
-  return A
-
 portal = context.getPortalObject()
 portal_catalog = portal.portal_catalog
 
@@ -34,10 +22,9 @@ for data_ingestion in portal_catalog(portal_type = "Data Ingestion",
   elif len(related_split_ingestions) > 1:
     try:
       query = Query(portal_type="Data Stream", reference=data_ingestion.getReference())
-      result_list = portal_catalog(query=query, sort_on=(('creation_date', 'DESC', 'date'),))
+      result_list = portal_catalog(query=query, sort_on=(('creation_date', 'ascending'),))
       index = 1
-      # for some reason, the sort query doesn't work (random order)
-      for data_stream in sorted_by_date(result_list):
+      for data_stream in result_list:
         if index == 1:
           full_data_stream = data_stream
         else:

bt5/erp5_wendelin_telecom_ingestion/WorkflowTemplateItem/portal_workflow/validation_workflow/scripts/Action_invalidateRelatedObjects.py

@@ -12,7 +12,7 @@ if object.getPortalType() == "Data Set":
   kw_dict = {"portal_type": "Data Stream",
              "query": reference_query}
   for data_stream in portal_catalog(**kw_dict):
-    if data_stream.getReference().startswith(data_set.getReference()+'/'):
+    if data_stream.getReference().startswith(data_set.getReference()+'/') and not data_stream.getReference().endswith("_invalid"):
       portal.ERP5Site_invalidateIngestionObjects(data_stream.getReference())
   data_set.setReference(data_set.getReference() + "_invalid")
   context.logEntry("Data set '%s' invalidated." % data_set.getReference())