erp5_wendelin_telecom_ingestion: fix handle invalids for split ingestions

bt5/erp5_wendelin_telecom_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_telecom/Alarm_handleInvalids.py

@@ -37,9 +37,21 @@ now = DateTime()
 now_minus_max = now - 1.0/24/60*9999
 now_minus_10 = now - 1.0/24/60*10
 
-catalog_kw = {'creation_date': {'query': (now_minus_max, now_minus_10), 'range': 'minmax'}, 'simulation_state': 'planned'}
+catalog_kw = {'creation_date': {'query': (now_minus_max, now_minus_10), 'range': 'minmax'}, 'simulation_state': 'planned', 'portal_type': 'Data Ingestion'}
 
-for data_ingestion in portal_catalog(portal_type="Data Ingestion", **catalog_kw):
+for data_ingestion in portal_catalog(**catalog_kw):
+  related_split_ingestions = portal_catalog(portal_type = "Data Ingestion",
+                                            simulation_state = "planned",
+                                            reference = data_ingestion.getReference())
+  invalidate = True
+  for related_ingestion in related_split_ingestions:
+    # only invalidate if all related ingestion are old
+    if len(portal_catalog(id=related_ingestion.getId(), **catalog_kw)) == 0:
+      context.logEntry("Data Ingestion %s is old, but it has related ingestion that are not. So it won't be invalidated yet." % data_ingestion.getId())
+      invalidate = False
+      break
+
+  if invalidate:
     # invalidate related Data Stream
     kw_dict = {"portal_type": "Data Stream",
                "reference": data_ingestion.getReference()}

bt5/erp5_wendelin_telecom_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_telecom/ERP5Site_startIngestionList.py

@@ -42,7 +42,6 @@ for data_ingestion in portal_catalog(portal_type = "Data Ingestion",
           full_data_stream = data_stream
         else:
           full_data_stream.appendData(data_stream.getData())
-          #data_stream.update_data("",0)
           portal.data_stream_module.deleteContent(data_stream.getId())
         index += 1
       full_data_stream.validate()

bt5/erp5_wendelin_telecom_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_telecom/IngestionPolicy_getIngestionOperationAndParameterDict.py

@@ -125,24 +125,9 @@ try:
   data_ingestion.plan()
   context.logEntry("Data Ingestion planned.")
 
-  #else:
-  #  log("Planned Data Ingestion found for reference: " + str(data_ingestion))
-  #  context.logEntry("Planned Data Ingestion found for reference: " + str(data_ingestion))
-  #  # find ingestion line for current resource
-  #  for line in data_ingestion.objectValues(portal_type="Data Ingestion Line"):
-  #    if line.getResourceReference() == resource_reference:
-  #      input_line = line
-  #    elif line.getResourceValue().getPortalType() == "Data Operation":
-  #      operation_line = line
-
   data_operation = operation_line.getResourceValue()
   data_stream = input_line.getAggregateDataStreamValue()
 
-  #if eof == "EOF":
-  #  data_ingestion.start()
-  #  log("Data Ingestion started.")
-  #  context.logEntry("Data Ingestion started.")
-
   return data_operation, {'data_stream': data_stream}
 except Exception as e:
   context.logEntry("[ERROR] Error during ingestion policy operation: " + str(e))