erp5_wendelin_telecom_ingestion: refactoring in data ingestion reference

bt5/erp5_wendelin_telecom_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_telecom/IngestionPolicy_getIngestionOperationAndParameterDict.py

@@ -6,34 +6,32 @@ today_string = now.strftime('%Y%m%d')
 portal = context.getPortalObject()
 portal_catalog = portal.portal_catalog
 
-# removing eof from reference
-reference = '_'.join(reference.split('.')[:-1])
-log("Reference: " + reference)
-context.logEntry("Data Ingestion reference: %s" % reference)
-
-data_ingestion_reference = reference
-data_ingestion_id =  '%s_%s' %(data_ingestion_reference, today_string)
-eof = movement_dict.get('eof', 'EOF')
-resource_reference = movement_dict.get('resource_reference', None)
-specialise_reference = movement_dict.get('specialise_reference', None)
-dataset_reference = movement_dict.get('aggregate_data_set_reference', None)
-
-# search for applicable data ingestion
-data_ingestion = portal_catalog.getResultValue(
+
+try:
+  # remove supplier and eof from reference
+  reference = '_'.join(reference.split('.')[1:-1]) #'_'.join(reference.split('.')[:-1])
+  log("Reference: " + reference)
+  context.logEntry("Data Ingestion reference: %s" % reference)
+
+  data_ingestion_reference = reference
+  eof = movement_dict.get('eof', 'EOF')
+  resource_reference = movement_dict.get('resource_reference', None)
+  specialise_reference = movement_dict.get('specialise_reference', None)
+  dataset_reference = movement_dict.get('aggregate_data_set_reference', None)
+  data_ingestion_id =  '%s_%s_%s' %(specialise_reference, data_ingestion_reference, today_string)
+  context.logEntry("Data Ingestion ID: %s" % reference)
+
+  # search for applicable data ingestion
+  data_ingestion = portal_catalog.getResultValue(
     portal_type = 'Data Ingestion',
-  simulation_state = 'planned',
     reference = data_ingestion_reference)
 
-if data_ingestion is None:
-  document = portal.data_ingestion_module.get(data_ingestion_id)
-  if (document is not None):
-    if document.getSimulationState() == 'planned':
-      data_ingestion = document
-    else:
+  if data_ingestion is not None:
+    if data_ingestion.getSimulationState() != 'planned':
       context.logEntry("An older ingestion for reference %s was already done." % data_ingestion_reference)
       raise ValueError("An older ingestion for reference %s was already done." % data_ingestion_reference)
 
-if data_ingestion is None:
+  if data_ingestion is None:
     specialise_value_list = [x.getObject() for x in portal_catalog.searchResults(
       portal_type = 'Data Supply',
       reference = 'embulk',
@@ -120,7 +118,7 @@ if data_ingestion is None:
     log("Data Ingestion planned.")
     context.logEntry("Data Ingestion planned.")
 
-else:
+  else:
     log("Planned Data Ingestion found for reference: " + str(data_ingestion))
     context.logEntry("Planned Data Ingestion found for reference: " + str(data_ingestion))
     # find ingestion line for current resource
@@ -130,12 +128,15 @@ else:
       elif line.getResourceValue().getPortalType() == "Data Operation":
         operation_line = line
 
-data_operation = operation_line.getResourceValue()
-data_stream = input_line.getAggregateDataStreamValue()
+  data_operation = operation_line.getResourceValue()
+  data_stream = input_line.getAggregateDataStreamValue()
 
-if eof == "EOF":
+  if eof == "EOF":
     data_ingestion.start()
     log("Data Ingestion started.")
     context.logEntry("Data Ingestion started.")
 
-return data_operation, {'data_stream': data_stream}
+  return data_operation, {'data_stream': data_stream}
+except Exception as e:
+  context.logEntry("[ERROR] At script IngestionPolicy_getIngestionOperationAndParameterDict \n[ERROR] " + str(e))
+  raise e

bt5/erp5_wendelin_telecom_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_telecom/ingestionReferenceExists.py

 from DateTime import DateTime
+from Products.ERP5Type.Log import log
 
-# reference parameter example: telecom.MNE.filename.fif.EOF
-# ingestion reference example: telecom_MNE_filename_fif
-# ingestion id example:        telecom_MNE_filename_fif_20170615
+# reference parameter example: supplier.dataset.filename.fif.EOF
+# ingestion reference example: dataset_filename_fif
 
 FALSE = "FALSE"
 TRUE = "TRUE"
 
-today_string = DateTime().strftime('%Y%m%d')
 portal = context.getPortalObject()
 portal_catalog = portal.portal_catalog
 
-data_ingestion_reference = '_'.join(reference.split('_')[:-1]) #'_'.join(reference.split('.')[:-1])
-data_ingestion_id =  '%s_%s' %(data_ingestion_reference, today_string)
+try:
+  context.logEntry("Attempt to do a new ingestion with reference: %s" % reference)
 
-# first search for applicable data ingestion
-data_ingestion = portal_catalog.getResultValue(
+  # remove supplier and eof from reference
+  data_ingestion_reference = '_'.join(reference.split('_')[1:-1])
+
+  data_ingestion = portal_catalog.getResultValue(
     portal_type = 'Data Ingestion',
-  simulation_state = 'planned',
     reference = data_ingestion_reference)
 
-if data_ingestion is None:
-  document = portal.data_ingestion_module.get(data_ingestion_id)
-  if (document is not None):
-    if document.getSimulationState() == 'planned':
-      return FALSE
-    else:
-      return TRUE
-  else:
+  if data_ingestion is None:
     return FALSE
-else:
+
+  if data_ingestion.getSimulationState() == 'planned':
     return FALSE
+
+  context.logEntry("[ERROR] Data Ingestion reference %s already exists" % data_ingestion_reference)
+  return TRUE
+except Exception as e:
+  context.logEntry("[ERROR] At script ingestionReferenceExists  \n[ERROR] " + str(e))
+  raise e