erp5_wendelin_telecom_ingestion: fixing multiple data set but and minor refactoring

bt5/erp5_wendelin_telecom_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_telecom/IngestionPolicy_getIngestionOperationAndParameterDict.py

@@ -96,9 +96,8 @@ try:
     input_line.setDefaultAggregateValue(data_stream)
 
     if dataset_reference is not None:
-      data_set = portal_catalog.getResultValue(
-        portal_type = "Data Set",
-        reference = dataset_reference)
+      data_set = portal.data_set_module.get(dataset_reference)
+      try:
         if data_set is None:
           log("Creating new Data Set")
           context.logEntry("Data Set created.")
@@ -106,10 +105,14 @@ try:
             portal_type = "Data Set",
             title = "Data set " + dataset_reference,
             reference = dataset_reference,
+            id = dataset_reference,
             description = "Default description.",
             version = "001"
           )
           data_set.validate()
+      except:
+        data_set = portal.data_set_module.get(dataset_reference)
+        log("Data Set found for dataset reference: " + data_set.getReference())
       else:
         log("Data Set found for dataset reference: " + data_set.getReference())
       input_line.setDefaultAggregateValue(data_set)
@@ -138,5 +141,6 @@ try:
 
   return data_operation, {'data_stream': data_stream}
 except Exception as e:
-  context.logEntry("[ERROR] At script IngestionPolicy_getIngestionOperationAndParameterDict \n[ERROR] " + str(e))
+  context.logEntry("[ERROR] At script IngestionPolicy_getIngestionOperationAndParameterDict")
+  context.logEntry("[ERROR] " + str(e))
   raise e

bt5/erp5_wendelin_telecom_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_telecom/getDataStreamList.py

 portal = context.getPortalObject()
 portal_catalog = portal.portal_catalog
 
-################################ workaround to force security access ################################
-check = context.restrictedTraverse("data_set_module/")
+try:
+  data_set = portal.data_set_module.get(data_set_reference)
+  if data_set is None:
+    return { "status_code": 0, "result": [] }
+except Exception as e: # fails because unauthorized access
+  context.logEntry("[ERROR] At script getDataStreamList")
+  context.logEntry("[ERROR] " + str(e))
+  return { "status_code": 1, "error_message": "401 - Unauthorized access" }
 
-################################ workaround for "multiple-datasets" bug ################################
-query_dict = {
-  "portal_type": "Data Set",
-  "reference" : data_set_reference}
-data_set_list = portal_catalog(**query_dict)
-if len(data_set_list) == 0:
+data_set = portal.data_set_module.get(data_set_reference)
+if data_set is None:
   return []
 
-def belongsToDatasets(line):
-  for data_set in data_set_list:
-    if line.getAggregateDeviceConfiguration() == "data_set_module/" + str(data_set.getId()):
-      return True
-  return False
-################################ workaround for "multiple-datasets" bug ################################
-
 query_dict = {
   "portal_type": "Data Ingestion Line",
   "resource_portal_type": "Data Product",
@@ -27,11 +22,11 @@ query_dict = {
 data_stream_list = []
 
 for line in portal_catalog(**query_dict):
-  if belongsToDatasets(line):
+  if line.getAggregateDeviceConfiguration() == "data_set_module/" + str(data_set.getId()):
     for item in line.getAggregateSet():
       if "data_stream_module/" in item:
         data_stream = context.restrictedTraverse(item)
         data_stream_list.append([item, data_stream.getReference()])
         break
 
-return data_stream_list
+return { "status_code": 0, "result": data_stream_list }

bt5/erp5_wendelin_telecom_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_telecom/ingestionReferenceExists.py

@@ -29,5 +29,6 @@ try:
   context.logEntry("[ERROR] Data Ingestion reference %s already exists" % data_ingestion_reference)
   return TRUE
 except Exception as e:
-  context.logEntry("[ERROR] At script ingestionReferenceExists  \n[ERROR] " + str(e))
+  context.logEntry("[ERROR] At script ingestionReferenceExists")
+  context.logEntry("[ERROR] " + str(e))
   raise e