erp5_wendelin_data_lake_ingestion: update unit tests

bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.py

@@ -70,6 +70,12 @@ class TestDataIngestion(SecurityTestCase):
                     reference = reference)
     return data_stream
 
+  def getDataStreamChunkList(self, reference):
+    data_stream_list = self.portal.portal_catalog(
+                        portal_type = 'Data Stream',
+                        reference = reference)
+    return data_stream_list
+
   def ingestRequest(self, reference, eof, data_chunk, ingestion_policy):
     encoded_data_chunk = base64.b64encode(data_chunk)
     request = self.portal.REQUEST
@@ -84,13 +90,10 @@ class TestDataIngestion(SecurityTestCase):
 
   def ingest(self, data_chunk, reference, extension, eof, randomize_ingestion_reference=False):
     ingestion_reference = self.getIngestionReference(reference, extension, randomize_ingestion_reference)
-    
     # use default ebulk policy
-    ingestion_policy = self.portal.portal_ingestion_policies.wendelin_embulk
-    
+    ingestion_policy = self.portal.portal_ingestion_policies.default_ebulk
     self.ingestRequest(ingestion_reference, eof, data_chunk, ingestion_policy)
     _, ingestion_reference = self.sanitizeReference(ingestion_reference)
-
     return ingestion_reference
 
   def stepIngest(self, extension, delimiter, randomize_ingestion_reference=False):
@@ -108,7 +111,6 @@ class TestDataIngestion(SecurityTestCase):
           chunk.append(line)
         else:
           break
-
     ingestion_reference = self.ingest(data_chunk, reference, extension, self.SINGLE_INGESTION_END, randomize_ingestion_reference=randomize_ingestion_reference)
 
     if os.path.exists(file_name):
@@ -127,8 +129,9 @@ class TestDataIngestion(SecurityTestCase):
     data_stream_data = data_stream.getData()
     self.assertEqual(data_chunk, data_stream_data)
 
-    # check Data Stream and Data Set are validated
-    self.assertEqual('validated', data_stream.getValidationState())
+    # check Data Set is validated and Data Stream is published
+    self.assertEqual('validated', data_set.getValidationState())
+    self.assertEqual('published', data_stream.getValidationState())
 
     return data_set, [data_stream]
 
@@ -140,7 +143,7 @@ class TestDataIngestion(SecurityTestCase):
 
   def test_02_DefaultSplitIngestion(self):
     """
-      Test multiple uploads from ebulk end up in same Data Stream concatenated 
+      Test multiple uploads from ebulk end up in multiple Data Streams
       (in case of large file upload when ebluk by default splits file to 50MBs
       chunks).
     """
@@ -152,7 +155,6 @@ class TestDataIngestion(SecurityTestCase):
                               for _ in xrange(250)])
     data_chunk_4 = ''.join([random.choice(string.ascii_letters + string.digits) \
                               for _ in xrange(250)])
-    data_chunk = data_chunk_1 + data_chunk_2 + data_chunk_3 + data_chunk_4
 
     reference = self.getRandomReference()
 
@@ -172,13 +174,20 @@ class TestDataIngestion(SecurityTestCase):
     time.sleep(1)
     self.tic()
 
-    # call explicitly alarm so all 4 Data Streams can be concatenated to one
-    self.portal.portal_alarms.wendelin_data_lake_handle_analysis.Alarm_dataLakeHandleAnalysis()
+    # call explicitly alarm so all 4 Data Streams are validated and published
+    self.portal.portal_alarms.wendelin_handle_analysis.Alarm_handleAnalysis()
     self.tic()
 
-    # check resulting Data Stream
-    data_stream = self.getDataStream(ingestion_reference)
-    self.assertEqual(data_chunk, data_stream.getData())
+    # check resulting Data Streams
+    data_stream_list = self.getDataStreamChunkList(ingestion_reference)
+    #one data stream per chunk
+    self.assertEqual(len(data_stream_list), 4)
+    #last datastream (EOF) published, the rest validated
+    for stream in data_stream_list:
+      if stream.getId().endswith(self.EOF.replace(self.REFERENCE_SEPARATOR, "")):
+        self.assertEqual('published', stream.getValidationState())
+      else:
+        self.assertEqual('validated', stream.getValidationState())
 
   def test_03_DefaultWendelinConfigurationExistency(self):
     """
@@ -186,7 +195,7 @@ class TestDataIngestion(SecurityTestCase):
     """
     # test default ebuk ingestion exists
     self.assertNotEqual(None,
-           getattr(self.portal.portal_ingestion_policies, "wendelin_embulk", None))
+           getattr(self.portal.portal_ingestion_policies, "default_ebulk", None))
     self.assertNotEqual(None,
            getattr(self.portal.data_supply_module, "embulk", None))
 
@@ -200,10 +209,8 @@ class TestDataIngestion(SecurityTestCase):
     # check data relation between Data Set and Data Streams work
     self.assertSameSet(data_stream_list, data_set.DataSet_getDataStreamList())
 
-    # publish data set and have all Data Streams publsihed automatically
-    data_set.publish()
-    self.tic()
-    self.assertEqual('published', data_set.getValidationState())
+    # check data set and all Data Streams states
+    self.assertEqual('validated', data_set.getValidationState())
     self.assertSameSet(['published' for x in data_stream_list],
                        [x.getValidationState() for x in data_stream_list])
 

bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.xml

@@ -46,8 +46,8 @@
             <key> <string>text_content_warning_message</string> </key>
             <value>
               <tuple>
-                <string>W: 99, 34: Unused variable \'i\' (unused-variable)</string>
-                <string>W: 99, 76: Unused variable \'j\' (unused-variable)</string>
+                <string>W:102, 34: Unused variable \'i\' (unused-variable)</string>
+                <string>W:102, 76: Unused variable \'j\' (unused-variable)</string>
               </tuple>
             </value>
         </item>