Updating unit tests.

bt5/erp5_wendelin_telecom_ingestion/ExtensionTemplateItem/portal_components/unit_test_external.xml

@@ -6,6 +6,12 @@
     </pickle>
     <pickle>
       <dictionary>
+        <item>
+            <key> <string>_recorded_property_dict</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
         <item>
             <key> <string>default_reference</string> </key>
             <value> <string>unit_test_external</string> </value>
@@ -53,13 +59,28 @@
         <item>
             <key> <string>workflow_history</string> </key>
             <value>
-              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+              <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
             </value>
         </item>
       </dictionary>
     </pickle>
   </record>
   <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary/>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
     <pickle>
       <global name="PersistentMapping" module="Persistence.mapping"/>
     </pickle>
@@ -72,7 +93,7 @@
                 <item>
                     <key> <string>component_validation_workflow</string> </key>
                     <value>
-                      <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+                      <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
                     </value>
                 </item>
               </dictionary>
@@ -81,7 +102,7 @@
       </dictionary>
     </pickle>
   </record>
-  <record id="3" aka="AAAAAAAAAAM=">
+  <record id="4" aka="AAAAAAAAAAQ=">
     <pickle>
       <global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
     </pickle>

bt5/erp5_wendelin_telecom_ingestion/TestTemplateItem/portal_components/test.erp5.wendelin.testDataIngestion.py

@@ -5,11 +5,17 @@ import time
 from DateTime import DateTime
 import numpy as np
 import math
+import base64
 
 class TestDataIngestion(SecurityTestCase):
 
-  PART = ".PART"
+  PART = ".xxx"
   EOF = ".EOF"
+  FIF = ".fif"
+  REF_PREFIX = "fake-supplier.fake-dataset."
+  INGESTION_SCRIPT = 'HandleFifEmbulkIngestion'
+  USER = 'zope'
+  PASS = 'roque5'
 
   def getTitle(self):
     return "DataIngestionTest"
@@ -23,7 +29,13 @@ class TestDataIngestion(SecurityTestCase):
 
   def getRandomReference(self):
     random_string = ''.join([random.choice(string.ascii_letters + string.digits) for _ in xrange(10)])
-    return 'UNIT_TEST-' + random_string
+    return 'UNIT-TEST-' + random_string
+
+  def getIngestionReference(self, reference):
+    return self.REF_PREFIX + reference + self.FIF
+
+  def sanitizeReference(self, reference):
+    return reference.replace(".", "_")
 
   def chunks(self, l, n):
     for i in xrange(0, len(l), n):
@@ -46,11 +58,12 @@ class TestDataIngestion(SecurityTestCase):
     return ingestion_policy
 
   def ingestRequest(self, requestMethod, authentication , reference, eof, data_chunk, ingestion):
+    encoded_data_chunk = base64.b64encode(data_chunk)
     request = self.portal.REQUEST
     request.method = requestMethod
     request.auth = authentication
     request.set('reference', reference + eof)
-    request.set('data_chunk', data_chunk)
+    request.set('data_chunk', encoded_data_chunk)
     ingestion.ingest()
     self.tic()
     return
@@ -64,12 +77,12 @@ class TestDataIngestion(SecurityTestCase):
   def getDataArray(self, reference):
     data_array = self.portal.portal_catalog.getResultValue(
                     portal_type = 'Data Array',
-                    reference = "Transform Raw FIF Data-" + reference)
+                    reference = reference)
     return data_array
 
   def manuallyStopIngestionWorkaround(self, reference):
     now = DateTime().strftime('%Y%m%d')
-    data_ingestion_id =  "%s-%s" %(now, reference)
+    data_ingestion_id =  "%s_%s" %(reference, now)
     url = 'data_ingestion_module/' + data_ingestion_id
     data_ingestion = self.context.restrictedTraverse(url)
     if data_ingestion.getSimulationState() == "started":
@@ -91,30 +104,34 @@ class TestDataIngestion(SecurityTestCase):
 
   def test_full_data_ingestion(self):
     reference = self.getRandomReference()
-    ingestion_policy = self.getIngestionPolicy(reference, 'HandleFifEmbulkIngestion')
+    ingestion_policy = self.getIngestionPolicy(reference, self.INGESTION_SCRIPT)
     data_chunk, nparray = self.generateRawDataBytesAndArray()
-    self.ingestRequest('POST', ('zope', 'roque5'), reference, self.EOF, data_chunk, ingestion_policy)
+    ingestion_reference = self.getIngestionReference(reference)
 
-    data_stream = self.getDataStream(reference)
+    self.ingestRequest('POST', (self.USER, self.PASS), ingestion_reference, self.EOF, data_chunk, ingestion_policy)
+    ingestion_reference = self.sanitizeReference(ingestion_reference)
+    data_stream = self.getDataStream(ingestion_reference)
     self.assertEqual(len(data_chunk), len(data_stream.getData()))
     self.assertEqual(data_chunk, data_stream.getData())
 
-    self.simulateIngestionAlarm(reference)
+    self.simulateIngestionAlarm(ingestion_reference)
 
-    data_array = self.getDataArray(reference)
+    data_array = self.getDataArray(ingestion_reference)
     self.assertTrue(np.allclose(nparray, data_array.getArray()))
 
 
   def test_data_ingestion_splitted_file(self):
     reference = self.getRandomReference()
-    ingestion_policy = self.getIngestionPolicy(reference, 'HandleFifEmbulkIngestion')
+    ingestion_policy = self.getIngestionPolicy(reference, self.INGESTION_SCRIPT)
     data_chunk, nparray = self.generateRawDataBytesAndArray()
     data_chunk_1 = data_chunk[:int(math.floor(len(data_chunk)/2))]
     data_chunk_2 = data_chunk[int(math.floor(len(data_chunk)/2)):]
+    ingestion_reference = self.getIngestionReference(reference)
 
-    self.ingestRequest('POST', ('zope', 'roque5'), reference, self.PART, data_chunk_1, ingestion_policy)
-    data_stream = self.getDataStream(reference)
-    self.ingestRequest('POST', ('zope', 'roque5'), reference, self.EOF, data_chunk_2, ingestion_policy)
+    self.ingestRequest('POST', (self.USER, self.PASS), ingestion_reference, self.PART, data_chunk_1, ingestion_policy)
+    self.ingestRequest('POST', (self.USER, self.PASS), ingestion_reference, self.EOF, data_chunk_2, ingestion_policy)
+    ingestion_reference = self.sanitizeReference(ingestion_reference)
+    data_stream = self.getDataStream(ingestion_reference)
     self.assertEqual(len(data_chunk), len(data_stream.getData()))
     self.assertEqual(data_chunk, data_stream.getData())
 

bt5/erp5_wendelin_telecom_ingestion/TestTemplateItem/portal_components/test.erp5.wendelin.testDataIngestion.xml

@@ -46,7 +46,7 @@
             <key> <string>text_content_warning_message</string> </key>
             <value>
               <tuple>
-                <string>W:111, 16: Unused variable \'nparray\' (unused-variable)</string>
+                <string>W:126, 16: Unused variable \'nparray\' (unused-variable)</string>
               </tuple>
             </value>
         </item>