Data lake changes

See merge request !58

bt5/erp5_wendelin_data_lake_ingestion/PathTemplateItem/data_supply_module/embulk.xml → bt5/erp5_wendelin_data_lake_ingestion/PathTemplateItem/data_supply_module/default_ebulk.xml

@@ -86,7 +86,7 @@
         </item>
         <item>
             <key> <string>default_reference</string> </key>
-            <value> <string>embulk</string> </value>
+            <value> <string>default_ebulk</string> </value>
         </item>
         <item>
             <key> <string>description</string> </key>
@@ -132,7 +132,7 @@
         </item>
         <item>
             <key> <string>id</string> </key>
-            <value> <string>embulk</string> </value>
+            <value> <string>default_ebulk</string> </value>
         </item>
         <item>
             <key> <string>language</string> </key>
@@ -146,7 +146,7 @@
         </item>
         <item>
             <key> <string>title</string> </key>
-            <value> <string>Embulk</string> </value>
+            <value> <string>Ebulk</string> </value>
         </item>
         <item>
             <key> <string>version</string> </key>
@@ -329,7 +329,7 @@
                   </item>
                   <item>
                       <key> <string>serial</string> </key>
-                      <value> <string>960.15698.49376.7287</string> </value>
+                      <value> <string>985.36429.29664.46233</string> </value>
                   </item>
                   <item>
                       <key> <string>state</string> </key>
@@ -347,7 +347,7 @@
                           </tuple>
                           <state>
                             <tuple>
-                              <float>1498225133.32</float>
+                              <float>1595958580.55</float>
                               <string>UTC</string>
                             </tuple>
                           </state>

bt5/erp5_wendelin_data_lake_ingestion/PathTemplateItem/data_supply_module/embulk/1.xml → bt5/erp5_wendelin_data_lake_ingestion/PathTemplateItem/data_supply_module/default_ebulk/1.xml

@@ -214,7 +214,7 @@
                   </item>
                   <item>
                       <key> <string>serial</string> </key>
-                      <value> <string>959.7816.8415.26299</string> </value>
+                      <value> <string>985.17805.27243.30754</string> </value>
                   </item>
                   <item>
                       <key> <string>state</string> </key>
@@ -232,7 +232,7 @@
                           </tuple>
                           <state>
                             <tuple>
-                              <float>1498225125.16</float>
+                              <float>1595958567.14</float>
                               <string>UTC</string>
                             </tuple>
                           </state>

bt5/erp5_wendelin_data_lake_ingestion/PathTemplateItem/data_supply_module/embulk/2.xml → bt5/erp5_wendelin_data_lake_ingestion/PathTemplateItem/data_supply_module/default_ebulk/2.xml

@@ -220,7 +220,7 @@
                   </item>
                   <item>
                       <key> <string>serial</string> </key>
-                      <value> <string>959.7816.8415.26299</string> </value>
+                      <value> <string>985.17805.27243.30754</string> </value>
                   </item>
                   <item>
                       <key> <string>state</string> </key>
@@ -238,7 +238,7 @@
                           </tuple>
                           <state>
                             <tuple>
-                              <float>1498225125.15</float>
+                              <float>1595958567.15</float>
                               <string>UTC</string>
                             </tuple>
                           </state>

bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_getUserName.py

 portal = context.getPortalObject()
-return portal.Base_getUserCaption()
+user_reference = portal.Base_getUserCaption()
+
+person = portal.portal_catalog.getResultValue(
+                portal_type = 'Person',
+                reference = user_reference)
+
+if person:
+  return '%s %s' % (person.getFirstName(), person.getLastName())
+else:
+  return user_reference

bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/IngestionPolicy_getIngestionOperationAndParameterDictEbulk.py

@@ -33,7 +33,7 @@ if data_ingestion is not None:
 
 specialise_value_list = [x.getObject() for x in portal_catalog.searchResults(
   portal_type = 'Data Supply',
-  reference = 'embulk',
+  reference = 'default_ebulk',
   validation_state = 'validated')]
 
 # create a new data ingestion

bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.py

@@ -216,11 +216,11 @@ class TestDataIngestion(SecurityTestCase):
     self.assertNotEqual(None,
            getattr(self.portal.portal_ingestion_policies, "default_ebulk", None))
     self.assertNotEqual(None,
-           getattr(self.portal.data_supply_module, "embulk", None))
+           getattr(self.portal.data_supply_module, "default_ebulk", None))
 
-  def test_04_DefaultModelSecurityModel(self):
+  def test_04_DatasetAndDatastreamsConsistency(self):
     """
-      Test default security model : 'All can download, only contributors can upload.'
+      Test that data set state transition also changes its data streams states
     """
     data_set, data_stream_list = self.stepIngest(self.CSV, ",", randomize_ingestion_reference=True)
     self.tic()

bt5/erp5_wendelin_data_lake_ingestion/bt/template_keep_last_workflow_history_only_path_list

@@ -3,8 +3,8 @@ data_operation_module/transform_fif
 data_product_module/fif_array
 data_product_module/fif_data
 data_product_module/fif_descriptor
-data_supply_module/embulk
-data_supply_module/embulk/**
+data_supply_module/default_ebulk
+data_supply_module/default_ebulk/**
 portal_ingestion_policies/wendelin_embulk
 portal_ingestion_policies/default_ebulk
 portal_categories/function/**

bt5/erp5_wendelin_data_lake_ingestion/bt/template_keep_workflow_path_list

@@ -3,8 +3,8 @@ data_operation_module/transform_fif
 data_product_module/fif_array
 data_product_module/fif_data
 data_product_module/fif_descriptor
-data_supply_module/embulk
-data_supply_module/embulk/**
+data_supply_module/default_ebulk
+data_supply_module/default_ebulk/**
 portal_ingestion_policies/wendelin_embulk
 portal_ingestion_policies/default_ebulk
 portal_categories/use/**
\ No newline at end of file

bt5/erp5_wendelin_data_lake_ingestion/bt/template_path_list

@@ -3,8 +3,8 @@ data_operation_module/transform_fif
 data_product_module/fif_array
 data_product_module/fif_data
 data_product_module/fif_descriptor
-data_supply_module/embulk
-data_supply_module/embulk/**
+data_supply_module/default_ebulk
+data_supply_module/default_ebulk/**
 portal_alarms/wendelin_check_datastream_consistency
 portal_alarms/wendelin_check_datastream_consistency/**
 portal_alarms/wendelin_data_lake_handle_analysis

bt5/erp5_wendelin_data_lake_ingestion_default_security_model/LocalRolesTemplateItem/data_ingestion_module.xml

@@ -10,8 +10,5 @@
    <item>Auditor</item>
    <item>Author</item>
   </role>
-  <role id='downloader'>
-   <item>Auditor</item>
-  </role>
  </local_roles>
 </local_roles_item>
\ No newline at end of file

bt5/erp5_wendelin_data_lake_ingestion_default_security_model/PortalTypeRolesTemplateItem/Data%20Ingestion%20Line.xml

@@ -7,8 +7,4 @@
    <property id='title'>contributor</property>
    <multi_property id='category'>function/contributor</multi_property>
   </role>
-  <role id='Auditor'>
-   <property id='title'>downloader</property>
-   <multi_property id='category'>function/downloader</multi_property>
-  </role>
 </type_roles>
\ No newline at end of file

bt5/erp5_wendelin_data_lake_ingestion_default_security_model/PortalTypeRolesTemplateItem/Data%20Ingestion%20Module.xml

@@ -7,8 +7,4 @@
    <property id='title'>contributor</property>
    <multi_property id='category'>function/contributor</multi_property>
   </role>
-  <role id='Auditor'>
-   <property id='title'>downloader</property>
-   <multi_property id='category'>function/downloader</multi_property>
-  </role>
 </type_roles>
\ No newline at end of file

bt5/erp5_wendelin_data_lake_ingestion_default_security_model/PortalTypeRolesTemplateItem/Data%20Ingestion.xml

@@ -7,8 +7,4 @@
    <property id='title'>contributor</property>
    <multi_property id='category'>function/contributor</multi_property>
   </role>
-  <role id='Auditor'>
-   <property id='title'>downloader</property>
-   <multi_property id='category'>function/downloader</multi_property>
-  </role>
 </type_roles>
\ No newline at end of file