From c771c6eaeaa58b3ce4a79cd9c5f51583f9a618a2 Mon Sep 17 00:00:00 2001 From: Roque Porchetto <roque.porchetto@nexedi.com> Date: Tue, 19 May 2020 14:33:37 +0000 Subject: [PATCH] erp5_wendelin_data_lake_ingestion: new default ebulk ingestion policy --- .../default_ebulk.xml | 246 ++++++++++++++++++ .../wendelin_embulk.xml | 10 +- ..._keep_last_workflow_history_only_path_list | 1 + .../bt/template_keep_workflow_path_list | 1 + .../bt/template_path_list | 1 + 5 files changed, 254 insertions(+), 5 deletions(-) create mode 100644 bt5/erp5_wendelin_data_lake_ingestion/PathTemplateItem/portal_ingestion_policies/default_ebulk.xml diff --git a/bt5/erp5_wendelin_data_lake_ingestion/PathTemplateItem/portal_ingestion_policies/default_ebulk.xml b/bt5/erp5_wendelin_data_lake_ingestion/PathTemplateItem/portal_ingestion_policies/default_ebulk.xml new file mode 100644 index 0000000..9b1051d --- /dev/null +++ b/bt5/erp5_wendelin_data_lake_ingestion/PathTemplateItem/portal_ingestion_policies/default_ebulk.xml @@ -0,0 +1,246 @@ +<?xml version="1.0"?> +<ZopeData> + <record id="1" aka="AAAAAAAAAAE="> + <pickle> + <global name="Ingestion Policy" module="erp5.portal_type"/> + </pickle> + <pickle> + <dictionary> + <item> + <key> <string>_Access_contents_information_Permission</string> </key> + <value> + <tuple> + <string>Anonymous</string> + <string>Assignee</string> + <string>Assignor</string> + <string>Associate</string> + <string>Auditor</string> + <string>Manager</string> + </tuple> + </value> + </item> + <item> + <key> <string>_Add_portal_content_Permission</string> </key> + <value> + <tuple> + <string>Assignee</string> + <string>Assignor</string> + <string>Associate</string> + <string>Manager</string> + </tuple> + </value> + </item> + <item> + <key> <string>_Modify_portal_content_Permission</string> </key> + <value> + <tuple> + <string>Assignee</string> + <string>Assignor</string> + <string>Associate</string> + <string>Manager</string> + </tuple> + </value> + </item> + <item> + <key> <string>_View_Permission</string> </key> + <value> + <tuple> + <string>Anonymous</string> + <string>Assignee</string> + <string>Assignor</string> + <string>Associate</string> + <string>Auditor</string> + <string>Manager</string> + </tuple> + </value> + </item> + <item> + <key> <string>data_operation_script_id</string> </key> + <value> <string>IngestionPolicy_getIngestionOperationAndParameterDictEbulk</string> </value> + </item> + <item> + <key> <string>default_reference</string> </key> + <value> <string>default_ebulk</string> </value> + </item> + <item> + <key> <string>description</string> </key> + <value> <string>Handles ingestion of raw files bytes sent to us from ebulk.</string> </value> + </item> + <item> + <key> <string>id</string> </key> + <value> <string>default_ebulk</string> </value> + </item> + <item> + <key> <string>portal_type</string> </key> + <value> <string>Ingestion Policy</string> </value> + </item> + <item> + <key> <string>script_id</string> </key> + <value> <string>IngestionPolicy_parseEbulkIngestionTag</string> </value> + </item> + <item> + <key> <string>title</string> </key> + <value> <string>Default Ebulk Ingestion Policy</string> </value> + </item> + <item> + <key> <string>version</string> </key> + <value> <string>001</string> </value> + </item> + <item> + <key> <string>workflow_history</string> </key> + <value> + <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent> + </value> + </item> + </dictionary> + </pickle> + </record> + <record id="2" aka="AAAAAAAAAAI="> + <pickle> + <global name="PersistentMapping" module="Persistence.mapping"/> + </pickle> + <pickle> + <dictionary> + <item> + <key> <string>data</string> </key> + <value> + <dictionary> + <item> + <key> <string>edit_workflow</string> </key> + <value> + <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent> + </value> + </item> + <item> + <key> <string>validation_workflow</string> </key> + <value> + <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent> + </value> + </item> + </dictionary> + </value> + </item> + </dictionary> + </pickle> + </record> + <record id="3" aka="AAAAAAAAAAM="> + <pickle> + <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/> + </pickle> + <pickle> + <dictionary> + <item> + <key> <string>_log</string> </key> + <value> + <list> + <dictionary> + <item> + <key> <string>action</string> </key> + <value> <string>edit</string> </value> + </item> + <item> + <key> <string>actor</string> </key> + <value> <string>zope</string> </value> + </item> + <item> + <key> <string>comment</string> </key> + <value> + <none/> + </value> + </item> + <item> + <key> <string>error_message</string> </key> + <value> <string></string> </value> + </item> + <item> + <key> <string>serial</string> </key> + <value> <string>984.990.49194.19609</string> </value> + </item> + <item> + <key> <string>state</string> </key> + <value> <string>current</string> </value> + </item> + <item> + <key> <string>time</string> </key> + <value> + <object> + <klass> + <global name="DateTime" module="DateTime.DateTime"/> + </klass> + <tuple> + <none/> + </tuple> + <state> + <tuple> + <float>1589986625.99</float> + <string>UTC</string> + </tuple> + </state> + </object> + </value> + </item> + </dictionary> + </list> + </value> + </item> + </dictionary> + </pickle> + </record> + <record id="4" aka="AAAAAAAAAAQ="> + <pickle> + <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/> + </pickle> + <pickle> + <dictionary> + <item> + <key> <string>_log</string> </key> + <value> + <list> + <dictionary> + <item> + <key> <string>action</string> </key> + <value> <string>validate</string> </value> + </item> + <item> + <key> <string>actor</string> </key> + <value> <string>zope</string> </value> + </item> + <item> + <key> <string>comment</string> </key> + <value> <string></string> </value> + </item> + <item> + <key> <string>error_message</string> </key> + <value> <string></string> </value> + </item> + <item> + <key> <string>time</string> </key> + <value> + <object> + <klass> + <global name="DateTime" module="DateTime.DateTime"/> + </klass> + <tuple> + <none/> + </tuple> + <state> + <tuple> + <float>1589898672.1</float> + <string>UTC</string> + </tuple> + </state> + </object> + </value> + </item> + <item> + <key> <string>validation_state</string> </key> + <value> <string>validated</string> </value> + </item> + </dictionary> + </list> + </value> + </item> + </dictionary> + </pickle> + </record> +</ZopeData> diff --git a/bt5/erp5_wendelin_data_lake_ingestion/PathTemplateItem/portal_ingestion_policies/wendelin_embulk.xml b/bt5/erp5_wendelin_data_lake_ingestion/PathTemplateItem/portal_ingestion_policies/wendelin_embulk.xml index 02189e4..f2b9ead 100644 --- a/bt5/erp5_wendelin_data_lake_ingestion/PathTemplateItem/portal_ingestion_policies/wendelin_embulk.xml +++ b/bt5/erp5_wendelin_data_lake_ingestion/PathTemplateItem/portal_ingestion_policies/wendelin_embulk.xml @@ -58,11 +58,11 @@ </item> <item> <key> <string>default_reference</string> </key> - <value> <string>wendelin_embulk</string> </value> + <value> <string>wendelin_ebulk</string> </value> </item> <item> <key> <string>description</string> </key> - <value> <string>Handles ingestion of raw files bytes sent to us from embulk.</string> </value> + <value> <string>[OBSOLETE - Kept for old ebulk clients] Handles ingestion of raw files bytes sent to us from ebulk.</string> </value> </item> <item> <key> <string>id</string> </key> @@ -78,7 +78,7 @@ </item> <item> <key> <string>title</string> </key> - <value> <string>Wendelin Embulk Ingestion Policy</string> </value> + <value> <string>[OBSOLETE] Wendelin Ebulk Ingestion Policy</string> </value> </item> <item> <key> <string>version</string> </key> @@ -152,7 +152,7 @@ </item> <item> <key> <string>serial</string> </key> - <value> <string>983.12768.14725.3720</string> </value> + <value> <string>983.63603.62266.8260</string> </value> </item> <item> <key> <string>state</string> </key> @@ -170,7 +170,7 @@ </tuple> <state> <tuple> - <float>1586946559.16</float> + <float>1589986571.48</float> <string>UTC</string> </tuple> </state> diff --git a/bt5/erp5_wendelin_data_lake_ingestion/bt/template_keep_last_workflow_history_only_path_list b/bt5/erp5_wendelin_data_lake_ingestion/bt/template_keep_last_workflow_history_only_path_list index 21c8279..fd6844f 100644 --- a/bt5/erp5_wendelin_data_lake_ingestion/bt/template_keep_last_workflow_history_only_path_list +++ b/bt5/erp5_wendelin_data_lake_ingestion/bt/template_keep_last_workflow_history_only_path_list @@ -6,6 +6,7 @@ data_product_module/fif_descriptor data_supply_module/embulk data_supply_module/embulk/** portal_ingestion_policies/wendelin_embulk +portal_ingestion_policies/default_ebulk portal_categories/function/** portal_categories/use/** portal_alarms/wendelin_data_lake_handle_analysis diff --git a/bt5/erp5_wendelin_data_lake_ingestion/bt/template_keep_workflow_path_list b/bt5/erp5_wendelin_data_lake_ingestion/bt/template_keep_workflow_path_list index 6a1c715..cfc17a0 100644 --- a/bt5/erp5_wendelin_data_lake_ingestion/bt/template_keep_workflow_path_list +++ b/bt5/erp5_wendelin_data_lake_ingestion/bt/template_keep_workflow_path_list @@ -6,4 +6,5 @@ data_product_module/fif_descriptor data_supply_module/embulk data_supply_module/embulk/** portal_ingestion_policies/wendelin_embulk +portal_ingestion_policies/default_ebulk portal_categories/use/** \ No newline at end of file diff --git a/bt5/erp5_wendelin_data_lake_ingestion/bt/template_path_list b/bt5/erp5_wendelin_data_lake_ingestion/bt/template_path_list index f143295..6b39ae4 100644 --- a/bt5/erp5_wendelin_data_lake_ingestion/bt/template_path_list +++ b/bt5/erp5_wendelin_data_lake_ingestion/bt/template_path_list @@ -10,4 +10,5 @@ portal_alarms/wendelin_data_lake_handle_analysis/** portal_callables/DataIngestionLine_writeEbulkIngestionToDataStream portal_callables/IngestionPolicy_parseEbulkIngestionTag portal_categories/use/** +portal_ingestion_policies/default_ebulk portal_ingestion_policies/wendelin_embulk \ No newline at end of file -- 2.30.9