Skip to content

W
wendelin
Commit c632c012 authored by Ivan Tyagov's avatar Ivan Tyagov
Browse files
Options

Keep naming conventions.

parent 3dc90e9d
Hide whitespace changes
Inline Side-by-side
Showing with 242 additions and 102 deletions
bt5/erp5_wendelin_data_lake_ingestion/PathTemplateItem/portal_callables/IngestionPolicy_handleEmbulkIngestion.py 0 → 100644
View file @ c632c012
portal = context.getPortalObject()
reference_separator = portal.getIngestionReferenceDictionary()["reference_separator"]
reference_length = portal.getIngestionReferenceDictionary()["reference_length"]
invalid_chars = portal.getIngestionReferenceDictionary()["invalid_chars"]
record = reference.rsplit(reference_separator)
length = len(record)
if (length < reference_length):
context.logEntry("[ERROR] In HandleFifEmbulkIngestion: Data Ingestion reference is not well formated")
raise ValueError("Data Ingestion reference is not well formated.")
for char in invalid_chars:
if char in reference:
context.logEntry("[ERROR] In HandleFifEmbulkIngestion: Data Ingestion reference contains chars that are not allowed")
raise ValueError("Data Ingestion reference contains chars that are not allowed.")
supplier = record[0]
dataset_reference = record[1]
filename = reference_separator.join(record[2:-4])
extension = record[length-4]
eof = record[length-3]
size = record[length-2]
hash = record[length-1]
dict = { 'filename': filename,
'extension': extension,
'eof': eof,
'supplier': supplier,
'dataset_reference': dataset_reference,
'resource_reference': 'fif',
'size': size,
'hash': hash
}
return dict
bt5/erp5_wendelin_data_lake_ingestion/PathTemplateItem/portal_callables/IngestionPolicy_handleEmbulkIngestion.xml 0 → 100644
View file @ c632c012
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PyData Script" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_local_properties</string> </key>
<value>
<tuple>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>reference</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>string</string> </value>
</item>
</dictionary>
</tuple>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>reference</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>Handles the ingestion and returns the dictionary with the corresponding references.</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>IngestionPolicy_handleEmbulkIngestion</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>PyData Script</string> </value>
</item>
<item>
<key> <string>reference</string> </key>
<value> <string>HandleFifEmbulkIngestion</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Handle Fif Embulk Ingestion</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_data_lake_ingestion/PathTemplateItem/portal_ingestion_policies/wendelin_embulk.xml
View file @ c632c012
...@@ -74,7 +74,7 @@ ...@@ -74,7 +74,7 @@
</item> </item>
<item> <item>
<key> <string>script_id</string> </key> <key> <string>script_id</string> </key>
<value> <string>HandleFifEmbulkIngestion</string> </value> <value> <string>IngestionPolicy_handleEmbulkIngestion</string> </value>
</item> </item>
<item> <item>
<key> <string>title</string> </key> <key> <string>title</string> </key>
...@@ -123,114 +123,122 @@ ...@@ -123,114 +123,122 @@
</record> </record>
<record id="3" aka="AAAAAAAAAAM="> <record id="3" aka="AAAAAAAAAAM=">
<pickle> <pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/> <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle> </pickle>
<pickle> <pickle>
<tuple> <dictionary>
<none/> <item>
<list> <key> <string>_log</string> </key>
<dictionary> <value>
<item> <list>
<key> <string>action</string> </key> <dictionary>
<value> <string>edit</string> </value> <item>
</item> <key> <string>action</string> </key>
<item> <value> <string>edit</string> </value>
<key> <string>actor</string> </key> </item>
<value> <string>zope</string> </value> <item>
</item> <key> <string>actor</string> </key>
<item> <value> <string>zope</string> </value>
<key> <string>comment</string> </key> </item>
<value> <item>
<none/> <key> <string>comment</string> </key>
</value> <value>
</item> <none/>
<item> </value>
<key> <string>error_message</string> </key> </item>
<value> <string></string> </value> <item>
</item> <key> <string>error_message</string> </key>
<item> <value> <string></string> </value>
<key> <string>serial</string> </key> </item>
<value> <string>976.34131.824.20292</string> </value> <item>
</item> <key> <string>serial</string> </key>
<item> <value> <string>982.48448.18949.19490</string> </value>
<key> <string>state</string> </key> </item>
<value> <string>current</string> </value> <item>
</item> <key> <string>state</string> </key>
<item> <value> <string>current</string> </value>
<key> <string>time</string> </key> </item>
<value> <item>
<object> <key> <string>time</string> </key>
<klass> <value>
<global name="DateTime" module="DateTime.DateTime"/> <object>
</klass> <klass>
<tuple> <global name="DateTime" module="DateTime.DateTime"/>
<none/> </klass>
</tuple> <tuple>
<state> <none/>
<tuple> </tuple>
<float>1561536855.39</float> <state>
<string>UTC</string> <tuple>
</tuple> <float>1586521981.09</float>
</state> <string>UTC</string>
</object> </tuple>
</value> </state>
</item> </object>
</dictionary> </value>
</list> </item>
</tuple> </dictionary>
</list>
</value>
</item>
</dictionary>
</pickle> </pickle>
</record> </record>
<record id="4" aka="AAAAAAAAAAQ="> <record id="4" aka="AAAAAAAAAAQ=">
<pickle> <pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/> <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle> </pickle>
<pickle> <pickle>
<tuple> <dictionary>
<none/> <item>
<list> <key> <string>_log</string> </key>
<dictionary> <value>
<item> <list>
<key> <string>action</string> </key> <dictionary>
<value> <string>validate</string> </value> <item>
</item> <key> <string>action</string> </key>
<item> <value> <string>validate</string> </value>
<key> <string>actor</string> </key> </item>
<value> <string>zope</string> </value> <item>
</item> <key> <string>actor</string> </key>
<item> <value> <string>zope</string> </value>
<key> <string>comment</string> </key> </item>
<value> <string></string> </value> <item>
</item> <key> <string>comment</string> </key>
<item> <value> <string></string> </value>
<key> <string>error_message</string> </key> </item>
<value> <string></string> </value> <item>
</item> <key> <string>error_message</string> </key>
<item> <value> <string></string> </value>
<key> <string>time</string> </key> </item>
<value> <item>
<object> <key> <string>time</string> </key>
<klass> <value>
<global name="DateTime" module="DateTime.DateTime"/> <object>
</klass> <klass>
<tuple> <global name="DateTime" module="DateTime.DateTime"/>
<none/> </klass>
</tuple> <tuple>
<state> <none/>
<tuple> </tuple>
<float>1489669293.19</float> <state>
<string>UTC</string> <tuple>
</tuple> <float>1489669293.19</float>
</state> <string>UTC</string>
</object> </tuple>
</value> </state>
</item> </object>
<item> </value>
<key> <string>validation_state</string> </key> </item>
<value> <string>validated</string> </value> <item>
</item> <key> <string>validation_state</string> </key>
</dictionary> <value> <string>validated</string> </value>
</list> </item>
</tuple> </dictionary>
</list>
</value>
</item>
</dictionary>
</pickle> </pickle>
</record> </record>
</ZopeData> </ZopeData>
bt5/erp5_wendelin_data_lake_ingestion/bt/template_path_list
View file @ c632c012
...@@ -9,7 +9,7 @@ portal_alarms/wendelin_data_lake_handle_analysis ...@@ -9,7 +9,7 @@ portal_alarms/wendelin_data_lake_handle_analysis
portal_alarms/wendelin_data_lake_handle_analysis/** portal_alarms/wendelin_data_lake_handle_analysis/**
portal_callables/DataIngestionLine_writeEmbulkIngestionToDataStream portal_callables/DataIngestionLine_writeEmbulkIngestionToDataStream
portal_callables/DataTransformation_transformFIFDataToDataArray portal_callables/DataTransformation_transformFIFDataToDataArray
portal_callables/HandleFifEmbulkIngestion portal_callables/IngestionPolicy_handleEmbulkIngestion
portal_categories/function/** portal_categories/function/**
portal_categories/use/** portal_categories/use/**
portal_ingestion_policies/wendelin_embulk portal_ingestion_policies/wendelin_embulk
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7