Skip to content

W
wendelin
Commit 02846171 authored by Ivan Tyagov's avatar Ivan Tyagov
Browse files
Options

Default ingestion policy 

See merge request nexedi/wendelin!60
parents 14897249 a9ba8af3
Expand all Show whitespace changes
Inline Side-by-side
Showing with 177 additions and 699 deletions
bt5/erp5_wendelin/PathTemplateItem/data_product_module/default.xml 0 → 100644
View file @ 02846171
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Data Product" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Access_contents_information_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_View_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>aggregated_portal_type</string> </key>
<value>
<tuple>
<string>Data Stream</string>
<string>Progress Indicator</string>
</tuple>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>use/big_data/ingestion/stream</string>
</tuple>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>test_product</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>default</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Data Product</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Append to Data Stream</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin/PathTemplateItem/data_supply_module/default_http_json.xml → bt5/erp5_wendelin/PathTemplateItem/data_supply_module/default.xml
View file @ 02846171
......@@ -74,7 +74,7 @@
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>default_http_json</string> </value>
<value> <string>test_sensor</string> </value>
</item>
<item>
<key> <string>effective_date</string> </key>
......@@ -88,7 +88,7 @@
</tuple>
<state>
<tuple>
<float>1585316915.85</float>
<float>1596024084.3</float>
<string>UTC</string>
</tuple>
</state>
......@@ -105,7 +105,7 @@
</tuple>
<state>
<tuple>
<float>1900676915.85</float>
<float>1627560084.3</float>
<string>UTC</string>
</tuple>
</state>
......@@ -114,7 +114,7 @@
</item>
<item>
<key> <string>id</string> </key>
<value> <string>default_http_json</string> </value>
<value> <string>default</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
......@@ -122,7 +122,7 @@
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Default HTTP / JSON policy</string> </value>
<value> <string>test_sensor.test_product</string> </value>
</item>
<item>
<key> <string>version</string> </key>
......@@ -206,7 +206,7 @@
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
<value> <string>ERP5TypeTestCase</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
......@@ -228,7 +228,7 @@
</tuple>
<state>
<tuple>
<float>1585316915.92</float>
<float>1596024084.32</float>
<string>UTC</string>
</tuple>
</state>
......@@ -247,7 +247,7 @@
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
<value> <string>ERP5TypeTestCase</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
......@@ -267,7 +267,7 @@
</tuple>
<state>
<tuple>
<float>1585316915.92</float>
<float>1596024084.32</float>
<string>UTC</string>
</tuple>
</state>
......@@ -302,7 +302,7 @@
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
<value> <string>ERP5TypeTestCase</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
......@@ -332,7 +332,7 @@
</tuple>
<state>
<tuple>
<float>1585316915.92</float>
<float>1596024084.32</float>
<string>UTC</string>
</tuple>
</state>
......
bt5/erp5_wendelin/PathTemplateItem/data_supply_module/default_http_json/1.xml → bt5/erp5_wendelin/PathTemplateItem/data_supply_module/default/1.xml
View file @ 02846171
......@@ -64,7 +64,7 @@
<key> <string>categories</string> </key>
<value>
<tuple>
<string>resource/data_operation_module/wendelin_1</string>
<string>resource/data_operation_module/wendelin_ingest_data</string>
</tuple>
</value>
</item>
......@@ -146,7 +146,7 @@
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
<value> <string>ERP5TypeTestCase</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
......@@ -176,7 +176,7 @@
</tuple>
<state>
<tuple>
<float>1585316915.94</float>
<float>1596024084.33</float>
<string>UTC</string>
</tuple>
</state>
......@@ -209,7 +209,7 @@
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
<value> <string>ERP5TypeTestCase</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
......@@ -231,7 +231,7 @@
</tuple>
<state>
<tuple>
<float>1585316915.94</float>
<float>1596024084.33</float>
<string>UTC</string>
</tuple>
</state>
......
bt5/erp5_wendelin/PathTemplateItem/data_supply_module/default_http_json/2.xml → bt5/erp5_wendelin/PathTemplateItem/data_supply_module/default/2.xml
View file @ 02846171
......@@ -76,8 +76,8 @@
<key> <string>categories</string> </key>
<value>
<tuple>
<string>resource/data_product_module/default_http_json</string>
<string>use/big_data/ingestion</string>
<string>resource/data_product_module/default</string>
<string>use/big_data/ingestion/stream</string>
</tuple>
</value>
</item>
......@@ -85,6 +85,12 @@
<key> <string>default_reference</string> </key>
<value> <string>out_stream</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>2</string> </value>
......@@ -189,7 +195,7 @@
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
<value> <string>ERP5TypeTestCase</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
......@@ -219,7 +225,52 @@
</tuple>
<state>
<tuple>
<float>1585316915.95</float>
<float>1596024084.34</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
</dictionary>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>edit</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>985.37521.26601.6024</string> </value>
</item>
<item>
<key> <string>state</string> </key>
<value> <string>current</string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass> <reference id="5.1"/> </klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1596025444.57</float>
<string>UTC</string>
</tuple>
</state>
......@@ -248,7 +299,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>982.52796.39292.30037</string> </value>
<value> <string>985.37544.5004.17339</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -264,7 +315,7 @@
</tuple>
<state>
<tuple>
<float>1585317304.32</float>
<float>1596025509.27</float>
<string>UTC</string>
</tuple>
</state>
......@@ -297,7 +348,7 @@
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
<value> <string>ERP5TypeTestCase</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
......@@ -319,7 +370,7 @@
</tuple>
<state>
<tuple>
<float>1585316915.95</float>
<float>1596024084.34</float>
<string>UTC</string>
</tuple>
</state>
......
bt5/erp5_wendelin/PathTemplateItem/portal_ingestion_policies/default_http_json.xml deleted 100644 → 0
View file @ 14897249
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Ingestion Policy" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Access_contents_information_Permission</string> </key>
<value>
<tuple>
<string>Anonymous</string>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_View_Permission</string> </key>
<value>
<tuple>
<string>Anonymous</string>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>default_http_json</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>default_http_json</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Ingestion Policy</string> </value>
</item>
<item>
<key> <string>script_id</string> </key>
<value> <string>IngestionPolicy_parseSimpleFluentdTag</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Default HTTP / JSON policy</string> </value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>001</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>edit_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_log</string> </key>
<value>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>edit</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>0.0.0.0</string> </value>
</item>
<item>
<key> <string>state</string> </key>
<value> <string>current</string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass>
<global name="DateTime" module="DateTime.DateTime"/>
</klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1585316915.87</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
</dictionary>
</list>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_log</string> </key>
<value>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass>
<global id="4.1" name="DateTime" module="DateTime.DateTime"/>
</klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1585316915.87</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>draft</string> </value>
</item>
</dictionary>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass> <reference id="4.1"/> </klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1585316915.88</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin/TestTemplateItem/portal_components/test.erp5.testWendelin.py
View file @ 02846171
......@@ -278,9 +278,9 @@ class Test(ERP5TypeTestCase):
"""
# the default json ingestion is usde in HowTo / Docs
self.assertNotEqual(None,
getattr(self.portal.portal_ingestion_policies, "default_http_json", None))
getattr(self.portal.portal_ingestion_policies, "default", None))
self.assertNotEqual(None,
getattr(self.portal.data_supply_module, "default_http_json", None))
getattr(self.portal.data_supply_module, "default", None))
def test_07_LinkedDataStreamList(self):
"""
......@@ -349,60 +349,15 @@ class Test(ERP5TypeTestCase):
Test ingestion using a POST Request containing a msgpack encoded message
simulating input from fluentd.
"""
from datetime import datetime, timedelta
from datetime import datetime
import time
portal = self.portal
now = datetime.now()
reference="test_sensor.test_product"
title = reference
ingestion_policy = portal.portal_ingestion_policies['default']
# create related data supply and etc.
use_category = portal.restrictedTraverse("portal_categories/use/big_data/ingestion/stream")
data_operation = portal.restrictedTraverse("data_operation_module/wendelin_ingest_data")
# create Data Product
data_product = portal.data_product_module.newContent(
portal_type = "Data Product",
title = "Append to Data Stream",
reference = reference.split('.')[1])
data_product.setUseValue(use_category)
data_product.setAggregatedPortalTypeList(["Data Stream", "Progress Indicator"])
data_product.validate()
# create Data Supply
data_supply_kw = {'title': title,
'reference': reference.split('.')[0],
'version': '001',
'effective_date': now,
'expiration_date': now + timedelta(days=365)}
data_supply = portal.data_supply_module.newContent( \
portal_type='Data Supply', **data_supply_kw)
data_supply.validate()
# add ingestion line
data_supply_line_kw = {'title': 'Ingest Data',
'reference': 'ingestion_operation',
'int_index': 1,
'quantity': 1.0}
data_supply_line = data_supply.newContent(portal_type='Data Supply Line', **data_supply_line_kw)
data_supply_line.setResourceValue(data_operation)
# add append to Data Stream line
data_supply_line_kw = {'title': 'Data Stream',
'reference': 'out_stream',
'int_index': 2,
'quantity': 1.0}
data_supply_line = data_supply.newContent(portal_type='Data Supply Line', \
**data_supply_line_kw)
data_supply_line.setResourceValue(data_product)
data_supply_line.setUseValue(use_category)
self.tic()
data_supply = portal.data_supply_module["default"]
data_list = []
int_date = int(time.mktime(now.timetuple()))
......
bt5/erp5_wendelin/bt/template_keep_workflow_path_list
View file @ 02846171
......@@ -6,13 +6,9 @@ web_page_module/rjs_gadget_erp5_pt_ingestion_policies_html
web_page_module/rjs_gadget_erp5_pt_callables_js
web_page_module/rjs_gadget_erp5_pt_callables_html
portal_gadgets/WendelinInformationGadget
portal_ingestion_policies/default_http_json
portal_ingestion_policies/default_http_json/**
portal_ingestion_policies/default
data_supply_module/default_http_json
data_supply_module/default_http_json/**
data_product_module/default_http_json
data_product_module/default_http_json/**
data_supply_module/default
data_supply_module/default/**
data_operation_module/wendelin_ingest_data
web_site_module/default_wendelin_front
web_site_module/default_wendelin_front/**
\ No newline at end of file
bt5/erp5_wendelin/bt/template_path_list
View file @ 02846171
data_operation_module/wendelin_ingest_data
data_operation_module/wendelin_ingest_data_conversion
data_product_module/default_http_json
data_product_module/default_http_json/**
data_supply_module/default_http_json
data_supply_module/default_http_json/**
data_product_module/default
data_product_module/default/**
data_supply_module/default
data_supply_module/default/**
portal_alarms/wendelin_handle_analysis
portal_callables/DataIngestionLine_writeFluentdIngestionToDataStream
portal_callables/DataIngestionLine_writeIngestionToDataStream
......@@ -22,8 +22,6 @@ portal_categories/use/**
portal_gadgets/WendelinInformationGadget
portal_gadgets/WendelinInformationGadget/**
portal_ingestion_policies/default
portal_ingestion_policies/default_http_json
portal_ingestion_policies/default_http_json/**
web_page_module/*_wendelin_*
web_page_module/rjs_gadget_erp5_pt_callables_html
web_page_module/rjs_gadget_erp5_pt_callables_js
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7