Skip to content

K
klaus_wendelin
Commit dc40d6a1 authored by Roque's avatar Roque
Browse files
Options

erp5_wendelin_data_sample: missing data operation

parent ddc8b1eb
Hide whitespace changes
Inline Side-by-side
Showing with 490 additions and 3 deletions
bt5/erp5_wendelin_data_sample/PathTemplateItem/data_analysis_module/wendelin_convert_environment_raw_data/1.xml
View file @ dc40d6a1
......@@ -10,9 +10,9 @@
<key> <string>categories</string> </key>
<value>
<tuple>
<string>resource/data_operation_module/49</string>
<string>quantity_unit/unit/piece</string>
<string>aggregate/data_acquisition_unit_module/wendelin_environment_sensor</string>
<string>resource/data_operation_module/wendelin_convert_environment_raw_data</string>
</tuple>
</value>
</item>
......@@ -20,6 +20,12 @@
<key> <string>default_reference</string> </key>
<value> <string>data_operation</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>1</string> </value>
......@@ -102,7 +108,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>989.23165.41501.63539</string> </value>
<value> <string>989.52330.10441.39304</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -120,7 +126,7 @@
</tuple>
<state>
<tuple>
<float>1610720792.52</float>
<float>1612468452.7</float>
<string>UTC</string>
</tuple>
</state>
......
bt5/erp5_wendelin_data_sample/PathTemplateItem/data_operation_module/wendelin_convert_environment_raw_data.xml 0 → 100644
View file @ dc40d6a1
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Data Operation" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Access_contents_information_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_View_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>quantity_unit/unit/piece</string>
</tuple>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>data-operation-convert-raw-data-to-array</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>wendelin_convert_environment_raw_data</string> </value>
</item>
<item>
<key> <string>language</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Data Operation</string> </value>
</item>
<item>
<key> <string>script_id</string> </key>
<value> <string>DataAnalysisLine_convertEnvironmentDataStreamToArray</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Convert Raw Environment Data to Array</string> </value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>001</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>edit_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_log</string> </key>
<value>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>edit</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>989.52329.44617.25873</string> </value>
</item>
<item>
<key> <string>state</string> </key>
<value> <string>current</string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass>
<global name="DateTime" module="DateTime.DateTime"/>
</klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1612468426.71</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
</dictionary>
</list>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_log</string> </key>
<value>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass>
<global name="DateTime" module="DateTime.DateTime"/>
</klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1612468433.24</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_data_sample/PathTemplateItem/portal_callables/DataAnalysisLine_convertEnvironmentDataStreamToArray.py 0 → 100644
View file @ dc40d6a1
import pandas as pd
progress_indicator = in_stream["Progress Indicator"]
in_data_stream = in_stream["Data Stream"]
out_data_array = out_array["Data Array"]
chunk_size = 20 * 10**6
start = progress_indicator.getIntOffsetIndex()
end = min(start+chunk_size, in_data_stream.getSize())
unpacked, end = in_data_stream.readMsgpackChunkList(start, end)
f = in_data_stream.extractDateTime
df = pd.DataFrame((dict(**o[1]) for o in unpacked), dtype="float64", index=(f(o[0]) for o in unpacked))
if df.shape[0] == 0:
return
df.index.name="date"
ndarray = df.to_records(convert_datetime64=False)
zbigarray = out_data_array.getArray()
if zbigarray is None:
zbigarray = out_data_array.initArray(shape=(0,), dtype=ndarray.dtype.fields)
zbigarray.append(ndarray)
if end > start:
progress_indicator.setIntOffsetIndex(end)
# tell caller to create new activity after processing if we did not reach end of stream
if end < in_data_stream.getSize():
return 1
bt5/erp5_wendelin_data_sample/PathTemplateItem/portal_callables/DataAnalysisLine_convertEnvironmentDataStreamToArray.xml 0 → 100644
View file @ dc40d6a1
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Python Script" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_local_properties</string> </key>
<value>
<tuple>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>reference</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>string</string> </value>
</item>
</dictionary>
</tuple>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>in_stream=None, out_array=None</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>DataAnalysisLine_convertEnvironmentDataStreamToArray</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Python Script</string> </value>
</item>
<item>
<key> <string>reference</string> </key>
<value> <string>DataAnalysisLine_convertEnvironmentDataStreamToArray</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>edit_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_log</string> </key>
<value>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>edit</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>989.52322.24707.52036</string> </value>
</item>
<item>
<key> <string>state</string> </key>
<value> <string>current</string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass>
<global name="DateTime" module="DateTime.DateTime"/>
</klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1612467989.09</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
</dictionary>
</list>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_data_sample/bt/template_keep_last_workflow_history_only_path_list
View file @ dc40d6a1
......@@ -10,6 +10,7 @@ data_array_module/wendelin_resample_generic_interval_array_30000
data_array_module/wendelin_resample_generic_interval_array_60
data_ingestion_module/wendelin_date-sensor_1
data_ingestion_module/wendelin_date-sensor_1/**
data_operation_module/wendelin_convert_environment_raw_data
data_operation_module/wendelin_ingest_data
data_product_module/wendelin_environment_raw_array
data_product_module/wendelin_environment_raw_array/**
......@@ -26,6 +27,7 @@ notebook_module/wendelin_environment_data_sample_graph
organisation_module/wendelin_nexedi_data_center
organisation_module/wendelin_sample_company
organisation_module/wendelin_sample_factory
portal_callables/DataAnalysisLine_convertEnvironmentDataStreamToArray
progress_indicator_module/wendelin_convert_environment_raw_data
progress_indicator_module/wendelin_resample_generic_interval_array
web_page_module/rjs_ndarray_bundle_js
\ No newline at end of file
bt5/erp5_wendelin_data_sample/bt/template_keep_path_list
View file @ dc40d6a1
......@@ -10,6 +10,7 @@ data_array_module/wendelin_resample_generic_interval_array_30000
data_array_module/wendelin_resample_generic_interval_array_60
data_ingestion_module/wendelin_date-sensor_1
data_ingestion_module/wendelin_date-sensor_1/**
data_operation_module/wendelin_convert_environment_raw_data
data_operation_module/wendelin_ingest_data
data_product_module/wendelin_environment_raw_array
data_product_module/wendelin_environment_raw_array/**
......@@ -26,6 +27,7 @@ notebook_module/wendelin_environment_data_sample_graph
organisation_module/wendelin_nexedi_data_center
organisation_module/wendelin_sample_company
organisation_module/wendelin_sample_factory
portal_callables/DataAnalysisLine_convertEnvironmentDataStreamToArray
progress_indicator_module/wendelin_convert_environment_raw_data
progress_indicator_module/wendelin_resample_generic_interval_array
web_page_module/rjs_ndarray_bundle_js
\ No newline at end of file
bt5/erp5_wendelin_data_sample/bt/template_keep_workflow_path_list
View file @ dc40d6a1
......@@ -10,6 +10,7 @@ data_array_module/wendelin_resample_generic_interval_array_30000
data_array_module/wendelin_resample_generic_interval_array_60
data_ingestion_module/wendelin_date-sensor_1
data_ingestion_module/wendelin_date-sensor_1/**
data_operation_module/wendelin_convert_environment_raw_data
data_operation_module/wendelin_ingest_data
data_product_module/wendelin_environment_raw_array
data_product_module/wendelin_environment_raw_array/**
......@@ -26,6 +27,7 @@ notebook_module/wendelin_environment_data_sample_graph
organisation_module/wendelin_nexedi_data_center
organisation_module/wendelin_sample_company
organisation_module/wendelin_sample_factory
portal_callables/DataAnalysisLine_convertEnvironmentDataStreamToArray
progress_indicator_module/wendelin_convert_environment_raw_data
progress_indicator_module/wendelin_resample_generic_interval_array
web_page_module/rjs_ndarray_bundle_js
\ No newline at end of file
bt5/erp5_wendelin_data_sample/bt/template_path_list
View file @ dc40d6a1
......@@ -10,6 +10,7 @@ data_array_module/wendelin_resample_generic_interval_array_30000
data_array_module/wendelin_resample_generic_interval_array_60
data_ingestion_module/wendelin_date-sensor_1
data_ingestion_module/wendelin_date-sensor_1/**
data_operation_module/wendelin_convert_environment_raw_data
data_operation_module/wendelin_ingest_data
data_product_module/wendelin_environment_raw_array
data_product_module/wendelin_environment_raw_array/**
......@@ -26,6 +27,7 @@ notebook_module/wendelin_environment_data_sample_graph
organisation_module/wendelin_nexedi_data_center
organisation_module/wendelin_sample_company
organisation_module/wendelin_sample_factory
portal_callables/DataAnalysisLine_convertEnvironmentDataStreamToArray
progress_indicator_module/wendelin_convert_environment_raw_data
progress_indicator_module/wendelin_resample_generic_interval_array
web_page_module/rjs_ndarray_bundle_js
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7