Skip to content

W
wendelin
Commit 0788d776 authored by Nikola Balog's avatar Nikola Balog
Browse files
Options

Update: Added recalculation of scores which returns the best 5 simulations,... 

Update: Added recalculation of scores which returns the best 5 simulations, new score function and improved notebook
parent 54949f6a
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 2146 additions and 401 deletions
bt5/erp5_wendelin_drone/PathTemplateItem/data_operation_module/recalculate_scores.xml 0 → 100644
View file @ 0788d776
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Data Operation" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Access_contents_information_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_View_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>aggregated_portal_type</string> </key>
<value>
<tuple>
<string>Data Array</string>
</tuple>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>quantity_unit/unit/piece</string>
</tuple>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>recalculate_scores</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>recalculate_scores</string> </value>
</item>
<item>
<key> <string>language</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>script_id</string> </key>
<value> <string>DataAnalysisLine_recalculateScore</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Recalcualte Scores</string> </value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>001</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_drone/PathTemplateItem/data_product_module/bouncy_flight_mavsdk_raw_data.xml
View file @ 0788d776
......@@ -14,9 +14,7 @@
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Author</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
......@@ -27,9 +25,7 @@
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Author</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
......@@ -40,9 +36,7 @@
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Author</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
......@@ -54,9 +48,7 @@
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Author</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
......
bt5/erp5_wendelin_drone/PathTemplateItem/data_product_module/drone_raw_array.xml
View file @ 0788d776
......@@ -14,9 +14,7 @@
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Author</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
......@@ -27,9 +25,7 @@
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Author</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
......@@ -40,9 +36,7 @@
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Author</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
......@@ -54,9 +48,7 @@
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Author</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
......
bt5/erp5_wendelin_drone/PathTemplateItem/data_product_module/drone_raw_data.xml
View file @ 0788d776
......@@ -14,9 +14,7 @@
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Author</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
......@@ -27,9 +25,7 @@
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Author</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
......@@ -40,9 +36,7 @@
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Author</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
......@@ -54,9 +48,7 @@
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Author</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
......
bt5/erp5_wendelin_drone/PathTemplateItem/data_product_module/recalculated_plots.xml 0 → 100644
View file @ 0788d776
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Data Product" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Access_contents_information_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_View_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>aggregated_portal_type</string> </key>
<value>
<tuple>
<string>Data Array</string>
<string>Progress Indicator</string>
<string>Data Bucket Stream</string>
</tuple>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>quantity_unit/unit/piece</string>
<string>use/big_data/ingestion/stream</string>
</tuple>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>recalculated_plots</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>recalculated_plots</string> </value>
</item>
<item>
<key> <string>language</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Recalculated Plots</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_drone/PathTemplateItem/data_product_module/recalculated_scores.xml 0 → 100644
View file @ 0788d776
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Data Product" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Access_contents_information_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_View_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>aggregated_portal_type</string> </key>
<value>
<tuple>
<string>Data Array</string>
<string>Progress Indicator</string>
<string>Data Bucket Stream</string>
</tuple>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>quantity_unit/unit/piece</string>
<string>use/big_data/ingestion/stream</string>
</tuple>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>recalculated_scores</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>recalculated_scores</string> </value>
</item>
<item>
<key> <string>language</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Recalculated Scores</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_drone/PathTemplateItem/data_product_module/simulation_scores.xml
View file @ 0788d776
......@@ -14,9 +14,7 @@
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Author</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
......@@ -27,9 +25,7 @@
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Author</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
......@@ -40,9 +36,7 @@
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Author</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
......@@ -54,9 +48,7 @@
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Author</string>
<string>Manager</string>
<string>Owner</string>
</tuple>
</value>
</item>
......@@ -66,6 +58,7 @@
<tuple>
<string>Data Array</string>
<string>Progress Indicator</string>
<string>Data Bucket Stream</string>
</tuple>
</value>
</item>
......@@ -73,8 +66,9 @@
<key> <string>categories</string> </key>
<value>
<tuple>
<string>use/big_data/analysis</string>
<string>quantity_unit/unit/piece</string>
<string>use/big_data/analysis</string>
<string>use/big_data/ingestion/stream</string>
</tuple>
</value>
</item>
......
bt5/erp5_wendelin_drone/PathTemplateItem/data_transformation_module/create_score_list_array/1.xml
View file @ 0788d776
......@@ -18,6 +18,14 @@
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>aggregated_portal_type</string> </key>
<value>
<tuple>
<string>Data Array</string>
</tuple>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
......
bt5/erp5_wendelin_drone/PathTemplateItem/data_transformation_module/create_score_list_array/2.xml
View file @ 0788d776
......@@ -31,10 +31,10 @@
<key> <string>categories</string> </key>
<value>
<tuple>
<string>trade_phase/data/convert</string>
<string>use/big_data/analysis</string>
<string>quantity_unit/unit/piece</string>
<string>resource/data_product_module/bouncy_flight_mavsdk</string>
<string>trade_phase/data/convert</string>
<string>use/big_data/ingestion/stream</string>
</tuple>
</value>
</item>
......
bt5/erp5_wendelin_drone/PathTemplateItem/data_transformation_module/create_score_list_array/5.xml
View file @ 0788d776
......@@ -31,10 +31,9 @@
<key> <string>categories</string> </key>
<value>
<tuple>
<string>resource/data_product_module/drone_simulation_performance_data</string>
<string>trade_phase/data/convert</string>
<string>use/big_data/analysis</string>
<string>resource/data_product_module/11</string>
<string>quantity_unit/unit/piece</string>
</tuple>
</value>
</item>
......
bt5/erp5_wendelin_drone/PathTemplateItem/data_transformation_module/recalculate-score-list-array.xml 0 → 100644
View file @ 0788d776
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Data Transformation" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Access_contents_information_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Add_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_Modify_portal_content_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_View_Permission</string> </key>
<value>
<tuple>
<string>Assignee</string>
<string>Assignor</string>
<string>Associate</string>
<string>Auditor</string>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>_count</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>_identity_criterion</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>_mt_index</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
<item>
<key> <string>_range_criterion</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAU=</string> </persistent>
</value>
</item>
<item>
<key> <string>_tree</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAY=</string> </persistent>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>resource/data_product_module/simulation_scores</string>
</tuple>
</value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>recalculate-score-list-array</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>This transformation recalcualtes the scores of every simulation we have processed. The output shows the best simulations we have currently seen.</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>recalculate-score-list-array</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Recalculate Score List Array</string> </value>
</item>
<item>
<key> <string>version</string> </key>
<value>
<none/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="Length" module="BTrees.Length"/>
</pickle>
<pickle> <int>0</int> </pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="OOBTree" module="BTrees.OOBTree"/>
</pickle>
<pickle>
<none/>
</pickle>
</record>
<record id="5" aka="AAAAAAAAAAU=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="6" aka="AAAAAAAAAAY=">
<pickle>
<global name="OOBTree" module="BTrees.OOBTree"/>
</pickle>
<pickle>
<none/>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_drone/PathTemplateItem/data_transformation_module/recalculate-score-list-array/1.xml 0 → 100644
View file @ 0788d776
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Data Transformation Operation Line" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_identity_criterion</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>_range_criterion</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>aggregated_portal_type</string> </key>
<value>
<tuple>
<string>Data Array</string>
</tuple>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>trade_phase/data/convert</string>
<string>use/big_data/analysis</string>
<string>quantity_unit/unit/piece</string>
<string>resource/data_operation_module/recalculate_scores</string>
</tuple>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>data_operation</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>1</string> </value>
</item>
<item>
<key> <string>int_index</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>quantity</string> </key>
<value> <float>1.0</float> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Recaculates Scores</string> </value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_drone/PathTemplateItem/data_transformation_module/recalculate-score-list-array/2.xml 0 → 100644
View file @ 0788d776
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Data Transformation Resource Line" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_identity_criterion</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>_range_criterion</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>aggregated_portal_type</string> </key>
<value>
<tuple>
<string>Data Array</string>
<string>Progress Indicator</string>
<string>Data Bucket Stream</string>
</tuple>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>trade_phase/data/convert</string>
<string>use/big_data/analysis</string>
<string>resource/data_product_module/recalculated_plots</string>
</tuple>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>out_array_plots</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>2</string> </value>
</item>
<item>
<key> <string>int_index</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>quantity</string> </key>
<value> <float>1.0</float> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Recalculated Plots</string> </value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_drone/PathTemplateItem/data_transformation_module/recalculate-score-list-array/3.xml 0 → 100644
View file @ 0788d776
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Data Transformation Resource Line" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_identity_criterion</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>_range_criterion</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>aggregated_portal_type</string> </key>
<value>
<tuple>
<string>Data Array</string>
<string>Progress Indicator</string>
</tuple>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>trade_phase/data/convert</string>
<string>use/big_data/analysis</string>
<string>quantity_unit/unit/piece</string>
<string>resource/data_product_module/simulation_scores</string>
</tuple>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>input_array_scores</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>3</string> </value>
</item>
<item>
<key> <string>int_index</string> </key>
<value> <int>2</int> </value>
</item>
<item>
<key> <string>quantity</string> </key>
<value> <float>-1.0</float> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Simulated Flights</string> </value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_drone/PathTemplateItem/data_transformation_module/recalculate-score-list-array/4.xml 0 → 100644
View file @ 0788d776
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Data Transformation Resource Line" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_identity_criterion</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>_range_criterion</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>aggregated_portal_type</string> </key>
<value>
<tuple>
<string>Data Array</string>
<string>Progress Indicator</string>
</tuple>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>trade_phase/data/convert</string>
<string>use/big_data/analysis</string>
<string>resource/data_product_module/recalculated_scores</string>
</tuple>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>out_array_scores</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>4</string> </value>
</item>
<item>
<key> <string>int_index</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>quantity</string> </key>
<value> <float>1.0</float> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Recalculated List</string> </value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_drone/PathTemplateItem/data_transformation_module/recalculate-score-list-array/5.xml 0 → 100644
View file @ 0788d776
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Data Transformation Resource Line" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_identity_criterion</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>_range_criterion</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>aggregated_portal_type</string> </key>
<value>
<tuple>
<string>Data Array</string>
<string>Progress Indicator</string>
</tuple>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>trade_phase/data/convert</string>
<string>use/big_data/analysis</string>
<string>quantity_unit/unit/piece</string>
<string>resource/data_product_module/drone_simulation_performance_data</string>
</tuple>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>input_array_plot</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>5</string> </value>
</item>
<item>
<key> <string>int_index</string> </key>
<value> <int>4</int> </value>
</item>
<item>
<key> <string>quantity</string> </key>
<value> <float>-1.0</float> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Simulated Flights Plot</string> </value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_drone/PathTemplateItem/portal_callables/DataAnalysisLine_convertRawDroneDataToArray.py
View file @ 0788d776
......@@ -12,74 +12,64 @@ out_data_array = out_array["Data Array"]
keys = in_data_stream.getKeyList()
start = progress_indicator.getIntOffsetIndex()
# Later we can add a chunk variable, which would tell us how many buckets we can process each call. But currently we do not have many buckets.
end = len(keys)
context.log(in_data_stream)
end = progress_indicator.getStringOffsetIndex()
if end is None:
end = ""
if end == 0:
if len(keys) == 0:
context.log("No Keys found")
return
for key in keys[start:end]: # should be start instead of 1, but because we have none in there we do it like this for the broken part
context.log(key)
log = in_data_stream.getBucketByKey(key)
df = pd.read_csv(log, sep=';', dtype=str)
if df.shape[0] == 0:
return
# Remove non-ASCII characters from DataFrame values
df = df.applymap(remove_non_ascii)
# Remove non-ASCII characters from column names (headers)
df.columns = df.columns.map(remove_non_ascii)
# Convert non-numeric columns to float64
non_numeric_columns = df.select_dtypes(exclude=[np.number]).columns
df[non_numeric_columns] = df[non_numeric_columns].apply(pd.to_numeric, errors='coerce')
ndarray = df.to_records(convert_datetime64=False)
zbigarray = out_data_array.getArray()
if zbigarray is None:
zbigarray = out_data_array.initArray(shape=(0,), dtype=ndarray.dtype.fields)
start_array = zbigarray.shape[0]
zbigarray.append(ndarray)
data_array_line = out_array.get(key)
if data_array_line is None:
data_array_line = out_data_array.newContent(id=key,
dtype= {'timestamp (ms)': '<f8',
'latitude ()': '<f8',
'longitude ()': '<f8',
'AMSL (m)': '<f8',
'rel altitude (m)': '<f8',
'yaw ()': '<f8',
'ground speed (m/s)': '<f8',
'climb rate (m/s)': '<f8'}
new_end = ""
for key in [x for x in keys if x not in end]:
try:
log = in_data_stream.getBucketByKey(key)
df = pd.read_csv(log, sep=';', dtype=str)
if df.shape[0] == 0:
return
# Remove non-ASCII characters from DataFrame values
df = df.applymap(remove_non_ascii)
# Remove non-ASCII characters from column names (headers)
df.columns = df.columns.map(remove_non_ascii)
non_numeric_columns = df.select_dtypes(exclude=[np.number]).columns
df[non_numeric_columns] = df[non_numeric_columns].apply(pd.to_numeric, errors='coerce')
ndarray = df.to_records(column_dtypes=dtype, index = False)
#Use to delete the array
#out_data_array = out_data_array.initArray(shape=(0,), dtype=ndarray.dtype.fields)
#return
zbigarray = out_data_array.getArray()
if zbigarray is None:
zbigarray = out_data_array.initArray(shape=(0,), dtype=ndarray.dtype.fields)
start_array = zbigarray.shape[0]
zbigarray.append(ndarray)
try:
data_array_line = out_array.get(key)
if data_array_line is None:
data_array_line = out_data_array.newContent(id=key,
portal_type="Data Array Line")
data_array_line.edit(reference=key,
index_expression="%s:%s" %(start_array, zbigarray.shape[0])
)
except:
context.log("Can not create Data Array Line")
except:
context.log("File "+str(key)+ " is not well formated")
new_end = new_end + str(key)
data_array_line.edit(reference=key,
index_expression="%s:%s" %(start_array, zbigarray.shape[0])
)
context.log("index_expression")
context.log("%s:%s" %(start_array, zbigarray.shape[0]))
# This is quite useless currently, as we are able to read all the buckets in one go. But in the case we want to only iterate over one bucket at a time (or a chunk of them) we can use this.
if end > start:
progress_indicator.setIntOffsetIndex(end)
if end < len(keys):
return 1
progress_indicator.setStringOffsetIndex(end + new_end)
bt5/erp5_wendelin_drone/PathTemplateItem/portal_callables/DataAnalysisLine_recalculateScore.py 0 → 100644
View file @ 0788d776
import math
import pandas as pd
import numpy as np
import re
# Take the data from the score list array. See if the array is larger than this one. If it is not, do not do anything. Else recalculate the scores for each simulation we have seen (just sort the data).
# Now, to save space, we just overwrite the old array with the new one, that contains the new scores
# This new array will give us the final overview of the ranking of the parameters. This can be used by our genetic algorithm to decide when we can stop and which parameters worked the best.
score_dtypes = {'name': 'S256', 'Param1': 'f16', 'Param2': 'f16',
'distance_reciprocal': 'f8', 'ASML_reciprocal': 'f8',
'ground_speed_reciprocal': 'f8', 'climb_rate_reciprocal': 'f8',
'score_reciprocal': 'f16', 'score_cosine_row': 'f16',
'score_cosine_column': 'f16'}
new_score_dtypes= {'name': 'S256', 'Param1': 'f16', 'Param2': 'f16',
'distance_reciprocal': 'f8', 'ASML_reciprocal': 'f8',
'ground_speed_reciprocal': 'f8', 'climb_rate_reciprocal': 'f8',
'score_reciprocal': 'f16', 'score_cosine_row': 'f16',
'score_cosine_column': 'f16',
'iteration': 'f16'}
plot_dtypes = {
'name': 'S256',
'Param1': 'f8',
'Param2': 'f8',
'timestamp': 'f8',
'distance_diff': 'f8',
'ASML_diff': 'f8',
'ground_speed_diff': 'f8',
'climb_rate_diff': 'f8',
'distance_reciprocal': 'f8',
'ASML_reciprocal': 'f8',
'ground_speed_reciprocal': 'f8',
'climb_rate_reciprocal': 'f8',
'score_reciprocal': 'f16',
'score_cosine_row': 'f16',
'score_cosine_column': 'f16',
'iteration': 'f16'
}
new_plot_dtypes = {
'Param1': 'f8',
'Param2': 'f8',
'timestamp': 'f8',
'distance_diff': 'f8',
'ASML_diff': 'f8',
'ground_speed_diff': 'f8',
'climb_rate_diff': 'f8',
'distance_reciprocal': 'f8',
'ASML_reciprocal': 'f8',
'ground_speed_reciprocal': 'f8',
'climb_rate_reciprocal': 'f8',
'score_reciprocal': 'f8',
#'score_cosine_row': 'f16',
#'score_cosine_column': 'f16',
'iteration': 'f8'
}
score_array = input_array_scores["Data Array"]
new_score_array = out_array_scores["Data Array"]
plot_array = input_array_plot["Data Array"]
new_plot_array = out_array_plots["Data Array"]
# Should only look at the newest few
score_nparray = score_array.getArray()
old_score_df = pd.DataFrame.from_records(score_nparray[:].copy())
plot_nparray = plot_array.getArray()
old_plot_df = pd.DataFrame.from_records(plot_nparray[:].copy())
progress_indicator = input_array_scores["Progress Indicator"]
seen_sims = progress_indicator.getStringOffsetIndex()
if seen_sims is None:
seen_sims = ""
sim_flight_names = list(old_score_df["name"])
if len([x for x in sim_flight_names if x not in seen_sims]) == 0:
return
new_score_nparray = new_score_array.getArray()
if new_score_nparray is None:
new_score_nparray = out_array_scores["Data Array"].initArray(shape=(0,), dtype=list(new_score_dtypes.items()))
new_plot_nparray = new_plot_array.getArray()
if new_plot_nparray is None:
new_plot_nparray = out_array_plots["Data Array"].initArray(shape=(0,), dtype=list(new_plot_dtypes.items()))
new_score_df = pd.DataFrame.from_records(new_score_nparray[:].copy())
new_plot_df = pd.DataFrame.from_records(new_plot_nparray[:].copy())
new_score_iteration = new_score_df["iteration"]
new_score_df = new_score_df.drop(columns=["iteration"])
new_plot_iteration = new_plot_df["iteration"]
#new_plot_df = new_plot_df.drop(columns=["iteration"])
answer_scores = pd.concat([old_score_df, new_score_df]).drop_duplicates(subset=['Param1','Param2'], keep='last')
answer_plots = pd.concat([old_plot_df, new_plot_df]).drop_duplicates(subset=['Param1','Param2'], keep='last')
answer_scores = answer_scores.nlargest(5, "score_reciprocal")
context.log(answer_scores["name"])
answer_plots = old_plot_df[old_plot_df["name"].isin(list(answer_scores["name"]))]
# Change this when everything works
if new_score_iteration.empty or new_plot_iteration.empty:
new_score_iteration = 0
new_plot_iteration = 0
answer_scores["iteration"] = new_score_iteration + 1
answer_plots["iteration"] = new_plot_iteration + 1
# We will remove all the data from the data arrays before we append the new data. Essentially we will be left with only the best few iterations
new_score_nparray = out_array_scores["Data Array"].initArray(shape=(0,), dtype=list(new_score_dtypes.items()))
new_plot_nparray = out_array_plots["Data Array"].initArray(shape=(0,), dtype=list(new_plot_dtypes.items()))
new_score_nparray.append(answer_scores.to_records(column_dtypes=new_score_dtypes, index = False))
# Group the DataFrame by the 'name' column
grouped = answer_plots.groupby(['Param1', 'Param2'])
# Initialize a list to store the resulting DataFrames
resulting_dfs = []
# Iterate over unique names and create DataFrames
for name, group in grouped:
resulting_dfs.append(group)
#context.log(resulting_dfs[0])
current_idx = 0
ranks = ["aasdfjalsd","basdfiu","clkjlj","dasdfawf", "easdfcvs"]
current_rank = 1
for df in resulting_dfs:
df = df.drop(columns=["name"])
data_array_line_plot = out_array_plots["Data Array"].get(str(current_rank))
context.log(data_array_line_plot)
new_plot_nparray.append(df.to_records(column_dtypes=new_plot_dtypes, index = False))
if data_array_line_plot is None:
data_array_line_plot = out_array_plots["Data Array"].newContent(id=current_rank,
portal_type="Data Array Line")
data_array_line_plot.edit(
reference=current_rank,
index_expression="%s:%s" %(current_idx, new_plot_nparray.shape[0])
)
current_idx = new_plot_nparray.shape[0]
current_rank = current_rank + 1
progress_indicator.setStringOffsetIndex(sim_flight_names)
return
bt5/erp5_wendelin_drone/PathTemplateItem/portal_callables/DataAnalysisLine_recalculateScore.xml 0 → 100644
View file @ 0788d776
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PyData Script" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="_reconstructor" module="copy_reg"/>
</klass>
<tuple>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
<global name="object" module="__builtin__"/>
<none/>
</tuple>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_local_properties</string> </key>
<value>
<tuple>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>reference</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>string</string> </value>
</item>
</dictionary>
</tuple>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>input_array_scores, input_array_plot, out_array_scores, out_array_plots</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>DataAnalysisLine_recalculateScore</string> </value>
</item>
<item>
<key> <string>reference</string> </key>
<value> <string>DataAnalysisLine_recalculateScore</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>DataAnalysisLine_recalculateScore</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_wendelin_drone/PathTemplateItem/portal_callables/DataIngestionLine_writeFluentdIngestionToDataBucketStream.py
View file @ 0788d776
......@@ -2,7 +2,7 @@
l=[(c[1]["filepath"].split("/")[-1], c[1]["message"]) for c in context.unpack_lazy(data_chunk, use_list=False)]
names_message_dir = {}
context.log("New data")
for file_name, message in l:
try:
names_message_dir[file_name].append(message)
......
bt5/erp5_wendelin_drone/bt/description
View file @ 0788d776
Ingest simulation drone data, score the simulations and display some relevant information.
\ No newline at end of file
Ingest simulation drone data, score the simulations and display some relevant information.
No Person data is present here.
\ No newline at end of file
bt5/erp5_wendelin_drone/bt/template_keep_path_list
View file @ 0788d776
data_acquisition_unit_module/drone_laptop
data_operation_module/convert_raw_data_to_array
data_operation_module/create_score_list
data_operation_module/ingest_fluent_bucket_data
data_operation_module/recalculate_scores
data_product_module/bouncy_flight_mavsdk
data_product_module/bouncy_flight_mavsdk_raw_data
data_product_module/drone_raw_array
data_product_module/drone_raw_data
data_product_module/drone_simulation_performance_data
data_product_module/recalculated_plots
data_product_module/recalculated_scores
data_product_module/simulation_scores
data_supply_module/drone_real
data_supply_module/drone_real/**
......@@ -18,7 +27,10 @@ data_transformation_module/convert_drone_raw_real_data
data_transformation_module/convert_drone_raw_real_data/**
data_transformation_module/create_score_list_array
data_transformation_module/create_score_list_array/**
data_transformation_module/recalculate-score-list-array
data_transformation_module/recalculate-score-list-array/**
notebook_module/drone_data_graph
portal_callables/DataAnalysisLine_convertDroneDataArrayToScoreArray
portal_callables/DataAnalysisLine_convertRawDroneDataToArray
portal_callables/DataAnalysisLine_recalculateScore
portal_callables/DataIngestionLine_writeFluentdIngestionToDataBucketStream
\ No newline at end of file
bt5/erp5_wendelin_drone/bt/template_path_list
View file @ 0788d776
data_acquisition_unit_module/drone_laptop
data_operation_module/convert_raw_data_to_array
data_operation_module/create_score_list
data_operation_module/ingest_fluent_bucket_data
data_operation_module/recalculate_scores
data_product_module/bouncy_flight_mavsdk
data_product_module/bouncy_flight_mavsdk_raw_data
data_product_module/drone_raw_array
data_product_module/drone_raw_data
data_product_module/drone_simulation_performance_data
data_product_module/recalculated_plots
data_product_module/recalculated_scores
data_product_module/simulation_scores
data_supply_module/drone_real
data_supply_module/drone_real/**
......@@ -18,7 +27,10 @@ data_transformation_module/convert_drone_raw_real_data
data_transformation_module/convert_drone_raw_real_data/**
data_transformation_module/create_score_list_array
data_transformation_module/create_score_list_array/**
data_transformation_module/recalculate-score-list-array
data_transformation_module/recalculate-score-list-array/**
notebook_module/drone_data_graph
portal_callables/DataAnalysisLine_convertDroneDataArrayToScoreArray
portal_callables/DataAnalysisLine_convertRawDroneDataToArray
portal_callables/DataAnalysisLine_recalculateScore
portal_callables/DataIngestionLine_writeFluentdIngestionToDataBucketStream
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7