Skip to content

W
wendelin
Commit 93dfd2fa authored by Léo-Paul Géneau's avatar Léo-Paul Géneau 👾
Browse files
Options

erp5_wendelin_drone: automatically fetch the amount of parameters

parent 477cf068
Hide whitespace changes
Inline Side-by-side
Showing with 183 additions and 389 deletions
bt5/erp5_wendelin_drone/PathTemplateItem/notebook_module/drone_data_graph.xml
View file @ 93dfd2fa
...@@ -116,7 +116,7 @@ The next few section will display some information about the best 5 parameters a ...@@ -116,7 +116,7 @@ The next few section will display some information about the best 5 parameters a
## Table of Contents\n ## Table of Contents\n
- [Flow Chart](#flowchart)\n - [Flow Chart](#flowchart)\n
- [Score Plot](#plot_div5)\n - [Score Plot](#plot_div5)\n
- [Distance Deviation Plot](#plot_div)\n - [Distance Deviation Plot](#plot_div1)\n
- [ASML Deviation Plot](#plot_div2)\n - [ASML Deviation Plot](#plot_div2)\n
- [Ground Speed Deviation Plot](#plot_div3)\n - [Ground Speed Deviation Plot](#plot_div3)\n
- [Climb Rate Deviation Plot](#plot_div4)\n - [Climb Rate Deviation Plot](#plot_div4)\n
...@@ -170,7 +170,7 @@ $$\n ...@@ -170,7 +170,7 @@ $$\n
# Deviation Plot\n # Deviation Plot\n
## Distance/Positional difference\n ## Distance/Positional difference\n
This line plot illustrates the deviation between the simulation data and the ground truth values, focusing on the common timestamps. It\'s important to note that the simulations may end at different times, resulting in a representation of a shorter timeframe for some simulations compared to others.\n This line plot illustrates the deviation between the simulation data and the ground truth values, focusing on the common timestamps. It\'s important to note that the simulations may end at different times, resulting in a representation of a shorter timeframe for some simulations compared to others.\n
<div class="plot_div" id="plot_div"/>\n <div class="plot_div1" id="plot_div1"/>\n
\n \n
%% md\n %% md\n
# Deviation Plot\n # Deviation Plot\n
...@@ -211,7 +211,7 @@ for (transformation of transformation_locations) {\n ...@@ -211,7 +211,7 @@ for (transformation of transformation_locations) {\n
.setAttribute("href", "https://" + window.location.host + transformation[1])\n .setAttribute("href", "https://" + window.location.host + transformation[1])\n
}\n }\n
\n \n
hateoas_url = "https://" + window.location.host + "/erp5/web_site_module/renderjs_runner/hateoas/";\n hateoas_url = "https://" + window.location.host + "/erp5/web_site_module/default_wendelin_front/hateoas/";\n
jio = jIO.createJIO({\n jio = jIO.createJIO({\n
type: "erp5",\n type: "erp5",\n
url: hateoas_url,\n url: hateoas_url,\n
...@@ -253,38 +253,29 @@ function unpack(rows, key) {\n ...@@ -253,38 +253,29 @@ function unpack(rows, key) {\n
\n \n
\n \n
function findNewestIteration(arrayID, rank) {\n function findNewestIteration(arrayID, rank) {\n
return jio.allDocs({\n return jio.allDocs({\n
query: \'portal_type:"Data Array Line" AND \' +\n query: \'portal_type:"Data Array Line" AND \' +\n
\'title: "\' + rank + \'"\'\n \'title: "\' + rank + \'"\'\n
\n })\n
})\n .push(function (result) {\n
.then(function (result) {\n array_id = result.data.rows[0].id; //would be the correct thing, but as we have multiple data arrays with the same name, we need to fix it like this\n
array_id = result.data.rows[0].id; //would be the correct thing, but as we have multiple data arrays with the same name, we need to fix it like this\n return gadget.jio_get(array_id);\n
return gadget.jio_get(array_id)\n });\n
})\n
.then(function (result) {\n
\n
return result.index_expression.split(":")\n
\n
})\n
\n
}\n }\n
\n \n
\n \n
\n \n
\n \n
function getAllData(arrayID, ranks, column_names) {\n function getAllData(arrayID, ranks, column_names) {\n
\n
intervals = []\n intervals = []\n
return RSVP.all(ranks.map((rank) => findNewestIteration(arrayID, rank)))\n return RSVP.all(ranks.map((rank) => findNewestIteration(arrayID, rank)))\n
.then(function (result) {\n .then(function (result) {\n
intervals = []\n intervals = []\n
for (interval of result) {\n for (data_array_line of result) {\n
start_index = interval[0];\n [start_index, stop_index] = data_array_line.index_expression.split(":");\n
stop_index = interval[1];\n
intervals.push([Number(start_index), Number(stop_index)]);\n intervals.push([Number(start_index), Number(stop_index)]);\n
}\n }\n
array_id = arrayID\n array_id = arrayID;\n
return wendelin.getArrayRawSlice(gadget, array_id, intervals[0][0], intervals[4][1]);\n return wendelin.getArrayRawSlice(gadget, array_id, intervals[0][0], intervals[4][1]);\n
})\n })\n
.then(function (result) {\n .then(function (result) {\n
...@@ -297,9 +288,6 @@ function getAllData(arrayID, ranks, column_names) {\n ...@@ -297,9 +288,6 @@ function getAllData(arrayID, ranks, column_names) {\n
table_data = table_data.concat([nj.unpack(result.pick(i, null))])\n table_data = table_data.concat([nj.unpack(result.pick(i, null))])\n
}\n }\n
\n \n
column_names = [\n
"ASML_diff", "ground_speed_diff", "climb_rate_diff", "climb_rate_reciprocal", "timestamp", "score_cosine_row", "score_cosine_column", "distance_reciprocal", "iteration", "score_reciprocal", "ground_speed_reciprocal", "distance_diff", "Param2", "Param1", "ASML_reciprocal"\n
]\n
for (i = 0; i < column_names.length; i += 1) {\n for (i = 0; i < column_names.length; i += 1) {\n
column_data = []\n column_data = []\n
for (row of table_data) {\n for (row of table_data) {\n
...@@ -319,8 +307,7 @@ function getAllData(arrayID, ranks, column_names) {\n ...@@ -319,8 +307,7 @@ function getAllData(arrayID, ranks, column_names) {\n
})\n })\n
}\n }\n
\n \n
\n function createPlotData(arrayData, column_names, label_list, parameters, type) {\n
function createLinePlotData(arrayData, column_names, label_list) {\n
all_trace_templates = []\n all_trace_templates = []\n
const plotlyColors = [\n const plotlyColors = [\n
\'rgb(31, 119, 180)\', // Blue\n \'rgb(31, 119, 180)\', // Blue\n
...@@ -329,82 +316,46 @@ function createLinePlotData(arrayData, column_names, label_list) {\n ...@@ -329,82 +316,46 @@ function createLinePlotData(arrayData, column_names, label_list) {\n
\'rgb(214, 39, 40)\', // Red\n \'rgb(214, 39, 40)\', // Red\n
\'rgb(148, 103, 189)\' // Purple\n \'rgb(148, 103, 189)\' // Purple\n
];\n ];\n
for (i = 0; i < arrayData.length; i = i + 1) {\n arrayData.forEach(function (data) {\n
trace_templates = []\n trace_templates = [];\n
for (j = 0; j < label_list.length; j = j + 1) {\n label_list.forEach(function (label) {\n
if (label_list[j] != "timestamp") {\n if (label != "timestamp") {\n
\n name_string = "The parameters are (" + parameters.join(\', \') + "): " + parameters.map((param_name) => data[column_names.indexOf(param_name)][0]).join(\', \');\n
name_string = "The parameters are (drone_speed, drone_max_roll): " + [arrayData[i][column_names.indexOf("Param1")][0], arrayData[i][column_names.indexOf("Param2")][0]].join(\', \')\n \n
\n trace_template = {\n
trace_template = {\n type: type,\n
type: "scatter",\n name: name_string,\n
name: name_string,\n x: data[column_names.indexOf("timestamp")],\n
x: arrayData[i][column_names.indexOf("timestamp")],\n y: data[column_names.indexOf(label)],\n
y: arrayData[i][column_names.indexOf(label_list[j])],\n line: {\n
line: {\n color: plotlyColors[i]\n
color: plotlyColors[i]\n }\n
}\n }\n
}\n trace_templates = trace_templates.concat(trace_template);\n
trace_templates = trace_templates.concat(trace_template);\n
}\n
}\n }\n
all_trace_templates.push(trace_template)\n });\n
}\n all_trace_templates.push(trace_template)\n
});\n
\n \n
return all_trace_templates;\n return all_trace_templates;\n
\n
}\n }\n
\n \n
\n function createLinePlotData(arrayData, column_names, label_list, parameters) {\n
\n return createPlotData(arrayData, column_names, label_list, parameters, "scatter");\n
\n
function createBarPlotData(arrayData, column_names, label_list) {\n
all_trace_templates = []\n
const plotlyColors = [\n
\'rgb(31, 119, 180)\', // Blue\n
\'rgb(255, 127, 14)\', // Orange\n
\'rgb(44, 160, 44)\', // Green\n
\'rgb(214, 39, 40)\', // Red\n
\'rgb(148, 103, 189)\' // Purple\n
];\n
for (i = 0; i < arrayData.length; i = i + 1) {\n
trace_templates = []\n
for (j = 0; j < label_list.length; j = j + 1) {\n
if (label_list[j] != "timestamp") {\n
\n
name_string = "The parameters are (drone_speed, drone_max_roll): " + [arrayData[i][column_names.indexOf("Param1")][0], arrayData[i][column_names.indexOf("Param2")][0]].join(\', \')\n
\n
trace_template = {\n
type: "bar", // Set type to "bar" for a bar plot\n
name: name_string,\n
x: ["Value"], //arrayData[i][column_names.indexOf("timestamp")],\n
y: [arrayData[i][column_names.indexOf(label_list[j])][0]],\n
marker: {\n
color: plotlyColors[i]\n
}\n
};\n
trace_templates = trace_templates.concat(trace_template);\n
}\n
}\n
all_trace_templates.push(trace_template)\n
}\n
\n
return all_trace_templates;\n
\n
}\n }\n
\n \n
\n function createBarPlotData(arrayData, column_names, label_list, parameters) {\n
\n return createPlotData(arrayData, column_names, label_list, parameters, "bar");\n
}\n
\n \n
function getColumnNames(arrayID) {\n function getColumnNames(arrayID) {\n
\n return gadget.jio_get(arrayID)\n
return gadget.jio_get(arrayID)\n .push(function (result) {\n
.then(function (result) {\n column_names = result.array_dtype.match(/\\(\'(.*?)\',/g);\n
column_names = result.array_dtype.match(/\\(\'(.*?)\',/g);\n column_names = column_names.map(match => match.match(/\\(\'(.*?)\',/));\n
column_names = column_names.map(match => match.match(/\\(\'(.*?)\',/))\n column_names = column_names.map(thing => thing[1]);\n
column_names = column_names.map(thing => thing[1]);\n return column_names;\n
return column_names\n });\n
})\n
}\n }\n
\n \n
function plot(data, plotTitle = null, graph = null, layout = null) {\n function plot(data, plotTitle = null, graph = null, layout = null) {\n
...@@ -424,36 +375,28 @@ function plot(data, plotTitle = null, graph = null, layout = null) {\n ...@@ -424,36 +375,28 @@ function plot(data, plotTitle = null, graph = null, layout = null) {\n
}\n }\n
\n \n
function prepareData() {\n function prepareData() {\n
return jio.allDocs({\n return jio.allDocs({\n
query: \'portal_type:"Data Analysis Line" AND \' +\n query: \'portal_type:"Data Analysis Line" AND \' +\n
\'title: "Recalculated Plots" AND \' +\n \'title: "Recalculated Plots" AND \' +\n
\'simulation_state:"started"\'\n \'simulation_state:"started"\'\n
})\n })\n
.then(function (result) {\n .push(function (result) {\n
\n var data_analysis_line_id = result.data.rows[0].id;\n
var data_analysis_line_id = result.data.rows[0].id;\n return jio.allDocs({\n
return jio.allDocs({\n query: \'portal_type:"Data Array" AND \' +\n
query: \'portal_type:"Data Array" AND \' +\n \'aggregate_related_relative_url:"\' + data_analysis_line_id + \'"\'\n
\'aggregate_related_relative_url:"\' + data_analysis_line_id + \'"\'\n });\n
});\n })\n
})\n .push(function (result) {\n
.then(function (result) {\n return RSVP.all([getColumnNames(result.data.rows[0].id), result.data.rows[0].id]);\n
\n })\n
column_names = getColumnNames(result.data.rows[0].id)\n .push(function (result) {\n
return [column_names, result.data.rows[0].id]\n return RSVP.all([result[0], getAllData(result[1], [1, 2, 3, 4, 5], result[0])]);\n
})\n });\n
.then(function (result) {\n
\n
return RSVP.all([result[0], getAllData(result[1], [1, 2, 3, 4, 5], result[0])])\n
})\n
}\n }\n
\n \n
\n
\n
\n
prepareData()\n prepareData()\n
.then(function (result) {\n .then(function (result) {\n
\n
const layout = {\n const layout = {\n
barmode: \'group\',\n barmode: \'group\',\n
title: \'Bar Chart\',\n title: \'Bar Chart\',\n
...@@ -465,31 +408,24 @@ prepareData()\n ...@@ -465,31 +408,24 @@ prepareData()\n
}\n }\n
};\n };\n
\n \n
graph = document.getElementById(\'plot_div\')\n const fixed_column_names = [\n
data = createLinePlotData(result[1], result[0], ["distance_diff"])\n "ASML_diff", "ground_speed_diff", "climb_rate_diff",\n
plot(data, "", graph, null)\n "climb_rate_reciprocal", "timestamp", "distance_reciprocal",\n
\n "score_cosine_row", "score_cosine_column", "ground_speed_reciprocal",\n
graph = document.getElementById(\'plot_div2\')\n "distance_diff", "score_reciprocal", "ASML_reciprocal", "name",\n
data = createLinePlotData(result[1], result[0], ["ASML_diff"])\n "iteration"\n
plot(data, "", graph, null)\n ];\n
\n const parameters = column_names.filter(n => !fixed_column_names.includes(n));\n
\n \n
graph = document.getElementById(\'plot_div3\')\n ["distance_diff", "ASML_diff", "ground_speed_diff", "climb_rate_diff"].forEach(function (label, index) {\n
data = createLinePlotData(result[1], result[0], ["ground_speed_diff"])\n plot(createLinePlotData(result[1], result[0], [label], parameters),\n
plot(data, "", graph, null)\n "", document.getElementById(\'plot_div\' + (index + 1)), null);\n
\n });\n
\n \n
graph = document.getElementById(\'plot_div4\')\n graph = document.getElementById(\'plot_div5\');\n
data = createLinePlotData(result[1], result[0], ["climb_rate_diff"])\n data = createBarPlotData(result[1], result[0], ["score_reciprocal"], parameters);\n
plot(data, "", graph, null)\n plot(data, "", graph, layout);\n
\n });\n
\n
graph = document.getElementById(\'plot_div5\')\n
data = createBarPlotData(result[1], result[0], ["score_reciprocal"])\n
plot(data, "", graph, layout)\n
\n
})\n
\n
]]></string> </value> ]]></string> </value>
...@@ -560,7 +496,7 @@ prepareData()\n ...@@ -560,7 +496,7 @@ prepareData()\n
</item> </item>
<item> <item>
<key> <string>serial</string> </key> <key> <string>serial</string> </key>
<value> <string>1013.42930.28344.22254</string> </value> <value> <string>1019.57407.12828.21128</string> </value>
</item> </item>
<item> <item>
<key> <string>state</string> </key> <key> <string>state</string> </key>
...@@ -580,7 +516,7 @@ prepareData()\n ...@@ -580,7 +516,7 @@ prepareData()\n
</tuple> </tuple>
<state> <state>
<tuple> <tuple>
<float>1704461947.15</float> <float>1728491943.46</float>
<string>UTC</string> <string>UTC</string>
</tuple> </tuple>
</state> </state>
......
bt5/erp5_wendelin_drone/PathTemplateItem/portal_callables/DataAnalysisLine_convertDroneDataArrayToScoreArray.py
View file @ 93dfd2fa
...@@ -74,32 +74,6 @@ seen_sims = progress_indicator_sim.getStringOffsetIndex() ...@@ -74,32 +74,6 @@ seen_sims = progress_indicator_sim.getStringOffsetIndex()
new_seen = "" new_seen = ""
if seen_sims is None: if seen_sims is None:
seen_sims = "" seen_sims = ""
score_dtypes = {'name': 'S256', 'Param1': 'f16', 'Param2': 'f16',
'distance_reciprocal': 'f8', 'ASML_reciprocal': 'f8',
'ground_speed_reciprocal': 'f8', 'climb_rate_reciprocal': 'f8',
'score_reciprocal': 'f16', 'score_cosine_row': 'f16',
'score_cosine_column': 'f16'}
_ = out_array_scores["Data Array"].initArray(shape=(0,), dtype=list(score_dtypes.items()))
plot_dtypes = {
'name': 'S256',
'Param1': 'f8',
'Param2': 'f8',
'timestamp': 'f8',
'distance_diff': 'f8',
'ASML_diff': 'f8',
'ground_speed_diff': 'f8',
'climb_rate_diff': 'f8',
'distance_reciprocal': 'f8',
'ASML_reciprocal': 'f8',
'ground_speed_reciprocal': 'f8',
'climb_rate_reciprocal': 'f8',
'score_reciprocal': 'f16',
'score_cosine_row': 'f16',
'score_cosine_column': 'f16'
}
_ = out_array_plot_data["Data Array"].initArray(shape=(0,), dtype=list(plot_dtypes.items()))
simulated_flight_list = [] simulated_flight_list = []
simulated_flights_value_dict_list = [] simulated_flights_value_dict_list = []
...@@ -135,27 +109,28 @@ for name in not_seen_list: ...@@ -135,27 +109,28 @@ for name in not_seen_list:
break break
try: try:
under_timestamp = simulated_flight[simulated_flight['timestamp (ms)'] - timestamp_diff < row['timestamp (ms)']].tail(1) under_timestamp = simulated_flight[simulated_flight['timestamp (ms)'] - timestamp_diff < row['timestamp (ms)']].tail(1).to_dict('records')[0]
except IndexError: except IndexError:
continue continue
over_timestamp = simulated_flight[simulated_flight['timestamp (ms)'] - timestamp_diff >= row['timestamp (ms)']].head(1) over_timestamp = simulated_flight[simulated_flight['timestamp (ms)'] - timestamp_diff >= row['timestamp (ms)']].head(1).to_dict('records')[0]
rate = (over_timestamp["timestamp (ms)"] - timestamp_diff - row["timestamp (ms)"])/(over_timestamp["timestamp (ms)"] - under_timestamp["timestamp (ms)"]) rate = (over_timestamp['timestamp (ms)'] - timestamp_diff - row['timestamp (ms)'])/(over_timestamp['timestamp (ms)'] - under_timestamp['timestamp (ms)'])
for criterion in distance_criterion_tuple: for criterion in distance_criterion_tuple:
tmp_sim[criterion].append(rated_value(over_timestamp[criterion], under_timestamp[criterion], rate)) tmp_sim[criterion].append(rated_value(over_timestamp[criterion], under_timestamp[criterion], rate))
tmp_real[criterion].append(row[criterion]) tmp_real[criterion].append(row[criterion])
for index, value in enumerate(( for index, value in enumerate((
row['timestamp (ms)'] / 1000, row['timestamp (ms)'] / 1000,
distance(row['latitude ()'], distance(row['latitude ()'],
row['longitude ()'], row['longitude ()'],
tmp_sim['latitude ()'][-1], tmp_sim['latitude ()'][-1],
tmp_sim['longitude ()'][-1], tmp_sim['longitude ()'][-1],
), ),
tmp_sim['AMSL (m)'][-1], tmp_sim['AMSL (m)'][-1],
tmp_sim['ground speed (m/s)'][-1], tmp_sim['ground speed (m/s)'][-1],
tmp_real['climb rate (m/s)'][-1], tmp_real['climb rate (m/s)'][-1]
)): )):
distance_list_tuple[index].append(value) distance_list_tuple[index].append(value)
# If we have at least 100 entries (timestamps) analysed, add the filename to the list # If we have at least 100 entries (timestamps) analysed, add the filename to the list
...@@ -163,51 +138,48 @@ for name in not_seen_list: ...@@ -163,51 +138,48 @@ for name in not_seen_list:
if len(distance_list_tuple[0]) > 0: if len(distance_list_tuple[0]) > 0:
# Add it to the dictionary # Add it to the dictionary
# A list of simulations, and their values over time # A list of simulations, and their values over time
pattern = r'\((.*?)\)' parameters = {k: float(v) for _, k, v in re.findall(r'[_-](([a-zA-Z_]+)_(\d+))', name)}
match = re.search(pattern, name) if parameters:
if match: reciprocal_of_difference = [1/(1+mean_operation(list(map(abs, x)))) for x in distance_list_tuple[1:]]
values_str = match.group(1) score_reciprocal = sum(reciprocal_of_difference)
parameters = [float(val.strip()) for val in values_str.split(',')]
tmp_df_sim = pd.DataFrame(tmp_sim)
reciprocal_of_difference = [1/(1+mean_operation(list(map(abs, x)))) for x in distance_list_tuple[1:]] tmp_df_real = pd.DataFrame(tmp_real)
score_reciprocal = sum(reciprocal_of_difference) # Uncomment once you can actually use np..norm
score_cosine_row = mean_cosine_similarity_custom(tmp_df_real, tmp_df_sim)# cosine_similarity_rows(tmp_df_real, tmp_df_sim).mean()
tmp_df_sim = pd.DataFrame(tmp_sim) score_cosine_column = mean_cosine_similarity_custom(tmp_df_real.T, tmp_df_sim.T)#cosine_similarity_rows(tmp_df_real.T, tmp_df_sim.T).mean()
tmp_df_real = pd.DataFrame(tmp_real)
# Uncomment once you can actually use np..norm flights_value_dict = {
score_cosine_row =mean_cosine_similarity_custom(tmp_df_real, tmp_df_sim)# cosine_similarity_rows(tmp_df_real, tmp_df_sim).mean() "name": name,
score_cosine_column = mean_cosine_similarity_custom(tmp_df_real.T, tmp_df_sim.T)#cosine_similarity_rows(tmp_df_real.T, tmp_df_sim.T).mean() "timestamp": distance_list_tuple[0],
"distance_diff": distance_list_tuple[1],
simulated_flights_value_dict_list.append({ "ASML_diff" : distance_list_tuple[2],
"name": name, "ground_speed_diff" : distance_list_tuple[3],
"Param1" : parameters[0], "climb_rate_diff" : distance_list_tuple[4],
"Param2": parameters[1], "distance_reciprocal": reciprocal_of_difference[0],
"timestamp": distance_list_tuple[0], "ASML_reciprocal" : reciprocal_of_difference[1],
"distance_diff": distance_list_tuple[1], "ground_speed_reciprocal" : reciprocal_of_difference[2],
"ASML_diff" : distance_list_tuple[2], "climb_rate_reciprocal" : reciprocal_of_difference[3],
"ground_speed_diff" : distance_list_tuple[3], "score_reciprocal": score_reciprocal/4,
"climb_rate_diff" : distance_list_tuple[4], "score_cosine_row" : score_cosine_row,
"distance_reciprocal": reciprocal_of_difference[0], "score_cosine_column" : score_cosine_column,
"ASML_reciprocal" : reciprocal_of_difference[1], }
"ground_speed_reciprocal" : reciprocal_of_difference[2],
"climb_rate_reciprocal" : reciprocal_of_difference[3], simulation_data_dict = {
"score_reciprocal": score_reciprocal/4, "name": name,
"score_cosine_row" : score_cosine_row, "distance_reciprocal": reciprocal_of_difference[0],
"score_cosine_column" : score_cosine_column "ASML_reciprocal" : reciprocal_of_difference[1],
}) "ground_speed_reciprocal" : reciprocal_of_difference[2],
"climb_rate_reciprocal" : reciprocal_of_difference[3],
selected_simulation_data_dict_list.append({ "score_reciprocal": score_reciprocal/4,
"name": name, "score_cosine_row" : score_cosine_row,
"Param1" : parameters[0], }
"Param2": parameters[1],
"distance_reciprocal": reciprocal_of_difference[0], for flight_parameter, value in parameters.items():
"ASML_reciprocal" : reciprocal_of_difference[1], flights_value_dict[flight_parameter] = value
"ground_speed_reciprocal" : reciprocal_of_difference[2], simulation_data_dict[flight_parameter] = value
"climb_rate_reciprocal" : reciprocal_of_difference[3], simulated_flights_value_dict_list.append(flights_value_dict)
"score_reciprocal": score_reciprocal/4, selected_simulation_data_dict_list.append(simulation_data_dict)
"score_cosine_row" : score_cosine_row,
"score_cosine_column" : score_cosine_column
})
combined_data = pd.DataFrame() # Initialize an empty DataFrame combined_data = pd.DataFrame() # Initialize an empty DataFrame
...@@ -217,26 +189,7 @@ for data_dict in simulated_flights_value_dict_list: ...@@ -217,26 +189,7 @@ for data_dict in simulated_flights_value_dict_list:
tmp = pd.DataFrame(data_dict) # Wrap data_dict in a list and create a DataFrame tmp = pd.DataFrame(data_dict) # Wrap data_dict in a list and create a DataFrame
combined_data = pd.concat([combined_data, tmp], ignore_index=True) combined_data = pd.concat([combined_data, tmp], ignore_index=True)
plot_dtypes = {k: 'float64' if k != 'name' else 'S256' for k in simulated_flights_value_dict_list[0]}
plot_dtypes = {
'name': 'S256',
'Param1': 'float64',
'Param2': 'float64',
'timestamp': 'float64',
'distance_diff': 'float64',
'ASML_diff': 'float64',
'ground_speed_diff': 'float64',
'climb_rate_diff': 'float64',
'distance_reciprocal': 'float64',
'ASML_reciprocal': 'float64',
'ground_speed_reciprocal': 'float64',
'climb_rate_reciprocal': 'float64',
'score_reciprocal': 'float64',
'score_cosine_row': 'float64',
'score_cosine_column': 'float64',
}
plots_df = combined_data plots_df = combined_data
if plots_df.empty: if plots_df.empty:
...@@ -261,11 +214,7 @@ for data_dict in selected_simulation_data_dict_list: ...@@ -261,11 +214,7 @@ for data_dict in selected_simulation_data_dict_list:
score_dtypes = {'name': 'S256', 'Param1': 'float64', 'Param2': 'float64', score_dtypes = {k: 'float64' if k != 'name' else 'S256' for k in selected_simulation_data_dict_list[0]}
'distance_reciprocal': 'float64', 'ASML_reciprocal': 'float64',
'ground_speed_reciprocal': 'float64', 'climb_rate_reciprocal': 'float64',
'score_reciprocal': 'float64', 'score_cosine_row': 'float64',
'score_cosine_column': 'float64'}
scores_df = pd.DataFrame(combined_data) scores_df = pd.DataFrame(combined_data)
scores_df = scores_df.astype(score_dtypes) scores_df = scores_df.astype(score_dtypes)
############################### ###############################
...@@ -273,6 +222,9 @@ scores_df = scores_df.astype(score_dtypes) ...@@ -273,6 +222,9 @@ scores_df = scores_df.astype(score_dtypes)
zbigarray_scores = out_array_scores["Data Array"].getArray() zbigarray_scores = out_array_scores["Data Array"].getArray()
if zbigarray_scores is None:
out_array_scores["Data Array"].initArray(shape=(0,), dtype=list(score_dtypes.items()))
out_array_plot_data["Data Array"].initArray(shape=(0,), dtype=list(plot_dtypes.items()))
score_names = list(out_array_scores["Data Array"]) score_names = list(out_array_scores["Data Array"])
...@@ -281,7 +233,7 @@ score_names = list(out_array_scores["Data Array"]) ...@@ -281,7 +233,7 @@ score_names = list(out_array_scores["Data Array"])
scores_ndarray = scores_df.to_records(index = False) scores_ndarray = scores_df.to_records(index = False)
scores_ndarray = scores_ndarray.astype([("name", "S256"),("Param1", "f8"), ("Param2", "f8"), ("distance_reciprocal", "f8"), ("ASML_reciprocal", "f8"), ("ground_speed_reciprocal", "f8"), ('climb_rate_reciprocal', 'f8'), ("score_reciprocal", "f8"), ("score_cosine_column", "f8"), ("score_cosine_row", "f8")]) scores_ndarray = scores_ndarray.astype([(k, 'f8' if k != 'name' else 'S256') for k in score_dtypes])
if zbigarray_scores is None: if zbigarray_scores is None:
...@@ -330,7 +282,7 @@ zbigarray_plots = out_array_plot_data["Data Array"].getArray() ...@@ -330,7 +282,7 @@ zbigarray_plots = out_array_plot_data["Data Array"].getArray()
plots_ndarray = plots_df.to_records(index = False) plots_ndarray = plots_df.to_records(index = False)
plots_ndarray = plots_ndarray.astype([('name', 'S256'), ('Param1', 'f8'), ('Param2', 'f8'), ('timestamp', 'f8'), ('distance_diff', 'f8'), ('ASML_diff', 'f8'), ('ground_speed_diff', 'f8'), ('climb_rate_diff', 'f8'), ('distance_reciprocal', 'f8'), ('ASML_reciprocal', 'f8'), ('ground_speed_reciprocal', 'f8'), ('climb_rate_reciprocal', 'f8'), ('score_reciprocal', 'f8'), ('score_cosine_row', 'f8'), ('score_cosine_column', 'f8')]) plots_ndarray = plots_ndarray.astype([(k, 'f8' if k != 'name' else 'S256') for k in plot_dtypes])
if zbigarray_plots is None: if zbigarray_plots is None:
zbigarray_plots = out_array_plot_data["Data Array"].initArray(shape=(0,), dtype=plots_ndarray.dtype.fields) zbigarray_plots = out_array_plot_data["Data Array"].initArray(shape=(0,), dtype=plots_ndarray.dtype.fields)
......
bt5/erp5_wendelin_drone/PathTemplateItem/portal_callables/DataAnalysisLine_recalculateScore.py
View file @ 93dfd2fa
...@@ -8,148 +8,64 @@ import re ...@@ -8,148 +8,64 @@ import re
# Now, to save space, we just overwrite the old array with the new one, that contains the new scores # Now, to save space, we just overwrite the old array with the new one, that contains the new scores
# This new array will give us the final overview of the ranking of the parameters. This can be used by our genetic algorithm to decide when we can stop and which parameters worked the best. # This new array will give us the final overview of the ranking of the parameters. This can be used by our genetic algorithm to decide when we can stop and which parameters worked the best.
score_array = input_array_scores['Data Array']
score_dtypes = {'name': 'S256', 'Param1': 'f8', 'Param2': 'f8',
'distance_reciprocal': 'f8', 'ASML_reciprocal': 'f8',
'ground_speed_reciprocal': 'f8', 'climb_rate_reciprocal': 'f8',
'score_reciprocal': 'f8', 'score_cosine_row': 'f8',
'score_cosine_column': 'f8'}
new_score_dtypes= {'name': 'S256', 'Param1': 'f8', 'Param2': 'f8',
'distance_reciprocal': 'f8', 'ASML_reciprocal': 'f8',
'ground_speed_reciprocal': 'f8', 'climb_rate_reciprocal': 'f8',
'score_reciprocal': 'f8', 'score_cosine_row': 'f8',
'score_cosine_column': 'f8',
'iteration': 'i8'}
plot_dtypes = {
'name': 'S256',
'Param1': 'f8',
'Param2': 'f8',
'timestamp': 'f8',
'distance_diff': 'f8',
'ASML_diff': 'f8',
'ground_speed_diff': 'f8',
'climb_rate_diff': 'f8',
'distance_reciprocal': 'f8',
'ASML_reciprocal': 'f8',
'ground_speed_reciprocal': 'f8',
'climb_rate_reciprocal': 'f8',
'score_reciprocal': 'f8',
'score_cosine_row': 'f8',
'score_cosine_column': 'f8',
'iteration': 'i8'
}
new_plot_dtypes = {
'Param1': 'f8',
'Param2': 'f8',
'timestamp': 'f8',
'distance_diff': 'f8',
'ASML_diff': 'f8',
'ground_speed_diff': 'f8',
'climb_rate_diff': 'f8',
'distance_reciprocal': 'f8',
'ASML_reciprocal': 'f8',
'ground_speed_reciprocal': 'f8',
'climb_rate_reciprocal': 'f8',
'score_reciprocal': 'f8',
'score_cosine_row': 'f8',
'score_cosine_column': 'f8',
'iteration': 'i8'
}
score_array = input_array_scores["Data Array"]
new_score_array = out_array_scores["Data Array"]
plot_array = input_array_plot["Data Array"]
new_plot_array = out_array_plots["Data Array"]
# Should only look at the newest few
score_nparray = score_array.getArray() score_nparray = score_array.getArray()
if score_nparray is None: if score_nparray is None:
score_nparray = out_array_scores["Data Array"].initArray(shape=(0,), dtype=list(new_score_dtypes.items())) return
plot_array = input_array_plot['Data Array']
plot_nparray = plot_array.getArray() plot_nparray = plot_array.getArray()
if plot_nparray is None: if plot_nparray is None:
plot_nparray = out_array_plots["Data Array"].initArray(shape=(0,), dtype=list(new_plot_dtypes.items())) return
old_score_df = pd.DataFrame.from_records(score_nparray[:].copy())
old_plot_df = pd.DataFrame.from_records(plot_nparray[:].copy())
progress_indicator = input_array_scores["Progress Indicator"]
seen_sims = progress_indicator.getStringOffsetIndex()
if seen_sims is None:
seen_sims = ""
sim_flight_names = list(old_score_df["name"])
# We will only continue if there is new data available.
if len([x for x in sim_flight_names if x not in seen_sims]) == 0:
return
new_score_dtypes = [(k, v[0]) for k, v in dict(score_nparray.dtype.fields).items()]
new_score_dtypes.append(('iteration', 'i8'))
new_score_array = out_array_scores['Data Array']
new_score_nparray = new_score_array.getArray() new_score_nparray = new_score_array.getArray()
if new_score_nparray is None: if new_score_nparray is None:
new_score_nparray = out_array_scores["Data Array"].initArray(shape=(0,), dtype=list(new_score_dtypes.items())) new_score_array.initArray(shape=(0,), dtype=new_score_dtypes)
new_plot_dtypes = [(k, v[0]) for k, v in dict(plot_nparray.dtype.fields).items()]
new_plot_dtypes.append(('iteration', 'i8'))
new_plot_array = out_array_plots['Data Array']
new_plot_nparray = new_plot_array.getArray() new_plot_nparray = new_plot_array.getArray()
if new_plot_nparray is None: if new_plot_nparray is None:
new_plot_nparray = out_array_plots["Data Array"].initArray(shape=(0,), dtype=list(new_plot_dtypes.items())) new_plot_array.initArray(shape=(0,), dtype=new_plot_dtypes)
new_score_df = pd.DataFrame.from_records(new_score_nparray[:].copy())
new_plot_df = pd.DataFrame.from_records(new_plot_nparray[:].copy())
old_score_df = pd.DataFrame.from_records(score_nparray[:].copy())
old_plot_df = pd.DataFrame.from_records(plot_nparray[:].copy())
progress_indicator = input_array_scores['Progress Indicator']
seen_sims = progress_indicator.getStringOffsetIndex()
if seen_sims is None:
seen_sims = ""
sim_flight_names = list(old_score_df['name'])
# We will only continue if there is new data available.
if len([x for x in sim_flight_names if x not in seen_sims]) == 0:
return
new_score_df = pd.DataFrame.from_records(new_score_nparray[:].copy())
new_plot_df = pd.DataFrame.from_records(new_plot_nparray[:].copy())
new_score_iteration = new_score_df["iteration"].max() new_score_iteration = new_score_df['iteration'].max()
if math.isnan(new_score_iteration): if math.isnan(new_score_iteration):
new_score_iteration = 0 new_score_iteration = 0
new_score_df = new_score_df.drop(['iteration'], axis=1)
new_score_df = new_score_df.drop(["iteration"],axis=1)
new_plot_iteration = new_plot_df["iteration"].max() new_plot_iteration = new_plot_df["iteration"].max()
if math.isnan(new_plot_iteration): if math.isnan(new_plot_iteration):
new_plot_iteration = 0 new_plot_iteration = 0
#new_plot_df = new_plot_df.drop(columns=["iteration"]) param_subset = [k for k in old_score_df.dtypes.keys().values if not 'reciprocal' in k and not 'score' in k and k != 'name']
answer_scores = pd.concat([old_score_df, new_score_df]).drop_duplicates(subset=param_subset, keep='last')
answer_plots = pd.concat([old_plot_df, new_plot_df]).drop_duplicates(subset=param_subset, keep='last')
answer_scores = pd.concat([old_score_df, new_score_df]).drop_duplicates(subset=['Param1','Param2'], keep='last')
answer_plots = pd.concat([old_plot_df, new_plot_df]).drop_duplicates(subset=['Param1','Param2'], keep='last')
answer_scores["score_reciprocal"] = answer_scores["score_reciprocal"].astype('float64')
answer_scores = answer_scores.nlargest(5, "score_reciprocal")
answer_plots = old_plot_df[old_plot_df["name"].isin(list(answer_scores["name"]))]
answer_scores['score_reciprocal'] = answer_scores['score_reciprocal'].astype('float64')
answer_scores = answer_scores.nlargest(5, 'score_reciprocal')
answer_plots = old_plot_df[old_plot_df['name'].isin(list(answer_scores['name']))]
new_score_iteration = new_score_iteration + 1 new_score_iteration = new_score_iteration + 1
new_plot_iteration = new_plot_iteration + 1 new_plot_iteration = new_plot_iteration + 1
...@@ -158,24 +74,14 @@ answer_scores["iteration"] = new_score_iteration ...@@ -158,24 +74,14 @@ answer_scores["iteration"] = new_score_iteration
answer_plots["iteration"] = new_plot_iteration answer_plots["iteration"] = new_plot_iteration
# We will remove all the data from the data arrays before we append the new data. Essentially we will be left with only the best few iterations # We will remove all the data from the data arrays before we append the new data. Essentially we will be left with only the best few iterations
new_score_nparray = out_array_scores["Data Array"].initArray(shape=(0,), dtype=list(new_score_dtypes.items())) new_score_nparray = new_score_array.initArray(shape=(0,), dtype=new_score_dtypes)
new_plot_nparray = new_plot_array.initArray(shape=(0,), dtype=new_plot_dtypes)
new_plot_nparray = out_array_plots["Data Array"].initArray(shape=(0,), dtype=list(new_plot_dtypes.items()))
new_score_nparray.append(answer_scores.to_records(index = False)) new_score_nparray.append(answer_scores.to_records(index = False))
# Group the DataFrame by the 'name' column # Group the DataFrame by the 'name' column
grouped = answer_plots.groupby(['Param1', 'Param2']) grouped = answer_plots.groupby(param_subset)
# Initialize a list to store the resulting DataFrames # Initialize a list to store the resulting DataFrames
resulting_dfs = [] resulting_dfs = []
...@@ -209,5 +115,5 @@ for df in resulting_dfs: ...@@ -209,5 +115,5 @@ for df in resulting_dfs:
progress_indicator.setStringOffsetIndex(sim_flight_names) progress_indicator.setStringOffsetIndex(sim_flight_names)
return return
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7