Skip to content

W
wendelin
Commit d17f59dd authored by Léo-Paul Géneau's avatar Léo-Paul Géneau 👾
Browse files
Options

erp5_wendelin_drone: tmp

parent 945ab3c9
Hide whitespace changes
Inline Side-by-side
Showing with 110 additions and 194 deletions
bt5/erp5_wendelin_drone/PathTemplateItem/notebook_module/drone_data_graph.xml
View file @ d17f59dd
......@@ -211,7 +211,7 @@ for (transformation of transformation_locations) {\n
.setAttribute("href", "https://" + window.location.host + transformation[1])\n
}\n
\n
hateoas_url = "https://" + window.location.host + "/erp5/web_site_module/renderjs_runner/hateoas/";\n
hateoas_url = "https://" + window.location.host + "/erp5/web_site_module/default_wendelin_front/hateoas/";\n
jio = jIO.createJIO({\n
type: "erp5",\n
url: hateoas_url,\n
......@@ -253,21 +253,21 @@ function unpack(rows, key) {\n
\n
\n
function findNewestIteration(arrayID, rank) {\n
return jio.allDocs({\n
query: \'portal_type:"Data Array Line" AND \' +\n
\'title: "\' + rank + \'"\'\n
\n
})\n
.then(function (result) {\n
array_id = result.data.rows[0].id; //would be the correct thing, but as we have multiple data arrays with the same name, we need to fix it like this\n
return gadget.jio_get(array_id)\n
})\n
.then(function (result) {\n
\n
return result.index_expression.split(":")\n
\n
})\n
\n
return jio.allDocs({\n
query: \'portal_type:"Data Array Line" AND \' +\n
\'title: "\' + rank + \'"\'\n
})\n
.push(function (result) {\n
array_id = result.data.rows[0].id; //would be the correct thing, but as we have multiple data arrays with the same name, we need to fix it like this\n
return gadget.jio_get(array_id);\n
}, function (error) {\n
throw error;\n
})\n
.push(function (result) {\n
return result.index_expression.split(":");\n
}, function (error) {\n
throw error;\n
});\n
}\n
\n
\n
......@@ -333,7 +333,6 @@ function createLinePlotData(arrayData, column_names, label_list) {\n
trace_templates = []\n
for (j = 0; j < label_list.length; j = j + 1) {\n
if (label_list[j] != "timestamp") {\n
\n
name_string = "The parameters are (drone_speed, drone_max_roll): " + [arrayData[i][column_names.indexOf("Param1")][0], arrayData[i][column_names.indexOf("Param2")][0]].join(\', \')\n
\n
trace_template = {\n
......@@ -397,14 +396,22 @@ function createBarPlotData(arrayData, column_names, label_list) {\n
\n
\n
function getColumnNames(arrayID) {\n
\n
return gadget.jio_get(arrayID)\n
.then(function (result) {\n
column_names = result.array_dtype.match(/\\(\'(.*?)\',/g);\n
column_names = column_names.map(match => match.match(/\\(\'(.*?)\',/))\n
column_names = column_names.map(thing => thing[1]);\n
return column_names\n
})\n
jio.get(arrayID)\n
.push(function (result) {\n
console.log("got", result);\n
}, function (error) {\n
throw error;\n
});\n
return gadget.jio_get(arrayID)\n
.push(function (result) {\n
console.log("getColumnName", result);\n
column_names = result.array_dtype.match(/\\(\'(.*?)\',/g);\n
column_names = column_names.map(match => match.match(/\\(\'(.*?)\',/))\n
column_names = column_names.map(thing => thing[1]);\n
return column_names;\n
}, function (error) {\n
throw error;\n
});\n
}\n
\n
function plot(data, plotTitle = null, graph = null, layout = null) {\n
......@@ -424,28 +431,29 @@ function plot(data, plotTitle = null, graph = null, layout = null) {\n
}\n
\n
function prepareData() {\n
return jio.allDocs({\n
query: \'portal_type:"Data Analysis Line" AND \' +\n
\'title: "Recalculated Plots" AND \' +\n
\'simulation_state:"started"\'\n
})\n
.then(function (result) {\n
\n
var data_analysis_line_id = result.data.rows[0].id;\n
return jio.allDocs({\n
query: \'portal_type:"Data Array" AND \' +\n
\'aggregate_related_relative_url:"\' + data_analysis_line_id + \'"\'\n
});\n
})\n
.then(function (result) {\n
\n
column_names = getColumnNames(result.data.rows[0].id)\n
return [column_names, result.data.rows[0].id]\n
})\n
.then(function (result) {\n
\n
return RSVP.all([result[0], getAllData(result[1], [1, 2, 3, 4, 5], result[0])])\n
})\n
return jio.allDocs({\n
query: \'portal_type:"Data Analysis Line" AND \' +\n
\'title: "Recalculated Plots" AND \' +\n
\'simulation_state:"started"\'\n
})\n
.push(function (result) {\n
var data_analysis_line_id = result.data.rows[0].id;\n
return jio.allDocs({\n
query: \'portal_type:"Data Array" AND \' +\n
\'aggregate_related_relative_url:"\' + data_analysis_line_id + \'"\'\n
});\n
}, function (error) {\n
throw error;\n
})\n
.push(function (result) {\n
column_names = getColumnNames(result.data.rows[0].id)\n
return [column_names, result.data.rows[0].id];\n
}, function (error) {\n
throw error;\n
})\n
.push(function (result) {\n
return RSVP.all([result[0], getAllData(result[1], [1, 2, 3, 4, 5], result[0])])\n
});\n
}\n
\n
\n
......@@ -560,7 +568,7 @@ prepareData()\n
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>1013.42930.28344.22254</string> </value>
<value> <string>1019.15648.51729.36078</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -580,7 +588,7 @@ prepareData()\n
</tuple>
<state>
<tuple>
<float>1704461947.15</float>
<float>1726072547.36</float>
<string>UTC</string>
</tuple>
</state>
......
bt5/erp5_wendelin_drone/PathTemplateItem/portal_callables/DataAnalysisLine_convertDroneDataArrayToScoreArray.py
View file @ d17f59dd
......@@ -109,27 +109,28 @@ for name in not_seen_list:
break
try:
under_timestamp = simulated_flight[simulated_flight['timestamp (ms)'] - timestamp_diff < row['timestamp (ms)']].tail(1)
under_timestamp = simulated_flight[simulated_flight['timestamp (ms)'] - timestamp_diff < row['timestamp (ms)']].tail(1).to_dict('records')[0]
except IndexError:
continue
over_timestamp = simulated_flight[simulated_flight['timestamp (ms)'] - timestamp_diff >= row['timestamp (ms)']].head(1)
rate = (over_timestamp["timestamp (ms)"] - timestamp_diff - row["timestamp (ms)"])/(over_timestamp["timestamp (ms)"] - under_timestamp["timestamp (ms)"])
over_timestamp = simulated_flight[simulated_flight['timestamp (ms)'] - timestamp_diff >= row['timestamp (ms)']].head(1).to_dict('records')[0]
rate = (over_timestamp['timestamp (ms)'] - timestamp_diff - row['timestamp (ms)'])/(over_timestamp['timestamp (ms)'] - under_timestamp['timestamp (ms)'])
for criterion in distance_criterion_tuple:
tmp_sim[criterion].append(rated_value(over_timestamp[criterion], under_timestamp[criterion], rate))
tmp_real[criterion].append(row[criterion])
for index, value in enumerate((
row['timestamp (ms)'] / 1000,
distance(row['latitude ()'],
row['longitude ()'],
tmp_sim['latitude ()'][-1],
tmp_sim['longitude ()'][-1],
),
tmp_sim['AMSL (m)'][-1],
tmp_sim['ground speed (m/s)'][-1],
tmp_real['climb rate (m/s)'][-1],
)):
row['timestamp (ms)'] / 1000,
distance(row['latitude ()'],
row['longitude ()'],
tmp_sim['latitude ()'][-1],
tmp_sim['longitude ()'][-1],
),
tmp_sim['AMSL (m)'][-1],
tmp_sim['ground speed (m/s)'][-1],
tmp_real['climb rate (m/s)'][-1]
)):
distance_list_tuple[index].append(value)
# If we have at least 100 entries (timestamps) analysed, add the filename to the list
......@@ -145,7 +146,7 @@ for name in not_seen_list:
tmp_df_sim = pd.DataFrame(tmp_sim)
tmp_df_real = pd.DataFrame(tmp_real)
# Uncomment once you can actually use np..norm
score_cosine_row =mean_cosine_similarity_custom(tmp_df_real, tmp_df_sim)# cosine_similarity_rows(tmp_df_real, tmp_df_sim).mean()
score_cosine_row = mean_cosine_similarity_custom(tmp_df_real, tmp_df_sim)# cosine_similarity_rows(tmp_df_real, tmp_df_sim).mean()
score_cosine_column = mean_cosine_similarity_custom(tmp_df_real.T, tmp_df_sim.T)#cosine_similarity_rows(tmp_df_real.T, tmp_df_sim.T).mean()
flights_value_dict = {
......
bt5/erp5_wendelin_drone/PathTemplateItem/portal_callables/DataAnalysisLine_convertRawDroneDataToArray.py
View file @ d17f59dd
......@@ -56,7 +56,6 @@ for key in key_list:
zbigarray = out_data_array.initArray(shape=(0,), dtype=ndarray.dtype.fields)
start_array = zbigarray.shape[0]
zbigarray.append(ndarray)
data_array_line = out_data_array.newContent(id=key,
portal_type="Data Array Line")
data_array_line.edit(
......
bt5/erp5_wendelin_drone/PathTemplateItem/portal_callables/DataAnalysisLine_recalculateScore.py
View file @ d17f59dd
......@@ -8,148 +8,64 @@ import re
# Now, to save space, we just overwrite the old array with the new one, that contains the new scores
# This new array will give us the final overview of the ranking of the parameters. This can be used by our genetic algorithm to decide when we can stop and which parameters worked the best.
score_dtypes = {'name': 'S256', 'Param1': 'f8', 'Param2': 'f8',
'distance_reciprocal': 'f8', 'ASML_reciprocal': 'f8',
'ground_speed_reciprocal': 'f8', 'climb_rate_reciprocal': 'f8',
'score_reciprocal': 'f8', 'score_cosine_row': 'f8',
'score_cosine_column': 'f8'}
new_score_dtypes= {'name': 'S256', 'Param1': 'f8', 'Param2': 'f8',
'distance_reciprocal': 'f8', 'ASML_reciprocal': 'f8',
'ground_speed_reciprocal': 'f8', 'climb_rate_reciprocal': 'f8',
'score_reciprocal': 'f8', 'score_cosine_row': 'f8',
'score_cosine_column': 'f8',
'iteration': 'i8'}
plot_dtypes = {
'name': 'S256',
'Param1': 'f8',
'Param2': 'f8',
'timestamp': 'f8',
'distance_diff': 'f8',
'ASML_diff': 'f8',
'ground_speed_diff': 'f8',
'climb_rate_diff': 'f8',
'distance_reciprocal': 'f8',
'ASML_reciprocal': 'f8',
'ground_speed_reciprocal': 'f8',
'climb_rate_reciprocal': 'f8',
'score_reciprocal': 'f8',
'score_cosine_row': 'f8',
'score_cosine_column': 'f8',
'iteration': 'i8'
}
new_plot_dtypes = {
'Param1': 'f8',
'Param2': 'f8',
'timestamp': 'f8',
'distance_diff': 'f8',
'ASML_diff': 'f8',
'ground_speed_diff': 'f8',
'climb_rate_diff': 'f8',
'distance_reciprocal': 'f8',
'ASML_reciprocal': 'f8',
'ground_speed_reciprocal': 'f8',
'climb_rate_reciprocal': 'f8',
'score_reciprocal': 'f8',
'score_cosine_row': 'f8',
'score_cosine_column': 'f8',
'iteration': 'i8'
}
score_array = input_array_scores["Data Array"]
new_score_array = out_array_scores["Data Array"]
plot_array = input_array_plot["Data Array"]
new_plot_array = out_array_plots["Data Array"]
# Should only look at the newest few
score_array = input_array_scores['Data Array']
score_nparray = score_array.getArray()
if score_nparray is None:
score_nparray = out_array_scores["Data Array"].initArray(shape=(0,), dtype=list(new_score_dtypes.items()))
return
plot_array = input_array_plot['Data Array']
plot_nparray = plot_array.getArray()
if plot_nparray is None:
plot_nparray = out_array_plots["Data Array"].initArray(shape=(0,), dtype=list(new_plot_dtypes.items()))
old_score_df = pd.DataFrame.from_records(score_nparray[:].copy())
old_plot_df = pd.DataFrame.from_records(plot_nparray[:].copy())
progress_indicator = input_array_scores["Progress Indicator"]
seen_sims = progress_indicator.getStringOffsetIndex()
if seen_sims is None:
seen_sims = ""
sim_flight_names = list(old_score_df["name"])
# We will only continue if there is new data available.
if len([x for x in sim_flight_names if x not in seen_sims]) == 0:
return
return
new_score_dtypes = dict(score_nparray.dtype.fields).items()
new_score_dtypes.append(('iteration', 'i8'))
new_score_array = out_array_scores['Data Array']
new_score_nparray = new_score_array.getArray()
if new_score_nparray is None:
new_score_nparray = out_array_scores["Data Array"].initArray(shape=(0,), dtype=list(new_score_dtypes.items()))
new_score_array.initArray(shape=(0,), dtype=new_score_dtypes)
new_plot_dtypes = dict(plot_nparray.dtype.fields).items()
new_plot_dtypes.append(('iteration', 'i8'))
new_plot_array = out_array_plots['Data Array']
new_plot_nparray = new_plot_array.getArray()
if new_plot_nparray is None:
new_plot_nparray = out_array_plots["Data Array"].initArray(shape=(0,), dtype=list(new_plot_dtypes.items()))
new_score_df = pd.DataFrame.from_records(new_score_nparray[:].copy())
new_plot_df = pd.DataFrame.from_records(new_plot_nparray[:].copy())
new_plot_array.initArray(shape=(0,), dtype=new_plot_dtypes)
old_score_df = pd.DataFrame.from_records(score_nparray[:].copy())
old_plot_df = pd.DataFrame.from_records(plot_nparray[:].copy())
progress_indicator = input_array_scores['Progress Indicator']
seen_sims = progress_indicator.getStringOffsetIndex()
if seen_sims is None:
seen_sims = ""
sim_flight_names = list(old_score_df['name'])
# We will only continue if there is new data available.
if len([x for x in sim_flight_names if x not in seen_sims]) == 0:
return
new_score_df = pd.DataFrame.from_records(new_score_nparray[:].copy())
new_plot_df = pd.DataFrame.from_records(new_plot_nparray[:].copy())
new_score_iteration = new_score_df["iteration"].max()
new_score_iteration = new_score_df['iteration'].max()
if math.isnan(new_score_iteration):
new_score_iteration = 0
new_score_df = new_score_df.drop(["iteration"],axis=1)
new_score_df = new_score_df.drop(['iteration'], axis=1)
new_plot_iteration = new_plot_df["iteration"].max()
if math.isnan(new_plot_iteration):
new_plot_iteration = 0
#new_plot_df = new_plot_df.drop(columns=["iteration"])
answer_scores = pd.concat([old_score_df, new_score_df]).drop_duplicates(subset=['Param1','Param2'], keep='last')
answer_plots = pd.concat([old_plot_df, new_plot_df]).drop_duplicates(subset=['Param1','Param2'], keep='last')
answer_scores["score_reciprocal"] = answer_scores["score_reciprocal"].astype('float64')
answer_scores = answer_scores.nlargest(5, "score_reciprocal")
answer_plots = old_plot_df[old_plot_df["name"].isin(list(answer_scores["name"]))]
param_subset = [k for k in old_score_df.dtypes.keys().values if not 'reciprocal' in k and not 'score' in k and k != 'name']
answer_scores = pd.concat([old_score_df, new_score_df]).drop_duplicates(subset=param_subset, keep='last')
answer_plots = pd.concat([old_plot_df, new_plot_df]).drop_duplicates(subset=param_subset, keep='last')
answer_scores['score_reciprocal'] = answer_scores['score_reciprocal'].astype('float64')
answer_scores = answer_scores.nlargest(5, 'score_reciprocal')
answer_plots = old_plot_df[old_plot_df['name'].isin(list(answer_scores['name']))]
new_score_iteration = new_score_iteration + 1
new_plot_iteration = new_plot_iteration + 1
......@@ -158,24 +74,14 @@ answer_scores["iteration"] = new_score_iteration
answer_plots["iteration"] = new_plot_iteration
# We will remove all the data from the data arrays before we append the new data. Essentially we will be left with only the best few iterations
new_score_nparray = out_array_scores["Data Array"].initArray(shape=(0,), dtype=list(new_score_dtypes.items()))
new_plot_nparray = out_array_plots["Data Array"].initArray(shape=(0,), dtype=list(new_plot_dtypes.items()))
new_score_nparray = new_score_array.initArray(shape=(0,), dtype=new_score_dtypes)
new_plot_nparray = new_plot_array.initArray(shape=(0,), dtype=new_plot_dtypes)
new_score_nparray.append(answer_scores.to_records(index = False))
# Group the DataFrame by the 'name' column
grouped = answer_plots.groupby(['Param1', 'Param2'])
grouped = answer_plots.groupby(param_subset)
# Initialize a list to store the resulting DataFrames
resulting_dfs = []
......@@ -192,9 +98,11 @@ current_idx = 0
current_rank = 1
for df in resulting_dfs:
df = df.drop(["name"], axis=1)
context.log(str(current_rank))
data_array_line_plot = out_array_plots["Data Array"].get(str(current_rank))
df = df.drop_duplicates()
new_plot_nparray.append(df.to_records(index = False))
context.log(new_plot_nparray.shape)
if data_array_line_plot is None:
data_array_line_plot = out_array_plots["Data Array"].newContent(id=current_rank,
......@@ -209,5 +117,5 @@ for df in resulting_dfs:
progress_indicator.setStringOffsetIndex(sim_flight_names)
return
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7