Commit 982d6b83 authored by Ekaterina's avatar Ekaterina

erp5_wendelin_tutorial: fix whitespaces in Resample.Data test script,...

erp5_wendelin_tutorial: fix whitespaces in Resample.Data test script, increased time of waiting till 350000
parent 76caa4fa
...@@ -2110,7 +2110,7 @@ function getPredictionData(start_date, stop_date){ ...@@ -2110,7 +2110,7 @@ function getPredictionData(start_date, stop_date){
<tr> <tr>
<td>type</td> <td>type</td>
<td>//input[@id="field_my_title"]</td> <td>//input[@id="field_my_title"]</td>
<td>Environment Data Graph</td> <td>Predicted Data</td>
</tr> </tr>
<tr> <tr>
<td>waitForElementPresent</td> <td>waitForElementPresent</td>
...@@ -2120,7 +2120,8 @@ function getPredictionData(start_date, stop_date){ ...@@ -2120,7 +2120,8 @@ function getPredictionData(start_date, stop_date){
<tr> <tr>
<td>type</td> <td>type</td>
<td>//div[contains(@class,'CodeMirror')]//div//textarea</td> <td>//div[contains(@class,'CodeMirror')]//div//textarea</td>
<td>%% md <td>
%% md
# Prediction # Prediction
&lt;div class=&quot;prediction_plot_div&quot; id=&quot;prediction_plot_div&quot;/&gt; &lt;div class=&quot;prediction_plot_div&quot; id=&quot;prediction_plot_div&quot;/&gt;
......
...@@ -1260,6 +1260,51 @@ ...@@ -1260,6 +1260,51 @@
</value> </value>
</item> </item>
</dictionary> </dictionary>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>edit</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>987.51843.4592.4676</string> </value>
</item>
<item>
<key> <string>state</string> </key>
<value> <string>current</string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass> <reference id="4.1"/> </klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1605282722.21</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
</dictionary>
</list> </list>
</value> </value>
</item> </item>
......
...@@ -1035,6 +1035,96 @@ ...@@ -1035,6 +1035,96 @@
</value> </value>
</item> </item>
</dictionary> </dictionary>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>edit</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>987.52109.39626.39048</string> </value>
</item>
<item>
<key> <string>state</string> </key>
<value> <string>current</string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass> <reference id="4.1"/> </klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1605288275.3</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
</dictionary>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>edit</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>987.62292.38572.16418</string> </value>
</item>
<item>
<key> <string>state</string> </key>
<value> <string>current</string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass> <reference id="4.1"/> </klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1605289283.36</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
</dictionary>
</list> </list>
</value> </value>
</item> </item>
......
...@@ -663,6 +663,141 @@ ...@@ -663,6 +663,141 @@
</value> </value>
</item> </item>
</dictionary> </dictionary>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>edit</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>P11</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>987.57587.28935.45619</string> </value>
</item>
<item>
<key> <string>state</string> </key>
<value> <string>current</string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass> <reference id="4.1"/> </klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1605287644.73</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
</dictionary>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>edit</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>P11</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>987.62282.5174.1245</string> </value>
</item>
<item>
<key> <string>state</string> </key>
<value> <string>current</string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass> <reference id="4.1"/> </klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1605287645.98</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
</dictionary>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>edit</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>987.62282.5174.1245</string> </value>
</item>
<item>
<key> <string>state</string> </key>
<value> <string>current</string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass> <reference id="4.1"/> </klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1605288251.79</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
</dictionary>
</list> </list>
</value> </value>
</item> </item>
......
...@@ -1314,7 +1314,8 @@ progress_indicator.setIntOffsetIndex(in_zbigarray.shape[0]) ...@@ -1314,7 +1314,8 @@ progress_indicator.setIntOffsetIndex(in_zbigarray.shape[0])
<tr> <tr>
<td>type</td> <td>type</td>
<td>//div[contains(@data-gadget-url,'gadget_editor.html')]//textarea</td> <td>//div[contains(@data-gadget-url,'gadget_editor.html')]//textarea</td>
<td>import numpy as np <td>
import numpy as np
import pandas as pd import pandas as pd
out_array_list = out_array out_array_list = out_array
...@@ -1324,10 +1325,10 @@ progress_indicator = in_array[&quot;Progress Indicator&quot;] ...@@ -1324,10 +1325,10 @@ progress_indicator = in_array[&quot;Progress Indicator&quot;]
in_zbigarray = in_data_array.getArray() in_zbigarray = in_data_array.getArray()
if in_zbigarray is None: if in_zbigarray is None:
return &nbsp; return
if in_zbigarray.shape[0] == 0: if in_zbigarray.shape[0] == 0:
return &nbsp; return
# first fill array with lowest resolution # first fill array with lowest resolution
default_data_array = out_array_list[0][&quot;Data Array&quot;] default_data_array = out_array_list[0][&quot;Data Array&quot;]
...@@ -1342,11 +1343,11 @@ df = pd.DataFrame.from_records(in_zbigarray[index:].copy(), index=&#39;date&#39; ...@@ -1342,11 +1343,11 @@ df = pd.DataFrame.from_records(in_zbigarray[index:].copy(), index=&#39;date&#39;
# ignore data before start date of output array # ignore data before start date of output array
if default_zbigarray is not None: if default_zbigarray is not None:
if default_zbigarray.shape[0] != 0: &nbsp; if default_zbigarray.shape[0] != 0:
df = df.loc[str(default_zbigarray[0][&#39;date&#39;]):] &nbsp; &nbsp; df = df.loc[str(default_zbigarray[0][&#39;date&#39;]):]
if len(df) == 0: if len(df) == 0:
return &nbsp; return
# resample # resample
df = df.resample(default_resolution).agg([&#39;min&#39;,&#39;mean&#39;,&#39;max&#39;]).fillna(0) df = df.resample(default_resolution).agg([&#39;min&#39;,&#39;mean&#39;,&#39;max&#39;]).fillna(0)
...@@ -1370,91 +1371,91 @@ mask_zero = (df==0).all(axis=1) ...@@ -1370,91 +1371,91 @@ mask_zero = (df==0).all(axis=1)
default_data[&#39;date&#39;][mask_zero] = 0 default_data[&#39;date&#39;][mask_zero] = 0
if default_zbigarray is None: if default_zbigarray is None:
default_zbigarray = default_data_array.initArray(shape=(0,), dtype=default_data.dtype.fields) &nbsp; default_zbigarray = default_data_array.initArray(shape=(0,), dtype=default_data.dtype.fields)
if default_zbigarray.shape[0] == 0: if default_zbigarray.shape[0] == 0:
default_zbigarray.append(default_data) &nbsp; default_zbigarray.append(default_data)
else: else:
# calculate start and stop index of new data in output array # calculate start and stop index of new data in output array
default_start_index = int((date_vector[0] - default_zbigarray[0][&#39;date&#39;]) / default_frequency) &nbsp; default_start_index = int((date_vector[0] - default_zbigarray[0][&#39;date&#39;]) / default_frequency)
default_stop_index = int((date_vector[-1] - default_zbigarray[0][&#39;date&#39;]) / default_frequency + 1) &nbsp; default_stop_index = int((date_vector[-1] - default_zbigarray[0][&#39;date&#39;]) / default_frequency + 1)
# make sure data fits in # make sure data fits in
if default_stop_index &gt; default_zbigarray.shape[0]: &nbsp; if default_stop_index &gt; default_zbigarray.shape[0]:
default_zbigarray.resize((default_stop_index,)) &nbsp; &nbsp; default_zbigarray.resize((default_stop_index,))
# fill holes in new data with values from old data # fill holes in new data with values from old data
old_data = default_zbigarray[default_start_index:default_stop_index] &nbsp; old_data = default_zbigarray[default_start_index:default_stop_index]
default_data[mask_zero ] = old_data[mask_zero] &nbsp; default_data[mask_zero ] = old_data[mask_zero]
# write new_data to zbigarray # write new_data to zbigarray
default_zbigarray[default_start_index:default_stop_index] = default_data &nbsp; default_zbigarray[default_start_index:default_stop_index] = default_data
# now use data in first resolution array for all other arrays # now use data in first resolution array for all other arrays
for out_array in out_array_list[1:]: for out_array in out_array_list[1:]:
out_data_array = out_array[&quot;Data Array&quot;] &nbsp; out_data_array = out_array[&quot;Data Array&quot;]
out_array_resolution = out_array[&quot;resolution&quot;] &nbsp; out_array_resolution = out_array[&quot;resolution&quot;]
out_zbigarray = out_data_array.getArray() &nbsp; out_zbigarray = out_data_array.getArray()
if out_zbigarray is None: &nbsp; if out_zbigarray is None:
out_zbigarray = out_data_array.initArray(shape=(0,), dtype=default_data.dtype.fields) &nbsp; &nbsp; out_zbigarray = out_data_array.initArray(shape=(0,), dtype=default_data.dtype.fields)
if out_zbigarray.shape[0] == 0: &nbsp; if out_zbigarray.shape[0] == 0:
start_index = 0 &nbsp; &nbsp; start_index = 0
else: &nbsp; else:
out_array_frequency = pd.to_timedelta(out_array_resolution) &nbsp; &nbsp; out_array_frequency = pd.to_timedelta(out_array_resolution)
new_stop_date = default_zbigarray[0][&#39;date&#39;] + default_zbigarray.shape[0] * default_frequency &nbsp; &nbsp; new_stop_date = default_zbigarray[0][&#39;date&#39;] + default_zbigarray.shape[0] * default_frequency
old_stop_date = out_zbigarray[0][&#39;date&#39;] + out_zbigarray.shape[0] * out_array_frequency &nbsp; &nbsp; old_stop_date = out_zbigarray[0][&#39;date&#39;] + out_zbigarray.shape[0] * out_array_frequency
start_date = old_stop_date - out_array_frequency &nbsp; &nbsp; start_date = old_stop_date - out_array_frequency
if old_stop_date &gt;= new_stop_date: &nbsp; &nbsp; if old_stop_date &gt;= new_stop_date:
continue &nbsp; &nbsp; &nbsp; continue
# find row index in in_array from where to start resampling # find row index in in_array from where to start resampling
start_index = int(max((start_date - default_zbigarray[0][&#39;date&#39;]) / default_frequency, 0)) &nbsp; &nbsp; start_index = int(max((start_date - default_zbigarray[0][&#39;date&#39;]) / default_frequency, 0))
# if we got data which has been already resampled, then we resample again and overwrite # if we got data which has been already resampled, then we resample again and overwrite
start_index = min(start_index, default_start_index) &nbsp; &nbsp; start_index = min(start_index, default_start_index)
data = default_zbigarray[start_index:].copy() &nbsp; data = default_zbigarray[start_index:].copy()
# convert data to DataFrame and resample # convert data to DataFrame and resample
df = pd.DataFrame.from_records(data, index=&#39;date&#39;) &nbsp; df = pd.DataFrame.from_records(data, index=&#39;date&#39;)
# set our own date range index so that we can resample and keep 0-dates # set our own date range index so that we can resample and keep 0-dates
resampling_start_date = default_zbigarray[0][&#39;date&#39;] + start_index * default_frequency &nbsp; resampling_start_date = default_zbigarray[0][&#39;date&#39;] + start_index * default_frequency
df.index = pd.date_range(start=resampling_start_date, &nbsp; df.index = pd.date_range(start=resampling_start_date,
periods=data.shape[0], periods=data.shape[0],
freq=default_frequency) freq=default_frequency)
df.index.name = &#39;date&#39; &nbsp; df.index.name = &#39;date&#39;
# resample each column with appropriate aggregation method # resample each column with appropriate aggregation method
aggregation_dict = {c: c.split(&#39;_&#39;)[-1] for c in df.columns} &nbsp; aggregation_dict = {c: c.split(&#39;_&#39;)[-1] for c in df.columns}
df = df.resample(out_array_resolution).agg(aggregation_dict).fillna(0) &nbsp; df = df.resample(out_array_resolution).agg(aggregation_dict).fillna(0)
# save date vector for later # save date vector for later
date_vector = df.index.values.copy() &nbsp; date_vector = df.index.values.copy()
# convert data back to ndarray # convert data back to ndarray
new_data = df.to_records(convert_datetime64=False) &nbsp; new_data = df.to_records(convert_datetime64=False)
# set date to zero where all values are 0 # set date to zero where all values are 0
new_data[&#39;date&#39;][(df==0).all(axis=1)] = 0 &nbsp; new_data[&#39;date&#39;][(df==0).all(axis=1)] = 0
if out_zbigarray.shape[0] == 0: &nbsp; if out_zbigarray.shape[0] == 0:
out_zbigarray.append(new_data) &nbsp; &nbsp; out_zbigarray.append(new_data)
else: &nbsp; else:
# calculate start and stop index of new data in output array # calculate start and stop index of new data in output array
start_index = int((date_vector[0] - out_zbigarray[0][&#39;date&#39;]) / out_array_frequency) &nbsp; &nbsp; start_index = int((date_vector[0] - out_zbigarray[0][&#39;date&#39;]) / out_array_frequency)
stop_index = int((date_vector[-1] - out_zbigarray[0][&#39;date&#39;]) / out_array_frequency + 1) &nbsp; &nbsp; stop_index = int((date_vector[-1] - out_zbigarray[0][&#39;date&#39;]) / out_array_frequency + 1)
# make sure data fits in # make sure data fits in
if stop_index &gt; out_zbigarray.shape[0]: &nbsp; &nbsp; if stop_index &gt; out_zbigarray.shape[0]:
out_zbigarray.resize((stop_index,)) &nbsp; &nbsp; &nbsp; out_zbigarray.resize((stop_index,))
# write new_data to zbigarray # write new_data to zbigarray
out_zbigarray[start_index:stop_index] = new_data &nbsp; &nbsp; out_zbigarray[start_index:stop_index] = new_data
progress_indicator.setIntOffsetIndex(in_zbigarray.shape[0]) progress_indicator.setIntOffsetIndex(in_zbigarray.shape[0])
</td> </td>
......
...@@ -767,6 +767,231 @@ ...@@ -767,6 +767,231 @@
</value> </value>
</item> </item>
</dictionary> </dictionary>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>edit</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>987.52042.38293.28160</string> </value>
</item>
<item>
<key> <string>state</string> </key>
<value> <string>current</string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass> <reference id="4.1"/> </klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1605196373.03</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
</dictionary>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>edit</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>987.52042.38293.28160</string> </value>
</item>
<item>
<key> <string>state</string> </key>
<value> <string>current</string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass> <reference id="4.1"/> </klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1605264338.45</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
</dictionary>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>edit</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>987.61893.42009.59477</string> </value>
</item>
<item>
<key> <string>state</string> </key>
<value> <string>current</string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass> <reference id="4.1"/> </klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1605272918.8</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
</dictionary>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>edit</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>987.62036.42385.47274</string> </value>
</item>
<item>
<key> <string>state</string> </key>
<value> <string>current</string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass> <reference id="4.1"/> </klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1605277747.85</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
</dictionary>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>edit</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>987.62117.8588.11468</string> </value>
</item>
<item>
<key> <string>state</string> </key>
<value> <string>current</string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass> <reference id="4.1"/> </klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1605277751.1</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
</dictionary>
</list> </list>
</value> </value>
</item> </item>
......
...@@ -1587,7 +1587,7 @@ if end &lt; in_data_stream.getSize(): ...@@ -1587,7 +1587,7 @@ if end &lt; in_data_stream.getSize():
</tr> </tr>
<tr> <tr>
<td>pause</td> <td>pause</td>
<td>200000</td> <td>350000</td>
<td></td> <td></td>
</tr> </tr>
<tr> <tr>
...@@ -1690,7 +1690,7 @@ if end &lt; in_data_stream.getSize(): ...@@ -1690,7 +1690,7 @@ if end &lt; in_data_stream.getSize():
</tr> </tr>
<tr> <tr>
<td>pause</td> <td>pause</td>
<td>200000</td> <td>350000</td>
<td></td> <td></td>
</tr> </tr>
<tr> <tr>
......
...@@ -859,6 +859,51 @@ ...@@ -859,6 +859,51 @@
</value> </value>
</item> </item>
</dictionary> </dictionary>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>edit</string> </value>
</item>
<item>
<key> <string>actor</string> </key>
<value> <string>zope</string> </value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>error_message</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>987.52108.51661.39645</string> </value>
</item>
<item>
<key> <string>state</string> </key>
<value> <string>current</string> </value>
</item>
<item>
<key> <string>time</string> </key>
<value>
<object>
<klass> <reference id="4.1"/> </klass>
<tuple>
<none/>
</tuple>
<state>
<tuple>
<float>1605189289.93</float>
<string>UTC</string>
</tuple>
</state>
</object>
</value>
</item>
</dictionary>
</list> </list>
</value> </value>
</item> </item>
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment