<span metal:use-macro="container/Zuite_viewWendelinTestMacroLibrary/macros/init_wendelin_test_environment">Init Wendlin Test Environment</span>
openAndWait | ${base_url}/Zuite_updateUserPreferenceForTutorialTests |

# Agenda

- Create Notebook
- Visualise Data

This tutorial teaches how to visualise data.

Before doing this tutorial make sure you read and completed the following tutorial and have data in your wendelin instance:

- HowTo Transform Data
- HowTo Resample Data

# Notebook Module

On Modules page click on Notebooks.

# Add Notebook

Click on Add to add a new Notebook.

# Create Notebook

Click on Create Document to continue.

# Notebook

Fill in the Title.

The visualisation code will be at the bottom text area.

Notebook uses the same technology as Iodide.

To learn more about programming in Iodide environment checkout Iodide homepage.

# Visualisation Script

    %% md
    # Time Series
    <div class="plot_div" id="plot_div"/>

    %% fetch
    js: jio.js
    js: ndarray_bundle.js
    js: wendelin.js
    js: https://cdn.plot.ly/plotly-latest.min.js

    %% js
    hateoas_url = "https://softinst133633.host.vifib.net/erp5/web_site_module/renderjs_runner/hateoas/";
    jio = jIO.createJIO({
      type: "erp5",
      url: hateoas_url,
      default_view_reference: "view"
    });

    gadget = {
      getSetting: function(property) {
        return new RSVP.Queue()
          .push(function () {
            if (property == "hateoas_url") {
              return hateoas_url;
            }
            return;
          });
      },
      jio_getAttachment: function(id, url, parameter_dict) {
        return jio.getAttachment(id, url, parameter_dict);
      },
      jio_get: function(id) {return jio.get(id);}
    }

    var graph = document.getElementById('plot_div'),
      graph_data = [],
      label_list = ["date", "humidity_min","humidity_mean","humidity_max", "pressure_min","pressure_mean","pressure_max", "temperature_min","temperature_mean", "temperature_max"];

    selectorOptions = {
        "buttons": [{
            "step": 'day',
            "stepmode": 'backward',
            "count": 1,
            "label": 'day'
        },/* {
            "step": 'day',
            "stepmode": 'backward',
            "count": 7,
            "label": 'week'
        },*/ {
            "step": 'month',
            "stepmode": 'backward',
            "count": 1,
            "label": 'month'
        }, {
            "step": 'year',
            "stepmode": 'backward',
            "count": 1,
            "label": 'year'
        }],
    }

    function update_graph(start_date, stop_date){
      return getData(start_date, stop_date)
        .push(function (data) {
          return Plotly.react(graph,
            data,
            { 'title' :'Environmental Sensor Data',
              'xaxis': {
                'autorange': true,
                'rangeselector': selectorOptions,
              }
            }
          );
        });
    }

    function unpack(rows, key) {
      return rows.map(function(row) { return row[key]; });
    }

    function getData(start_date, stop_date){
      var graph_data=[],
        ratio = screen.availWidth / 1024,
        interval,
        frequency,
        start_index,
        stop_index,
        array_id;

       if (stop_date === undefined) {
         frequency = 30000;
         start_index = 0;
         stop_index = 0;
       } else {
        interval = stop_date - start_date;
        if (interval >= 1000 * 60 * 60 * 24 * 100 * ratio) {
          frequency = 30000;
        } else if (interval >= 1000 * 60 * 60 * 24 * 28 * ratio) {
          frequency = 3000;
        } else if (interval >= 1000 * 60 * 60 * 24 * 7 * ratio) {
          frequency = 300;
        } else {
          frequency = 60;
        }
      }
      return jio.allDocs({
        query: 'portal_type:"Data Analysis Line" AND ' +
               'title: "Resampled Array (' + frequency + ' Seconds)" AND ' +
               'resource_reference:"GENERIC-INTERVAL-RESAMPLED-ARRAY" AND ' +
               'parent_source_title:"Sample Factory" AND ' +
               'simulation_state:"started"'
      })
        .push(function (result) {
          var data_analysis_line_id = result.data.rows[0].id;
          return jio.allDocs({
            query: 'portal_type:"Data Array" AND ' +
                   'aggregate_related_relative_url:"' + data_analysis_line_id +'"'
          });
        })
        .push(function (result) {
          array_id = result.data.rows[0].id;
          return wendelin.getArrayRawSlice(gadget, array_id, 0, 1);
        })
        .push(function (result) {
          array_start_date = wendelin.convertFirstColToDate([[result.data[0]]])[0][0];
          if (start_index === undefined) {
            start_index = Math.max(0, Math.ceil((start_date - array_start_date) / (frequency*1000))),
            stop_index = Math.ceil((stop_date - array_start_date) / (frequency*1000));
          }
          return wendelin.getArrayRawSlice(gadget, array_id, start_index, stop_index);
        })
        .push(function(result) {
          for (i = 0; i < label_list.length; i += 1) {
            graph_data = graph_data.concat(nj.unpack(result.pick( null, label_list[i])));
          }
          return graph_data
        })
        .push(function(result){
          var filtered_graph_data = [];
          for (var i=0; i<result.length; i++) {
            if (result[i][0] != 0) {
              filtered_graph_data.push(result[i]);
            }
          }
          return wendelin.convertFirstColToDate(filtered_graph_data)
        })
        .push(function (graph_data) {
          var humidity_trace = {
              type: "scattergl",
              mode: "markers",
              name: 'Humidity(%)',
              x: unpack(graph_data, label_list.indexOf("date")),
              y: unpack(graph_data, label_list.indexOf("humidity_mean")),
              line: {color: '#17BECF'}
          }

          var pressure_trace = {
              type: "scattergl",
              mode: "markers",
              name: 'Pressure(Pa)',
              x: unpack(graph_data, label_list.indexOf("date")),
              y: unpack(graph_data, label_list.indexOf("pressure_mean")),
              line: {color: '#7F7F7F'}
          }

          var temperature_trace = {
              type: "scattergl",
              mode: "markers",
              name: 'Temperature(C°)',
              x: unpack(graph_data, label_list.indexOf("date")),
              y: unpack(graph_data, label_list.indexOf("temperature_mean")),
              line: {color: '#FF0000'}
          }

          var data = [humidity_trace,pressure_trace,temperature_trace];
          return data
        });
    }

    function plot () {
      return getData()
        .push(function (data) {
          Plotly.newPlot(
            graph,
            data,
            { 'title' :'Environmental Sensor Data',
              'xaxis': {
              'autorange': true,
              'rangeselector': selectorOptions,}
            });
          graph.on('plotly_relayout', function(eventdata) {
            var start_date = new Date(eventdata["xaxis.range[0]"]),
              stop_date = new Date(eventdata["xaxis.range[1]"]);

            return update_graph(start_date, stop_date);
          });
        });
    }
    plot();

Copy/paste the code to your notebook and save the changes.

Note: you will need to provide the URL of your instance in the script.

For that change the line :

    hateoas_url = "https://softinstXXX.host.vifib.net/erp5/web_site_module/renderjs_runner/hateoas/";

The script will find automatically find the resampled arrays based on the Data Product and plot the data using plotly library.

# Notebook

After writing the script and saving the changes click on the Editable checkbox to (uncheck it) on the left side panel to see the result.

# Data Visualisation

If everything was done correctly you will see the plot of your data.

openAndWait | ${default_front_url} |
<span metal:use-macro="container/Zuite_CommonTemplateForRenderjsUi/macros/wait_for_app_loaded"></span>
<tal:block tal:define="click_configuration python: {'text': 'Modules'}"> <tal:block metal:use-macro="here/Zuite_CommonTemplateForRenderjsUi/macros/click_on_panel_link"> </tal:block> </tal:block>
<tal:block metal:use-macro="here/Zuite_CommonTemplateForRenderjsUi/macros/wait_for_content_loaded"> </tal:block>
waitForElementPresent | //a[contains(@href,'key=notebook_module')] |
click | //a[contains(@href,'key=notebook_module')] |
<tal:block metal:use-macro="here/Zuite_CommonTemplateForRenderjsUi/macros/wait_for_content_loaded"> </tal:block>
waitForElementPresent | //a[@data-i18n="Add"] |
click | //a[@data-i18n="Add"] |
<tal:block metal:use-macro="here/Zuite_CommonTemplateForRenderjsUi/macros/wait_for_content_loaded"> </tal:block>
waitForElementPresent | //input[@type="submit" and @value="Create Document"] |
click | //input[@type="submit" and @value="Create Document"] |
<tal:block metal:use-macro="here/Zuite_CommonTemplateForRenderjsUi/macros/wait_for_content_loaded"> </tal:block>
waitForElementPresent | //a[text()='Edit'] |
click | //a[text()='Edit'] |
waitForElementPresent | //input[@id="field_my_title"] |
type | //input[@id="field_my_title"] | Environment Data Graph
waitForElementPresent | //div[contains(@data-gadget-url,'gadget_editor.html')]//textarea |
type | //div[contains(@data-gadget-url,'gadget_editor.html')]//textarea |
    %% md
    # Time Series
    <div class="plot_div" id="plot_div"/>

    %% fetch
    js: jio.js
    js: ndarray_bundle.js
    js: wendelin.js
    js: https://cdn.plot.ly/plotly-latest.min.js

    %% js
    hateoas_url = "${base_url}/web_site_module/renderjs_runner/hateoas/";
    jio = jIO.createJIO({
      type: "erp5",
      url: hateoas_url,
      default_view_reference: "view"
    });

    gadget = {
      getSetting: function(property) {
        return new RSVP.Queue()
          .push(function () {
            if (property == "hateoas_url") {
              return hateoas_url;
            }
            return;
          });
      },
      jio_getAttachment: function(id, url, parameter_dict) {
        return jio.getAttachment(id, url, parameter_dict);
      },
      jio_get: function(id) {return jio.get(id);}
    }

    var graph = document.getElementById('plot_div'),
      graph_data = [],
      label_list = ["date", "humidity_min","humidity_mean","humidity_max", "pressure_min","pressure_mean","pressure_max", "temperature_min","temperature_mean", "temperature_max"];

    selectorOptions = {
        "buttons": [{
            "step": 'day',
            "stepmode": 'backward',
            "count": 1,
            "label": 'day'
        },/* {
            "step": 'day',
            "stepmode": 'backward',
            "count": 7,
            "label": 'week'
        },*/ {
            "step": 'month',
            "stepmode": 'backward',
            "count": 1,
            "label": 'month'
        }, {
            "step": 'year',
            "stepmode": 'backward',
            "count": 1,
            "label": 'year'
        }],
    }

    function update_graph(start_date, stop_date){
      return getData(start_date, stop_date)
        .push(function (data) {
          return Plotly.react(graph,
            data,
            { 'title' :'Environmental Sensor Data',
              'xaxis': {
                'autorange': true,
                'rangeselector': selectorOptions,
              }
            }
          );
        });
    }

    function unpack(rows, key) {
      return rows.map(function(row) { return row[key]; });
    }

    function getData(start_date, stop_date){
      var graph_data=[],
        ratio = screen.availWidth / 1024,
        interval,
        frequency,
        start_index,
        stop_index,
        array_id;

       if (stop_date === undefined) {
         frequency = 30000;
         start_index = 0;
         stop_index = 0;
       } else {
        interval = stop_date - start_date;
        if (interval >= 1000 * 60 * 60 * 24 * 100 * ratio) {
          frequency = 30000;
        } else if (interval >= 1000 * 60 * 60 * 24 * 28 * ratio) {
          frequency = 3000;
        } else if (interval >= 1000 * 60 * 60 * 24 * 7 * ratio) {
          frequency = 300;
        } else {
          frequency = 60;
        }
      }
      return jio.allDocs({
        query: 'portal_type:"Data Analysis Line" AND ' +
               'title: "Resampled Array (' + frequency + ' Seconds)" AND ' +
               'resource_reference:"GENERIC-INTERVAL-RESAMPLED-ARRAY" AND ' +
               'parent_source_title:"Sample Factory" AND ' +
               'simulation_state:"started"'
      })
        .push(function (result) {
          var data_analysis_line_id = result.data.rows[0].id;
          return jio.allDocs({
            query: 'portal_type:"Data Array" AND ' +
                   'aggregate_related_relative_url:"' + data_analysis_line_id +'"'
          });
        })
        .push(function (result) {
          array_id = result.data.rows[0].id;
          return wendelin.getArrayRawSlice(gadget, array_id, 0, 1);
        })
        .push(function (result) {
          array_start_date = wendelin.convertFirstColToDate([[result.data[0]]])[0][0];
          if (start_index === undefined) {
            start_index = Math.max(0, Math.ceil((start_date - array_start_date) / (frequency*1000))),
            stop_index = Math.ceil((stop_date - array_start_date) / (frequency*1000));
          }
          return wendelin.getArrayRawSlice(gadget, array_id, start_index, stop_index);
        })
        .push(function(result) {
          for (i = 0; i < label_list.length; i += 1) {
            graph_data = graph_data.concat(nj.unpack(result.pick( null, label_list[i])));
          }
          return graph_data
        })
        .push(function(result){
          var filtered_graph_data = [];
          for (var i=0; i<result.length; i++) {
            if (result[i][0] != 0) {
              filtered_graph_data.push(result[i]);
            }
          }
          return wendelin.convertFirstColToDate(filtered_graph_data)
        })
        .push(function (graph_data) {
          var humidity_trace = {
              type: "scattergl",
              mode: "markers",
              name: 'Humidity(%)',
              x: unpack(graph_data, label_list.indexOf("date")),
              y: unpack(graph_data, label_list.indexOf("humidity_mean")),
              line: {color: '#17BECF'}
          }

          var pressure_trace = {
              type: "scattergl",
              mode: "markers",
              name: 'Pressure(Pa)',
              x: unpack(graph_data, label_list.indexOf("date")),
              y: unpack(graph_data, label_list.indexOf("pressure_mean")),
              line: {color: '#7F7F7F'}
          }

          var temperature_trace = {
              type: "scattergl",
              mode: "markers",
              name: 'Temperature(C°)',
              x: unpack(graph_data, label_list.indexOf("date")),
              y: unpack(graph_data, label_list.indexOf("temperature_mean")),
              line: {color: '#FF0000'}
          }

          var data = [humidity_trace,pressure_trace,temperature_trace];
          return data
        });
    }

    function plot () {
      return getData()
        .push(function (data) {
          Plotly.newPlot(
            graph,
            data,
            { 'title' :'Environmental Sensor Data',
              'xaxis': {
              'autorange': true,
              'rangeselector': selectorOptions,}
            });
          graph.on('plotly_relayout', function(eventdata) {
            var start_date = new Date(eventdata["xaxis.range[0]"]),
              stop_date = new Date(eventdata["xaxis.range[1]"]);

            return update_graph(start_date, stop_date);
          });
        });
    }
    plot();
pause | 5000 |
click | //button[@name="submit" and @data-i18n="Save"] |
<tal:block metal:use-macro="here/Zuite_CommonTemplateForRenderjsUi/macros/wait_for_content_loaded"> </tal:block>
waitForElementPresent | //button[@type="submit" and @class="success"] |
click | //a[text()="View"] |
<tal:block metal:use-macro="here/Zuite_CommonTemplateForRenderjsUi/macros/wait_for_content_loaded"> </tal:block>
waitForElementPresent | //div[@id="plot_div"] |