wendelin_telecom_base: Refactor KPI processing pipeline

bt5/wendelin_telecom_base/ExtensionTemplateItem/portal_components/extension.erp5.OrsKpiUtils.py → bt5/wendelin_telecom_base/ExtensionTemplateItem/portal_components/extension.erp5.WendelinTelecomCalcEnbKpi.py

 from xlte.amari import kpi as amari_kpi
 from xlte import kpi
+from zLOG import LOG, WARNING
 import io
 
 def load_measurements(alogm):
@@ -9,13 +10,13 @@ def load_measurements(alogm):
     while measurement is not None:
       try:
         mlog.append(measurement)
-      except AssertionError:
+      except Exception as e:
         # Invalid measurement: simply skip it
-        pass
+        LOG('WendelinTelecomCalcOrsKpi.loadMeasurements', WARNING, "Skipped data during KPI calculation: %s" % e)
       measurement = alogm.read()
-  except amari_kpi.LogError:
+  except amari_kpi.LogError as e:
     # Invalid measurement: simply skip it
-    pass
+    LOG('WendelinTelecomCalcOrsKpi.loadMeasurements', WARNING, "Skipped data during KPI calculation: %s" % e)
   finally:
     alogm.close()
   return mlog
@@ -33,32 +34,49 @@ def calc_periods(mlog, tperiod):
     # No data to read: exit
     return
 
-def getOrsKpiValues(data):
+def calcEnbKpi(data, t_period):
   fxlog = io.StringIO(data)
   alogm = amari_kpi.LogMeasure(fxlog, open('/dev/null', 'r'))
   mlog = load_measurements(alogm)
 
+  # E-RAB Accessibility KPI
   vt = []
   v_initial_epsb_estab_sr = []
   v_added_epsb_estab_sr = []
 
-  tperiod = 60 # seconds 
-  for calc in calc_periods(mlog, tperiod):
+  for calc in calc_periods(mlog, t_period):
+    try:
+      erab_accessibility = calc.erab_accessibility()
+    except AssertionError as e:
+      LOG('WendelinTelecomCalcOrsKpi.calcEnbKpi', WARNING, "Skipped data during KPI calculation: %s" % e)
+      continue
     vt.append(calc.tau_lo)
-    erab_accessibility = calc.erab_accessibility()
     v_initial_epsb_estab_sr.append((erab_accessibility[0]['lo'], erab_accessibility[0]['hi']))
     v_added_epsb_estab_sr.append((erab_accessibility[1]['lo'], erab_accessibility[1]['hi']))
 
+  # E-UTRAN IP Throughput KPI
   evt = []
   v_ip_throughput_qci = []
 
-  tperiod = 3 # seconds
-  for calc in calc_periods(mlog, tperiod):
-    evt.append(calc.tau_lo)
-    eutran_ip_throughput = calc.eutran_ip_throughput()
+  for calc in calc_periods(mlog, t_period):
+    try:
+      eutran_ip_throughput = calc.eutran_ip_throughput()
+    except AssertionError as e:
+      LOG('WendelinTelecomCalcOrsKpi.calcEnbKpi', WARNING, "Skipped data during KPI calculation: %s" % e)
+      continue
     period_qci_data = []
-    for qci, qci_measurement in enumerate(eutran_ip_throughput):
-      period_qci_data.append((qci, qci_measurement['dl']['lo'], qci_measurement['dl']['hi'], qci_measurement['ul']['lo'], qci_measurement['ul']['hi']))
+    for qci_measurement in eutran_ip_throughput:
+      period_qci_data.append((
+        qci_measurement['dl']['lo'],
+        qci_measurement['dl']['hi'],
+        qci_measurement['ul']['lo'],
+        qci_measurement['ul']['hi']
+       ))
+    evt.append(calc.tau_lo)
     v_ip_throughput_qci.append(period_qci_data)
 
-  return vt, v_initial_epsb_estab_sr, v_added_epsb_estab_sr, evt, v_ip_throughput_qci
+  kpi_data_dict = dict(
+    e_rab_accessibility=(vt, v_initial_epsb_estab_sr, v_added_epsb_estab_sr),
+    e_utran_ip_throughput=(evt, v_ip_throughput_qci)
+  )
+  return kpi_data_dict
\ No newline at end of file

bt5/wendelin_telecom_base/ExtensionTemplateItem/portal_components/extension.erp5.WendelinTelecomCalcEnbKpi.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Extension Component" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>default_reference</string> </key>
+            <value> <string>WendelinTelecomCalcEnbKpi</string> </value>
+        </item>
+        <item>
+            <key> <string>default_source_reference</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>extension.erp5.WendelinTelecomCalcEnbKpi</string> </value>
+        </item>
+        <item>
+            <key> <string>sid</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_error_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_warning_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>version</string> </key>
+            <value> <string>erp5</string> </value>
+        </item>
+        <item>
+            <key> <string>workflow_history</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>component_validation_workflow</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+                    </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_log</string> </key>
+            <value>
+              <list>
+                <dictionary>
+                  <item>
+                      <key> <string>action</string> </key>
+                      <value> <string>validate</string> </value>
+                  </item>
+                  <item>
+                      <key> <string>validation_state</string> </key>
+                      <value> <string>validated</string> </value>
+                  </item>
+                </dictionary>
+              </list>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/wendelin_telecom_base/ExtensionTemplateItem/portal_components/extension.erp5.WendelinTelecomGetOrsKpi.py

+import json
+import numpy as np
+import pandas as pd
+
+QCI_COUNT = 256
+# The maximum number of points to show on a KPI graph
+RESAMPLE_SIZE = 1000
+# Values with which to replace NaNs for each data field
+NA_VALUES_REPLACEMENTS = {
+  'vInitialEPSBEstabSR_lo': 0.,
+  'vInitialEPSBEstabSR_hi': 100.,
+  'vAddedEPSBEstabSR_lo': 0.,
+  'vAddedEPSBEstabSR_hi': 100.,
+  'dl_lo': 0.,
+  'dl_hi': 0.,
+  'ul_lo': 0.,
+  'ul_hi': 0.
+}
+
+def resample_data_array(
+  np_data_array, time_field,
+  time_start, time_end, resample_size
+):
+  '''
+    Resample the provided numpy data array to the provided new size
+    by averaging values in regular time intervals, if it is long enough to be downsized.
+    Return a dictionary with the data arrays keyed to their respective data fields.
+  '''
+  data_frame = pd.DataFrame.from_records(np_data_array)
+  data_frame[time_field] = pd.to_datetime(data_frame[time_field], unit='s')
+  data_frame = data_frame.sort_values(by=time_field)
+
+  # Resample data
+  if len(np_data_array) > resample_size:
+    resample_period = '%ss' % int((time_end - time_start) / resample_size)
+    data_frame = data_frame.resample(resample_period, on=time_field).mean()
+    data_frame = data_frame.fillna(value=NA_VALUES_REPLACEMENTS)
+
+  data_frame = data_frame.reset_index()
+  data_frame[time_field] = data_frame[time_field].map(pd.Timestamp.timestamp)
+  return data_frame.to_dict(orient='list')
+
+def getOrsEnbKpi(
+  self, data_array_url, kpi_type,
+  time_start=None, time_end=None, REQUEST=None, RESPONSE=None
+):
+  '''
+    Return the requested KPI data in the response as a JSON object
+    with the data arrays keyed to their respective data fields.
+    Several optimizations are made:
+    - If required, the data will be resampled to at most RESAMPLE_SIZE data points.
+    - For the E-UTRAN IP Throughput KPI, optimize the data format by removing data
+    for "silent" QCIs, which have only zero values.
+  '''
+  portal = self.getPortalObject()
+
+  try:
+    data_array = portal.restrictedTraverse(data_array_url)
+    data_array = np.array(data_array.getArray())
+  except KeyError:
+    # Data Array does not exist
+    RESPONSE.setStatus(404)
+    return
+  # Data Array is empty
+  if data_array.shape in [(), (0,)]:
+    RESPONSE.write(json.dumps(dict()))
+    return
+
+  RESPONSE.setHeader("Content-Type", "application/json")
+
+  time_field = data_array.dtype.names[0]
+
+  if time_start is None or time_start == 'NaN':
+    time_start = data_array[time_field][0]
+  else:
+    time_start = float(time_start)
+  if time_end is None or time_end == 'NaN':
+    time_end = data_array[time_field][-1]
+  else:
+    time_end = float(time_end)
+  time_mask = (data_array[time_field] >= time_start) \
+    & (data_array[time_field] <= time_end)
+  if isinstance(time_mask, np.ndarray):
+    data_array = data_array[time_mask]
+
+  if kpi_type == 'e_rab_accessibility':
+    resampled_data_dict = resample_data_array(
+      data_array,
+      time_field,
+      time_start,
+      time_end,
+      RESAMPLE_SIZE
+    )
+    response_dict = dict(
+      vt=resampled_data_dict['vt'],
+      v_initial_epsb_estab_sr_lo=resampled_data_dict['vInitialEPSBEstabSR_lo'],
+      v_initial_epsb_estab_sr_hi=resampled_data_dict['vInitialEPSBEstabSR_hi'],
+      v_added_epsb_estab_sr_lo=resampled_data_dict['vAddedEPSBEstabSR_lo'],
+      v_added_epsb_estab_sr_hi=resampled_data_dict['vAddedEPSBEstabSR_hi']
+    )
+    RESPONSE.write(json.dumps(response_dict))
+
+  elif kpi_type == 'e_utran_ip_throughput':
+    evt = []
+    active_qci = []
+    dl_lo = []
+    dl_hi = []
+    ul_lo = []
+    ul_hi = []
+
+    for qci in range(QCI_COUNT):
+      qci_data_array = data_array[qci::QCI_COUNT]
+      qci_dl_hi = qci_data_array['dl_hi']
+      qci_ul_hi = qci_data_array['ul_hi']
+
+      # QCI is silent: skip it
+      if (qci_dl_hi == 0.).all() and (qci_ul_hi == 0.).all():
+        continue
+
+      resampled_qci_data_dict = resample_data_array(
+        qci_data_array,
+        time_field,
+        time_start,
+        time_end,
+        RESAMPLE_SIZE
+      )
+      evt = resampled_qci_data_dict['evt']
+      qci_dl_lo = resampled_qci_data_dict['dl_lo']
+      qci_dl_hi = resampled_qci_data_dict['dl_hi']
+      qci_ul_lo = resampled_qci_data_dict['ul_lo']
+      qci_ul_hi = resampled_qci_data_dict['ul_hi']
+
+      active_qci.append(qci)
+      dl_lo.append(qci_dl_lo)
+      dl_hi.append(qci_dl_hi)
+      ul_lo.append(qci_ul_lo)
+      ul_hi.append(qci_ul_hi)
+
+    response_dict = dict(
+      evt=evt,
+      active_qci=active_qci,
+      dl_lo=dl_lo,
+      dl_hi=dl_hi,
+      ul_lo=ul_lo,
+      ul_hi=ul_hi
+    )
+    RESPONSE.write(json.dumps(response_dict))

bt5/wendelin_telecom_base/ExtensionTemplateItem/portal_components/extension.erp5.OrsKpiUtils.xml → bt5/wendelin_telecom_base/ExtensionTemplateItem/portal_components/extension.erp5.WendelinTelecomGetOrsKpi.xml

@@ -8,7 +8,13 @@
       <dictionary>
         <item>
             <key> <string>default_reference</string> </key>
-            <value> <string>OrsKpiUtils</string> </value>
+            <value> <string>WendelinTelecomGetOrsKpi</string> </value>
+        </item>
+        <item>
+            <key> <string>default_source_reference</string> </key>
+            <value>
+              <none/>
+            </value>
         </item>
         <item>
             <key> <string>description</string> </key>
@@ -18,7 +24,7 @@
         </item>
         <item>
             <key> <string>id</string> </key>
-            <value> <string>extension.erp5.OrsKpiUtils</string> </value>
+            <value> <string>extension.erp5.WendelinTelecomGetOrsKpi</string> </value>
         </item>
         <item>
             <key> <string>sid</string> </key>

bt5/wendelin_telecom_base/PathTemplateItem/data_operation_module/calculate_ors_kpi.xml → bt5/wendelin_telecom_base/PathTemplateItem/data_operation_module/calculate_ors_enb_kpi.xml

@@ -62,7 +62,7 @@
         </item>
         <item>
             <key> <string>default_reference</string> </key>
-            <value> <string>calculate_ors_kpi</string> </value>
+            <value> <string>calculate_ors_enb_kpi</string> </value>
         </item>
         <item>
             <key> <string>description</string> </key>
@@ -72,7 +72,7 @@
         </item>
         <item>
             <key> <string>id</string> </key>
-            <value> <string>calculate_ors_kpi</string> </value>
+            <value> <string>calculate_ors_enb_kpi</string> </value>
         </item>
         <item>
             <key> <string>language</string> </key>
@@ -82,11 +82,11 @@
         </item>
         <item>
             <key> <string>script_id</string> </key>
-            <value> <string>DataAnalysisLine_calculateOrsKpi</string> </value>
+            <value> <string>DataAnalysisLine_calculateOrsEnbKpi</string> </value>
         </item>
         <item>
             <key> <string>title</string> </key>
-            <value> <string>Calculate ORS KPI</string> </value>
+            <value> <string>Calculate ORS eNB KPI</string> </value>
         </item>
         <item>
             <key> <string>version</string> </key>
@@ -160,7 +160,7 @@
                   </item>
                   <item>
                       <key> <string>serial</string> </key>
-                      <value> <string>1018.50460.57093.18414</string> </value>
+                      <value> <string>1021.29636.31750.64409</string> </value>
                   </item>
                   <item>
                       <key> <string>state</string> </key>
@@ -180,7 +180,7 @@
                           </tuple>
                           <state>
                             <tuple>
-                              <float>1724228941.27</float>
+                              <float>1734605116.76</float>
                               <string>UTC</string>
                             </tuple>
                           </state>

bt5/wendelin_telecom_base/PathTemplateItem/data_product_module/ors_kpi.xml → bt5/wendelin_telecom_base/PathTemplateItem/data_product_module/ors_enb_kpi.xml

@@ -89,7 +89,7 @@
         </item>
         <item>
             <key> <string>default_reference</string> </key>
-            <value> <string>ors_kpi</string> </value>
+            <value> <string>ors_enb_kpi</string> </value>
         </item>
         <item>
             <key> <string>description</string> </key>
@@ -99,7 +99,7 @@
         </item>
         <item>
             <key> <string>id</string> </key>
-            <value> <string>ors_kpi</string> </value>
+            <value> <string>ors_enb_kpi</string> </value>
         </item>
         <item>
             <key> <string>individual_variation_base_category_list</string> </key>
@@ -109,9 +109,15 @@
               </tuple>
             </value>
         </item>
+        <item>
+            <key> <string>language</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
         <item>
             <key> <string>title</string> </key>
-            <value> <string>ORS KPI</string> </value>
+            <value> <string>ORS eNB KPI</string> </value>
         </item>
         <item>
             <key> <string>workflow_history</string> </key>
@@ -203,7 +209,7 @@
                   </item>
                   <item>
                       <key> <string>serial</string> </key>
-                      <value> <string>1007.25758.5119.31470</string> </value>
+                      <value> <string>1021.29669.43776.60125</string> </value>
                   </item>
                   <item>
                       <key> <string>state</string> </key>
@@ -223,7 +229,7 @@
                           </tuple>
                           <state>
                             <tuple>
-                              <float>1681374109.0</float>
+                              <float>1734605141.66</float>
                               <string>UTC</string>
                             </tuple>
                           </state>

bt5/wendelin_telecom_base/PathTemplateItem/data_product_module/ors_enb_kpi/2.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Product Individual Variation" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>2</string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string>e_utran</string> </value>
+        </item>
+        <item>
+            <key> <string>variation_base_category_list</string> </key>
+            <value>
+              <tuple>
+                <string>variation</string>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>workflow_history</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>edit_workflow</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+                    </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_log</string> </key>
+            <value>
+              <list>
+                <dictionary>
+                  <item>
+                      <key> <string>action</string> </key>
+                      <value> <string>edit</string> </value>
+                  </item>
+                  <item>
+                      <key> <string>actor</string> </key>
+                      <value> <unicode>zope</unicode> </value>
+                  </item>
+                  <item>
+                      <key> <string>comment</string> </key>
+                      <value>
+                        <none/>
+                      </value>
+                  </item>
+                  <item>
+                      <key> <string>error_message</string> </key>
+                      <value> <string></string> </value>
+                  </item>
+                  <item>
+                      <key> <string>serial</string> </key>
+                      <value> <string>1020.53498.7256.14643</string> </value>
+                  </item>
+                  <item>
+                      <key> <string>state</string> </key>
+                      <value> <string>current</string> </value>
+                  </item>
+                  <item>
+                      <key> <string>time</string> </key>
+                      <value>
+                        <object>
+                          <klass>
+                            <global name="_reconstructor" module="copy_reg"/>
+                          </klass>
+                          <tuple>
+                            <global name="DateTime" module="DateTime.DateTime"/>
+                            <global name="object" module="__builtin__"/>
+                            <none/>
+                          </tuple>
+                          <state>
+                            <tuple>
+                              <float>1734603116.58</float>
+                              <string>UTC</string>
+                            </tuple>
+                          </state>
+                        </object>
+                      </value>
+                  </item>
+                </dictionary>
+              </list>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/wendelin_telecom_base/PathTemplateItem/data_product_module/ors_kpi/2.xml

-<?xml version="1.0"?>
-<ZopeData>
-  <record id="1" aka="AAAAAAAAAAE=">
-    <pickle>
-      <global name="Product Individual Variation" module="erp5.portal_type"/>
-    </pickle>
-    <pickle>
-      <dictionary>
-        <item>
-            <key> <string>id</string> </key>
-            <value> <string>2</string> </value>
-        </item>
-        <item>
-            <key> <string>title</string> </key>
-            <value> <string>e_utran</string> </value>
-        </item>
-        <item>
-            <key> <string>variation_base_category_list</string> </key>
-            <value>
-              <tuple>
-                <string>variation</string>
-              </tuple>
-            </value>
-        </item>
-      </dictionary>
-    </pickle>
-  </record>
-</ZopeData>

bt5/wendelin_telecom_base/PathTemplateItem/data_transformation_module/ors_enb_log_data_transformation/convert_raw_data_to_array.xml

@@ -22,7 +22,7 @@
             <key> <string>categories</string> </key>
             <value>
               <tuple>
-                <string>resource/data_operation_module/calculate_ors_kpi</string>
+                <string>resource/data_operation_module/calculate_ors_enb_kpi</string>
                 <string>trade_phase/data/convert</string>
               </tuple>
             </value>

bt5/wendelin_telecom_base/PathTemplateItem/data_transformation_module/ors_enb_log_data_transformation/e_rab_out_array.xml

@@ -31,8 +31,8 @@
             <value>
               <tuple>
                 <string>quantity_unit/unit/piece</string>
-                <string>resource/data_product_module/ors_kpi</string>
-                <string>variation/data_product_module/ors_kpi/1</string>
+                <string>resource/data_product_module/ors_enb_kpi</string>
+                <string>variation/data_product_module/ors_enb_kpi/1</string>
                 <string>trade_phase/data/subsample</string>
                 <string>use/big_data/ingestion/stream</string>
               </tuple>

bt5/wendelin_telecom_base/PathTemplateItem/data_transformation_module/ors_enb_log_data_transformation/e_utran_out_array.xml

@@ -31,10 +31,10 @@
             <value>
               <tuple>
                 <string>quantity_unit/unit/piece</string>
-                <string>resource/data_product_module/ors_kpi</string>
+                <string>resource/data_product_module/ors_enb_kpi</string>
                 <string>trade_phase/data/subsample</string>
                 <string>use/big_data/ingestion/stream</string>
-                <string>variation/data_product_module/ors_kpi/2</string>
+                <string>variation/data_product_module/ors_enb_kpi/2</string>
               </tuple>
             </value>
         </item>

bt5/wendelin_telecom_base/PathTemplateItem/portal_callables/DataAnalysisLine_calculateOrsEnbKpi.py

+import json
+import numpy as np
+
+HISTORY_LINE_COUNT = 64
+T_PERIOD = 30 # seconds
+QCI_COUNT = 256
+
+in_data_stream = in_stream['Data Stream']
+progress_indicator = in_stream['Progress Indicator']
+offset_index = progress_indicator.getIntOffsetIndex()
+start = 0
+end = in_data_stream.getSize()
+
+# No new data to process
+if offset_index >= end:
+  return
+
+previous_log_data = ''.join(in_data_stream.readChunkList(start, offset_index))
+new_log_data = ''.join(in_data_stream.readChunkList(offset_index, end))
+
+previous_log_data_line_list = previous_log_data.splitlines()
+new_log_data_line_list = new_log_data.splitlines()
+
+log_data_line_list = previous_log_data_line_list + new_log_data_line_list
+
+# Sort data lines by UTC timestamp
+def get_log_line_timestamp(log_line):
+  try:
+    log_line_json = json.loads(log_line)
+  except ValueError:
+    # Invalid JSON
+    return 0.0
+  if 'meta' in log_line_json:
+    if 'srv_utc' in log_line_json['meta']:
+      return float(log_line_json['meta']['srv_utc'])
+    else:
+      return float(log_line_json['meta']['time'])
+  else:
+    return float(log_line_json.get('utc', 0.0))
+log_data_line_list.sort(key=get_log_line_timestamp)
+
+# Find the index of the earliest timestamp in the new data
+new_log_data_line_first_index = min([
+  log_data_line_list.index(new_log_line) for new_log_line in new_log_data_line_list
+])
+
+# Use partial history of previous logs starting from the earliest timestamp of the new data
+# This also gives some tolerance to log errors
+history_start_index = max(0, new_log_data_line_first_index - HISTORY_LINE_COUNT)
+
+log_data = '\n'.join(log_data_line_list[history_start_index:])
+log_data = log_data.decode('utf8')
+
+e_rab_data_array = None
+e_utran_data_array = None
+
+for array in out_array:
+  if array['variation'] == 'e_rab':
+    e_rab_data_array = array['Data Array']
+  if array['variation'] == 'e_utran':
+    e_utran_data_array = array['Data Array']
+
+# Calculate the KPI data
+kpi_data_dict = context.Base_calcEnbKpi(log_data, T_PERIOD)
+vt, v_initial_epsb_estab_sr, v_added_epsb_estab_sr = kpi_data_dict['e_rab_accessibility']
+evt, v_ip_throughput_qci = kpi_data_dict['e_utran_ip_throughput']
+
+e_rab_dtype = np.dtype([
+  ('vt', 'float'),
+  ('vInitialEPSBEstabSR_lo', 'float64'),
+  ('vInitialEPSBEstabSR_hi', 'float64'),
+  ('vAddedEPSBEstabSR_lo', 'float64'),
+  ('vAddedEPSBEstabSR_hi', 'float64'),
+])
+e_utran_dtype = np.dtype([
+  ('evt', 'float'),
+  ('dl_lo', 'float64'),
+  ('dl_hi', 'float64'),
+  ('ul_lo', 'float64'),
+  ('ul_hi', 'float64'),
+])
+
+e_rab_array = e_rab_data_array.getArray()
+if not e_rab_array:
+  e_rab_array = e_rab_data_array.initArray(shape=(0,), dtype=e_rab_dtype)
+e_rab_array_data = []
+
+e_utran_array = e_utran_data_array.getArray()
+if not e_utran_array:
+  e_utran_array = e_utran_data_array.initArray(shape=(0,), dtype=e_utran_dtype)
+e_utran_array_data = []
+
+# Don't duplicate KPI data:
+# search and start inserting new timestamps from the first one
+vt_column = e_rab_array[:]['vt']
+first_new_row_vt = 0
+while (first_new_row_vt < len(vt) and vt[first_new_row_vt] in vt_column):
+  first_new_row_vt += 1
+
+for i in range(first_new_row_vt, len(vt)):
+  if vt[i] not in vt_column:
+    e_rab_array_data.append((
+      vt[i],
+      v_initial_epsb_estab_sr[i][0],
+      v_initial_epsb_estab_sr[i][1],
+      v_added_epsb_estab_sr[i][0],
+      v_added_epsb_estab_sr[i][1]
+    ))
+
+evt_column = e_utran_array[::QCI_COUNT]['evt']
+first_new_row_evt = 0
+while (first_new_row_evt < len(evt) and evt[first_new_row_evt] in evt_column):
+  first_new_row_evt += 1
+
+for i in range(first_new_row_evt, len(evt)):
+  if evt[i] not in evt_column:
+    for qci_data in v_ip_throughput_qci[i]:
+      e_utran_array_data.append((evt[i], qci_data[0], qci_data[1], qci_data[2], qci_data[3]))
+
+if e_rab_array_data:
+  e_rab_array_data = np.ndarray((len(e_rab_array_data),), e_rab_dtype, np.array(e_rab_array_data))
+  e_rab_array.append(e_rab_array_data)
+
+if e_utran_array_data:
+  e_utran_array_data = np.ndarray((len(e_utran_array_data),), e_utran_dtype, np.array(e_utran_array_data))
+  e_utran_array.append(e_utran_array_data)
+
+progress_indicator.setIntOffsetIndex(end)
+return

bt5/wendelin_telecom_base/PathTemplateItem/portal_callables/DataAnalysisLine_calculateOrsKpi.xml → bt5/wendelin_telecom_base/PathTemplateItem/portal_callables/DataAnalysisLine_calculateOrsEnbKpi.xml

@@ -77,7 +77,7 @@
         </item>
         <item>
             <key> <string>id</string> </key>
-            <value> <string>DataAnalysisLine_calculateOrsKpi</string> </value>
+            <value> <string>DataAnalysisLine_calculateOrsEnbKpi</string> </value>
         </item>
         <item>
             <key> <string>reference</string> </key>
@@ -85,7 +85,7 @@
         </item>
         <item>
             <key> <string>title</string> </key>
-            <value> <string>DataAnalysisLine_calculateOrsKpi</string> </value>
+            <value> <string>DataAnalysisLine_calculateOrsEnbKpi</string> </value>
         </item>
       </dictionary>
     </pickle>

bt5/wendelin_telecom_base/PathTemplateItem/portal_callables/DataAnalysisLine_calculateOrsKpi.py

-import numpy as np
-
-progress_indicator = in_stream["Progress Indicator"]
-in_data_stream = in_stream["Data Stream"]
-start = progress_indicator.getIntOffsetIndex()
-total_size = in_data_stream.getSize()
-
-if start >= total_size:
-  return
-
-chunk_size = 512 * 512
-end = min(start + chunk_size, total_size)
-
-log_data = ''.join(in_data_stream.readChunkList(start, end))
-
-# Some lines may contain several contiguous JSON objects
-# Normalize to one JSON object per line
-newlines_offset = log_data.count('}{')
-log_data = log_data.replace('}{', '}\n{')
-
-# Last chunk may contain an incomplete JSON object: leave it for next iteration
-log_data_lines = log_data.splitlines()
-if len(log_data_lines) < 2:
-  return
-log_data = '\n'.join(log_data_lines[:-1])
-
-end = start + len(log_data) - newlines_offset + 1
-
-log_data = log_data.decode('utf8')
-
-e_rab_data_array = None
-e_utran_data_array = None
-
-for array in out_array:
-  if array['variation'] == 'e_rab':
-    e_rab_data_array = array['Data Array']
-  if array['variation'] == 'e_utran':
-    e_utran_data_array = array['Data Array']
-
-vt, v_initial_epsb_estab_sr, v_added_epsb_estab_sr, evt, v_ip_throughput_qci = context.Base_getOrsKpiValues(log_data)
-
-e_rab_dtype = np.dtype([
-  ('vt', 'float'),
-  ('vInitialEPSBEstabSR_lo', 'float64'),
-  ('vInitialEPSBEstabSR_hi', 'float64'),
-  ('vAddedEPSBEstabSR_lo', 'float64'),
-  ('vAddedEPSBEstabSR_hi', 'float64'),
-])
-
-e_utran_dtype = np.dtype([
-  ('evt', 'float'),
-  ('qci', 'float'),
-  ('dl_lo', 'float64'),
-  ('dl_hi', 'float64'),
-  ('ul_lo', 'float64'),
-  ('ul_hi', 'float64'),
-])
-
-e_rab_array = e_rab_data_array.getArray()
-if not e_rab_array:
-  e_rab_array = e_rab_data_array.initArray(shape=(0,), dtype=e_rab_dtype)
-e_rab_array_data = []
-
-for i in range(len(vt)):
-  e_rab_array_data.append((vt[i], v_initial_epsb_estab_sr[i][0], v_initial_epsb_estab_sr[i][1], v_added_epsb_estab_sr[i][0], v_added_epsb_estab_sr[i][1]))
-
-if e_rab_array_data:
-  e_rab_array_data =  np.ndarray((len(e_rab_array_data), ), e_rab_dtype, np.array(e_rab_array_data))
-  e_rab_array.append(e_rab_array_data)
-
-e_utran_array = e_utran_data_array.getArray()
-if not e_utran_array:
-  e_utran_array = e_utran_data_array.initArray(shape=(0,), dtype=e_utran_dtype)
-e_utran_array_data = []
-seen_qci_list = []
-i = e_utran_array.shape[0] - 1
-qci_idx = 1
-while (i >= 0) and (e_utran_array[i][qci_idx] not in seen_qci_list):
-  seen_qci_list.append(e_utran_array[i][qci_idx])
-  i -= 1
-
-for i in range(len(evt)):
-  for qci_data in v_ip_throughput_qci[i]:
-    qci = qci_data[0]
-    qci_data_hi = [qci_data[2], qci_data[4]]
-    if qci in seen_qci_list or any([data != 0 for data in qci_data_hi]):
-      e_utran_array_data.append((evt[i], qci, qci_data[1], qci_data[2], qci_data[3], qci_data[4]))
-
-if e_utran_array_data:
-  e_utran_array_data =  np.ndarray((len(e_utran_array_data), ), e_utran_dtype, np.array(e_utran_array_data))
-  e_utran_array.append(e_utran_array_data)
-
-progress_indicator.setIntOffsetIndex(end)
-return

bt5/wendelin_telecom_base/PathTemplateItem/portal_callables/DataIngestionLine_writeOrsFluentdIngestionToDataStream.py

@@ -5,4 +5,7 @@ It will first unpack the MsgPack, then remove the first item of the tuple (times
 get the actual data stored under the 'log' key and append the corresponding string to "Data Stream".
 """
 
-out_stream["Data Stream"].appendData('\n'.join([c[1]['log'] for c in context.unpack(data_chunk)]))
+unpacked_data = '\n'.join([c[1]["log"] for c in context.unpack(data_chunk)])
+if unpacked_data == '':
+  return
+out_stream["Data Stream"].appendData(unpacked_data + '\n')

bt5/wendelin_telecom_base/PathTemplateItem/web_page_module/ors_e_rab_kpi_html.html → bt5/wendelin_telecom_base/PathTemplateItem/web_page_module/ors_e_rab_accessibility_kpi_html.html

@@ -10,10 +10,8 @@
     <script src="renderjs.js" type="text/javascript"></script>
 
     <!-- custom script -->
-    <script src="ndarray_bundle.js" type="text/javascript"></script>
-    <script src="wendelin.js" type="text/javascript"></script>
     <script src="plotly_strict_v2.34.0.js" type="text/javascript"></script>
-    <script src="gadget_ors_e_rab_kpi.js" type="text/javascript"></script>
+    <script src="gadget_ors_e_rab_accessibility_kpi.js" type="text/javascript"></script>
     <link href="gadget_ors_kpi.css" rel="stylesheet" type="text/css">
   </head>
   <body>

bt5/wendelin_telecom_base/PathTemplateItem/web_page_module/ors_e_rab_kpi_html.xml → bt5/wendelin_telecom_base/PathTemplateItem/web_page_module/ors_e_rab_accessibility_kpi_html.xml

@@ -73,7 +73,7 @@
         </item>
         <item>
             <key> <string>default_reference</string> </key>
-            <value> <string>gadget_ors_e_rab_kpi.html</string> </value>
+            <value> <string>gadget_ors_e_rab_accessibility_kpi.html</string> </value>
         </item>
         <item>
             <key> <string>description</string> </key>
@@ -83,7 +83,7 @@
         </item>
         <item>
             <key> <string>id</string> </key>
-            <value> <string>ors_e_rab_kpi_html</string> </value>
+            <value> <string>ors_e_rab_accessibility_kpi_html</string> </value>
         </item>
         <item>
             <key> <string>language</string> </key>
@@ -101,7 +101,7 @@
         </item>
         <item>
             <key> <string>title</string> </key>
-            <value> <string>Gadget ORS E RAB KPI HTML</string> </value>
+            <value> <string>Gadget ORS E RAB Accessibility KPI HTML</string> </value>
         </item>
         <item>
             <key> <string>version</string> </key>
@@ -132,18 +132,6 @@
                       <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
                     </value>
                 </item>
-                <item>
-                    <key> <string>edit_workflow</string> </key>
-                    <value>
-                      <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
-                    </value>
-                </item>
-                <item>
-                    <key> <string>processing_status_workflow</string> </key>
-                    <value>
-                      <persistent> <string encoding="base64">AAAAAAAAAAU=</string> </persistent>
-                    </value>
-                </item>
               </dictionary>
             </value>
         </item>
@@ -167,7 +155,7 @@
                   </item>
                   <item>
                       <key> <string>actor</string> </key>
-                      <value> <string>zope</string> </value>
+                      <value> <unicode>zope</unicode> </value>
                   </item>
                   <item>
                       <key> <string>comment</string> </key>
@@ -191,7 +179,7 @@
                           </tuple>
                           <state>
                             <tuple>
-                              <float>1680075443.84</float>
+                              <float>1734617150.07</float>
                               <string>UTC</string>
                             </tuple>
                           </state>
@@ -209,132 +197,4 @@
       </dictionary>
     </pickle>
   </record>
-  <record id="4" aka="AAAAAAAAAAQ=">
-    <pickle>
-      <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
-    </pickle>
-    <pickle>
-      <dictionary>
-        <item>
-            <key> <string>_log</string> </key>
-            <value>
-              <list>
-                <dictionary>
-                  <item>
-                      <key> <string>action</string> </key>
-                      <value> <string>edit</string> </value>
-                  </item>
-                  <item>
-                      <key> <string>actor</string> </key>
-                      <value> <unicode>zope</unicode> </value>
-                  </item>
-                  <item>
-                      <key> <string>comment</string> </key>
-                      <value>
-                        <none/>
-                      </value>
-                  </item>
-                  <item>
-                      <key> <string>error_message</string> </key>
-                      <value> <string></string> </value>
-                  </item>
-                  <item>
-                      <key> <string>serial</string> </key>
-                      <value> <string>1018.50745.28244.6348</string> </value>
-                  </item>
-                  <item>
-                      <key> <string>state</string> </key>
-                      <value> <string>current</string> </value>
-                  </item>
-                  <item>
-                      <key> <string>time</string> </key>
-                      <value>
-                        <object>
-                          <klass>
-                            <global name="_reconstructor" module="copy_reg"/>
-                          </klass>
-                          <tuple>
-                            <global name="DateTime" module="DateTime.DateTime"/>
-                            <global name="object" module="__builtin__"/>
-                            <none/>
-                          </tuple>
-                          <state>
-                            <tuple>
-                              <float>1724246207.17</float>
-                              <string>UTC</string>
-                            </tuple>
-                          </state>
-                        </object>
-                      </value>
-                  </item>
-                </dictionary>
-              </list>
-            </value>
-        </item>
-      </dictionary>
-    </pickle>
-  </record>
-  <record id="5" aka="AAAAAAAAAAU=">
-    <pickle>
-      <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
-    </pickle>
-    <pickle>
-      <dictionary>
-        <item>
-            <key> <string>_log</string> </key>
-            <value>
-              <list>
-                <dictionary>
-                  <item>
-                      <key> <string>action</string> </key>
-                      <value> <string>detect_converted_file</string> </value>
-                  </item>
-                  <item>
-                      <key> <string>actor</string> </key>
-                      <value> <string>zope</string> </value>
-                  </item>
-                  <item>
-                      <key> <string>comment</string> </key>
-                      <value> <string></string> </value>
-                  </item>
-                  <item>
-                      <key> <string>error_message</string> </key>
-                      <value> <string></string> </value>
-                  </item>
-                  <item>
-                      <key> <string>external_processing_state</string> </key>
-                      <value> <string>converted</string> </value>
-                  </item>
-                  <item>
-                      <key> <string>serial</string> </key>
-                      <value> <string>0.0.0.0</string> </value>
-                  </item>
-                  <item>
-                      <key> <string>time</string> </key>
-                      <value>
-                        <object>
-                          <klass>
-                            <global name="_reconstructor" module="copy_reg"/>
-                          </klass>
-                          <tuple>
-                            <global name="DateTime" module="DateTime.DateTime"/>
-                            <global name="object" module="__builtin__"/>
-                            <none/>
-                          </tuple>
-                          <state>
-                            <tuple>
-                              <float>1680075412.12</float>
-                              <string>UTC</string>
-                            </tuple>
-                          </state>
-                        </object>
-                      </value>
-                  </item>
-                </dictionary>
-              </list>
-            </value>
-        </item>
-      </dictionary>
-    </pickle>
-  </record>
 </ZopeData>

bt5/wendelin_telecom_base/PathTemplateItem/web_page_module/ors_e_rab_accessibility_kpi_js.js

+/*global window, rJS, RSVP, console, Plotly */
+/*jslint indent: 2, maxlen: 80, nomen: true */
+(function (window, rJS, RSVP, Plotly, loopEventListener) {
+  'use strict';
+
+  var gadget_klass = rJS(window);
+
+  function getNoDataPlotLayout(title, annotation) {
+    return {
+      'title' : title,
+      'xaxis': {
+        'fixedrange': true
+      },
+      'yaxis': {
+        'fixedrange': true
+      },
+      'annotations': [
+        {
+          'text': annotation,
+          'xref': 'paper',
+          'yref': 'paper',
+          'showarrow': false,
+          'font': {
+            'size': 28
+          }
+        }
+      ]
+    };
+  }
+
+  function getDataPlotLayout(title) {
+    return {
+      'title' : title,
+      'xaxis': {
+        title: {
+          text: 'Date'
+        },
+        'autorange': true,
+        'autosize': true
+      },
+      'yaxis': {
+        'fixedrange': true,
+        title: {
+          text: '%'
+        }
+      }
+    };
+  }
+
+  function plotFromResponse(response, element, data_type) {
+    var kpi_data_dict,
+      date = [],
+      epsb_estab_sr_lo = [],
+      epsb_estab_sr_hi = [],
+      success_rate_data = [],
+      data_name = '';
+
+    kpi_data_dict = response;
+
+    if (Object.keys(kpi_data_dict).length === 0 ||
+        kpi_data_dict.vt.length === 0) {
+      Plotly.react(
+        element,
+        [],
+        getNoDataPlotLayout(data_type, 'No data found')
+      );
+      return;
+    }
+
+    kpi_data_dict.vt.forEach(function (element) {
+      date.push(new Date(element * 1000));
+    });
+
+    if (data_type === 'Initial E-RAB establishment success rate') {
+      epsb_estab_sr_lo = kpi_data_dict.v_initial_epsb_estab_sr_lo;
+      epsb_estab_sr_hi = kpi_data_dict.v_initial_epsb_estab_sr_hi;
+      data_name = 'InitialEPSBEstabSR';
+    } else if (data_type === 'Added E-RAB establishment success rate') {
+      epsb_estab_sr_lo = kpi_data_dict.v_added_epsb_estab_sr_lo;
+      epsb_estab_sr_hi = kpi_data_dict.v_added_epsb_estab_sr_hi;
+      data_name = 'AddedEPSBEstabSR';
+    }
+
+    success_rate_data = [
+      {
+        x: date,
+        mode: 'lines+markers',
+        y: epsb_estab_sr_lo,
+        type: 'scatter',
+        line: {shape: 'hv'},
+        hovertemplate: 'Date: %{x}<br>Rate: %{y}%',
+        name: data_name
+      },
+      {
+        x: date,
+        mode: 'lines+markers',
+        fill: 'tonexty',
+        y: epsb_estab_sr_hi,
+        type: 'scatter',
+        line: {
+          color: '#6cb9e5',
+          shape: 'hv'
+        },
+        hovertemplate: 'Date: %{x}<br>Rate: %{y}%',
+        name: data_name + ' uncertainty'
+      }
+    ];
+    Plotly.react(
+      element,
+      success_rate_data,
+      getDataPlotLayout(data_type)
+    );
+  }
+
+  gadget_klass
+    .declareAcquiredMethod('getSetting', 'getSetting')
+    .declareAcquiredMethod('jio_get', 'jio_get')
+    .declareAcquiredMethod('jio_getAttachment', 'jio_getAttachment')
+    .declareService(function () {
+      var gadget = this;
+      return loopEventListener(window, 'resize', false, function () {
+        Plotly.Plots.resize(
+          gadget.element.querySelector('.graph-initial-success-rate')
+        );
+        Plotly.Plots.resize(
+          gadget.element.querySelector('.graph-added-success-rate')
+        );
+      });
+    })
+    .declareMethod('render', function (option_dict) {
+      var gadget = this,
+        kpi_url,
+        initial_success_rate_element =
+          gadget.element.querySelector('.graph-initial-success-rate'),
+        added_success_rate_element =
+          gadget.element.querySelector('.graph-added-success-rate');;
+
+      return new RSVP.Queue().push(function () {
+        return gadget.getSetting('hateoas_url');
+      })
+        .push(function (hateoas_url) {
+          kpi_url =
+            (new URI(hateoas_url)).absoluteTo(location.href).toString() +
+            'Base_getOrsEnbKpi?data_array_url=' +
+            option_dict.data_array_url +
+            '&kpi_type=' + option_dict.kpi_type;
+          return gadget.jio_getAttachment('erp5', kpi_url, {
+            format: 'json'
+          });
+        })
+        .push(function (response) {
+          plotFromResponse(
+            response,
+            initial_success_rate_element,
+            'Initial E-RAB establishment success rate'
+          );
+          plotFromResponse(
+            response,
+            added_success_rate_element,
+            'Added E-RAB establishment success rate'
+          );
+          gadget.element.querySelector('.ui-icon-spinner').hidden = true;
+
+          initial_success_rate_element.on(
+            'plotly_relayout',
+            function (eventdata) {
+              var x_start = new Date(eventdata['xaxis.range[0]']).getTime()
+                / 1000,
+                x_end = new Date(eventdata['xaxis.range[1]']).getTime()
+                / 1000,
+                update_kpi_url = kpi_url +
+                '&time_start=' + x_start +
+                '&time_end=' + x_end;
+
+              return new RSVP.Queue().push(function () {
+                return gadget.jio_getAttachment('erp5', update_kpi_url, {
+                  format: 'json'
+                });
+              })
+                .push(function (response) {
+                  plotFromResponse(
+                    response,
+                    initial_success_rate_element,
+                    'Initial E-RAB establishment success rate'
+                  );
+                });
+            }
+          );
+          added_success_rate_element.on(
+            'plotly_relayout',
+            function (eventdata) {
+              var x_start = new Date(eventdata['xaxis.range[0]']).getTime()
+                / 1000,
+                x_end = new Date(eventdata['xaxis.range[1]']).getTime()
+                / 1000,
+                update_kpi_url = kpi_url +
+                '&time_start=' + x_start +
+                '&time_end=' + x_end;
+
+              return new RSVP.Queue().push(function () {
+                return gadget.jio_getAttachment('erp5', update_kpi_url, {
+                  format: 'json'
+                });
+              })
+                .push(function (response) {
+                  plotFromResponse(
+                    response,
+                    added_success_rate_element,
+                    'Added E-RAB establishment success rate'
+                  );
+                });
+            }
+          );
+        }, function () {
+          // On request error, show empty plots
+          plotFromResponse(
+            {},
+            initial_success_rate_element,
+            'Initial E-RAB establishment success rate'
+          );
+          plotFromResponse(
+            {},
+            added_success_rate_element,
+            'Added E-RAB establishment success rate'
+          );
+          gadget.element.querySelector('.ui-icon-spinner').hidden = true;
+        });
+    });
+
+}(window, rJS, RSVP, Plotly, loopEventListener));

bt5/wendelin_telecom_base/PathTemplateItem/web_page_module/ors_e_rab_kpi_js.xml → bt5/wendelin_telecom_base/PathTemplateItem/web_page_module/ors_e_rab_accessibility_kpi_js.xml

@@ -75,7 +75,7 @@
         </item>
         <item>
             <key> <string>default_reference</string> </key>
-            <value> <string>gadget_ors_e_rab_kpi.js</string> </value>
+            <value> <string>gadget_ors_e_rab_accessibility_kpi.js</string> </value>
         </item>
         <item>
             <key> <string>description</string> </key>
@@ -85,7 +85,7 @@
         </item>
         <item>
             <key> <string>id</string> </key>
-            <value> <string>ors_e_rab_kpi_js</string> </value>
+            <value> <string>ors_e_rab_accessibility_kpi_js</string> </value>
         </item>
         <item>
             <key> <string>language</string> </key>
@@ -99,7 +99,7 @@
         </item>
         <item>
             <key> <string>title</string> </key>
-            <value> <string>Gadget ORS E RAB KPI JS</string> </value>
+            <value> <string>Gadget ORS E RAB Accessibility KPI JS</string> </value>
         </item>
         <item>
             <key> <string>version</string> </key>
@@ -130,18 +130,6 @@
                       <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
                     </value>
                 </item>
-                <item>
-                    <key> <string>edit_workflow</string> </key>
-                    <value>
-                      <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
-                    </value>
-                </item>
-                <item>
-                    <key> <string>processing_status_workflow</string> </key>
-                    <value>
-                      <persistent> <string encoding="base64">AAAAAAAAAAU=</string> </persistent>
-                    </value>
-                </item>
               </dictionary>
             </value>
         </item>
@@ -165,7 +153,7 @@
                   </item>
                   <item>
                       <key> <string>actor</string> </key>
-                      <value> <string>zope</string> </value>
+                      <value> <unicode>zope</unicode> </value>
                   </item>
                   <item>
                       <key> <string>comment</string> </key>
@@ -189,7 +177,7 @@
                           </tuple>
                           <state>
                             <tuple>
-                              <float>1680075654.87</float>
+                              <float>1734617161.66</float>
                               <string>UTC</string>
                             </tuple>
                           </state>
@@ -207,134 +195,4 @@
       </dictionary>
     </pickle>
   </record>
-  <record id="4" aka="AAAAAAAAAAQ=">
-    <pickle>
-      <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
-    </pickle>
-    <pickle>
-      <dictionary>
-        <item>
-            <key> <string>_log</string> </key>
-            <value>
-              <list>
-                <dictionary>
-                  <item>
-                      <key> <string>action</string> </key>
-                      <value> <string>edit</string> </value>
-                  </item>
-                  <item>
-                      <key> <string>actor</string> </key>
-                      <value> <unicode>zope</unicode> </value>
-                  </item>
-                  <item>
-                      <key> <string>comment</string> </key>
-                      <value>
-                        <none/>
-                      </value>
-                  </item>
-                  <item>
-                      <key> <string>error_message</string> </key>
-                      <value> <string></string> </value>
-                  </item>
-                  <item>
-                      <key> <string>serial</string> </key>
-                      <value> <string>1019.54468.57368.35892</string> </value>
-                  </item>
-                  <item>
-                      <key> <string>state</string> </key>
-                      <value> <string>current</string> </value>
-                  </item>
-                  <item>
-                      <key> <string>time</string> </key>
-                      <value>
-                        <object>
-                          <klass>
-                            <global name="_reconstructor" module="copy_reg"/>
-                          </klass>
-                          <tuple>
-                            <global name="DateTime" module="DateTime.DateTime"/>
-                            <global name="object" module="__builtin__"/>
-                            <none/>
-                          </tuple>
-                          <state>
-                            <tuple>
-                              <float>1728632990.63</float>
-                              <string>UTC</string>
-                            </tuple>
-                          </state>
-                        </object>
-                      </value>
-                  </item>
-                </dictionary>
-              </list>
-            </value>
-        </item>
-      </dictionary>
-    </pickle>
-  </record>
-  <record id="5" aka="AAAAAAAAAAU=">
-    <pickle>
-      <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
-    </pickle>
-    <pickle>
-      <dictionary>
-        <item>
-            <key> <string>_log</string> </key>
-            <value>
-              <list>
-                <dictionary>
-                  <item>
-                      <key> <string>action</string> </key>
-                      <value>
-                        <none/>
-                      </value>
-                  </item>
-                  <item>
-                      <key> <string>actor</string> </key>
-                      <value> <string>zope</string> </value>
-                  </item>
-                  <item>
-                      <key> <string>comment</string> </key>
-                      <value> <string></string> </value>
-                  </item>
-                  <item>
-                      <key> <string>error_message</string> </key>
-                      <value> <string></string> </value>
-                  </item>
-                  <item>
-                      <key> <string>external_processing_state</string> </key>
-                      <value> <string>empty</string> </value>
-                  </item>
-                  <item>
-                      <key> <string>serial</string> </key>
-                      <value> <string>0.0.0.0</string> </value>
-                  </item>
-                  <item>
-                      <key> <string>time</string> </key>
-                      <value>
-                        <object>
-                          <klass>
-                            <global name="_reconstructor" module="copy_reg"/>
-                          </klass>
-                          <tuple>
-                            <global name="DateTime" module="DateTime.DateTime"/>
-                            <global name="object" module="__builtin__"/>
-                            <none/>
-                          </tuple>
-                          <state>
-                            <tuple>
-                              <float>1680075617.54</float>
-                              <string>UTC</string>
-                            </tuple>
-                          </state>
-                        </object>
-                      </value>
-                  </item>
-                </dictionary>
-              </list>
-            </value>
-        </item>
-      </dictionary>
-    </pickle>
-  </record>
 </ZopeData>

bt5/wendelin_telecom_base/PathTemplateItem/web_page_module/ors_e_rab_kpi_js.js

-/*global window, rJS, RSVP, console, Plotly */
-/*jslint nomen: true, indent: 2 */
-(function (window, rJS, RSVP, Plotly, wendelin, loopEventListener) {
-  "use strict";
-
-  var gadget_klass = rJS(window);
-
-  gadget_klass
-    .declareAcquiredMethod("getSetting", "getSetting")
-    .declareAcquiredMethod("jio_get", "jio_get")
-    .declareAcquiredMethod("jio_getAttachment", "jio_getAttachment")
-    .declareService(function () {
-      var gadget = this;
-      return loopEventListener(window, 'resize', false, function () {
-        Plotly.Plots.resize(gadget.element.querySelector('.graph-initial-success-rate'));
-        Plotly.Plots.resize(gadget.element.querySelector('.graph-added-success-rate'));
-      });
-    })
-    .declareMethod('render', function (option_dict) {
-      var gadget = this,
-        i,
-        initial_success_data,
-        added_success_data,
-        label_list = ["vt", "vInitialEPSBEstabSR_lo", "vInitialEPSBEstabSR_hi", "vAddedEPSBEstabSR_lo", "vAddedEPSBEstabSR_hi"];
-      return new RSVP.Queue().push(function () {
-        return wendelin.getArrayRawSlice(gadget, option_dict.data_array_key);
-      })
-        .push(function (result) {
-          var graph_data = nj.unpack(result.pick(null, label_list)),
-            date = [],
-            vInitialEPSBEstabSR_lo = [],
-            vInitialEPSBEstabSR_hi = [],
-            vAddedEPSBEstabSR_lo = [],
-            vAddedEPSBEstabSR_hi = [];
-          graph_data.sort(function (a, b) {
-            return (a[0] - b[0]);
-          });
-          for (i = 0; i < graph_data.length; i += 1) {
-            date.push(new Date(graph_data[i][0] * 1000));
-            vInitialEPSBEstabSR_lo.push(graph_data[i][1]);
-            vInitialEPSBEstabSR_hi.push(graph_data[i][2]);
-            vAddedEPSBEstabSR_lo.push(graph_data[i][3]);
-            vAddedEPSBEstabSR_hi.push(graph_data[i][4]);
-          }
-          initial_success_data = [
-            {
-              x: date,
-              mode: 'lines+markers',
-              y: vInitialEPSBEstabSR_lo,
-              type: 'scatter',
-              line: {shape: 'hv'},
-              hovertemplate: 'Date: %{x}<br>Rate: %{y}%',
-              name: 'InitialEPSBEstabSR'
-            },
-            {
-              x: date,
-              mode: 'lines+markers',
-              fill: 'tonexty',
-              y: vInitialEPSBEstabSR_hi,
-              type: 'scatter',
-              line: {
-                color: "#6cb9e5",
-                shape: 'hv'
-              },
-              hovertemplate: 'Date: %{x}<br>Rate: %{y}%',
-              name: 'InitialEPSBEstabSR uncertainty'
-            }
-          ];
-          added_success_data = [
-            {
-              x: date,
-              mode: 'lines+markers',
-              y: vAddedEPSBEstabSR_lo,
-              type: 'scatter',
-              line: {shape: 'hv'},
-              hovertemplate: 'Date: %{x}<br>Rate: %{y}%',
-              name: 'AddedEPSBEstabSR'
-            },
-            {
-              x: date,
-              mode: 'lines+markers',
-              fill: 'tonexty',
-              y: vAddedEPSBEstabSR_hi,
-              type: 'scatter',
-              line: {
-                color: "#6cb9e5",
-                shape: 'hv'
-              },
-              hovertemplate: 'Date: %{x}<br>Rate: %{y}%',
-              name: 'AddedEPSBEstabSR uncertainty'
-            }
-          ];
-          Plotly.newPlot(
-            gadget.element.querySelector('.graph-initial-success-rate'),
-            initial_success_data,
-            {
-              'title' : 'Initial E-RAB establishment success rate',
-              'xaxis': {
-                'autorange': true,
-                'autosize': true
-              },
-              'yaxis': {
-                title: {
-                  text: '%'
-                }
-              }
-            }
-          );
-          Plotly.newPlot(
-            gadget.element.querySelector('.graph-added-success-rate'),
-            added_success_data,
-            {
-              'title' : 'Added E-RAB establishment success rate',
-              'xaxis': {
-                'autorange': true,
-                'autosize': true
-              },
-              'yaxis': {
-                title: {
-                  text: '%'
-                }
-              }
-            }
-          );
-          gadget.element.querySelector('.ui-icon-spinner').hidden = true;
-        });
-    });
-
-}(window, rJS, RSVP, Plotly, wendelin, loopEventListener));

bt5/wendelin_telecom_base/PathTemplateItem/web_page_module/ors_e_utran_ip_throughput_kpi_html.html

@@ -10,15 +10,13 @@
     <script src="renderjs.js" type="text/javascript"></script>
 
     <!-- custom script -->
-    <script src="ndarray_bundle.js" type="text/javascript"></script>
-    <script src="wendelin.js" type="text/javascript"></script>
     <script src="plotly_strict_v2.34.0.js" type="text/javascript"></script>
     <script src="gadget_ors_e_utran_ip_throughput_kpi.js" type="text/javascript"></script>
     <link href="gadget_ors_kpi.css" rel="stylesheet" type="text/css">
   </head>
   <body>
-    <div class="graph-download-link"></div>
-    <div class="graph-upload-link"></div>
+    <div class="graph-downlink"></div>
+    <div class="graph-uplink"></div>
     <div class="ui-icon-spinner ui-btn-icon-notext first-loader"></div>
   </body>
 </html>
\ No newline at end of file

bt5/wendelin_telecom_base/PathTemplateItem/web_page_module/ors_e_utran_ip_throughput_kpi_html.xml

@@ -240,7 +240,7 @@
                   </item>
                   <item>
                       <key> <string>serial</string> </key>
-                      <value> <string>1018.60805.51967.7987</string> </value>
+                      <value> <string>1020.64896.36157.8226</string> </value>
                   </item>
                   <item>
                       <key> <string>state</string> </key>
@@ -260,7 +260,7 @@
                           </tuple>
                           <state>
                             <tuple>
-                              <float>1724849643.71</float>
+                              <float>1732873094.4</float>
                               <string>UTC</string>
                             </tuple>
                           </state>

bt5/wendelin_telecom_base/PathTemplateItem/web_page_module/ors_e_utran_ip_throughput_kpi_js.xml

@@ -238,7 +238,7 @@
                   </item>
                   <item>
                       <key> <string>serial</string> </key>
-                      <value> <string>1019.54468.57368.35892</string> </value>
+                      <value> <string>1021.10867.58743.39594</string> </value>
                   </item>
                   <item>
                       <key> <string>state</string> </key>
@@ -258,7 +258,7 @@
                           </tuple>
                           <state>
                             <tuple>
-                              <float>1728633034.89</float>
+                              <float>1733477211.82</float>
                               <string>UTC</string>
                             </tuple>
                           </state>

bt5/wendelin_telecom_base/PathTemplateItem/web_page_module/plotly_strict_v2.34.0.js.xml

@@ -238,7 +238,7 @@
                   </item>
                   <item>
                       <key> <string>serial</string> </key>
-                      <value> <string>1018.50736.46413.16776</string> </value>
+                      <value> <string>1020.52201.24472.32051</string> </value>
                   </item>
                   <item>
                       <key> <string>state</string> </key>
@@ -258,7 +258,7 @@
                           </tuple>
                           <state>
                             <tuple>
-                              <float>1724245506.16</float>
+                              <float>1732111313.58</float>
                               <string>UTC</string>
                             </tuple>
                           </state>

bt5/wendelin_telecom_base/SkinTemplateItem/portal_skins/wendelin_telecom_base/Base_getOrsKpiValues.xml → bt5/wendelin_telecom_base/SkinTemplateItem/portal_skins/wendelin_telecom_base/Base_calcEnbKpi.xml

@@ -8,15 +8,15 @@
       <dictionary>
         <item>
             <key> <string>_function</string> </key>
-            <value> <string>getOrsKpiValues</string> </value>
+            <value> <string>calcEnbKpi</string> </value>
         </item>
         <item>
             <key> <string>_module</string> </key>
-            <value> <string>OrsKpiUtils</string> </value>
+            <value> <string>WendelinTelecomCalcEnbKpi</string> </value>
         </item>
         <item>
             <key> <string>id</string> </key>
-            <value> <string>Base_getOrsKpiValues</string> </value>
+            <value> <string>Base_calcEnbKpi</string> </value>
         </item>
         <item>
             <key> <string>title</string> </key>

bt5/wendelin_telecom_base/PathTemplateItem/data_product_module/ors_kpi/1.xml → bt5/wendelin_telecom_base/SkinTemplateItem/portal_skins/wendelin_telecom_base/Base_getOrsEnbKpi.xml

@@ -2,25 +2,25 @@
 <ZopeData>
   <record id="1" aka="AAAAAAAAAAE=">
     <pickle>
-      <global name="Product Individual Variation" module="erp5.portal_type"/>
+      <global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
     </pickle>
     <pickle>
       <dictionary>
         <item>
-            <key> <string>id</string> </key>
-            <value> <string>1</string> </value>
+            <key> <string>_function</string> </key>
+            <value> <string>getOrsEnbKpi</string> </value>
         </item>
         <item>
-            <key> <string>title</string> </key>
-            <value> <string>e_rab</string> </value>
+            <key> <string>_module</string> </key>
+            <value> <string>WendelinTelecomGetOrsKpi</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>Base_getOrsEnbKpi</string> </value>
         </item>
         <item>
-            <key> <string>variation_base_category_list</string> </key>
-            <value>
-              <tuple>
-                <string>variation</string>
-              </tuple>
-            </value>
+            <key> <string>title</string> </key>
+            <value> <string></string> </value>
         </item>
       </dictionary>
     </pickle>

bt5/wendelin_telecom_base/SkinTemplateItem/portal_skins/wendelin_telecom_base/DataAcquisitionUnit_getERabDataArrayKey.xml → bt5/wendelin_telecom_base/SkinTemplateItem/portal_skins/wendelin_telecom_base/DataAcquisitionUnit_getERabDataArrayUrl.xml

@@ -54,7 +54,7 @@
         </item>
         <item>
             <key> <string>id</string> </key>
-            <value> <string>DataAcquisitionUnit_getERabDataArrayKey</string> </value>
+            <value> <string>DataAcquisitionUnit_getERabDataArrayUrl</string> </value>
         </item>
       </dictionary>
     </pickle>

bt5/wendelin_telecom_base/SkinTemplateItem/portal_skins/wendelin_telecom_base/DataAcquisitionUnit_getEUtranDataArrayKey.xml → bt5/wendelin_telecom_base/SkinTemplateItem/portal_skins/wendelin_telecom_base/DataAcquisitionUnit_getEUtranDataArrayUrl.xml

@@ -54,7 +54,7 @@
         </item>
         <item>
             <key> <string>id</string> </key>
-            <value> <string>DataAcquisitionUnit_getEUtranDataArrayKey</string> </value>
+            <value> <string>DataAcquisitionUnit_getEUtranDataArrayUrl</string> </value>
         </item>
       </dictionary>
     </pickle>

bt5/wendelin_telecom_base/SkinTemplateItem/portal_skins/wendelin_telecom_base/DataAcquisitionUnit_viewERabAccessibilityKpi/your_e_rab_accessibility_kpi.xml

@@ -211,7 +211,7 @@
                 </item>
                 <item>
                     <key> <string>gadget_url</string> </key>
-                    <value> <string>gadget_ors_e_rab_kpi.html</string> </value>
+                    <value> <string>gadget_ors_e_rab_accessibility_kpi.html</string> </value>
                 </item>
                 <item>
                     <key> <string>hidden</string> </key>
@@ -253,7 +253,7 @@
       <dictionary>
         <item>
             <key> <string>_text</string> </key>
-            <value> <string>python: [(\'data_array_key\', context.DataAcquisitionUnit_getERabDataArrayKey())]</string> </value>
+            <value> <string>python: [(\'data_array_url\', context.DataAcquisitionUnit_getERabDataArrayUrl()), (\'kpi_type\', \'e_rab_accessibility\')]</string> </value>
         </item>
       </dictionary>
     </pickle>

bt5/wendelin_telecom_base/SkinTemplateItem/portal_skins/wendelin_telecom_base/DataAcquisitionUnit_viewEUtranIpThroughputKpi/your_e_utran_ip_throughput_kpi.xml

@@ -253,7 +253,7 @@
       <dictionary>
         <item>
             <key> <string>_text</string> </key>
-            <value> <string>python: [(\'data_array_key\', context.DataAcquisitionUnit_getEUtranDataArrayKey())]</string> </value>
+            <value> <string>python: [(\'data_array_url\', context.DataAcquisitionUnit_getEUtranDataArrayUrl()), (\'kpi_type\', \'e_utran_ip_throughput\')]</string> </value>
         </item>
       </dictionary>
     </pickle>

bt5/wendelin_telecom_base/bt/template_extension_id_list

-extension.erp5.OrsKpiUtils
\ No newline at end of file
+extension.erp5.WendelinTelecomCalcEnbKpi
+extension.erp5.WendelinTelecomGetOrsKpi
\ No newline at end of file

bt5/wendelin_telecom_base/bt/template_keep_last_workflow_history_only_path_list

-data_operation_module/calculate_ors_kpi
+data_operation_module/calculate_ors_enb_kpi
 data_operation_module/ingest_ors_enb_log_data
+data_product_module/ors_enb_kpi
+data_product_module/ors_enb_kpi/**
 data_product_module/ors_enb_log_data
-data_product_module/ors_kpi
-data_product_module/ors_kpi/**
 data_transformation_module/ors_enb_log_data_transformation
 data_transformation_module/ors_enb_log_data_transformation/**
 organisation_module/open_radio_station
 organisation_module/rapid_space_data_center
-portal_callables/DataAnalysisLine_calculateOrsKpi
+portal_callables/DataAnalysisLine_calculateOrsEnbKpi
 portal_callables/DataIngestionLine_writeOrsFluentdIngestionToDataStream
 portal_callables/IngestionPolicy_parseOrsFluentdTag
 portal_ingestion_policies/ors_enb_log_ingestion
-web_page_module/ndarray_bundle.js
-web_page_module/ors_e_rab_kpi_*
+web_page_module/ors_e_rab_accessibility_kpi_*
 web_page_module/ors_e_utran_ip_throughput_kpi_*
 web_page_module/plotly_strict_v2.34.0.js
 web_page_module/rjs_gadget_ors_kpi_css
\ No newline at end of file

bt5/wendelin_telecom_base/bt/template_path_list

-data_operation_module/calculate_ors_kpi
+data_operation_module/calculate_ors_enb_kpi
 data_operation_module/ingest_ors_enb_log_data
+data_product_module/ors_enb_kpi
+data_product_module/ors_enb_kpi/**
 data_product_module/ors_enb_log_data
-data_product_module/ors_kpi
-data_product_module/ors_kpi/**
 data_transformation_module/ors_enb_log_data_transformation
 data_transformation_module/ors_enb_log_data_transformation/**
 organisation_module/open_radio_station
 organisation_module/rapid_space_data_center
-portal_callables/DataAnalysisLine_calculateOrsKpi
+portal_callables/DataAnalysisLine_calculateOrsEnbKpi
 portal_callables/DataIngestionLine_writeOrsFluentdIngestionToDataStream
 portal_callables/IngestionPolicy_parseOrsFluentdTag
 portal_ingestion_policies/ors_enb_log_ingestion
-web_page_module/ndarray_bundle.js
-web_page_module/ors_e_rab_kpi_*
+web_page_module/ors_e_rab_accessibility_kpi_*
 web_page_module/ors_e_utran_ip_throughput_kpi_*
 web_page_module/plotly_strict_v2.34.0.js
 web_page_module/rjs_gadget_ors_kpi_css
\ No newline at end of file