Skip to content

W
wendelin-telecom
Commit e105c711 authored by Paul Graydon's avatar Paul Graydon
Browse files
Options

wendelin_telecom_base: Improve handling of active QCIs for E-UTRAN IP Throughput KPI 

Use Data Array Lines to track and view active QCI data
parent c8178be7
Hide whitespace changes
Inline Side-by-side
Showing with 181 additions and 39 deletions
bt5/wendelin_telecom_base/ExtensionTemplateItem/portal_components/extension.erp5.WendelinTelecomGetOrsKpi.py
View file @ e105c711
......@@ -17,6 +17,25 @@ NA_VALUES_REPLACEMENTS = {
'ul_hi': 0.
}
def get_data_zarray_in_time_range(np_data_zarray, time_field, time_start, time_end):
'''
Return a view of the provided numpy data array restricted to the provided time range.
'''
if time_start is None or time_start == 'NaN':
time_start = np_data_zarray[time_field][0]
else:
time_start = float(time_start)
if time_end is None or time_end == 'NaN':
time_end = np_data_zarray[time_field][-1]
else:
time_end = float(time_end)
time_mask = (np_data_zarray[time_field] >= time_start) \
& (np_data_zarray[time_field] <= time_end)
if isinstance(time_mask, np.ndarray):
return np_data_zarray[time_mask]
return np_data_zarray
def resample_data_zarray(
np_data_zarray, time_field,
time_start, time_end, resample_size
......@@ -49,8 +68,8 @@ def getOrsEnbKpi(
with the data arrays keyed to their respective data fields.
Several optimizations are made:
- If required, the data will be resampled to at most RESAMPLE_SIZE data points.
- For the E-UTRAN IP Throughput KPI, optimize the data format by removing data
for "silent" QCIs, which have only zero values.
- For the E-UTRAN IP Throughput KPI, only fetch and send data for "active" QCIs,
which is typically only 1 out of all 256 for an ORS.
'''
portal = self.getPortalObject()
......@@ -71,23 +90,13 @@ def getOrsEnbKpi(
RESPONSE.setHeader("Content-Type", "application/json")
data_zarray = data_array.getArray()[:]
time_field = data_array_dtype[0]
if time_start is None or time_start == 'NaN':
time_start = data_zarray[time_field][0]
else:
time_start = float(time_start)
if time_end is None or time_end == 'NaN':
time_end = data_zarray[time_field][-1]
else:
time_end = float(time_end)
time_mask = (data_zarray[time_field] >= time_start) \
& (data_zarray[time_field] <= time_end)
if isinstance(time_mask, np.ndarray):
data_zarray = data_zarray[time_mask]
if kpi_type == 'e_rab_accessibility':
data_zarray = data_array.getArray()[:]
time_field = data_array_dtype[0]
data_zarray = get_data_zarray_in_time_range(
data_zarray, time_field, time_start, time_end
)
resampled_data_dict = resample_data_zarray(
data_zarray,
time_field,
......@@ -112,15 +121,22 @@ def getOrsEnbKpi(
ul_lo = []
ul_hi = []
for qci in range(QCI_COUNT):
qci_data_zarray = data_zarray[qci::QCI_COUNT]
qci_dl_hi = qci_data_zarray['dl_hi']
qci_ul_hi = qci_data_zarray['ul_hi']
# QCI is silent: skip it
if (qci_dl_hi == 0.).all() and (qci_ul_hi == 0.).all():
for data_array_line in data_array.contentValues(portal_type='Data Array Line'):
line_id_split = data_array_line.getId().split('_')
if not (len(line_id_split) == 3 \
and line_id_split[0] == 'active' \
and line_id_split[1] == 'qci'):
continue
qci = int(line_id_split[2])
active_qci.append(qci)
qci_data_zarray = data_array_line.getArray()[:]
time_field = data_array_line.getArrayDtypeNames()[0]
qci_data_zarray = get_data_zarray_in_time_range(
qci_data_zarray, time_field, time_start, time_end
)
resampled_qci_data_dict = resample_data_zarray(
qci_data_zarray,
time_field,
......@@ -134,7 +150,6 @@ def getOrsEnbKpi(
qci_ul_lo = resampled_qci_data_dict['ul_lo']
qci_ul_hi = resampled_qci_data_dict['ul_hi']
active_qci.append(qci)
dl_lo.append(qci_dl_lo)
dl_hi.append(qci_dl_hi)
ul_lo.append(qci_ul_lo)
......@@ -148,4 +163,4 @@ def getOrsEnbKpi(
ul_lo=ul_lo,
ul_hi=ul_hi
)
RESPONSE.write(json.dumps(response_dict))
RESPONSE.write(json.dumps(response_dict))
\ No newline at end of file
bt5/wendelin_telecom_base/PathTemplateItem/portal_callables/DataAnalysisLine_calculateOrsEnbKpi.py
View file @ e105c711
......@@ -13,6 +13,17 @@ offset_index = progress_indicator.getIntOffsetIndex()
start = max(0, offset_index - MAX_PREV_CHUNK_SIZE)
end = min(in_data_stream.getSize(), offset_index + MAX_NEW_CHUNK_SIZE)
e_rab_data_array = None
e_utran_data_array = None
for array in out_array:
if array['variation'] == 'e_rab':
e_rab_data_array = array['Data Array']
if array['variation'] == 'e_utran':
e_utran_data_array = array['Data Array']
# Queue active QCI updating in all cases
e_utran_data_array.activate().DataArray_updateActiveQciLines()
# No new data to process
if offset_index >= end:
return
......@@ -75,15 +86,6 @@ history_start_index = max(0, new_log_data_line_first_index - HISTORY_LINE_COUNT)
log_data = '\n'.join(log_data_line_list[history_start_index:])
log_data = log_data.decode('utf8')
e_rab_data_array = None
e_utran_data_array = None
for array in out_array:
if array['variation'] == 'e_rab':
e_rab_data_array = array['Data Array']
if array['variation'] == 'e_utran':
e_utran_data_array = array['Data Array']
# Calculate the KPI data
kpi_data_dict = context.Base_calcEnbKpi(log_data, T_PERIOD)
vt, v_initial_epsb_estab_sr, v_added_epsb_estab_sr = kpi_data_dict['e_rab_accessibility']
......
bt5/wendelin_telecom_base/PathTemplateItem/portal_callables/DataArray_updateActiveQciLines.py 0 → 100644
View file @ e105c711
'''
Find every active QCI in the Data Array's data,
and create a Data Array Line indexed to each active QCI's data
for easy data retrieval and viewing if needed.
Should only be used on Data Arrays containing data for a per-QCI 3GPP KPI.
'''
QCI_COUNT = 256
data_zarray = context.getArray()
if data_zarray is None:
return
data_zarray = data_zarray[:]
for qci in range(QCI_COUNT):
qci_data_view = data_zarray[qci::QCI_COUNT]
qci_dl_hi = qci_data_view['dl_hi']
qci_ul_hi = qci_data_view['ul_hi']
# Silent QCI: skip
if (qci_dl_hi == 0.).all() and (qci_ul_hi == 0.).all():
continue
active_qci_id = "active_qci_%s" % qci
if context.get(active_qci_id) is None:
context.newContent(
id=active_qci_id,
portal_type='Data Array Line',
reference=active_qci_id,
index_expression='%s::%s' % (qci, QCI_COUNT)
)
bt5/wendelin_telecom_base/PathTemplateItem/portal_callables/DataArray_updateActiveQciLines.xml 0 → 100644
View file @ e105c711
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PyData Script" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="_reconstructor" module="copy_reg"/>
</klass>
<tuple>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
<global name="object" module="__builtin__"/>
<none/>
</tuple>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_local_properties</string> </key>
<value>
<tuple>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>reference</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>string</string> </value>
</item>
</dictionary>
</tuple>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>DataArray_updateActiveQciLines</string> </value>
</item>
<item>
<key> <string>reference</string> </key>
<value> <string>DataArray_updateActiveQciLines</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>DataArray_updateActiveQciLines</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/wendelin_telecom_base/PathTemplateItem/web_page_module/ors_e_utran_ip_throughput_kpi_js.js
View file @ e105c711
......@@ -134,7 +134,7 @@
kpi_data_dict = response;
if (Object.keys(kpi_data_dict).length === 0 ||
kpi_data_dict.active_qci.length === 0) {
kpi_data_dict.evt.length === 0) {
Plotly.react(
element,
[],
......
bt5/wendelin_telecom_base/PathTemplateItem/web_page_module/ors_e_utran_ip_throughput_kpi_js.xml
View file @ e105c711
......@@ -238,7 +238,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>1022.36430.45868.57753</string> </value>
<value> <string>1023.9871.29527.38382</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -258,7 +258,7 @@
</tuple>
<state>
<tuple>
<float>1738943046.37</float>
<float>1741022444.15</float>
<string>UTC</string>
</tuple>
</state>
......
bt5/wendelin_telecom_base/bt/template_keep_last_workflow_history_only_path_list
View file @ e105c711
......@@ -9,6 +9,7 @@ organisation_module/open_radio_station
organisation_module/rapid_space_data_center
portal_alarms/wendelin_telecom_*
portal_callables/DataAnalysisLine_calculateOrsEnbKpi
portal_callables/DataArray_updateActiveQciLines
portal_callables/DataIngestionLine_writeOrsFluentdIngestionToDataStream
portal_callables/IngestionPolicy_parseOrsFluentdTag
portal_ingestion_policies/ors_enb_log_ingestion
......
bt5/wendelin_telecom_base/bt/template_path_list
View file @ e105c711
......@@ -9,6 +9,7 @@ organisation_module/open_radio_station
organisation_module/rapid_space_data_center
portal_alarms/wendelin_telecom_*
portal_callables/DataAnalysisLine_calculateOrsEnbKpi
portal_callables/DataArray_updateActiveQciLines
portal_callables/DataIngestionLine_writeOrsFluentdIngestionToDataStream
portal_callables/IngestionPolicy_parseOrsFluentdTag
portal_ingestion_policies/ors_enb_log_ingestion
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7