Commit 840a266b authored by Lu Xu's avatar Lu Xu 👀 Committed by Lu Xu

software/ors-amarisoft: add Resource activation timeout alarm

parent c2bfe70f
Pipeline #25881 failed with stage
in 0 seconds
......@@ -16,7 +16,7 @@
[template]
filename = instance.cfg
md5sum = 8a6e6e065e3eeffc3d5d4d8bb68f4a43
md5sum = 61e9685d6aa973ae0c45c45ca357458a
[amarisoft-stats.jinja2.py]
_update_hash_filename_ = amarisoft-stats.jinja2.py
......@@ -28,7 +28,7 @@ md5sum = 833667743c693b8d5f78a2527b275a9e
[template-lte-enb]
_update_hash_filename_ = instance-enb.jinja2.cfg
md5sum = ec127e2da004e52a516a7f499d7c7652
md5sum = afd8867c4dec5c5c68a4005dec502ebb
[template-lte-gnb-epc]
_update_hash_filename_ = instance-gnb-epc.jinja2.cfg
......@@ -40,7 +40,7 @@ md5sum = 99c05a34678adb5a70aa64ecf2ee4e35
[template-lte-gnb]
_update_hash_filename_ = instance-gnb.jinja2.cfg
md5sum = 58ee1663f86db0bbc4ccaf27831d2f0c
md5sum = cd0527037b7cb0ae53f2f822fcf551ac
[template-lte-mme]
_update_hash_filename_ = instance-mme.jinja2.cfg
......@@ -102,6 +102,10 @@ md5sum = 764ca8913ea40964382848cf3233c1f7
_update_hash_filename_ = promise/check_rx_saturated.py
md5sum = 40cf5389fd91845f4426ef159fb20fc0
[baseband-latency-promise]
_update_hash_filename_ = promise/check_baseband_latency.py
md5sum = 983b179735527f6b38393d28bd858400
[amarisoft-stats-log-promise]
_update_hash_filename_ = promise/check_amarisoft_stats_log.py
md5sum = 5bf57a9074ea5b054d999789cbbe2c87
......
......@@ -132,6 +132,18 @@
"type": "number",
"default": 0
},
"min_txrx_delay": {
"title": "Minimum available time for radio front end processing (ms)",
"description": "Baseband latency promise will fail if minimum TX/RX diff reaches threshold (higher than this value)",
"type": "number",
"default": 5
},
"avg_txrx_delay": {
"title": "Average available time for radio front end processing (ms)",
"description": "Baseband latency promise will fail if average TX/RX diff reaches threshold (higher than this value)",
"type": "number",
"default": 7
},
"promise_cpu_temperature_threshold": {
"title": "CPU temperature promise threshold",
"description": "Temperature threshold above which CPU temperature promise will fail",
......
......@@ -8,6 +8,7 @@ parts =
sdr-busy-promise
cell-gain-saturated-promise
rx-saturated-promise
baseband-latency-promise
amarisoft-stats-log-promise
{% if not slapparameter_dict.get("sub-instance", False) %}
cpu-temperature-promise
......@@ -192,6 +193,19 @@ config-amarisoft-stats-log = ${amarisoft-stats-template:log-output}
config-stats-period = {{ slapparameter_dict.get("enb_stats_fetch_period", 60) }}
config-max-rx-sample-db = {{ slapparameter_dict.get("max_rx_sample_db", 0) }}
[baseband-latency-promise]
recipe = slapos.cookbook:promise.plugin
eggs =
slapos.core
python-dateutil
file = {{ baseband_latency_promise }}
output = ${directory:plugins}/check-baseband-latency.py
config-testing = {{ slapparameter_dict.get("testing", False) }}
config-amarisoft-stats-log = ${amarisoft-stats-template:log-output}
config-stats-period = {{ slapparameter_dict.get("enb_stats_fetch_period", 60) }}
config-min-txrx-delay = {{ slapparameter_dict.get("min_txrx_delay", 5) }}
config-avg-txrx-delay = {{ slapparameter_dict.get("avg_txrx_delay", 7) }}
[amarisoft-stats-log-promise]
recipe = slapos.cookbook:promise.plugin
eggs =
......
......@@ -184,6 +184,18 @@
"type": "number",
"default": 0
},
"min_txrx_delay": {
"title": "Minimum available time for radio front end processing (ms)",
"description": "Minimum TX/RX diff threshold above which baseband latency promise will fail",
"type": "number",
"default": 5
},
"avg_txrx_delay": {
"title": "Average available time for radio front end processing (ms)",
"description": "Average TX/RX diff threshold above which baseband latency promise will fail",
"type": "number",
"default": 7
},
"promise_cpu_temperature_threshold": {
"title": "CPU temperature promise threshold",
"description": "Temperature threshold above which CPU temperature promise will fail",
......
......@@ -192,6 +192,19 @@ config-amarisoft-stats-log = ${amarisoft-stats-template:log-output}
config-stats-period = {{ slapparameter_dict.get("enb_stats_fetch_period", 60) }}
config-max-rx-sample-db = {{ slapparameter_dict.get("max_rx_sample_db", 0) }}
[baseband-latency-promise]
recipe = slapos.cookbook:promise.plugin
eggs =
slapos.core
python-dateutil
file = {{ baseband_latency_promise }}
output = ${directory:plugins}/check-baseband-latency.py
config-testing = {{ slapparameter_dict.get("testing", False) }}
config-amarisoft-stats-log = ${amarisoft-stats-template:log-output}
config-stats-period = {{ slapparameter_dict.get("enb_stats_fetch_period", 60) }}
config-min-txrx-delay = {{ slapparameter_dict.get("min_txrx_delay", 5) }}
config-avg-txrx-delay = {{ slapparameter_dict.get("avg_txrx_delay", 7) }}
[amarisoft-stats-log-promise]
recipe = slapos.cookbook:promise.plugin
eggs =
......
......@@ -130,6 +130,18 @@
"type": "number",
"default": 0
},
"min_txrx_delay": {
"title": "Minimum available time for radio front end processing (ms)",
"description": "Baseband latency promise will fail if minimum TX/RX diff reaches threshold (higher than this value)",
"type": "number",
"default": 5
},
"avg_txrx_delay": {
"title": "Average available time for radio front end processing (ms)",
"description": "Baseband latency promise will fail if average TX/RX diff reaches threshold (higher than this value)",
"type": "number",
"default": 7
},
"promise_cpu_temperature_threshold": {
"title": "CPU temperature promise threshold",
"description": "Temperature threshold above which CPU temperature promise will fail",
......
......@@ -112,6 +112,18 @@
"type": "number",
"default": 0
},
"min_txrx_delay": {
"title": "Minimum available time for radio front end processing (ms)",
"description": "Baseband latency promise will fail if minimum TX/RX diff reaches threshold (higher than this value)",
"type": "number",
"default": 5
},
"avg_txrx_delay": {
"title": "Average available time for radio front end processing (ms)",
"description": "Baseband latency promise will fail if average TX/RX diff reaches threshold (higher than this value)",
"type": "number",
"default": 7
},
"promise_cpu_temperature_threshold": {
"title": "CPU temperature promise threshold",
"description": "Temperature threshold above which CPU temperature promise will fail",
......
......@@ -182,6 +182,18 @@
"type": "number",
"default": 0
},
"min_txrx_delay": {
"title": "Minimum available time for radio front end processing (ms)",
"description": "Minimum TX/RX diff threshold above which baseband latency promise will fail",
"type": "number",
"default": 5
},
"avg_txrx_delay": {
"title": "Average available time for radio front end processing (ms)",
"description": "Average TX/RX diff threshold above which baseband latency promise will fail",
"type": "number",
"default": 7
},
"promise_cpu_temperature_threshold": {
"title": "CPU temperature promise threshold",
"description": "Temperature threshold above which CPU temperature promise will fail",
......
......@@ -164,6 +164,18 @@
"type": "number",
"default": 0
},
"min_txrx_delay": {
"title": "Minimum available time for radio front end processing (ms)",
"description": "Minimum TX/RX diff threshold above which baseband latency promise will fail",
"type": "number",
"default": 5
},
"avg_txrx_delay": {
"title": "Average available time for radio front end processing (ms)",
"description": "Average TX/RX diff threshold above which baseband latency promise will fail",
"type": "number",
"default": 7
},
"promise_cpu_temperature_threshold": {
"title": "CPU temperature promise threshold",
"description": "Temperature threshold above which CPU temperature promise will fail",
......
......@@ -130,6 +130,18 @@
"type": "number",
"default": 0
},
"min_txrx_delay": {
"title": "Minimum available time for radio front end processing (ms)",
"description": "Baseband latency promise will fail if minimum TX/RX diff reaches threshold (higher than this value)",
"type": "number",
"default": 5
},
"avg_txrx_delay": {
"title": "Average available time for radio front end processing (ms)",
"description": "Baseband latency promise will fail if average TX/RX diff reaches threshold (higher than this value)",
"type": "number",
"default": 7
},
"promise_cpu_temperature_threshold": {
"title": "CPU temperature promise threshold",
"description": "Temperature threshold above which CPU temperature promise will fail",
......
......@@ -112,6 +112,18 @@
"type": "number",
"default": 0
},
"min_txrx_delay": {
"title": "Minimum available time for radio front end processing (ms)",
"description": "Baseband latency promise will fail if minimum TX/RX diff reaches threshold (higher than this value)",
"type": "number",
"default": 5
},
"avg_txrx_delay": {
"title": "Average available time for radio front end processing (ms)",
"description": "Baseband latency promise will fail if average TX/RX diff reaches threshold (higher than this value)",
"type": "number",
"default": 7
},
"promise_cpu_temperature_threshold": {
"title": "CPU temperature promise threshold",
"description": "Temperature threshold above which CPU temperature promise will fail",
......
......@@ -182,6 +182,18 @@
"type": "number",
"default": 0
},
"min_txrx_delay": {
"title": "Minimum available time for radio front end processing (ms)",
"description": "Minimum TX/RX diff threshold above which baseband latency promise will fail",
"type": "number",
"default": 5
},
"avg_txrx_delay": {
"title": "Average available time for radio front end processing (ms)",
"description": "Average TX/RX diff threshold above which baseband latency promise will fail",
"type": "number",
"default": 7
},
"promise_cpu_temperature_threshold": {
"title": "CPU temperature promise threshold",
"description": "Temperature threshold above which CPU temperature promise will fail",
......
......@@ -164,6 +164,18 @@
"type": "number",
"default": 0
},
"min_txrx_delay": {
"title": "Minimum available time for radio front end processing (ms)",
"description": "Minimum TX/RX diff threshold above which baseband latency promise will fail",
"type": "number",
"default": 5
},
"avg_txrx_delay": {
"title": "Average available time for radio front end processing (ms)",
"description": "Average TX/RX diff threshold above which baseband latency promise will fail",
"type": "number",
"default": 7
},
"promise_cpu_temperature_threshold": {
"title": "CPU temperature promise threshold",
"description": "Temperature threshold above which CPU temperature promise will fail",
......
......@@ -130,6 +130,18 @@
"type": "number",
"default": 0
},
"min_txrx_delay": {
"title": "Minimum available time for radio front end processing (ms)",
"description": "Baseband latency promise will fail if minimum TX/RX diff reaches threshold (higher than this value)",
"type": "number",
"default": 5
},
"avg_txrx_delay": {
"title": "Average available time for radio front end processing (ms)",
"description": "Baseband latency promise will fail if average TX/RX diff reaches threshold (higher than this value)",
"type": "number",
"default": 7
},
"promise_cpu_temperature_threshold": {
"title": "CPU temperature promise threshold",
"description": "Temperature threshold above which CPU temperature promise will fail",
......
......@@ -112,6 +112,18 @@
"type": "number",
"default": 0
},
"min_txrx_delay": {
"title": "Minimum available time for radio front end processing (ms)",
"description": "Baseband latency promise will fail if minimum TX/RX diff reaches threshold (higher than this value)",
"type": "number",
"default": 5
},
"avg_txrx_delay": {
"title": "Average available time for radio front end processing (ms)",
"description": "Baseband latency promise will fail if average TX/RX diff reaches threshold (higher than this value)",
"type": "number",
"default": 7
},
"promise_cpu_temperature_threshold": {
"title": "CPU temperature promise threshold",
"description": "Temperature threshold above which CPU temperature promise will fail",
......
......@@ -182,6 +182,18 @@
"type": "number",
"default": 0
},
"min_txrx_delay": {
"title": "Minimum available time for radio front end processing (ms)",
"description": "Minimum TX/RX diff threshold above which baseband latency promise will fail",
"type": "number",
"default": 5
},
"avg_txrx_delay": {
"title": "Average available time for radio front end processing (ms)",
"description": "Average TX/RX diff threshold above which baseband latency promise will fail",
"type": "number",
"default": 7
},
"promise_cpu_temperature_threshold": {
"title": "CPU temperature promise threshold",
"description": "Temperature threshold above which CPU temperature promise will fail",
......
......@@ -164,6 +164,18 @@
"type": "number",
"default": 0
},
"min_txrx_delay": {
"title": "Minimum available time for radio front end processing (ms)",
"description": "Minimum TX/RX diff threshold above which baseband latency promise will fail",
"type": "number",
"default": 5
},
"avg_txrx_delay": {
"title": "Average available time for radio front end processing (ms)",
"description": "Average TX/RX diff threshold above which baseband latency promise will fail",
"type": "number",
"default": 7
},
"promise_cpu_temperature_threshold": {
"title": "CPU temperature promise threshold",
"description": "Temperature threshold above which CPU temperature promise will fail",
......
......@@ -130,6 +130,18 @@
"type": "number",
"default": 0
},
"min_txrx_delay": {
"title": "Minimum available time for radio front end processing (ms)",
"description": "Baseband latency promise will fail if minimum TX/RX diff reaches threshold (higher than this value)",
"type": "number",
"default": 5
},
"avg_txrx_delay": {
"title": "Average available time for radio front end processing (ms)",
"description": "Baseband latency promise will fail if average TX/RX diff reaches threshold (higher than this value)",
"type": "number",
"default": 7
},
"promise_cpu_temperature_threshold": {
"title": "CPU temperature promise threshold",
"description": "Temperature threshold above which CPU temperature promise will fail",
......
......@@ -112,6 +112,18 @@
"type": "number",
"default": 0
},
"min_txrx_delay": {
"title": "Minimum available time for radio front end processing (ms)",
"description": "Baseband latency promise will fail if minimum TX/RX diff reaches threshold (higher than this value)",
"type": "number",
"default": 5
},
"avg_txrx_delay": {
"title": "Average available time for radio front end processing (ms)",
"description": "Baseband latency promise will fail if average TX/RX diff reaches threshold (higher than this value)",
"type": "number",
"default": 7
},
"promise_cpu_temperature_threshold": {
"title": "CPU temperature promise threshold",
"description": "Temperature threshold above which CPU temperature promise will fail",
......
......@@ -182,6 +182,18 @@
"type": "number",
"default": 0
},
"min_txrx_delay": {
"title": "Minimum available time for radio front end processing (ms)",
"description": "Minimum TX/RX diff threshold above which baseband latency promise will fail",
"type": "number",
"default": 5
},
"avg_txrx_delay": {
"title": "Average available time for radio front end processing (ms)",
"description": "Average TX/RX diff threshold above which baseband latency promise will fail",
"type": "number",
"default": 7
},
"promise_cpu_temperature_threshold": {
"title": "CPU temperature promise threshold",
"description": "Temperature threshold above which CPU temperature promise will fail",
......
......@@ -164,6 +164,18 @@
"type": "number",
"default": 0
},
"min_txrx_delay": {
"title": "Minimum available time for radio front end processing (ms)",
"description": "Minimum TX/RX diff threshold above which baseband latency promise will fail",
"type": "number",
"default": 5
},
"avg_txrx_delay": {
"title": "Average available time for radio front end processing (ms)",
"description": "Average TX/RX diff threshold above which baseband latency promise will fail",
"type": "number",
"default": 7
},
"promise_cpu_temperature_threshold": {
"title": "CPU temperature promise threshold",
"description": "Temperature threshold above which CPU temperature promise will fail",
......
......@@ -113,6 +113,7 @@ extra-context =
raw sdr_busy_promise ${sdr-busy-promise:target}
raw cell_gain_saturated_promise ${cell-gain-saturated-promise:target}
raw rx_saturated_promise ${rx-saturated-promise:target}
raw baseband_latency_promise ${baseband-latency-promise:target}
raw amarisoft_stats_log_promise ${amarisoft-stats-log-promise:target}
raw cpu_temperature_promise ${cpu-temperature-promise:target}
raw openssl_location ${openssl:location}
......@@ -141,6 +142,7 @@ extra-context =
raw sdr_busy_promise ${sdr-busy-promise:target}
raw cell_gain_saturated_promise ${cell-gain-saturated-promise:target}
raw rx_saturated_promise ${rx-saturated-promise:target}
raw baseband_latency_promise ${baseband-latency-promise:target}
raw amarisoft_stats_log_promise ${amarisoft-stats-log-promise:target}
raw cpu_temperature_promise ${cpu-temperature-promise:target}
raw openssl_location ${openssl:location}
......
import errno
import json
import logging
import os
from dateutil import parser
from zope.interface import implementer
from slapos.grid.promise import interface
from slapos.grid.promise.generic import GenericPromise
# Get all data in the last "interval" seconds from JSON log
def get_data_interval(log, interval):
log_number = 0
latest_timestamp = 0
data_list = []
while True:
try:
f = open("{}.{}".format(log, log_number) if log_number else log, "rb")
except OSError:
return data_list
try:
f.seek(0, os.SEEK_END)
while True:
try:
while f.seek(-2, os.SEEK_CUR) and f.read(1) != b'\n':
pass
except OSError:
break
pos = f.tell()
l = json.loads(f.readline().decode().replace("'", '"'))
timestamp = parser.parse(l['time'])
data_list.append(l['data'])
if not latest_timestamp:
latest_timestamp = timestamp
if (latest_timestamp - timestamp).total_seconds() > interval:
return data_list
f.seek(pos, os.SEEK_SET)
finally:
f.close()
log_number += 1
@implementer(interface.IPromise)
class RunPromise(GenericPromise):
def __init__(self, config):
self.__name = config.get('name', None)
self.__log_folder = config.get('log-folder', None)
super(RunPromise, self).__init__(config)
self.setPeriodicity(minute=1)
self.__title = os.path.splitext(self.__name)[0]
self.__log_file = os.path.join(self.__log_folder, '%s.json.log' % self.__title)
self.json_logger = logging.getLogger('json_logger')
self.json_logger.setLevel(logging.INFO)
handler = logging.FileHandler(self.__log_file)
formatter = logging.Formatter('{"time": "%(asctime)s", "log_level": "%(levelname)s", "message": "%(message)s", "data": %(data)s}')
handler.setFormatter(formatter)
self.json_logger.addHandler(handler)
def sense(self):
testing = self.getConfig('testing') == "True"
amarisoft_stats_log = self.getConfig('amarisoft-stats-log')
stats_period = int(self.getConfig('stats-period'))
min_txrx_delay = float(self.getConfig('min-txrx-delay', 5))
avg_txrx_delay = float(self.getConfig('avg-txrx-delay', 7))
data_list = get_data_interval(amarisoft_stats_log, stats_period * 2)
min_txrx_delay_list = []
avg_txrx_delay_list = []
min_unavailable = False
avg_unavailable = False
for data in data_list:
rf_list = data['rf']
if not min_txrx_delay_list:
min_txrx_delay_list = [99 for x in rf_list]
if not avg_txrx_delay_list:
avg_txrx_delay_list = [99 for x in rf_list]
for i, rxtx_delay_avg in enumerate(rf_list):
min_txrx_delay_list[i] = min(min_txrx_delay_list[i], float(rf_list['rxtx_delay_min']))
min_delay = min_txrx_delay_list[i]
if min_txrx_delay_list[i] >= min_txrx_delay:
min_unavailable = True
for i, rxtx_delay_avg in enumerate(rf_list):
avg_txrx_delay_list[i] = min(avg_txrx_delay_list[i], float(rf_list['rxtx_delay_avg']))
avg_delay = avg_txrx_delay_list[i]
if avg_txrx_delay_list[i] >= avg_txrx_delay:
avg_unavailable = True
self.json_logger.info("The minimum available time for radio front end processing (ms)",
extra={'data': min_txrx_delay_list})
if not min_txrx_delay_list or not avg_txrx_delay:
self.logger.error("No TX/RX diff data available")
elif min_unavailable:
self.logger.error("The minimum available time %s (ms) for radio front end processing is higher than a threshold %s (ms) depending on the radio front end." % (min_delay, min_txrx_delay))
elif avg_unavailable:
self.logger.error("The average available time %s (ms) for radio front end processing is higher than a threshold %s (ms) depending on the radio front end." % (avg_delay, avg_txrx_delay))
else:
self.logger.info("The minimum %s (ms) and average %s (ms) available time for radio front end processing OK" % (min_delay, avg_delay))
def test(self):
"""
Called after sense() if the instance is still converging.
Returns success or failure based on sense results.
In this case, fail if the previous sensor result is negative.
"""
return self._test(result_count=1, failure_amount=1)
def anomaly(self):
"""
Called after sense() if the instance has finished converging.
Returns success or failure based on sense results.
Failure signals the instance has diverged.
In this case, fail if two out of the last three results are negative.
"""
return self._anomaly(result_count=1, failure_amount=1)
......@@ -70,6 +70,8 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_}
<= download-base
[rx-saturated-promise]
<= download-base
[baseband-latency-promise]
<= download-base
[amarisoft-stats-log-promise]
<= download-base
[cpu-temperature-promise]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment