Commit e6a80ee2 authored by Łukasz Nowak's avatar Łukasz Nowak

Fix/promise check disk drop until full

See merge request !79
parents a61da413 587e5657
Pipeline #11104 failed with stage
in 0 seconds
...@@ -22,60 +22,7 @@ class RunPromise(GenericPromise): ...@@ -22,60 +22,7 @@ class RunPromise(GenericPromise):
# check disk space at least every 3 minutes # check disk space at least every 3 minutes
self.setPeriodicity(minute=3) self.setPeriodicity(minute=3)
def getDaysUntilFull(self, disk_partition, database, date, time, day_range):
"""Returns estimation of days until the disk_partition would become full
It uses date and time in order to find current disk free percentage, then rewinds
day_range back in history and calculates average speed of losing free space, which
is assumed constant and used to predict in how many days the disk would become full.
"""
database = Database(database, create=False, timeout=10)
try:
# fetch free disk space
database.connect()
result_max = database.select(
"disk",
date,
columns="free*1.0/(used+free) AS percent, max(datetime(date || ' ' || time))",
where="time between '%s:00' and '%s:30' and partition='%s'" % (time, time, disk_partition),
limit=1,
).fetchone()
if not result_max or not result_max[1]:
return None
result_min = database.select(
"disk",
columns="free*1.0/(used+free) AS percent, min(datetime(date || ' ' || time))",
where="datetime(date || ' ' || time) >= datetime('%s', '-%s days') and partition='%s'" % (result_max[1], day_range, disk_partition,),
limit=1,
).fetchone()
if not result_min or not result_min[1] or result_min == result_max:
return None
change = result_max[0] - result_min[0]
if change > 0.:
return None
timep = '%Y-%m-%d %H:%M:%S'
timespan = datetime.datetime.strptime(
result_max[1], timep) - datetime.datetime.strptime(
result_min[1], timep)
delta_days = timespan.total_seconds() / (3600.*24)
try:
return (-result_max[0] / (change / delta_days), result_min[1], result_min[0], result_max[1], result_max[0], delta_days)
except ZeroDivisionError as e:
# no data
return None
except sqlite3.OperationalError as e:
# if database is still locked after timeout expiration (another process is using it)
# we print warning message and try the promise at next run until max warn count
locked_message = "database is locked"
if locked_message in str(e) and \
not self.raiseOnDatabaseLocked(locked_message):
return None
raise
finally:
try:
database.close()
except Exception:
pass
def getDiskSize(self, disk_partition, database): def getDiskSize(self, disk_partition, database):
database = Database(database, create=False, timeout=10) database = Database(database, create=False, timeout=10)
...@@ -221,16 +168,6 @@ is assumed constant and used to predict in how many days the disk would become f ...@@ -221,16 +168,6 @@ is assumed constant and used to predict in how many days the disk would become f
free_space = self.getFreeSpace(disk_partition, db_path, currentdate, free_space = self.getFreeSpace(disk_partition, db_path, currentdate,
currenttime) currenttime)
days_until_full_tuple = self.getDaysUntilFull(disk_partition, db_path, currentdate, currenttime, threshold_days/2)
if days_until_full_tuple is not None:
days_until_full, min_date, min_free, max_date, max_free, day_span = days_until_full_tuple
message = "Disk will become full in %.2f days (threshold: %.2f days), checked from %s to %s, %.2f days span" % (
days_until_full, threshold_days, min_date, max_date, day_span)
if days_until_full < threshold_days:
self.logger.error(message + ', free space dropped from %.1f%% to %.1f%%: ERROR' % (min_free*100, max_free*100))
else:
self.logger.info(message + ': OK')
if free_space == 0: if free_space == 0:
return return
elif free_space > threshold*1024*1024*1024: elif free_space > threshold*1024*1024*1024:
......
BEGIN TRANSACTION; BEGIN TRANSACTION;
CREATE TABLE disk (partition text, used text, free text, mountpoint text, date text, time text, reported integer NULL DEFAULT 0); CREATE TABLE disk (partition text, used text, free text, mountpoint text, date text, time text, reported integer NULL DEFAULT 0);
INSERT INTO "disk" VALUES('/dev/sda1','155510666368','292658396032','/','2017-10-02','01:16:01',1);
INSERT INTO "disk" VALUES('/dev/sda1','159220666368','288948396032','/','2017-10-02','09:17:01',1); INSERT INTO "disk" VALUES('/dev/sda1','159220666368','288948396032','/','2017-10-02','09:17:01',1);
INSERT INTO "disk" VALUES('/dev/sda1','159237537792','288931524608','/','2017-10-02','09:18:01',1); INSERT INTO "disk" VALUES('/dev/sda1','159237537792','288931524608','/','2017-10-02','09:18:01',1);
INSERT INTO "disk" VALUES('/dev/sda1','159238090752','288930971648','/','2017-10-02','09:19:02',1); INSERT INTO "disk" VALUES('/dev/sda1','159238090752','288930971648','/','2017-10-02','09:19:02',1);
......
...@@ -54,7 +54,6 @@ class TestCheckFreeDiskSpace(TestPromisePluginMixin): ...@@ -54,7 +54,6 @@ class TestCheckFreeDiskSpace(TestPromisePluginMixin):
extra_config_dict = { extra_config_dict = {
'collectordb': '%(collectordb)s', 'collectordb': '%(collectordb)s',
'test-check-date': '2017-10-02', 'test-check-date': '2017-10-02',
'threshold-days': '25'
} }
""" % {'collectordb': self.db_file} """ % {'collectordb': self.db_file}
self.writePromise(self.promise_name, content) self.writePromise(self.promise_name, content)
...@@ -70,8 +69,7 @@ extra_config_dict = { ...@@ -70,8 +69,7 @@ extra_config_dict = {
extra_config_dict = { extra_config_dict = {
'collectordb': '%(collectordb)s', 'collectordb': '%(collectordb)s',
'test-check-date': '2017-09-14', 'test-check-date': '2017-09-14',
'test-check-time': '18:00', 'test-check-time': '18:00'
'threshold-days': '25'
} }
""" % {'collectordb': self.db_file} """ % {'collectordb': self.db_file}
self.writePromise(self.promise_name, content) self.writePromise(self.promise_name, content)
...@@ -87,26 +85,7 @@ extra_config_dict = { ...@@ -87,26 +85,7 @@ extra_config_dict = {
self.launcher.run() self.launcher.run()
result = self.getPromiseResult(self.promise_name) result = self.getPromiseResult(self.promise_name)
self.assertEqual(result['result']['failed'], False) self.assertEqual(result['result']['failed'], False)
self.assertEqual(result['result']['message'], self.assertEqual(result['result']['message'], "Disk usage: OK")
"Disk will become full in 26.02 days (threshold: 25.00 days), checked from 2017-10-02 01:16:01 to 2017-10-02 09:17:01, 0.33 days span: OK\nDisk usage: OK")
def test_disk_space_ok_will_full_soon(self):
content = """from slapos.promise.plugin.check_free_disk_space import RunPromise
extra_config_dict = {
'collectordb': '%(collectordb)s',
'test-check-date': '2017-10-02',
}
""" % {'collectordb': self.db_file}
self.writePromise(self.promise_name, content)
self.configureLauncher()
with self.assertRaises(PromiseError):
self.launcher.run()
result = self.getPromiseResult(self.promise_name)
self.assertEqual(result['result']['failed'], True)
self.assertEqual(
result['result']['message'],
"Disk will become full in 26.02 days (threshold: 30.00 days), checked from 2017-10-02 01:16:01 to 2017-10-02 09:17:01, 0.33 days span, free space dropped from 65.3% to 64.5%: ERROR\nDisk usage: OK")
def test_disk_space_nok(self): def test_disk_space_nok(self):
content = """from slapos.promise.plugin.check_free_disk_space import RunPromise content = """from slapos.promise.plugin.check_free_disk_space import RunPromise
...@@ -115,7 +94,6 @@ extra_config_dict = { ...@@ -115,7 +94,6 @@ extra_config_dict = {
'collectordb': '%(collectordb)s', 'collectordb': '%(collectordb)s',
'test-check-date': '2017-10-02', 'test-check-date': '2017-10-02',
'threshold': '278', 'threshold': '278',
'threshold-days': '25',
} }
""" % {'collectordb': self.db_file} """ % {'collectordb': self.db_file}
self.writePromise(self.promise_name, content) self.writePromise(self.promise_name, content)
...@@ -126,26 +104,14 @@ extra_config_dict = { ...@@ -126,26 +104,14 @@ extra_config_dict = {
result = self.getPromiseResult(self.promise_name) result = self.getPromiseResult(self.promise_name)
self.assertEqual(result['result']['failed'], True) self.assertEqual(result['result']['failed'], True)
self.assertEqual(result['result']['message'], self.assertEqual(result['result']['message'],
"Disk will become full in 26.02 days (threshold: 25.00 days), checked from 2017-10-02 01:16:01 to 2017-10-02 09:17:01, 0.33 days span: OK\nFree disk space low: remaining 269.1 G (threshold: 278.0 G)") "Free disk space low: remaining 269.1 G (threshold: 278.0 G)")
def test_disk_space_nok_will_full_soon(self):
content = """from slapos.promise.plugin.check_free_disk_space import RunPromise
extra_config_dict = {
'collectordb': '%(collectordb)s',
'test-check-date': '2017-10-02',
'threshold': '278',
}
""" % {'collectordb': self.db_file}
self.writePromise(self.promise_name, content)
self.configureLauncher() self.configureLauncher()
with self.assertRaises(PromiseError): with self.assertRaises(PromiseError):
self.launcher.run() self.launcher.run()
result = self.getPromiseResult(self.promise_name) result = self.getPromiseResult(self.promise_name)
self.assertEqual(result['result']['failed'], True) self.assertEqual(result['result']['failed'], True)
self.assertEqual(result['result']['message'], self.assertEqual(result['result']['message'], "Free disk space low: remaining 269.1 G (threshold: 278.0 G)")
"Disk will become full in 26.02 days (threshold: 30.00 days), checked from 2017-10-02 01:16:01 to 2017-10-02 09:17:01, 0.33 days span, free space dropped from 65.3% to 64.5%: ERROR\nFree disk space low: remaining 269.1 G (threshold: 278.0 G)")
def test_check_free_disk_with_unicode_string_path(self): def test_check_free_disk_with_unicode_string_path(self):
# set path unicode # set path unicode
...@@ -154,7 +120,7 @@ extra_config_dict = { ...@@ -154,7 +120,7 @@ extra_config_dict = {
self.launcher.run() self.launcher.run()
result = self.getPromiseResult(self.promise_name) result = self.getPromiseResult(self.promise_name)
self.assertEqual(result['result']['failed'], False) self.assertEqual(result['result']['failed'], False)
self.assertEqual(result['result']['message'], "Disk will become full in 26.02 days (threshold: 25.00 days), checked from 2017-10-02 01:16:01 to 2017-10-02 09:17:01, 0.33 days span: OK\nDisk usage: OK") self.assertEqual(result['result']['message'], "Disk usage: OK")
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment