Skip to content

S
slapos.toolbox
Commit e6a80ee2 authored by Łukasz Nowak's avatar Łukasz Nowak
Browse files
Options

Fix/promise check disk drop until full 

See merge request !79
parents a61da413 587e5657
Pipeline #11104 failed with stage
    in 0 seconds
    Show whitespace changes
    Inline Side-by-side
    Showing with 5 additions and 103 deletions
    slapos/promise/plugin/check_free_disk_space.py
    View file @ e6a80ee2
    ...@@ -22,60 +22,7 @@ class RunPromise(GenericPromise): ...@@ -22,60 +22,7 @@ class RunPromise(GenericPromise):
    # check disk space at least every 3 minutes # check disk space at least every 3 minutes
    self.setPeriodicity(minute=3) self.setPeriodicity(minute=3)
    def getDaysUntilFull(self, disk_partition, database, date, time, day_range):
    """Returns estimation of days until the disk_partition would become full
    It uses date and time in order to find current disk free percentage, then rewinds
    day_range back in history and calculates average speed of losing free space, which
    is assumed constant and used to predict in how many days the disk would become full.
    """
    database = Database(database, create=False, timeout=10)
    try:
    # fetch free disk space
    database.connect()
    result_max = database.select(
    "disk",
    date,
    columns="free*1.0/(used+free) AS percent, max(datetime(date || ' ' || time))",
    where="time between '%s:00' and '%s:30' and partition='%s'" % (time, time, disk_partition),
    limit=1,
    ).fetchone()
    if not result_max or not result_max[1]:
    return None
    result_min = database.select(
    "disk",
    columns="free*1.0/(used+free) AS percent, min(datetime(date || ' ' || time))",
    where="datetime(date || ' ' || time) >= datetime('%s', '-%s days') and partition='%s'" % (result_max[1], day_range, disk_partition,),
    limit=1,
    ).fetchone()
    if not result_min or not result_min[1] or result_min == result_max:
    return None
    change = result_max[0] - result_min[0]
    if change > 0.:
    return None
    timep = '%Y-%m-%d %H:%M:%S'
    timespan = datetime.datetime.strptime(
    result_max[1], timep) - datetime.datetime.strptime(
    result_min[1], timep)
    delta_days = timespan.total_seconds() / (3600.*24)
    try:
    return (-result_max[0] / (change / delta_days), result_min[1], result_min[0], result_max[1], result_max[0], delta_days)
    except ZeroDivisionError as e:
    # no data
    return None
    except sqlite3.OperationalError as e:
    # if database is still locked after timeout expiration (another process is using it)
    # we print warning message and try the promise at next run until max warn count
    locked_message = "database is locked"
    if locked_message in str(e) and \
    not self.raiseOnDatabaseLocked(locked_message):
    return None
    raise
    finally:
    try:
    database.close()
    except Exception:
    pass
    def getDiskSize(self, disk_partition, database): def getDiskSize(self, disk_partition, database):
    database = Database(database, create=False, timeout=10) database = Database(database, create=False, timeout=10)
    ...@@ -221,16 +168,6 @@ is assumed constant and used to predict in how many days the disk would become f ...@@ -221,16 +168,6 @@ is assumed constant and used to predict in how many days the disk would become f
    free_space = self.getFreeSpace(disk_partition, db_path, currentdate, free_space = self.getFreeSpace(disk_partition, db_path, currentdate,
    currenttime) currenttime)
    days_until_full_tuple = self.getDaysUntilFull(disk_partition, db_path, currentdate, currenttime, threshold_days/2)
    if days_until_full_tuple is not None:
    days_until_full, min_date, min_free, max_date, max_free, day_span = days_until_full_tuple
    message = "Disk will become full in %.2f days (threshold: %.2f days), checked from %s to %s, %.2f days span" % (
    days_until_full, threshold_days, min_date, max_date, day_span)
    if days_until_full < threshold_days:
    self.logger.error(message + ', free space dropped from %.1f%% to %.1f%%: ERROR' % (min_free*100, max_free*100))
    else:
    self.logger.info(message + ': OK')
    if free_space == 0: if free_space == 0:
    return return
    elif free_space > threshold*1024*1024*1024: elif free_space > threshold*1024*1024*1024:
    ......
    slapos/test/promise/data/disktest.sql
    View file @ e6a80ee2
    BEGIN TRANSACTION; BEGIN TRANSACTION;
    CREATE TABLE disk (partition text, used text, free text, mountpoint text, date text, time text, reported integer NULL DEFAULT 0); CREATE TABLE disk (partition text, used text, free text, mountpoint text, date text, time text, reported integer NULL DEFAULT 0);
    INSERT INTO "disk" VALUES('/dev/sda1','155510666368','292658396032','/','2017-10-02','01:16:01',1);
    INSERT INTO "disk" VALUES('/dev/sda1','159220666368','288948396032','/','2017-10-02','09:17:01',1); INSERT INTO "disk" VALUES('/dev/sda1','159220666368','288948396032','/','2017-10-02','09:17:01',1);
    INSERT INTO "disk" VALUES('/dev/sda1','159237537792','288931524608','/','2017-10-02','09:18:01',1); INSERT INTO "disk" VALUES('/dev/sda1','159237537792','288931524608','/','2017-10-02','09:18:01',1);
    INSERT INTO "disk" VALUES('/dev/sda1','159238090752','288930971648','/','2017-10-02','09:19:02',1); INSERT INTO "disk" VALUES('/dev/sda1','159238090752','288930971648','/','2017-10-02','09:19:02',1);
    ......
    slapos/test/promise/plugin/test_check_free_disk_space.py
    View file @ e6a80ee2
    ...@@ -54,7 +54,6 @@ class TestCheckFreeDiskSpace(TestPromisePluginMixin): ...@@ -54,7 +54,6 @@ class TestCheckFreeDiskSpace(TestPromisePluginMixin):
    extra_config_dict = { extra_config_dict = {
    'collectordb': '%(collectordb)s', 'collectordb': '%(collectordb)s',
    'test-check-date': '2017-10-02', 'test-check-date': '2017-10-02',
    'threshold-days': '25'
    } }
    """ % {'collectordb': self.db_file} """ % {'collectordb': self.db_file}
    self.writePromise(self.promise_name, content) self.writePromise(self.promise_name, content)
    ...@@ -70,8 +69,7 @@ extra_config_dict = { ...@@ -70,8 +69,7 @@ extra_config_dict = {
    extra_config_dict = { extra_config_dict = {
    'collectordb': '%(collectordb)s', 'collectordb': '%(collectordb)s',
    'test-check-date': '2017-09-14', 'test-check-date': '2017-09-14',
    'test-check-time': '18:00', 'test-check-time': '18:00'
    'threshold-days': '25'
    } }
    """ % {'collectordb': self.db_file} """ % {'collectordb': self.db_file}
    self.writePromise(self.promise_name, content) self.writePromise(self.promise_name, content)
    ...@@ -87,26 +85,7 @@ extra_config_dict = { ...@@ -87,26 +85,7 @@ extra_config_dict = {
    self.launcher.run() self.launcher.run()
    result = self.getPromiseResult(self.promise_name) result = self.getPromiseResult(self.promise_name)
    self.assertEqual(result['result']['failed'], False) self.assertEqual(result['result']['failed'], False)
    self.assertEqual(result['result']['message'], self.assertEqual(result['result']['message'], "Disk usage: OK")
    "Disk will become full in 26.02 days (threshold: 25.00 days), checked from 2017-10-02 01:16:01 to 2017-10-02 09:17:01, 0.33 days span: OK\nDisk usage: OK")
    def test_disk_space_ok_will_full_soon(self):
    content = """from slapos.promise.plugin.check_free_disk_space import RunPromise
    extra_config_dict = {
    'collectordb': '%(collectordb)s',
    'test-check-date': '2017-10-02',
    }
    """ % {'collectordb': self.db_file}
    self.writePromise(self.promise_name, content)
    self.configureLauncher()
    with self.assertRaises(PromiseError):
    self.launcher.run()
    result = self.getPromiseResult(self.promise_name)
    self.assertEqual(result['result']['failed'], True)
    self.assertEqual(
    result['result']['message'],
    "Disk will become full in 26.02 days (threshold: 30.00 days), checked from 2017-10-02 01:16:01 to 2017-10-02 09:17:01, 0.33 days span, free space dropped from 65.3% to 64.5%: ERROR\nDisk usage: OK")
    def test_disk_space_nok(self): def test_disk_space_nok(self):
    content = """from slapos.promise.plugin.check_free_disk_space import RunPromise content = """from slapos.promise.plugin.check_free_disk_space import RunPromise
    ...@@ -115,7 +94,6 @@ extra_config_dict = { ...@@ -115,7 +94,6 @@ extra_config_dict = {
    'collectordb': '%(collectordb)s', 'collectordb': '%(collectordb)s',
    'test-check-date': '2017-10-02', 'test-check-date': '2017-10-02',
    'threshold': '278', 'threshold': '278',
    'threshold-days': '25',
    } }
    """ % {'collectordb': self.db_file} """ % {'collectordb': self.db_file}
    self.writePromise(self.promise_name, content) self.writePromise(self.promise_name, content)
    ...@@ -126,26 +104,14 @@ extra_config_dict = { ...@@ -126,26 +104,14 @@ extra_config_dict = {
    result = self.getPromiseResult(self.promise_name) result = self.getPromiseResult(self.promise_name)
    self.assertEqual(result['result']['failed'], True) self.assertEqual(result['result']['failed'], True)
    self.assertEqual(result['result']['message'], self.assertEqual(result['result']['message'],
    "Disk will become full in 26.02 days (threshold: 25.00 days), checked from 2017-10-02 01:16:01 to 2017-10-02 09:17:01, 0.33 days span: OK\nFree disk space low: remaining 269.1 G (threshold: 278.0 G)") "Free disk space low: remaining 269.1 G (threshold: 278.0 G)")
    def test_disk_space_nok_will_full_soon(self):
    content = """from slapos.promise.plugin.check_free_disk_space import RunPromise
    extra_config_dict = {
    'collectordb': '%(collectordb)s',
    'test-check-date': '2017-10-02',
    'threshold': '278',
    }
    """ % {'collectordb': self.db_file}
    self.writePromise(self.promise_name, content)
    self.configureLauncher() self.configureLauncher()
    with self.assertRaises(PromiseError): with self.assertRaises(PromiseError):
    self.launcher.run() self.launcher.run()
    result = self.getPromiseResult(self.promise_name) result = self.getPromiseResult(self.promise_name)
    self.assertEqual(result['result']['failed'], True) self.assertEqual(result['result']['failed'], True)
    self.assertEqual(result['result']['message'], self.assertEqual(result['result']['message'], "Free disk space low: remaining 269.1 G (threshold: 278.0 G)")
    "Disk will become full in 26.02 days (threshold: 30.00 days), checked from 2017-10-02 01:16:01 to 2017-10-02 09:17:01, 0.33 days span, free space dropped from 65.3% to 64.5%: ERROR\nFree disk space low: remaining 269.1 G (threshold: 278.0 G)")
    def test_check_free_disk_with_unicode_string_path(self): def test_check_free_disk_with_unicode_string_path(self):
    # set path unicode # set path unicode
    ...@@ -154,7 +120,7 @@ extra_config_dict = { ...@@ -154,7 +120,7 @@ extra_config_dict = {
    self.launcher.run() self.launcher.run()
    result = self.getPromiseResult(self.promise_name) result = self.getPromiseResult(self.promise_name)
    self.assertEqual(result['result']['failed'], False) self.assertEqual(result['result']['failed'], False)
    self.assertEqual(result['result']['message'], "Disk will become full in 26.02 days (threshold: 25.00 days), checked from 2017-10-02 01:16:01 to 2017-10-02 09:17:01, 0.33 days span: OK\nDisk usage: OK") self.assertEqual(result['result']['message'], "Disk usage: OK")
    if __name__ == '__main__': if __name__ == '__main__':
    unittest.main() unittest.main()
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment
    GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7