promise: Support threshold-days in check_free_disk_space

The most important information is if the disk won't become full in some time,
by default 30 days. This check uses collector db history to calculate speed
of filling the disk and predicts when it will become full.

slapos/promise/plugin/check_free_disk_space.py

@@ -22,7 +22,60 @@ class RunPromise(GenericPromise):
     # check disk space at least every 3 minutes
     self.setPeriodicity(minute=3)
 
+  def getDaysUntilFull(self, disk_partition, database, date, time, day_range):
+    """Returns estimation of days until the disk_partition would become full
 
+It uses date and time in order to find current disk free percentage, then rewinds
+day_range back in history and calculates average speed of losing free space, which
+is assumed constant and used to predict in how many days the disk would become full.
+"""
+    database = Database(database, create=False, timeout=10)
+    try:
+      # fetch free disk space
+      database.connect()
+      result_max = database.select(
+        "disk",
+        date,
+        columns="free*1.0/(used+free) AS percent, max(datetime(date || ' ' || time))",
+        where="time between '%s:00' and '%s:30' and partition='%s'" % (time, time, disk_partition),
+        limit=1,
+      ).fetchone()
+      if not result_max or not result_max[1]:
+        return None
+      result_min = database.select(
+        "disk",
+        columns="free*1.0/(used+free) AS percent, min(datetime(date || ' ' || time))",
+        where="datetime(date || ' ' || time) >= datetime('%s', '-%s days')  and partition='%s'" % (result_max[1], day_range, disk_partition,),
+        limit=1,
+      ).fetchone()
+      if not result_min or not result_min[1] or result_min == result_max:
+        return None
+      change = result_max[0] - result_min[0]
+      if change > 0.:
+        return None
+      timep = '%Y-%m-%d %H:%M:%S'
+      timespan = datetime.datetime.strptime(
+        result_max[1], timep) - datetime.datetime.strptime(
+        result_min[1], timep)
+      delta_days = timespan.total_seconds() / (3600.*24)
+      try:
+        return (-(1. - result_max[0]) / (change / delta_days), result_min[1], result_max[1], delta_days)
+      except ZeroDivisionError as e:
+        # no data
+        return None
+    except sqlite3.OperationalError as e:
+      # if database is still locked after timeout expiration (another process is using it)
+      # we print warning message and try the promise at next run until max warn count
+      locked_message = "database is locked"
+      if locked_message in str(e) and \
+          not self.raiseOnDatabaseLocked(locked_message):
+        return None
+      raise
+    finally:
+      try:
+        database.close()
+      except Exception:
+        pass
 
   def getDiskSize(self, disk_partition, database):
     database = Database(database, create=False, timeout=10)
@@ -168,6 +221,16 @@ class RunPromise(GenericPromise):
 
     free_space = self.getFreeSpace(disk_partition, db_path, currentdate,
                                    currenttime)
+    days_until_full_tuple = self.getDaysUntilFull(disk_partition, db_path, currentdate, currenttime, threshold_days/2)
+    if days_until_full_tuple is not None:
+      days_until_full, min_date, max_date, day_span = days_until_full_tuple
+      message = "Disk will become full in %.2f days (threshold: %.2f days), checked from %s to %s, %.2f days span" % (
+          days_until_full, threshold_days, min_date, max_date, day_span)
+      if days_until_full < threshold_days:
+        self.logger.error(message + ': ERROR')
+      else:
+        self.logger.info(message + ': OK')
+
     if free_space == 0:
       return
     elif free_space > threshold*1024*1024*1024:

slapos/test/promise/data/disktest.sql

 BEGIN TRANSACTION;
 CREATE TABLE disk (partition text, used text, free text, mountpoint text,  date text, time text, reported integer NULL DEFAULT 0);
+INSERT INTO "disk" VALUES('/dev/sda1','157210666368','290958396032','/','2017-10-02','01:16:01',1);
 INSERT INTO "disk" VALUES('/dev/sda1','159220666368','288948396032','/','2017-10-02','09:17:01',1);
 INSERT INTO "disk" VALUES('/dev/sda1','159237537792','288931524608','/','2017-10-02','09:18:01',1);
 INSERT INTO "disk" VALUES('/dev/sda1','159238090752','288930971648','/','2017-10-02','09:19:02',1);

slapos/test/promise/plugin/test_check_free_disk_space.py

@@ -54,6 +54,7 @@ class TestCheckFreeDiskSpace(TestPromisePluginMixin):
 extra_config_dict = {
   'collectordb': '%(collectordb)s',
   'test-check-date': '2017-10-02',
+  'threshold-days': '25'
 }
 """ % {'collectordb': self.db_file}
     self.writePromise(self.promise_name, content)
@@ -69,7 +70,8 @@ extra_config_dict = {
 extra_config_dict = {
   'collectordb': '%(collectordb)s',
   'test-check-date': '2017-09-14',
-  'test-check-time': '18:00'
+  'test-check-time': '18:00',
+  'threshold-days': '25'
 }
 """ % {'collectordb': self.db_file}
     self.writePromise(self.promise_name, content)
@@ -85,7 +87,26 @@ extra_config_dict = {
     self.launcher.run()
     result = self.getPromiseResult(self.promise_name)
     self.assertEqual(result['result']['failed'], False)
-    self.assertEqual(result['result']['message'], "Disk usage: OK")
+    self.assertEqual(result['result']['message'],
+      "Disk will become full in 26.46 days (threshold: 25.00 days), checked from 2017-10-02 01:16:01 to 2017-10-02 09:17:01, 0.33 days span: OK\nDisk usage: OK")
+
+  def test_disk_space_ok_will_full_soon(self):
+    content = """from slapos.promise.plugin.check_free_disk_space import RunPromise
+
+extra_config_dict = {
+  'collectordb': '%(collectordb)s',
+  'test-check-date': '2017-10-02',
+}
+""" % {'collectordb': self.db_file}
+    self.writePromise(self.promise_name, content)
+    self.configureLauncher()
+    with self.assertRaises(PromiseError):
+      self.launcher.run()
+    result = self.getPromiseResult(self.promise_name)
+    self.assertEqual(result['result']['failed'], True)
+    self.assertEqual(
+      result['result']['message'],
+      "Disk will become full in 26.46 days (threshold: 30.00 days), checked from 2017-10-02 01:16:01 to 2017-10-02 09:17:01, 0.33 days span: ERROR\nDisk usage: OK")
 
   def test_disk_space_nok(self):
     content = """from slapos.promise.plugin.check_free_disk_space import RunPromise
@@ -94,6 +115,7 @@ extra_config_dict = {
   'collectordb': '%(collectordb)s',
   'test-check-date': '2017-10-02',
   'threshold': '278',
+  'threshold-days': '25',
 }
 """ % {'collectordb': self.db_file}
     self.writePromise(self.promise_name, content)
@@ -104,14 +126,26 @@ extra_config_dict = {
     result = self.getPromiseResult(self.promise_name)
     self.assertEqual(result['result']['failed'], True)
     self.assertEqual(result['result']['message'],
-      "Free disk space low: remaining 269.1 G (threshold: 278.0 G)")
+      "Disk will become full in 26.46 days (threshold: 25.00 days), checked from 2017-10-02 01:16:01 to 2017-10-02 09:17:01, 0.33 days span: OK\nFree disk space low: remaining 269.1 G (threshold: 278.0 G)")
+
+  def test_disk_space_nok_will_full_soon(self):
+    content = """from slapos.promise.plugin.check_free_disk_space import RunPromise
+
+extra_config_dict = {
+  'collectordb': '%(collectordb)s',
+  'test-check-date': '2017-10-02',
+  'threshold': '278',
+}
+""" % {'collectordb': self.db_file}
+    self.writePromise(self.promise_name, content)
 
     self.configureLauncher()
     with self.assertRaises(PromiseError):
       self.launcher.run()
     result = self.getPromiseResult(self.promise_name)
     self.assertEqual(result['result']['failed'], True)
-    self.assertEqual(result['result']['message'], "Free disk space low: remaining 269.1 G (threshold: 278.0 G)")
+    self.assertEqual(result['result']['message'],
+      "Disk will become full in 26.46 days (threshold: 30.00 days), checked from 2017-10-02 01:16:01 to 2017-10-02 09:17:01, 0.33 days span: ERROR\nFree disk space low: remaining 269.1 G (threshold: 278.0 G)")
 
   def test_check_free_disk_with_unicode_string_path(self):
     # set path unicode
@@ -120,7 +154,7 @@ extra_config_dict = {
     self.launcher.run()
     result = self.getPromiseResult(self.promise_name)
     self.assertEqual(result['result']['failed'], False)
-    self.assertEqual(result['result']['message'], "Disk usage: OK")
+    self.assertEqual(result['result']['message'], "Disk will become full in 26.46 days (threshold: 25.00 days), checked from 2017-10-02 01:16:01 to 2017-10-02 09:17:01, 0.33 days span: OK\nDisk usage: OK")
 
 if __name__ == '__main__':
   unittest.main()

[Thomas Gambier]

result_max[0] is already the current free disk space so I don't see why you'd do 1. - result_max[0]

[Łukasz Nowak]

You're right. I will apply it and update tests.