[slapos.collect] Implement garbage collection for the sqlite

  Remove older registers, already reported (or exported to csv)
  (older them 3 days)

slapos/collect/__init__.py

@@ -61,6 +61,9 @@ def do_collect(conf):
     SystemCSVReporterDumper(database).dump(log_directory)
     RawCSVDumper(database).dump(log_directory)
 
+    # Drop older entries already reported
+    database.garbageCollect()
+
   except AccessDenied:
     print "You HAVE TO execute this script with root permission."
 

slapos/collect/db.py

 import sqlite3
 import os
 from time import time, strftime
+import datetime
 
 class Database:
 
@@ -150,6 +151,34 @@ class Database:
     return [i[0] for i in self._execute(
            "SELECT name FROM sqlite_master WHERE type='table'")]
 
+  def _getGarbageCollectionDateList(self, days_to_preserve=3):
+    """ Return the list of dates to Preserve when data collect
+    """
+    base = datetime.datetime.today()
+    date_list = []
+    for x in range(0, days_to_preserve):
+      date_list.append((base - datetime.timedelta(days=x)).strftime("%Y-%m-%d"))
+    return date_list
+
+  def garbageCollect(self):
+    """ Garbase collect the database, by removing older records already
+        reported.
+    """
+    date_list = self._getGarbageCollectionDateList()
+    print date_list 
+    where_clause = "reported = 1" 
+    for _date in date_list:
+      where_clause += " AND date != '%s' " % _date
+    
+    delete_sql = "DELETE FROM %s WHERE %s"
+
+    self.connect()
+    for table in self.table_list:
+      self._execute(delete_sql % (table, where_clause))
+
+    self.commit()
+    self.close()
+
   def getDateScopeList(self, ignore_date=None, reported=0):
     """ Get from the present unique dates from the system
         Use a smaller table to sabe time.

slapos/tests/collect.py

@@ -167,6 +167,40 @@ class TestCollectDatabase(unittest.TestCase):
         finally:
           database.close()
 
+    def test_garbage_collection_date_list(self):
+        database = db.Database(self.instance_root)
+        self.assertEquals(len(database._getGarbageCollectionDateList()), 3)
+        self.assertEquals(len(database._getGarbageCollectionDateList(1)), 1)
+        self.assertEquals(len(database._getGarbageCollectionDateList(0)), 0)
+
+        self.assertEquals(database._getGarbageCollectionDateList(1), 
+                           [strftime("%Y-%m-%d")])
+
+    def test_garbage(self):
+
+        database = db.Database(self.instance_root)
+        database.connect()
+        database.insertSystemSnapshot("0.1", '10.0', '100.0', '100.0', 
+                         '10.0', '1', '2', '12.0', '1', '1', '1983-01-10', 'TIME')
+        database.insertDiskPartitionSnapshot(
+                 '/dev/sdx1', '10', '20', '/mnt', '1983-01-10', 'TIME')
+        database.insertComputerSnapshot(
+              '1', '0', '0', '100', '0', '/dev/sdx1', '1983-01-10', 'TIME')
+        database.commit()
+        database.markDayAsReported(date_scope="1983-01-10", 
+                                       table_list=database.table_list)
+        database.commit()
+        self.assertEquals(len([x for x in database.select('system')]), 1)
+        self.assertEquals(len([x for x in database.select('computer')]), 1)
+        self.assertEquals(len([x for x in database.select('disk')]), 1)
+        database.close()
+
+        database.garbageCollect()
+        database.connect()
+
+        self.assertEquals(len([x for x in database.select('system')]), 0)
+        self.assertEquals(len([x for x in database.select('computer')]), 0)
+        self.assertEquals(len([x for x in database.select('disk')]), 0)
 
     def test_mark_day_as_reported(self):