slapos/collect: wip

slapos/cli/collect.py

@@ -44,4 +44,5 @@ class CollectCommand(ConfigCommand):
     @must_be_root
     def take_action(self, args):
         configp = self.fetch_config(args)
+        print("Configuration file :", configp.sections())
         do_collect(configp)

slapos/collect/__init__.py

@@ -80,6 +80,7 @@ def do_collect(conf):
     Each user object is a dict, indexed on timestamp. We add every snapshot
     matching the user so that we get informations for each users
   """
+
   try:
     collected_date, collected_time = _get_time()
     user_dict = get_user_list(conf)
@@ -87,6 +88,9 @@ def do_collect(conf):
       for snapshot in current_state(user_dict):
         if snapshot:
           user_dict[snapshot.username].append(snapshot)
+      for user, info in user_dict.items():
+        print(_get_time()[0], _get_time()[1], user, "\tprocesses in progress :", len(info.snapshot_list))
+        #print(user, "\t", '\n'.join([str(p.get("process")) for p in info.snapshot_list]))
     except (KeyboardInterrupt, SystemExit, NoSuchProcess):
       raise
     days_to_preserve =  15
@@ -94,30 +98,40 @@ def do_collect(conf):
     if conf.has_option("slapos", "collect_cache"):
       days_to_preserve = conf.getint("slapos", "collect_cache")
     log_directory = "%s/var/data-log" % conf.get("slapos", "instance_root")
+    print(_get_time()[0], _get_time()[1], "log_directory :", log_directory)
     mkdir_p(log_directory, 0o755)
     
     consumption_report_directory = "%s/var/consumption-report" % \
-                                        conf.get("slapos", "instance_root") 
+                                        conf.get("slapos", "instance_root")
     mkdir_p(consumption_report_directory, 0o755)
 
+    print(_get_time()[0], _get_time()[1], "consumption_report_directory :", consumption_report_directory)
+
     xml_report_directory = "%s/var/xml_report/%s" % \
                     (conf.get("slapos", "instance_root"), 
                      conf.get("slapos", "computer_id"))
     mkdir_p(xml_report_directory, 0o755)
 
+    print(_get_time()[0], _get_time()[1], "xml_report_directory :", xml_report_directory)
+
     if stat.S_IMODE(os.stat(log_directory).st_mode) != 0o755:
       os.chmod(log_directory, 0o755)    
 
     database = Database(log_directory, create=True)
 
+    database.connect()
+    print(_get_time()[0], _get_time()[1], "Database tables :", database.getTableList())
+    database.close()
+
     if conf.has_option("slapformat", "computer_model_id"):
       computer_model_id = conf.get("slapformat", 
                                   "computer_model_id")
- 
     else:
       computer_model_id = "no_model"
+    print(_get_time()[0], _get_time()[1], "computer model id :", computer_model_id)
 
     uptime = _get_uptime()
+
     if conf.has_option("slapformat", "heating_sensor_id"):
       heating_sensor_id = conf.get("slapformat", 
                                   "heating_sensor_id")
@@ -130,21 +144,38 @@ def do_collect(conf):
     else:
       heating_sensor_id = "no_sensor"
       test_heating = False
+    
+    print(_get_time()[0], _get_time()[1], "heating sensor id :", heating_sensor_id)
+
+    print(_get_time()[0], _get_time()[1], "Inserting computer information into database...")
 
     computer = Computer(ComputerSnapshot(model_id=computer_model_id, 
                                      sensor_id = heating_sensor_id,
                                      test_heating=test_heating))
 
+    # Insert computer's data
     computer.save(database, collected_date, collected_time)
 
+    print(_get_time()[0], _get_time()[1], "Done")
+
+    print(_get_time()[0], _get_time()[1], "Inserting user information into database...")
+
+    # Insert TABLE user + TABLE folder
     for user in user_dict.values():
       user.save(database, collected_date, collected_time)
+
+    print(_get_time()[0], _get_time()[1], "Done")
+
+    print(_get_time()[0], _get_time()[1], "Writing csv, XML and JSON files...")
     
+    # Write a csv with dumped data in the log_directory
     SystemCSVReporterDumper(database).dump(log_directory)
     RawCSVDumper(database).dump(log_directory)
+
+    # Write xml files
     consumption_report = ConsumptionReport(
                       computer_id=conf.get("slapos", "computer_id"), 
-                      user_list=user_dict, 
+                      user_list=user_dict,
                       database=database,
                       location=consumption_report_directory)
     
@@ -156,11 +187,16 @@ def do_collect(conf):
       if report_file is not None:
         shutil.copy(report_file, xml_report_directory)
 
+    # write json
     partition_report = PartitionReport(
             database=database,
             user_list=user_dict)
 
     partition_report.buildJSONMonitorReport()
+
+    print(_get_time()[0], _get_time()[1], "Done")
+
+    # Put dumped csv in a current_date.tar.gz
     compressLogFolder(log_directory)
 
     # Drop older entries already reported

slapos/collect/db.py

@@ -290,6 +290,8 @@ class Database:
       where_clause += " AND (date < '%s' " % min(date_list)
       where_clause += " OR date > '%s') " % max(date_list)
 
+    #print(where_clause)
+
     vacuum = False
     delete_sql = "DELETE FROM %s WHERE %s"
     select_sql = "SELECT date FROM %s WHERE %s LIMIT 1"

slapos/collect/entity.py

@@ -31,21 +31,32 @@ import os
 from datetime import datetime, timedelta
 from slapos.collect.snapshot import FolderSizeSnapshot
 
+from time import strftime, gmtime
+
+def _get_time():
+  # add by Lisa to print
+  return strftime("%Y-%m-%d -- %H:%M:%S", gmtime()).split(" -- ")
+
+
 def get_user_list(config):
   nb_user = int(config.get("slapformat", "partition_amount"))
   name_prefix = config.get("slapformat", "user_base_name")
   path_prefix = config.get("slapformat", "partition_base_name")
   instance_root = config.get("slapos", "instance_root")
+  print(nb_user, name_prefix, path_prefix, instance_root)
   # By default, enable disk snapshot,
   # and set time_cycle to 24hours after the first disk snapshot run
-  disk_snapshot_params = {'enable': False, 'time_cycle': 86400}
+  pid_folder_tmp = instance_root+"/var/run/"
+  disk_snapshot_params = {'enable': True, 'time_cycle': 86400, \
+    'pid_folder': pid_folder_tmp, 'use_quota': False}
+    
   if config.has_section('collect'):
     collect_section = dict(config.items("collect"))
     disk_snapshot_params = dict(
-      enable=collect_section.get("report_disk_usage", "False").lower() in ('true', 'on', '1'),
-      pid_folder=collect_section.get("disk_snapshot_process_pid_foder", None),
+      enable=collect_section.get("report_disk_usage", "True").lower() in ('true', 'on', '1'),
+      pid_folder=collect_section.get("disk_snapshot_process_pid_foder", pid_folder_tmp),
       time_cycle=int(collect_section.get("disk_snapshot_time_cycle", 86400)),
-      use_quota=collect_section.get("disk_snapshot_use_quota", "True").lower() in ('true', 'on', '1'),
+      use_quota=collect_section.get("disk_snapshot_use_quota", "False").lower() in ('true', 'on', '1'),
     )
   user_dict = {name: User(name, path, disk_snapshot_params)
       for name, path in [
@@ -71,21 +82,29 @@ class User(object):
 
   def _insertDiskSnapShot(self, database, collected_date, collected_time):
     if self.disk_snapshot_params['enable']:
+      #print('\nInside "insert Disk Snapshot.."')
       time_cycle = self.disk_snapshot_params.get('time_cycle', 0)
+
       database.connect()
       if time_cycle:
+        # check if the time spent between the two cycles is enough
         for date_time in database.select(table="folder", columns="date, time",
                                     order='date DESC, time DESC', limit=1,
                                     where="partition='%s'" % self.name):
           latest_date = datetime.strptime('%s %s' % date_time,
                                           "%Y-%m-%d %H:%M:%S")
-          if (datetime.utcnow() - latest_date).seconds < time_cycle:
+          time_spent = (datetime.utcnow() - latest_date).seconds
+          if time_spent < time_cycle:
             # wait the time cycle
+            print("Too soon... wait ", time_cycle - time_spent, "seconds")
             return
           break
       pid_file = self.disk_snapshot_params.get('pid_folder', None)
       if pid_file is not None:
         pid_file = os.path.join(pid_file, '%s_disk_size.pid' % self.name)
+      #print("path", self.path)
+      print("pid_file", pid_file)
+
       disk_snapshot = FolderSizeSnapshot(self.path, pid_file)
       disk_snapshot.update_folder_size()
       # Skeep insert empty partition: size <= 1Mb
@@ -102,7 +121,7 @@ class User(object):
   def save(self, database, collected_date, collected_time):
     """ Insert collected data on user collector """
     database.connect()
-    snapshot_counter = len(self.snapshot_list)
+    #snapshot_counter = len(self.snapshot_list)
     for snapshot_item in self.snapshot_list:
       snapshot_item.update_cpu_percent()
       database.insertUserSnapshot(self.name,

slapos/collect/reporter.py

@@ -127,8 +127,8 @@ class RawCSVDumper(RawDumper):
 
 def compressLogFolder(log_directory):
   
-    initial_folder = os.getcwd()
-    os.chdir(log_directory)
+    initial_folder = os.getcwd() # current working directory of a process
+    os.chdir(log_directory) # changes the current working directory to the given path
     try:
       for backup_to_archive in glob.glob("*-*-*/"):
         filename = '%s.tar.gz' % backup_to_archive.strip("/")

slapos/collect/snapshot.py

@@ -103,22 +103,27 @@ class FolderSizeSnapshot(_Snapshot):
     # If extra disk added to partition
     data_dir = os.path.join(self.folder_path, 'DATA')
     if os.path.exists(data_dir):
+      print("extra disk added to partition")
       for filename in os.listdir(data_dir):
         extra_path = os.path.join(data_dir, filename)
         if os.path.islink(extra_path) and os.path.isdir('%s/' % extra_path):
           self.disk_usage += self._getSize('%s/' % extra_path)
+          print(self.disk_usage)
+          
 
   def _getSize(self, file_path):
+
     size = 0
     command = 'du -s %s' % file_path
     process = subprocess.Popen(command, stdout=subprocess.PIPE, 
                               stderr=subprocess.PIPE, shell=True)
+    # write pid process : du -s /srv/slapgrid/slappart[i]
     if self.pid_file:
       with open(self.pid_file, 'w') as fpid:
         pid = fpid.write(str(process.pid))
     result = process.communicate()[0]
     if process.returncode == 0:
-      size, _  = result.strip().split()
+      size, _  = result.strip().split() # retourne la taille + path
     return float(size)
 
 class SystemSnapshot(_Snapshot):
@@ -126,18 +131,19 @@ class SystemSnapshot(_Snapshot):
   """
   def __init__(self, interval=MEASURE_INTERVAL):
 
-    cpu_idle_percentage = psutil.cpu_times_percent(interval=interval).idle
+    # provides utilization percentages for each specific CPU time
+    cpu_idle_percentage = psutil.cpu_times_percent(interval=interval).idle # long
     load_percent = 100 - cpu_idle_percentage
 
     memory = psutil.virtual_memory()
-    net_io = psutil.net_io_counters()
+    net_io = psutil.net_io_counters() # system-wide network I/O statistics as a named tuple
 
     self.memory_free = available = memory.available
     self.memory_used = memory.total - available
     self.memory_percent = memory.percent
     #self.cpu_percent = psutil.cpu_percent()
     self.cpu_percent = load_percent
-    self.load = os.getloadavg()[0]
+    self.load = os.getloadavg()[0] # number of processes in the system run queue
     self.net_in_bytes = net_io.bytes_recv
     self.net_in_errors = net_io.errin
     self.net_in_dropped = net_io.dropin
@@ -208,14 +214,14 @@ class ComputerSnapshot(_Snapshot):
     self.cpu_num_core = psutil.cpu_count()
     self.cpu_frequency = 0
     self.cpu_type = 0
-    self.memory_size = psutil.virtual_memory().total
+    self.memory_size = psutil.virtual_memory().total #total physical memory (exclusive swap)
     self.memory_type = 0
 
     #
     # Include a SystemSnapshot and a list DiskPartitionSnapshot
     # on a Computer Snapshot
     #
-    self.system_snapshot = SystemSnapshot()
+    self.system_snapshot = SystemSnapshot() # take few seconds
     self.temperature_snapshot_list = self._get_temperature_snapshot_list()
     self._get_physical_disk_info()