wip

slapos/apachedex.py

@@ -100,6 +100,7 @@ def main():
   process_handler = subprocess.Popen(argument_list,
                                      stdout=subprocess.PIPE,
                                      stderr=subprocess.PIPE,
+                                     universal_newlines=True,
                                      )
 
   stdout, stderr = process_handler.communicate()
@@ -108,7 +109,8 @@ def main():
       print(stderr)
     return 1
   
-  with open(output_file, 'r') as f:
+  # Check that output_file is a readable file.
+  with open(output_file, 'r'):
     print(base_url + '/ApacheDex-%s.html' % today)
   return 0
 

slapos/equeue/__init__.py

@@ -81,9 +81,9 @@ class EqueueServer(socketserver.ThreadingUnixStreamServer):
 
   def __init__(self, *args, **kw):
     self.options = kw.pop('equeue_options')
-    socketserver.ThreadingUnixStreamServer.__init__(self,
-                                                    RequestHandlerClass=None,
-                                                    *args, **kw)
+    super(EqueueServer, self).__init__(self,
+                                       RequestHandlerClass=None,
+                                       *args, **kw)
     # Equeue Specific elements
     self.setLogger(self.options.logfile[0], self.options.loglevel[0])
     self.setDB(self.options.database[0])

slapos/monitor/collect.py

@@ -27,8 +27,6 @@
 #
 ##############################################################################
 
-from six.moves import zip
-
 import sqlite3
 import os
 import pwd
@@ -82,10 +80,8 @@ class ResourceCollect:
       table="sqlite_master",
       columns='name',
       where="type='table' AND name='%s'" % name)
-    table_exists_result = list(zip(*check_result_cursor))
-    if not len(table_exists_result) or table_exists_result[0][0] is None:
-      return False
-    return True
+    r = check_result_cursor.fetchone()
+    return r and r[0] is not None
 
   def getPartitionCPULoadAverage(self, partition_id, date_scope):
     return self.consumption_utils.getPartitionCPULoadAverage(partition_id, date_scope)
@@ -161,38 +157,35 @@ class ResourceCollect:
     query_result = self.db.select('user', date_scope, colums, 
                                   where="partition='%s' and (time between '%s' and '%s') %s" % 
                                   (partition_id, min_time, max_time, where))
-    result_list = list(zip(*query_result))
-
-    process_dict = memory_dict = io_dict = {}
-    if len(result_list):
-      result = result_list
-      process_dict = {'total_process': result[0][0],
-        'cpu_percent': round((result[1][0] or 0), 2),
-        'cpu_time': round((result[2][0] or 0)/(60.0), 2),
-        'cpu_num_threads': round((result[3][0] or 0), 2),
-        'date': '%s %s' % (date_scope, min_time)
-      }
-      memory_dict = {'memory_percent': round((result[4][0] or 0), 2),
-        'memory_rss': round((result[5][0] or 0)/(1024*1024.0), 2),
-        'date': '%s %s' % (date_scope, min_time)
-      }
-      io_dict = {'io_rw_counter': round((result[6][0] or 0), 2),
-        'io_cycles_counter': round((result[7][0] or 0), 2),
-        'disk_used': 0,
-        'date': '%s %s' % (date_scope, min_time)
-      }
-      if self.has_table('folder'):
-        disk_result_cursor = self.db.select(
-          "folder", date_scope,
-          columns="SUM(disk_used)", 
-          where="partition='%s' and (time between '%s' and '%s') %s" % (
-            partition_id, min_time, max_time, where
-          )
+    result = query_result.fetchone()
+
+    process_dict = {'total_process': result[0],
+      'cpu_percent': round((result[1] or 0), 2),
+      'cpu_time': round((result[2] or 0)/(60.0), 2),
+      'cpu_num_threads': round((result[3] or 0), 2),
+      'date': '%s %s' % (date_scope, min_time)
+    }
+    memory_dict = {'memory_percent': round((result[4] or 0), 2),
+      'memory_rss': round((result[5] or 0)/(1024*1024.0), 2),
+      'date': '%s %s' % (date_scope, min_time)
+    }
+    io_dict = {'io_rw_counter': round((result[6] or 0), 2),
+      'io_cycles_counter': round((result[7] or 0), 2),
+      'disk_used': 0,
+      'date': '%s %s' % (date_scope, min_time)
+    }
+    if self.has_table('folder'):
+      disk_result_cursor = self.db.select(
+        "folder", date_scope,
+        columns="SUM(disk_used)", 
+        where="partition='%s' and (time between '%s' and '%s') %s" % (
+          partition_id, min_time, max_time, where
         )
-    
-        disk_used_sum = list(zip(*disk_result_cursor))
-        if len(disk_used_sum) and disk_used_sum[0][0] is not None:
-          io_dict['disk_used'] = round(disk_used_sum[0][0]/1024.0, 2)
+      )
+
+      disk_used_sum, = disk_result_cursor.fetchone()
+      if disk_used_sum is not None:
+        io_dict['disk_used'] = round(disk_used_sum/1024.0, 2)
     self.db.close()
     return (process_dict, memory_dict, io_dict)
 

slapos/monitor/monitor.py

@@ -133,7 +133,7 @@ class Monitoring(object):
               with open(config_list[2]) as cfile:
                 param_value = cfile.read()
             except OSError as e:
-              print('Cannot read file %s, Error is: %s' % (config_list[2], str(e)))
+              print('Cannot read file %s, Error is: %s' % (config_list[2], e))
               pass
           else:
             param_value = ""
@@ -180,7 +180,7 @@ class Monitoring(object):
             )
             configuration_list.append(parameter)
           except OSError as e:
-            print('Cannot read file at %s, Error is: %s' % (old_cors_file, str(e)))
+            print('Cannot read file at %s, Error is: %s' % (old_cors_file, e))
             pass
     return configuration_list
 
@@ -213,9 +213,9 @@ class Monitoring(object):
       # XXX - working here with public url
       if hasattr(ssl, '_create_unverified_context'):
         context = ssl._create_unverified_context()
-        response = urllib.urlopen(url, context=context, timeout=timeout)
+        response = urllib.request.urlopen(url, context=context, timeout=timeout)
       else:
-        response = urllib.urlopen(url, timeout=timeout)
+        response = urllib.request.urlopen(url, timeout=timeout)
     except urllib.HTTPError:
       print("ERROR: Failed to get Monitor configuration file at %s " % url)
     except socket.timeout as e:

slapos/promise/check_computer_memory/__init__.py

@@ -16,8 +16,6 @@ import datetime
 
 from slapos.collect.db import Database
 
-from six.moves import zip
-
 def getMemoryInfo(database, time, date):
 
   memory_info = {}
@@ -25,21 +23,21 @@ def getMemoryInfo(database, time, date):
   try:
     database.connect()
     query_result = database.select("computer", date, "memory_size", limit=1) 
-    result = list(zip(*query_result))
-    if not result or not result[0][0]:
+    r = query_result.fetchone()
+    if not r or not r[0]:
       return (None, "couldn't fetch total memory, collectordb is empty?")
-    memory_info['total'] = int(result[0][0])  # in byte
+    memory_info['total'] = int(r[0])  # in byte
 
     # fetch free and used memory 
     where_query = "time between '%s:00' and '%s:30' " % (time, time)
     query_result = database.select("system", date, "memory_free, memory_used", where=where_query)
-    result = list(zip(*query_result))
-    if not result or not result[0][0]: 
+    r = query_result.fetchone()
+    if not r or not r[0]: 
       return (None, "couldn't fetch free memory")
-    memory_info['free'] = int(result[0][0])  # in byte
-    if not result or not result[1][0]: 
+    memory_info['free'] = int(r[0])  # in byte
+    if not r or not r[1]: 
       return (None, "couldn't fetch used memory")
-    memory_info['used'] = int(result[1][0])  # in byte
+    memory_info['used'] = int(r[1])  # in byte
   finally:
     database.close()
 

slapos/promise/check_error_on_apache_log/__init__.py

@@ -7,7 +7,7 @@ import gzip
 import argparse
 import os
 
-r = re.compile("^(\[[^\]]+\]) (\[[^\]]+\]) (.*)$")
+r = re.compile(b"^(\[[^\]]+\]) (\[[^\]]+\]) (.*)$")
 
 def test(log_file, maximum_delay):
   error_amount = 0
@@ -19,11 +19,11 @@ def test(log_file, maximum_delay):
     # file don't exist, nothing to check
     return "OK"
 
-  with open(log_file) as f:
+  with open(log_file, "rb") as f:
 
     f.seek(0, 2)
     block_end_byte = f.tell()
-    f.seek(block_end_byte - min(block_end_byte, 4096), 0)
+    f.seek(-min(block_end_byte, 4096), 1)
 
     data = f.read()
     for line in reversed(data.splitlines()):
@@ -50,7 +50,7 @@ def test(log_file, maximum_delay):
         # no result in the latest hour
         break
 
-      if level != "[error]":
+      if level != b"[error]":
         continue
 
       # Classify the types of errors

slapos/qemuqmpclient/__init__.py

@@ -561,7 +561,7 @@ class QemuQMPWrapper(object):
     if (mem_size / slot_size) > slot_amount:
       raise ValueError("No enough slots available to add %sMB of RAM" % mem_size)
 
-    current_size = current_size // (1024 * 1024)
+    current_size //= (1024 * 1024)
     if current_size == mem_size:
       print("Hotplug Memory size is up to date.")
       return

slapos/securedelete.py

@@ -98,7 +98,8 @@ def shred(options):
   arg_list.extend(getFileList(options.file_list, options.check_exist))
 
   pshred = subprocess.Popen(arg_list, stdout=subprocess.PIPE,
-                            stderr=subprocess.STDOUT)
+                            stderr=subprocess.STDOUT,
+                            universal_newlines=True)
   result, stderr = pshred.communicate()
   if pshred.returncode is None:
     pshred.kill()

slapos/test/monitor/test_config_document.py

@@ -106,7 +106,7 @@ echo "htpasswd $@" > %s/monitor-htpasswd
 
     for config in config_json:
       if config["key"]:
-        self.assertTrue(config["key"] in config_parameter_json)
+        self.assertIn(config["key"], config_parameter_json)
         parameter = config_parameter_json[config["key"]]
       else:
         continue

slapos/test/test_securedelete.py

@@ -58,32 +58,32 @@ class TestSecureDelete(unittest.TestCase):
     passes = 2 + 1 # Option -z is used, plus one more pass
     result = shred(options)
     self.assertFalse(os.path.exists(self.remove_file))
-    self.assertTrue(b"pass %d/%d" % (passes, passes) in result)
-    self.assertTrue(b"%s: removed" % os.path.basename(self.remove_file).encode('utf-8') in result)
+    self.assertIn("pass %d/%d" % (passes, passes), result)
+    self.assertIn("%s: removed" % os.path.basename(self.remove_file), result)
 
   def test_secure_remove_file_keep_file(self):
     options = getAgumentParser().parse_args(['-n', '2', '-z', '--file', self.remove_file])
     passes = 2 + 1 # Option -z is used, plus one more pass
     result = shred(options)
     self.assertTrue(os.path.exists(self.remove_file))
-    self.assertTrue(b"pass %d/%d" % (passes, passes) in result)
-    self.assertFalse(b"%s: removed" % os.path.basename(self.remove_file).encode('utf-8') in result)
+    self.assertIn("pass %d/%d" % (passes, passes), result)
+    self.assertNotIn("%s: removed" % os.path.basename(self.remove_file), result)
 
   def test_secure_remove_file_non_zero(self):
     options = getAgumentParser().parse_args(['-n', '2', '-u', '--file', self.remove_file])
     passes = 2
     result = shred(options)
     self.assertFalse(os.path.exists(self.remove_file))
-    self.assertTrue(b"pass %d/%d" % (passes, passes) in result)
-    self.assertTrue(b"%s: removed" % os.path.basename(self.remove_file).encode('utf-8') in result)
+    self.assertIn("pass %d/%d" % (passes, passes), result)
+    self.assertIn("%s: removed" % os.path.basename(self.remove_file), result)
 
   def test_secure_remove_file_check_exist(self):
     options = getAgumentParser().parse_args(['-n', '2', '-u', '-s', '--file', 'random.txt', self.remove_file])
     passes = 2
     result = shred(options)
     self.assertFalse(os.path.exists(self.remove_file))
-    self.assertTrue(b"pass %d/%d" % (passes, passes) in result)
-    self.assertTrue(b"%s: removed" % os.path.basename(self.remove_file).encode('utf-8') in result)
+    self.assertIn("pass %d/%d" % (passes, passes), result)
+    self.assertIn("%s: removed" % os.path.basename(self.remove_file), result)
 
   def test_secure_remove_file_check_exist_false(self):
     options = getAgumentParser().parse_args(['-n', '2', '-u', '--file', 'random.txt'])
@@ -99,19 +99,19 @@ class TestSecureDelete(unittest.TestCase):
     # shred removed link and target file
     self.assertFalse(os.path.exists(self.remove_file))
     self.assertFalse(os.path.exists(self.link_name))
-    self.assertTrue(b"pass %d/%d" % (passes, passes) in result)
-    self.assertTrue(b"%s: removed" % os.path.basename(self.remove_file).encode('utf-8') in result)
+    self.assertIn("pass %d/%d" % (passes, passes), result)
+    self.assertIn("%s: removed" % os.path.basename(self.remove_file), result)
 
   def test_secure_remove_file_multiple_files(self):
     options = getAgumentParser().parse_args(['-n', '2', '-u', '-z', '--file', self.remove_file, self.remove_file2])
     passes = 2 + 1 # Option -z is used, plus one more pass
     result = shred(options)
     self.assertFalse(os.path.exists(self.remove_file))
-    self.assertTrue(b"pass %d/%d" % (passes, passes) in result)
-    self.assertTrue(b"%s: removed" % os.path.basename(self.remove_file).encode('utf-8') in result)
+    self.assertIn("pass %d/%d" % (passes, passes), result)
+    self.assertIn("%s: removed" % os.path.basename(self.remove_file), result)
 
     self.assertFalse(os.path.exists(self.remove_file2))
-    self.assertTrue(b"%s: removed" % os.path.basename(self.remove_file2).encode('utf-8') in result)
+    self.assertIn("%s: removed" % os.path.basename(self.remove_file2), result)
 
 if __name__ == '__main__':
   unittest.main()