Commit f9edadf3 authored by Rafael Monnerat's avatar Rafael Monnerat

slapos.collect: Compress historical logs and fix folder permissions.

parent 9414e8e6
...@@ -27,16 +27,17 @@ ...@@ -27,16 +27,17 @@
# #
############################################################################## ##############################################################################
from psutil import process_iter, NoSuchProcess, AccessDenied from psutil import process_iter, NoSuchProcess, AccessDenied
from time import strftime from time import strftime
from slapos.collect.db import Database from slapos.collect.db import Database
from slapos.util import mkdir_p from slapos.util import mkdir_p
import os
import stat
from slapos.collect.snapshot import ProcessSnapshot, ComputerSnapshot from slapos.collect.snapshot import ProcessSnapshot, ComputerSnapshot
from slapos.collect.reporter import RawCSVDumper, \ from slapos.collect.reporter import RawCSVDumper, \
SystemCSVReporterDumper SystemCSVReporterDumper, \
compressLogFolder
from entity import get_user_list, Computer from entity import get_user_list, Computer
...@@ -80,6 +81,10 @@ def do_collect(conf): ...@@ -80,6 +81,10 @@ def do_collect(conf):
log_directory = "%s/var/data-log" % conf.get("slapos", "instance_root") log_directory = "%s/var/data-log" % conf.get("slapos", "instance_root")
mkdir_p(log_directory, 0o755) mkdir_p(log_directory, 0o755)
if stat.S_IMODE(os.stat(log_directory).st_mode) != 0o755:
os.chmod(log_directory, 0o755)
database = Database(log_directory) database = Database(log_directory)
computer = Computer(ComputerSnapshot()) computer = Computer(ComputerSnapshot())
...@@ -91,6 +96,8 @@ def do_collect(conf): ...@@ -91,6 +96,8 @@ def do_collect(conf):
SystemCSVReporterDumper(database).dump(log_directory) SystemCSVReporterDumper(database).dump(log_directory)
RawCSVDumper(database).dump(log_directory) RawCSVDumper(database).dump(log_directory)
compressLogFolder(log_directory)
# Drop older entries already reported # Drop older entries already reported
database.garbageCollect() database.garbageCollect()
......
...@@ -31,6 +31,9 @@ ...@@ -31,6 +31,9 @@
from slapos.util import mkdir_p from slapos.util import mkdir_p
import os.path import os.path
import json import json
import tarfile
import glob
import shutil
import csv import csv
from time import strftime from time import strftime
...@@ -39,6 +42,12 @@ class Dumper(object): ...@@ -39,6 +42,12 @@ class Dumper(object):
def __init__(self, database): def __init__(self, database):
self.db = database self.db = database
def dump(self, folder):
raise NotImplemented("Implemented on Subclass")
def writeFile(self, **kw):
raise NotImplemented("Implemented on Subclass")
class SystemReporter(Dumper): class SystemReporter(Dumper):
def dump(self, folder): def dump(self, folder):
...@@ -94,9 +103,25 @@ class RawDumper(Dumper): ...@@ -94,9 +103,25 @@ class RawDumper(Dumper):
class RawCSVDumper(RawDumper): class RawCSVDumper(RawDumper):
def writeFile(self, name, folder, date_scope, rows): def writeFile(self, name, folder, date_scope, rows):
mkdir_p(os.path.join(folder, date_scope)) mkdir_p(os.path.join(folder, date_scope), 0o755)
file_io = open(os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name)), "w") file_io = open(os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name)), "w")
csv_output = csv.writer(file_io) csv_output = csv.writer(file_io)
csv_output.writerows(rows) csv_output.writerows(rows)
file_io.close() file_io.close()
def compressLogFolder(log_directory):
initial_folder = os.getcwd()
os.chdir(log_directory)
try:
for backup_to_archive in glob.glob("*-*-*/"):
filename = '%s.tar.gz' % backup_to_archive.strip("/")
with tarfile.open(filename, 'w:gz') as tfile:
tfile.add(backup_to_archive)
tfile.close()
shutil.rmtree(backup_to_archive)
finally:
os.chdir(initial_folder)
...@@ -29,6 +29,7 @@ import os ...@@ -29,6 +29,7 @@ import os
import glob import glob
import unittest import unittest
import shutil import shutil
import tarfile
import tempfile import tempfile
import slapos.slap import slapos.slap
import psutil import psutil
...@@ -290,6 +291,32 @@ class TestCollectReport(unittest.TestCase): ...@@ -290,6 +291,32 @@ class TestCollectReport(unittest.TestCase):
self.assertEquals(set(glob.glob("%s/*.csv" % self.instance_root)), set(csv_path_list)) self.assertEquals(set(glob.glob("%s/*.csv" % self.instance_root)), set(csv_path_list))
def test_compress_log_directory(self):
log_directory = "%s/test_compress" % self.instance_root
dump_folder = "%s/1990-01-01" % log_directory
if not os.path.exists(log_directory):
os.mkdir(log_directory)
if os.path.exists(dump_folder):
shutil.rmtree(dump_folder)
os.mkdir("%s/1990-01-01" % log_directory)
with open("%s/test.txt" % dump_folder, "w") as dump_file:
dump_file.write("hi")
dump_file.close()
reporter.compressLogFolder(log_directory)
self.assertFalse(os.path.exists(dump_folder))
self.assertTrue(os.path.exists("%s.tar.gz" % dump_folder))
with tarfile.open("%s.tar.gz" % dump_folder) as tf:
self.assertEquals(tf.getmembers()[0].name, "1990-01-01")
self.assertEquals(tf.getmembers()[1].name, "1990-01-01/test.txt")
self.assertEquals(tf.extractfile(tf.getmembers()[1]).read(), 'hi')
class TestCollectSnapshot(unittest.TestCase): class TestCollectSnapshot(unittest.TestCase):
def setUp(self): def setUp(self):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment