Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos.core
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Klaus Wölfel
slapos.core
Commits
f9edadf3
Commit
f9edadf3
authored
Sep 10, 2014
by
Rafael Monnerat
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
slapos.collect: Compress historical logs and fix folder permissions.
parent
9414e8e6
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
63 additions
and
4 deletions
+63
-4
slapos/collect/__init__.py
slapos/collect/__init__.py
+10
-3
slapos/collect/reporter.py
slapos/collect/reporter.py
+26
-1
slapos/tests/collect.py
slapos/tests/collect.py
+27
-0
No files found.
slapos/collect/__init__.py
View file @
f9edadf3
...
...
@@ -27,16 +27,17 @@
#
##############################################################################
from
psutil
import
process_iter
,
NoSuchProcess
,
AccessDenied
from
time
import
strftime
from
slapos.collect.db
import
Database
from
slapos.util
import
mkdir_p
import
os
import
stat
from
slapos.collect.snapshot
import
ProcessSnapshot
,
ComputerSnapshot
from
slapos.collect.reporter
import
RawCSVDumper
,
\
SystemCSVReporterDumper
SystemCSVReporterDumper
,
\
compressLogFolder
from
entity
import
get_user_list
,
Computer
...
...
@@ -80,6 +81,10 @@ def do_collect(conf):
log_directory
=
"%s/var/data-log"
%
conf
.
get
(
"slapos"
,
"instance_root"
)
mkdir_p
(
log_directory
,
0o755
)
if
stat
.
S_IMODE
(
os
.
stat
(
log_directory
).
st_mode
)
!=
0o755
:
os
.
chmod
(
log_directory
,
0o755
)
database
=
Database
(
log_directory
)
computer
=
Computer
(
ComputerSnapshot
())
...
...
@@ -91,6 +96,8 @@ def do_collect(conf):
SystemCSVReporterDumper
(
database
).
dump
(
log_directory
)
RawCSVDumper
(
database
).
dump
(
log_directory
)
compressLogFolder
(
log_directory
)
# Drop older entries already reported
database
.
garbageCollect
()
...
...
slapos/collect/reporter.py
View file @
f9edadf3
...
...
@@ -31,6 +31,9 @@
from
slapos.util
import
mkdir_p
import
os.path
import
json
import
tarfile
import
glob
import
shutil
import
csv
from
time
import
strftime
...
...
@@ -39,6 +42,12 @@ class Dumper(object):
def
__init__
(
self
,
database
):
self
.
db
=
database
def
dump
(
self
,
folder
):
raise
NotImplemented
(
"Implemented on Subclass"
)
def
writeFile
(
self
,
**
kw
):
raise
NotImplemented
(
"Implemented on Subclass"
)
class
SystemReporter
(
Dumper
):
def
dump
(
self
,
folder
):
...
...
@@ -94,9 +103,25 @@ class RawDumper(Dumper):
class
RawCSVDumper
(
RawDumper
):
def
writeFile
(
self
,
name
,
folder
,
date_scope
,
rows
):
mkdir_p
(
os
.
path
.
join
(
folder
,
date_scope
))
mkdir_p
(
os
.
path
.
join
(
folder
,
date_scope
)
,
0o755
)
file_io
=
open
(
os
.
path
.
join
(
folder
,
"%s/dump_%s.csv"
%
(
date_scope
,
name
)),
"w"
)
csv_output
=
csv
.
writer
(
file_io
)
csv_output
.
writerows
(
rows
)
file_io
.
close
()
def
compressLogFolder
(
log_directory
):
initial_folder
=
os
.
getcwd
()
os
.
chdir
(
log_directory
)
try
:
for
backup_to_archive
in
glob
.
glob
(
"*-*-*/"
):
filename
=
'%s.tar.gz'
%
backup_to_archive
.
strip
(
"/"
)
with
tarfile
.
open
(
filename
,
'w:gz'
)
as
tfile
:
tfile
.
add
(
backup_to_archive
)
tfile
.
close
()
shutil
.
rmtree
(
backup_to_archive
)
finally
:
os
.
chdir
(
initial_folder
)
slapos/tests/collect.py
View file @
f9edadf3
...
...
@@ -29,6 +29,7 @@ import os
import
glob
import
unittest
import
shutil
import
tarfile
import
tempfile
import
slapos.slap
import
psutil
...
...
@@ -290,6 +291,32 @@ class TestCollectReport(unittest.TestCase):
self
.
assertEquals
(
set
(
glob
.
glob
(
"%s/*.csv"
%
self
.
instance_root
)),
set
(
csv_path_list
))
def
test_compress_log_directory
(
self
):
log_directory
=
"%s/test_compress"
%
self
.
instance_root
dump_folder
=
"%s/1990-01-01"
%
log_directory
if
not
os
.
path
.
exists
(
log_directory
):
os
.
mkdir
(
log_directory
)
if
os
.
path
.
exists
(
dump_folder
):
shutil
.
rmtree
(
dump_folder
)
os
.
mkdir
(
"%s/1990-01-01"
%
log_directory
)
with
open
(
"%s/test.txt"
%
dump_folder
,
"w"
)
as
dump_file
:
dump_file
.
write
(
"hi"
)
dump_file
.
close
()
reporter
.
compressLogFolder
(
log_directory
)
self
.
assertFalse
(
os
.
path
.
exists
(
dump_folder
))
self
.
assertTrue
(
os
.
path
.
exists
(
"%s.tar.gz"
%
dump_folder
))
with
tarfile
.
open
(
"%s.tar.gz"
%
dump_folder
)
as
tf
:
self
.
assertEquals
(
tf
.
getmembers
()[
0
].
name
,
"1990-01-01"
)
self
.
assertEquals
(
tf
.
getmembers
()[
1
].
name
,
"1990-01-01/test.txt"
)
self
.
assertEquals
(
tf
.
extractfile
(
tf
.
getmembers
()[
1
]).
read
(),
'hi'
)
class
TestCollectSnapshot
(
unittest
.
TestCase
):
def
setUp
(
self
):
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment