Commit cff209d0 authored by Łukasz Nowak's avatar Łukasz Nowak

promise: Cleanup stale monitoring files of removed promises

/reviewed-on nexedi/slapos.core!174
parent 63b17738
Pipeline #7415 failed with stage
in 0 seconds
...@@ -29,6 +29,8 @@ ...@@ -29,6 +29,8 @@
############################################################################## ##############################################################################
import os import os
import glob
import fnmatch
import sys import sys
import logging import logging
import time import time
...@@ -728,6 +730,7 @@ class PromiseLauncher(object): ...@@ -728,6 +730,7 @@ class PromiseLauncher(object):
} }
error = 0 error = 0
success = 0 success = 0
promise_name_list = []
if os.path.exists(self.promise_folder) and os.path.isdir(self.promise_folder): if os.path.exists(self.promise_folder) and os.path.isdir(self.promise_folder):
for promise_name in os.listdir(self.promise_folder): for promise_name in os.listdir(self.promise_folder):
for suffix in ['.pyc', '.pyo']: for suffix in ['.pyc', '.pyo']:
...@@ -745,6 +748,7 @@ class PromiseLauncher(object): ...@@ -745,6 +748,7 @@ class PromiseLauncher(object):
not promise_name.endswith('.py'): not promise_name.endswith('.py'):
continue continue
promise_name_list.append(promise_name)
if self.run_only_promise_list is not None and not \ if self.run_only_promise_list is not None and not \
promise_name in self.run_only_promise_list: promise_name in self.run_only_promise_list:
continue continue
...@@ -787,6 +791,31 @@ class PromiseLauncher(object): ...@@ -787,6 +791,31 @@ class PromiseLauncher(object):
else: else:
success += 1 success += 1
if not self.run_only_promise_list and len(promise_name_list) > 0:
# cleanup stale json files
promise_output_dir_content = glob.glob(os.path.join(self.promise_output_dir, '*.status.json'))
promise_history_output_dir_content = glob.glob(os.path.join(self.promise_history_output_dir, '*.history*.json'))
for promise_file_name in promise_name_list:
promise_name = promise_file_name[:-3]
promise_status_json_name = promise_name + '.status.json'
promise_history_json_match = promise_name + '.history*.json'
promise_output_dir_content = [q for q in promise_output_dir_content if os.path.basename(q) != promise_status_json_name]
promise_history_output_dir_content = [q for q in promise_history_output_dir_content if not fnmatch.fnmatch(os.path.basename(q), promise_history_json_match)]
promise_output_dir_cleanup = [os.path.join(self.promise_output_dir, q) for q in promise_output_dir_content]
promise_history_output_dir_cleanup = [os.path.join(self.promise_history_output_dir, q) for q in promise_history_output_dir_content]
for path in promise_output_dir_cleanup + promise_history_output_dir_cleanup:
try:
os.unlink(path)
except Exception:
self.logger.exception('Problem while removing stale file %s', path)
else:
self.logger.info('Removed stale file %s', path)
# drop old promises from new_state_dict
for key in list(new_state_dict.keys()):
if key not in promise_name_list:
new_state_dict.pop(key, None)
if not self.run_only_promise_list and os.path.exists(self.legacy_promise_folder) \ if not self.run_only_promise_list and os.path.exists(self.legacy_promise_folder) \
and os.path.isdir(self.legacy_promise_folder): and os.path.isdir(self.legacy_promise_folder):
# run legacy promise styles # run legacy promise styles
......
...@@ -1588,6 +1588,84 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin): ...@@ -1588,6 +1588,84 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
self.assertFalse(os.path.exists(stale_pyc)) self.assertFalse(os.path.exists(stale_pyc))
self.assertFalse(os.path.exists(stale_pyo)) self.assertFalse(os.path.exists(stale_pyo))
def test_promise_cleanup_output_history(self):
promise_name = 'dummy.py'
promise_content = """from zope.interface import implementer
from slapos.grid.promise import interface
from slapos.grid.promise import GenericPromise
@implementer(interface.IPromise)
class RunPromise(GenericPromise):
def sense(self):
self.logger.info("success")
def anomaly(self):
return self._anomaly()
def test(self):
return self._test()
"""
promise_path = os.path.join(self.plugin_dir, promise_name)
self.writeFile(promise_path, promise_content)
self.writeInit()
# output .slapgrid/promise/result
# history .slapgrid/promise/history'
promise_dir = os.path.join(self.partition_dir, '.slapgrid', 'promise')
result_dir = os.path.join(promise_dir, 'result')
history_dir = os.path.join(promise_dir, 'history')
os.makedirs(result_dir)
os.makedirs(history_dir)
def createFile(path, name, content=''):
filepath = os.path.join(path, name)
with open(filepath, 'w') as fh:
fh.write(content)
return filepath
promise_status_json = createFile(
promise_dir, 'promise_status.json',
'{"stale.py": ["OK", "2020-01-09T08:09:44+0000", '
'"260ca9dd8a4577fc00b7bd5810298076"]}')
dummy_result = createFile(result_dir, 'dummy.status.json')
dummy_history = createFile(history_dir, 'dummy.history.json')
dummy_history_old = createFile(history_dir, 'dummy.history.old.json')
stale_result = createFile(result_dir, 'stale.status.json')
stale_history = createFile(history_dir, 'stale.history.json')
stale_history_old = createFile(history_dir, 'stale.history.old.json')
just_result_file = createFile(result_dir, 'doesnotmatch')
just_history_file = createFile(history_dir, 'doesnotmatch')
def assertPathExists(path):
self.assertTrue(os.path.exists(path))
def assertPathNotExists(path):
self.assertFalse(os.path.exists(path))
self.initialisePromise()
self.launcher.run()
assertPathExists(dummy_result)
assertPathExists(dummy_history)
assertPathExists(dummy_history_old)
# check that it does not clean up too much
assertPathExists(just_result_file)
assertPathExists(just_history_file)
assertPathNotExists(stale_result)
assertPathNotExists(stale_history)
assertPathNotExists(stale_history_old)
with open(promise_status_json) as fh:
promise_status = json.load(fh)
self.assertNotIn('stale.py', promise_status)
def test_promise_anomaly_disabled(self): def test_promise_anomaly_disabled(self):
self.initialisePromise() self.initialisePromise()
promise_process = self.createPromiseProcess() promise_process = self.createPromiseProcess()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment