Commit 0b5cdd66 authored by Alain Takoudjou's avatar Alain Takoudjou

pre-delete pluging: update to naming conventions, add more tests

This pluging can be used a add a script which wipe data before the partition is removed
The script run after all partition services are stopped. The script is ran by supervisord
then slapos node report will wait until it finish before destroy the partition.

/reviewed-on slapos.core!23
parent 7e99dfc3
...@@ -354,6 +354,7 @@ class Partition(object): ...@@ -354,6 +354,7 @@ class Partition(object):
self.instance_path = instance_path self.instance_path = instance_path
self.run_path = os.path.join(self.instance_path, 'etc', 'run') self.run_path = os.path.join(self.instance_path, 'etc', 'run')
self.service_path = os.path.join(self.instance_path, 'etc', 'service') self.service_path = os.path.join(self.instance_path, 'etc', 'service')
self.prerm_path = os.path.join(self.instance_path, 'etc', 'prerm')
self.supervisord_partition_configuration_path = \ self.supervisord_partition_configuration_path = \
supervisord_partition_configuration_path supervisord_partition_configuration_path
self.supervisord_socket = supervisord_socket self.supervisord_socket = supervisord_socket
...@@ -453,16 +454,18 @@ class Partition(object): ...@@ -453,16 +454,18 @@ class Partition(object):
'USER': pwd.getpwuid(uid).pw_name, 'USER': pwd.getpwuid(uid).pw_name,
} }
def addServiceToCustomGroup(self, group_id, runner_list, path): def addServiceToCustomGroup(self, group_suffix, partition_id, runner_list,
path, extension=''):
"""Add new services to supervisord that belong to specific group""" """Add new services to supervisord that belong to specific group"""
group_partition_template = pkg_resources.resource_stream(__name__, group_partition_template = pkg_resources.resource_stream(__name__,
'templates/group_partition_supervisord.conf.in').read() 'templates/group_partition_supervisord.conf.in').read()
self.supervisor_configuration_groups += group_partition_template % { group_id = '-'.join([partition_id, group_suffix])
self.supervisor_configuration_group += group_partition_template % {
'instance_id': group_id, 'instance_id': group_id,
'program_list': ','.join(['_'.join([group_id, runner]) 'program_list': ','.join(['_'.join([group_id, runner])
for runner in runner_list]) for runner in runner_list])
} }
return self.addServiceToGroup(group_id, runner_list, path) return self.addServiceToGroup(group_id, runner_list, path, extension)
def updateSymlink(self, sr_symlink, software_path): def updateSymlink(self, sr_symlink, software_path):
if os.path.lexists(sr_symlink): if os.path.lexists(sr_symlink):
...@@ -612,7 +615,7 @@ class Partition(object): ...@@ -612,7 +615,7 @@ class Partition(object):
runner_list = [] runner_list = []
service_list = [] service_list = []
self.partition_supervisor_configuration = "" self.partition_supervisor_configuration = ""
self.supervisor_configuration_groups = "" self.supervisor_configuration_group = ""
if os.path.exists(self.run_path): if os.path.exists(self.run_path):
if os.path.isdir(self.run_path): if os.path.isdir(self.run_path):
runner_list = os.listdir(self.run_path) runner_list = os.listdir(self.run_path)
...@@ -628,7 +631,7 @@ class Partition(object): ...@@ -628,7 +631,7 @@ class Partition(object):
partition_id = self.computer_partition.getId() partition_id = self.computer_partition.getId()
group_partition_template = pkg_resources.resource_stream(__name__, group_partition_template = pkg_resources.resource_stream(__name__,
'templates/group_partition_supervisord.conf.in').read() 'templates/group_partition_supervisord.conf.in').read()
self.supervisor_configuration_groups = group_partition_template % { self.supervisor_configuration_group = group_partition_template % {
'instance_id': partition_id, 'instance_id': partition_id,
'program_list': ','.join(['_'.join([partition_id, runner]) 'program_list': ','.join(['_'.join([partition_id, runner])
for runner in runner_list + service_list]) for runner in runner_list + service_list])
...@@ -642,10 +645,10 @@ class Partition(object): ...@@ -642,10 +645,10 @@ class Partition(object):
""" """
Write supervisord configuration file and update supervisord Write supervisord configuration file and update supervisord
""" """
if self.supervisor_configuration_groups and \ if self.supervisor_configuration_group and \
self.partition_supervisor_configuration: self.partition_supervisor_configuration:
updateFile(self.supervisord_partition_configuration_path, updateFile(self.supervisord_partition_configuration_path,
self.supervisor_configuration_groups + self.supervisor_configuration_group +
self.partition_supervisor_configuration) self.partition_supervisor_configuration)
self.updateSupervisor() self.updateSupervisor()
......
...@@ -80,7 +80,7 @@ PROMISE_TIMEOUT = 3 ...@@ -80,7 +80,7 @@ PROMISE_TIMEOUT = 3
COMPUTER_PARTITION_TIMESTAMP_FILENAME = '.timestamp' COMPUTER_PARTITION_TIMESTAMP_FILENAME = '.timestamp'
COMPUTER_PARTITION_LATEST_BANG_TIMESTAMP_FILENAME = '.slapos_latest_bang_timestamp' COMPUTER_PARTITION_LATEST_BANG_TIMESTAMP_FILENAME = '.slapos_latest_bang_timestamp'
COMPUTER_PARTITION_INSTALL_ERROR_FILENAME = '.slapgrid-%s-error.log' COMPUTER_PARTITION_INSTALL_ERROR_FILENAME = '.slapgrid-%s-error.log'
COMPUTER_PARTITION_WAIT_LIST_FILENAME = '.slapos-wait-services' COMPUTER_PARTITION_WAIT_LIST_FILENAME = '.slapos-report-wait-service-list'
# XXX hardcoded watchdog_path # XXX hardcoded watchdog_path
WATCHDOG_PATH = '/opt/slapos/bin/slapos-watchdog' WATCHDOG_PATH = '/opt/slapos/bin/slapos-watchdog'
...@@ -1273,7 +1273,7 @@ stderr_logfile_backups=1 ...@@ -1273,7 +1273,7 @@ stderr_logfile_backups=1
if os.path.exists(wait_file) and os.path.isfile(wait_file): if os.path.exists(wait_file) and os.path.isfile(wait_file):
with open(wait_file) as wait_f: with open(wait_file) as wait_f:
processes_list = [name.strip() for name in wait_f.readlines() if name] processes_list = [name.strip() for name in wait_f if name]
# return True if one of process in the list is running # return True if one of process in the list is running
return partition.checkProcessesFromStateList(processes_list, return partition.checkProcessesFromStateList(processes_list,
state_list) state_list)
......
...@@ -9,7 +9,6 @@ from slapos.manager import interface ...@@ -9,7 +9,6 @@ from slapos.manager import interface
from slapos.grid.slapgrid import COMPUTER_PARTITION_WAIT_LIST_FILENAME from slapos.grid.slapgrid import COMPUTER_PARTITION_WAIT_LIST_FILENAME
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
WIPE_WRAPPER_BASE_PATH = "var/run/slapos/pre-destroy/"
class Manager(object): class Manager(object):
"""Manager is called in every step of preparation of the computer.""" """Manager is called in every step of preparation of the computer."""
...@@ -45,32 +44,31 @@ class Manager(object): ...@@ -45,32 +44,31 @@ class Manager(object):
wait_filepath = os.path.join(partition.instance_path, wait_filepath = os.path.join(partition.instance_path,
COMPUTER_PARTITION_WAIT_LIST_FILENAME) COMPUTER_PARTITION_WAIT_LIST_FILENAME)
wipe_base_folder = os.path.join(partition.instance_path, if not os.path.exists(partition.prerm_path):
WIPE_WRAPPER_BASE_PATH)
if not os.path.exists(wipe_base_folder):
return return
wipe_wrapper_list = [f for f in os.listdir(wipe_base_folder) partition_id = partition.partition_id
if os.path.isfile(os.path.join(wipe_base_folder, f))] wrapper_list = [f for f in os.listdir(partition.prerm_path)
if len(wipe_wrapper_list) > 0: if os.path.isfile(os.path.join(partition.prerm_path, f))]
group_name = partition.partition_id + '-' + "destroy" if len(wrapper_list) > 0:
logger.info("Adding pre-destroy scripts to supervisord...") group_suffix = "prerm"
logger.info("Adding pre-delete scripts to supervisord...")
partition.generateSupervisorConfiguration() partition.generateSupervisorConfiguration()
partition.addServiceToCustomGroup(group_name, partition.addServiceToCustomGroup(group_suffix,
wipe_wrapper_list, partition_id,
wipe_base_folder) wrapper_list,
partition.prerm_path)
partition.writeSupervisorConfigurationFile() partition.writeSupervisorConfigurationFile()
# check the state of all process, if the process is not started yes, start it # check the state of all process, if the process is not started yes, start it
supervisord = partition.getSupervisorRPC() supervisord = partition.getSupervisorRPC()
process_list_string = "" process_list_string = ""
for name in wipe_wrapper_list: for name in wrapper_list:
process_name = group_name + ':' + name process_name = '-'.join([partition_id, group_suffix]) + ':' + name
process_list_string += process_name + '\n' process_list_string += '%s\n' % process_name
status = supervisord.getProcessInfo(process_name) status = supervisord.getProcessInfo(process_name)
if status['start'] == 0: if status['start'] == 0:
# process is not started yet # process is not started yet
logger.info("Starting pre-destroy process %r..." % name) logger.info("Starting pre-delete process %r..." % name)
supervisord.startProcess(process_name, False) supervisord.startProcess(process_name, False)
# ask to slapgrid to check theses scripts before destroy partition # ask to slapgrid to check theses scripts before destroy partition
......
...@@ -40,6 +40,7 @@ import time ...@@ -40,6 +40,7 @@ import time
import unittest import unittest
import urlparse import urlparse
import json import json
import re
import xml_marshaller import xml_marshaller
from mock import patch from mock import patch
...@@ -53,6 +54,7 @@ from slapos.grid import SlapObject ...@@ -53,6 +54,7 @@ from slapos.grid import SlapObject
from slapos.grid.SlapObject import WATCHDOG_MARK from slapos.grid.SlapObject import WATCHDOG_MARK
from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME from slapos.slap.slap import COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME
import slapos.grid.SlapObject import slapos.grid.SlapObject
from slapos import manager as slapmanager
import httmock import httmock
...@@ -2415,69 +2417,214 @@ class TestSlapgridCPWithTransaction(MasterMixin, unittest.TestCase): ...@@ -2415,69 +2417,214 @@ class TestSlapgridCPWithTransaction(MasterMixin, unittest.TestCase):
self.assertFalse(os.path.exists(request_list_file)) self.assertFalse(os.path.exists(request_list_file))
class TestSlapgridCPWithPreDeleteScript(MasterMixin, unittest.TestCase): class TestSlapgridReportWithPreDeleteScript(MasterMixin, unittest.TestCase):
def test_one_partition_pre_destroy_service(self): prerm_script_content = """#!/bin/sh
from slapos import manager as slapmanager
from slapos.manager.predestroy import WIPE_WRAPPER_BASE_PATH echo "Running prerm script for this partition..."
touch etc/prerm.txt
for i in {1..2}
do
echo "sleeping for 1s..."
sleep 1
done
echo "finished prerm script."
rm etc/prerm.txt
exit 0
"""
def _wait_prerm_script_finished(self, base_path):
check_file = os.path.join(base_path, 'etc/prerm.txt')
limit = 10
count = 0
time.sleep(1)
while (count < limit) and os.path.exists(check_file):
time.sleep(1)
count += 1
def test_partition_destroy_with_pre_remove_service(self):
computer = ComputerForTest(self.software_root, self.instance_root) computer = ComputerForTest(self.software_root, self.instance_root)
with httmock.HTTMock(computer.request_handler): with httmock.HTTMock(computer.request_handler):
partition = computer.instance_list[0] partition = computer.instance_list[0]
pre_delete_dir = os.path.join(partition.partition_path, WIPE_WRAPPER_BASE_PATH) pre_delete_dir = os.path.join(partition.partition_path, 'etc/prerm')
pre_delete_script = os.path.join(pre_delete_dir, 'slapos_pre_delete') pre_delete_script = os.path.join(pre_delete_dir, 'slapos_pre_delete')
partition.requested_state = 'started' partition.requested_state = 'started'
partition.software.setBuildout(WRAPPER_CONTENT) partition.software.setBuildout(WRAPPER_CONTENT)
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
os.makedirs(pre_delete_dir, 0o700) os.makedirs(pre_delete_dir, 0o700)
with open(pre_delete_script, 'w') as f: with open(pre_delete_script, 'w') as f:
f.write("""#!/bin/sh f.write(self.prerm_script_content)
echo "Running script to wipe this partition..."
for i in {1..3}
do
echo "sleeping for 1s..."
sleep 1
done
echo "finished wipe disk."
exit 0
""")
os.chmod(pre_delete_script, 0754) os.chmod(pre_delete_script, 0754)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(partition.partition_path), self.assertItemsEqual(os.listdir(partition.partition_path),
['.slapgrid', '.0_wrapper.log', 'buildout.cfg', 'var', ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
'etc', 'software_release', 'worked', '.slapos-retention-lock-delay']) 'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
wrapper_log = os.path.join(partition.partition_path, '.0_wrapper.log')
self.assertLogContent(wrapper_log, 'Working')
self.assertItemsEqual(os.listdir(self.software_root), [partition.software.software_hash])
self.assertEqual(computer.sequence, self.assertEqual(computer.sequence,
['/getFullComputerInformation', '/availableComputerPartition', ['/getFullComputerInformation', '/availableComputerPartition',
'/startedComputerPartition']) '/startedComputerPartition'])
self.assertEqual(partition.state, 'started') self.assertEqual(partition.state, 'started')
partition.requested_state = 'stopped' manager_list = slapmanager.from_config({'manager_list': 'prerm'})
self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
self.assertEqual(partition.state, 'stopped')
manager_list = slapmanager.from_config({'manager_list': 'predestroy'})
self.grid._manager_list = manager_list self.grid._manager_list = manager_list
partition.requested_state = 'destroyed' partition.requested_state = 'destroyed'
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is not destroyed (pre-destroy is running) # Assert partition directory is not destroyed (pre-delete is running)
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(partition.partition_path), self.assertItemsEqual(os.listdir(partition.partition_path),
['.slapgrid', '.0_wrapper.log', 'buildout.cfg', 'var', ['.slapgrid', '.0_wrapper.log', 'buildout.cfg',
'etc', 'software_release', 'worked', '.slapos-retention-lock-delay', 'etc', 'software_release', 'worked', '.slapos-retention-lock-delay',
'.0-destroy_slapos_pre_delete.log', '.slapos-wait-services', '.0-prerm_slapos_pre_delete.log', '.slapos-report-wait-service-list',
'.slapos-request-transaction-0']) '.slapos-request-transaction-0'])
self.assertItemsEqual(os.listdir(self.software_root), self.assertItemsEqual(os.listdir(self.software_root),
[partition.software.software_hash]) [partition.software.software_hash])
# wait until the pre-destroy script is finished # wait until the pre-delete script is finished
time.sleep(5) self._wait_prerm_script_finished(partition.partition_path)
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is empty
self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(partition.partition_path), [])
def test_partition_destroy_pre_remove_with_retention_lock(self):
computer = ComputerForTest(self.software_root, self.instance_root)
with httmock.HTTMock(computer.request_handler):
partition = computer.instance_list[0]
pre_delete_dir = os.path.join(partition.partition_path, 'etc/prerm')
pre_delete_script = os.path.join(pre_delete_dir, 'slapos_pre_delete')
partition.requested_state = 'started'
partition.filter_dict = {'retention_delay': 1.0 / (3600 * 24)}
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
self.assertTrue(os.path.exists(os.path.join(
partition.partition_path,
slapos.grid.SlapObject.Partition.retention_lock_delay_filename
)))
os.makedirs(pre_delete_dir, 0o700)
with open(pre_delete_script, 'w') as f:
f.write(self.prerm_script_content)
os.chmod(pre_delete_script, 0754)
self.assertTrue(os.path.exists(pre_delete_script))
manager_list = slapmanager.from_config({'manager_list': 'prerm'})
self.grid._manager_list = manager_list
partition.requested_state = 'destroyed'
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is not destroyed (retention-delay-lock)
self.assertItemsEqual(os.listdir(partition.partition_path),
['.slapgrid', 'buildout.cfg', 'etc', 'software_release',
'worked', '.slapos-retention-lock-delay',
'.slapos-retention-lock-date', '.slapos-request-transaction-0'])
self.assertTrue(os.path.exists(pre_delete_script))
self.assertTrue(os.path.exists(os.path.join(
partition.partition_path,
slapos.grid.SlapObject.Partition.retention_lock_date_filename
)))
time.sleep(1)
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is not destroyed (pre-delete is running)
self.assertItemsEqual(os.listdir(partition.partition_path),
['.slapgrid', 'buildout.cfg', 'etc', 'software_release',
'worked', '.slapos-retention-lock-delay', '.slapos-retention-lock-date',
'.0-prerm_slapos_pre_delete.log', '.slapos-report-wait-service-list',
'.slapos-request-transaction-0'])
# wait until the pre-delete script is finished
self._wait_prerm_script_finished(partition.partition_path)
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is empty
self.assertItemsEqual(os.listdir(partition.partition_path), [])
def test_partition_destroy_pre_remove_script_not_stopped(self):
computer = ComputerForTest(self.software_root, self.instance_root)
with httmock.HTTMock(computer.request_handler):
partition = computer.instance_list[0]
pre_delete_dir = os.path.join(partition.partition_path, 'etc/prerm')
pre_delete_script = os.path.join(pre_delete_dir, 'slapos_pre_delete')
partition.requested_state = 'started'
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
os.makedirs(pre_delete_dir, 0o700)
with open(pre_delete_script, 'w') as f:
f.write(self.prerm_script_content)
os.chmod(pre_delete_script, 0754)
self.assertEqual(partition.state, 'started')
manager_list = slapmanager.from_config({'manager_list': 'prerm'})
self.grid._manager_list = manager_list
partition.requested_state = 'destroyed'
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is not destroyed (pre-delete is running)
self.assertItemsEqual(os.listdir(partition.partition_path),
['.slapgrid', 'buildout.cfg', 'etc', 'software_release',
'worked', '.slapos-retention-lock-delay', '.slapos-request-transaction-0',
'.0-prerm_slapos_pre_delete.log', '.slapos-report-wait-service-list'])
# wait until the pre-delete script is finished
self._wait_prerm_script_finished(partition.partition_path)
with open(os.path.join(partition.partition_path, '.0-prerm_slapos_pre_delete.log')) as f:
# the script is well finished...
self.assertTrue("finished prerm script." in f.read())
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS) self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is empty # Assert partition directory is empty
self.assertInstanceDirectoryListEqual(['0']) self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(partition.partition_path), []) self.assertItemsEqual(os.listdir(partition.partition_path), [])
def test_partition_destroy_pre_remove_script_run_as_partition_user(self):
computer = ComputerForTest(self.software_root, self.instance_root)
with httmock.HTTMock(computer.request_handler):
partition = computer.instance_list[0]
pre_delete_dir = os.path.join(partition.partition_path, 'etc/prerm')
pre_delete_script = os.path.join(pre_delete_dir, 'slapos_pre_delete')
partition.requested_state = 'started'
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
os.makedirs(pre_delete_dir, 0o700)
with open(pre_delete_script, 'w') as f:
f.write(self.prerm_script_content)
os.chmod(pre_delete_script, 0754)
manager_list = slapmanager.from_config({'manager_list': 'prerm'})
self.grid._manager_list = manager_list
partition.requested_state = 'destroyed'
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is not destroyed (pre-delete is running)
self.assertItemsEqual(os.listdir(partition.partition_path),
['.slapgrid', 'buildout.cfg', 'etc', 'software_release',
'worked', '.slapos-retention-lock-delay', '.slapos-request-transaction-0',
'.0-prerm_slapos_pre_delete.log', '.slapos-report-wait-service-list'])
stat_info = os.stat(partition.partition_path)
uid = stat_info.st_uid
gid = stat_info.st_gid
supervisor_conf_file = os.path.join(self.instance_root,
'etc/supervisord.conf.d',
'%s.conf' % partition.name)
self.assertTrue(os.path.exists(supervisor_conf_file))
regex_user = r"user=(\d+)"
regex_group = r"group=(\d+)"
with open(supervisor_conf_file) as f:
config = f.read()
# search user uid in conf file
result = re.search(regex_user, config, re.DOTALL)
self.assertTrue(result is not None)
self.assertEqual(int(result.groups()[0]), uid)
# search user group gid in conf file
result = re.search(regex_group, config, re.DOTALL)
self.assertTrue(result is not None)
self.assertEqual(int(result.groups()[0]), gid)
# wait until the pre-delete script is finished
self._wait_prerm_script_finished(partition.partition_path)
self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS)
# Assert partition directory is empty
self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(partition.partition_path), [])
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment