Commit f7a543bb authored by Alain Takoudjou's avatar Alain Takoudjou

share rmtree from cli/prune with slapgrid and use it to drop partition

See merge request !227
parents f5407c3a b0447adf
...@@ -34,6 +34,7 @@ import six.moves.configparser as configparser ...@@ -34,6 +34,7 @@ import six.moves.configparser as configparser
from slapos.cli.config import ConfigCommand from slapos.cli.config import ConfigCommand
from slapos.grid.slapgrid import merged_options from slapos.grid.slapgrid import merged_options
from slapos.util import rmtree
class PruneCommand(ConfigCommand): class PruneCommand(ConfigCommand):
...@@ -229,53 +230,3 @@ def getUsageSignatureFromSoftwareAndSharedPart( ...@@ -229,53 +230,3 @@ def getUsageSignatureFromSoftwareAndSharedPart(
with open(shared_signature) as f: with open(shared_signature) as f:
signatures[shared_signature] = f.read() signatures[shared_signature] = f.read()
return signatures return signatures
# XXX copied from https://lab.nexedi.com/nexedi/erp5/blob/31804f683fd36322fb38aeb9654bee70cebe4fdb/erp5/util/testnode/Utils.py
# TODO: move to shared place or ... isn't there already such an utility function in slapos.core ?
import shutil
import errno
import six
from six.moves import map
try:
PermissionError
except NameError: # make pylint happy on python2...
PermissionError = Exception
def rmtree(path):
"""Delete a path recursively.
Like shutil.rmtree, but supporting the case that some files or folder
might have been marked read only. """
def chmod_retry(func, failed_path, exc_info):
"""Make sure the directories are executable and writable.
"""
# Depending on the Python version, the following items differ.
if six.PY3:
expected_error_type = PermissionError
expected_func = os.lstat
else:
expected_error_type = OSError
expected_func = os.listdir
e = exc_info[1]
if isinstance(e, expected_error_type):
if e.errno == errno.ENOENT:
# because we are calling again rmtree on listdir errors, this path might
# have been already deleted by the recursive call to rmtree.
return
if e.errno == errno.EACCES:
if func is expected_func:
os.chmod(failed_path, 0o700)
# corner case to handle errors in listing directories.
# https://bugs.python.org/issue8523
return shutil.rmtree(failed_path, onerror=chmod_retry)
# If parent directory is not writable, we still cannot delete the file.
# But make sure not to change the parent of the folder we are deleting.
if failed_path != path:
os.chmod(os.path.dirname(failed_path), 0o700)
return func(failed_path)
raise e # XXX make pylint happy
shutil.rmtree(path, onerror=chmod_retry)
# / erp5/util/testnode/Utils.py code
...@@ -52,7 +52,7 @@ from slapos.grid.exception import (BuildoutFailedError, WrongPermissionError, ...@@ -52,7 +52,7 @@ from slapos.grid.exception import (BuildoutFailedError, WrongPermissionError,
PathDoesNotExistError, DiskSpaceError) PathDoesNotExistError, DiskSpaceError)
from slapos.grid.networkcache import download_network_cached, upload_network_cached from slapos.grid.networkcache import download_network_cached, upload_network_cached
from slapos.human import bytes2human from slapos.human import bytes2human
from slapos.util import bytes2str from slapos.util import bytes2str, rmtree
WATCHDOG_MARK = '-on-watch' WATCHDOG_MARK = '-on-watch'
...@@ -354,17 +354,10 @@ class Software(object): ...@@ -354,17 +354,10 @@ class Software(object):
def destroy(self): def destroy(self):
"""Removes software release.""" """Removes software release."""
def retry(func, path, exc):
# inspired by slapos.buildout hard remover
if func == os.path.islink:
os.unlink(path)
else:
os.chmod(path, 0o600)
func(path)
try: try:
if os.path.exists(self.software_path): if os.path.exists(self.software_path):
self.logger.info('Removing path %r' % self.software_path) self.logger.info('Removing path %r' % self.software_path)
shutil.rmtree(self.software_path, onerror=retry) rmtree(self.software_path)
else: else:
self.logger.info('Path %r does not exists, no need to remove.' % self.logger.info('Path %r does not exists, no need to remove.' %
self.software_path) self.software_path)
......
...@@ -397,6 +397,8 @@ class ComputerForTest(object): ...@@ -397,6 +397,8 @@ class ComputerForTest(object):
return {'status_code': 200} return {'status_code': 200}
if url.path == '/buildingSoftwareRelease': if url.path == '/buildingSoftwareRelease':
return {'status_code': 200} return {'status_code': 200}
if url.path == '/destroyedSoftwareRelease':
return {'status_code': 200}
if url.path == '/softwareReleaseError': if url.path == '/softwareReleaseError':
software.error_log = '\n'.join( software.error_log = '\n'.join(
[ [
...@@ -1921,6 +1923,41 @@ echo %s; echo %s; exit 42""" % (line1, line2)) ...@@ -1921,6 +1923,41 @@ echo %s; echo %s; exit 42""" % (line1, line2))
self.launchSlapgridSoftware() self.launchSlapgridSoftware()
self.assertIn('shared-part-list = %s' % self.shared_parts_root, software.error_log) self.assertIn('shared-part-list = %s' % self.shared_parts_root, software.error_log)
def test_remove_software(self):
computer = ComputerForTest(self.software_root, self.instance_root, 1, 1)
with httmock.HTTMock(computer.request_handler):
software = computer.software_list[0]
software.setBuildout("""#!/bin/sh
mkdir directory
touch directory/file
""")
self.launchSlapgridSoftware()
self.assertIn('directory', os.listdir(os.path.join(self.software_root, software.software_hash)))
software.requested_state = 'destroyed'
self.launchSlapgridSoftware()
self.assertEqual(os.listdir(self.software_root), [])
def test_remove_software_chmod(self):
# This software is "hard" to remove, as permissions have been changed
computer = ComputerForTest(self.software_root, self.instance_root, 1, 1)
with httmock.HTTMock(computer.request_handler):
software = computer.software_list[0]
software.setBuildout("""#!/bin/sh
mkdir directory
touch directory/file
chmod a-rxw directory/file
chmod a-rxw directory
""")
self.launchSlapgridSoftware()
self.assertIn('directory', os.listdir(os.path.join(self.software_root, software.software_hash)))
software.requested_state = 'destroyed'
self.launchSlapgridSoftware()
self.assertEqual(os.listdir(self.software_root), [])
class SlapgridInitialization(unittest.TestCase): class SlapgridInitialization(unittest.TestCase):
""" """
......
...@@ -39,6 +39,14 @@ import six ...@@ -39,6 +39,14 @@ import six
from six.moves.urllib import parse from six.moves.urllib import parse
import hashlib import hashlib
import netaddr import netaddr
import shutil
try:
PermissionError
except NameError: # make pylint happy on python2...
PermissionError = Exception
def mkdir_p(path, mode=0o700): def mkdir_p(path, mode=0o700):
...@@ -221,3 +229,39 @@ def _addIpv6Brackets(url): ...@@ -221,3 +229,39 @@ def _addIpv6Brackets(url):
api_netloc = '%s:%s' % (api_netloc, port) api_netloc = '%s:%s' % (api_netloc, port)
url = parse.urlunsplit((api_scheme, api_netloc, api_path, api_query, api_fragment)) url = parse.urlunsplit((api_scheme, api_netloc, api_path, api_query, api_fragment))
return url return url
def rmtree(path):
"""Delete a path recursively.
Like shutil.rmtree, but supporting the case that some files or folder
might have been marked read only. """
def chmod_retry(func, failed_path, exc_info):
"""Make sure the directories are executable and writable.
"""
# Depending on the Python version, the following items differ.
if six.PY3:
expected_error_type = PermissionError
expected_func = os.lstat
else:
expected_error_type = OSError
expected_func = os.listdir
e = exc_info[1]
if isinstance(e, expected_error_type):
if e.errno == errno.ENOENT:
# because we are calling again rmtree on listdir errors, this path might
# have been already deleted by the recursive call to rmtree.
return
if e.errno == errno.EACCES:
if func is expected_func:
os.chmod(failed_path, 0o700)
# corner case to handle errors in listing directories.
# https://bugs.python.org/issue8523
return shutil.rmtree(failed_path, onerror=chmod_retry)
# If parent directory is not writable, we still cannot delete the file.
# But make sure not to change the parent of the folder we are deleting.
if failed_path != path:
os.chmod(os.path.dirname(failed_path), 0o700)
return func(failed_path)
raise e # XXX make pylint happy
shutil.rmtree(path, onerror=chmod_retry)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment