Commit 23f35307 authored by Xavier Thompson's avatar Xavier Thompson

slapproxy: Fix software URL migration

The `local-software-release-url` option allows migrating the software
URLs which are local paths by rebasing them on the path provided by
the option.

Do not migrate software release URLs if the old root path and the
new root path are subpaths or superpaths one of the other.

In addition, do not migrate an URL if the old one refers to an
existing file and the new one doesn't.

Also, create a backup of the database before migrating.

See merge request !338
parent 7cb0769c
Pipeline #17921 failed with stage
in 0 seconds
......@@ -203,23 +203,45 @@ def _updateLocalSoftwareReleaseRootPathIfNeeded():
current_root_path = execute_db('local_software_release_root', 'SELECT * from %s', one=True)['path'] or os.sep
new_root_path = app.config['local_software_release_root'] or os.sep
execute_db('local_software_release_root', 'UPDATE %s SET path=?', [new_root_path])
# Check whether one is the same as or a subpath of the other
if current_root_path == new_root_path:
return
relpath = os.path.relpath(new_root_path, current_root_path)
if not relpath.startswith(os.pardir + os.sep):
app.logger.info('Do not rebase any URLs because %s is a subpath of %s', new_root_path, current_root_path)
return
elif os.path.basename(relpath) == os.pardir:
app.logger.info('Do not rebase any URLs because %s is a superpath of %s', new_root_path, current_root_path)
return
# Backup the database before migrating
database_path = app.config['DATABASE_URI']
backup_path = database_path + "-backup-%s.sql" % datetime.now().isoformat()
app.logger.info("Backuping database to %s", backup_path)
with open(backup_path, 'w') as f:
for line in g.db.iterdump():
f.write('%s\n' % line)
# Rebase all URLs relative to the new root path
if current_root_path != new_root_path:
app.logger.info('Updating local software release root path: %s --> %s', current_root_path, new_root_path)
def migrate_url(url):
if not url or urlparse(url).scheme:
app.logger.debug('Migrate URL ? N: %s is not a path', url)
return url
rel = os.path.relpath(url, current_root_path)
if rel.startswith(os.pardir + os.sep):
app.logger.debug('Migrate URL ? N: %s is not a subpath', url)
return url
new = os.path.join(new_root_path, rel)
app.logger.debug('Migrate URL ? Y: %s -> %s', url, new)
return new
g.db.create_function('migrate_url', 1, migrate_url)
execute_db('software', 'UPDATE %s SET url=migrate_url(url)')
execute_db('partition', 'UPDATE %s SET software_release=migrate_url(software_release)')
app.logger.info('Rebase URLs on local software release root path')
app.logger.info('Old root path: %s', current_root_path)
app.logger.info('New root path: %s', new_root_path)
def migrate_url(url):
app.logger.debug('Examining URL %s', url)
if not url or urlparse(url).scheme:
app.logger.debug(' Do not rebase because it is not a path')
return url
rel = os.path.relpath(url, current_root_path)
if rel.startswith(os.pardir + os.sep):
app.logger.debug(' Do not rebase because it is not a subpath of %s', current_root_path)
return url
new = os.path.join(new_root_path, rel)
if not os.path.isfile(new) and os.path.isfile(url):
app.logger.debug(' Do not rebase because it refers to an existing file but %s does not', new)
return url
app.logger.debug(' Migrate to rebased URL %s', new)
return new
g.db.create_function('migrate_url', 1, migrate_url)
execute_db('software', 'UPDATE %s SET url=migrate_url(url)')
execute_db('partition', 'UPDATE %s SET software_release=migrate_url(software_release)')
is_schema_already_executed = False
@app.before_request
......
......@@ -2060,19 +2060,21 @@ class TestLocalSoftwareReleaseRootPathMigration(MasterMixin):
def newRootDir(self):
return os.path.join(self._tempdir, str(1 + int(os.path.basename(self._rootdir))))
def createSlapOSConfigurationFile(self):
def createSlapOSConfigurationFile(self, localdir=None):
localdir = localdir or os.path.join(self._rootdir, 'opt')
super(TestLocalSoftwareReleaseRootPathMigration, self).createSlapOSConfigurationFile()
with open(self.slapos_cfg, 'a') as f:
f.write("\nlocal_software_release_root = %s/opt" % self._rootdir)
def moveProxy(self, rootdir=None):
if not rootdir:
rootdir = self.newRootDir()
os.rename(self._rootdir, rootdir)
self._rootdir = rootdir
f.write("\nlocal_software_release_root = %s" % localdir)
def moveProxy(self, rootdir=None, localdir=None):
if not localdir:
if not rootdir:
rootdir = self.newRootDir()
os.rename(self._rootdir, rootdir)
self._rootdir = rootdir
self.slapos_cfg = os.path.join(self._rootdir, 'slapos.cfg')
self.proxy_db = os.path.join(self._rootdir, 'lib', 'proxy.db')
self.createSlapOSConfigurationFile()
self.createSlapOSConfigurationFile(localdir)
views.is_schema_already_executed = False
self.startProxy()
os.environ.pop('SLAPGRID_INSTANCE_ROOT', None)
......@@ -2084,23 +2086,23 @@ class TestLocalSoftwareReleaseRootPathMigration(MasterMixin):
def assertPartitionUrl(self, partition_id, expected_url):
self.assertEqual(self.getPartitionInformation(partition_id).getSoftwareRelease().getURI(), expected_url)
def checkSupplyUrl(self, initial_url, expected_url, rootdir=None):
def checkSupplyUrl(self, initial_url, expected_url, rootdir=None, localdir=None):
self.supply(initial_url)
self.assertSoftwareUrls(initial_url)
self.moveProxy(rootdir)
self.moveProxy(rootdir, localdir)
self.assertSoftwareUrls(expected_url)
def checkRequestUrl(self, initial_url, expected_url, rootdir=None):
def checkRequestUrl(self, initial_url, expected_url, rootdir=None, localdir=None):
self.format_for_number_of_partitions(1)
partition = self.request(initial_url, None, 'MyInstance', 'slappart0')
self.assertPartitionUrl(partition._partition_id, initial_url)
self.moveProxy(rootdir)
self.moveProxy(rootdir, localdir)
self.assertPartitionUrl(partition._partition_id, expected_url)
def test_supply_local_url(self):
initial_url = os.path.join(self._rootdir, 'opt', 'software.cfg')
initial_url = os.path.join(self._rootdir, 'opt', 'soft', 'software.cfg')
new_rootdir = self.newRootDir()
expected_url = os.path.join(new_rootdir, 'opt', 'software.cfg')
expected_url = os.path.join(new_rootdir, 'opt', 'soft', 'software.cfg')
self.checkSupplyUrl(initial_url, expected_url, new_rootdir)
def test_supply_not_in_root_url(self):
......@@ -2116,13 +2118,13 @@ class TestLocalSoftwareReleaseRootPathMigration(MasterMixin):
self.checkSupplyUrl(url, url)
def test_request_local_url(self):
initial_url = os.path.join(self._rootdir, 'opt', 'software.cfg')
initial_url = os.path.join(self._rootdir, 'opt', 'soft', 'software.cfg')
new_rootdir = self.newRootDir()
expected_url = os.path.join(new_rootdir, 'opt', 'software.cfg')
expected_url = os.path.join(new_rootdir, 'opt', 'soft', 'software.cfg')
self.checkRequestUrl(initial_url, expected_url, new_rootdir)
def test_request_not_in_root_url(self):
url = os.path.join(self._rootdir, 'srv', 'software.cfg')
url = os.path.join(self._rootdir, 'srv', 'soft', 'software.cfg')
self.checkRequestUrl(url, url)
def test_request_http_url(self):
......@@ -2134,10 +2136,10 @@ class TestLocalSoftwareReleaseRootPathMigration(MasterMixin):
self.checkRequestUrl(url, url)
def checkMultipleMoves(self, checkUrl):
initial_url = os.path.join(self._rootdir, 'opt', 'software.cfg')
initial_url = os.path.join(self._rootdir, 'opt', 'soft', 'software.cfg')
for _ in range(5):
new_rootdir = self.newRootDir()
expected_url = os.path.join(new_rootdir, 'opt', 'software.cfg')
expected_url = os.path.join(new_rootdir, 'opt', 'soft', 'software.cfg')
checkUrl(initial_url, expected_url, new_rootdir)
initial_url = expected_url
......@@ -2149,8 +2151,8 @@ class TestLocalSoftwareReleaseRootPathMigration(MasterMixin):
def test_move_logs(self):
local_sr_root = os.path.join(self._rootdir, 'opt')
subpath_url = os.path.join(local_sr_root, 'software.cfg')
path_not_subpath_url = os.path.join(self._rootdir, 'srv', 'software.cfg')
subpath_url = os.path.join(local_sr_root, 'soft', 'software.cfg')
path_not_subpath_url = os.path.join(self._rootdir, 'srv', 'soft', 'software.cfg')
http_url = "https://sr//"
self.format_for_number_of_partitions(3)
......@@ -2160,23 +2162,73 @@ class TestLocalSoftwareReleaseRootPathMigration(MasterMixin):
self.moveProxy()
new_local_sr_root = os.path.join(self._rootdir, 'opt')
new_subpath_url = os.path.join(new_local_sr_root, 'software.cfg')
new_subpath_url = os.path.join(new_local_sr_root, 'soft', 'software.cfg')
with mock.patch.object(views.app, 'logger') as logger:
# Request something to trigger update
self.getFullComputerInformation()
logger.info.assert_called_once_with(
'Updating local software release root path: %s --> %s',
local_sr_root,
new_local_sr_root,
)
logger.info.assert_has_calls([
mock.call('Backuping database to %s' , mock.ANY),
mock.call('Rebase URLs on local software release root path'),
mock.call('Old root path: %s', local_sr_root),
mock.call('New root path: %s', new_local_sr_root)
])
backup_path = logger.info.call_args_list[0][0][1]
with open(backup_path) as f:
dump = f.read()
self.assertIn("CREATE TABLE", dump)
self.assertIn('INSERT INTO', dump)
logger.debug.assert_has_calls([
mock.call('Migrate URL ? Y: %s -> %s', subpath_url, new_subpath_url),
mock.call('Migrate URL ? N: %s is not a subpath', path_not_subpath_url),
mock.call('Migrate URL ? N: %s is not a path', http_url)
mock.call('Examining URL %s', subpath_url),
mock.call(' Migrate to rebased URL %s', new_subpath_url),
mock.call(' Do not rebase because it is not a subpath of %s', local_sr_root),
mock.call(' Do not rebase because it is not a path')
]*2, any_order=True)
def checkSupplyAndRequestUrl(self, initial_url, expected_url, rootdir=None, localdir=None):
self.format_for_number_of_partitions(1)
self.supply(initial_url)
partition = self.request(initial_url, None, 'MyInstance', 'slappart0')
self.assertSoftwareUrls(initial_url)
self.assertPartitionUrl(partition._partition_id, initial_url)
self.moveProxy(rootdir, localdir)
self.assertSoftwareUrls(expected_url)
self.assertPartitionUrl(partition._partition_id, expected_url)
def test_move_to_subpath(self):
initial_url = os.path.join(self._rootdir, 'opt', 'soft', 'software.cfg')
new_localdir = os.path.join(self._rootdir, 'opt', 'soft')
self.checkSupplyAndRequestUrl(initial_url, initial_url, None, localdir=new_localdir)
def test_move_to_superpath(self):
initial_url = os.path.join(self._rootdir, 'opt', 'soft', 'software.cfg')
self.checkSupplyAndRequestUrl(initial_url, initial_url, None, localdir=self._rootdir)
def createSoftware(self, rootdir):
softdir = os.path.join(rootdir, 'opt', 'soft')
url = os.path.join(softdir, 'software.cfg')
os.makedirs(softdir)
with open(url, 'w'):
pass
return url
def test_move_initial_exists(self):
initial_url = self.createSoftware(self._rootdir)
new_rootdir = self.newRootDir()
new_localdir = os.path.join(new_rootdir, 'opt')
os.mkdir(new_rootdir)
self.checkSupplyAndRequestUrl(initial_url, initial_url, new_rootdir, new_localdir)
def test_move_both_exist(self):
initial_url = self.createSoftware(self._rootdir)
new_rootdir = self.newRootDir()
expected_url = self.createSoftware(new_rootdir)
new_localdir = os.path.join(new_rootdir, 'opt')
self.checkSupplyAndRequestUrl(initial_url, expected_url, new_rootdir, new_localdir)
class _MigrationTestCase(TestInformation, TestRequest, TestSlaveRequest, TestMultiNodeSupport):
"""
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment