slapproxy: add automatic migration procedure to new schema.

Migration to new database schema is automatically done when an old version is detected.
It keeps the old tables untouched in case of needed rollback to an older version.

slapos/proxy/schema.sql

 --version:11
 CREATE TABLE IF NOT EXISTS software%(version)s (
   url VARCHAR(255),
-  computer_reference VARCHAR(255) DEFAULT 'computer',
+  computer_reference VARCHAR(255) DEFAULT '%(computer)s',
   CONSTRAINT uniq PRIMARY KEY (url, computer_reference)
 );
 
 CREATE TABLE IF NOT EXISTS computer%(version)s (
-  reference VARCHAR(255),
+  reference VARCHAR(255) DEFAULT '%(computer)s',
   address VARCHAR(255),
   netmask VARCHAR(255),
   CONSTRAINT uniq PRIMARY KEY (reference)
@@ -14,7 +14,7 @@ CREATE TABLE IF NOT EXISTS computer%(version)s (
 
 CREATE TABLE IF NOT EXISTS partition%(version)s (
   reference VARCHAR(255),
-  computer_reference VARCHAR(255) DEFAULT 'computer',
+  computer_reference VARCHAR(255) DEFAULT '%(computer)s',
   slap_state VARCHAR(255) DEFAULT 'free',
   software_release VARCHAR(255),
   xml TEXT,
@@ -30,7 +30,7 @@ CREATE TABLE IF NOT EXISTS partition%(version)s (
 
 CREATE TABLE IF NOT EXISTS slave%(version)s (
   reference VARCHAR(255),
-  computer_reference VARCHAR(255),
+  computer_reference VARCHAR(255) DEFAULT '%(computer)s',
   connection_xml TEXT,
   hosted_by VARCHAR(255),
   asked_by VARCHAR(255) -- only used for debugging,
@@ -39,8 +39,9 @@ CREATE TABLE IF NOT EXISTS slave%(version)s (
 
 CREATE TABLE IF NOT EXISTS partition_network%(version)s (
   partition_reference VARCHAR(255),
-  computer_reference VARCHAR(255),
+  computer_reference VARCHAR(255) DEFAULT '%(computer)s',
   reference VARCHAR(255),
   address VARCHAR(255),
   netmask VARCHAR(255)
 );
+

slapos/proxy/views.py

@@ -108,9 +108,14 @@ def partitiondict2partition(partition):
   return slap_partition
 
 
-def execute_db(table, query, args=(), one=False):
+def execute_db(table, query, args=(), one=False, db_version=None, log=False):
+  if not db_version:
+    db_version = DB_VERSION
+  query = query % (table + db_version,)
+  if log:
+    print query
   try:
-    cur = g.db.execute(query % (table + DB_VERSION,), args)
+    cur = g.db.execute(query, args)
   except:
     app.logger.error('There was some issue during processing query %r on table %r with args %r' % (query, table, args))
     raise
@@ -122,15 +127,66 @@ def execute_db(table, query, args=(), one=False):
 def connect_db():
   return sqlite3.connect(app.config['DATABASE_URI'])
 
+def _getTableList():
+  return g.db.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY Name").fetchall()
+
+def _getCurrentDatabaseSchemaVersion():
+  """
+  Return version of database schema.
+  As there is no actual definition of version, analyse
+  name of all tables (containing version) and take the
+  highest version (as several versions can live in the db).
+  """
+  # XXX: define an actual version and proper migration/repair procedure.
+  version = -1
+  for table_name in _getTableList():
+    try:
+      table_version = int(table_name[0][-2:])
+    except ValueError:
+      table_version = int(table_name[0][-1:])
+    if table_version > version:
+      version = table_version
+  return str(version)
+
+def _upgradeDatabaseIfNeeded():
+  """
+  Analyses current database compared to defined schema,
+  and adapt tables/data it if needed.
+  """
+  current_schema_version = _getCurrentDatabaseSchemaVersion()
+  # If version of current database is not old, do nothing
+  if current_schema_version == DB_VERSION:
+    return
  
-@app.before_request
-def before_request():
-  g.db = connect_db()
   schema = app.open_resource('schema.sql')
-  schema = schema.read() % dict(version=DB_VERSION)
+  schema = schema.read() % dict(version=DB_VERSION, computer=app.config['computer_id'])
   g.db.cursor().executescript(schema)
   g.db.commit()
 
+  if current_schema_version == '-1':
+    return
+
+  # Migrate all data to new tables
+  app.logger.info('Old schema detected: Migrating old tables...')
+  app.logger.info('Note that old tables are not alterated.')
+  for table in ('software', 'computer', 'partition', 'slave', 'partition_network'):
+    for row in execute_db(table, 'SELECT * from %s', db_version=current_schema_version):
+      columns = ', '.join(row.keys())
+      placeholders = ':'+', :'.join(row.keys())
+      query = 'INSERT INTO %s (%s) VALUES (%s)' % ('%s', columns, placeholders)
+      execute_db(table, query, row, log=True)
+
+  g.db.commit()
+
+is_schema_already_executed = False
+@app.before_request
+def before_request():
+  g.db = connect_db()
+  global is_schema_already_executed
+  if not is_schema_already_executed:
+    _upgradeDatabaseIfNeeded()
+    is_schema_already_executed = True
+
 
 @app.after_request
 def after_request(response):

slapos/tests/slapproxy.py

@@ -40,6 +40,9 @@ import slapos.proxy
 import slapos.proxy.views as views
 import slapos.slap.slap
 
+import sqlite3
+import pkg_resources
+
 
 class WrongFormat(Exception):
   pass
@@ -137,6 +140,7 @@ database_uri = %(tempdir)s/lib/proxy.db
     Remove files generated for test
     """
     shutil.rmtree(self._tempdir, True)
+    views.is_schema_already_executed = False
 
 
 class TestInformation(BasicMixin, unittest.TestCase):
@@ -841,3 +845,81 @@ class TestMultiNodeSupport(MasterMixin):
     except slapos.slap.NotFoundError:
       self.fail('Could not fetch informations for registered computer.')
 
+
+class TestMigrateVersion10To11(TestInformation, TestRequest, TestSlaveRequest, TestMultiNodeSupport):
+  """
+  Test that old database version are automatically migrated without failure
+  """
+  def setUp(self):
+    super(TestMigrateVersion10To11, self).setUp()
+    schema = pkg_resources.resource_stream('slapos.tests.slapproxy', 'database_dump_version_10.sql')
+    schema = schema.read() % dict(version='11')
+    self.db = sqlite3.connect(self.proxy_db)
+    self.db.cursor().executescript(schema)
+    self.db.commit()
+
+  def test_automatic_migration(self):
+    table_list = ('software11', 'computer11', 'partition11', 'slave11', 'partition_network11')
+    for table in table_list:
+      self.assertRaises(sqlite3.OperationalError, self.db.execute, "SELECT name FROM computer11")
+    # Run a dummy request to cause migration
+    self.app.get('/getComputerInformation?computer_id=computer')
+
+    # Check some partition parameters
+    self.assertEqual(
+        loads(self.app.get('/getComputerInformation?computer_id=computer').data)._computer_partition_list[0]._parameter_dict['slap_software_type'],
+        'production'
+    )
+    
+    # Lower level tests
+    computer_list = self.db.execute("SELECT * FROM computer11").fetchall()
+    self.assertEqual(
+        computer_list,
+        [(u'computer', u'127.0.0.1', u'255.255.255.255')]
+    )
+
+    software_list = self.db.execute("SELECT * FROM software11").fetchall()
+    self.assertEqual(
+        software_list,
+        [(u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u'computer')]
+    )
+
+    partition_list = self.db.execute("select * from partition11").fetchall()
+    self.assertEqual(
+        partition_list,
+        [(u'slappart0', u'computer', u'busy', u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n  <parameter id="json">{\n  "site-id": "erp5"\n  }\n}</parameter>\n</instance>\n', None, None, u'production', u'slapos', None, u'started'), (u'slappart1', u'computer', u'busy', u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u"<?xml version='1.0' encoding='utf-8'?>\n<instance/>\n", u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n  <parameter id="url">mysql://127.0.0.1:45678/erp5</parameter>\n</instance>\n', None, u'mariadb', u'MariaDB DataBase', u'slappart0', u'started'), (u'slappart2', u'computer', u'busy', u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n  <parameter id="cloudooo-json"></parameter>\n</instance>\n', u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n  <parameter id="url">cloudooo://127.0.0.1:23000/</parameter>\n</instance>\n', None, u'cloudooo', u'Cloudooo', u'slappart0', u'started'), (u'slappart3', u'computer', u'busy', u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u"<?xml version='1.0' encoding='utf-8'?>\n<instance/>\n", u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n  <parameter id="url">memcached://127.0.0.1:11000/</parameter>\n</instance>\n', None, u'memcached', u'Memcached', u'slappart0', u'started'), (u'slappart4', u'computer', u'busy', u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u"<?xml version='1.0' encoding='utf-8'?>\n<instance/>\n", u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n  <parameter id="url">memcached://127.0.0.1:13301/</parameter>\n</instance>\n', None, u'kumofs', u'KumoFS', u'slappart0', u'started'), (u'slappart5', u'computer', u'busy', u'/srv/slapgrid//srv//runner/project//slapos/software.cfg', u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n  <parameter id="kumofs-url">memcached://127.0.0.1:13301/</parameter>\n  <parameter id="memcached-url">memcached://127.0.0.1:11000/</parameter>\n  <parameter id="cloudooo-url">cloudooo://127.0.0.1:23000/</parameter>\n</instance>\n', u'<?xml version=\'1.0\' encoding=\'utf-8\'?>\n<instance>\n  <parameter id="url">https://[fc00::1]:10001</parameter>\n</instance>\n', None, u'tidstorage', u'TidStorage', u'slappart0', u'started'), (u'slappart6', u'computer', u'free', None, None, None, None, None, None, None, u'started'), (u'slappart7', u'computer', u'free', None, None, None, None, None, None, None, u'started'), (u'slappart8', u'computer', u'free', None, None, None, None, None, None, None, u'started'), (u'slappart9', u'computer', u'free', None, None, None, None, None, None, None, u'started')]
+    )
+
+    slave_list = self.db.execute("select * from slave11").fetchall()
+    self.assertEqual(
+        slave_list,
+        []
+    )
+
+    partition_network_list = self.db.execute("select * from partition_network11").fetchall()
+    self.assertEqual(
+        partition_network_list,
+        [(u'slappart0', u'computer', u'slappart0', u'127.0.0.1', u'255.255.255.255'), (u'slappart0', u'computer', u'slappart0', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart1', u'computer', u'slappart1', u'127.0.0.1', u'255.255.255.255'), (u'slappart1', u'computer', u'slappart1', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart2', u'computer', u'slappart2', u'127.0.0.1', u'255.255.255.255'), (u'slappart2', u'computer', u'slappart2', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart3', u'computer', u'slappart3', u'127.0.0.1', u'255.255.255.255'), (u'slappart3', u'computer', u'slappart3', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart4', u'computer', u'slappart4', u'127.0.0.1', u'255.255.255.255'), (u'slappart4', u'computer', u'slappart4', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart5', u'computer', u'slappart5', u'127.0.0.1', u'255.255.255.255'), (u'slappart5', u'computer', u'slappart5', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart6', u'computer', u'slappart6', u'127.0.0.1', u'255.255.255.255'), (u'slappart6', u'computer', u'slappart6', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart7', u'computer', u'slappart7', u'127.0.0.1', u'255.255.255.255'), (u'slappart7', u'computer', u'slappart7', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart8', u'computer', u'slappart8', u'127.0.0.1', u'255.255.255.255'), (u'slappart8', u'computer', u'slappart8', u'fc00::1', u'ffff:ffff:ffff::'), (u'slappart9', u'computer', u'slappart9', u'127.0.0.1', u'255.255.255.255'), (u'slappart9', u'computer', u'slappart9', u'fc00::1', u'ffff:ffff:ffff::')]
+    )
+
+  # Override several tests that needs an empty database
+  @unittest.skip("Not implemented")
+  def test_multi_node_support_different_software_release_list(self):
+    pass
+
+  @unittest.skip("Not implemented")
+  def test_multi_node_support_instance_default_computer(self):
+    pass
+
+  @unittest.skip("Not implemented")
+  def test_multi_node_support_instance_guid(self):
+    pass
+
+  @unittest.skip("Not implemented")
+  def test_partition_are_empty(self):
+    pass
+
+  @unittest.skip("Not implemented")
+  def test_request_consistent_parameters(self):
+    pass
+

slapos/tests/slapproxy/database_dump_version_10.sql

+-- Real world example of webrunner database running version 10 of proxy db.
+PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE software10 (url VARCHAR(255) UNIQUE);
+INSERT INTO "software10" VALUES('/srv/slapgrid//srv//runner/project//slapos/software.cfg');
+CREATE TABLE computer10 (
+  address VARCHAR(255),
+  netmask VARCHAR(255),
+  CONSTRAINT uniq PRIMARY KEY (address, netmask));
+INSERT INTO "computer10" VALUES('127.0.0.1','255.255.255.255');
+CREATE TABLE partition10 (
+  reference VARCHAR(255) UNIQUE,
+  slap_state VARCHAR(255) DEFAULT 'free',
+  software_release VARCHAR(255),
+  xml TEXT,
+  connection_xml TEXT,
+  slave_instance_list TEXT,
+  software_type VARCHAR(255),
+  partition_reference VARCHAR(255),
+  requested_by VARCHAR(255), -- only used for debugging,
+                             -- slapproxy does not support proper scope
+  requested_state VARCHAR(255) NOT NULL DEFAULT 'started'
+);
+INSERT INTO "partition10" VALUES('slappart0','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="json">{
+  "site-id": "erp5"
+  }
+}</parameter>
+</instance>
+',NULL,NULL,'production','slapos',NULL,'started');
+INSERT INTO "partition10" VALUES('slappart1','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance/>
+','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="url">mysql://127.0.0.1:45678/erp5</parameter>
+</instance>
+',NULL,'mariadb','MariaDB DataBase','slappart0','started');
+INSERT INTO "partition10" VALUES('slappart2','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="cloudooo-json"></parameter>
+</instance>
+','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="url">cloudooo://127.0.0.1:23000/</parameter>
+</instance>
+',NULL,'cloudooo','Cloudooo','slappart0','started');
+INSERT INTO "partition10" VALUES('slappart3','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance/>
+','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="url">memcached://127.0.0.1:11000/</parameter>
+</instance>
+',NULL,'memcached','Memcached','slappart0','started');
+INSERT INTO "partition10" VALUES('slappart4','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance/>
+','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="url">memcached://127.0.0.1:13301/</parameter>
+</instance>
+',NULL,'kumofs','KumoFS','slappart0','started');
+INSERT INTO "partition10" VALUES('slappart5','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="kumofs-url">memcached://127.0.0.1:13301/</parameter>
+  <parameter id="memcached-url">memcached://127.0.0.1:11000/</parameter>
+  <parameter id="cloudooo-url">cloudooo://127.0.0.1:23000/</parameter>
+</instance>
+','<?xml version=''1.0'' encoding=''utf-8''?>
+<instance>
+  <parameter id="url">https://[fc00::1]:10001</parameter>
+</instance>
+',NULL,'tidstorage','TidStorage','slappart0','started');
+INSERT INTO "partition10" VALUES('slappart6','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
+INSERT INTO "partition10" VALUES('slappart7','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
+INSERT INTO "partition10" VALUES('slappart8','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
+INSERT INTO "partition10" VALUES('slappart9','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
+CREATE TABLE slave10 (
+  reference VARCHAR(255) UNIQUE,
+  connection_xml TEXT,
+  hosted_by VARCHAR(255),
+  asked_by VARCHAR(255) -- only used for debugging,
+                        -- slapproxy does not support proper scope
+);
+CREATE TABLE partition_network10 (
+  partition_reference VARCHAR(255),
+  reference VARCHAR(255),
+  address VARCHAR(255),
+  netmask VARCHAR(255)
+);
+INSERT INTO "partition_network10" VALUES('slappart0','slappart0','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network10" VALUES('slappart0','slappart0','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network10" VALUES('slappart1','slappart1','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network10" VALUES('slappart1','slappart1','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network10" VALUES('slappart2','slappart2','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network10" VALUES('slappart2','slappart2','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network10" VALUES('slappart3','slappart3','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network10" VALUES('slappart3','slappart3','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network10" VALUES('slappart4','slappart4','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network10" VALUES('slappart4','slappart4','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network10" VALUES('slappart5','slappart5','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network10" VALUES('slappart5','slappart5','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network10" VALUES('slappart6','slappart6','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network10" VALUES('slappart6','slappart6','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network10" VALUES('slappart7','slappart7','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network10" VALUES('slappart7','slappart7','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network10" VALUES('slappart8','slappart8','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network10" VALUES('slappart8','slappart8','fc00::1','ffff:ffff:ffff::');
+INSERT INTO "partition_network10" VALUES('slappart9','slappart9','127.0.0.1','255.255.255.255');
+INSERT INTO "partition_network10" VALUES('slappart9','slappart9','fc00::1','ffff:ffff:ffff::');
+COMMIT;
+