mysql: new configuration parameter to choose engine (InnoDB or TokuDB)

neo.conf

@@ -48,6 +48,7 @@ partitions: 12
 #   - MySQL: [user[:password]@]database[unix_socket]
 #     Database must be created manually.
 #   - SQLite: path
+# engine: Optionnal parameter for MySQL. Can be InnoDB (default) or TokuDB.
 
 # Admin node
 [admin]

neo/lib/config.py

@@ -61,6 +61,9 @@ class ConfigurationManager(object):
     def getDatabase(self):
         return self.__get('database')
 
+    def getEngine(self):
+        return self.__get('engine', True)
+
     def getWait(self):
         # BUG
         return self.__get('wait')

neo/scripts/neostorage.py

@@ -36,6 +36,7 @@ parser.add_option('-c', '--cluster', help = 'the cluster name')
 parser.add_option('-m', '--masters', help = 'master node list')
 parser.add_option('-a', '--adapter', help = 'database adapter to use')
 parser.add_option('-d', '--database', help = 'database connections string')
+parser.add_option('-e', '--engine', help = 'database engine')
 parser.add_option('-D', '--dynamic-master-list', help='path of the file '
     'containing dynamic master node list')
 parser.add_option('-w', '--wait', help='seconds to wait for backend to be '
@@ -59,6 +60,7 @@ def main(args=None):
         cluster = options.cluster,
         masters = options.masters,
         database = options.database,
+        engine = options.engine,
         reset = options.reset,
         adapter = options.adapter,
         wait = options.wait,

neo/storage/app.py

@@ -50,7 +50,7 @@ class Application(object):
         self.nm = NodeManager(config.getDynamicMasterList())
         self.tm = TransactionManager(self)
         self.dm = buildDatabaseManager(config.getAdapter(),
-            (config.getDatabase(), config.getWait())
+            (config.getDatabase(), config.getEngine(), config.getWait()),
         )
 
         # load master nodes

neo/storage/database/importer.py

@@ -294,7 +294,7 @@ class ImporterDatabaseManager(DatabaseManager):
         self.zodb = ((x, dict(config.items(x))) for x in sections)
         self.compress = main.get('compress', 1)
         self.db = buildDatabaseManager(main['adapter'],
-            (main['database'], main['wait']))
+            (main['database'], main.get('engine'), main['wait']))
         for x in """query erase getConfiguration _setConfiguration
                     getPartitionTable changePartitionTable getUnfinishedTIDList
                     dropUnfinishedData storeTransaction finishTransaction

neo/storage/database/manager.py

@@ -42,10 +42,17 @@ class CreationUndone(Exception):
 class DatabaseManager(object):
     """This class only describes an interface for database managers."""
 
-    def __init__(self, database, wait=0):
+    ENGINES = ()
+
+    def __init__(self, database, engine=None, wait=0):
         """
             Initialize the object.
         """
+        if engine:
+            if engine not in self.ENGINES:
+                raise ValueError("Unsupported engine: %r not in %r"
+                                 % (engine, self.ENGINES))
+            self._engine = engine
         self._wait = wait
         self._parse(database)
 

neo/storage/database/mysqldb.py

@@ -44,8 +44,8 @@ def splitOIDField(tid, oids):
 class MySQLDatabaseManager(DatabaseManager):
     """This class manages a database on MySQL."""
 
-    # WARNING: some parts are not concurrent safe (ex: holdData)
-    # (there must be only 1 writable connection per DB)
+    ENGINES = "InnoDB", "TokuDB"
+    _engine = ENGINES[0] # default engine
 
     # Disabled even on MySQL 5.1-5.5 and MariaDB 5.2-5.3 because
     # 'select count(*) from obj' sometimes returns incorrect values
@@ -142,12 +142,13 @@ class MySQLDatabaseManager(DatabaseManager):
     def _setup(self):
         self._config.clear()
         q = self.query
+        p = engine = self._engine
         # The table "config" stores configuration parameters which affect the
         # persistent data.
         q("""CREATE TABLE IF NOT EXISTS config (
                  name VARBINARY(255) NOT NULL PRIMARY KEY,
                  value VARBINARY(255) NULL
-             ) ENGINE = InnoDB""")
+             ) ENGINE=""" + engine)
 
         # The table "pt" stores a partition table.
         q("""CREATE TABLE IF NOT EXISTS pt (
@@ -155,10 +156,11 @@ class MySQLDatabaseManager(DatabaseManager):
                  nid INT NOT NULL,
                  state TINYINT UNSIGNED NOT NULL,
                  PRIMARY KEY (rid, nid)
-             ) ENGINE = InnoDB""")
+             ) ENGINE=""" + engine)
 
-        p = self._use_partition and """ PARTITION BY LIST (`partition`) (
-            PARTITION dummy VALUES IN (NULL))""" or ''
+        if self._use_partition:
+            p += """ PARTITION BY LIST (`partition`) (
+                PARTITION dummy VALUES IN (NULL))"""
 
         # The table "trans" stores information on committed transactions.
         q("""CREATE TABLE IF NOT EXISTS trans (
@@ -171,7 +173,7 @@ class MySQLDatabaseManager(DatabaseManager):
                  ext BLOB NOT NULL,
                  ttid BIGINT UNSIGNED NOT NULL,
                  PRIMARY KEY (`partition`, tid)
-             ) ENGINE = InnoDB""" + p)
+             ) ENGINE=""" + p)
 
         # The table "obj" stores committed object metadata.
         q("""CREATE TABLE IF NOT EXISTS obj (
@@ -183,7 +185,10 @@ class MySQLDatabaseManager(DatabaseManager):
                  PRIMARY KEY (`partition`, tid, oid),
                  KEY (`partition`, oid, tid),
                  KEY (data_id)
-             ) ENGINE = InnoDB""" + p)
+             ) ENGINE=""" + p)
+
+        if engine == "TokuDB":
+            engine += " compression='tokudb_uncompressed'"
 
         # The table "data" stores object data.
         # We'd like to have partial index on 'hash' colum (e.g. hash(4))
@@ -193,7 +198,7 @@ class MySQLDatabaseManager(DatabaseManager):
                  hash BINARY(20) NOT NULL UNIQUE,
                  compression TINYINT UNSIGNED NULL,
                  value LONGBLOB NULL
-             ) ENGINE = InnoDB""")
+             ) ENGINE=""" + engine)
 
         # The table "ttrans" stores information on uncommitted transactions.
         q("""CREATE TABLE IF NOT EXISTS ttrans (
@@ -205,7 +210,7 @@ class MySQLDatabaseManager(DatabaseManager):
                  description BLOB NOT NULL,
                  ext BLOB NOT NULL,
                  ttid BIGINT UNSIGNED NOT NULL
-             ) ENGINE = InnoDB""")
+             ) ENGINE=""" + engine)
 
         # The table "tobj" stores uncommitted object metadata.
         q("""CREATE TABLE IF NOT EXISTS tobj (
@@ -215,7 +220,7 @@ class MySQLDatabaseManager(DatabaseManager):
                  data_id BIGINT UNSIGNED NULL,
                  value_tid BIGINT UNSIGNED NULL,
                  PRIMARY KEY (tid, oid)
-             ) ENGINE = InnoDB""")
+             ) ENGINE=""" + engine)
 
         self._uncommitted_data.update(q("SELECT data_id, count(*)"
             " FROM tobj WHERE data_id IS NOT NULL GROUP BY data_id"))

neo/tests/storage/testStorageMySQL.py

@@ -24,24 +24,19 @@ from neo.storage.database.mysqldb import MySQLDatabaseManager
 NEO_SQL_DATABASE = 'test_mysqldb0'
 NEO_SQL_USER = 'test'
 
-class StorageMySQSLdbTests(StorageDBTests):
+class StorageMySQLdbTests(StorageDBTests):
+
+    engine = None
 
     def getDB(self, reset=0):
         self.prepareDatabase(number=1, prefix=NEO_SQL_DATABASE[:-1])
         # db manager
         database = '%s@%s' % (NEO_SQL_USER, NEO_SQL_DATABASE)
-        db = MySQLDatabaseManager(database, 0)
-        db.setup(reset)
-        return db
-
-    def test_MySQLDatabaseManagerInit(self):
-        db = MySQLDatabaseManager('%s@%s' % (NEO_SQL_USER, NEO_SQL_DATABASE),
-            0)
-        # init
+        db = MySQLDatabaseManager(database, self.engine)
         self.assertEqual(db.db, NEO_SQL_DATABASE)
         self.assertEqual(db.user, NEO_SQL_USER)
-        # & connect
-        self.assertTrue(isinstance(db.conn, MySQLdb.connection))
+        db.setup(reset)
+        return db
 
     def test_query1(self):
         # fake result object
@@ -96,6 +91,10 @@ class StorageMySQSLdbTests(StorageDBTests):
         self.assertEqual(self.db.escape('a"b'), 'a\\"b')
         self.assertEqual(self.db.escape("a'b"), "a\\'b")
 
+class StorageMySQLdbTokuDBTests(StorageMySQLdbTests):
+
+    engine = "TokuDB"
+
 del StorageDBTests
 
 if __name__ == "__main__":

neo/tests/storage/testStorageSQLite.py

@@ -21,7 +21,7 @@ from neo.storage.database.sqlite import SQLiteDatabaseManager
 class StorageSQLiteTests(StorageDBTests):
 
     def getDB(self, reset=0):
-        db = SQLiteDatabaseManager(':memory:', 0)
+        db = SQLiteDatabaseManager(':memory:')
         db.setup(reset)
         return db