Commit 8535b9cc authored by Julien Muchembled's avatar Julien Muchembled

mysql: drop support for horizontal partitioning of trans/obj

It has never been enabled and the code to drop partitions will be
changed in a way that only 'trans' may still benefit of partitioning.
We'll see in the future if we have cases where 'trans' is too big to
delete all rows (of a given partition) in a single query.
parent 0e43dd1f
...@@ -18,6 +18,7 @@ import os, re, string, struct, sys, time ...@@ -18,6 +18,7 @@ import os, re, string, struct, sys, time
from binascii import a2b_hex from binascii import a2b_hex
from collections import OrderedDict from collections import OrderedDict
from functools import wraps from functools import wraps
from hashlib import sha1
from . import useMySQLdb from . import useMySQLdb
if useMySQLdb(): if useMySQLdb():
binding_name = 'MySQLdb' binding_name = 'MySQLdb'
...@@ -48,12 +49,6 @@ else: ...@@ -48,12 +49,6 @@ else:
# for tests # for tests
from pymysql import NotSupportedError from pymysql import NotSupportedError
from pymysql.constants.ER import BAD_DB_ERROR, UNKNOWN_STORAGE_ENGINE from pymysql.constants.ER import BAD_DB_ERROR, UNKNOWN_STORAGE_ENGINE
# BBB: the following 2 constants were added to mysqlclient 1.3.8
DROP_LAST_PARTITION = 1508
SAME_NAME_PARTITION = 1517
from array import array
from hashlib import sha1
from . import LOG_QUERIES, DatabaseFailure from . import LOG_QUERIES, DatabaseFailure
from .manager import DatabaseManager, splitOIDField from .manager import DatabaseManager, splitOIDField
from neo.lib import logging, util from neo.lib import logging, util
...@@ -118,8 +113,6 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -118,8 +113,6 @@ class MySQLDatabaseManager(DatabaseManager):
ENGINES = "InnoDB", "RocksDB" ENGINES = "InnoDB", "RocksDB"
_engine = ENGINES[0] # default engine _engine = ENGINES[0] # default engine
_use_partition = False
_max_allowed_packet = 32769 * 1024 _max_allowed_packet = 32769 * 1024
def _parse(self, database): def _parse(self, database):
...@@ -324,10 +317,6 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -324,10 +317,6 @@ class MySQLDatabaseManager(DatabaseManager):
PRIMARY KEY (`partition`, nid) PRIMARY KEY (`partition`, nid)
) ENGINE=""" + engine ) ENGINE=""" + engine
if self._use_partition:
p += """ PARTITION BY LIST (`partition`) (
PARTITION dummy VALUES IN (NULL))"""
if engine == "RocksDB": if engine == "RocksDB":
cf = lambda name, rev=False: " COMMENT '%scf_neo_%s'" % ( cf = lambda name, rev=False: " COMMENT '%scf_neo_%s'" % (
'rev:' if rev else '', name) 'rev:' if rev else '', name)
...@@ -529,7 +518,6 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -529,7 +518,6 @@ class MySQLDatabaseManager(DatabaseManager):
compression, checksum, data, value_serial) compression, checksum, data, value_serial)
def _changePartitionTable(self, cell_list, reset=False): def _changePartitionTable(self, cell_list, reset=False):
offset_list = []
q = self.query q = self.query
if reset: if reset:
q("DELETE FROM pt") q("DELETE FROM pt")
...@@ -540,20 +528,9 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -540,20 +528,9 @@ class MySQLDatabaseManager(DatabaseManager):
q("DELETE FROM pt WHERE `partition` = %d AND nid = %d" q("DELETE FROM pt WHERE `partition` = %d AND nid = %d"
% (offset, nid)) % (offset, nid))
else: else:
offset_list.append(offset)
q("INSERT INTO pt VALUES (%d, %d, %d)" q("INSERT INTO pt VALUES (%d, %d, %d)"
" ON DUPLICATE KEY UPDATE tid = %d" " ON DUPLICATE KEY UPDATE tid = %d"
% (offset, nid, tid, tid)) % (offset, nid, tid, tid))
if self._use_partition:
for offset in offset_list:
add = """ALTER TABLE %%s ADD PARTITION (
PARTITION p%u VALUES IN (%u))""" % (offset, offset)
for table in 'trans', 'obj':
try:
self.query(add % table)
except MysqlError as e:
if e.code != SAME_NAME_PARTITION:
raise
def dropPartitions(self, offset_list): def dropPartitions(self, offset_list):
q = self.query q = self.query
...@@ -565,19 +542,9 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -565,19 +542,9 @@ class MySQLDatabaseManager(DatabaseManager):
data_id_list = [x for x, in data_id_list = [x for x, in
q("SELECT DISTINCT data_id FROM obj FORCE INDEX(tid)" q("SELECT DISTINCT data_id FROM obj FORCE INDEX(tid)"
"%s AND data_id IS NOT NULL" % where)] "%s AND data_id IS NOT NULL" % where)]
if not self._use_partition: q("DELETE FROM obj" + where)
q("DELETE FROM obj" + where) q("DELETE FROM trans" + where)
q("DELETE FROM trans" + where)
self._pruneData(data_id_list) self._pruneData(data_id_list)
if self._use_partition:
drop = "ALTER TABLE %s DROP PARTITION" + \
','.join(' p%u' % i for i in offset_list)
for table in 'trans', 'obj':
try:
self.query(drop % table)
except MysqlError as e:
if e.code != DROP_LAST_PARTITION:
raise
def _getUnfinishedDataIdList(self): def _getUnfinishedDataIdList(self):
return [x for x, in self.query( return [x for x, in self.query(
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment