Commit a8b9ec0c authored by Julien Muchembled's avatar Julien Muchembled

mysql: split queries to avoid exceeding max_allowed_packet when pruning data

parent e02dffd2
...@@ -85,6 +85,10 @@ def auto_reconnect(wrapped): ...@@ -85,6 +85,10 @@ def auto_reconnect(wrapped):
retry -= 1 retry -= 1
return wraps(wrapped)(wrapper) return wraps(wrapped)(wrapper)
def splitList(x, n):
for i in xrange(0, len(x), n):
yield x[i:i+n]
@implements @implements
class MySQLDatabaseManager(DatabaseManager): class MySQLDatabaseManager(DatabaseManager):
...@@ -633,20 +637,29 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -633,20 +637,29 @@ class MySQLDatabaseManager(DatabaseManager):
q = self.query q = self.query
id_list = [] id_list = []
bigid_list = [] bigid_list = []
for id, value in q("SELECT id, IF(compression < 128, NULL, value)" # Split the query to avoid exceeding max_allowed_packet.
" FROM data LEFT JOIN obj ON (id = data_id)" # Each id is 20 chars maximum.
" WHERE id IN (%s) AND data_id IS NULL" for data_id_list in splitList(sorted(data_id_list), 1000000):
% ",".join(map(str, data_id_list))): for id, value in q(
id_list.append(str(id)) "SELECT id, IF(compression < 128, NULL, value)"
if value: " FROM data LEFT JOIN obj ON (id = data_id)"
bigdata_id, length = self._unpackLL(value) " WHERE id IN (%s) AND data_id IS NULL"
bigid_list += xrange(bigdata_id, % ",".join(map(str, data_id_list))):
bigdata_id + (length + 0x7fffff >> 23)) id_list.append(id)
if value:
bigdata_id, length = self._unpackLL(value)
bigid_list += xrange(
bigdata_id,
bigdata_id + (length + 0x7fffff >> 23))
if id_list: if id_list:
q("DELETE FROM data WHERE id IN (%s)" % ",".join(id_list)) def delete(table, id_list):
for id_list in splitList(id_list, 1000000):
q("DELETE FROM %s WHERE id IN (%s)"
% (table, ",".join(map(str, id_list))))
delete('data', id_list)
if bigid_list: if bigid_list:
q("DELETE FROM bigdata WHERE id IN (%s)" bigid_list.sort()
% ",".join(map(str, bigid_list))) delete('bigdata', bigid_list)
return len(id_list) return len(id_list)
return 0 return 0
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment