Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
N
neoppod
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Vincent Pelletier
neoppod
Commits
a8b9ec0c
Commit
a8b9ec0c
authored
Apr 22, 2019
by
Julien Muchembled
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
mysql: split queries to avoid exceeding max_allowed_packet when pruning data
parent
e02dffd2
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
25 additions
and
12 deletions
+25
-12
neo/storage/database/mysqldb.py
neo/storage/database/mysqldb.py
+25
-12
No files found.
neo/storage/database/mysqldb.py
View file @
a8b9ec0c
...
@@ -85,6 +85,10 @@ def auto_reconnect(wrapped):
...
@@ -85,6 +85,10 @@ def auto_reconnect(wrapped):
retry
-=
1
retry
-=
1
return
wraps
(
wrapped
)(
wrapper
)
return
wraps
(
wrapped
)(
wrapper
)
def
splitList
(
x
,
n
):
for
i
in
xrange
(
0
,
len
(
x
),
n
):
yield
x
[
i
:
i
+
n
]
@
implements
@
implements
class
MySQLDatabaseManager
(
DatabaseManager
):
class
MySQLDatabaseManager
(
DatabaseManager
):
...
@@ -633,20 +637,29 @@ class MySQLDatabaseManager(DatabaseManager):
...
@@ -633,20 +637,29 @@ class MySQLDatabaseManager(DatabaseManager):
q
=
self
.
query
q
=
self
.
query
id_list
=
[]
id_list
=
[]
bigid_list
=
[]
bigid_list
=
[]
for
id
,
value
in
q
(
"SELECT id, IF(compression < 128, NULL, value)"
# Split the query to avoid exceeding max_allowed_packet.
" FROM data LEFT JOIN obj ON (id = data_id)"
# Each id is 20 chars maximum.
" WHERE id IN (%s) AND data_id IS NULL"
for
data_id_list
in
splitList
(
sorted
(
data_id_list
),
1000000
):
%
","
.
join
(
map
(
str
,
data_id_list
))):
for
id
,
value
in
q
(
id_list
.
append
(
str
(
id
))
"SELECT id, IF(compression < 128, NULL, value)"
if
value
:
" FROM data LEFT JOIN obj ON (id = data_id)"
bigdata_id
,
length
=
self
.
_unpackLL
(
value
)
" WHERE id IN (%s) AND data_id IS NULL"
bigid_list
+=
xrange
(
bigdata_id
,
%
","
.
join
(
map
(
str
,
data_id_list
))):
bigdata_id
+
(
length
+
0x7fffff
>>
23
))
id_list
.
append
(
id
)
if
value
:
bigdata_id
,
length
=
self
.
_unpackLL
(
value
)
bigid_list
+=
xrange
(
bigdata_id
,
bigdata_id
+
(
length
+
0x7fffff
>>
23
))
if
id_list
:
if
id_list
:
q
(
"DELETE FROM data WHERE id IN (%s)"
%
","
.
join
(
id_list
))
def
delete
(
table
,
id_list
):
for
id_list
in
splitList
(
id_list
,
1000000
):
q
(
"DELETE FROM %s WHERE id IN (%s)"
%
(
table
,
","
.
join
(
map
(
str
,
id_list
))))
delete
(
'data'
,
id_list
)
if
bigid_list
:
if
bigid_list
:
q
(
"DELETE FROM bigdata WHERE id IN (%s)"
bigid_list
.
sort
()
%
","
.
join
(
map
(
str
,
bigid_list
))
)
delete
(
'bigdata'
,
bigid_list
)
return
len
(
id_list
)
return
len
(
id_list
)
return
0
return
0
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment