Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
N
neoppod
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
1
Issues
1
List
Boards
Labels
Milestones
Merge Requests
2
Merge Requests
2
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
nexedi
neoppod
Commits
dd1d6b30
Commit
dd1d6b30
authored
8 years ago
by
Julien Muchembled
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
storage: new --prune-orphan option
parent
ccbf7bce
Changes
6
Show whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
40 additions
and
2 deletions
+40
-2
neo/lib/config.py
neo/lib/config.py
+4
-0
neo/scripts/neostorage.py
neo/scripts/neostorage.py
+5
-1
neo/storage/database/importer.py
neo/storage/database/importer.py
+1
-1
neo/storage/database/manager.py
neo/storage/database/manager.py
+16
-0
neo/storage/database/mysqldb.py
neo/storage/database/mysqldb.py
+7
-0
neo/storage/database/sqlite.py
neo/storage/database/sqlite.py
+7
-0
No files found.
neo/lib/config.py
View file @
dd1d6b30
...
...
@@ -130,6 +130,10 @@ class ConfigurationManager(object):
# only from command line
return
self
.
argument_list
.
get
(
'reset'
,
False
)
def
getPruneOrphan
(
self
):
# only from command line
return
self
.
argument_list
.
get
(
'prune_orphan'
,
False
)
def
getUUID
(
self
):
# only from command line
uuid
=
self
.
argument_list
.
get
(
'uuid'
,
None
)
...
...
This diff is collapsed.
Click to expand it.
neo/scripts/neostorage.py
View file @
dd1d6b30
...
...
@@ -30,6 +30,8 @@ parser.add_option('-d', '--database', help = 'database connections string')
parser
.
add_option
(
'-e'
,
'--engine'
,
help
=
'database engine'
)
parser
.
add_option
(
'-w'
,
'--wait'
,
help
=
'seconds to wait for backend to be '
'available, before erroring-out (-1 = infinite)'
,
type
=
'float'
,
default
=
0
)
parser
.
add_option
(
'--prune-orphan'
,
action
=
'store_true'
,
help
=
'fix database'
' by deleting unreferenced raw data, and exit (this can take a long time)'
)
parser
.
add_option
(
'--reset'
,
action
=
'store_true'
,
help
=
'remove an existing database if any, and exit'
)
...
...
@@ -53,5 +55,7 @@ def main(args=None):
# and then, load and run the application
from
neo.storage.app
import
Application
app
=
Application
(
config
)
if
not
config
.
getReset
():
if
config
.
getPruneOrphan
():
print
app
.
dm
.
pruneOrphan
(),
'deleted record(s)'
elif
not
config
.
getReset
():
app
.
run
()
This diff is collapsed.
Click to expand it.
neo/storage/database/importer.py
View file @
dd1d6b30
...
...
@@ -304,7 +304,7 @@ class ImporterDatabaseManager(DatabaseManager):
getPartitionTable changePartitionTable
getUnfinishedTIDDict dropUnfinishedData abortTransaction
storeTransaction lockTransaction unlockTransaction
storeData _pruneData deferCommit
storeData
getOrphanList
_pruneData deferCommit
"""
.
split
():
setattr
(
self
,
x
,
getattr
(
self
.
db
,
x
))
...
...
This diff is collapsed.
Click to expand it.
neo/storage/database/manager.py
View file @
dd1d6b30
...
...
@@ -409,6 +409,20 @@ class DatabaseManager(object):
is always the case at tpc_vote.
"""
@
abstract
def
getOrphanList
(
self
):
"""Return the list of data id that is not referenced by the obj table
This is a repair method, and it's usually expensive.
There was a bug that did not free data of transactions that were
aborted before vote. This method is used to reclaim the wasted space.
"""
def
pruneOrphan
(
self
):
n
=
self
.
_pruneData
(
self
.
getOrphanList
())
self
.
commit
()
return
n
@
abstract
def
_pruneData
(
self
,
data_id_list
):
"""To be overridden by the backend to delete any unreferenced data
...
...
@@ -417,6 +431,8 @@ class DatabaseManager(object):
- not in self._uncommitted_data
- and not referenced by a fully-committed object (storage should have
an index or a refcount of all data ids of all objects)
The returned value is the number of deleted rows from the data table.
"""
@
abstract
...
...
This diff is collapsed.
Click to expand it.
neo/storage/database/mysqldb.py
View file @
dd1d6b30
...
...
@@ -471,6 +471,11 @@ class MySQLDatabaseManager(DatabaseManager):
_structLL
=
struct
.
Struct
(
">LL"
)
_unpackLL
=
_structLL
.
unpack
def
getOrphanList
(
self
):
return
[
x
for
x
,
in
self
.
query
(
"SELECT id FROM data LEFT JOIN obj ON (id=data_id)"
" WHERE data_id IS NULL"
)]
def
_pruneData
(
self
,
data_id_list
):
data_id_list
=
set
(
data_id_list
).
difference
(
self
.
_uncommitted_data
)
if
data_id_list
:
...
...
@@ -491,6 +496,8 @@ class MySQLDatabaseManager(DatabaseManager):
if
bigid_list
:
q
(
"DELETE FROM bigdata WHERE id IN (%s)"
%
","
.
join
(
map
(
str
,
bigid_list
)))
return
len
(
id_list
)
return
0
def
_bigData
(
self
,
value
):
bigdata_id
,
length
=
self
.
_unpackLL
(
value
)
...
...
This diff is collapsed.
Click to expand it.
neo/storage/database/sqlite.py
View file @
dd1d6b30
...
...
@@ -376,6 +376,11 @@ class SQLiteDatabaseManager(DatabaseManager):
packed
,
buffer
(
''
.
join
(
oid_list
)),
buffer
(
user
),
buffer
(
desc
),
buffer
(
ext
),
u64
(
ttid
)))
def
getOrphanList
(
self
):
return
[
x
for
x
,
in
self
.
query
(
"SELECT id FROM data LEFT JOIN obj ON (id=data_id)"
" WHERE data_id IS NULL"
)]
def
_pruneData
(
self
,
data_id_list
):
data_id_list
=
set
(
data_id_list
).
difference
(
self
.
_uncommitted_data
)
if
data_id_list
:
...
...
@@ -385,6 +390,8 @@ class SQLiteDatabaseManager(DatabaseManager):
%
","
.
join
(
map
(
str
,
data_id_list
))))
q
(
"DELETE FROM data WHERE id IN (%s)"
%
","
.
join
(
map
(
str
,
data_id_list
)))
return
len
(
data_id_list
)
return
0
def
storeData
(
self
,
checksum
,
data
,
compression
,
_dup
=
unique_constraint_message
(
"data"
,
"hash"
,
"compression"
)):
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment