Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
N
neoppod
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Carlos Ramos Carreño
neoppod
Commits
1eed0239
Commit
1eed0239
authored
Mar 09, 2017
by
Julien Muchembled
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Code clean up: PartitionTable
parent
b61ee7f1
Changes
6
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
95 additions
and
118 deletions
+95
-118
neo/lib/pt.py
neo/lib/pt.py
+18
-41
neo/master/pt.py
neo/master/pt.py
+6
-4
neo/storage/app.py
neo/storage/app.py
+7
-9
neo/tests/master/testMasterPT.py
neo/tests/master/testMasterPT.py
+26
-26
neo/tests/storage/testStorageApp.py
neo/tests/storage/testStorageApp.py
+4
-4
neo/tests/testPT.py
neo/tests/testPT.py
+34
-34
No files found.
neo/lib/pt.py
View file @
1eed0239
...
...
@@ -15,12 +15,10 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import
math
from
functools
import
wraps
from
.
import
logging
,
protocol
from
.locking
import
Lock
from
.protocol
import
uuid_str
,
CellStates
from
.util
import
u64
from
.locking
import
RLock
class
PartitionTableException
(
Exception
):
"""
...
...
@@ -163,7 +161,7 @@ class PartitionTable(object):
if
cell
.
getUUID
()
==
uuid
:
return
cell
def
setCell
(
self
,
offset
,
node
,
state
):
def
_
setCell
(
self
,
offset
,
node
,
state
):
if
state
==
CellStates
.
DISCARDED
:
return
self
.
removeCell
(
offset
,
node
)
if
node
.
isBroken
()
or
node
.
isDown
():
...
...
@@ -182,7 +180,6 @@ class PartitionTable(object):
row
.
append
(
Cell
(
node
,
state
))
if
state
!=
CellStates
.
FEEDING
:
self
.
count_dict
[
node
]
+=
1
return
offset
,
node
.
getUUID
(),
state
def
removeCell
(
self
,
offset
,
node
):
row
=
self
.
partition_list
[
offset
]
...
...
@@ -193,7 +190,6 @@ class PartitionTable(object):
if
not
cell
.
isFeeding
():
self
.
count_dict
[
node
]
-=
1
break
return
(
offset
,
node
.
getUUID
(),
CellStates
.
DISCARDED
)
def
load
(
self
,
ptid
,
row_list
,
nm
):
"""
...
...
@@ -209,24 +205,21 @@ class PartitionTable(object):
node
=
nm
.
getByUUID
(
uuid
)
# the node must be known by the node manager
assert
node
is
not
None
self
.
setCell
(
offset
,
node
,
state
)
self
.
_
setCell
(
offset
,
node
,
state
)
logging
.
debug
(
'partition table loaded (ptid=%s)'
,
ptid
)
self
.
log
()
def
update
(
self
,
ptid
,
cell_list
,
nm
):
"""
Update the partition with the cell list supplied. Ignore those changes
if the partition table ID is not greater than the current one. If a node
Update the partition with the cell list supplied. If a node
is not known, it is created in the node manager and set as unavailable
"""
if
ptid
<=
self
.
_id
:
logging
.
warning
(
'ignoring older partition changes'
)
return
assert
self
.
_id
<
ptid
,
(
self
.
_id
,
ptid
)
self
.
_id
=
ptid
for
offset
,
uuid
,
state
in
cell_list
:
node
=
nm
.
getByUUID
(
uuid
)
assert
node
is
not
None
,
'No node found for uuid '
+
uuid_str
(
uuid
)
self
.
setCell
(
offset
,
node
,
state
)
self
.
_
setCell
(
offset
,
node
,
state
)
logging
.
debug
(
'partition table updated (ptid=%s)'
,
ptid
)
self
.
log
()
...
...
@@ -310,38 +303,22 @@ class PartitionTable(object):
getRow
=
self
.
getRow
return
[(
x
,
getRow
(
x
))
for
x
in
xrange
(
self
.
np
)]
def
thread_safe
(
method
):
def
wrapper
(
self
,
*
args
,
**
kwargs
):
self
.
lock
()
try
:
return
method
(
self
,
*
args
,
**
kwargs
)
finally
:
self
.
unlock
()
return
wraps
(
method
)(
wrapper
)
class
MTPartitionTable
(
PartitionTable
):
""" Thread-safe aware version of the partition table, override only methods
used in the client """
def
__init__
(
self
,
*
args
,
**
kwargs
):
self
.
_lock
=
RLock
()
PartitionTable
.
__init__
(
self
,
*
args
,
**
kwargs
)
def
lock
(
self
):
self
.
_lock
.
acquire
()
def
unlock
(
self
):
self
.
_lock
.
release
()
def
__init__
(
self
,
*
args
,
**
kw
):
self
.
_lock
=
Lock
()
PartitionTable
.
__init__
(
self
,
*
args
,
**
kw
)
@
thread_safe
def
setCell
(
self
,
*
args
,
**
kwargs
)
:
return
PartitionTable
.
setCell
(
self
,
*
args
,
**
kwargs
)
def
update
(
self
,
*
args
,
**
kw
):
with
self
.
_lock
:
return
PartitionTable
.
update
(
self
,
*
args
,
**
kw
)
@
thread_safe
def
clear
(
self
,
*
args
,
**
kwargs
)
:
return
PartitionTable
.
clear
(
self
,
*
args
,
**
kwargs
)
def
clear
(
self
,
*
args
,
**
kw
):
with
self
.
_lock
:
return
PartitionTable
.
clear
(
self
,
*
args
,
**
kw
)
@
thread_safe
def
operational
(
self
,
*
args
,
**
kwargs
)
:
return
PartitionTable
.
operational
(
self
,
*
args
,
**
kwargs
)
def
operational
(
self
,
*
args
,
**
kw
):
with
self
.
_lock
:
return
PartitionTable
.
operational
(
self
,
*
args
,
**
kw
)
neo/master/pt.py
View file @
1eed0239
...
...
@@ -147,7 +147,7 @@ class PartitionTable(neo.lib.pt.PartitionTable):
if
node
is
None
:
node
=
nm
.
createStorage
(
uuid
=
uuid
)
new_nodes
.
append
(
node
.
asTuple
())
self
.
setCell
(
offset
,
node
,
state
)
self
.
_
setCell
(
offset
,
node
,
state
)
return
new_nodes
def
setUpToDate
(
self
,
node
,
offset
):
...
...
@@ -163,13 +163,15 @@ class PartitionTable(neo.lib.pt.PartitionTable):
raise
neo
.
lib
.
pt
.
PartitionTableException
(
'Non-assigned partition'
)
# update the partition table
cell_list
=
[
self
.
setCell
(
offset
,
node
,
CellStates
.
UP_TO_DATE
)]
self
.
_setCell
(
offset
,
node
,
CellStates
.
UP_TO_DATE
)
cell_list
=
[(
offset
,
uuid
,
CellStates
.
UP_TO_DATE
)]
# If the partition contains a feeding cell, drop it now.
for
feeding_cell
in
self
.
getCellList
(
offset
):
if
feeding_cell
.
isFeeding
():
cell_list
.
append
(
self
.
removeCell
(
offset
,
feeding_cell
.
getNode
()))
node
=
feeding_cell
.
getNode
()
self
.
removeCell
(
offset
,
node
)
cell_list
.
append
((
offset
,
node
.
getUUID
(),
CellStates
.
DISCARDED
))
break
return
cell_list
...
...
neo/storage/app.py
View file @
1eed0239
...
...
@@ -133,19 +133,17 @@ class Application(BaseApplication):
def
loadPartitionTable
(
self
):
"""Load a partition table from the database."""
self
.
pt
.
clear
()
ptid
=
self
.
dm
.
getPTID
()
cell_list
=
self
.
dm
.
getPartitionTable
()
new_cell_list
=
[]
for
offset
,
uuid
,
state
in
cell_list
:
# convert from int to Enum
state
=
CellStates
[
state
]
if
ptid
is
None
:
return
cell_list
=
[]
for
offset
,
uuid
,
state
in
self
.
dm
.
getPartitionTable
():
# register unknown nodes
if
self
.
nm
.
getByUUID
(
uuid
)
is
None
:
self
.
nm
.
createStorage
(
uuid
=
uuid
)
new_cell_list
.
append
((
offset
,
uuid
,
state
))
# load the partition table in manager
self
.
pt
.
clear
()
self
.
pt
.
update
(
ptid
,
new_cell_list
,
self
.
nm
)
cell_list
.
append
((
offset
,
uuid
,
CellStates
[
state
]))
self
.
pt
.
update
(
ptid
,
cell_list
,
self
.
nm
)
def
run
(
self
):
try
:
...
...
neo/tests/master/testMasterPT.py
View file @
1eed0239
...
...
@@ -70,15 +70,15 @@ class MasterPartitionTableTests(NeoUnitTestBase):
num_partitions
=
5
num_replicas
=
3
pt
=
PartitionTable
(
num_partitions
,
num_replicas
)
pt
.
setCell
(
0
,
sn1
,
CellStates
.
OUT_OF_DATE
)
pt
.
_
setCell
(
0
,
sn1
,
CellStates
.
OUT_OF_DATE
)
sn1
.
setState
(
NodeStates
.
RUNNING
)
pt
.
setCell
(
1
,
sn2
,
CellStates
.
UP_TO_DATE
)
pt
.
_
setCell
(
1
,
sn2
,
CellStates
.
UP_TO_DATE
)
sn2
.
setState
(
NodeStates
.
TEMPORARILY_DOWN
)
pt
.
setCell
(
2
,
sn3
,
CellStates
.
UP_TO_DATE
)
pt
.
_
setCell
(
2
,
sn3
,
CellStates
.
UP_TO_DATE
)
sn3
.
setState
(
NodeStates
.
DOWN
)
pt
.
setCell
(
3
,
sn4
,
CellStates
.
UP_TO_DATE
)
pt
.
_
setCell
(
3
,
sn4
,
CellStates
.
UP_TO_DATE
)
sn4
.
setState
(
NodeStates
.
BROKEN
)
pt
.
setCell
(
4
,
sn5
,
CellStates
.
UP_TO_DATE
)
pt
.
_
setCell
(
4
,
sn5
,
CellStates
.
UP_TO_DATE
)
sn5
.
setState
(
NodeStates
.
RUNNING
)
# outdate nodes
cells_outdated
=
pt
.
outdate
()
...
...
@@ -118,12 +118,12 @@ class MasterPartitionTableTests(NeoUnitTestBase):
sn
=
[
self
.
createStorage
(
None
,
i
+
1
,
NodeStates
.
RUNNING
)
for
i
in
xrange
(
3
)]
pt
=
PartitionTable
(
3
,
0
)
pt
.
setCell
(
0
,
sn
[
0
],
CellStates
.
OUT_OF_DATE
)
pt
.
setCell
(
1
,
sn
[
1
],
CellStates
.
FEEDING
)
pt
.
setCell
(
1
,
sn
[
2
],
CellStates
.
OUT_OF_DATE
)
pt
.
setCell
(
2
,
sn
[
0
],
CellStates
.
OUT_OF_DATE
)
pt
.
setCell
(
2
,
sn
[
1
],
CellStates
.
FEEDING
)
pt
.
setCell
(
2
,
sn
[
2
],
CellStates
.
UP_TO_DATE
)
pt
.
_
setCell
(
0
,
sn
[
0
],
CellStates
.
OUT_OF_DATE
)
pt
.
_
setCell
(
1
,
sn
[
1
],
CellStates
.
FEEDING
)
pt
.
_
setCell
(
1
,
sn
[
2
],
CellStates
.
OUT_OF_DATE
)
pt
.
_
setCell
(
2
,
sn
[
0
],
CellStates
.
OUT_OF_DATE
)
pt
.
_
setCell
(
2
,
sn
[
1
],
CellStates
.
FEEDING
)
pt
.
_
setCell
(
2
,
sn
[
2
],
CellStates
.
UP_TO_DATE
)
self
.
assertEqual
(
sorted
(
pt
.
dropNodeList
(
sn
[:
1
],
True
)),
[
(
0
,
1
,
CellStates
.
DISCARDED
),
...
...
@@ -137,7 +137,7 @@ class MasterPartitionTableTests(NeoUnitTestBase):
(
2
,
3
,
CellStates
.
DISCARDED
)])
self
.
assertRaises
(
PartitionTableException
,
pt
.
dropNodeList
,
sn
[
1
:
2
])
pt
.
setCell
(
1
,
sn
[
2
],
CellStates
.
UP_TO_DATE
)
pt
.
_
setCell
(
1
,
sn
[
2
],
CellStates
.
UP_TO_DATE
)
self
.
assertEqual
(
sorted
(
pt
.
dropNodeList
(
sn
[
1
:
2
])),
[
(
1
,
2
,
CellStates
.
DISCARDED
),
(
2
,
2
,
CellStates
.
DISCARDED
)])
...
...
@@ -233,24 +233,24 @@ class MasterPartitionTableTests(NeoUnitTestBase):
for
i
in
xrange
(
5
)]
pt
=
PartitionTable
(
5
,
2
)
# part 0
pt
.
setCell
(
0
,
sn
[
0
],
CellStates
.
DISCARDED
)
pt
.
setCell
(
0
,
sn
[
1
],
CellStates
.
UP_TO_DATE
)
pt
.
_
setCell
(
0
,
sn
[
0
],
CellStates
.
DISCARDED
)
pt
.
_
setCell
(
0
,
sn
[
1
],
CellStates
.
UP_TO_DATE
)
# part 1
pt
.
setCell
(
1
,
sn
[
0
],
CellStates
.
FEEDING
)
pt
.
setCell
(
1
,
sn
[
1
],
CellStates
.
FEEDING
)
pt
.
setCell
(
1
,
sn
[
2
],
CellStates
.
OUT_OF_DATE
)
pt
.
_
setCell
(
1
,
sn
[
0
],
CellStates
.
FEEDING
)
pt
.
_
setCell
(
1
,
sn
[
1
],
CellStates
.
FEEDING
)
pt
.
_
setCell
(
1
,
sn
[
2
],
CellStates
.
OUT_OF_DATE
)
# part 2
pt
.
setCell
(
2
,
sn
[
0
],
CellStates
.
FEEDING
)
pt
.
setCell
(
2
,
sn
[
1
],
CellStates
.
UP_TO_DATE
)
pt
.
setCell
(
2
,
sn
[
2
],
CellStates
.
UP_TO_DATE
)
pt
.
_
setCell
(
2
,
sn
[
0
],
CellStates
.
FEEDING
)
pt
.
_
setCell
(
2
,
sn
[
1
],
CellStates
.
UP_TO_DATE
)
pt
.
_
setCell
(
2
,
sn
[
2
],
CellStates
.
UP_TO_DATE
)
# part 3
pt
.
setCell
(
3
,
sn
[
0
],
CellStates
.
UP_TO_DATE
)
pt
.
setCell
(
3
,
sn
[
1
],
CellStates
.
UP_TO_DATE
)
pt
.
setCell
(
3
,
sn
[
2
],
CellStates
.
UP_TO_DATE
)
pt
.
setCell
(
3
,
sn
[
3
],
CellStates
.
UP_TO_DATE
)
pt
.
_
setCell
(
3
,
sn
[
0
],
CellStates
.
UP_TO_DATE
)
pt
.
_
setCell
(
3
,
sn
[
1
],
CellStates
.
UP_TO_DATE
)
pt
.
_
setCell
(
3
,
sn
[
2
],
CellStates
.
UP_TO_DATE
)
pt
.
_
setCell
(
3
,
sn
[
3
],
CellStates
.
UP_TO_DATE
)
# part 4
pt
.
setCell
(
4
,
sn
[
0
],
CellStates
.
UP_TO_DATE
)
pt
.
setCell
(
4
,
sn
[
4
],
CellStates
.
UP_TO_DATE
)
pt
.
_
setCell
(
4
,
sn
[
0
],
CellStates
.
UP_TO_DATE
)
pt
.
_
setCell
(
4
,
sn
[
4
],
CellStates
.
UP_TO_DATE
)
count_dict
=
defaultdict
(
int
)
change_list
=
self
.
tweak
(
pt
)
...
...
neo/tests/storage/testStorageApp.py
View file @
1eed0239
...
...
@@ -62,8 +62,8 @@ class StorageAppTests(NeoUnitTestBase):
storage
=
self
.
app
.
nm
.
createStorage
(
uuid
=
storage_uuid
)
client_uuid
=
self
.
getClientUUID
()
self
.
app
.
pt
.
setCell
(
0
,
master
,
CellStates
.
UP_TO_DATE
)
self
.
app
.
pt
.
setCell
(
0
,
storage
,
CellStates
.
UP_TO_DATE
)
self
.
app
.
pt
.
_
setCell
(
0
,
master
,
CellStates
.
UP_TO_DATE
)
self
.
app
.
pt
.
_
setCell
(
0
,
storage
,
CellStates
.
UP_TO_DATE
)
self
.
assertEqual
(
len
(
self
.
app
.
pt
.
getNodeSet
()),
2
)
self
.
assertFalse
(
self
.
app
.
pt
.
filled
())
for
x
in
xrange
(
num_partitions
):
...
...
@@ -79,8 +79,8 @@ class StorageAppTests(NeoUnitTestBase):
self
.
assertFalse
(
self
.
app
.
pt
.
hasOffset
(
x
))
# add some node
self
.
app
.
pt
.
setCell
(
0
,
master
,
CellStates
.
UP_TO_DATE
)
self
.
app
.
pt
.
setCell
(
0
,
storage
,
CellStates
.
UP_TO_DATE
)
self
.
app
.
pt
.
_
setCell
(
0
,
master
,
CellStates
.
UP_TO_DATE
)
self
.
app
.
pt
.
_
setCell
(
0
,
storage
,
CellStates
.
UP_TO_DATE
)
self
.
assertEqual
(
len
(
self
.
app
.
pt
.
getNodeSet
()),
2
)
self
.
assertFalse
(
self
.
app
.
pt
.
filled
())
for
x
in
xrange
(
num_partitions
):
...
...
neo/tests/testPT.py
View file @
1eed0239
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment