Commit 1eed0239 authored by Julien Muchembled's avatar Julien Muchembled

Code clean up: PartitionTable

parent b61ee7f1
......@@ -15,12 +15,10 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import math
from functools import wraps
from . import logging, protocol
from .locking import Lock
from .protocol import uuid_str, CellStates
from .util import u64
from .locking import RLock
class PartitionTableException(Exception):
"""
......@@ -163,7 +161,7 @@ class PartitionTable(object):
if cell.getUUID() == uuid:
return cell
def setCell(self, offset, node, state):
def _setCell(self, offset, node, state):
if state == CellStates.DISCARDED:
return self.removeCell(offset, node)
if node.isBroken() or node.isDown():
......@@ -182,7 +180,6 @@ class PartitionTable(object):
row.append(Cell(node, state))
if state != CellStates.FEEDING:
self.count_dict[node] += 1
return offset, node.getUUID(), state
def removeCell(self, offset, node):
row = self.partition_list[offset]
......@@ -193,7 +190,6 @@ class PartitionTable(object):
if not cell.isFeeding():
self.count_dict[node] -= 1
break
return (offset, node.getUUID(), CellStates.DISCARDED)
def load(self, ptid, row_list, nm):
"""
......@@ -209,24 +205,21 @@ class PartitionTable(object):
node = nm.getByUUID(uuid)
# the node must be known by the node manager
assert node is not None
self.setCell(offset, node, state)
self._setCell(offset, node, state)
logging.debug('partition table loaded (ptid=%s)', ptid)
self.log()
def update(self, ptid, cell_list, nm):
"""
Update the partition with the cell list supplied. Ignore those changes
if the partition table ID is not greater than the current one. If a node
Update the partition with the cell list supplied. If a node
is not known, it is created in the node manager and set as unavailable
"""
if ptid <= self._id:
logging.warning('ignoring older partition changes')
return
assert self._id < ptid, (self._id, ptid)
self._id = ptid
for offset, uuid, state in cell_list:
node = nm.getByUUID(uuid)
assert node is not None, 'No node found for uuid ' + uuid_str(uuid)
self.setCell(offset, node, state)
self._setCell(offset, node, state)
logging.debug('partition table updated (ptid=%s)', ptid)
self.log()
......@@ -310,38 +303,22 @@ class PartitionTable(object):
getRow = self.getRow
return [(x, getRow(x)) for x in xrange(self.np)]
def thread_safe(method):
def wrapper(self, *args, **kwargs):
self.lock()
try:
return method(self, *args, **kwargs)
finally:
self.unlock()
return wraps(method)(wrapper)
class MTPartitionTable(PartitionTable):
""" Thread-safe aware version of the partition table, override only methods
used in the client """
def __init__(self, *args, **kwargs):
self._lock = RLock()
PartitionTable.__init__(self, *args, **kwargs)
def lock(self):
self._lock.acquire()
def unlock(self):
self._lock.release()
def __init__(self, *args, **kw):
self._lock = Lock()
PartitionTable.__init__(self, *args, **kw)
@thread_safe
def setCell(self, *args, **kwargs):
return PartitionTable.setCell(self, *args, **kwargs)
def update(self, *args, **kw):
with self._lock:
return PartitionTable.update(self, *args, **kw)
@thread_safe
def clear(self, *args, **kwargs):
return PartitionTable.clear(self, *args, **kwargs)
def clear(self, *args, **kw):
with self._lock:
return PartitionTable.clear(self, *args, **kw)
@thread_safe
def operational(self, *args, **kwargs):
return PartitionTable.operational(self, *args, **kwargs)
def operational(self, *args, **kw):
with self._lock:
return PartitionTable.operational(self, *args, **kw)
......@@ -147,7 +147,7 @@ class PartitionTable(neo.lib.pt.PartitionTable):
if node is None:
node = nm.createStorage(uuid=uuid)
new_nodes.append(node.asTuple())
self.setCell(offset, node, state)
self._setCell(offset, node, state)
return new_nodes
def setUpToDate(self, node, offset):
......@@ -163,13 +163,15 @@ class PartitionTable(neo.lib.pt.PartitionTable):
raise neo.lib.pt.PartitionTableException('Non-assigned partition')
# update the partition table
cell_list = [self.setCell(offset, node, CellStates.UP_TO_DATE)]
self._setCell(offset, node, CellStates.UP_TO_DATE)
cell_list = [(offset, uuid, CellStates.UP_TO_DATE)]
# If the partition contains a feeding cell, drop it now.
for feeding_cell in self.getCellList(offset):
if feeding_cell.isFeeding():
cell_list.append(self.removeCell(offset,
feeding_cell.getNode()))
node = feeding_cell.getNode()
self.removeCell(offset, node)
cell_list.append((offset, node.getUUID(), CellStates.DISCARDED))
break
return cell_list
......
......@@ -133,19 +133,17 @@ class Application(BaseApplication):
def loadPartitionTable(self):
"""Load a partition table from the database."""
self.pt.clear()
ptid = self.dm.getPTID()
cell_list = self.dm.getPartitionTable()
new_cell_list = []
for offset, uuid, state in cell_list:
# convert from int to Enum
state = CellStates[state]
if ptid is None:
return
cell_list = []
for offset, uuid, state in self.dm.getPartitionTable():
# register unknown nodes
if self.nm.getByUUID(uuid) is None:
self.nm.createStorage(uuid=uuid)
new_cell_list.append((offset, uuid, state))
# load the partition table in manager
self.pt.clear()
self.pt.update(ptid, new_cell_list, self.nm)
cell_list.append((offset, uuid, CellStates[state]))
self.pt.update(ptid, cell_list, self.nm)
def run(self):
try:
......
......@@ -70,15 +70,15 @@ class MasterPartitionTableTests(NeoUnitTestBase):
num_partitions = 5
num_replicas = 3
pt = PartitionTable(num_partitions, num_replicas)
pt.setCell(0, sn1, CellStates.OUT_OF_DATE)
pt._setCell(0, sn1, CellStates.OUT_OF_DATE)
sn1.setState(NodeStates.RUNNING)
pt.setCell(1, sn2, CellStates.UP_TO_DATE)
pt._setCell(1, sn2, CellStates.UP_TO_DATE)
sn2.setState(NodeStates.TEMPORARILY_DOWN)
pt.setCell(2, sn3, CellStates.UP_TO_DATE)
pt._setCell(2, sn3, CellStates.UP_TO_DATE)
sn3.setState(NodeStates.DOWN)
pt.setCell(3, sn4, CellStates.UP_TO_DATE)
pt._setCell(3, sn4, CellStates.UP_TO_DATE)
sn4.setState(NodeStates.BROKEN)
pt.setCell(4, sn5, CellStates.UP_TO_DATE)
pt._setCell(4, sn5, CellStates.UP_TO_DATE)
sn5.setState(NodeStates.RUNNING)
# outdate nodes
cells_outdated = pt.outdate()
......@@ -118,12 +118,12 @@ class MasterPartitionTableTests(NeoUnitTestBase):
sn = [self.createStorage(None, i + 1, NodeStates.RUNNING)
for i in xrange(3)]
pt = PartitionTable(3, 0)
pt.setCell(0, sn[0], CellStates.OUT_OF_DATE)
pt.setCell(1, sn[1], CellStates.FEEDING)
pt.setCell(1, sn[2], CellStates.OUT_OF_DATE)
pt.setCell(2, sn[0], CellStates.OUT_OF_DATE)
pt.setCell(2, sn[1], CellStates.FEEDING)
pt.setCell(2, sn[2], CellStates.UP_TO_DATE)
pt._setCell(0, sn[0], CellStates.OUT_OF_DATE)
pt._setCell(1, sn[1], CellStates.FEEDING)
pt._setCell(1, sn[2], CellStates.OUT_OF_DATE)
pt._setCell(2, sn[0], CellStates.OUT_OF_DATE)
pt._setCell(2, sn[1], CellStates.FEEDING)
pt._setCell(2, sn[2], CellStates.UP_TO_DATE)
self.assertEqual(sorted(pt.dropNodeList(sn[:1], True)), [
(0, 1, CellStates.DISCARDED),
......@@ -137,7 +137,7 @@ class MasterPartitionTableTests(NeoUnitTestBase):
(2, 3, CellStates.DISCARDED)])
self.assertRaises(PartitionTableException, pt.dropNodeList, sn[1:2])
pt.setCell(1, sn[2], CellStates.UP_TO_DATE)
pt._setCell(1, sn[2], CellStates.UP_TO_DATE)
self.assertEqual(sorted(pt.dropNodeList(sn[1:2])), [
(1, 2, CellStates.DISCARDED),
(2, 2, CellStates.DISCARDED)])
......@@ -233,24 +233,24 @@ class MasterPartitionTableTests(NeoUnitTestBase):
for i in xrange(5)]
pt = PartitionTable(5, 2)
# part 0
pt.setCell(0, sn[0], CellStates.DISCARDED)
pt.setCell(0, sn[1], CellStates.UP_TO_DATE)
pt._setCell(0, sn[0], CellStates.DISCARDED)
pt._setCell(0, sn[1], CellStates.UP_TO_DATE)
# part 1
pt.setCell(1, sn[0], CellStates.FEEDING)
pt.setCell(1, sn[1], CellStates.FEEDING)
pt.setCell(1, sn[2], CellStates.OUT_OF_DATE)
pt._setCell(1, sn[0], CellStates.FEEDING)
pt._setCell(1, sn[1], CellStates.FEEDING)
pt._setCell(1, sn[2], CellStates.OUT_OF_DATE)
# part 2
pt.setCell(2, sn[0], CellStates.FEEDING)
pt.setCell(2, sn[1], CellStates.UP_TO_DATE)
pt.setCell(2, sn[2], CellStates.UP_TO_DATE)
pt._setCell(2, sn[0], CellStates.FEEDING)
pt._setCell(2, sn[1], CellStates.UP_TO_DATE)
pt._setCell(2, sn[2], CellStates.UP_TO_DATE)
# part 3
pt.setCell(3, sn[0], CellStates.UP_TO_DATE)
pt.setCell(3, sn[1], CellStates.UP_TO_DATE)
pt.setCell(3, sn[2], CellStates.UP_TO_DATE)
pt.setCell(3, sn[3], CellStates.UP_TO_DATE)
pt._setCell(3, sn[0], CellStates.UP_TO_DATE)
pt._setCell(3, sn[1], CellStates.UP_TO_DATE)
pt._setCell(3, sn[2], CellStates.UP_TO_DATE)
pt._setCell(3, sn[3], CellStates.UP_TO_DATE)
# part 4
pt.setCell(4, sn[0], CellStates.UP_TO_DATE)
pt.setCell(4, sn[4], CellStates.UP_TO_DATE)
pt._setCell(4, sn[0], CellStates.UP_TO_DATE)
pt._setCell(4, sn[4], CellStates.UP_TO_DATE)
count_dict = defaultdict(int)
change_list = self.tweak(pt)
......
......@@ -62,8 +62,8 @@ class StorageAppTests(NeoUnitTestBase):
storage = self.app.nm.createStorage(uuid=storage_uuid)
client_uuid = self.getClientUUID()
self.app.pt.setCell(0, master, CellStates.UP_TO_DATE)
self.app.pt.setCell(0, storage, CellStates.UP_TO_DATE)
self.app.pt._setCell(0, master, CellStates.UP_TO_DATE)
self.app.pt._setCell(0, storage, CellStates.UP_TO_DATE)
self.assertEqual(len(self.app.pt.getNodeSet()), 2)
self.assertFalse(self.app.pt.filled())
for x in xrange(num_partitions):
......@@ -79,8 +79,8 @@ class StorageAppTests(NeoUnitTestBase):
self.assertFalse(self.app.pt.hasOffset(x))
# add some node
self.app.pt.setCell(0, master, CellStates.UP_TO_DATE)
self.app.pt.setCell(0, storage, CellStates.UP_TO_DATE)
self.app.pt._setCell(0, master, CellStates.UP_TO_DATE)
self.app.pt._setCell(0, storage, CellStates.UP_TO_DATE)
self.assertEqual(len(self.app.pt.getNodeSet()), 2)
self.assertFalse(self.app.pt.filled())
for x in xrange(num_partitions):
......
......@@ -53,7 +53,7 @@ class PartitionTableTests(NeoUnitTestBase):
self.assertEqual(len(pt.partition_list[x]), 0)
# add a cell to an empty row
self.assertFalse(pt.count_dict.has_key(sn1))
pt.setCell(0, sn1, CellStates.UP_TO_DATE)
pt._setCell(0, sn1, CellStates.UP_TO_DATE)
self.assertTrue(pt.count_dict.has_key(sn1))
self.assertEqual(pt.count_dict[sn1], 1)
for x in xrange(num_partitions):
......@@ -64,14 +64,14 @@ class PartitionTableTests(NeoUnitTestBase):
else:
self.assertEqual(len(pt.partition_list[x]), 0)
# try to add to a nonexistent partition
self.assertRaises(IndexError, pt.setCell, 10, sn1, CellStates.UP_TO_DATE)
self.assertRaises(IndexError, pt._setCell, 10, sn1, CellStates.UP_TO_DATE)
# if we add in discards state, must be removed
pt.setCell(0, sn1, CellStates.DISCARDED)
pt._setCell(0, sn1, CellStates.DISCARDED)
for x in xrange(num_partitions):
self.assertEqual(len(pt.partition_list[x]), 0)
self.assertEqual(pt.count_dict[sn1], 0)
# add a feeding node into empty row
pt.setCell(0, sn1, CellStates.FEEDING)
pt._setCell(0, sn1, CellStates.FEEDING)
self.assertTrue(pt.count_dict.has_key(sn1))
self.assertEqual(pt.count_dict[sn1], 0)
for x in xrange(num_partitions):
......@@ -82,7 +82,7 @@ class PartitionTableTests(NeoUnitTestBase):
else:
self.assertEqual(len(pt.partition_list[x]), 0)
# re-add it as feeding, nothing change
pt.setCell(0, sn1, CellStates.FEEDING)
pt._setCell(0, sn1, CellStates.FEEDING)
self.assertTrue(pt.count_dict.has_key(sn1))
self.assertEqual(pt.count_dict[sn1], 0)
for x in xrange(num_partitions):
......@@ -93,7 +93,7 @@ class PartitionTableTests(NeoUnitTestBase):
else:
self.assertEqual(len(pt.partition_list[x]), 0)
# now add it as up to date
pt.setCell(0, sn1, CellStates.UP_TO_DATE)
pt._setCell(0, sn1, CellStates.UP_TO_DATE)
self.assertTrue(pt.count_dict.has_key(sn1))
self.assertEqual(pt.count_dict[sn1], 1)
for x in xrange(num_partitions):
......@@ -105,18 +105,18 @@ class PartitionTableTests(NeoUnitTestBase):
self.assertEqual(len(pt.partition_list[x]), 0)
# now add broken and down state, must not be taken into account
pt.setCell(0, sn1, CellStates.DISCARDED)
pt._setCell(0, sn1, CellStates.DISCARDED)
for x in xrange(num_partitions):
self.assertEqual(len(pt.partition_list[x]), 0)
self.assertEqual(pt.count_dict[sn1], 0)
sn1.setState(NodeStates.BROKEN)
self.assertRaises(PartitionTableException, pt.setCell,
self.assertRaises(PartitionTableException, pt._setCell,
0, sn1, CellStates.UP_TO_DATE)
for x in xrange(num_partitions):
self.assertEqual(len(pt.partition_list[x]), 0)
self.assertEqual(pt.count_dict[sn1], 0)
sn1.setState(NodeStates.DOWN)
self.assertRaises(PartitionTableException, pt.setCell,
self.assertRaises(PartitionTableException, pt._setCell,
0, sn1, CellStates.UP_TO_DATE)
for x in xrange(num_partitions):
self.assertEqual(len(pt.partition_list[x]), 0)
......@@ -134,7 +134,7 @@ class PartitionTableTests(NeoUnitTestBase):
self.assertEqual(len(pt.partition_list[x]), 0)
# add a cell to an empty row
self.assertFalse(pt.count_dict.has_key(sn1))
pt.setCell(0, sn1, CellStates.UP_TO_DATE)
pt._setCell(0, sn1, CellStates.UP_TO_DATE)
self.assertTrue(pt.count_dict.has_key(sn1))
self.assertEqual(pt.count_dict[sn1], 1)
for x in xrange(num_partitions):
......@@ -148,7 +148,7 @@ class PartitionTableTests(NeoUnitTestBase):
for x in xrange(num_partitions):
self.assertEqual(len(pt.partition_list[x]), 0)
# add a feeding cell
pt.setCell(0, sn1, CellStates.FEEDING)
pt._setCell(0, sn1, CellStates.FEEDING)
self.assertTrue(pt.count_dict.has_key(sn1))
self.assertEqual(pt.count_dict[sn1], 0)
for x in xrange(num_partitions):
......@@ -170,19 +170,19 @@ class PartitionTableTests(NeoUnitTestBase):
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = self.createStorage(server1, uuid1)
pt.setCell(0, sn1, CellStates.UP_TO_DATE)
pt._setCell(0, sn1, CellStates.UP_TO_DATE)
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19001)
sn2 = self.createStorage(server2, uuid2)
pt.setCell(0, sn2, CellStates.OUT_OF_DATE)
pt._setCell(0, sn2, CellStates.OUT_OF_DATE)
uuid3 = self.getStorageUUID()
server3 = ("127.0.0.3", 19001)
sn3 = self.createStorage(server3, uuid3)
pt.setCell(0, sn3, CellStates.FEEDING)
pt._setCell(0, sn3, CellStates.FEEDING)
uuid4 = self.getStorageUUID()
server4 = ("127.0.0.4", 19001)
sn4 = self.createStorage(server4, uuid4)
pt.setCell(0, sn4, CellStates.DISCARDED) # won't be added
pt._setCell(0, sn4, CellStates.DISCARDED) # won't be added
# now checks result
self.assertEqual(len(pt.partition_list[0]), 3)
for x in xrange(num_partitions):
......@@ -216,15 +216,15 @@ class PartitionTableTests(NeoUnitTestBase):
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = self.createStorage(server1, uuid1)
pt.setCell(0, sn1, CellStates.UP_TO_DATE)
pt._setCell(0, sn1, CellStates.UP_TO_DATE)
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19001)
sn2 = self.createStorage(server2, uuid2)
pt.setCell(1, sn2, CellStates.OUT_OF_DATE)
pt._setCell(1, sn2, CellStates.OUT_OF_DATE)
uuid3 = self.getStorageUUID()
server3 = ("127.0.0.3", 19001)
sn3 = self.createStorage(server3, uuid3)
pt.setCell(2, sn3, CellStates.FEEDING)
pt._setCell(2, sn3, CellStates.FEEDING)
# now checks result
self.assertEqual(len(pt.partition_list[0]), 1)
self.assertEqual(len(pt.partition_list[1]), 1)
......@@ -246,19 +246,19 @@ class PartitionTableTests(NeoUnitTestBase):
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = self.createStorage(server1, uuid1)
pt.setCell(0, sn1, CellStates.UP_TO_DATE)
pt._setCell(0, sn1, CellStates.UP_TO_DATE)
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19001)
sn2 = self.createStorage(server2, uuid2)
pt.setCell(0, sn2, CellStates.OUT_OF_DATE)
pt._setCell(0, sn2, CellStates.OUT_OF_DATE)
uuid3 = self.getStorageUUID()
server3 = ("127.0.0.3", 19001)
sn3 = self.createStorage(server3, uuid3)
pt.setCell(0, sn3, CellStates.FEEDING)
pt._setCell(0, sn3, CellStates.FEEDING)
uuid4 = self.getStorageUUID()
server4 = ("127.0.0.4", 19001)
sn4 = self.createStorage(server4, uuid4)
pt.setCell(0, sn4, CellStates.DISCARDED) # won't be added
pt._setCell(0, sn4, CellStates.DISCARDED) # won't be added
# must get only two node as feeding and discarded not taken
# into account
self.assertEqual(pt.getNodeSet(True), {sn1, sn3})
......@@ -276,7 +276,7 @@ class PartitionTableTests(NeoUnitTestBase):
server1 = ("127.0.0.1", 19001)
sn1 = self.createStorage(server1, uuid1)
for x in xrange(num_partitions):
pt.setCell(x, sn1, CellStates.UP_TO_DATE)
pt._setCell(x, sn1, CellStates.UP_TO_DATE)
self.assertEqual(pt.num_filled_rows, num_partitions)
self.assertTrue(pt.filled())
......@@ -288,7 +288,7 @@ class PartitionTableTests(NeoUnitTestBase):
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = self.createStorage(server1, uuid1)
pt.setCell(0, sn1, CellStates.UP_TO_DATE)
pt._setCell(0, sn1, CellStates.UP_TO_DATE)
# now test
self.assertTrue(pt.hasOffset(0))
self.assertFalse(pt.hasOffset(1))
......@@ -307,7 +307,7 @@ class PartitionTableTests(NeoUnitTestBase):
# adding a node in all partition
sn1 = createStorage()
for x in xrange(num_partitions):
pt.setCell(x, sn1, CellStates.UP_TO_DATE)
pt._setCell(x, sn1, CellStates.UP_TO_DATE)
self.assertTrue(pt.filled())
# it's up to date and running, so operational
sn1.setState(NodeStates.RUNNING)
......@@ -319,7 +319,7 @@ class PartitionTableTests(NeoUnitTestBase):
# adding a node in all partition
sn1 = createStorage()
for x in xrange(num_partitions):
pt.setCell(x, sn1, CellStates.FEEDING)
pt._setCell(x, sn1, CellStates.FEEDING)
self.assertTrue(pt.filled())
# it's feeding and running, so operational
sn1.setState(NodeStates.RUNNING)
......@@ -333,7 +333,7 @@ class PartitionTableTests(NeoUnitTestBase):
sn1 = createStorage()
sn1.setState(NodeStates.TEMPORARILY_DOWN)
for x in xrange(num_partitions):
pt.setCell(x, sn1, CellStates.FEEDING)
pt._setCell(x, sn1, CellStates.FEEDING)
self.assertTrue(pt.filled())
# it's up to date and not running, so not operational
self.assertFalse(pt.operational())
......@@ -345,7 +345,7 @@ class PartitionTableTests(NeoUnitTestBase):
# adding a node in all partition
sn1 = createStorage()
for x in xrange(num_partitions):
pt.setCell(x, sn1, CellStates.OUT_OF_DATE)
pt._setCell(x, sn1, CellStates.OUT_OF_DATE)
self.assertTrue(pt.filled())
# it's not up to date and running, so not operational
self.assertFalse(pt.operational())
......@@ -358,18 +358,18 @@ class PartitionTableTests(NeoUnitTestBase):
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = self.createStorage(server1, uuid1)
pt.setCell(0, sn1, CellStates.UP_TO_DATE)
pt.setCell(1, sn1, CellStates.UP_TO_DATE)
pt.setCell(2, sn1, CellStates.UP_TO_DATE)
pt._setCell(0, sn1, CellStates.UP_TO_DATE)
pt._setCell(1, sn1, CellStates.UP_TO_DATE)
pt._setCell(2, sn1, CellStates.UP_TO_DATE)
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19001)
sn2 = self.createStorage(server2, uuid2)
pt.setCell(0, sn2, CellStates.UP_TO_DATE)
pt.setCell(1, sn2, CellStates.UP_TO_DATE)
pt._setCell(0, sn2, CellStates.UP_TO_DATE)
pt._setCell(1, sn2, CellStates.UP_TO_DATE)
uuid3 = self.getStorageUUID()
server3 = ("127.0.0.3", 19001)
sn3 = self.createStorage(server3, uuid3)
pt.setCell(0, sn3, CellStates.UP_TO_DATE)
pt._setCell(0, sn3, CellStates.UP_TO_DATE)
# test
row_0 = pt.getRow(0)
self.assertEqual(len(row_0), 3)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment