Skip to content

N
neo
Commit a1539219 authored by Julien Muchembled's avatar Julien Muchembled
Browse files
Options

Do not change partition table when adding node and reimplement pt.tweak()

parent 07b1ce47
Show whitespace changes
Inline Side-by-side
Showing with 258 additions and 381 deletions
neo/admin/handler.py
View file @ a1539219
...@@ -78,6 +78,7 @@ class AdminEventHandler(EventHandler): ...@@ -78,6 +78,7 @@ class AdminEventHandler(EventHandler):
conn.answer(Packets.AnswerPrimary(master_node.getUUID())) conn.answer(Packets.AnswerPrimary(master_node.getUUID()))
addPendingNodes = forward_ask(Packets.AddPendingNodes) addPendingNodes = forward_ask(Packets.AddPendingNodes)
tweakPartitionTable = forward_ask(Packets.TweakPartitionTable)
setClusterState = forward_ask(Packets.SetClusterState) setClusterState = forward_ask(Packets.SetClusterState)
checkReplicas = forward_ask(Packets.CheckReplicas) checkReplicas = forward_ask(Packets.CheckReplicas)
......
neo/lib/protocol.py
View file @ a1539219
...@@ -26,7 +26,7 @@ except ImportError: ...@@ -26,7 +26,7 @@ except ImportError:
pass pass
# The protocol version (major, minor). # The protocol version (major, minor).
PROTOCOL_VERSION = (11, 1) PROTOCOL_VERSION = (12, 1)
# Size restrictions. # Size restrictions.
MIN_PACKET_SIZE = 10 MIN_PACKET_SIZE = 10
...@@ -1150,6 +1150,16 @@ class AddPendingNodes(Packet): ...@@ -1150,6 +1150,16 @@ class AddPendingNodes(Packet):
_answer = Error _answer = Error
class TweakPartitionTable(Packet):
"""
Ask the primary to optimize the partition table. A -> PM.
"""
_fmt = PStruct('tweak_partition_table',
PFUUIDList,
)
_answer = Error
class NotifyNodeInformation(Packet): class NotifyNodeInformation(Packet):
""" """
Notify information about one or more nodes. PM -> Any. Notify information about one or more nodes. PM -> Any.
...@@ -1666,6 +1676,8 @@ class Packets(dict): ...@@ -1666,6 +1676,8 @@ class Packets(dict):
SetNodeState, ignore_when_closed=False) SetNodeState, ignore_when_closed=False)
AddPendingNodes = register( AddPendingNodes = register(
AddPendingNodes, ignore_when_closed=False) AddPendingNodes, ignore_when_closed=False)
TweakPartitionTable = register(
TweakPartitionTable, ignore_when_closed=False)
AskNodeInformation, AnswerNodeInformation = register( AskNodeInformation, AnswerNodeInformation = register(
NodeInformation) NodeInformation)
SetClusterState = register( SetClusterState = register(
......
neo/master/handlers/administration.py
View file @ a1539219
...@@ -130,37 +130,40 @@ class AdministrationHandler(MasterHandler): ...@@ -130,37 +130,40 @@ class AdministrationHandler(MasterHandler):
uuids = ', '.join(map(uuid_str, uuid_list)) uuids = ', '.join(map(uuid_str, uuid_list))
logging.debug('Add nodes %s', uuids) logging.debug('Add nodes %s', uuids)
app = self.app app = self.app
nm = app.nm state = app.getClusterState()
em = app.em # XXX: Would it be safe to allow more states ?
pt = app.pt if state not in (ClusterStates.RUNNING,
cell_list = [] ClusterStates.STARTING_BACKUP,
uuid_set = set() ClusterStates.BACKINGUP):
if app.getClusterState() == ClusterStates.RUNNING: raise ProtocolError('Can not add nodes in %s state' % state)
# take all pending nodes # take all pending nodes
for node in nm.getStorageList(): node_list = list(app.pt.addNodeList(node
if node.isPending(): for node in app.nm.getStorageList()
uuid_set.add(node.getUUID()) if node.isPending() and node.getUUID() in uuid_list))
# keep only selected nodes if node_list:
if uuid_list: p = Packets.StartOperation()
uuid_set = uuid_set.intersection(set(uuid_list))
# nothing to do
if not uuid_set:
logging.warning('No nodes added')
conn.answer(Errors.Ack('No nodes added'))
return
uuids = ', '.join(map(uuid_str, uuid_set))
logging.info('Adding nodes %s', uuids)
# switch nodes to running state
node_list = map(nm.getByUUID, uuid_set)
for node in node_list: for node in node_list:
new_cells = pt.addNode(node)
cell_list.extend(new_cells)
node.setRunning() node.setRunning()
node.getConnection().notify(Packets.StartOperation()) node.notify(p)
app.broadcastNodesInformation(node_list) app.broadcastNodesInformation(node_list)
# broadcast the new partition table conn.answer(Errors.Ack('Nodes added: %s' %
app.broadcastPartitionChanges(cell_list) ', '.join(uuid_str(x.getUUID()) for x in node_list)))
conn.answer(Errors.Ack('Nodes added: %s' % (uuids, ))) else:
logging.warning('No node added')
conn.answer(Errors.Ack('No node added'))
def tweakPartitionTable(self, conn, uuid_list):
app = self.app
state = app.getClusterState()
# XXX: Would it be safe to allow more states ?
if state not in (ClusterStates.RUNNING,
ClusterStates.STARTING_BACKUP,
ClusterStates.BACKINGUP):
raise ProtocolError('Can not tweak partition table in %s state'
% state)
app.broadcastPartitionChanges(app.pt.tweak(
map(app.nm.getByUUID, uuid_list)))
conn.answer(Errors.Ack(''))
def checkReplicas(self, conn, partition_dict, min_tid, max_tid): def checkReplicas(self, conn, partition_dict, min_tid, max_tid):
app = self.app app = self.app
......
neo/master/pt.py
View file @ a1539219
...@@ -35,6 +35,16 @@ class Cell(neo.lib.pt.Cell): ...@@ -35,6 +35,16 @@ class Cell(neo.lib.pt.Cell):
neo.lib.pt.Cell = Cell neo.lib.pt.Cell = Cell
class MappedNode(object):
def __init__(self, node):
self.node = node
self.assigned = set()
def __getattr__(self, attr):
return getattr(self.node, attr)
class PartitionTable(neo.lib.pt.PartitionTable): class PartitionTable(neo.lib.pt.PartitionTable):
"""This class manages a partition table for the primary master node""" """This class manages a partition table for the primary master node"""
...@@ -164,130 +174,101 @@ class PartitionTable(neo.lib.pt.PartitionTable): ...@@ -164,130 +174,101 @@ class PartitionTable(neo.lib.pt.PartitionTable):
return cell_list return cell_list
def addNode(self, node): def addNodeList(self, node_list):
"""Add a node. Take it into account that it might not be really a new """Add nodes"""
node. The strategy is, if a row does not contain a good number of added_list = []
cells, add this node to the row, unless the node is already present for node in node_list:
in the same row. Otherwise, check if this node should replace another if node not in self.count_dict:
cell.""" self.count_dict[node] = 0
cell_list = [] added_list.append(node)
node_count = self.count_dict.get(node, 0) return added_list
for offset, row in enumerate(self.partition_list):
max_count = 0
max_cell = None
num_cells = 0
for cell in row:
if cell.getNode() is node:
break
if not cell.isFeeding():
num_cells += 1
count = self.count_dict[cell.getNode()]
if count > max_count:
max_count = count
max_cell = cell
else:
if self.nr < num_cells:
if node_count + 1 >= max_count:
continue
if max_cell.isReadable():
max_cell.setState(CellStates.FEEDING)
cell_list.append((offset, max_cell.getUUID(),
CellStates.FEEDING))
else:
row.remove(max_cell)
cell_list.append((offset, max_cell.getUUID(),
CellStates.DISCARDED))
self.count_dict[max_cell.getNode()] -= 1
row.append(Cell(node, CellStates.OUT_OF_DATE))
cell_list.append((offset, node.getUUID(),
CellStates.OUT_OF_DATE))
node_count += 1
self.count_dict[node] = node_count
self.log()
return cell_list
def tweak(self): def tweak(self, drop_list=()):
"""Test if nodes are distributed uniformly. Otherwise, correct the """Optimize partition table
partition table."""
changed_cell_list = []
This is done by computing a minimal diff between current partition table
and what make() would do.
"""
assigned_dict = dict((x, {}) for x in self.count_dict)
readable_dict = dict((i, set()) for i in xrange(self.np))
for offset, row in enumerate(self.partition_list): for offset, row in enumerate(self.partition_list):
removed_cell_list = []
feeding_cell = None
out_of_date_cell_list = []
up_to_date_cell_list = []
for cell in row: for cell in row:
if cell.getNode().isBroken(): if cell.isReadable():
# Remove a broken cell. readable_dict[offset].add(cell)
removed_cell_list.append(cell) assigned_dict[cell.getNode()][offset] = cell
elif cell.isFeeding(): pt = PartitionTable(self.np, self.nr)
if feeding_cell is None: drop_list = set(x for x in drop_list if x in assigned_dict)
feeding_cell = cell node_set = set(MappedNode(x) for x in assigned_dict
else: if x not in drop_list)
# Remove an excessive feeding cell. pt.make(node_set)
removed_cell_list.append(cell) for offset, row in enumerate(pt.partition_list):
elif cell.isUpToDate():
up_to_date_cell_list.append(cell)
else:
out_of_date_cell_list.append(cell)
# If all cells are up-to-date, a feeding cell is not required.
if len(out_of_date_cell_list) == 0 and feeding_cell is not None:
removed_cell_list.append(feeding_cell)
ideal_num = self.nr + 1
while len(out_of_date_cell_list) + len(up_to_date_cell_list) > \
ideal_num:
# This row contains too many cells.
if len(up_to_date_cell_list) > 1:
# There are multiple up-to-date cells, so choose whatever
# used too much.
cell_list = out_of_date_cell_list + up_to_date_cell_list
else:
# Drop an out-of-date cell.
cell_list = out_of_date_cell_list
max_count = 0
chosen_cell = None
for cell in cell_list:
count = self.count_dict[cell.getNode()]
if max_count < count:
max_count = count
chosen_cell = cell
removed_cell_list.append(chosen_cell)
try:
out_of_date_cell_list.remove(chosen_cell)
except ValueError:
up_to_date_cell_list.remove(chosen_cell)
# Now remove cells really.
for cell in removed_cell_list:
row.remove(cell)
if not cell.isFeeding():
self.count_dict[cell.getNode()] -= 1
changed_cell_list.append((offset, cell.getUUID(),
CellStates.DISCARDED))
# Add cells, if a row contains less than the number of replicas.
for offset, row in enumerate(self.partition_list):
num_cells = 0
for cell in row: for cell in row:
if cell.isReadable():
cell.getNode().assigned.add(offset)
def map_nodes():
node_list = []
for node, assigned in assigned_dict.iteritems():
if node in drop_list:
yield node, frozenset()
continue
readable = set(offset for offset, cell in assigned.iteritems()
if cell.isReadable())
# the criterion on UUID is purely cosmetic
node_list.append((len(readable), len(assigned),
-node.getUUID(), readable, node))
node_list.sort(reverse=1)
for _, _, _, readable, node in node_list:
assigned = assigned_dict[node]
mapped = min(node_set, key=lambda m: (
len(m.assigned.symmetric_difference(assigned)),
len(m.assigned ^ readable)))
node_set.remove(mapped)
yield node, mapped.assigned
assert not node_set
changed_list = []
uptodate_set = set()
remove_dict = dict((i, []) for i in xrange(self.np))
for node, mapped in map_nodes():
uuid = node.getUUID()
assigned = assigned_dict[node]
for offset, cell in assigned.iteritems():
if offset in mapped:
if cell.isReadable():
uptodate_set.add(offset)
readable_dict[offset].remove(cell)
if cell.isFeeding():
self.count_dict[node] += 1
state = CellStates.UP_TO_DATE
cell.setState(state)
changed_list.append((offset, uuid, state))
else:
if not cell.isFeeding(): if not cell.isFeeding():
num_cells += 1 self.count_dict[node] -= 1
while num_cells <= self.nr: remove_dict[offset].append(cell)
node = self.findLeastUsedNode([cell.getNode() for cell in row]) for offset in mapped.difference(assigned):
if node is None:
break
row.append(Cell(node, CellStates.OUT_OF_DATE))
changed_cell_list.append((offset, node.getUUID(),
CellStates.OUT_OF_DATE))
self.count_dict[node] += 1 self.count_dict[node] += 1
num_cells += 1 state = CellStates.OUT_OF_DATE
self.partition_list[offset].append(Cell(node, state))
self.log() changed_list.append((offset, uuid, state))
return changed_cell_list count_dict = self.count_dict.copy()
for offset, cell_list in remove_dict.iteritems():
if not cell_list:
continue
row = self.partition_list[offset]
feeding = None if offset in uptodate_set else min(
readable_dict[offset], key=lambda x: count_dict[x.getNode()])
for cell in cell_list:
if cell is feeding:
count_dict[cell.getNode()] += 1
if cell.isFeeding():
continue
state = CellStates.FEEDING
cell.setState(state)
else:
state = CellStates.DISCARDED
row.remove(cell)
changed_list.append((offset, cell.getUUID(), state))
return changed_list
def outdate(self, lost_node=None): def outdate(self, lost_node=None):
"""Outdate all non-working nodes """Outdate all non-working nodes
......
neo/master/verification.py
View file @ a1539219
...@@ -93,7 +93,6 @@ class VerificationManager(BaseServiceHandler): ...@@ -93,7 +93,6 @@ class VerificationManager(BaseServiceHandler):
return state, self return state, self
def run(self): def run(self):
self.app.changeClusterState(ClusterStates.VERIFYING) self.app.changeClusterState(ClusterStates.VERIFYING)
while True: while True:
try: try:
...@@ -102,14 +101,7 @@ class VerificationManager(BaseServiceHandler): ...@@ -102,14 +101,7 @@ class VerificationManager(BaseServiceHandler):
continue continue
break break
# At this stage, all non-working nodes are out-of-date. # At this stage, all non-working nodes are out-of-date.
cell_list = self.app.pt.outdate() self.app.broadcastPartitionChanges(self.app.pt.outdate())
# Tweak the partition table, if the distribution of storage nodes
# is not uniform.
cell_list.extend(self.app.pt.tweak())
# If anything changed, send the changes.
self.app.broadcastPartitionChanges(cell_list)
def verifyData(self): def verifyData(self):
"""Verify the data in storage nodes and clean them up, if necessary.""" """Verify the data in storage nodes and clean them up, if necessary."""
......
neo/neoctl/app.py
View file @ a1539219
...@@ -34,6 +34,7 @@ action_dict = { ...@@ -34,6 +34,7 @@ action_dict = {
'check': 'checkReplicas', 'check': 'checkReplicas',
'start': 'startCluster', 'start': 'startCluster',
'add': 'enableStorageList', 'add': 'enableStorageList',
'tweak': 'tweakPartitionTable',
'drop': 'dropNode', 'drop': 'dropNode',
} }
...@@ -169,6 +170,14 @@ class TerminalNeoCTL(object): ...@@ -169,6 +170,14 @@ class TerminalNeoCTL(object):
uuid_list = map(self.asNode, params) uuid_list = map(self.asNode, params)
return self.neoctl.enableStorageList(uuid_list) return self.neoctl.enableStorageList(uuid_list)
def tweakPartitionTable(self, params):
"""
Optimize partition table.
No partitition will be assigned to specified storage nodes.
Parameters: [node [...]]
"""
return self.neoctl.tweakPartitionTable(map(self.asNode, params))
def dropNode(self, params): def dropNode(self, params):
""" """
Set node into DOWN state. Set node into DOWN state.
......
neo/neoctl/neoctl.py
View file @ a1539219
...@@ -84,6 +84,12 @@ class NeoCTL(object): ...@@ -84,6 +84,12 @@ class NeoCTL(object):
raise RuntimeError(response) raise RuntimeError(response)
return response[2] return response[2]
def tweakPartitionTable(self, uuid_list=()):
response = self.__ask(Packets.TweakPartitionTable(uuid_list))
if response[0] != Packets.Error or response[1] != ErrorCodes.ACK:
raise RuntimeError(response)
return response[2]
def setClusterState(self, state): def setClusterState(self, state):
""" """
Set cluster state. Set cluster state.
......
neo/tests/functional/testStorage.py
View file @ a1539219
...@@ -160,6 +160,7 @@ class StorageTests(NEOFunctionalTest): ...@@ -160,6 +160,7 @@ class StorageTests(NEOFunctionalTest):
# add it to the partition table # add it to the partition table
self.neo.neoctl.enableStorageList([stopped[0].getUUID()]) self.neo.neoctl.enableStorageList([stopped[0].getUUID()])
self.neo.expectRunning(stopped[0]) self.neo.expectRunning(stopped[0])
self.neo.neoctl.tweakPartitionTable()
self.neo.expectAssignedCells(stopped[0], number=10) self.neo.expectAssignedCells(stopped[0], number=10)
self.neo.expectClusterRunning() self.neo.expectClusterRunning()
...@@ -298,6 +299,7 @@ class StorageTests(NEOFunctionalTest): ...@@ -298,6 +299,7 @@ class StorageTests(NEOFunctionalTest):
stopped[0].start() stopped[0].start()
self.neo.expectPending(stopped[0]) self.neo.expectPending(stopped[0])
self.neo.neoctl.enableStorageList([stopped[0].getUUID()]) self.neo.neoctl.enableStorageList([stopped[0].getUUID()])
self.neo.neoctl.tweakPartitionTable()
self.neo.expectRunning(stopped[0]) self.neo.expectRunning(stopped[0])
self.neo.expectClusterRunning() self.neo.expectClusterRunning()
self.neo.expectOudatedCells(number=0) self.neo.expectOudatedCells(number=0)
...@@ -356,6 +358,7 @@ class StorageTests(NEOFunctionalTest): ...@@ -356,6 +358,7 @@ class StorageTests(NEOFunctionalTest):
stopped[0].start() stopped[0].start()
self.neo.expectPending(stopped[0]) self.neo.expectPending(stopped[0])
self.neo.neoctl.enableStorageList([stopped[0].getUUID()]) self.neo.neoctl.enableStorageList([stopped[0].getUUID()])
self.neo.neoctl.tweakPartitionTable()
self.neo.expectRunning(stopped[0]) self.neo.expectRunning(stopped[0])
self.neo.expectClusterRunning() self.neo.expectClusterRunning()
self.neo.expectAssignedCells(started[0], 10) self.neo.expectAssignedCells(started[0], 10)
...@@ -469,6 +472,7 @@ class StorageTests(NEOFunctionalTest): ...@@ -469,6 +472,7 @@ class StorageTests(NEOFunctionalTest):
stopped[0].start() stopped[0].start()
self.neo.expectPending(stopped[0]) self.neo.expectPending(stopped[0])
self.neo.neoctl.enableStorageList([stopped[0].getUUID()]) self.neo.neoctl.enableStorageList([stopped[0].getUUID()])
self.neo.neoctl.tweakPartitionTable()
self.neo.expectRunning(stopped[0]) self.neo.expectRunning(stopped[0])
self.neo.expectClusterRunning() self.neo.expectClusterRunning()
self.neo.expectAssignedCells(started[0], 10) self.neo.expectAssignedCells(started[0], 10)
......
neo/tests/master/testMasterPT.py
View file @ a1539219
...@@ -15,6 +15,7 @@ ...@@ -15,6 +15,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
import unittest import unittest
from collections import defaultdict
from mock import Mock from mock import Mock
from .. import NeoUnitTestBase from .. import NeoUnitTestBase
from neo.lib.protocol import NodeStates, CellStates from neo.lib.protocol import NodeStates, CellStates
...@@ -142,130 +143,6 @@ class MasterPartitionTableTests(NeoUnitTestBase): ...@@ -142,130 +143,6 @@ class MasterPartitionTableTests(NeoUnitTestBase):
cell = cells[0] cell = cells[0]
self.assertEqual(cell.getState(), CellStates.UP_TO_DATE) self.assertEqual(cell.getState(), CellStates.UP_TO_DATE)
def test_14_addNode(self):
num_partitions = 5
num_replicas = 2
pt = PartitionTable(num_partitions, num_replicas)
# add nodes
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
# add it to an empty pt
cell_list = pt.addNode(sn1)
self.assertEqual(len(cell_list), 5)
# it must be added to all partitions
for x in xrange(num_partitions):
self.assertEqual(len(pt.getCellList(x)), 1)
self.assertEqual(pt.getCellList(x)[0].getState(), CellStates.OUT_OF_DATE)
self.assertEqual(pt.getCellList(x)[0].getNode(), sn1)
self.assertEqual(pt.count_dict[sn1], 5)
# add same node again, must remain the same
cell_list = pt.addNode(sn1)
self.assertEqual(len(cell_list), 0)
for x in xrange(num_partitions):
self.assertEqual(len(pt.getCellList(x)), 1)
self.assertEqual(pt.getCellList(x)[0].getState(), CellStates.OUT_OF_DATE)
self.assertEqual(pt.getCellList(x)[0].getNode(), sn1)
self.assertEqual(pt.count_dict[sn1], 5)
# add a second node to fill the partition table
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19002)
sn2 = StorageNode(Mock(), server2, uuid2)
# add it
cell_list = pt.addNode(sn2)
self.assertEqual(len(cell_list), 5)
for x in xrange(num_partitions):
self.assertEqual(len(pt.getCellList(x)), 2)
self.assertEqual(pt.getCellList(x)[0].getState(), CellStates.OUT_OF_DATE)
self.assertTrue(pt.getCellList(x)[0].getNode() in (sn1, sn2))
# test the most used node is remove from some partition
uuid3 = self.getStorageUUID()
server3 = ("127.0.0.3", 19001)
sn3 = StorageNode(Mock(), server3, uuid3)
uuid4 = self.getStorageUUID()
server4 = ("127.0.0.4", 19001)
sn4 = StorageNode(Mock(), server4, uuid4)
uuid5 = self.getStorageUUID()
server5 = ("127.0.0.5", 1900)
sn5 = StorageNode(Mock(), server5, uuid5)
# partition looks like:
# 0 : sn1, sn2
# 1 : sn1, sn3
# 2 : sn1, sn4
# 3 : sn1, sn5
num_partitions = 4
num_replicas = 1
pt = PartitionTable(num_partitions, num_replicas)
# node most used is out of date, just dropped
pt.setCell(0, sn1, CellStates.OUT_OF_DATE)
pt.setCell(0, sn2, CellStates.UP_TO_DATE)
pt.setCell(1, sn1, CellStates.OUT_OF_DATE)
pt.setCell(1, sn3, CellStates.UP_TO_DATE)
pt.setCell(2, sn1, CellStates.OUT_OF_DATE)
pt.setCell(2, sn4, CellStates.UP_TO_DATE)
pt.setCell(3, sn1, CellStates.OUT_OF_DATE)
pt.setCell(3, sn5, CellStates.UP_TO_DATE)
uuid6 = self.getStorageUUID()
server6 = ("127.0.0.6", 19006)
sn6 = StorageNode(Mock(), server6, uuid6)
# sn1 is removed twice and sn6 is added twice
self.assertEqual(sorted(pt.addNode(sn6)), [
(0, uuid1, CellStates.DISCARDED),
(0, uuid6, CellStates.OUT_OF_DATE),
(1, uuid1, CellStates.DISCARDED),
(1, uuid6, CellStates.OUT_OF_DATE)])
for x in xrange(num_partitions):
self.assertEqual(len(pt.getCellList(x)), 2)
# there is a feeding cell, just dropped
pt.clear()
pt.setCell(0, sn1, CellStates.OUT_OF_DATE)
pt.setCell(0, sn2, CellStates.UP_TO_DATE)
pt.setCell(0, sn3, CellStates.FEEDING)
pt.setCell(1, sn1, CellStates.OUT_OF_DATE)
pt.setCell(1, sn2, CellStates.FEEDING)
pt.setCell(1, sn3, CellStates.UP_TO_DATE)
pt.setCell(2, sn1, CellStates.UP_TO_DATE)
pt.setCell(2, sn4, CellStates.UP_TO_DATE)
pt.setCell(2, sn5, CellStates.UP_TO_DATE)
pt.setCell(3, sn1, CellStates.UP_TO_DATE)
pt.setCell(3, sn4, CellStates.UP_TO_DATE)
pt.setCell(3, sn5, CellStates.UP_TO_DATE)
# sn1 is removed twice and sn6 is added twice
self.assertEqual(sorted(pt.addNode(sn6)), [
(0, uuid1, CellStates.DISCARDED),
(0, uuid6, CellStates.OUT_OF_DATE),
(1, uuid1, CellStates.DISCARDED),
(1, uuid6, CellStates.OUT_OF_DATE)])
for x in xrange(num_partitions):
self.assertEqual(len(pt.getCellList(x)), 3)
# there is no feeding cell, marked as feeding
pt.clear()
pt.setCell(0, sn1, CellStates.UP_TO_DATE)
pt.setCell(0, sn2, CellStates.UP_TO_DATE)
pt.setCell(1, sn1, CellStates.UP_TO_DATE)
pt.setCell(1, sn3, CellStates.UP_TO_DATE)
pt.setCell(2, sn1, CellStates.UP_TO_DATE)
pt.setCell(2, sn4, CellStates.UP_TO_DATE)
pt.setCell(3, sn1, CellStates.UP_TO_DATE)
pt.setCell(3, sn5, CellStates.UP_TO_DATE)
# sn1 is removed twice and sn6 is added twice
self.assertEqual(sorted(pt.addNode(sn6)), [
(0, uuid1, CellStates.FEEDING),
(0, uuid6, CellStates.OUT_OF_DATE),
(1, uuid1, CellStates.FEEDING),
(1, uuid6, CellStates.OUT_OF_DATE)])
pt.setUpToDate(sn6, 0)
pt.setUpToDate(sn6, 1)
for x in xrange(num_partitions):
self.assertEqual(len(pt.getCellList(x)), 2)
pt = PartitionTable(12, 0)
node_count = 0
for node in sn1, sn2, sn3, sn4:
pt.addNode(node)
node_count += 1
self.assertEqual(pt.count_dict.values(),
[12/node_count] * node_count)
def test_15_dropNode(self): def test_15_dropNode(self):
num_partitions = 4 num_partitions = 4
num_replicas = 2 num_replicas = 2
...@@ -376,108 +253,100 @@ class MasterPartitionTableTests(NeoUnitTestBase): ...@@ -376,108 +253,100 @@ class MasterPartitionTableTests(NeoUnitTestBase):
self.assertTrue(pt.filled()) self.assertTrue(pt.filled())
self.assertTrue(pt.operational()) self.assertTrue(pt.operational())
def test_17_tweak(self): def _pt_states(self, pt):
# remove broken node node_dict = defaultdict(list)
# remove if too many feeding nodes for offset, row in enumerate(pt.partition_list):
# remove feeding if all cells are up to date for cell in row:
# if too many cells, remove most used cell state_list = node_dict[cell.getNode()]
# if not enought cell, add least used node if state_list:
self.assertTrue(state_list[-1][0] < offset)
state_list.append((offset, str(cell.getState())[0]))
return map(dict, sorted(node_dict.itervalues()))
# create nodes def checkPT(self, pt, exclude_empty=False):
uuid1 = self.getStorageUUID() new_pt = PartitionTable(pt.np, pt.nr)
server1 = ("127.0.0.1", 19001) new_pt.make(pt.getNodeList() if exclude_empty else pt.count_dict)
sn1 = StorageNode(Mock(), server1, uuid1, NodeStates.RUNNING) self.assertEqual(self._pt_states(pt), self._pt_states(new_pt))
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19002) def update(self, pt, change_list=None):
sn2 = StorageNode(Mock(), server2, uuid2, NodeStates.RUNNING) if change_list is None:
uuid3 = self.getStorageUUID() for offset, row in enumerate(pt.partition_list):
server3 = ("127.0.0.3", 19003) for cell in list(row):
sn3 = StorageNode(Mock(), server3, uuid3, NodeStates.RUNNING) if cell.isOutOfDate():
uuid4 = self.getStorageUUID() pt.setUpToDate(cell.getNode(), offset)
server4 = ("127.0.0.4", 19004) else:
sn4 = StorageNode(Mock(), server4, uuid4, NodeStates.RUNNING) node_dict = dict((x.getUUID(), x) for x in pt.count_dict)
uuid5 = self.getStorageUUID() for offset, uuid, state in change_list:
server5 = ("127.0.0.5", 19005) if state is CellStates.OUT_OF_DATE:
sn5 = StorageNode(Mock(), server5, uuid5, NodeStates.RUNNING) pt.setUpToDate(node_dict[uuid], offset)
# create partition table
# 0 : sn1(discarded), sn2(up), -> sn2 must remain def tweak(self, pt, drop_list=()):
# 1 : sn1(feeding), sn2(feeding), sn3(up) -> one feeding and sn3 must remain change_list = pt.tweak(drop_list)
# 2 : sn1(feeding), sn2(up), sn3(up) -> sn2 and sn3 must remain, feeding must go away self.assertFalse(pt.tweak(drop_list))
# 3 : sn1(up), sn2(up), sn3(up), sn4(up) -> only 3 cell must remain return change_list
# 4 : sn1(up), sn5(up) -> one more cell must be added
num_partitions = 5 def test_17_tweak(self):
num_replicas = 2 sn = [StorageNode(Mock(), None, i + 1, NodeStates.RUNNING)
pt = PartitionTable(num_partitions, num_replicas) for i in xrange(5)]
pt = PartitionTable(5, 2)
# part 0 # part 0
pt.setCell(0, sn1, CellStates.DISCARDED) pt.setCell(0, sn[0], CellStates.DISCARDED)
pt.setCell(0, sn2, CellStates.UP_TO_DATE) pt.setCell(0, sn[1], CellStates.UP_TO_DATE)
# part 1 # part 1
pt.setCell(1, sn1, CellStates.FEEDING) pt.setCell(1, sn[0], CellStates.FEEDING)
pt.setCell(1, sn2, CellStates.FEEDING) pt.setCell(1, sn[1], CellStates.FEEDING)
pt.setCell(1, sn3, CellStates.OUT_OF_DATE) pt.setCell(1, sn[2], CellStates.OUT_OF_DATE)
# part 2 # part 2
pt.setCell(2, sn1, CellStates.FEEDING) pt.setCell(2, sn[0], CellStates.FEEDING)
pt.setCell(2, sn2, CellStates.UP_TO_DATE) pt.setCell(2, sn[1], CellStates.UP_TO_DATE)
pt.setCell(2, sn3, CellStates.UP_TO_DATE) pt.setCell(2, sn[2], CellStates.UP_TO_DATE)
# part 3 # part 3
pt.setCell(3, sn1, CellStates.UP_TO_DATE) pt.setCell(3, sn[0], CellStates.UP_TO_DATE)
pt.setCell(3, sn2, CellStates.UP_TO_DATE) pt.setCell(3, sn[1], CellStates.UP_TO_DATE)
pt.setCell(3, sn3, CellStates.UP_TO_DATE) pt.setCell(3, sn[2], CellStates.UP_TO_DATE)
pt.setCell(3, sn4, CellStates.UP_TO_DATE) pt.setCell(3, sn[3], CellStates.UP_TO_DATE)
# part 4 # part 4
pt.setCell(4, sn1, CellStates.UP_TO_DATE) pt.setCell(4, sn[0], CellStates.UP_TO_DATE)
pt.setCell(4, sn5, CellStates.UP_TO_DATE) pt.setCell(4, sn[4], CellStates.UP_TO_DATE)
# now tweak the table
pt.tweak()
# check part 1
cells = pt.getCellList(0)
self.assertEqual(len(cells), 3)
for cell in cells:
self.assertNotEqual(cell.getState(), CellStates.DISCARDED)
if cell.getNode() == sn2:
self.assertEqual(cell.getState(), CellStates.UP_TO_DATE)
else:
self.assertEqual(cell.getState(), CellStates.OUT_OF_DATE)
self.assertTrue(sn2 in [x.getNode() for x in cells])
# check part 2 count_dict = defaultdict(int)
cells = pt.getCellList(1) change_list = self.tweak(pt)
self.assertEqual(len(cells), 4) for offset, uuid, state in change_list:
for cell in cells: count_dict[state] += 1
if cell.getNode() == sn1: self.assertEqual(count_dict, {CellStates.DISCARDED: 3,
self.assertEqual(cell.getState(), CellStates.FEEDING) CellStates.OUT_OF_DATE: 5,
else: CellStates.UP_TO_DATE: 3})
self.assertEqual(cell.getState(), CellStates.OUT_OF_DATE) self.update(pt, change_list)
self.assertTrue(sn3 in [x.getNode() for x in cells]) self.checkPT(pt)
self.assertTrue(sn1 in [x.getNode() for x in cells])
# check part 3 for offset in pt.getAssignedPartitionList(sn[1].getUUID()):
cells = pt.getCellList(2) pt.removeCell(offset, sn[1])
self.assertEqual(len(cells), 3) change_list = self.tweak(pt)
for cell in cells: self.assertEqual(3, len(change_list))
if cell.getNode() in (sn2, sn3): self.update(pt, change_list)
self.assertEqual(cell.getState(), CellStates.UP_TO_DATE) self.checkPT(pt)
else:
self.assertEqual(cell.getState(), CellStates.OUT_OF_DATE)
self.assertTrue(sn3 in [x.getNode() for x in cells])
self.assertTrue(sn2 in [x.getNode() for x in cells])
# check part 4 for np, i in (12, 0), (12, 1), (13, 2):
cells = pt.getCellList(3) pt = PartitionTable(np, i)
self.assertEqual(len(cells), 3) i += 1
for cell in cells: pt.make(sn[:i])
self.assertEqual(cell.getState(), CellStates.UP_TO_DATE) for n in sn[i:i+3]:
self.assertEqual([n], pt.addNodeList([n]))
self.update(pt, self.tweak(pt))
self.checkPT(pt)
pt.clear()
pt.make(sn[:i])
for n in sn[i:i+3]:
self.assertEqual([n], pt.addNodeList([n]))
self.tweak(pt)
self.update(pt)
self.checkPT(pt)
# check part 5 pt = PartitionTable(7, 0)
cells = pt.getCellList(4) pt.make(sn[:1])
self.assertEqual(len(cells), 3) pt.addNodeList(sn[1:3])
for cell in cells: self.update(pt, self.tweak(pt, sn[:1]))
if cell.getNode() in (sn1, sn5): self.checkPT(pt, True)
self.assertEqual(cell.getState(), CellStates.UP_TO_DATE)
else:
self.assertEqual(cell.getState(), CellStates.OUT_OF_DATE)
self.assertTrue(sn1 in [x.getNode() for x in cells])
self.assertTrue(sn5 in [x.getNode() for x in cells])
if __name__ == '__main__': if __name__ == '__main__':
......
neo/tests/threaded/testReplication.py
View file @ a1539219
...@@ -184,13 +184,12 @@ class ReplicationTests(NEOThreadedTest): ...@@ -184,13 +184,12 @@ class ReplicationTests(NEOThreadedTest):
another source. another source.
Here are the different states of partitions over time: Here are the different states of partitions over time:
pt: 0: U|U|U pt: 0: U|U|U
pt: 0: UO|UO|UO pt: 0: UO.|U.O|FOO
pt: 0: FOO|UO.|U.O # node 1 replicates from node 0 pt: 0: UU.|U.O|FOO
pt: 0: .OU|UO.|U.O # here node 0 lost partition 0 pt: 0: UU.|U.U|FOO # nodes 1 & 2 replicate from node 0
pt: 0: UU.|U.U|.OU # here node 0 lost partition 2
# and node 1 must switch to node 2 # and node 1 must switch to node 2
pt: 0: .OU|UO.|U.U pt: 0: UU.|U.U|.UU
pt: 0: .OU|UU.|U.U
pt: 0: .UU|UU.|U.U
""" """
def connected(orig, *args, **kw): def connected(orig, *args, **kw):
patch[0] = s1.filterConnection(s0) patch[0] = s1.filterConnection(s0)
...@@ -218,6 +217,7 @@ class ReplicationTests(NEOThreadedTest): ...@@ -218,6 +217,7 @@ class ReplicationTests(NEOThreadedTest):
s2.start() s2.start()
cluster.tic() cluster.tic()
cluster.neoctl.enableStorageList([s1.uuid, s2.uuid]) cluster.neoctl.enableStorageList([s1.uuid, s2.uuid])
cluster.neoctl.tweakPartitionTable()
offset, = [offset for offset, row in enumerate( offset, = [offset for offset, row in enumerate(
cluster.master.pt.partition_list) cluster.master.pt.partition_list)
for cell in row if cell.isFeeding()] for cell in row if cell.isFeeding()]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7