Commit 23fad3af authored by Julien Muchembled's avatar Julien Muchembled

Reduce size of UUIDs to 4-bytes

parent 6bc3318d
......@@ -31,6 +31,8 @@ Other changes are:
- Adding and removing master nodes is now easier: unknown incoming master nodes
are now accepted instead of rejected, and nodes can be given a path to a file
that maintains a list of known master nodes.
- Node UUIDs have been shortened from 16 to 4 bytes, for better performance and
easier debugging.
Also contains code clean-ups and bugfixes.
......
......@@ -30,11 +30,6 @@ RC - Review output of pylint (CODE)
Consider the need to implement a keep-alive system (packets sent
automatically when there is no activity on the connection for a period
of time).
- Factorise packet data when sending partition table cells (BANDWITH)
Currently, each cell in a partition table update contains UUIDs of all
involved nodes.
It must be changed to a correspondance table using shorter keys (sent
in the packet) to avoid repeating the same UUIDs many times.
- Consider using multicast for cluster-wide notifications. (BANDWITH)
Currently, multi-receivers notifications are sent in unicast to each
receiver. Multicast should be used.
......
......@@ -6,6 +6,7 @@ compatibility or transparent migration, so you will have to use the following
SQL commands to migrate each storage from NEO 0.10.x::
-- make sure 'tobj' & 'ttrans' are empty first
- and all storages have up-to-date partition tables
CREATE TABLE new_data (id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, hash BINARY(20) NOT NULL UNIQUE, compression TINYINT UNSIGNED NULL, value LONGBLOB NULL) ENGINE = InnoDB SELECT DISTINCT obj.hash as hash, compression, value FROM obj, data WHERE obj.hash=data.hash ORDER BY serial;
DROP TABLE data;
RENAME TABLE new_data TO data;
......@@ -17,6 +18,12 @@ SQL commands to migrate each storage from NEO 0.10.x::
UPDATE trans SET ttid=tid;
ALTER TABLE ttrans ADD COLUMN ttid BIGINT UNSIGNED NOT NULL;
CREATE TEMPORARY TABLE uuid (new INT NOT NULL AUTO_INCREMENT PRIMARY KEY, old CHAR(32) NOT NULL, KEY (old)) ENGINE = InnoDB SELECT DISTINCT uuid as old FROM pt ORDER BY uuid;
ALTER TABLE pt DROP PRIMARY KEY, CHANGE uuid old CHAR(32) NOT NULL, ADD uuid INT NOT NULL after rid;
UPDATE pt, uuid SET pt.uuid=uuid.new WHERE pt.old=uuid.old;
ALTER TABLE pt DROP old, ADD PRIMARY KEY (rid, uuid);
UPDATE config, uuid SET config.value=uuid.new WHERE config.name='uuid' AND uuid.old=config.value;
NEO 0.10
========
......
......@@ -16,9 +16,8 @@
from neo.lib import logging, protocol
from neo.lib.handler import EventHandler
from neo.lib.protocol import Packets, Errors
from neo.lib.protocol import uuid_str, Packets, Errors
from neo.lib.exception import PrimaryFailure
from neo.lib.util import dump
def check_primary_master(func):
def wrapper(self, *args, **kw):
......@@ -38,7 +37,7 @@ class AdminEventHandler(EventHandler):
@check_primary_master
def askPartitionList(self, conn, min_offset, max_offset, uuid):
logging.info("ask partition list from %s to %s for %s",
min_offset, max_offset, dump(uuid))
min_offset, max_offset, uuid_str(uuid))
self.app.sendPartitionTable(conn, min_offset, max_offset, uuid)
@check_primary_master
......@@ -56,7 +55,7 @@ class AdminEventHandler(EventHandler):
@check_primary_master
def setNodeState(self, conn, uuid, state, modify_partition_table):
logging.info("set node state for %s-%s", dump(uuid), state)
logging.info("set node state for %s-%s", uuid_str(uuid), state)
node = self.app.nm.getByUUID(uuid)
if node is None:
raise protocol.ProtocolError('invalid uuid')
......
......@@ -18,8 +18,7 @@ from time import sleep
from . import logging
from .handler import EventHandler
from .protocol import Packets
from .util import dump
from .protocol import uuid_str, Packets
from .connection import ClientConnection
NO_SERVER = ('0.0.0.0', 0)
......@@ -115,7 +114,7 @@ class BootstrapManager(EventHandler):
if self.uuid != your_uuid:
# got an uuid from the primary master
self.uuid = your_uuid
logging.info('Got a new UUID: %s', dump(self.uuid))
logging.info('Got a new UUID: %s', uuid_str(self.uuid))
self.accepted = True
def getPrimaryConnection(self, connector_handler):
......
......@@ -84,7 +84,9 @@ class ConfigurationManager(object):
def getUUID(self):
# only from command line
return util.bin(self.argument_list.get('uuid', None))
uuid = self.argument_list.get('uuid', None)
if uuid:
return int(uuid)
def getUpstreamCluster(self):
return self.__get('upstream_cluster', True)
......
......@@ -23,8 +23,9 @@ from .connector import ConnectorException, ConnectorTryAgainException, \
ConnectorConnectionClosedException
from .locking import RLock
from .profiling import profiler_decorator
from .protocol import Errors, PacketMalformedError, Packets, ParserState
from .util import dump, ReadBuffer
from .protocol import uuid_str, Errors, \
PacketMalformedError, Packets, ParserState
from .util import ReadBuffer
CRITICAL_TIMEOUT = 30
......@@ -297,7 +298,7 @@ class BaseConnection(object):
address = self.addr and '%s:%d' % self.addr or '?'
return '<%s(uuid=%s, address=%s, closed=%s, handler=%s) at %x>' % (
self.__class__.__name__,
dump(self.getUUID()),
uuid_str(self.getUUID()),
address,
int(self.isClosed()),
self.getHandler(),
......
......@@ -21,7 +21,6 @@
# Fortunately, SQLite allow multiple process to access the same DB,
# so an external tool should be able to dump and empty tables.
from binascii import b2a_hex
from collections import deque
from functools import wraps
from logging import getLogger, Formatter, Logger, LogRecord, StreamHandler, \
......@@ -117,6 +116,9 @@ class NEOLogger(Logger):
def setup(self, filename=None, reset=False):
self._acquire()
try:
from . import protocol as p
global uuid_str
uuid_str = p.uuid_str
if self.db is not None:
self.db.close()
if not filename:
......@@ -153,7 +155,6 @@ class NEOLogger(Logger):
date REAL PRIMARY KEY NOT NULL,
text BLOB NOT NULL)
""")
from . import protocol as p
with open(inspect.getsourcefile(p)) as p:
p = buffer(bz2.compress(p.read()))
for t, in q("SELECT text FROM protocol ORDER BY date DESC"):
......@@ -172,7 +173,7 @@ class NEOLogger(Logger):
if type(r) is PacketRecord:
ip, port = r.addr
peer = '%s %s (%s:%u)' % ('>' if r.outgoing else '<',
r.uuid and b2a_hex(r.uuid), ip, port)
uuid_str(r.uuid), ip, port)
self.db.execute("INSERT INTO packet VALUES (?,?,?,?,?,?)",
(r.created, r._name, r.msg_id, r.code, peer, buffer(r.msg)))
else:
......
......@@ -19,8 +19,7 @@ from os.path import exists, getsize
import json
from . import attributeTracker, logging
from .util import dump
from .protocol import NodeTypes, NodeStates, ProtocolError
from .protocol import uuid_str, NodeTypes, NodeStates, ProtocolError
class Node(object):
......@@ -166,7 +165,7 @@ class Node(object):
def __repr__(self):
return '<%s(uuid=%s, address=%s, state=%s, connection=%r) at %x>' % (
self.__class__.__name__,
dump(self._uuid),
uuid_str(self._uuid),
self._address,
self._state,
self._connection,
......@@ -511,16 +510,16 @@ class NodeManager(object):
elif by_address is None:
node = by_uuid
else:
raise ValueError('Got different nodes for uuid %r: %r and '
'address %r: %r.' % (dump(uuid), by_uuid, address,
raise ValueError('Got different nodes for uuid %s: %r and '
'address %r: %r.' % (uuid_str(uuid), by_uuid, address,
by_address))
if uuid is not None:
node_uuid = node.getUUID()
if node_uuid is None:
node.setUUID(uuid)
elif node_uuid != uuid:
raise ValueError('Expected uuid %r on node %r' % (
dump(uuid), node))
raise ValueError('Expected uuid %s on node %r' % (
uuid_str(uuid), node))
if address is not None:
node_address = node.getAddress()
if node_address is None:
......@@ -574,7 +573,7 @@ class NodeManager(object):
node_by_addr = self.getByAddress(addr)
node = node_by_uuid or node_by_addr
log_args = (node_type, dump(uuid), addr, state)
log_args = node_type, uuid_str(uuid), addr, state
if node is None:
if state == NodeStates.DOWN:
logging.debug('NOT creating node %s %s %s %s', *log_args)
......@@ -606,10 +605,12 @@ class NodeManager(object):
def log(self):
logging.info('Node manager : %u nodes', len(self._node_set))
for node in sorted(list(self._node_set)):
uuid = dump(node.getUUID()) or '-' * 32
node_list = [(node, uuid_str(node.getUUID()))
for node in sorted(self._node_set)]
max_len = max(len(x[1]) for x in node_list)
for node, uuid in node_list:
address = node.getAddress() or ''
if address:
address = '%s:%d' % address
logging.info(' * %32s | %8s | %22s | %s',
uuid, node.getType(), address, node.getState())
logging.info(' * %*s | %8s | %22s | %s',
max_len, uuid, node.getType(), address, node.getState())
......@@ -26,7 +26,7 @@ except ImportError:
pass
# The protocol version (major, minor).
PROTOCOL_VERSION = (9, 1)
PROTOCOL_VERSION = (10, 1)
# Size restrictions.
MIN_PACKET_SIZE = 10
......@@ -154,7 +154,7 @@ cell_state_prefix_dict = {
}
# Other constants.
INVALID_UUID = '\0' * 16
INVALID_UUID = 0
INVALID_TID = '\xff' * 8
INVALID_OID = '\xff' * 8
INVALID_PARTITION = 0xffffffff
......@@ -166,12 +166,24 @@ OID_LEN = len(INVALID_OID)
TID_LEN = len(INVALID_TID)
MAX_TID = '\x7f' + '\xff' * 7 # SQLite does not accept numbers above 2^63-1
# High-order byte:
# 7 6 5 4 3 2 1 0
# | | | | +-+-+-+-- reserved (0)
# | +-+-+---------- node type
# +---------------- temporary if negative
# UUID namespaces are required to prevent conflicts when the master generate
# new uuid before it knows uuid of existing storage nodes. So only the high
# order bit is really important and the 31 other bits could be random.
# Extra namespace information and non-randomness of 3 LOB help to read logs.
UUID_NAMESPACES = {
NodeTypes.STORAGE: 'S',
NodeTypes.MASTER: 'M',
NodeTypes.CLIENT: 'C',
NodeTypes.ADMIN: 'A',
NodeTypes.STORAGE: 0x00,
NodeTypes.MASTER: -0x10,
NodeTypes.CLIENT: -0x20,
NodeTypes.ADMIN: -0x30,
}
uuid_str = (lambda ns: lambda uuid:
ns[uuid >> 24] + str(uuid & 0xffffff) if uuid else str(uuid)
)(dict((v, str(k)[0]) for k, v in UUID_NAMESPACES.iteritems()))
class ProtocolError(Exception):
""" Base class for protocol errors, close the connection """
......@@ -586,21 +598,18 @@ class PChecksum(PItem):
def _decode(self, reader):
return reader(20)
class PUUID(PItem):
class PUUID(PStructItem):
"""
An UUID (node identifier)
An UUID (node identifier, 4-bytes signed integer)
"""
def __init__(self, name):
PStructItem.__init__(self, name, '!l')
def _encode(self, writer, uuid):
if uuid is None:
uuid = INVALID_UUID
assert len(uuid) == 16, (len(uuid), uuid)
writer(uuid)
writer(self.pack(uuid or 0))
def _decode(self, reader):
uuid = reader(16)
if uuid == INVALID_UUID:
uuid = None
return uuid
return self.unpack(reader(self.size))[0] or None
class PTID(PItem):
"""
......
......@@ -18,8 +18,8 @@ import math
from functools import wraps
from . import logging, protocol
from .protocol import CellStates
from .util import dump, u64
from .protocol import uuid_str, CellStates
from .util import u64
from .locking import RLock
class PartitionTableException(Exception):
......@@ -36,7 +36,7 @@ class Cell(object):
def __repr__(self):
return "<Cell(uuid=%s, address=%s, state=%s)>" % (
dump(self.getUUID()),
uuid_str(self.getUUID()),
self.getAddress(),
self.getState(),
)
......@@ -226,7 +226,7 @@ class PartitionTable(object):
self._id = ptid
for offset, uuid, state in cell_list:
node = nm.getByUUID(uuid)
assert node is not None, 'No node found for uuid %r' % (dump(uuid), )
assert node is not None, 'No node found for uuid ' + uuid_str(uuid)
self.setCell(offset, node, state)
logging.debug('partition table updated (ptid=%s)', ptid)
self.log()
......@@ -260,7 +260,7 @@ class PartitionTable(object):
width under 80 column).
"""
node_list = sorted(self.count_dict)
result = ['pt: node %u: %s, %s' % (i, dump(node.getUUID()),
result = ['pt: node %u: %s, %s' % (i, uuid_str(node.getUUID()),
protocol.node_state_prefix_dict[node.getState()])
for i, node in enumerate(node_list)]
append = result.append
......
......@@ -20,7 +20,7 @@ from time import time
from neo.lib import logging
from neo.lib.connector import getConnectorHandler
from neo.lib.debug import register as registerLiveDebugger
from neo.lib.protocol import UUID_NAMESPACES, ZERO_TID, NotReadyError
from neo.lib.protocol import uuid_str, UUID_NAMESPACES, ZERO_TID, NotReadyError
from neo.lib.protocol import ClusterStates, NodeStates, NodeTypes, Packets
from neo.lib.node import NodeManager
from neo.lib.event import EventManager
......@@ -335,7 +335,7 @@ class Application(object):
if self.uuid is None:
self.uuid = self.getNewUUID(None, self.server, NodeTypes.MASTER)
logging.info('My UUID: ' + dump(self.uuid))
logging.info('My UUID: ' + uuid_str(self.uuid))
else:
in_conflict = self.nm.getByUUID(self.uuid)
if in_conflict is not None:
......@@ -443,14 +443,16 @@ class Application(object):
self.cluster_state = state
def getNewUUID(self, uuid, address, node_type):
getByUUID = self.nm.getByUUID
if None != uuid != self.uuid:
node = self.nm.getByUUID(uuid)
node = getByUUID(uuid)
if node is None or node.getAddress() == address:
return uuid
while True:
uuid = UUID_NAMESPACES[node_type] + os.urandom(15)
if uuid != self.uuid and self.nm.getByUUID(uuid) is None:
hob = UUID_NAMESPACES[node_type]
for uuid in xrange((hob << 24) + 1, hob + 0x10 << 24):
if uuid != self.uuid and getByUUID(uuid) is None:
return uuid
raise RuntimeError
def getClusterState(self):
return self.cluster_state
......
......@@ -22,7 +22,7 @@ from neo.lib.connector import getConnectorHandler
from neo.lib.exception import PrimaryFailure
from neo.lib.node import NodeManager
from neo.lib.protocol import CellStates, ClusterStates, NodeTypes, Packets
from neo.lib.protocol import INVALID_TID, ZERO_TID
from neo.lib.protocol import uuid_str, INVALID_TID, ZERO_TID
from neo.lib.util import add64, dump
from .app import StateChangedException
from .pt import PartitionTable
......@@ -154,9 +154,9 @@ class BackupApplication(object):
cell.replicating = tid
if cell.backup_tid < tid:
logging.debug(
"ask %s to replicate partition %u up to %s from %r",
dump(cell.getUUID()), offset, dump(tid),
dump(primary_node.getUUID()))
"ask %s to replicate partition %u up to %s from %s",
uuid_str(cell.getUUID()), offset, dump(tid),
uuid_str(primary_node.getUUID()))
cell.getNode().getConnection().notify(p)
trigger_set.add(primary_node)
for node in trigger_set:
......@@ -238,7 +238,7 @@ class BackupApplication(object):
address_set.add(addr)
source_dict[offset] = addr
logging.debug("ask %s to replicate partition %u up to %s from %r",
dump(node.getUUID()), offset, dump(tid), addr)
uuid_str(node.getUUID()), offset, dump(tid), addr)
node.getConnection().notify(Packets.Replicate(
tid, self.name, source_dict))
......@@ -272,9 +272,9 @@ class BackupApplication(object):
if tid < max_tid:
cell.replicating = max_tid
logging.debug(
"ask %s to replicate partition %u up to %s from %r",
dump(node.getUUID()), offset, dump(max_tid),
dump(primary_node.getUUID()))
"ask %s to replicate partition %u up to %s from %s",
uuid_str(node.getUUID()), offset, dump(max_tid),
uuid_str(primary_node.getUUID()))
node.getConnection().notify(Packets.Replicate(max_tid,
'', {offset: primary_node.getAddress()}))
else:
......@@ -288,7 +288,7 @@ class BackupApplication(object):
cell.replicating = tid
logging.debug(
"ask %s to replicate partition %u up to %s from"
" %r", dump(cell.getUUID()), offset, dump(tid),
dump(node.getUUID()))
" %s", uuid_str(cell.getUUID()), offset,
dump(tid), uuid_str(node.getUUID()))
cell.getNode().getConnection().notify(p)
return result
......@@ -16,10 +16,9 @@
from neo.lib import logging
from neo.lib.handler import EventHandler
from neo.lib.protocol import (NodeTypes, NodeStates, Packets,
from neo.lib.protocol import (uuid_str, NodeTypes, NodeStates, Packets,
BrokenNodeDisallowedError,
)
from neo.lib.util import dump
class MasterHandler(EventHandler):
"""This class implements a generic part of the event handlers."""
......@@ -100,7 +99,7 @@ class BaseServiceHandler(MasterHandler):
assert new_state in (NodeStates.TEMPORARILY_DOWN, NodeStates.DOWN,
NodeStates.BROKEN), new_state
assert node.getState() not in (NodeStates.TEMPORARILY_DOWN,
NodeStates.DOWN, NodeStates.BROKEN), (dump(self.app.uuid),
NodeStates.DOWN, NodeStates.BROKEN), (uuid_str(self.app.uuid),
node.whoSetState(), new_state)
was_pending = node.isPending()
node.setState(new_state)
......
......@@ -20,7 +20,7 @@ from . import MasterHandler
from ..app import StateChangedException
from neo.lib import logging
from neo.lib.protocol import ClusterStates, NodeStates, Packets, ProtocolError
from neo.lib.protocol import Errors
from neo.lib.protocol import Errors, uuid_str
from neo.lib.util import dump
CLUSTER_STATE_WORKFLOW = {
......@@ -73,7 +73,7 @@ class AdministrationHandler(MasterHandler):
def setNodeState(self, conn, uuid, state, modify_partition_table):
logging.info("set node state for %s-%s : %s",
dump(uuid), state, modify_partition_table)
uuid_str(uuid), state, modify_partition_table)
app = self.app
node = app.nm.getByUUID(uuid)
if node is None:
......@@ -127,7 +127,7 @@ class AdministrationHandler(MasterHandler):
app.broadcastNodesInformation([node])
def addPendingNodes(self, conn, uuid_list):
uuids = ', '.join(map(dump, uuid_list))
uuids = ', '.join(map(uuid_str, uuid_list))
logging.debug('Add nodes %s', uuids)
app = self.app
nm = app.nm
......@@ -148,7 +148,7 @@ class AdministrationHandler(MasterHandler):
logging.warning('No nodes added')
conn.answer(Errors.Ack('No nodes added'))
return
uuids = ', '.join(map(dump, uuid_set))
uuids = ', '.join(map(uuid_str, uuid_set))
logging.info('Adding nodes %s', uuids)
# switch nodes to running state
node_list = map(nm.getByUUID, uuid_set)
......
......@@ -15,12 +15,11 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from neo.lib import logging
from neo.lib.protocol import NodeTypes, NodeStates, Packets
from neo.lib.protocol import uuid_str, NodeTypes, NodeStates, Packets
from neo.lib.protocol import NotReadyError, ProtocolError, \
UnexpectedPacketError
from neo.lib.exception import ElectionFailure
from neo.lib.handler import EventHandler
from neo.lib.util import dump
from . import MasterHandler
class BaseElectionHandler(EventHandler):
......@@ -58,8 +57,8 @@ class ClientElectionHandler(BaseElectionHandler):
def connectionFailed(self, conn):
addr = conn.getAddress()
node = self.app.nm.getByAddress(addr)
assert node is not None, (dump(self.app.uuid), addr)
assert node.isUnknown(), (dump(self.app.uuid), node.whoSetState(),
assert node is not None, (uuid_str(self.app.uuid), addr)
assert node.isUnknown(), (uuid_str(self.app.uuid), node.whoSetState(),
node)
# connection never success, node is still in unknown state
self.app.negotiating_master_node_set.discard(addr)
......@@ -92,7 +91,7 @@ class ClientElectionHandler(BaseElectionHandler):
if app.server == address:
# This is self.
assert node.getAddress() != primary or uuid == your_uuid, (
dump(uuid), dump(your_uuid))
uuid_str(uuid), uuid_str(your_uuid))
continue
n = app.nm.getByAddress(address)
if n is None:
......
......@@ -16,9 +16,8 @@
import neo
from neo.lib import logging
from neo.lib.util import dump
from neo.lib.protocol import ClusterStates, NodeStates, NodeTypes, Packets, \
NotReadyError, ProtocolError
NotReadyError, ProtocolError, uuid_str
from . import MasterHandler
class IdentificationHandler(MasterHandler):
......@@ -68,7 +67,7 @@ class IdentificationHandler(MasterHandler):
raise NotImplementedError(node_type)
uuid = app.getNewUUID(uuid, address, node_type)
logging.info('Accept a' + human_readable_node_type + dump(uuid))
logging.info('Accept a' + human_readable_node_type + uuid_str(uuid))
if node is None:
node = node_ctor(uuid=uuid, address=address)
node.setUUID(uuid)
......
......@@ -17,9 +17,8 @@
from . import MasterHandler
from neo.lib.handler import EventHandler
from neo.lib.exception import ElectionFailure, PrimaryFailure
from neo.lib.protocol import NodeTypes, Packets
from neo.lib.protocol import NodeTypes, Packets, uuid_str
from neo.lib import logging
from neo.lib.util import dump
class SecondaryMasterHandler(MasterHandler):
""" Handler used by primary to handle secondary masters"""
......@@ -97,7 +96,7 @@ class PrimaryHandler(EventHandler):
if your_uuid != app.uuid:
app.uuid = your_uuid
logging.info('My UUID: ' + dump(your_uuid))
logging.info('My UUID: ' + uuid_str(your_uuid))
node.setUUID(uuid)
......@@ -19,6 +19,7 @@ from struct import pack, unpack
from neo.lib.protocol import ZERO_TID
from datetime import timedelta, datetime
from neo.lib import logging
from neo.lib.protocol import uuid_str
from neo.lib.util import dump, u64, p64
TID_LOW_OVERFLOW = 2**32
......@@ -118,7 +119,7 @@ class Transaction(object):
self._node,
dump(self._tid),
map(dump, self._oid_list or ()),
map(dump, self._uuid_set or ()),
map(uuid_str, self._uuid_set or ()),
time() - self._birth,
id(self),
)
......@@ -417,7 +418,7 @@ class TransactionManager(object):
If transaction is completely locked, calls function given at
instanciation time.
"""
logging.debug('Lock TXN %s for %s', dump(ttid), dump(uuid))
logging.debug('Lock TXN %s for %s', dump(ttid), uuid_str(uuid))
assert ttid in self._ttid_dict, "Transaction not started"
txn = self._ttid_dict[ttid]
if txn.lock(uuid) and self._queue[0][1] == ttid:
......
......@@ -15,8 +15,9 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from .neoctl import NeoCTL, NotReadyException
from neo.lib.util import bin, dump, p64
from neo.lib.protocol import ClusterStates, NodeStates, NodeTypes, ZERO_TID
from neo.lib.util import bin, p64
from neo.lib.protocol import uuid_str, ClusterStates, NodeStates, NodeTypes, \
ZERO_TID
action_dict = {
'print': {
......@@ -57,7 +58,7 @@ class TerminalNeoCTL(object):
def formatRowList(self, row_list):
return '\n'.join('%03d | %s' % (offset,
''.join('%s - %s |' % (dump(uuid), state)
''.join('%s - %s |' % (uuid_str(uuid), state)
for (uuid, state) in cell_list))
for (offset, cell_list) in row_list)
......@@ -69,13 +70,10 @@ class TerminalNeoCTL(object):
if address is None:
address = (None, None)
ip, port = address
result.append('%s - %s - %s:%s - %s' % (node_type, dump(uuid), ip,
port, state))
result.append('%s - %s - %s:%s - %s' % (node_type, uuid_str(uuid),
ip, port, state))
return '\n'.join(result)
def formatUUID(self, uuid):
return dump(uuid)
# Actual actions
def getPartitionRowList(self, params):
"""
......@@ -185,7 +183,7 @@ class TerminalNeoCTL(object):
"""
Get primary master node.
"""
return self.formatUUID(self.neoctl.getPrimary())
return uuid_str(self.neoctl.getPrimary())
def checkReplicas(self, params):
"""
......
......@@ -50,8 +50,8 @@ class main(object):
g = {}
exec bz2.decompress(*q("SELECT text FROM protocol WHERE date<?"
" ORDER BY date DESC", (date,)).next()) in g
self.Packets = g['Packets']
self.PacketMalformedError = g['PacketMalformedError']
for x in 'uuid_str', 'Packets', 'PacketMalformedError':
setattr(self, x, g[x])
try:
self._next_protocol, = q("SELECT date FROM protocol WHERE date>=?",
(date,)).next()
......@@ -96,11 +96,17 @@ class main(object):
return "%s (%s)" % (code, message),
def notifyNodeInformation(self, node_list):
for node_type, address, uuid, state in node_list:
address = '%s:%u' % address if address else '?'
if uuid is not None:
uuid = b2a_hex(uuid)
yield ' ! %s | %8s | %22s | %s' % (uuid, node_type, address, state)
node_list.sort(key=lambda x: x[2])
node_list = [(self.uuid_str(uuid), str(node_type),
'%s:%u' % address if address else '?', state)
for node_type, address, uuid, state in node_list]
if node_list:
t = ' ! %%%us | %%%us | %%%us | %%s' % (
max(len(x[0]) for x in node_list),
max(len(x[1]) for x in node_list),
max(len(x[2]) for x in node_list))
return map(t.__mod__, node_list)
return ()
if __name__ == "__main__":
......
......@@ -18,7 +18,7 @@ import sys
from collections import deque
from neo.lib import logging
from neo.lib.protocol import NodeTypes, CellStates, Packets
from neo.lib.protocol import uuid_str, CellStates, NodeTypes, Packets
from neo.lib.node import NodeManager
from neo.lib.event import EventManager
from neo.lib.connection import ListeningConnection
......@@ -138,7 +138,7 @@ class Application(object):
self.pt = PartitionTable(num_partitions, num_replicas)
logging.info('Configuration loaded:')
logging.info('UUID : %s', dump(self.uuid))
logging.info('UUID : %s', uuid_str(self.uuid))
logging.info('PTID : %s', dump(ptid))
logging.info('Name : %s', self.name)
logging.info('Partitions: %s', num_partitions)
......@@ -233,7 +233,7 @@ class Application(object):
(node, conn, uuid, num_partitions, num_replicas) = data
self.master_node = node
self.master_conn = conn
logging.info('I am %s', dump(uuid))
logging.info('I am %s', uuid_str(uuid))
self.uuid = uuid
self.dm.setUUID(uuid)
......
......@@ -104,13 +104,15 @@ class DatabaseManager(object):
"""
Load an UUID from a database.
"""
return util.bin(self.getConfiguration('uuid'))
uuid = self.getConfiguration('uuid')
if uuid is not None:
return int(uuid)
def setUUID(self, uuid):
"""
Store an UUID into a database.
"""
self.setConfiguration('uuid', util.dump(uuid))
self.setConfiguration('uuid', str(uuid))
def getNumPartitions(self):
"""
......@@ -188,7 +190,7 @@ class DatabaseManager(object):
return self.setConfiguration('backup_tid', util.dump(backup_tid))
def getPartitionTable(self):
"""Return a whole partition table as a tuple of rows. Each row
"""Return a whole partition table as a sequence of rows. Each row
is again a tuple of an offset (row ID), an UUID of a storage
node, and a cell state."""
raise NotImplementedError
......
......@@ -165,7 +165,7 @@ class MySQLDatabaseManager(DatabaseManager):
# The table "pt" stores a partition table.
q("""CREATE TABLE IF NOT EXISTS pt (
rid INT UNSIGNED NOT NULL,
uuid CHAR(32) NOT NULL,
uuid INT NOT NULL,
state TINYINT UNSIGNED NOT NULL,
PRIMARY KEY (rid, uuid)
) ENGINE = InnoDB""")
......@@ -267,13 +267,7 @@ class MySQLDatabaseManager(DatabaseManager):
return -1
def getPartitionTable(self):
q = self.query
cell_list = q("""SELECT rid, uuid, state FROM pt""")
pt = []
for offset, uuid, state in cell_list:
uuid = util.bin(uuid)
pt.append((offset, uuid, state))
return pt
return self.query("SELECT * FROM pt")
def _getLastTIDs(self, all=True):
p64 = util.p64
......@@ -340,21 +334,19 @@ class MySQLDatabaseManager(DatabaseManager):
return serial, next_serial, compression, checksum, data, value_serial
def doSetPartitionTable(self, ptid, cell_list, reset):
e = self.escape
offset_list = []
with self as q:
if reset:
q("""TRUNCATE pt""")
for offset, uuid, state in cell_list:
uuid = e(util.dump(uuid))
# TODO: this logic should move out of database manager
# add 'dropCells(cell_list)' to API and use one query
if state == CellStates.DISCARDED:
q("""DELETE FROM pt WHERE rid = %d AND uuid = '%s'""" \
q("""DELETE FROM pt WHERE rid = %d AND uuid = %d"""
% (offset, uuid))
else:
offset_list.append(offset)
q("""INSERT INTO pt VALUES (%d, '%s', %d)
q("""INSERT INTO pt VALUES (%d, %d, %d)
ON DUPLICATE KEY UPDATE state = %d""" \
% (offset, uuid, state, state))
self.setPTID(ptid)
......
......@@ -106,7 +106,7 @@ class SQLiteDatabaseManager(DatabaseManager):
# The table "pt" stores a partition table.
q("""CREATE TABLE IF NOT EXISTS pt (
rid INTEGER NOT NULL,
uuid BLOB NOT NULL,
uuid INTEGER NOT NULL,
state INTEGER NOT NULL,
PRIMARY KEY (rid, uuid))
""")
......@@ -203,9 +203,7 @@ class SQLiteDatabaseManager(DatabaseManager):
return -1
def getPartitionTable(self):
return [(offset, util.bin(uuid), state)
for offset, uuid, state in self.query(
"SELECT rid, uuid, state FROM pt")]
return self.query("SELECT * FROM pt")
def _getLastTIDs(self, all=True):
p64 = util.p64
......@@ -275,7 +273,6 @@ class SQLiteDatabaseManager(DatabaseManager):
if reset:
q("DELETE FROM pt")
for offset, uuid, state in cell_list:
uuid = buffer(util.dump(uuid))
# TODO: this logic should move out of database manager
# add 'dropCells(cell_list)' to API and use one query
# WKRD: Why does SQLite need a statement journal file
......
......@@ -16,9 +16,8 @@
from neo.lib import logging
from neo.lib.handler import EventHandler
from neo.lib.util import dump
from neo.lib.exception import PrimaryFailure, OperationFailure
from neo.lib.protocol import NodeStates, NodeTypes
from neo.lib.protocol import uuid_str, NodeStates, NodeTypes
class BaseMasterHandler(EventHandler):
......@@ -55,8 +54,8 @@ class BaseMasterHandler(EventHandler):
elif state == NodeStates.HIDDEN:
raise OperationFailure
elif node_type == NodeTypes.CLIENT and state != NodeStates.RUNNING:
logging.info('Notified of non-running client, abort (%r)',
dump(uuid))
logging.info('Notified of non-running client, abort (%s)',
uuid_str(uuid))
self.app.tm.abortFor(uuid)
def answerUnfinishedTransactions(self, conn, *args, **kw):
......
......@@ -16,9 +16,8 @@
from neo.lib import logging
from neo.lib.handler import EventHandler
from neo.lib.protocol import NodeTypes, Packets, NotReadyError
from neo.lib.protocol import uuid_str, NodeTypes, NotReadyError, Packets
from neo.lib.protocol import ProtocolError, BrokenNodeDisallowedError
from neo.lib.util import dump
from .storage import StorageOperationHandler
from .client import ClientOperationHandler
......@@ -60,7 +59,7 @@ class IdentificationHandler(EventHandler):
elif node_type == NodeTypes.STORAGE:
if node is None:
logging.error('reject an unknown storage node %s',
dump(uuid))
uuid_str(uuid))
raise NotReadyError
handler = StorageOperationHandler
else:
......
......@@ -17,7 +17,7 @@
from time import time
from neo.lib import logging
from neo.lib.util import dump
from neo.lib.protocol import ZERO_TID
from neo.lib.protocol import uuid_str, ZERO_TID
class ConflictError(Exception):
"""
......@@ -55,15 +55,14 @@ class Transaction(object):
self._checked_set = set()
def __repr__(self):
return "<%s(ttid=%r, tid=%r, uuid=%r, locked=%r, age=%.2fs)> at %x" % (
self.__class__.__name__,
return "<%s(ttid=%r, tid=%r, uuid=%r, locked=%r, age=%.2fs) at 0x%x>" \
% (self.__class__.__name__,
dump(self._ttid),
dump(self._tid),
dump(self._uuid),
uuid_str(self._uuid),
self.isLocked(),
time() - self._birth,
id(self),
)
id(self))
def addCheckedObject(self, oid):
assert oid not in self._object_dict, dump(oid)
......@@ -148,7 +147,7 @@ class TransactionManager(object):
"""
Register a transaction, it may be already registered
"""
logging.debug('Register TXN %s for %s', dump(ttid), dump(uuid))
logging.debug('Register TXN %s for %s', dump(ttid), uuid_str(uuid))
transaction = self._transaction_dict.get(ttid, None)
if transaction is None:
transaction = Transaction(uuid, ttid)
......@@ -353,7 +352,7 @@ class TransactionManager(object):
"""
Abort any non-locked transaction of a node
"""
logging.debug('Abort for %s', dump(uuid))
logging.debug('Abort for %s', uuid_str(uuid))
# abort any non-locked transaction of this node
for ttid in [x.getTTID() for x in self._uuid_dict.get(uuid, [])]:
self.abort(ttid)
......
......@@ -27,7 +27,7 @@ import transaction
from mock import Mock
from neo.lib import debug, logging, protocol
from neo.lib.protocol import Packets
from neo.lib.protocol import NodeTypes, Packets, UUID_NAMESPACES
from neo.lib.util import getAddressType
from time import time
from struct import pack, unpack
......@@ -159,6 +159,10 @@ class NeoUnitTestBase(NeoTestBase):
local_ip = IP_VERSION_FORMAT_DICT[ADDRESS_TYPE]
def setUp(self):
self.uuid_dict = {}
NeoTestBase.setUp(self)
def prepareDatabase(self, number, prefix='test_neo'):
""" create empties databases """
setupMySQLdb(['%s%u' % (prefix, i) for i in xrange(number)])
......@@ -196,30 +200,26 @@ class NeoUnitTestBase(NeoTestBase):
'getAdapter': 'MySQL',
})
def _makeUUID(self, prefix):
def getNewUUID(self, node_type):
"""
Retuns a 16-bytes UUID according to namespace 'prefix'
"""
assert len(prefix) == 1
uuid = protocol.INVALID_UUID
while uuid[1:] == protocol.INVALID_UUID[1:]:
uuid = prefix + os.urandom(15)
return uuid
def getNewUUID(self):
return self._makeUUID('\0')
if node_type is None:
node_type = random.choice(NodeTypes)
self.uuid_dict[node_type] = uuid = 1 + self.uuid_dict.get(node_type, 0)
return uuid + (UUID_NAMESPACES[node_type] << 24)
def getClientUUID(self):
return self._makeUUID('C')
return self.getNewUUID(NodeTypes.CLIENT)
def getMasterUUID(self):
return self._makeUUID('M')
return self.getNewUUID(NodeTypes.MASTER)
def getStorageUUID(self):
return self._makeUUID('S')
return self.getNewUUID(NodeTypes.STORAGE)
def getAdminUUID(self):
return self._makeUUID('A')
return self.getNewUUID(NodeTypes.ADMIN)
def getNextTID(self, ltid=None):
return newTid(ltid)
......@@ -236,12 +236,6 @@ class NeoUnitTestBase(NeoTestBase):
return os.urandom(8)
return pack('!Q', i)
def getTwoIDs(self):
""" Return a tuple of two sorted UUIDs """
# generate two ptid, first is lower
uuids = self.getNewUUID(), self.getNewUUID()
return min(uuids), max(uuids)
def getFakeConnector(self, descriptor=None):
return Mock({
'__repr__': 'FakeConnector',
......
......@@ -22,8 +22,8 @@ from .. import NeoUnitTestBase, buildUrlFromString, ADDRESS_TYPE
from neo.client.app import Application
from neo.client.exception import NEOStorageError, NEOStorageNotFoundError
from neo.client.exception import NEOStorageDoesNotExistError
from neo.lib.protocol import Packet, Packets, Errors, INVALID_TID, \
INVALID_PARTITION
from neo.lib.protocol import NodeTypes, Packet, Packets, Errors, INVALID_TID, \
INVALID_PARTITION, UUID_NAMESPACES
from neo.lib.util import makeChecksum, SOCKET_CONNECTORS_DICT
import time
......@@ -37,7 +37,7 @@ class Dispatcher(object):
def _getMasterConnection(self):
if self.master_conn is None:
self.uuid = 'C' * 16
self.uuid = 1 + (UUID_NAMESPACES[NodeTypes.CLIENT] << 24)
self.num_partitions = 10
self.num_replicas = 1
self.pt = Mock({'getCellList': ()})
......@@ -323,7 +323,7 @@ class ClientApplicationTests(NeoUnitTestBase):
def test_store3(self):
app = self.getApp()
uuid = self.getNewUUID()
uuid = self.getStorageUUID()
oid = self.makeOID(11)
tid = self.makeTID()
txn = self.makeTransactionObject()
......@@ -429,10 +429,9 @@ class ClientApplicationTests(NeoUnitTestBase):
oid1 = self.makeOID(num_partitions + 1) # on partition 1, conflicting
oid2 = self.makeOID(num_partitions + 2) # on partition 2
# storage nodes
uuid1, uuid2, uuid3 = [self.getNewUUID() for _ in range(3)]
address1 = ('127.0.0.1', 10000)
address2 = ('127.0.0.1', 10001)
address3 = ('127.0.0.1', 10002)
address1 = ('127.0.0.1', 10000); uuid1 = self.getMasterUUID()
address2 = ('127.0.0.1', 10001); uuid2 = self.getStorageUUID()
address3 = ('127.0.0.1', 10002); uuid3 = self.getStorageUUID()
app.nm.createMaster(address=address1, uuid=uuid1)
app.nm.createStorage(address=address2, uuid=uuid2)
app.nm.createStorage(address=address3, uuid=uuid3)
......@@ -699,7 +698,7 @@ class ClientApplicationTests(NeoUnitTestBase):
def test_undoLog(self):
app = self.getApp()
app.num_partitions = 2
uuid1, uuid2 = '\x00' * 15 + '\x01', '\x00' * 15 + '\x02'
uuid1, uuid2 = self.getStorageUUID(), self.getStorageUUID()
# two nodes, two partition, two transaction, two objects :
tid1, tid2 = self.makeTID(1), self.makeTID(2)
oid1, oid2 = self.makeOID(1), self.makeOID(2)
......@@ -778,7 +777,7 @@ class ClientApplicationTests(NeoUnitTestBase):
# fifth packet : request node identification succeeded
def _ask6(conn):
app.master_conn = conn
app.uuid = 'C' * 16
app.uuid = 1 + (UUID_NAMESPACES[NodeTypes.CLIENT] << 24)
app.trying_master_node = app.primary_master_node = Mock({
'getAddress': ('127.0.0.1', 10011),
'__str__': 'Fake master node',
......
......@@ -26,10 +26,8 @@ class ConnectionPoolTests(NeoUnitTestBase):
def test_removeConnection(self):
app = None
pool = ConnectionPool(app)
test_node_uuid = self.getNewUUID()
other_node_uuid = test_node_uuid
while other_node_uuid == test_node_uuid:
other_node_uuid = self.getNewUUID()
test_node_uuid = self.getStorageUUID()
other_node_uuid = self.getStorageUUID()
test_node = Mock({'getUUID': test_node_uuid})
other_node = Mock({'getUUID': other_node_uuid})
# Test sanity check
......@@ -50,9 +48,9 @@ class ConnectionPoolTests(NeoUnitTestBase):
def test_CellSortKey(self):
pool = ConnectionPool(None)
node_uuid_1 = self.getNewUUID()
node_uuid_2 = self.getNewUUID()
node_uuid_3 = self.getNewUUID()
node_uuid_1 = self.getStorageUUID()
node_uuid_2 = self.getStorageUUID()
node_uuid_3 = self.getStorageUUID()
# We are connected to node 1
pool.connection_dict[node_uuid_1] = None
# A connection to node 3 failed, will be forgotten at 5
......
......@@ -69,7 +69,7 @@ class MasterBootstrapHandlerTests(MasterHandlerTests):
def test_acceptIdentification2(self):
""" No UUID supplied """
node, conn = self.getKnownMaster()
uuid = self.getNewUUID()
uuid = self.getMasterUUID()
addr = conn.getAddress()
self.checkProtocolErrorRaised(self.handler.acceptIdentification,
conn, NodeTypes.MASTER, uuid, 100, 0, None,
......@@ -79,9 +79,9 @@ class MasterBootstrapHandlerTests(MasterHandlerTests):
def test_acceptIdentification3(self):
""" identification accepted """
node, conn = self.getKnownMaster()
uuid = self.getNewUUID()
uuid = self.getMasterUUID()
addr = conn.getAddress()
your_uuid = self.getNewUUID()
your_uuid = self.getClientUUID()
self.handler.acceptIdentification(conn, NodeTypes.MASTER, uuid,
100, 2, your_uuid, addr, [(addr, uuid)])
self.assertEqual(self.app.uuid, your_uuid)
......
......@@ -38,14 +38,14 @@ class StorageBootstrapHandlerTests(NeoUnitTestBase):
self.handler = StorageBootstrapHandler(self.app)
self.app.primary_master_node = node = Mock({
'getConnection': self.getFakeConnection(),
'getUUID': self.getNewUUID(),
'getUUID': self.getMasterUUID(),
'getAddress': (self.local_ip, 2999)
})
self._next_port = 3000
def getKnownStorage(self):
node = self.app.nm.createStorage(
uuid=self.getNewUUID(),
uuid=self.getStorageUUID(),
address=(self.local_ip, self._next_port),
)
self._next_port += 1
......@@ -172,7 +172,7 @@ class StorageAnswerHandlerTests(NeoUnitTestBase):
{oid: set([tid, ])}).answerStoreObject(conn, 1, oid, tid_2)
def test_answerStoreObject_4(self):
uuid = self.getNewUUID()
uuid = self.getStorageUUID()
conn = self.getFakeConnection(uuid=uuid)
oid = self.getOID(0)
tid = self.getNextTID()
......@@ -223,7 +223,7 @@ class StorageAnswerHandlerTests(NeoUnitTestBase):
conn, 'message')
def test_answerTIDs(self):
uuid = self.getNewUUID()
uuid = self.getStorageUUID()
tid1 = self.getNextTID()
tid2 = self.getNextTID(tid1)
tid_list = [tid1, tid2]
......@@ -233,7 +233,7 @@ class StorageAnswerHandlerTests(NeoUnitTestBase):
self.assertEqual(tid_set, set(tid_list))
def test_answerObjectUndoSerial(self):
uuid = self.getNewUUID()
uuid = self.getStorageUUID()
conn = self.getFakeConnection(uuid=uuid)
oid1 = self.getOID(1)
oid2 = self.getOID(2)
......
......@@ -34,16 +34,19 @@ import psutil
import neo.scripts
from neo.neoctl.neoctl import NeoCTL, NotReadyException
from neo.lib import logging
from neo.lib.protocol import ClusterStates, NodeTypes, CellStates, NodeStates
from neo.lib.protocol import ClusterStates, NodeTypes, CellStates, NodeStates, \
UUID_NAMESPACES
from neo.lib.util import dump
from .. import DB_USER, setupMySQLdb, NeoTestBase, buildUrlFromString, \
ADDRESS_TYPE, IP_VERSION_FORMAT_DICT, getTempDirectory
from ..cluster import SocketLock
from neo.client.Storage import Storage
NEO_MASTER = 'neomaster'
NEO_STORAGE = 'neostorage'
NEO_ADMIN = 'neoadmin'
command_dict = {
NodeTypes.MASTER: 'neomaster',
NodeTypes.STORAGE: 'neostorage',
NodeTypes.ADMIN: 'neoadmin',
}
DELAY_SAFETY_MARGIN = 10
MAX_START_TIME = 30
......@@ -231,7 +234,7 @@ class NEOProcess(object):
Note: for this change to take effect, the node must be restarted.
"""
self.uuid = uuid
self.arg_dict['--uuid'] = dump(uuid)
self.arg_dict['--uuid'] = str(uuid)
def isAlive(self):
try:
......@@ -252,7 +255,7 @@ class NEOCluster(object):
self.adapter = adapter
self.zodb_storage_list = []
self.cleanup_on_delete = cleanup_on_delete
self.uuid_set = set()
self.uuid_dict = {}
self.db_list = db_list
if temp_dir is None:
temp_dir = tempfile.mkdtemp(prefix='neo_')
......@@ -280,7 +283,7 @@ class NEOCluster(object):
for x in master_node_list)
# create admin node
self.__newProcess(NEO_ADMIN, {
self.__newProcess(NodeTypes.ADMIN, {
'--cluster': self.cluster_name,
'--logfile': os.path.join(self.temp_dir, 'admin.log'),
'--bind': '%s:%d' % (buildUrlFromString(
......@@ -289,7 +292,7 @@ class NEOCluster(object):
})
# create master nodes
for i, port in enumerate(master_node_list):
self.__newProcess(NEO_MASTER, {
self.__newProcess(NodeTypes.MASTER, {
'--cluster': self.cluster_name,
'--logfile': os.path.join(self.temp_dir, 'master_%u.log' % i),
'--bind': '%s:%d' % (buildUrlFromString(
......@@ -300,7 +303,7 @@ class NEOCluster(object):
})
# create storage nodes
for i, db in enumerate(db_list):
self.__newProcess(NEO_STORAGE, {
self.__newProcess(NodeTypes.STORAGE, {
'--cluster': self.cluster_name,
'--logfile': os.path.join(self.temp_dir, 'storage_%u.log' % i),
'--bind': '%s:%d' % (buildUrlFromString(
......@@ -313,11 +316,12 @@ class NEOCluster(object):
# create neoctl
self.neoctl = NeoCTL((self.local_ip, admin_port))
def __newProcess(self, command, arguments):
uuid = self.__allocateUUID()
def __newProcess(self, node_type, arguments):
self.uuid_dict[node_type] = uuid = 1 + self.uuid_dict.get(node_type, 0)
uuid += UUID_NAMESPACES[node_type] << 24
arguments['--uuid'] = uuid
self.process_dict.setdefault(command, []).append(
NEOProcess(command, uuid, arguments))
self.process_dict.setdefault(node_type, []).append(
NEOProcess(command_dict[node_type], uuid, arguments))
def __allocateUUID(self):
uuid = ('%032x' % random.getrandbits(128)).decode('hex')
......@@ -433,17 +437,14 @@ class NEOCluster(object):
conn = sqlite3.connect(self.db_template % db, isolation_level=None)
return conn
def _getProcessList(self, type):
return self.process_dict.get(type)
def getMasterProcessList(self):
return self._getProcessList(NEO_MASTER)
return self.process_dict.get(NodeTypes.MASTER)
def getStorageProcessList(self):
return self._getProcessList(NEO_STORAGE)
return self.process_dict.get(NodeTypes.STORAGE)
def getAdminProcessList(self):
return self._getProcessList(NEO_ADMIN)
return self.process_dict.get(NodeTypes.ADMIN)
def _killMaster(self, primary=False, all=False):
killed_uuid_list = []
......
......@@ -42,7 +42,7 @@ class MasterClientHandlerTests(NeoUnitTestBase):
self.master_address = ('127.0.0.1', self.master_port)
self.client_address = ('127.0.0.1', self.client_port)
self.storage_address = ('127.0.0.1', self.storage_port)
self.storage_uuid = self.getNewUUID()
self.storage_uuid = self.getStorageUUID()
# register the storage
self.app.nm.createStorage(
uuid=self.storage_uuid,
......@@ -53,7 +53,7 @@ class MasterClientHandlerTests(NeoUnitTestBase):
port=10021):
"""Do first step of identification to MN """
# register the master itself
uuid = self.getNewUUID()
uuid = self.getNewUUID(node_type)
self.app.nm.createFromNodeType(
node_type,
address=(ip, port),
......
......@@ -37,10 +37,8 @@ class MasterClientElectionTestBase(NeoUnitTestBase):
super(MasterClientElectionTestBase, self).setUp()
self._master_port = 3001
def identifyToMasterNode(self, uuid=True):
if uuid is True:
uuid = self.getNewUUID()
node = self.app.nm.createMaster(uuid=uuid)
def identifyToMasterNode(self):
node = self.app.nm.createMaster(uuid=self.getMasterUUID())
node.setAddress((self.local_ip, self._master_port))
self._master_port += 1
conn = self.getFakeConnection(
......@@ -59,7 +57,7 @@ class MasterClientElectionTests(MasterClientElectionTestBase):
self.app = Application(config)
self.app.pt.clear()
self.app.em = Mock()
self.app.uuid = self._makeUUID('M')
self.app.uuid = self.getMasterUUID()
self.app.server = (self.local_ip, 10000)
self.app.name = 'NEOCLUSTER'
self.election = ClientElectionHandler(self.app)
......@@ -264,13 +262,13 @@ class MasterServerElectionTests(MasterClientElectionTestBase):
return [x.asTuple() for x in self.app.nm.getList()]
def __getClient(self):
uuid = self.getNewUUID()
uuid = self.getClientUUID()
conn = self.getFakeConnection(uuid=uuid, address=self.client_address)
self.app.nm.createClient(uuid=uuid, address=self.client_address)
return conn
def __getMaster(self, port=1000, register=True):
uuid = self.getNewUUID()
uuid = self.getMasterUUID()
address = ('127.0.0.1', port)
conn = self.getFakeConnection(uuid=uuid, address=address)
if register:
......@@ -291,7 +289,7 @@ class MasterServerElectionTests(MasterClientElectionTestBase):
def _requestIdentification(self):
conn = self.getFakeConnection()
peer_uuid = self.getNewUUID()
peer_uuid = self.getMasterUUID()
address = (self.local_ip, 2001)
self.election.requestIdentification(
conn,
......
......@@ -31,11 +31,11 @@ class MasterAppTests(NeoUnitTestBase):
def test_06_broadcastNodeInformation(self):
# defined some nodes to which data will be send
master_uuid = self.getNewUUID()
master_uuid = self.getMasterUUID()
master = self.app.nm.createMaster(uuid=master_uuid)
storage_uuid = self.getNewUUID()
storage_uuid = self.getStorageUUID()
storage = self.app.nm.createStorage(uuid=storage_uuid)
client_uuid = self.getNewUUID()
client_uuid = self.getClientUUID()
client = self.app.nm.createClient(uuid=client_uuid)
# create conn and patch em
master_conn = self.getFakeConnection()
......@@ -51,7 +51,7 @@ class MasterAppTests(NeoUnitTestBase):
self.app.nm.add(client)
# no address defined, not send to client node
c_node = self.app.nm.createClient(uuid = self.getNewUUID())
c_node = self.app.nm.createClient(uuid=self.getClientUUID())
self.app.broadcastNodesInformation([c_node])
# check conn
self.checkNoPacketSent(client_conn)
......@@ -60,7 +60,7 @@ class MasterAppTests(NeoUnitTestBase):
# address defined and client type
s_node = self.app.nm.createClient(
uuid = self.getNewUUID(),
uuid=self.getClientUUID(),
address=("127.1.0.1", 3361)
)
self.app.broadcastNodesInformation([c_node])
......@@ -71,7 +71,7 @@ class MasterAppTests(NeoUnitTestBase):
# address defined and storage type
s_node = self.app.nm.createStorage(
uuid=self.getNewUUID(),
uuid=self.getStorageUUID(),
address=("127.0.0.1", 1351)
)
......@@ -91,8 +91,8 @@ class MasterAppTests(NeoUnitTestBase):
self.checkNotifyNodeInformation(storage_conn)
def test_storageReadinessAPI(self):
uuid_1 = self.getNewUUID()
uuid_2 = self.getNewUUID()
uuid_1 = self.getStorageUUID()
uuid_2 = self.getStorageUUID()
self.assertFalse(self.app.isStorageReady(uuid_1))
self.assertFalse(self.app.isStorageReady(uuid_2))
# Must not raise, nor change readiness
......
......@@ -54,18 +54,18 @@ class MasterPartitionTableTests(NeoUnitTestBase):
num_replicas = 2
pt = PartitionTable(num_partitions, num_replicas)
# add nodes
uuid1 = self.getNewUUID()
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1, NodeStates.RUNNING)
pt.setCell(0, sn1, CellStates.UP_TO_DATE)
pt.setCell(1, sn1, CellStates.UP_TO_DATE)
pt.setCell(2, sn1, CellStates.UP_TO_DATE)
uuid2 = self.getNewUUID()
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19001)
sn2 = StorageNode(Mock(), server2, uuid2, NodeStates.RUNNING)
pt.setCell(0, sn2, CellStates.UP_TO_DATE)
pt.setCell(1, sn2, CellStates.UP_TO_DATE)
uuid3 = self.getNewUUID()
uuid3 = self.getStorageUUID()
server3 = ("127.0.0.3", 19001)
sn3 = StorageNode(Mock(), server3, uuid3, NodeStates.RUNNING)
pt.setCell(0, sn3, CellStates.UP_TO_DATE)
......@@ -79,19 +79,19 @@ class MasterPartitionTableTests(NeoUnitTestBase):
def test_13_outdate(self):
# create nodes
uuid1 = self.getNewUUID()
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
uuid2 = self.getNewUUID()
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19002)
sn2 = StorageNode(Mock(), server2, uuid2)
uuid3 = self.getNewUUID()
uuid3 = self.getStorageUUID()
server3 = ("127.0.0.3", 19003)
sn3 = StorageNode(Mock(), server3, uuid3)
uuid4 = self.getNewUUID()
uuid4 = self.getStorageUUID()
server4 = ("127.0.0.4", 19004)
sn4 = StorageNode(Mock(), server4, uuid4)
uuid5 = self.getNewUUID()
uuid5 = self.getStorageUUID()
server5 = ("127.0.0.5", 19005)
sn5 = StorageNode(Mock(), server5, uuid5)
# create partition table
......@@ -147,7 +147,7 @@ class MasterPartitionTableTests(NeoUnitTestBase):
num_replicas = 2
pt = PartitionTable(num_partitions, num_replicas)
# add nodes
uuid1 = self.getNewUUID()
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
# add it to an empty pt
......@@ -168,7 +168,7 @@ class MasterPartitionTableTests(NeoUnitTestBase):
self.assertEqual(pt.getCellList(x)[0].getNode(), sn1)
self.assertEqual(pt.count_dict[sn1], 5)
# add a second node to fill the partition table
uuid2 = self.getNewUUID()
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19002)
sn2 = StorageNode(Mock(), server2, uuid2)
# add it
......@@ -179,13 +179,13 @@ class MasterPartitionTableTests(NeoUnitTestBase):
self.assertEqual(pt.getCellList(x)[0].getState(), CellStates.OUT_OF_DATE)
self.assertTrue(pt.getCellList(x)[0].getNode() in (sn1, sn2))
# test the most used node is remove from some partition
uuid3 = self.getNewUUID()
uuid3 = self.getStorageUUID()
server3 = ("127.0.0.3", 19001)
sn3 = StorageNode(Mock(), server3, uuid3)
uuid4 = self.getNewUUID()
uuid4 = self.getStorageUUID()
server4 = ("127.0.0.4", 19001)
sn4 = StorageNode(Mock(), server4, uuid4)
uuid5 = self.getNewUUID()
uuid5 = self.getStorageUUID()
server5 = ("127.0.0.5", 1900)
sn5 = StorageNode(Mock(), server5, uuid5)
# partition looks like:
......@@ -205,7 +205,7 @@ class MasterPartitionTableTests(NeoUnitTestBase):
pt.setCell(2, sn4, CellStates.UP_TO_DATE)
pt.setCell(3, sn1, CellStates.OUT_OF_DATE)
pt.setCell(3, sn5, CellStates.UP_TO_DATE)
uuid6 = self.getNewUUID()
uuid6 = self.getStorageUUID()
server6 = ("127.0.0.6", 19006)
sn6 = StorageNode(Mock(), server6, uuid6)
cell_list = pt.addNode(sn6)
......@@ -283,16 +283,16 @@ class MasterPartitionTableTests(NeoUnitTestBase):
num_replicas = 2
pt = PartitionTable(num_partitions, num_replicas)
# add nodes
uuid1 = self.getNewUUID()
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1, NodeStates.RUNNING)
uuid2 = self.getNewUUID()
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19002)
sn2 = StorageNode(Mock(), server2, uuid2, NodeStates.RUNNING)
uuid3 = self.getNewUUID()
uuid3 = self.getStorageUUID()
server3 = ("127.0.0.3", 19001)
sn3 = StorageNode(Mock(), server3, uuid3, NodeStates.RUNNING)
uuid4 = self.getNewUUID()
uuid4 = self.getStorageUUID()
server4 = ("127.0.0.4", 19001)
sn4 = StorageNode(Mock(), server4, uuid4, NodeStates.RUNNING)
# partition looks like:
......@@ -350,11 +350,11 @@ class MasterPartitionTableTests(NeoUnitTestBase):
num_replicas = 1
pt = PartitionTable(num_partitions, num_replicas)
# add nodes
uuid1 = self.getNewUUID()
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1, NodeStates.RUNNING)
# add not running node
uuid2 = self.getNewUUID()
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19001)
sn2 = StorageNode(Mock(), server2, uuid2)
sn2.setState(NodeStates.TEMPORARILY_DOWN)
......@@ -362,10 +362,10 @@ class MasterPartitionTableTests(NeoUnitTestBase):
server3 = ("127.0.0.3", 19001)
sn3 = StorageNode(Mock(), server3, None, NodeStates.RUNNING)
# add clear node
uuid4 = self.getNewUUID()
uuid4 = self.getStorageUUID()
server4 = ("127.0.0.4", 19001)
sn4 = StorageNode(Mock(), server4, uuid4, NodeStates.RUNNING)
uuid5 = self.getNewUUID()
uuid5 = self.getStorageUUID()
server5 = ("127.0.0.5", 1900)
sn5 = StorageNode(Mock(), server5, uuid5, NodeStates.RUNNING)
# make the table
......@@ -404,19 +404,19 @@ class MasterPartitionTableTests(NeoUnitTestBase):
# if not enought cell, add least used node
# create nodes
uuid1 = self.getNewUUID()
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1, NodeStates.RUNNING)
uuid2 = self.getNewUUID()
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19002)
sn2 = StorageNode(Mock(), server2, uuid2, NodeStates.RUNNING)
uuid3 = self.getNewUUID()
uuid3 = self.getStorageUUID()
server3 = ("127.0.0.3", 19003)
sn3 = StorageNode(Mock(), server3, uuid3, NodeStates.RUNNING)
uuid4 = self.getNewUUID()
uuid4 = self.getStorageUUID()
server4 = ("127.0.0.4", 19004)
sn4 = StorageNode(Mock(), server4, uuid4, NodeStates.RUNNING)
uuid5 = self.getNewUUID()
uuid5 = self.getStorageUUID()
server5 = ("127.0.0.5", 19005)
sn5 = StorageNode(Mock(), server5, uuid5, NodeStates.RUNNING)
# create partition table
......
......@@ -49,7 +49,7 @@ class MasterRecoveryTests(NeoUnitTestBase):
"""Do first step of identification to MN
"""
address = (ip, port)
uuid = self.getNewUUID()
uuid = self.getNewUUID(node_type)
self.app.nm.createFromNodeType(node_type, address=address, uuid=uuid,
state=NodeStates.RUNNING)
return uuid
......
......@@ -60,7 +60,7 @@ class MasterStorageHandlerTests(NeoUnitTestBase):
"""Do first step of identification to MN
"""
nm = self.app.nm
uuid = self.getNewUUID()
uuid = self.getNewUUID(node_type)
node = nm.createFromNodeType(node_type, address=(ip, port),
uuid=uuid)
conn = self.getFakeConnection(node.getUUID(),node.getAddress())
......
......@@ -18,7 +18,7 @@ import unittest
from mock import Mock
from struct import pack, unpack
from .. import NeoUnitTestBase
from neo.lib.protocol import ZERO_TID
from neo.lib.protocol import NodeTypes, ZERO_TID
from neo.master.transactions import Transaction, TransactionManager
from neo.master.transactions import packTID, unpackTID, addTID, DelayedError
......@@ -31,12 +31,9 @@ class testTransactionManager(NeoUnitTestBase):
def makeOID(self, i):
return pack('!Q', i)
def makeUUID(self, i):
return '\0' * 12 + pack('!Q', i)
def makeNode(self, i):
uuid = self.makeUUID(i)
node = Mock({'getUUID': uuid, '__hash__': i, '__repr__': 'FakeNode'})
def makeNode(self, node_type):
uuid = self.getNewUUID(node_type)
node = Mock({'getUUID': uuid, '__hash__': uuid, '__repr__': 'FakeNode'})
return uuid, node
def testTransaction(self):
......@@ -45,7 +42,8 @@ class testTransactionManager(NeoUnitTestBase):
tid = self.makeTID(1)
ttid = self.makeTID(2)
oid_list = (oid1, oid2) = [self.makeOID(1), self.makeOID(2)]
uuid_list = (uuid1, uuid2) = [self.makeUUID(1), self.makeUUID(2)]
uuid_list = (uuid1, uuid2) = [self.getStorageUUID(),
self.getStorageUUID()]
msg_id = 1
# create transaction object
txn = Transaction(node, ttid)
......@@ -63,8 +61,8 @@ class testTransactionManager(NeoUnitTestBase):
node = Mock({'__hash__': 1})
msg_id = 1
oid_list = (oid1, oid2) = self.makeOID(1), self.makeOID(2)
uuid_list = (uuid1, uuid2) = self.makeUUID(1), self.makeUUID(2)
client_uuid = self.makeUUID(3)
uuid_list = uuid1, uuid2 = self.getStorageUUID(), self.getStorageUUID()
client_uuid = self.getClientUUID()
# create transaction manager
callback = Mock()
txnman = TransactionManager(on_commit=callback)
......@@ -94,9 +92,9 @@ class testTransactionManager(NeoUnitTestBase):
def testAbortFor(self):
oid_list = [self.makeOID(1), ]
storage_1_uuid, node1 = self.makeNode(1)
storage_2_uuid, node2 = self.makeNode(2)
client_uuid, client = self.makeNode(3)
storage_1_uuid, node1 = self.makeNode(NodeTypes.STORAGE)
storage_2_uuid, node2 = self.makeNode(NodeTypes.STORAGE)
client_uuid, client = self.makeNode(NodeTypes.CLIENT)
txnman = TransactionManager(lambda tid, txn: None)
# register 4 transactions made by two nodes
self.assertEqual(txnman.registerForNotification(storage_1_uuid), [])
......@@ -132,10 +130,10 @@ class testTransactionManager(NeoUnitTestBase):
client1 = Mock({'__hash__': 1})
client2 = Mock({'__hash__': 2})
client3 = Mock({'__hash__': 3})
storage_1_uuid = self.makeUUID(1)
storage_2_uuid = self.makeUUID(2)
storage_1_uuid = self.getStorageUUID()
storage_2_uuid = self.getStorageUUID()
oid_list = [self.makeOID(1), ]
client_uuid = self.makeUUID(3)
client_uuid = self.getClientUUID()
tm = TransactionManager(lambda tid, txn: None)
# Transaction 1: 2 storage nodes involved, one will die and the other
......@@ -213,7 +211,7 @@ class testTransactionManager(NeoUnitTestBase):
strictly increasing order.
Note: this implementation might change later, to allow more paralelism.
"""
client_uuid, client = self.makeNode(1)
client_uuid, client = self.makeNode(NodeTypes.CLIENT)
tm = TransactionManager(lambda tid, txn: None)
# With a requested TID, lock spans from begin to remove
ttid1 = self.getNextTID()
......@@ -230,7 +228,7 @@ class testTransactionManager(NeoUnitTestBase):
tm.prepare(ttid3, 1, [], [], 0)
def testClientDisconectsAfterBegin(self):
client_uuid1, node1 = self.makeNode(1)
client_uuid1, node1 = self.makeNode(NodeTypes.CLIENT)
tm = TransactionManager(lambda tid, txn: None)
tid1 = self.getNextTID()
tid2 = self.getNextTID()
......@@ -240,9 +238,9 @@ class testTransactionManager(NeoUnitTestBase):
def testUnlockPending(self):
callback = Mock()
uuid1, node1 = self.makeNode(1)
uuid2, node2 = self.makeNode(2)
storage_uuid = self.makeUUID(3)
uuid1, node1 = self.makeNode(NodeTypes.CLIENT)
uuid2, node2 = self.makeNode(NodeTypes.CLIENT)
storage_uuid = self.getStorageUUID()
tm = TransactionManager(callback)
ttid1 = tm.begin(node1)
ttid2 = tm.begin(node2)
......
......@@ -49,7 +49,7 @@ class MasterVerificationTests(NeoUnitTestBase):
port=10021):
"""Do first step of identification to MN
"""
uuid = self.getNewUUID()
uuid = self.getNewUUID(node_type)
self.app.nm.createFromNodeType(
node_type,
address=(ip, port),
......
......@@ -49,7 +49,7 @@ class StorageClientHandlerTests(NeoUnitTestBase):
# handler
self.operation = ClientOperationHandler(self.app)
# set pmn
self.master_uuid = self.getNewUUID()
self.master_uuid = self.getMasterUUID()
pmn = self.app.nm.getMasterList()[0]
pmn.setUUID(self.master_uuid)
self.app.primary_master_node = pmn
......@@ -69,7 +69,7 @@ class StorageClientHandlerTests(NeoUnitTestBase):
calls[0].checkArgs(uuid)
def test_connectionLost(self):
uuid = self.getNewUUID()
uuid = self.getClientUUID()
self.app.nm.createClient(uuid=uuid)
conn = self._getConnection(uuid=uuid)
self.operation.connectionClosed(conn)
......@@ -192,8 +192,7 @@ class StorageClientHandlerTests(NeoUnitTestBase):
self.checkAnswerObjectHistory(conn)
def test_askStoreTransaction(self):
uuid = self.getNewUUID()
conn = self._getConnection(uuid=uuid)
conn = self._getConnection(uuid=self.getClientUUID())
tid = self.getNextTID()
user = 'USER'
desc = 'DESC'
......@@ -217,8 +216,7 @@ class StorageClientHandlerTests(NeoUnitTestBase):
def test_askStoreObject1(self):
# no conflict => answer
uuid = self.getNewUUID()
conn = self._getConnection(uuid=uuid)
conn = self._getConnection(uuid=self.getClientUUID())
tid = self.getNextTID()
oid, serial, comp, checksum, data = self._getObject()
self.operation.askStoreObject(conn, oid, serial, comp, checksum,
......@@ -233,8 +231,7 @@ class StorageClientHandlerTests(NeoUnitTestBase):
def test_askStoreObjectWithDataTID(self):
# same as test_askStoreObject1, but with a non-None data_tid value
uuid = self.getNewUUID()
conn = self._getConnection(uuid=uuid)
conn = self._getConnection(uuid=self.getClientUUID())
tid = self.getNextTID()
oid, serial, comp, checksum, data = self._getObject()
data_tid = self.getNextTID()
......@@ -250,8 +247,7 @@ class StorageClientHandlerTests(NeoUnitTestBase):
def test_askStoreObject2(self):
# conflict error
uuid = self.getNewUUID()
conn = self._getConnection(uuid=uuid)
conn = self._getConnection(uuid=self.getClientUUID())
tid = self.getNextTID()
locking_tid = self.getNextTID(tid)
def fakeStoreObject(*args):
......@@ -275,8 +271,7 @@ class StorageClientHandlerTests(NeoUnitTestBase):
calls[0].checkArgs(tid)
def test_askObjectUndoSerial(self):
uuid = self.getNewUUID()
conn = self._getConnection(uuid=uuid)
conn = self._getConnection(uuid=self.getClientUUID())
tid = self.getNextTID()
ltid = self.getNextTID()
undone_tid = self.getNextTID()
......
......@@ -47,7 +47,7 @@ class StorageIdentificationHandlerTests(NeoUnitTestBase):
self.identification.requestIdentification,
self.getFakeConnection(),
NodeTypes.CLIENT,
self.getNewUUID(),
self.getClientUUID(),
None,
self.app.name,
)
......@@ -57,14 +57,14 @@ class StorageIdentificationHandlerTests(NeoUnitTestBase):
self.identification.requestIdentification,
self.getFakeConnection(),
NodeTypes.STORAGE,
self.getNewUUID(),
self.getStorageUUID(),
None,
self.app.name,
)
def test_requestIdentification3(self):
""" broken nodes must be rejected """
uuid = self.getNewUUID()
uuid = self.getClientUUID()
conn = self.getFakeConnection(uuid=uuid)
node = self.app.nm.createClient(uuid=uuid)
node.setBroken()
......@@ -79,7 +79,7 @@ class StorageIdentificationHandlerTests(NeoUnitTestBase):
def test_requestIdentification2(self):
""" accepted client must be connected and running """
uuid = self.getNewUUID()
uuid = self.getClientUUID()
conn = self.getFakeConnection(uuid=uuid)
node = self.app.nm.createClient(uuid=uuid)
master = (self.local_ip, 3000)
......
......@@ -49,7 +49,8 @@ class StorageInitializationHandlerTests(NeoUnitTestBase):
def getClientConnection(self):
address = ("127.0.0.1", self.client_port)
return self.getFakeConnection(uuid=self.getNewUUID(), address=address)
return self.getFakeConnection(uuid=self.getClientUUID(),
address=address)
def test_03_connectionClosed(self):
conn = self.getClientConnection()
......@@ -62,15 +63,15 @@ class StorageInitializationHandlerTests(NeoUnitTestBase):
# send a table
conn = self.getClientConnection()
self.app.pt = PartitionTable(3, 2)
node_1 = self.getNewUUID()
node_2 = self.getNewUUID()
node_3 = self.getNewUUID()
node_1 = self.getStorageUUID()
node_2 = self.getStorageUUID()
node_3 = self.getStorageUUID()
self.app.uuid = node_1
# SN already know all nodes
self.app.nm.createStorage(uuid=node_1)
self.app.nm.createStorage(uuid=node_2)
self.app.nm.createStorage(uuid=node_3)
self.assertEqual(self.app.dm.getPartitionTable(), [])
self.assertEqual(self.app.dm.getPartitionTable(), ())
row_list = [(0, ((node_1, CellStates.UP_TO_DATE), (node_2, CellStates.UP_TO_DATE))),
(1, ((node_3, CellStates.UP_TO_DATE), (node_1, CellStates.UP_TO_DATE))),
(2, ((node_2, CellStates.UP_TO_DATE), (node_3, CellStates.UP_TO_DATE)))]
......
......@@ -40,7 +40,7 @@ class StorageMasterHandlerTests(NeoUnitTestBase):
# handler
self.operation = MasterOperationHandler(self.app)
# set pmn
self.master_uuid = self.getNewUUID()
self.master_uuid = self.getMasterUUID()
pmn = self.app.nm.getMasterList()[0]
pmn.setUUID(self.master_uuid)
self.app.primary_master_node = pmn
......@@ -79,7 +79,7 @@ class StorageMasterHandlerTests(NeoUnitTestBase):
def test_14_notifyPartitionChanges2(self):
# cases :
uuid1, uuid2, uuid3 = [self.getNewUUID() for i in range(3)]
uuid1, uuid2, uuid3 = [self.getStorageUUID() for i in range(3)]
cells = (
(0, uuid1, CellStates.UP_TO_DATE),
(1, uuid2, CellStates.DISCARDED),
......
......@@ -61,11 +61,11 @@ class StorageAppTests(NeoUnitTestBase):
self.assertFalse(self.app.pt.hasOffset(x))
# add some node, will be remove when loading table
master_uuid = self.getNewUUID()
master_uuid = self.getMasterUUID()
master = self.app.nm.createMaster(uuid=master_uuid)
storage_uuid = self.getNewUUID()
storage_uuid = self.getStorageUUID()
storage = self.app.nm.createStorage(uuid=storage_uuid)
client_uuid = self.getNewUUID()
client_uuid = self.getClientUUID()
self.app.pt.setCell(0, master, CellStates.UP_TO_DATE)
self.app.pt.setCell(0, storage, CellStates.UP_TO_DATE)
......
......@@ -67,7 +67,7 @@ class StorageDBTests(NeoUnitTestBase):
db.dropPartitions(n)
db.setNumPartitions(num_partitions)
self.assertEqual(num_partitions, db.getNumPartitions())
uuid = self.getNewUUID()
uuid = self.getStorageUUID()
db.setUUID(uuid)
self.assertEqual(uuid, db.getUUID())
db.setPartitionTable(1,
......@@ -83,7 +83,7 @@ class StorageDBTests(NeoUnitTestBase):
def test_UUID(self):
db = self.getDB()
self.checkConfigEntry(db.getUUID, db.setUUID, 'TEST_VALUE')
self.checkConfigEntry(db.getUUID, db.setUUID, 123)
def test_Name(self):
db = self.getDB()
......@@ -119,7 +119,7 @@ class StorageDBTests(NeoUnitTestBase):
def test_getPartitionTable(self):
db = self.getDB()
ptid = self.getPTID(1)
uuid1, uuid2 = self.getNewUUID(), self.getNewUUID()
uuid1, uuid2 = self.getStorageUUID(), self.getStorageUUID()
cell1 = (0, uuid1, CellStates.OUT_OF_DATE)
cell2 = (1, uuid1, CellStates.UP_TO_DATE)
db.setPartitionTable(ptid, [cell1, cell2])
......@@ -249,38 +249,38 @@ class StorageDBTests(NeoUnitTestBase):
def test_setPartitionTable(self):
db = self.getDB()
ptid = self.getPTID(1)
uuid1, uuid2 = self.getNewUUID(), self.getNewUUID()
cell1 = (0, uuid1, CellStates.OUT_OF_DATE)
cell2 = (1, uuid1, CellStates.UP_TO_DATE)
cell3 = (1, uuid1, CellStates.DISCARDED)
uuid = self.getStorageUUID()
cell1 = 0, uuid, CellStates.OUT_OF_DATE
cell2 = 1, uuid, CellStates.UP_TO_DATE
cell3 = 1, uuid, CellStates.DISCARDED
# no partition table
self.assertEqual(db.getPartitionTable(), [])
self.assertEqual(list(db.getPartitionTable()), [])
# set one
db.setPartitionTable(ptid, [cell1])
result = db.getPartitionTable()
self.assertEqual(result, [cell1])
self.assertEqual(list(result), [cell1])
# then another
db.setPartitionTable(ptid, [cell2])
result = db.getPartitionTable()
self.assertEqual(result, [cell2])
self.assertEqual(list(result), [cell2])
# drop discarded cells
db.setPartitionTable(ptid, [cell2, cell3])
result = db.getPartitionTable()
self.assertEqual(result, [])
self.assertEqual(list(result), [])
def test_changePartitionTable(self):
db = self.getDB()
ptid = self.getPTID(1)
uuid1, uuid2 = self.getNewUUID(), self.getNewUUID()
cell1 = (0, uuid1, CellStates.OUT_OF_DATE)
cell2 = (1, uuid1, CellStates.UP_TO_DATE)
cell3 = (1, uuid1, CellStates.DISCARDED)
uuid = self.getStorageUUID()
cell1 = 0, uuid, CellStates.OUT_OF_DATE
cell2 = 1, uuid, CellStates.UP_TO_DATE
cell3 = 1, uuid, CellStates.DISCARDED
# no partition table
self.assertEqual(db.getPartitionTable(), [])
self.assertEqual(list(db.getPartitionTable()), [])
# set one
db.changePartitionTable(ptid, [cell1])
result = db.getPartitionTable()
self.assertEqual(result, [cell1])
self.assertEqual(list(result), [cell1])
# add more entries
db.changePartitionTable(ptid, [cell2])
result = db.getPartitionTable()
......@@ -288,7 +288,7 @@ class StorageDBTests(NeoUnitTestBase):
# drop discarded cells
db.changePartitionTable(ptid, [cell2, cell3])
result = db.getPartitionTable()
self.assertEqual(result, [cell1])
self.assertEqual(list(result), [cell1])
def test_dropUnfinishedData(self):
oid1, oid2 = self.getOIDs(2)
......
......@@ -25,7 +25,7 @@ from neo.storage.transactions import ConflictError, DelayedError
class TransactionTests(NeoUnitTestBase):
def testInit(self):
uuid = self.getNewUUID()
uuid = self.getClientUUID()
ttid = self.getNextTID()
tid = self.getNextTID()
txn = Transaction(uuid, ttid)
......@@ -37,15 +37,8 @@ class TransactionTests(NeoUnitTestBase):
self.assertEqual(txn.getObjectList(), [])
self.assertEqual(txn.getOIDList(), [])
def testRepr(self):
""" Just check if the __repr__ implementation will not raise """
uuid = self.getNewUUID()
tid = self.getNextTID()
txn = Transaction(uuid, tid)
repr(txn)
def testLock(self):
txn = Transaction(self.getNewUUID(), self.getNextTID())
txn = Transaction(self.getClientUUID(), self.getNextTID())
self.assertFalse(txn.isLocked())
txn.lock()
self.assertTrue(txn.isLocked())
......@@ -53,7 +46,8 @@ class TransactionTests(NeoUnitTestBase):
self.assertRaises(AssertionError, txn.lock)
def testTransaction(self):
txn = Transaction(self.getNewUUID(), self.getNextTID())
txn = Transaction(self.getClientUUID(), self.getNextTID())
repr(txn) # check __repr__ does not raise
oid_list = [self.getOID(1), self.getOID(2)]
txn_info = (oid_list, 'USER', 'DESC', 'EXT', False)
txn.prepare(*txn_info)
......@@ -61,7 +55,7 @@ class TransactionTests(NeoUnitTestBase):
txn_info + (txn.getTTID(),))
def testObjects(self):
txn = Transaction(self.getNewUUID(), self.getNextTID())
txn = Transaction(self.getClientUUID(), self.getNextTID())
oid1, oid2 = self.getOID(1), self.getOID(2)
object1 = oid1, "0" * 20, None
object2 = oid2, "1" * 20, None
......@@ -77,7 +71,7 @@ class TransactionTests(NeoUnitTestBase):
def test_getObject(self):
oid_1 = self.getOID(1)
oid_2 = self.getOID(2)
txn = Transaction(self.getNewUUID(), self.getNextTID())
txn = Transaction(self.getClientUUID(), self.getNextTID())
object_info = oid_1, None, None
txn.addObject(*object_info)
self.assertRaises(KeyError, txn.getObject, oid_2)
......@@ -127,7 +121,7 @@ class TransactionManagerTests(NeoUnitTestBase):
""" One node, one transaction, not abort """
data_id_list = random.random(), random.random()
self.app.dm.mockAddReturnValues(storeData=ReturnValues(*data_id_list))
uuid = self.getNewUUID()
uuid = self.getClientUUID()
ttid = self.getNextTID()
tid, txn = self._getTransaction()
serial1, object1 = self._getObject(1)
......@@ -148,7 +142,7 @@ class TransactionManagerTests(NeoUnitTestBase):
def testDelayed(self):
""" Two transactions, the first cause the second to be delayed """
uuid = self.getNewUUID()
uuid = self.getClientUUID()
ttid1 = self.getNextTID()
ttid2 = self.getNextTID()
tid1, txn1 = self._getTransaction()
......@@ -169,7 +163,7 @@ class TransactionManagerTests(NeoUnitTestBase):
def testUnresolvableConflict(self):
""" A newer transaction has already modified an object """
uuid = self.getNewUUID()
uuid = self.getClientUUID()
ttid1 = self.getNextTID()
ttid2 = self.getNextTID()
tid1, txn1 = self._getTransaction()
......@@ -190,7 +184,7 @@ class TransactionManagerTests(NeoUnitTestBase):
def testResolvableConflict(self):
""" Try to store an object with the lastest revision """
uuid = self.getNewUUID()
uuid = self.getClientUUID()
tid, txn = self._getTransaction()
serial, obj = self._getObject(1)
next_serial = self.getNextTID(serial)
......@@ -203,8 +197,8 @@ class TransactionManagerTests(NeoUnitTestBase):
def testLockDelayed(self):
""" Check lock delay """
uuid1 = self.getNewUUID()
uuid2 = self.getNewUUID()
uuid1 = self.getClientUUID()
uuid2 = self.getClientUUID()
self.assertNotEqual(uuid1, uuid2)
ttid1 = self.getNextTID()
ttid2 = self.getNextTID()
......@@ -230,8 +224,8 @@ class TransactionManagerTests(NeoUnitTestBase):
def testLockConflict(self):
""" Check lock conflict """
uuid1 = self.getNewUUID()
uuid2 = self.getNewUUID()
uuid1 = self.getClientUUID()
uuid2 = self.getClientUUID()
self.assertNotEqual(uuid1, uuid2)
ttid1 = self.getNextTID()
ttid2 = self.getNextTID()
......@@ -257,7 +251,7 @@ class TransactionManagerTests(NeoUnitTestBase):
def testAbortUnlocked(self):
""" Abort a non-locked transaction """
uuid = self.getNewUUID()
uuid = self.getClientUUID()
tid, txn = self._getTransaction()
serial, obj = self._getObject(1)
self.manager.register(uuid, tid)
......@@ -272,7 +266,7 @@ class TransactionManagerTests(NeoUnitTestBase):
def testAbortLockedDoNothing(self):
""" Try to abort a locked transaction """
uuid = self.getNewUUID()
uuid = self.getClientUUID()
ttid = self.getNextTID()
tid, txn = self._getTransaction()
self.manager.register(uuid, ttid)
......@@ -289,8 +283,8 @@ class TransactionManagerTests(NeoUnitTestBase):
def testAbortForNode(self):
""" Abort transaction for a node """
uuid1 = self.getNewUUID()
uuid2 = self.getNewUUID()
uuid1 = self.getClientUUID()
uuid2 = self.getClientUUID()
self.assertNotEqual(uuid1, uuid2)
ttid1 = self.getNextTID()
ttid2 = self.getNextTID()
......@@ -319,7 +313,7 @@ class TransactionManagerTests(NeoUnitTestBase):
def testReset(self):
""" Reset the manager """
uuid = self.getNewUUID()
uuid = self.getClientUUID()
tid, txn = self._getTransaction()
ttid = self.getNextTID()
self.manager.register(uuid, ttid)
......@@ -335,7 +329,7 @@ class TransactionManagerTests(NeoUnitTestBase):
def test_getObjectFromTransaction(self):
data_id = random.random()
self.app.dm.mockAddReturnValues(storeData=ReturnValues(data_id))
uuid = self.getNewUUID()
uuid = self.getClientUUID()
tid1, txn1 = self._getTransaction()
tid2, txn2 = self._getTransaction()
serial1, obj1 = self._getObject(1)
......@@ -350,7 +344,7 @@ class TransactionManagerTests(NeoUnitTestBase):
(obj1[0], data_id, obj1[4]))
def test_getLockingTID(self):
uuid = self.getNewUUID()
uuid = self.getClientUUID()
serial1, obj1 = self._getObject(1)
oid1 = obj1[0]
tid1, txn1 = self._getTransaction()
......@@ -363,7 +357,7 @@ class TransactionManagerTests(NeoUnitTestBase):
ram_serial = self.getNextTID()
oid = self.getOID(1)
orig_serial = self.getNextTID()
uuid = self.getNewUUID()
uuid = self.getClientUUID()
locking_serial = self.getNextTID()
other_serial = self.getNextTID()
new_serial = self.getNextTID()
......
......@@ -49,16 +49,12 @@ class StorageVerificationHandlerTests(NeoUnitTestBase):
super(StorageVerificationHandlerTests, self)._tearDown(success)
# Common methods
def getClientConnection(self):
address = ("127.0.0.1", self.client_port)
return self.getFakeConnection(uuid=self.getNewUUID(), address=address)
def getMasterConnection(self):
return self.getFakeConnection(address=("127.0.0.1", self.master_port))
# Tests
def test_03_connectionClosed(self):
conn = self.getClientConnection()
conn = self.getMasterConnection()
self.app.listening_conn = object() # mark as running
self.assertRaises(PrimaryFailure, self.verification.connectionClosed, conn,)
# nothing happens
......@@ -67,11 +63,11 @@ class StorageVerificationHandlerTests(NeoUnitTestBase):
def test_08_askPartitionTable(self):
node = self.app.nm.createStorage(
address=("127.7.9.9", 1),
uuid=self.getNewUUID()
uuid=self.getStorageUUID()
)
self.app.pt.setCell(1, node, CellStates.UP_TO_DATE)
self.assertTrue(self.app.pt.hasOffset(1))
conn = self.getClientConnection()
conn = self.getMasterConnection()
self.verification.askPartitionTable(conn)
ptid, row_list = self.checkAnswerPartitionTable(conn, decode=True)
self.assertEqual(len(row_list), 1009)
......@@ -85,12 +81,12 @@ class StorageVerificationHandlerTests(NeoUnitTestBase):
# new node
conn = self.getMasterConnection()
new_uuid = self.getNewUUID()
new_uuid = self.getStorageUUID()
cell = (0, new_uuid, CellStates.UP_TO_DATE)
self.app.nm.createStorage(uuid=new_uuid)
self.app.pt = PartitionTable(1, 1)
self.app.dm = Mock({ })
ptid, self.ptid = self.getTwoIDs()
ptid = self.getPTID()
# pt updated
self.verification.notifyPartitionChanges(conn, ptid, (cell, ))
# check db update
......@@ -150,7 +146,7 @@ class StorageVerificationHandlerTests(NeoUnitTestBase):
self.app.dm = Mock({
'getTransaction': ([p64(2)], 'u2', 'd2', 'e2', False),
})
conn = self.getClientConnection()
conn = self.getMasterConnection()
self.verification.askTransactionInformation(conn, p64(1))
tid, user, desc, ext, packed, oid_list = self.checkAnswerTransactionInformation(conn, decode=True)
self.assertEqual(u64(tid), 1)
......
......@@ -207,7 +207,7 @@ class ConnectionTests(NeoUnitTestBase):
# test uuid
self.assertEqual(bc.uuid, None)
self.assertEqual(bc.getUUID(), None)
uuid = self.getNewUUID()
uuid = self.getNewUUID(None)
bc.setUUID(uuid)
self.assertEqual(bc.getUUID(), uuid)
# test next id
......@@ -423,7 +423,7 @@ class ConnectionTests(NeoUnitTestBase):
self._checkPacketReceived(0)
self._checkReadBuf(bc, '')
p = Packets.AnswerPrimary(self.getNewUUID())
p = Packets.AnswerPrimary(self.getNewUUID(None))
p.setId(1)
p_data = ''.join(p.encode())
data_edge = len(p_data) - 1
......@@ -450,10 +450,10 @@ class ConnectionTests(NeoUnitTestBase):
# give multiple packet
bc = self._makeConnection()
bc._queue = Mock()
p1 = Packets.AnswerPrimary(self.getNewUUID())
p1 = Packets.AnswerPrimary(self.getNewUUID(None))
p1.setId(1)
self._appendPacketToReadBuf(bc, p1)
p2 = Packets.AnswerPrimary( self.getNewUUID())
p2 = Packets.AnswerPrimary( self.getNewUUID(None))
p2.setId(2)
self._appendPacketToReadBuf(bc, p2)
self.assertEqual(len(bc.read_buf), len(p1) + len(p2))
......@@ -488,7 +488,7 @@ class ConnectionTests(NeoUnitTestBase):
# give an expected packet
bc = self._makeConnection()
bc._queue = Mock()
p = Packets.AnswerPrimary(self.getNewUUID())
p = Packets.AnswerPrimary(self.getNewUUID(None))
p.setId(1)
self._appendPacketToReadBuf(bc, p)
bc.analyse()
......@@ -587,7 +587,7 @@ class ConnectionTests(NeoUnitTestBase):
# With aborted set to false
# patch receive method to return data
def receive(self):
p = Packets.AnswerPrimary(self.getNewUUID())
p = Packets.AnswerPrimary(self.getNewUUID(None))
p.setId(1)
return ''.join(p.encode())
DoNothingConnector.receive = receive
......@@ -757,7 +757,7 @@ class ConnectionTests(NeoUnitTestBase):
# test uuid
self.assertEqual(bc.uuid, None)
self.assertEqual(bc.getUUID(), None)
uuid = self.getNewUUID()
uuid = self.getNewUUID(None)
bc.setUUID(uuid)
self.assertEqual(bc.getUUID(), uuid)
# test next id
......
......@@ -44,7 +44,7 @@ class NodesTests(NeoUnitTestBase):
def testInit(self):
""" Check the node initialization """
address = ('127.0.0.1', 10000)
uuid = self.getNewUUID()
uuid = self.getNewUUID(None)
node = Node(self.manager, address=address, uuid=uuid)
self.assertEqual(node.getState(), NodeStates.UNKNOWN)
self.assertEqual(node.getAddress(), address)
......@@ -74,7 +74,7 @@ class NodesTests(NeoUnitTestBase):
""" As for Address but UUID """
node = Node(self.manager)
self.assertEqual(node.getAddress(), None)
uuid = self.getNewUUID()
uuid = self.getNewUUID(None)
node.setUUID(uuid)
self._updatedByUUID(node)
......@@ -131,16 +131,16 @@ class NodeManagerTests(NeoUnitTestBase):
self.manager = NodeManager()
def _addStorage(self):
self.storage = StorageNode(self.manager, ('127.0.0.1', 1000), self.getNewUUID())
self.storage = StorageNode(self.manager, ('127.0.0.1', 1000), self.getStorageUUID())
def _addMaster(self):
self.master = MasterNode(self.manager, ('127.0.0.1', 2000), self.getNewUUID())
self.master = MasterNode(self.manager, ('127.0.0.1', 2000), self.getMasterUUID())
def _addClient(self):
self.client = ClientNode(self.manager, None, self.getNewUUID())
self.client = ClientNode(self.manager, None, self.getClientUUID())
def _addAdmin(self):
self.admin = AdminNode(self.manager, ('127.0.0.1', 4000), self.getNewUUID())
self.admin = AdminNode(self.manager, ('127.0.0.1', 4000), self.getAdminUUID())
def checkNodes(self, node_list):
manager = self.manager
......@@ -180,7 +180,7 @@ class NodeManagerTests(NeoUnitTestBase):
address = ('127.0.0.1', 10000)
self.assertEqual(manager.getByAddress(address), None)
self.assertEqual(manager.getByAddress(None), None)
uuid = self.getNewUUID()
uuid = self.getNewUUID(None)
self.assertEqual(manager.getByUUID(uuid), None)
self.assertEqual(manager.getByUUID(None), None)
......@@ -261,7 +261,7 @@ class NodeManagerTests(NeoUnitTestBase):
old_address = self.master.getAddress()
new_address = ('127.0.0.1', 2001)
old_uuid = self.storage.getUUID()
new_uuid = self.getNewUUID()
new_uuid = self.getStorageUUID()
node_list = (
(NodeTypes.CLIENT, None, self.client.getUUID(), NodeStates.DOWN),
(NodeTypes.MASTER, new_address, self.master.getUUID(), NodeStates.RUNNING),
......
......@@ -24,7 +24,7 @@ from . import NeoUnitTestBase
class PartitionTableTests(NeoUnitTestBase):
def test_01_Cell(self):
uuid = self.getNewUUID()
uuid = self.getStorageUUID()
server = ("127.0.0.1", 19001)
sn = StorageNode(Mock(), server, uuid)
cell = Cell(sn)
......@@ -48,7 +48,7 @@ class PartitionTableTests(NeoUnitTestBase):
num_partitions = 5
num_replicas = 2
pt = PartitionTable(num_partitions, num_replicas)
uuid1 = self.getNewUUID()
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
for x in xrange(num_partitions):
......@@ -129,7 +129,7 @@ class PartitionTableTests(NeoUnitTestBase):
num_partitions = 5
num_replicas = 2
pt = PartitionTable(num_partitions, num_replicas)
uuid1 = self.getNewUUID()
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
for x in xrange(num_partitions):
......@@ -169,19 +169,19 @@ class PartitionTableTests(NeoUnitTestBase):
num_replicas = 2
pt = PartitionTable(num_partitions, num_replicas)
# add two kind of node, usable and unsable
uuid1 = self.getNewUUID()
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
pt.setCell(0, sn1, CellStates.UP_TO_DATE)
uuid2 = self.getNewUUID()
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19001)
sn2 = StorageNode(Mock(), server2, uuid2)
pt.setCell(0, sn2, CellStates.OUT_OF_DATE)
uuid3 = self.getNewUUID()
uuid3 = self.getStorageUUID()
server3 = ("127.0.0.3", 19001)
sn3 = StorageNode(Mock(), server3, uuid3)
pt.setCell(0, sn3, CellStates.FEEDING)
uuid4 = self.getNewUUID()
uuid4 = self.getStorageUUID()
server4 = ("127.0.0.4", 19001)
sn4 = StorageNode(Mock(), server4, uuid4)
pt.setCell(0, sn4, CellStates.DISCARDED) # won't be added
......@@ -215,15 +215,15 @@ class PartitionTableTests(NeoUnitTestBase):
num_replicas = 2
pt = PartitionTable(num_partitions, num_replicas)
# add two kind of node, usable and unsable
uuid1 = self.getNewUUID()
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
pt.setCell(0, sn1, CellStates.UP_TO_DATE)
uuid2 = self.getNewUUID()
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19001)
sn2 = StorageNode(Mock(), server2, uuid2)
pt.setCell(1, sn2, CellStates.OUT_OF_DATE)
uuid3 = self.getNewUUID()
uuid3 = self.getStorageUUID()
server3 = ("127.0.0.3", 19001)
sn3 = StorageNode(Mock(), server3, uuid3)
pt.setCell(2, sn3, CellStates.FEEDING)
......@@ -245,19 +245,19 @@ class PartitionTableTests(NeoUnitTestBase):
num_replicas = 2
pt = PartitionTable(num_partitions, num_replicas)
# add two kind of node, usable and unsable
uuid1 = self.getNewUUID()
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
pt.setCell(0, sn1, CellStates.UP_TO_DATE)
uuid2 = self.getNewUUID()
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19001)
sn2 = StorageNode(Mock(), server2, uuid2)
pt.setCell(0, sn2, CellStates.OUT_OF_DATE)
uuid3 = self.getNewUUID()
uuid3 = self.getStorageUUID()
server3 = ("127.0.0.3", 19001)
sn3 = StorageNode(Mock(), server3, uuid3)
pt.setCell(0, sn3, CellStates.FEEDING)
uuid4 = self.getNewUUID()
uuid4 = self.getStorageUUID()
server4 = ("127.0.0.4", 19001)
sn4 = StorageNode(Mock(), server4, uuid4)
pt.setCell(0, sn4, CellStates.DISCARDED) # won't be added
......@@ -278,7 +278,7 @@ class PartitionTableTests(NeoUnitTestBase):
self.assertEqual(pt.num_filled_rows, 0)
self.assertFalse(pt.filled())
# adding a node in all partition
uuid1 = self.getNewUUID()
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
for x in xrange(num_partitions):
......@@ -291,7 +291,7 @@ class PartitionTableTests(NeoUnitTestBase):
num_replicas = 2
pt = PartitionTable(num_partitions, num_replicas)
# add two kind of node, usable and unsable
uuid1 = self.getNewUUID()
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
pt.setCell(0, sn1, CellStates.UP_TO_DATE)
......@@ -308,7 +308,7 @@ class PartitionTableTests(NeoUnitTestBase):
self.assertFalse(pt.filled())
self.assertFalse(pt.operational())
# adding a node in all partition
uuid1 = self.getNewUUID()
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
for x in xrange(num_partitions):
......@@ -322,7 +322,7 @@ class PartitionTableTests(NeoUnitTestBase):
self.assertFalse(pt.filled())
self.assertFalse(pt.operational())
# adding a node in all partition
uuid1 = self.getNewUUID()
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
for x in xrange(num_partitions):
......@@ -337,7 +337,7 @@ class PartitionTableTests(NeoUnitTestBase):
self.assertFalse(pt.filled())
self.assertFalse(pt.operational())
# adding a node in all partition
uuid1 = self.getNewUUID()
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
sn1.setState(NodeStates.TEMPORARILY_DOWN)
......@@ -352,7 +352,7 @@ class PartitionTableTests(NeoUnitTestBase):
self.assertFalse(pt.filled())
self.assertFalse(pt.operational())
# adding a node in all partition
uuid1 = self.getNewUUID()
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
for x in xrange(num_partitions):
......@@ -366,18 +366,18 @@ class PartitionTableTests(NeoUnitTestBase):
num_replicas = 2
pt = PartitionTable(num_partitions, num_replicas)
# add nodes
uuid1 = self.getNewUUID()
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
pt.setCell(0, sn1, CellStates.UP_TO_DATE)
pt.setCell(1, sn1, CellStates.UP_TO_DATE)
pt.setCell(2, sn1, CellStates.UP_TO_DATE)
uuid2 = self.getNewUUID()
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19001)
sn2 = StorageNode(Mock(), server2, uuid2)
pt.setCell(0, sn2, CellStates.UP_TO_DATE)
pt.setCell(1, sn2, CellStates.UP_TO_DATE)
uuid3 = self.getNewUUID()
uuid3 = self.getStorageUUID()
server3 = ("127.0.0.3", 19001)
sn3 = StorageNode(Mock(), server3, uuid3)
pt.setCell(0, sn3, CellStates.UP_TO_DATE)
......
......@@ -32,8 +32,7 @@ from neo.lib.connection import BaseConnection, Connection
from neo.lib.connector import SocketConnector, \
ConnectorConnectionRefusedException, ConnectorTryAgainException
from neo.lib.event import EventManager
from neo.lib.protocol import CellStates, ClusterStates, NodeStates, NodeTypes, \
UUID_NAMESPACES, INVALID_UUID
from neo.lib.protocol import CellStates, ClusterStates, NodeStates, NodeTypes
from neo.lib.util import SOCKET_CONNECTORS_DICT, parseMasterList, p64
from .. import NeoTestBase, getTempDirectory, setupMySQLdb, \
ADDRESS_TYPE, IP_VERSION_FORMAT_DICT, DB_PREFIX, DB_USER
......@@ -802,27 +801,10 @@ def predictable_random(seed=None):
logging.info("using seed %r", s)
r = random.Random(s)
try:
def getNewUUID(self, uuid, address, node_type):
if node_type == NodeTypes.CLIENT:
return super(MasterApplication, self).getNewUUID(uuid,
address, node_type)
if None != uuid != self.uuid and \
self.nm.getByAddress(address) is \
self.nm.getByUUID(uuid):
return uuid
while True:
uuid = UUID_NAMESPACES[node_type] + ''.join(
chr(r.randrange(256)) for _ in xrange(15))
if uuid != self.uuid and \
self.nm.getByUUID(uuid) is None:
return uuid
MasterApplication.getNewUUID = getNewUUID
administration.random = backup_app.random = replicator.random \
= r
return wrapped(*args, **kw)
finally:
del MasterApplication.getNewUUID
administration.random = backup_app.random = replicator.random \
= random
return wraps(wrapped)(wrapper)
......
......@@ -495,13 +495,13 @@ class Test(NEOThreadedTest):
cluster.reset() # reopen DB to check partition tables
dm = cluster.storage_list[0].dm
self.assertEqual(1, dm.getPTID())
pt = dm.getPartitionTable()
pt = list(dm.getPartitionTable())
self.assertEqual(20, len(pt))
for _, _, state in pt:
self.assertEqual(state, CellStates.UP_TO_DATE)
for s in cluster.storage_list[1:]:
self.assertEqual(s.dm.getPTID(), 1)
self.assertEqual(s.dm.getPartitionTable(), pt)
self.assertEqual(list(s.dm.getPartitionTable()), pt)
if __name__ == "__main__":
......
......@@ -26,8 +26,8 @@ from neo.storage.transactions import TransactionManager, \
DelayedError, ConflictError
from neo.lib.connection import MTClientConnection
from neo.lib.protocol import CellStates, ClusterStates, NodeStates, Packets, \
ZERO_OID, ZERO_TID, MAX_TID
from neo.lib.util import dump, p64
ZERO_OID, ZERO_TID, MAX_TID, uuid_str
from neo.lib.util import p64
from . import NEOCluster, NEOThreadedTest, Patch, predictable_random
from neo.client.pool import CELL_CONNECTED, CELL_GOOD
......@@ -239,7 +239,8 @@ class ReplicationTests(NEOThreadedTest):
def corrupt(offset):
s0, s1, s2 = (storage_dict[cell.getUUID()]
for cell in cluster.master.pt.getCellList(offset, True))
logging.info('corrupt partition %u of %s', offset, dump(s1.uuid))
logging.info('corrupt partition %u of %s',
offset, uuid_str(s1.uuid))
s1.dm.deleteObject(p64(np+offset), p64(corrupt_tid))
return s0.uuid
def check(expected_state, expected_count):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment