Skip to content

N
neoppod
Commit f11bafed authored by Grégory Wisniewski's avatar Grégory Wisniewski
Browse files
Options

First modification of packet management. Encoders are no more instance methods 

but module methods that return Packet instances. Next operation is to set msg_id
related to connection only. Updated tests follow in next commit.


git-svn-id: https://svn.erp5.org/repos/neo/branches/prototype3@481 71dcc9de-d417-0410-9af5-da40c76e7ee4
parent 58edc07e
Expand all Show whitespace changes
Inline Side-by-side
Showing with 424 additions and 626 deletions
neo/admin/handler.py
View file @ f11bafed
......@@ -23,6 +23,7 @@ from neo.protocol import INVALID_UUID, RUNNING_STATE, BROKEN_STATE, \
ADMIN_NODE_TYPE
from neo.node import MasterNode, StorageNode, ClientNode
from neo.connection import ClientConnection
from neo import protocol
from neo.protocol import Packet
from neo.pt import PartitionTable
from neo.exception import PrimaryFailure
......@@ -56,10 +57,8 @@ class MonitoringEventHandler(BaseEventHandler):
# Should not happen.
raise RuntimeError('connection completed while not trying to connect')
p = Packet()
msg_id = conn.getNextId()
p.requestNodeIdentification(msg_id, ADMIN_NODE_TYPE, app.uuid,
app.server[0], app.server[1], app.name)
p = protocol.requestNodeIdentification(conn.getNextId(), ADMIN_NODE_TYPE,
app.uuid, app.server[0], app.server[1], app.name)
conn.addPacket(p)
conn.expectMessage(msg_id)
EventHandler.connectionCompleted(self, conn)
......@@ -173,8 +172,8 @@ class MonitoringEventHandler(BaseEventHandler):
app.uuid = your_uuid
# Ask a primary master.
msg_id = conn.getNextId()
conn.addPacket(Packet().askPrimaryMaster(msg_id))
p = protocol.askPrimaryMaster(conn.getNextId())
conn.addPacket(p)
conn.expectMessage(msg_id)
def handleAnswerPrimaryMaster(self, conn, packet, primary_uuid,
......
neo/client/app.py
View file @ f11bafed
......@@ -27,6 +27,7 @@ from time import sleep
from neo.client.mq import MQ
from neo.node import NodeManager, MasterNode, StorageNode
from neo.connection import MTClientConnection
from neo import protocol
from neo.protocol import Packet, INVALID_UUID, INVALID_TID, INVALID_PARTITION, \
INVALID_PTID, STORAGE_NODE_TYPE, CLIENT_NODE_TYPE, \
RUNNING_STATE, TEMPORARILY_DOWN_STATE, \
......@@ -85,10 +86,8 @@ class ConnectionPool(object):
return None
msg_id = conn.getNextId()
p = Packet()
p.requestNodeIdentification(msg_id, CLIENT_NODE_TYPE,
app.uuid, addr[0],
addr[1], app.name)
p = protocol.requestNodeIdentification(msg_id, CLIENT_NODE_TYPE,
app.uuid, addr[0], addr[1], app.name)
conn.addPacket(p)
conn.expectMessage(msg_id)
app.dispatcher.register(conn, msg_id, app.getQueue())
......@@ -305,8 +304,7 @@ class Application(object):
conn.lock()
try:
msg_id = conn.getNextId()
p = Packet()
p.askNewOIDs(msg_id, 25)
p = protocol.askNewOIDs(msg_id, 25)
conn.addPacket(p)
conn.expectMessage(msg_id)
self.dispatcher.register(conn, msg_id, self.getQueue())
......@@ -365,8 +363,7 @@ class Application(object):
try:
msg_id = conn.getNextId()
p = Packet()
p.askObject(msg_id, oid, serial, tid)
p = protocol.askObject(msg_id, oid, serial, tid)
conn.addPacket(p)
conn.expectMessage(msg_id)
self.dispatcher.register(conn, msg_id, self.getQueue())
......@@ -471,8 +468,7 @@ class Application(object):
conn.lock()
try:
msg_id = conn.getNextId()
p = Packet()
p.askNewTID(msg_id)
p = protocol.askNewTID(msg_id)
conn.addPacket(p)
conn.expectMessage(msg_id)
self.dispatcher.register(conn, msg_id, self.getQueue())
......@@ -516,8 +512,7 @@ class Application(object):
try:
msg_id = conn.getNextId()
p = Packet()
p.askStoreObject(msg_id, oid, serial, 1,
p = protocol.askStoreObject(msg_id, oid, serial, 1,
checksum, compressed_data, self.local_var.tid)
conn.addPacket(p)
conn.expectMessage(msg_id)
......@@ -568,9 +563,8 @@ class Application(object):
try:
msg_id = conn.getNextId()
p = Packet()
p.askStoreTransaction(msg_id, self.local_var.tid, user, desc, ext,
oid_list)
p = protocol.askStoreTransaction(msg_id, self.local_var.tid,
user, desc, ext, oid_list)
conn.addPacket(p)
conn.expectMessage(msg_id)
self.dispatcher.register(conn, msg_id, self.getQueue())
......@@ -616,7 +610,7 @@ class Application(object):
if conn is None:
continue
try:
conn.addPacket(Packet().abortTransaction(conn.getNextId(), self.local_var.tid))
conn.addPacket(protocol.abortTransaction(conn.getNextId(), self.local_var.tid))
finally:
conn.unlock()
......@@ -624,7 +618,7 @@ class Application(object):
conn = self.master_conn
conn.lock()
try:
conn.addPacket(Packet().abortTransaction(conn.getNextId(), self.local_var.tid))
conn.addPacket(protocol.abortTransaction(conn.getNextId(), self.local_var.tid))
finally:
conn.unlock()
......@@ -646,8 +640,7 @@ class Application(object):
conn.lock()
try:
msg_id = conn.getNextId()
p = Packet()
p.finishTransaction(msg_id, oid_list, self.local_var.tid)
p = protocol.finishTransaction(msg_id, oid_list, self.local_var.tid)
conn.addPacket(p)
conn.expectMessage(msg_id, additional_timeout = 300)
self.dispatcher.register(conn, msg_id, self.getQueue())
......@@ -693,8 +686,7 @@ class Application(object):
try:
msg_id = conn.getNextId()
p = Packet()
p.askTransactionInformation(msg_id, transaction_id)
p = protocol.askTransactionInformation(msg_id, transaction_id)
conn.addPacket(p)
conn.expectMessage(msg_id)
self.dispatcher.register(conn, msg_id, self.getQueue())
......@@ -770,8 +762,7 @@ class Application(object):
try:
msg_id = conn.getNextId()
p = Packet()
p.askTIDs(msg_id, first, last, INVALID_PARTITION)
p = protocol.askTIDs(msg_id, first, last, INVALID_PARTITION)
conn.addPacket(p)
conn.expectMessage(msg_id)
self.dispatcher.register(conn, msg_id, self.getQueue())
......@@ -809,8 +800,7 @@ class Application(object):
try:
msg_id = conn.getNextId()
p = Packet()
p.askTransactionInformation(msg_id, tid)
p = protocol.askTransactionInformation(msg_id, tid)
conn.addPacket(p)
conn.expectMessage(msg_id)
self.dispatcher.register(conn, msg_id, self.getQueue())
......@@ -865,8 +855,7 @@ class Application(object):
try:
msg_id = conn.getNextId()
p = Packet()
p.askObjectHistory(msg_id, oid, 0, length)
p = protocol.askObjectHistory(msg_id, oid, 0, length)
conn.addPacket(p)
conn.expectMessage(msg_id)
self.dispatcher.register(conn, msg_id, self.getQueue())
......@@ -906,8 +895,7 @@ class Application(object):
try:
msg_id = conn.getNextId()
p = Packet()
p.askTransactionInformation(msg_id, serial)
p = protocol.askTransactionInformation(msg_id, serial)
conn.addPacket(p)
conn.expectMessage(msg_id)
self.dispatcher.register(conn, msg_id, self.getQueue())
......@@ -989,9 +977,8 @@ class Application(object):
conn.lock()
try:
msg_id = conn.getNextId()
p = Packet()
p.requestNodeIdentification(msg_id, CLIENT_NODE_TYPE, self.uuid,
'0.0.0.0', 0, self.name)
p = protocol.requestNodeIdentification(msg_id, CLIENT_NODE_TYPE,
self.uuid, '0.0.0.0', 0, self.name)
conn.addPacket(p)
conn.expectMessage(msg_id)
self.dispatcher.register(conn, msg_id, self.getQueue())
......
neo/client/handler.py
View file @ f11bafed
......@@ -19,6 +19,7 @@ import logging
from neo.handler import EventHandler
from neo.connection import MTClientConnection
from neo import protocol
from neo.protocol import Packet, \
MASTER_NODE_TYPE, STORAGE_NODE_TYPE, CLIENT_NODE_TYPE, \
INVALID_UUID, RUNNING_STATE, TEMPORARILY_DOWN_STATE, \
......@@ -78,12 +79,10 @@ class BaseClientEventHandler(EventHandler):
if conn is not None:
conn.lock()
try:
msg_id = conn.getNextId()
p = Packet()
ip_address, port = node.getServer()
node_list = [(STORAGE_NODE_TYPE, ip_address, port,
node.getUUID(), state)]
p.notifyNodeInformation(msg_id, node_list)
p = protocol.notifyNodeInformation(conn.getNextId(), node_list)
conn.addPacket(p)
finally:
conn.unlock()
......@@ -151,8 +150,7 @@ class PrimaryBoostrapEventHandler(BaseClientEventHandler):
conn.lock()
try:
msg_id = conn.getNextId()
p = Packet()
p.askPrimaryMaster(msg_id)
p = protocol.askPrimaryMaster(msg_id)
conn.addPacket(p)
conn.expectMessage(msg_id)
self.dispatcher.register(conn, msg_id, app.getQueue())
......
neo/connection.py
View file @ f11bafed
......@@ -20,6 +20,7 @@ from neo.locking import RLock
import sys
import traceback
from neo import protocol
from neo.protocol import Packet, ProtocolError
from neo.event import IdleEvent
from neo.connector import ConnectorTryAgainException, ConnectorInProgressException
......@@ -286,7 +287,7 @@ class Connection(BaseConnection):
self.write_buf += packet.encode()
except ProtocolError, m:
logging.critical('trying to send a too big message')
return self.addPacket(packet.internalError(packet.getId(), m[0]))
return self.addPacket(protocol.internalError(packet.getId(), m[0]))
# If this is the first time, enable polling for writing.
if self.write_buf:
......
neo/event.py
View file @ f11bafed
......@@ -19,6 +19,7 @@ import logging
from select import select
from time import time
from neo import protocol
from neo.protocol import Packet
from neo.epoll import Epoll
......@@ -65,7 +66,7 @@ class IdleEvent(object):
logging.info('sending a ping to %s:%d',
*(conn.getAddress()))
msg_id = conn.getNextId()
conn.addPacket(Packet().ping(msg_id))
conn.addPacket(protocol.ping(msg_id))
conn.expectMessage(msg_id, 5, 0)
else:
conn.expectMessage(self._id, self._additional_timeout, 0)
......
neo/handler.py
View file @ f11bafed
......@@ -17,6 +17,7 @@
import logging
from neo import protocol
from neo.protocol import Packet, ProtocolError
from neo.connection import ServerConnection
......@@ -82,7 +83,7 @@ class EventHandler(object):
logging.info('malformed packet %x from %s:%d: %s',
packet.getType(), conn.getAddress()[0],
conn.getAddress()[1], error_message)
conn.addPacket(Packet().protocolError(packet.getId(), error_message))
conn.addPacket(protocol.protocolError(packet.getId(), error_message))
conn.abort()
self.peerBroken(conn)
......@@ -109,7 +110,7 @@ class EventHandler(object):
else:
message = 'unexpected packet: ' + message
logging.info('%s', message)
conn.addPacket(Packet().protocolError(packet.getId(), message))
conn.addPacket(protocol.protocolError(packet.getId(), message))
conn.abort()
self.peerBroken(conn)
......@@ -133,7 +134,7 @@ class EventHandler(object):
def handlePing(self, conn, packet):
logging.info('got a ping packet; am I overloaded?')
conn.addPacket(Packet().pong(packet.getId()))
conn.addPacket(protocol.pong(packet.getId()))
def handlePong(self, conn, packet):
pass
......
neo/master/app.py
View file @ f11bafed
......@@ -21,6 +21,7 @@ from time import time, gmtime
from struct import pack, unpack
from neo.config import ConfigurationManager
from neo import protocol
from neo.protocol import Packet, \
RUNNING_STATE, TEMPORARILY_DOWN_STATE, DOWN_STATE, BROKEN_STATE, \
INVALID_UUID, INVALID_OID, INVALID_TID, INVALID_PTID, \
......@@ -188,7 +189,7 @@ class Application(object):
logging.info('I am the primary, so sending an announcement')
for conn in em.getConnectionList():
if isinstance(conn, ClientConnection):
p = Packet().announcePrimaryMaster(conn.getNextId())
p = protocol.announcePrimaryMaster(conn.getNextId())
conn.addPacket(p)
conn.abort()
closed = False
......@@ -238,7 +239,7 @@ class Application(object):
# Ask all connected nodes to reelect a single primary master.
for conn in em.getConnectionList():
if isinstance(conn, ClientConnection):
conn.addPacket(Packet().reelectPrimaryMaster(conn.getNextId()))
conn.addPacket(protocol.reelectPrimaryMaster(conn.getNextId()))
conn.abort()
# Wait until the connections are closed.
......@@ -293,16 +294,14 @@ class Application(object):
if c.getUUID() is not None:
n = self.nm.getNodeByUUID(c.getUUID())
if n.getNodeType() in (MASTER_NODE_TYPE, STORAGE_NODE_TYPE, ADMIN_NODE_TYPE):
p = Packet()
node_list = [(node_type, ip_address, port, uuid, state)]
p.notifyNodeInformation(c.getNextId(), node_list)
p = protocol.notifyNodeInformation(c.getNextId(), node_list)
c.addPacket(p)
elif node.getNodeType() in (MASTER_NODE_TYPE, STORAGE_NODE_TYPE):
for c in self.em.getConnectionList():
if c.getUUID() is not None:
p = Packet()
node_list = [(node_type, ip_address, port, uuid, state)]
p.notifyNodeInformation(c.getNextId(), node_list)
p = protocol.notifyNodeInformation(c.getNextId(), node_list)
c.addPacket(p)
elif node.getNodeType() != ADMIN_NODE_TYPE:
raise RuntimeError('unknown node type')
......@@ -319,8 +318,7 @@ class Application(object):
start = 0
while size:
amt = min(10000, size)
p = Packet()
p.notifyPartitionChanges(c.getNextId(), ptid,
p = protocol.notifyPartitionChanges(c.getNextId(), ptid,
cell_list[start:start+amt])
c.addPacket(p)
size -= amt
......@@ -356,9 +354,8 @@ class Application(object):
node = nm.getNodeByUUID(uuid)
if node.getNodeType() == STORAGE_NODE_TYPE \
and node.getState() == RUNNING_STATE:
p = Packet()
msg_id = conn.getNextId()
p.askLastIDs(msg_id)
p = protocol.askLastIDs(msg_id)
conn.addPacket(p)
conn.expectMessage(msg_id)
......@@ -401,8 +398,7 @@ class Application(object):
while size:
amt = min(1000, size)
msg_id = conn.getNextId()
p = Packet()
p.askPartitionTable(msg_id, range(start, start + amt))
p = protocol.askPartitionTable(msg_id, range(start, start + amt))
conn.addPacket(p)
conn.expectMessage(msg_id)
size -= amt
......@@ -459,9 +455,8 @@ class Application(object):
uuid = conn.getUUID()
if uuid in transaction_uuid_list:
self.asking_uuid_dict[uuid] = False
p = Packet()
msg_id = conn.getNextId()
p.askTransactionInformation(msg_id, tid)
p = protocol.askTransactionInformation(msg_id, tid)
conn.addPacket(p)
conn.expectMessage(msg_id)
if len(self.asking_uuid_dict) == 0:
......@@ -493,9 +488,8 @@ class Application(object):
uuid = conn.getUUID()
if uuid in object_uuid_list:
self.asking_uuid_dict[uuid] = False
p = Packet()
msg_id = conn.getNextId()
p.askObjectPresent(msg_id, oid, tid)
p = protocol.askObjectPresent(msg_id, oid, tid)
conn.addPacket(p)
conn.expectMessage(msg_id)
......@@ -540,14 +534,12 @@ class Application(object):
for offset in xrange(self.num_partitions):
row_list.append((offset, self.pt.getRow(offset)))
if len(row_list) == 1000:
p = Packet()
p.sendPartitionTable(conn.getNextId(),
p = protocol.sendPartitionTable(conn.getNextId(),
self.lptid, row_list)
conn.addPacket(p)
del row_list[:]
if len(row_list) != 0:
p = Packet()
p.sendPartitionTable(conn.getNextId(),
p = protocol.sendPartitionTable(conn.getNextId(),
self.lptid, row_list)
conn.addPacket(p)
......@@ -575,9 +567,8 @@ class Application(object):
node = nm.getNodeByUUID(uuid)
if node.getNodeType() == STORAGE_NODE_TYPE:
self.asking_uuid_dict[uuid] = False
p = Packet()
msg_id = conn.getNextId()
p.askUnfinishedTransactions(msg_id)
p = protocol.askUnfinishedTransactions(msg_id)
conn.addPacket(p)
conn.expectMessage(msg_id)
......@@ -600,15 +591,13 @@ class Application(object):
if uuid is not None:
node = nm.getNodeByUUID(uuid)
if node.getNodeType() == STORAGE_NODE_TYPE:
p = Packet()
p.deleteTransaction(conn.getNextId(), tid)
p = protocol.deleteTransaction(conn.getNextId(), tid)
conn.addPacket(p)
else:
for conn in em.getConnectionList():
uuid = conn.getUUID()
if uuid in uuid_set:
p = Packet()
p.commitTransaction(conn.getNextId(), tid)
p = protocol.commitTransaction(conn.getNextId(), tid)
conn.addPacket(p)
# If possible, send the packets now.
......@@ -655,7 +644,7 @@ class Application(object):
if uuid is not None:
node = nm.getNodeByUUID(uuid)
if node.getNodeType() == STORAGE_NODE_TYPE:
conn.addPacket(Packet().startOperation(conn.getNextId()))
conn.addPacket(protocol.startOperation(conn.getNextId()))
# Now everything is passive.
expiration = 10
......@@ -693,7 +682,7 @@ class Application(object):
if uuid is not None:
node = nm.getNodeByUUID(uuid)
if node.getNodeType() in (STORAGE_NODE_TYPE, CLIENT_NODE_TYPE):
conn.addPacket(Packet().stopOperation(conn.getNextId()))
conn.addPacket(protocol.stopOperation(conn.getNextId()))
if node.getNodeType() == CLIENT_NODE_TYPE:
conn.abort()
......
neo/master/election.py
View file @ f11bafed
......@@ -17,6 +17,7 @@
import logging
from neo import protocol
from neo.protocol import MASTER_NODE_TYPE, \
RUNNING_STATE, BROKEN_STATE, TEMPORARILY_DOWN_STATE, \
DOWN_STATE, ADMIN_NODE_TYPE
......@@ -39,9 +40,8 @@ class ElectionEventHandler(MasterEventHandler):
def connectionCompleted(self, conn):
app = self.app
# Request a node idenfitication.
p = Packet()
msg_id = conn.getNextId()
p.requestNodeIdentification(msg_id, MASTER_NODE_TYPE, app.uuid,
p = protocol.requestNodeIdentification(msg_id, MASTER_NODE_TYPE, app.uuid,
app.server[0], app.server[1], app.name)
conn.addPacket(p)
conn.expectMessage(msg_id)
......@@ -122,7 +122,7 @@ class ElectionEventHandler(MasterEventHandler):
# Ask a primary master.
msg_id = conn.getNextId()
conn.addPacket(Packet().askPrimaryMaster(msg_id))
conn.addPacket(protocol.askPrimaryMaster(msg_id))
conn.expectMessage(msg_id)
else:
self.handleUnexpectedPacket(conn, packet)
......@@ -183,12 +183,12 @@ class ElectionEventHandler(MasterEventHandler):
app = self.app
if node_type != MASTER_NODE_TYPE:
logging.info('reject a connection from a non-master')
conn.addPacket(Packet().notReady(packet.getId(), 'retry later'))
conn.addPacket(protocol.notReady(packet.getId(), 'retry later'))
conn.abort()
return
if name != app.name:
logging.error('reject an alien cluster')
conn.addPacket(Packet().protocolError(packet.getId(),
conn.addPacket(protocol.protocolError(packet.getId(),
'invalid cluster name'))
conn.abort()
return
......@@ -203,8 +203,7 @@ class ElectionEventHandler(MasterEventHandler):
# If this node is broken, reject it.
if node.getUUID() == uuid:
if node.getState() == BROKEN_STATE:
p = Packet()
p.brokenNodeDisallowedError(packet.getId(), 'go away')
p = protocol.brokenNodeDisallowedError(packet.getId(), 'go away')
conn.addPacket(p)
conn.abort()
return
......@@ -216,8 +215,7 @@ class ElectionEventHandler(MasterEventHandler):
node.setUUID(uuid)
conn.setUUID(uuid)
p = Packet()
p.acceptNodeIdentification(packet.getId(), MASTER_NODE_TYPE,
p = protocol.acceptNodeIdentification(packet.getId(), MASTER_NODE_TYPE,
app.uuid, app.server[0], app.server[1],
app.num_partitions, app.num_replicas,
uuid)
......@@ -248,8 +246,7 @@ class ElectionEventHandler(MasterEventHandler):
continue
info = n.getServer() + (n.getUUID() or INVALID_UUID,)
known_master_list.append(info)
p = Packet()
p.answerPrimaryMaster(packet.getId(), primary_uuid, known_master_list)
p = protocol.answerPrimaryMaster(packet.getId(), primary_uuid, known_master_list)
conn.addPacket(p)
def handleAnnouncePrimaryMaster(self, conn, packet):
......
neo/master/recovery.py
View file @ f11bafed
......@@ -17,6 +17,7 @@
import logging
from neo import protocol
from neo.protocol import MASTER_NODE_TYPE, \
RUNNING_STATE, BROKEN_STATE, TEMPORARILY_DOWN_STATE, DOWN_STATE, \
STORAGE_NODE_TYPE, CLIENT_NODE_TYPE, ADMIN_NODE_TYPE
......@@ -67,12 +68,12 @@ class RecoveryEventHandler(MasterEventHandler):
app = self.app
if node_type not in (MASTER_NODE_TYPE, STORAGE_NODE_TYPE, ADMIN_NODE_TYPE):
logging.info('reject a connection from a client')
conn.addPacket(Packet().notReady(packet.getId(), 'retry later'))
conn.addPacket(protocol.notReady(packet.getId(), 'retry later'))
conn.abort()
return
if name != app.name:
logging.error('reject an alien cluster')
conn.addPacket(Packet().protocolError(packet.getId(),
conn.addPacket(protocol.protocolError(packet.getId(),
'invalid cluster name'))
conn.abort()
return
......@@ -117,7 +118,7 @@ class RecoveryEventHandler(MasterEventHandler):
if node.getNodeType() != MASTER_NODE_TYPE or node_type != MASTER_NODE_TYPE:
# Error. This node uses the same server address as a master
# node.
conn.addPacket(Packet().protocolError(packet.getId(),
conn.addPacket(protocol.protocolError(packet.getId(),
'invalid server address'))
conn.abort()
return
......@@ -130,7 +131,7 @@ class RecoveryEventHandler(MasterEventHandler):
# This node has a different UUID.
if node.getState() == RUNNING_STATE:
# If it is still running, reject this node.
conn.addPacket(Packet().protocolError(packet.getId(),
conn.addPacket(protocol.protocolError(packet.getId(),
'invalid server address'))
conn.abort()
return
......@@ -149,7 +150,7 @@ class RecoveryEventHandler(MasterEventHandler):
if node.getState() == RUNNING_STATE:
# If it is still running, reject this node.
conn.addPacket(Packet().protocolError(packet.getId(),
conn.addPacket(protocol.protocolError(packet.getId(),
'invalid server address'))
conn.abort()
return
......@@ -165,8 +166,7 @@ class RecoveryEventHandler(MasterEventHandler):
# If this node is broken, reject it. Otherwise, assume that it is
# working again.
if node.getState() == BROKEN_STATE:
p = Packet()
p.brokenNodeDisallowedError(packet.getId(), 'go away')
p = protocol.brokenNodeDisallowedError(packet.getId(), 'go away')
conn.addPacket(p)
conn.abort()
return
......@@ -177,8 +177,7 @@ class RecoveryEventHandler(MasterEventHandler):
conn.setUUID(uuid)
p = Packet()
p.acceptNodeIdentification(packet.getId(), MASTER_NODE_TYPE,
p = protocol.acceptNodeIdentification(packet.getId(), MASTER_NODE_TYPE,
app.uuid, app.server[0], app.server[1],
app.num_partitions, app.num_replicas, uuid)
conn.addPacket(p)
......@@ -196,8 +195,7 @@ class RecoveryEventHandler(MasterEventHandler):
# Merely tell the peer that I am the primary master node.
# It is not necessary to send known master nodes, because
# I must send all node information immediately.
p = Packet()
p.answerPrimaryMaster(packet.getId(), app.uuid, [])
p = protocol.answerPrimaryMaster(packet.getId(), app.uuid, [])
conn.addPacket(p)
# Send the information.
......@@ -211,20 +209,17 @@ class RecoveryEventHandler(MasterEventHandler):
n.getUUID() or INVALID_UUID, n.getState()))
if len(node_list) == 10000:
# Ugly, but it is necessary to split a packet, if it is too big.
p = Packet()
p.notifyNodeInformation(conn.getNextId(), node_list)
p = protocol.notifyNodeInformation(conn.getNextId(), node_list)
conn.addPacket(p)
del node_list[:]
p = Packet()
p.notifyNodeInformation(conn.getNextId(), node_list)
p = protocol.notifyNodeInformation(conn.getNextId(), node_list)
conn.addPacket(p)
# If this is a storage node, ask the last IDs.
node = app.nm.getNodeByUUID(uuid)
if node.getNodeType() == STORAGE_NODE_TYPE:
p = Packet()
msg_id = conn.getNextId()
p.askLastIDs(msg_id)
p = protocol.askLastIDs(msg_id)
conn.addPacket(p)
conn.expectMessage(msg_id)
elif node.getNodeType() == ADMIN_NODE_TYPE and app.lptid != INVALID_PTID:
......@@ -232,16 +227,15 @@ class RecoveryEventHandler(MasterEventHandler):
logging.info('sending partition table %s to %s' % (dump(app.lptid),
conn.getAddress()))
# Split the packet if too huge.
p = Packet()
row_list = []
for offset in xrange(app.num_partitions):
row_list.append((offset, app.pt.getRow(offset)))
if len(row_list) == 1000:
p.sendPartitionTable(conn.getNextId(), app.lptid, row_list)
p = protocol.sendPartitionTable(conn.getNextId(), app.lptid, row_list)
conn.addPacket(p)
del row_list[:]
if len(row_list) != 0:
p.sendPartitionTable(conn.getNextId(), app.lptid, row_list)
p = protocol.sendPartitionTable(conn.getNextId(), app.lptid, row_list)
conn.addPacket(p)
def handleAnnouncePrimaryMaster(self, conn, packet):
......
neo/master/secondary.py
View file @ f11bafed
......@@ -17,6 +17,7 @@
import logging
from neo import protocol
from neo.protocol import MASTER_NODE_TYPE, \
RUNNING_STATE, BROKEN_STATE, TEMPORARILY_DOWN_STATE, \
DOWN_STATE, ADMIN_NODE_TYPE
......@@ -62,7 +63,7 @@ class SecondaryEventHandler(MasterEventHandler):
app = self.app
if name != app.name:
logging.error('reject an alien cluster')
conn.addPacket(Packet().protocolError(packet.getId(),
conn.addPacket(protocol.protocolError(packet.getId(),
'invalid cluster name'))
conn.abort()
return
......@@ -80,8 +81,7 @@ class SecondaryEventHandler(MasterEventHandler):
conn.setUUID(uuid)
p = Packet()
p.acceptNodeIdentification(packet.getId(), MASTER_NODE_TYPE,
p = protocol.acceptNodeIdentification(packet.getId(), MASTER_NODE_TYPE,
app.uuid, app.server[0], app.server[1],
app.num_partitions, app.num_replicas,
uuid)
......@@ -108,8 +108,7 @@ class SecondaryEventHandler(MasterEventHandler):
info = n.getServer() + (n.getUUID() or INVALID_UUID,)
known_master_list.append(info)
p = Packet()
p.answerPrimaryMaster(packet.getId(), primary_uuid, known_master_list)
p = protocol.answerPrimaryMaster(packet.getId(), primary_uuid, known_master_list)
conn.addPacket(p)
def handleAnnouncePrimaryMaster(self, conn, packet):
......
neo/master/service.py
View file @ f11bafed
......@@ -18,6 +18,7 @@
import logging
from copy import copy
from neo import protocol
from neo.protocol import MASTER_NODE_TYPE, CLIENT_NODE_TYPE, \
RUNNING_STATE, BROKEN_STATE, TEMPORARILY_DOWN_STATE, DOWN_STATE, \
UP_TO_DATE_STATE, FEEDING_STATE, DISCARDED_STATE, \
......@@ -152,7 +153,7 @@ class ServiceEventHandler(MasterEventHandler):
app = self.app
if name != app.name:
logging.error('reject an alien cluster')
conn.addPacket(Packet().protocolError(packet.getId(),
conn.addPacket(protocol.protocolError(packet.getId(),
'invalid cluster name'))
conn.abort()
return
......@@ -201,8 +202,7 @@ class ServiceEventHandler(MasterEventHandler):
or node_type != MASTER_NODE_TYPE:
# Error. This node uses the same server address as
# a master node.
p = Packet()
p.protocolError(packet.getId(),
p = protocol.protocolError(packet.getId(),
'invalid server address')
conn.addPacket(p)
conn.abort()
......@@ -217,8 +217,7 @@ class ServiceEventHandler(MasterEventHandler):
# This node has a different UUID.
if node.getState() == RUNNING_STATE:
# If it is still running, reject this node.
p = Packet()
p.protocolError(packet.getId(),
p = protocol.protocolError(packet.getId(),
'invalid server address')
conn.addPacket(p)
conn.abort()
......@@ -247,8 +246,7 @@ class ServiceEventHandler(MasterEventHandler):
# This node has a different server address.
if node.getState() == RUNNING_STATE:
# If it is still running, reject this node.
p = Packet()
p.protocolError(packet.getId(),
p = protocol.protocolError(packet.getId(),
'invalid server address')
conn.addPacket(p)
conn.abort()
......@@ -271,8 +269,7 @@ class ServiceEventHandler(MasterEventHandler):
# If this node is broken, reject it. Otherwise, assume that
# it is working again.
if node.getState() == BROKEN_STATE:
p = Packet()
p.brokenNodeDisallowedError(packet.getId(), 'go away')
p = protocol.brokenNodeDisallowedError(packet.getId(), 'go away')
conn.addPacket(p)
conn.abort()
return
......@@ -300,8 +297,7 @@ class ServiceEventHandler(MasterEventHandler):
ptid = app.getNextPartitionTableID()
app.broadcastPartitionChanges(ptid, cell_list)
p = Packet()
p.acceptNodeIdentification(packet.getId(), MASTER_NODE_TYPE,
p = protocol.acceptNodeIdentification(packet.getId(), MASTER_NODE_TYPE,
app.uuid, app.server[0], app.server[1],
app.num_partitions, app.num_replicas, uuid)
conn.addPacket(p)
......@@ -319,8 +315,7 @@ class ServiceEventHandler(MasterEventHandler):
# Merely tell the peer that I am the primary master node.
# It is not necessary to send known master nodes, because
# I must send all node information immediately.
p = Packet()
p.answerPrimaryMaster(packet.getId(), app.uuid, [])
p = protocol.answerPrimaryMaster(packet.getId(), app.uuid, [])
conn.addPacket(p)
# Send the information.
......@@ -336,12 +331,10 @@ class ServiceEventHandler(MasterEventHandler):
n.getUUID() or INVALID_UUID, n.getState()))
if len(node_list) == 10000:
# Ugly, but it is necessary to split a packet, if it is too big.
p = Packet()
p.notifyNodeInformation(conn.getNextId(), node_list)
p = protocol.notifyNodeInformation(conn.getNextId(), node_list)
conn.addPacket(p)
del node_list[:]
p = Packet()
p.notifyNodeInformation(conn.getNextId(), node_list)
p = protocol.notifyNodeInformation(conn.getNextId(), node_list)
conn.addPacket(p)
# If this is a storage node or a client node or an admin node, send the partition table.
......@@ -350,21 +343,20 @@ class ServiceEventHandler(MasterEventHandler):
logging.info('sending partition table to %s:%d',
*(conn.getAddress()))
# Split the packet if too huge.
p = Packet()
row_list = []
for offset in xrange(app.num_partitions):
row_list.append((offset, app.pt.getRow(offset)))
if len(row_list) == 1000:
p.sendPartitionTable(conn.getNextId(), app.lptid, row_list)
p = protocol.sendPartitionTable(conn.getNextId(), app.lptid, row_list)
conn.addPacket(p)
del row_list[:]
if len(row_list) != 0:
p.sendPartitionTable(conn.getNextId(), app.lptid, row_list)
p = protocol.sendPartitionTable(conn.getNextId(), app.lptid, row_list)
conn.addPacket(p)
# If this is a storage node, ask it to start.
if node.getNodeType() == STORAGE_NODE_TYPE:
conn.addPacket(Packet().startOperation(conn.getNextId()))
conn.addPacket(protocol.startOperation(conn.getNextId()))
def handleAnnouncePrimaryMaster(self, conn, packet):
uuid = conn.getUUID()
......@@ -474,7 +466,7 @@ class ServiceEventHandler(MasterEventHandler):
return
tid = app.getNextTID()
app.finishing_transaction_dict[tid] = FinishingTransaction(conn)
conn.addPacket(Packet().answerNewTID(packet.getId(), tid))
conn.addPacket(protocol.answerNewTID(packet.getId(), tid))
def handleAskNewOIDs(self, conn, packet, num_oids):
uuid = conn.getUUID()
......@@ -490,7 +482,7 @@ class ServiceEventHandler(MasterEventHandler):
return
oid_list = app.getNewOIDList(num_oids)
conn.addPacket(Packet().answerNewOIDs(packet.getId(), oid_list))
conn.addPacket(protocol.answerNewOIDs(packet.getId(), oid_list))
def handleFinishTransaction(self, conn, packet, oid_list, tid):
uuid = conn.getUUID()
......@@ -527,7 +519,7 @@ class ServiceEventHandler(MasterEventHandler):
for c in app.em.getConnectionList():
if c.getUUID() in uuid_set:
msg_id = c.getNextId()
c.addPacket(Packet().lockInformation(msg_id, tid))
c.addPacket(protocol.lockInformation(msg_id, tid))
c.expectMessage(msg_id, timeout = 60)
try:
......@@ -569,20 +561,19 @@ class ServiceEventHandler(MasterEventHandler):
for c in app.em.getConnectionList():
uuid = c.getUUID()
if uuid is not None:
p = Packet()
node = app.nm.getNodeByUUID(uuid)
if node.getNodeType() == CLIENT_NODE_TYPE:
if c is t.getConnection():
p.notifyTransactionFinished(t.getMessageId(),
tid)
p = protocol.notifyTransactionFinished(
t.getMessageId(), tid)
c.addPacket(p)
else:
p.invalidateObjects(c.getNextId(),
p = protocol.invalidateObjects(c.getNextId(),
t.getOIDList(), tid)
c.addPacket(p)
elif node.getNodeType() == STORAGE_NODE_TYPE:
if uuid in t.getUUIDSet():
p.unlockInformation(c.getNextId(), tid)
p = protocol.unlockInformation(c.getNextId(), tid)
c.addPacket(p)
del app.finishing_transaction_dict[tid]
except KeyError:
......@@ -615,8 +606,7 @@ class ServiceEventHandler(MasterEventHandler):
return
app = self.app
p = Packet()
p.answerLastIDs(packet.getId(), app.loid, app.ltid, app.lptid)
p = protocol.answerLastIDs(packet.getId(), app.loid, app.ltid, app.lptid)
conn.addPacket(p)
def handleAskUnfinishedTransactions(self, conn, packet):
......@@ -626,8 +616,7 @@ class ServiceEventHandler(MasterEventHandler):
return
app = self.app
p = Packet()
p.answerUnfinishedTransactions(packet.getId(),
p = protocol.answerUnfinishedTransactions(packet.getId(),
app.finishing_transaction_dict.keys())
conn.addPacket(p)
......
neo/master/verification.py
View file @ f11bafed
......@@ -17,6 +17,7 @@
import logging
from neo import protocol
from neo.protocol import MASTER_NODE_TYPE, STORAGE_NODE_TYPE, CLIENT_NODE_TYPE, \
RUNNING_STATE, BROKEN_STATE, TEMPORARILY_DOWN_STATE, DOWN_STATE, \
ADMIN_NODE_TYPE
......@@ -91,12 +92,12 @@ class VerificationEventHandler(MasterEventHandler):
app = self.app
if node_type not in (MASTER_NODE_TYPE, STORAGE_NODE_TYPE, ADMIN_NODE_TYPE):
logging.info('reject a connection from a client')
conn.addPacket(Packet().notReady(packet.getId(), 'retry later'))
conn.addPacket(protocol.notReady(packet.getId(), 'retry later'))
conn.abort()
return
if name != app.name:
logging.error('reject an alien cluster')
conn.addPacket(Packet().protocolError(packet.getId(),
conn.addPacket(protocol.protocolError(packet.getId(),
'invalid cluster name'))
conn.abort()
return
......@@ -141,7 +142,7 @@ class VerificationEventHandler(MasterEventHandler):
if node.getNodeType() != MASTER_NODE_TYPE or node_type != MASTER_NODE_TYPE:
# Error. This node uses the same server address as a master
# node.
conn.addPacket(Packet().protocolError(packet.getId(),
conn.addPacket(protocol.protocolError(packet.getId(),
'invalid server address'))
conn.abort()
return
......@@ -154,7 +155,7 @@ class VerificationEventHandler(MasterEventHandler):
# This node has a different UUID.
if node.getState() == RUNNING_STATE:
# If it is still running, reject this node.
conn.addPacket(Packet().protocolError(packet.getId(),
conn.addPacket(protocol.protocolError(packet.getId(),
'invalid server address'))
conn.abort()
return
......@@ -172,7 +173,7 @@ class VerificationEventHandler(MasterEventHandler):
# This node has a different server address.
if node.getState() == RUNNING_STATE:
# If it is still running, reject this node.
conn.addPacket(Packet().protocolError(packet.getId(),
conn.addPacket(protocol.protocolError(packet.getId(),
'invalid server address'))
conn.abort()
return
......@@ -188,8 +189,7 @@ class VerificationEventHandler(MasterEventHandler):
# If this node is broken, reject it. Otherwise, assume that it is
# working again.
if node.getState() == BROKEN_STATE:
p = Packet()
p.brokenNodeDisallowedError(packet.getId(), 'go away')
p = protocol.brokenNodeDisallowedError(packet.getId(), 'go away')
conn.addPacket(p)
conn.abort()
return
......@@ -200,8 +200,7 @@ class VerificationEventHandler(MasterEventHandler):
conn.setUUID(uuid)
p = Packet()
p.acceptNodeIdentification(packet.getId(), MASTER_NODE_TYPE,
p = protocol.acceptNodeIdentification(packet.getId(), MASTER_NODE_TYPE,
app.uuid, app.server[0], app.server[1],
app.num_partitions, app.num_replicas, uuid)
conn.addPacket(p)
......@@ -219,8 +218,7 @@ class VerificationEventHandler(MasterEventHandler):
# Merely tell the peer that I am the primary master node.
# It is not necessary to send known master nodes, because
# I must send all node information immediately.
p = Packet()
p.answerPrimaryMaster(packet.getId(), app.uuid, [])
p = protocol.answerPrimaryMaster(packet.getId(), app.uuid, [])
conn.addPacket(p)
# Send the information.
......@@ -234,28 +232,25 @@ class VerificationEventHandler(MasterEventHandler):
n.getUUID() or INVALID_UUID, n.getState()))
if len(node_list) == 10000:
# Ugly, but it is necessary to split a packet, if it is too big.
p = Packet()
p.notifyNodeInformation(conn.getNextId(), node_list)
p = protocol.notifyNodeInformation(conn.getNextId(), node_list)
conn.addPacket(p)
del node_list[:]
p = Packet()
p.notifyNodeInformation(conn.getNextId(), node_list)
p = protocol.notifyNodeInformation(conn.getNextId(), node_list)
conn.addPacket(p)
# If this is a storage node or an admin node, send the partition table.
node = app.nm.getNodeByUUID(uuid)
if node.getNodeType() in (STORAGE_NODE_TYPE, ADMIN_NODE_TYPE):
# Split the packet if too huge.
p = Packet()
row_list = []
for offset in xrange(app.num_partitions):
row_list.append((offset, app.pt.getRow(offset)))
if len(row_list) == 1000:
p.sendPartitionTable(conn.getNextId(), app.lptid, row_list)
p = protocol.sendPartitionTable(conn.getNextId(), app.lptid, row_list)
conn.addPacket(p)
del row_list[:]
if len(row_list) != 0:
p.sendPartitionTable(conn.getNextId(), app.lptid, row_list)
p = protocol.sendPartitionTable(conn.getNextId(), app.lptid, row_list)
conn.addPacket(p)
def handleAnnouncePrimaryMaster(self, conn, packet):
......
neo/protocol.py
View file @ f11bafed
This diff is collapsed.
neo/storage/bootstrap.py
View file @ f11bafed
......@@ -17,6 +17,7 @@
import logging
from neo import protocol
from neo.storage.handler import StorageEventHandler
from neo.protocol import INVALID_UUID, RUNNING_STATE, BROKEN_STATE, \
MASTER_NODE_TYPE, STORAGE_NODE_TYPE, CLIENT_NODE_TYPE
......@@ -36,9 +37,8 @@ class BootstrapEventHandler(StorageEventHandler):
# Should not happen.
raise RuntimeError('connection completed while not trying to connect')
p = Packet()
msg_id = conn.getNextId()
p.requestNodeIdentification(msg_id, STORAGE_NODE_TYPE, app.uuid,
p = protocol.requestNodeIdentification(msg_id, STORAGE_NODE_TYPE, app.uuid,
app.server[0], app.server[1], app.name)
conn.addPacket(p)
conn.expectMessage(msg_id)
......@@ -115,12 +115,12 @@ class BootstrapEventHandler(StorageEventHandler):
app = self.app
if node_type != MASTER_NODE_TYPE:
logging.info('reject a connection from a non-master')
conn.addPacket(Packet().notReady(packet.getId(), 'retry later'))
conn.addPacket(protocol.notReady(packet.getId(), 'retry later'))
conn.abort()
return
if name != app.name:
logging.error('reject an alien cluster')
conn.addPacket(Packet().protocolError(packet.getId(),
conn.addPacket(protocol.protocolError(packet.getId(),
'invalid cluster name'))
conn.abort()
return
......@@ -134,8 +134,7 @@ class BootstrapEventHandler(StorageEventHandler):
# If this node is broken, reject it.
if node.getUUID() == uuid:
if node.getState() == BROKEN_STATE:
p = Packet()
p.brokenNodeDisallowedError(packet.getId(), 'go away')
p = protocol.brokenNodeDisallowedError(packet.getId(), 'go away')
conn.addPacket(p)
conn.abort()
return
......@@ -144,8 +143,7 @@ class BootstrapEventHandler(StorageEventHandler):
node.setUUID(uuid)
conn.setUUID(uuid)
p = Packet()
p.acceptNodeIdentification(packet.getId(), STORAGE_NODE_TYPE,
p = protocol.acceptNodeIdentification(packet.getId(), STORAGE_NODE_TYPE,
app.uuid, app.server[0], app.server[1],
0, 0, uuid)
conn.addPacket(p)
......@@ -202,7 +200,7 @@ class BootstrapEventHandler(StorageEventHandler):
# Ask a primary master.
msg_id = conn.getNextId()
conn.addPacket(Packet().askPrimaryMaster(msg_id))
conn.addPacket(protocol.askPrimaryMaster(msg_id))
conn.expectMessage(msg_id)
def handleAnswerPrimaryMaster(self, conn, packet, primary_uuid,
......
neo/storage/handler.py
View file @ f11bafed
......@@ -62,8 +62,7 @@ class StorageEventHandler(EventHandler):
info = n.getServer() + (n.getUUID() or INVALID_UUID,)
known_master_list.append(info)
p = Packet()
p.answerPrimaryMaster(packet.getId(), primary_uuid, known_master_list)
p = protocol.answerPrimaryMaster(packet.getId(), primary_uuid, known_master_list)
conn.addPacket(p)
def handleAnswerPrimaryMaster(self, conn, packet, primary_uuid,
......
neo/storage/mysqldb.py
View file @ f11bafed
......@@ -77,14 +77,14 @@ class MySQLDatabaseManager(DatabaseManager):
"""Query data from a database."""
conn = self.conn
try:
printable_char_list = []
for c in query.split('\n', 1)[0][:70]:
if c not in string.printable or c in '\t\x0b\x0c\r':
c = '\\x%02x' % ord(c)
printable_char_list.append(c)
query_part = ''.join(printable_char_list)
logging.debug('querying %s...', query_part)
# printable_char_list = []
# for c in query.split('\n', 1)[0][:70]:
# if c not in string.printable or c in '\t\x0b\x0c\r':
# c = '\\x%02x' % ord(c)
# printable_char_list.append(c)
# query_part = ''.join(printable_char_list)
#
# logging.debug('querying %s...', query_part)
conn.query(query)
r = conn.store_result()
if r is not None:
......
neo/storage/operation.py
View file @ f11bafed
......@@ -17,6 +17,7 @@
import logging
from neo import protocol
from neo.storage.handler import StorageEventHandler
from neo.protocol import INVALID_UUID, INVALID_SERIAL, INVALID_TID, \
INVALID_PARTITION, \
......@@ -138,7 +139,7 @@ class OperationEventHandler(StorageEventHandler):
app = self.app
if name != app.name:
logging.error('reject an alien cluster')
conn.addPacket(Packet().protocolError(packet.getId(),
conn.addPacket(protocol.protocolError(packet.getId(),
'invalid cluster name'))
conn.abort()
return
......@@ -155,7 +156,7 @@ class OperationEventHandler(StorageEventHandler):
# If I do not know such a node, and it is not even a master
# node, simply reject it.
logging.error('reject an unknown node %s', dump(uuid))
conn.addPacket(Packet().notReady(packet.getId(),
conn.addPacket(protocol.notReady(packet.getId(),
'unknown node'))
conn.abort()
return
......@@ -163,8 +164,7 @@ class OperationEventHandler(StorageEventHandler):
# If this node is broken, reject it.
if node.getUUID() == uuid:
if node.getState() == BROKEN_STATE:
p = Packet()
p.brokenNodeDisallowedError(packet.getId(), 'go away')
p = protocol.brokenNodeDisallowedError(packet.getId(), 'go away')
conn.addPacket(p)
conn.abort()
return
......@@ -173,8 +173,7 @@ class OperationEventHandler(StorageEventHandler):
node.setUUID(uuid)
conn.setUUID(uuid)
p = Packet()
p.acceptNodeIdentification(packet.getId(), STORAGE_NODE_TYPE,
p = protocol.acceptNodeIdentification(packet.getId(), STORAGE_NODE_TYPE,
app.uuid, app.server[0], app.server[1],
app.num_partitions, app.num_replicas,
uuid)
......@@ -256,11 +255,10 @@ class OperationEventHandler(StorageEventHandler):
app = self.app
t = app.dm.getTransaction(tid)
p = Packet()
if t is None:
p.tidNotFound(packet.getId(), '%s does not exist' % dump(tid))
p = protocol.tidNotFound(packet.getId(), '%s does not exist' % dump(tid))
else:
p.answerTransactionInformation(packet.getId(), tid,
p = protocol.answerTransactionInformation(packet.getId(), tid,
t[1], t[2], t[3], t[0])
conn.addPacket(p)
......@@ -286,7 +284,7 @@ class OperationEventHandler(StorageEventHandler):
except KeyError:
pass
conn.addPacket(Packet().notifyInformationLocked(packet.getId(), tid))
conn.addPacket(protocol.notifyInformationLocked(packet.getId(), tid))
else:
self.handleUnexpectedPacket(conn, packet)
......@@ -324,25 +322,24 @@ class OperationEventHandler(StorageEventHandler):
if tid == INVALID_TID:
tid = None
o = app.dm.getObject(oid, serial, tid)
p = Packet()
if o is not None:
serial, next_serial, compression, checksum, data = o
if next_serial is None:
next_serial = INVALID_SERIAL
logging.debug('oid = %s, serial = %s, next_serial = %s',
dump(oid), dump(serial), dump(next_serial))
p.answerObject(packet.getId(), oid, serial, next_serial,
p = protocol.answerObject(packet.getId(), oid, serial, next_serial,
compression, checksum, data)
else:
logging.debug('oid = %s not found', dump(oid))
p.oidNotFound(packet.getId(), '%s does not exist' % dump(oid))
p = protocol.oidNotFound(packet.getId(), '%s does not exist' % dump(oid))
conn.addPacket(p)
def handleAskTIDs(self, conn, packet, first, last, partition):
# This method is complicated, because I must return TIDs only
# about usable partitions assigned to me.
if first >= last:
conn.addPacket(Packet().protocolError(packet.getId(),
conn.addPacket(protocol.protocolError(packet.getId(),
'invalid offsets'))
return
......@@ -362,11 +359,11 @@ class OperationEventHandler(StorageEventHandler):
tid_list = app.dm.getTIDList(first, last - first,
app.num_partitions, partition_list)
conn.addPacket(Packet().answerTIDs(packet.getId(), tid_list))
conn.addPacket(protocol.answerTIDs(packet.getId(), tid_list))
def handleAskObjectHistory(self, conn, packet, oid, first, last):
if first >= last:
conn.addPacket(Packet().protocolError(packet.getId(),
conn.addPacket(protocol.protocolError(packet.getId(),
'invalid offsets'))
return
......@@ -374,7 +371,7 @@ class OperationEventHandler(StorageEventHandler):
history_list = app.dm.getObjectHistory(oid, first, last - first)
if history_list is None:
history_list = []
conn.addPacket(Packet().answerObjectHistory(packet.getId(), oid,
conn.addPacket(protocol.answerObjectHistory(packet.getId(), oid,
history_list))
def handleAskStoreTransaction(self, conn, packet, tid, user, desc,
......@@ -388,7 +385,7 @@ class OperationEventHandler(StorageEventHandler):
t = app.transaction_dict.setdefault(tid, TransactionInformation(uuid))
t.addTransaction(oid_list, user, desc, ext)
conn.addPacket(Packet().answerStoreTransaction(packet.getId(), tid))
conn.addPacket(protocol.answerStoreTransaction(packet.getId(), tid))
def handleAskStoreObject(self, conn, packet, oid, serial,
compression, checksum, data, tid):
......@@ -409,7 +406,7 @@ class OperationEventHandler(StorageEventHandler):
# If a newer transaction already locks this object,
# do not try to resolve a conflict, so return immediately.
logging.info('unresolvable conflict in %s', dump(oid))
conn.addPacket(Packet().answerStoreObject(packet.getId(), 1,
conn.addPacket(protocol.answerStoreObject(packet.getId(), 1,
oid, locking_tid))
return
......@@ -419,13 +416,13 @@ class OperationEventHandler(StorageEventHandler):
last_serial = history_list[0][0]
if last_serial != serial:
logging.info('resolvable conflict in %s', dump(oid))
conn.addPacket(Packet().answerStoreObject(packet.getId(), 1,
conn.addPacket(protocol.answerStoreObject(packet.getId(), 1,
oid, last_serial))
return
# Now store the object.
t = app.transaction_dict.setdefault(tid, TransactionInformation(uuid))
t.addObject(oid, compression, checksum, data)
conn.addPacket(Packet().answerStoreObject(packet.getId(), 0,
conn.addPacket(protocol.answerStoreObject(packet.getId(), 0,
oid, serial))
app.store_lock_dict[oid] = tid
......@@ -470,7 +467,7 @@ class OperationEventHandler(StorageEventHandler):
# This method is complicated, because I must return OIDs only
# about usable partitions assigned to me.
if first >= last:
conn.addPacket(Packet().protocolError(packet.getId(),
conn.addPacket(protocol.protocolError(packet.getId(),
'invalid offsets'))
return
......@@ -490,4 +487,4 @@ class OperationEventHandler(StorageEventHandler):
oid_list = app.dm.getOIDList(first, last - first,
app.num_partitions, partition_list)
conn.addPacket(Packet().answerOIDs(packet.getId(), oid_list))
conn.addPacket(protocol.answerOIDs(packet.getId(), oid_list))
neo/storage/replicator.py
View file @ f11bafed
......@@ -90,8 +90,7 @@ class ReplicationEventHandler(StorageEventHandler):
tid_set = set(tid_list) - set(present_tid_list)
for tid in tid_set:
msg_id = conn.getNextId()
p = Packet()
p.askTransactionInformation(msg_id, tid)
p = protocol.askTransactionInformation(msg_id, tid)
conn.addPacket(p)
conn.expectMessage(msg_id, timeout = 300)
......@@ -99,8 +98,7 @@ class ReplicationEventHandler(StorageEventHandler):
app.replicator.tid_offset += 1000
offset = app.replicator.tid_offset
msg_id = conn.getNextId()
p = Packet()
p.askTIDs(msg_id, offset, offset + 1000,
p = protocol.askTIDs(msg_id, offset, offset + 1000,
app.replicator.current_partition.getRID())
conn.addPacket(p)
conn.expectMessage(msg_id, timeout = 300)
......@@ -108,8 +106,7 @@ class ReplicationEventHandler(StorageEventHandler):
# If no more TID, a replication of transactions is finished.
# So start to replicate objects now.
msg_id = conn.getNextId()
p = Packet()
p.askOIDs(msg_id, 0, 1000,
p = protocol.askOIDs(msg_id, 0, 1000,
app.replicator.current_partition.getRID())
conn.addPacket(p)
conn.expectMessage(msg_id, timeout = 300)
......@@ -133,8 +130,7 @@ class ReplicationEventHandler(StorageEventHandler):
# Pick one up, and ask the history.
oid = oid_list.pop()
msg_id = conn.getNextId()
p = Packet()
p.askObjectHistory(msg_id, oid, 0, 1000)
p = protocol.askObjectHistory(msg_id, oid, 0, 1000)
conn.addPacket(p)
conn.expectMessage(msg_id, timeout = 300)
app.replicator.serial_offset = 0
......@@ -156,8 +152,7 @@ class ReplicationEventHandler(StorageEventHandler):
serial_set = set(serial_list) - set(present_serial_list)
for serial in serial_set:
msg_id = conn.getNextId()
p = Packet()
p.askObject(msg_id, oid, serial, INVALID_TID)
p = protocol.askObject(msg_id, oid, serial, INVALID_TID)
conn.addPacket(p)
conn.expectMessage(msg_id, timeout = 300)
......@@ -165,8 +160,7 @@ class ReplicationEventHandler(StorageEventHandler):
app.replicator.serial_offset += 1000
offset = app.replicator.serial_offset
msg_id = conn.getNextId()
p = Packet()
p.askObjectHistory(msg_id, oid, offset, offset + 1000)
p = protocol.askObjectHistory(msg_id, oid, offset, offset + 1000)
conn.addPacket(p)
conn.expectMessage(msg_id, timeout = 300)
else:
......@@ -176,8 +170,7 @@ class ReplicationEventHandler(StorageEventHandler):
# If I have more pending OIDs, pick one up.
oid = oid_list.pop()
msg_id = conn.getNextId()
p = Packet()
p.askObjectHistory(msg_id, oid, 0, 1000)
p = protocol.askObjectHistory(msg_id, oid, 0, 1000)
conn.addPacket(p)
conn.expectMessage(msg_id, timeout = 300)
app.replicator.serial_offset = 0
......@@ -186,8 +179,7 @@ class ReplicationEventHandler(StorageEventHandler):
app.replicator.oid_offset += 1000
offset = app.replicator.oid_offset
msg_id = conn.getNextId()
p = Packet()
p.askOIDs(msg_id, offset, offset + 1000,
p = protocol.askOIDs(msg_id, offset, offset + 1000,
app.replicator.current_partition.getRID())
conn.addPacket(p)
conn.expectMessage(msg_id, timeout = 300)
......@@ -293,7 +285,7 @@ class Replicator(object):
def _askCriticalTID(self):
conn = self.primary_master_connection
msg_id = conn.getNextId()
conn.addPacket(Packet().askLastIDs(msg_id))
conn.addPacket(protocol.askLastIDs(msg_id))
conn.expectMessage(msg_id)
self.critical_tid_dict[msg_id] = self.new_partition_dict.values()
self.partition_dict.update(self.new_partition_dict)
......@@ -309,7 +301,7 @@ class Replicator(object):
def _askUnfinishedTIDs(self):
conn = self.primary_master_connection
msg_id = conn.getNextId()
conn.addPacket(Packet().askUnfinishedTransactions(msg_id))
conn.addPacket(protocol.askUnfinishedTransactions(msg_id))
conn.expectMessage(msg_id)
self.waiting_for_unfinished_tids = True
......@@ -343,16 +335,14 @@ class Replicator(object):
addr = addr,
connector_handler = app.connector_handler)
msg_id = self.current_connection.getNextId()
p = Packet()
p.requestNodeIdentification(msg_id, STORAGE_NODE_TYPE, app.uuid,
p = protocol.requestNodeIdentification(msg_id, STORAGE_NODE_TYPE, app.uuid,
app.server[0], app.server[1], app.name)
self.current_connection.addPacket(p)
self.current_connection.expectMessage(msg_id)
self.tid_offset = 0
msg_id = self.current_connection.getNextId()
p = Packet()
p.askTIDs(msg_id, 0, 1000, self.current_partition.getRID())
p = protocol.askTIDs(msg_id, 0, 1000, self.current_partition.getRID())
self.current_connection.addPacket(p)
self.current_connection.expectMessage(msg_id, timeout = 300)
......@@ -364,8 +354,7 @@ class Replicator(object):
self.partition_dict.pop(self.current_partition.getRID())
# Notify to a primary master node that my cell is now up-to-date.
conn = self.primary_master_connection
p = Packet()
p.notifyPartitionChanges(conn.getNextId(),
p = protocol.notifyPartitionChanges(conn.getNextId(),
app.ptid,
[(self.current_partition.getRID(),
app.uuid,
......
neo/storage/verification.py
View file @ f11bafed
......@@ -17,6 +17,7 @@
import logging
from neo import protocol
from neo.storage.handler import StorageEventHandler
from neo.protocol import INVALID_OID, INVALID_TID, \
RUNNING_STATE, BROKEN_STATE, TEMPORARILY_DOWN_STATE, \
......@@ -68,12 +69,12 @@ class VerificationEventHandler(StorageEventHandler):
app = self.app
if node_type != MASTER_NODE_TYPE:
logging.info('reject a connection from a non-master')
conn.addPacket(Packet().notReady(packet.getId(), 'retry later'))
conn.addPacket(protocol.notReady(packet.getId(), 'retry later'))
conn.abort()
return
if name != app.name:
logging.error('reject an alien cluster')
conn.addPacket(Packet().protocolError(packet.getId(),
conn.addPacket(protocol.protocolError(packet.getId(),
'invalid cluster name'))
conn.abort()
return
......@@ -87,8 +88,7 @@ class VerificationEventHandler(StorageEventHandler):
# If this node is broken, reject it.
if node.getUUID() == uuid:
if node.getState() == BROKEN_STATE:
p = Packet()
p.brokenNodeDisallowedError(packet.getId(), 'go away')
p = protocol.brokenNodeDisallowedError(packet.getId(), 'go away')
conn.addPacket(p)
conn.abort()
return
......@@ -97,8 +97,7 @@ class VerificationEventHandler(StorageEventHandler):
node.setUUID(uuid)
conn.setUUID(uuid)
p = Packet()
p.acceptNodeIdentification(packet.getId(), STORAGE_NODE_TYPE,
p = protocol.acceptNodeIdentification(packet.getId(), STORAGE_NODE_TYPE,
app.uuid, app.server[0], app.server[1],
app.num_partitions, app.num_replicas,
uuid)
......@@ -127,10 +126,9 @@ class VerificationEventHandler(StorageEventHandler):
def handleAskLastIDs(self, conn, packet):
if not conn.isServerConnection():
app = self.app
p = Packet()
oid = app.dm.getLastOID() or INVALID_OID
tid = app.dm.getLastTID() or INVALID_TID
p.answerLastIDs(packet.getId(), oid, tid, app.ptid)
p = protocol.answerLastIDs(packet.getId(), oid, tid, app.ptid)
conn.addPacket(p)
else:
self.handleUnexpectedPacket(conn, packet)
......@@ -149,14 +147,12 @@ class VerificationEventHandler(StorageEventHandler):
pass
row_list.append((offset, row))
except IndexError:
p = Packet()
p.protocolError(packet.getId(),
p = protocol.protocolError(packet.getId(),
'invalid partition table offset')
conn.addPacket(p)
return
p = Packet()
p.answerPartitionTable(packet.getId(), app.ptid, row_list)
p = protocol.answerPartitionTable(packet.getId(), app.ptid, row_list)
conn.addPacket(p)
else:
self.handleUnexpectedPacket(conn, packet)
......@@ -240,8 +236,7 @@ class VerificationEventHandler(StorageEventHandler):
if not conn.isServerConnection():
app = self.app
tid_list = app.dm.getUnfinishedTIDList()
p = Packet()
p.answerUnfinishedTransactions(packet.getId(), tid_list)
p = protocol.answerUnfinishedTransactions(packet.getId(), tid_list)
conn.addPacket(p)
else:
self.handleUnexpectedPacket(conn, packet)
......@@ -256,22 +251,20 @@ class VerificationEventHandler(StorageEventHandler):
else:
t = app.dm.getTransaction(tid)
p = Packet()
if t is None:
p.tidNotFound(packet.getId(), '%s does not exist' % dump(tid))
p = protocol.tidNotFound(packet.getId(), '%s does not exist' % dump(tid))
else:
p.answerTransactionInformation(packet.getId(), tid,
p = protocol.answerTransactionInformation(packet.getId(), tid,
t[1], t[2], t[3], t[0])
conn.addPacket(p)
def handleAskObjectPresent(self, conn, packet, oid, tid):
if not conn.isServerConnection():
app = self.app
p = Packet()
if app.dm.objectPresent(oid, tid):
p.answerObjectPresent(packet.getId(), oid, tid)
p = protocol.answerObjectPresent(packet.getId(), oid, tid)
else:
p.oidNotFound(packet.getId(),
p = protocol.oidNotFound(packet.getId(),
'%s:%s do not exist' % (dump(oid), dump(tid)))
conn.addPacket(p)
else:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7