Review API betweeen connections and connectors

- Review error handling. Only 2 exceptions remain in connector.py:

  - Drop useless exception handling for EAGAIN since it should not happen
    if the kernel says the socket is ready.
  - Do not distinguish other socket errors. Just close and log in a generic way.
  - No need to raise a specific exception for EOF.
  - Make 'connect' return a boolean instead of raising an exception.
  - Raise appropriate exception when answer/ask/notify is called on a closed
    non-MT connection.

- Add support for more complex connectors, which may need to write for a read
  operation, or to read when there's pending data to send. This will be
  required for SSL support (more exactly, the handshake will be done in
  a transparent way):

  - Move write buffer to connector.
  - Make 'receive' fill the read buffer, instead of returning the read data.
  - Make 'receive' & 'send' return a boolean to switch polling for writing.
  - Tolerate that sockets return 0 as number of bytes sent.

- In testConnection, simply delete all failing tests, as announced
  in commit 71e30fb9.

neo/lib/connector.py

@@ -17,6 +17,7 @@
 import socket
 import errno
 from time import time
+from . import logging
 
 # Global connector registry.
 # Fill by calling registerConnectorHandler.
@@ -56,8 +57,19 @@ class SocketConnector(object):
         s.setblocking(0)
         # disable Nagle algorithm to reduce latency
         s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+        self.queued = []
         return self
 
+    def queue(self, data):
+        was_empty = not self.queued
+        self.queued += data
+        return was_empty
+
+    def _error(self, op, exc):
+        logging.debug("%s failed for %s: %s (%s)",
+            op, self, errno.errorcode[exc.errno], exc.strerror)
+        raise ConnectorException
+
     # Threaded tests monkey-patch the following 2 operations.
     _connect = lambda self, addr: self.socket.connect(addr)
     _bind = lambda self, addr: self.socket.bind(addr)
@@ -68,20 +80,23 @@ class SocketConnector(object):
         try:
             connect_limit = self.connect_limit[addr]
             if time() < connect_limit:
+                # Next call to queue() must return False
+                # in order not to enable polling for writing.
+                self.queued or self.queued.append('')
                 raise ConnectorDelayedConnection(connect_limit)
+            if self.queued and not self.queued[0]:
+                del self.queued[0]
         except KeyError:
             pass
         self.connect_limit[addr] = time() + self.CONNECT_LIMIT
         self.is_server = self.is_closed = False
         try:
             self._connect(addr)
-        except socket.error, (err, errmsg):
-            if err == errno.EINPROGRESS:
-                raise ConnectorInProgressException
-            if err == errno.ECONNREFUSED:
-                raise ConnectorConnectionRefusedException
-            raise ConnectorException, 'makeClientConnection to %s failed:' \
-                ' %s:%s' % (addr, err, errmsg)
+        except socket.error, e:
+            if e.errno == errno.EINPROGRESS:
+                return False
+            self._error('connect', e)
+        return True
 
     def makeListeningConnection(self):
         assert self.is_closed is None
@@ -90,10 +105,9 @@ class SocketConnector(object):
             self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
             self._bind(self.addr)
             self.socket.listen(5)
-        except socket.error, (err, errmsg):
+        except socket.error, e:
             self.socket.close()
-            raise ConnectorException, 'makeListeningConnection on %s failed:' \
-                    ' %s:%s' % (addr, err, errmsg)
+            self._error('listen', e)
 
     def getError(self):
         return self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
@@ -116,33 +130,39 @@ class SocketConnector(object):
             s, addr = self.socket.accept()
             s = self.__class__(addr, s)
             return s, s.addr
-        except socket.error, (err, errmsg):
-            if err == errno.EAGAIN:
-                raise ConnectorTryAgainException
-            raise ConnectorException, 'accept failed: %s:%s' % \
-                (err, errmsg)
+        except socket.error, e:
+            self._error('accept', e)
 
-    def receive(self):
+    def receive(self, read_buf):
         try:
-            return self.socket.recv(4096)
-        except socket.error, (err, errmsg):
-            if err == errno.EAGAIN:
-                raise ConnectorTryAgainException
-            if err in (errno.ECONNREFUSED, errno.EHOSTUNREACH):
-                raise ConnectorConnectionRefusedException
-            if err in (errno.ECONNRESET, errno.ETIMEDOUT):
-                raise ConnectorConnectionClosedException
-            raise ConnectorException, 'receive failed: %s:%s' % (err, errmsg)
-
-    def send(self, msg):
-        try:
-            return self.socket.send(msg)
-        except socket.error, (err, errmsg):
-            if err == errno.EAGAIN:
-                raise ConnectorTryAgainException
-            if err in (errno.ECONNRESET, errno.ETIMEDOUT, errno.EPIPE):
-                raise ConnectorConnectionClosedException
-            raise ConnectorException, 'send failed: %s:%s' % (err, errmsg)
+            data = self.socket.recv(4096)
+        except socket.error, e:
+            self._error('recv', e)
+        if data:
+            read_buf.append(data)
+            return
+        logging.debug('%r closed in recv', self)
+        raise ConnectorException
+
+    def send(self):
+        msg = ''.join(self.queued)
+        if msg:
+            try:
+                n = self.socket.send(msg)
+            except socket.error, e:
+                self._error('send', e)
+            # Do nothing special if n == 0:
+            # - it never happens for simple sockets;
+            # - for SSL sockets, this is always the case unless everything
+            #   could be sent.
+            if n != len(msg):
+                self.queued[:] = msg[n:],
+                return False
+            del self.queued[:]
+        else:
+            assert not self.queued
+        return True
+
 
     def close(self):
         self.is_closed = True
@@ -195,17 +215,5 @@ registerConnectorHandler(SocketConnectorIPv6)
 class ConnectorException(Exception):
     pass
 
-class ConnectorTryAgainException(ConnectorException):
-    pass
-
-class ConnectorInProgressException(ConnectorException):
-    pass
-
-class ConnectorConnectionClosedException(ConnectorException):
-    pass
-
-class ConnectorConnectionRefusedException(ConnectorException):
-    pass
-
 class ConnectorDelayedConnection(ConnectorException):
     pass

neo/storage/checker.py

@@ -16,8 +16,7 @@
 
 from collections import deque
 from neo.lib import logging
-from neo.lib.connection import ClientConnection
-from neo.lib.connector import ConnectorConnectionClosedException
+from neo.lib.connection import ClientConnection, ConnectionClosed
 from neo.lib.protocol import NodeTypes, Packets, ZERO_OID
 from neo.lib.util import add64, dump
 from .handlers.storage import StorageOperationHandler
@@ -85,7 +84,7 @@ class Checker(object):
                         if self.conn_dict:
                             break
                         msg = "no replica"
-                    except ConnectorConnectionClosedException:
+                    except ConnectionClosed:
                         msg = "connection closed"
                     finally:
                         conn_set.update(self.conn_dict)

neo/storage/handlers/storage.py

@@ -16,7 +16,7 @@
 
 import weakref
 from functools import wraps
-from neo.lib.connector import ConnectorConnectionClosedException
+from neo.lib.connection import ConnectionClosed
 from neo.lib.handler import EventHandler
 from neo.lib.protocol import Errors, NodeStates, Packets, ProtocolError, \
     ZERO_HASH
@@ -154,7 +154,7 @@ class StorageOperationHandler(EventHandler):
             r = app.dm.checkTIDRange(*args)
             try:
                 conn.answer(Packets.AnswerCheckTIDRange(*r), msg_id)
-            except (weakref.ReferenceError, ConnectorConnectionClosedException):
+            except (weakref.ReferenceError, ConnectionClosed):
                 pass
             yield
         app.newTask(check())
@@ -170,7 +170,7 @@ class StorageOperationHandler(EventHandler):
             r = app.dm.checkSerialRange(*args)
             try:
                 conn.answer(Packets.AnswerCheckSerialRange(*r), msg_id)
-            except (weakref.ReferenceError, ConnectorConnectionClosedException):
+            except (weakref.ReferenceError, ConnectionClosed):
                 pass
             yield
         app.newTask(check())
@@ -211,7 +211,7 @@ class StorageOperationHandler(EventHandler):
                 conn.answer(Packets.AnswerFetchTransactions(
                     pack_tid, next_tid, peer_tid_set), msg_id)
                 yield
-            except (weakref.ReferenceError, ConnectorConnectionClosedException):
+            except (weakref.ReferenceError, ConnectionClosed):
                 pass
         app.newTask(push())
 
@@ -253,6 +253,6 @@ class StorageOperationHandler(EventHandler):
                 conn.answer(Packets.AnswerFetchObjects(
                     pack_tid, next_tid, next_oid, object_dict), msg_id)
                 yield
-            except (weakref.ReferenceError, ConnectorConnectionClosedException):
+            except (weakref.ReferenceError, ConnectionClosed):
                 pass
         app.newTask(push())

neo/tests/threaded/__init__.py

@@ -31,8 +31,7 @@ import neo.client.app, neo.neoctl.app
 from neo.client import Storage
 from neo.lib import logging
 from neo.lib.connection import BaseConnection, Connection
-from neo.lib.connector import SocketConnector, \
-    ConnectorConnectionRefusedException
+from neo.lib.connector import SocketConnector, ConnectorException
 from neo.lib.locking import SimpleQueue
 from neo.lib.protocol import CellStates, ClusterStates, NodeStates, NodeTypes
 from neo.lib.util import cached_property, parseMasterList, p64
@@ -322,7 +321,7 @@ class ServerNode(Node):
         try:
             return self.listening_conn.getAddress()
         except AttributeError:
-            raise ConnectorConnectionRefusedException
+            raise ConnectorException
 
 class AdminApplication(ServerNode, neo.admin.app.Application):
     pass