Skip to content

N
neoppod
Commit 65b17bdc authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

X rework Conn acceptance to be explicit via NodeLink.Accept 

This way programming is more straightforward e.g. for case where we
first have to process identification and only then serve.

This is also more similar to how usual sockets work.
parent a024d393
Hide whitespace changes
Inline Side-by-side
Showing with 132 additions and 62 deletions
go/neo/connection.go
View file @ 65b17bdc
......@@ -33,11 +33,11 @@ import (
//
// New connection can be created with .NewConn() . Once connection is
// created and data is sent over it, on peer's side another corresponding
// new connection will be created - accepting first packet "request" - and all
// new connection will be created - accepting first packet "request" - and all XXX -> Accept
// further communication send/receive exchange will be happening in between
// those 2 connections.
//
// For a node to be able to accept new incoming connection it has to register
// For a node to be able to accept new incoming connection it has to register XXX -> Accept
// corresponding handler with .HandleNewConn() . Without such handler
// registered the node will be able to only initiate new connections, not
// accept new ones from its peer.
......@@ -53,8 +53,11 @@ type NodeLink struct {
nextConnId uint32 // next connId to use for Conn initiated by us
serveWg sync.WaitGroup // for serve{Send,Recv}
handleWg sync.WaitGroup // for spawned handlers
handleNewConn func(conn *Conn) // handler for new connections
// handleWg sync.WaitGroup // for spawned handlers
// handleNewConn func(conn *Conn) // handler for new connections
acceptq chan *Conn // queue of incoming connections for Accept
// = nil if NodeLink is not accepting connections
txreq chan txReq // tx requests from Conns go via here
closed chan struct{}
......@@ -78,7 +81,7 @@ type Conn struct {
closed chan struct{}
}
// A role our end of NodeLink is intended to play
// LinkRole is a role an end of NodeLink is intended to play
type LinkRole int
const (
LinkServer LinkRole = iota // link created as server
......@@ -89,10 +92,10 @@ const (
linkFlagsMask LinkRole = (1<<32 - 1) << 16
)
// Make a new NodeLink from already established net.Conn
// NewNodeLink makes a new NodeLink from already established net.Conn
//
// Role specifies how to treat our role on the link - either as client or
// server one. The difference in between client and server roles is only in
// server one. The difference in between client and server roles is only in XXX + acceptq
// how connection ids are allocated for connections initiated at our side:
// there is no conflict in identifiers if one side always allocates them as
// even (server) and its peer as odd (client).
......@@ -101,11 +104,14 @@ const (
// net.Listen/net.Accept and client role for connections created via net.Dial.
func NewNodeLink(conn net.Conn, role LinkRole) *NodeLink {
var nextConnId uint32
var acceptq chan *Conn
switch role&^linkFlagsMask {
case LinkServer:
nextConnId = 0 // all initiated by us connId will be even
nextConnId = 0 // all initiated by us connId will be even
acceptq = make(chan *Conn) // accept queue; TODO use backlog
case LinkClient:
nextConnId = 1 // ----//---- odd
acceptq = nil // not accepting incoming connections
default:
panic("invalid conn role")
}
......@@ -114,6 +120,7 @@ func NewNodeLink(conn net.Conn, role LinkRole) *NodeLink {
peerLink: conn,
connTab: map[uint32]*Conn{},
nextConnId: nextConnId,
acceptq: acceptq,
txreq: make(chan txReq),
closed: make(chan struct{}),
}
......@@ -125,9 +132,11 @@ func NewNodeLink(conn net.Conn, role LinkRole) *NodeLink {
return nl
}
// Close node-node link.
// Close closes node-node link.
// IO on connections established over it is automatically interrupted with an error.
func (nl *NodeLink) Close() error {
// XXX what with .acceptq ?
// mark all active Conns as closed
nl.connMu.Lock()
defer nl.connMu.Unlock()
......@@ -150,7 +159,7 @@ func (nl *NodeLink) Close() error {
return err
}
// send raw packet to peer
// sendPkt sends raw packet to peer
func (nl *NodeLink) sendPkt(pkt *PktBuf) error {
if true {
// XXX -> log
......@@ -165,7 +174,7 @@ func (nl *NodeLink) sendPkt(pkt *PktBuf) error {
return err
}
// receive raw packet from peer
// recvPkt receives raw packet from peer
func (nl *NodeLink) recvPkt() (*PktBuf, error) {
// TODO organize rx buffers management (freelist etc)
// TODO cleanup lots of ntoh32(...)
......@@ -174,7 +183,7 @@ func (nl *NodeLink) recvPkt() (*PktBuf, error) {
// first read to read pkt header and hopefully up to page of data in 1 syscall
pkt := &PktBuf{make([]byte, 4096)}
// TODO reenable, but NOTE next packet can be also prefetched here
// TODO reenable, but NOTE next packet can be also prefetched here -> use buffering ?
//n, err := io.ReadAtLeast(nl.peerLink, pkt.Data, PktHeadLen)
n, err := io.ReadFull(nl.peerLink, pkt.Data[:PktHeadLen])
if err != nil {
......@@ -193,6 +202,7 @@ func (nl *NodeLink) recvPkt() (*PktBuf, error) {
panic("TODO message too big")
}
//pkt.Data = xbytes.Resize32(pkt.Data, ntoh32(pkth.Len))
if ntoh32(pkth.Len) > uint32(cap(pkt.Data)) {
// grow rxbuf
rxbuf2 := make([]byte, ntoh32(pkth.Len))
......@@ -231,7 +241,7 @@ func (nl *NodeLink) newConn(connId uint32) *Conn {
return c
}
// Create a connection on top of node-node link
// NewConn creates a connection on top of node-node link
func (nl *NodeLink) NewConn() *Conn {
nl.connMu.Lock()
defer nl.connMu.Unlock()
......@@ -243,6 +253,27 @@ func (nl *NodeLink) NewConn() *Conn {
return c
}
// ErrClosedLink is the error indicated for opertions on closed NodeLink
var ErrLinkClosed = errors.New("node link closed")
var ErrLinkNoListen = errors.New("node link is not listening for incoming connections")
// Accept waits for and accepts incoming connection on top of node-node link
func (nl *NodeLink) Accept() (*Conn, error) {
// this node link is not accepting connections
if nl.acceptq == nil {
return nil, ErrLinkNoListen
}
select {
case <-nl.closed:
return nil, ErrLinkClosed // XXX + op = Accept ?
// XXX check acceptq != nil ?
case c := <-nl.acceptq:
return c, nil
}
}
// serveRecv handles incoming packets routing them to either appropriate
// already-established connection or to new handling goroutine.
......@@ -263,14 +294,17 @@ func (nl *NodeLink) serveRecv() {
// pkt.ConnId -> Conn
connId := ntoh32(pkt.Header().ConnId)
var handleNewConn func(conn *Conn)
// var handleNewConn func(conn *Conn)
nl.connMu.Lock()
conn := nl.connTab[connId]
if conn == nil {
handleNewConn = nl.handleNewConn
if handleNewConn != nil {
//handleNewConn = nl.handleNewConn // XXX -> Accept
//if handleNewConn != nil {
if nl.acceptq != nil {
conn = nl.newConn(connId)
// XXX what if Accept exited because of just recently close(nl.closed)?
nl.acceptq <- conn
}
}
nl.connMu.Unlock()
......@@ -281,29 +315,31 @@ func (nl *NodeLink) serveRecv() {
continue
}
// we are accepting new incoming connection - spawn
// connection-serving goroutine
if handleNewConn != nil {
// TODO avoid spawning goroutine for each new Ask request -
// - by keeping pool of read inactive goroutine / conn pool ?
// XXX rework interface for this to be Accept-like ?
go func() {
nl.handleWg.Add(1)
defer nl.handleWg.Done()
handleNewConn(conn)
}()
}
// // XXX -> accept
// // we are accepting new incoming connection - spawn
// // connection-serving goroutine
// if handleNewConn != nil {
// // TODO avoid spawning goroutine for each new Ask request -
// // - by keeping pool of read inactive goroutine / conn pool ?
// // XXX rework interface for this to be Accept-like ?
// go func() {
// nl.handleWg.Add(1)
// defer nl.handleWg.Done()
// handleNewConn(conn)
// }()
// }
// route packet to serving goroutine handler
// XXX what if Conn.Recv exited because of just recently close(nl.closed) ?
conn.rxq <- pkt
}
}
// wait for all handlers spawned for accepted connections to complete
// XXX naming -> WaitHandlers ?
func (nl *NodeLink) Wait() {
nl.handleWg.Wait()
}
// // wait for all handlers spawned for accepted connections to complete
// // XXX naming -> WaitHandlers ?
// func (nl *NodeLink) Wait() {
// nl.handleWg.Wait()
// }
// request to transmit a packet. Result error goes back to errch
......@@ -333,13 +369,13 @@ runloop:
}
}
// XXX move to NodeLink ctor ?
// Set handler for new incoming connections
func (nl *NodeLink) HandleNewConn(h func(*Conn)) {
nl.connMu.Lock()
defer nl.connMu.Unlock()
nl.handleNewConn = h // NOTE can change handler at runtime XXX do we need this?
}
// // XXX move to NodeLink ctor ?
// // Set handler for new incoming connections
// func (nl *NodeLink) HandleNewConn(h func(*Conn)) {
// nl.connMu.Lock()
// defer nl.connMu.Unlock()
// nl.handleNewConn = h // NOTE can change handler at runtime XXX do we need this?
// }
// ErrClosedConn is the error indicated for read/write operations on closed Conn
......@@ -426,7 +462,7 @@ func (c *Conn) Close() error {
// for convinience: Dial/Listen
// Connect to address on named network and wrap the connection as NodeLink
// Dial connects to address on named network and wrap the connection as NodeLink
// TODO +tls.Config
func Dial(ctx context.Context, network, address string) (*NodeLink, error) {
d := net.Dialer{}
......
go/neo/connection_test.go
View file @ 65b17bdc
......@@ -177,6 +177,25 @@ func TestNodeLink(t *testing.T) {
xwait(wg)
xclose(nl2)
// Close vs Accept
nl1, nl2 = _nodeLinkPipe(linkNoRecvSend, linkNoRecvSend)
wg = WorkGroup()
wg.Gox(func() {
tdelay()
xclose(nl2)
})
c, err := nl2.Accept()
if !(c == nil && err == ErrLinkClosed) {
t.Fatalf("NodeLink.Accept() after close: conn = %v, err = %v", c, err)
}
// nl1 is not accepting connections - because it has LinkClient role
// check Accept behaviour.
c, err = nl1.Accept()
if !(c == nil && err == ErrLinkNoListen) {
t.Fatalf("NodeLink.Accept() on non-listening node link: conn = %v, err = %v", c, err)
}
xclose(nl1)
// raw exchange
nl1, nl2 = _nodeLinkPipe(linkNoRecvSend, linkNoRecvSend)
......@@ -212,7 +231,7 @@ func TestNodeLink(t *testing.T) {
// Close vs Recv
nl1, nl2 = _nodeLinkPipe(0, linkNoRecvSend)
c := nl1.NewConn()
c = nl1.NewConn()
wg = WorkGroup()
wg.Gox(func() {
tdelay()
......@@ -267,8 +286,12 @@ func TestNodeLink(t *testing.T) {
// Conn accept + exchange
nl1, nl2 = nodeLinkPipe()
nl2.HandleNewConn(func(c *Conn) {
// TODO raised err -> errch
wg = WorkGroup()
//nl2.HandleNewConn(func(c *Conn) {
wg.Gox(func() {
c, err := nl2.Accept() // XXX -> xaccept ?
exc.Raiseif(err)
pkt := xrecv(c)
xverifyPkt(pkt, c.connId, 33, []byte("ping"))
......@@ -289,7 +312,8 @@ func TestNodeLink(t *testing.T) {
xsend(c, mkpkt(35, []byte("ping2")))
pkt = xrecv(c)
xverifyPkt(pkt, c.connId, 36, []byte("pong2"))
nl2.Wait()
//nl2.Wait()
xwait(wg)
xclose(c)
xclose(nl1)
......@@ -297,6 +321,7 @@ func TestNodeLink(t *testing.T) {
// test 2 channels with replies comming in reversed time order
nl1, nl2 = nodeLinkPipe()
wg = WorkGroup()
replyOrder := map[uint16]struct { // "order" in which to process requests
start chan struct{} // processing starts when start chan is ready
next uint16 // after processing this switch to next
......@@ -306,21 +331,28 @@ func TestNodeLink(t *testing.T) {
}
close(replyOrder[2].start)
nl2.HandleNewConn(func(c *Conn) {
// TODO raised err -> errch
pkt := xrecv(c)
n := ntoh16(pkt.Header().MsgCode)
x := replyOrder[n]
// wait before it is our turn & echo pkt back
<-x.start
xsend(c, pkt)
xclose(c)
// tell next it can start
if x.next != 0 {
close(replyOrder[x.next].start)
//nl2.HandleNewConn(func(c *Conn) {
wg.Gox(func() {
for _ = range replyOrder {
c, err := nl2.Accept()
exc.Raiseif(err)
wg.Gox(func() {
pkt := xrecv(c)
n := ntoh16(pkt.Header().MsgCode)
x := replyOrder[n]
// wait before it is our turn & echo pkt back
<-x.start
xsend(c, pkt)
xclose(c)
// tell next it can start
if x.next != 0 {
close(replyOrder[x.next].start)
}
})
}
})
......@@ -336,7 +368,8 @@ func TestNodeLink(t *testing.T) {
}
xechoWait(c2, 2)
xechoWait(c1, 1)
nl2.Wait()
//nl2.Wait()
xwait(wg)
xclose(c1)
xclose(c2)
......
go/neo/server.go
View file @ 65b17bdc
......@@ -80,7 +80,8 @@ func ListenAndServe(ctx context.Context, net_, laddr string, srv Server) error {
// ----------------------------------------
// Identify identifies peer on the link
// it expects peer to send RequestIdentification packet and TODO
// it expects peer to send RequestIdentification packet and replies with AcceptIdentification if identification parameters are ok.
// returns information about identified node or error.
func Identify(link *NodeLink) (nodeInfo RequestIdentification /*TODO -> NodeInfo*/, err error) {
// the first conn must come with RequestIdentification packet
conn, err := link.Accept()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7