X Clarified Request.Close semantics - tests working again

go/neo/client/client.go

@@ -281,8 +281,17 @@ func (c *Client) recvMaster(ctx context.Context, mlink *neo.NodeLink) (err error
 		if err != nil {
 			return err
 		}
+		err = c.recvMaster1(ctx, req)
 		req.Close()
+		if err != nil {
+			return err
+		}
 
+	}
+}
+
+// recvMaster1 handles 1 message from master
+func (c *Client) recvMaster1(ctx context.Context, req neo.Request) error {
 	c.node.StateMu.Lock()
 
 	switch msg := req.Msg.(type) {
@@ -309,7 +318,8 @@ func (c *Client) recvMaster(ctx context.Context, mlink *neo.NodeLink) (err error
 	opready := c.updateOperational()
 	c.node.StateMu.Unlock()
 	opready()
-	}
+
+	return nil
 }
 
 func (c *Client) initFromMaster(ctx context.Context, mlink *neo.NodeLink) (err error) {

go/neo/connection.go

@@ -1427,6 +1427,8 @@ func (c *Conn) Ask(req Msg, resp Msg) error {
 //
 // No Send or Recv must be in flight.
 // The caller must not use c after call to close - the connection is returned to freelist.
+//
+// XXX must be called only once.
 func (c *Conn) lightClose() {
 	nl := c.link
 	nl.connMu.Lock()
@@ -1460,7 +1462,7 @@ func (link *NodeLink) Recv1() (Request, error) {
 	// NOTE serveRecv guaranty that when a conn is accepted, there is 1 message in conn.rxq
 	msg, err := conn.Recv()		// XXX directly from <-rxq
 	if err != nil {
-		conn.Close() // XXX -> conn.release
+		conn.Close() // XXX -> conn.lightClose()
 		return Request{}, err
 	}
 

go/neo/server/master.go

@@ -860,7 +860,7 @@ func storCtlService(ctx context.Context, stor *neo.Node) (err error) {
 	//if err != nil {
 	//	return err
 	//}
-	//req.Close()
+	//req.Close()	XXX must be after req handling
 	//switch msg := req.Msg.(type) {
 	//case *neo.NotifyReady:
 	//	// ok
@@ -910,6 +910,7 @@ func (m *Master) serveClient(ctx context.Context, cli *neo.Node) (err error) {
 
 			resp := m.serveClient1(ctx, req.Msg)
 			err = req.Reply(resp)
+			req.Close()
 			if err != nil {
 				return err
 			}

go/neo/server/storage.go

@@ -238,16 +238,32 @@ func (stor *Storage) m1initialize(ctx context.Context, mlink *neo.NodeLink) (req
 		if err != nil {
 			return nil, err
 		}
+		err = stor.m1initialize1(ctx, req)
+		if err == cmdStart {
+			// start - transition to serve
+			return &req, nil
+		}
+		req.Close()
+		if err != nil {
+			return nil, err
+		}
+	}
+}
 
+var cmdStart = errors.New("start requested")
+
+// m1initialize1 handles one message from master from under m1initialize
+func (stor *Storage) m1initialize1(ctx context.Context, req neo.Request) error {
 	// XXX vvv move Send out of reply preparing logic
+	var err error
 
 	switch msg := req.Msg.(type) {
 	default:
-			return nil, fmt.Errorf("unexpected message: %T", msg)
+		return fmt.Errorf("unexpected message: %T", msg)
 
 	case *neo.StartOperation:
 		// ok, transition to serve
-			return &req, nil
+		return cmdStart
 
 	case *neo.Recovery:
 		err = req.Reply(&neo.AnswerRecovery{
@@ -271,7 +287,7 @@ func (stor *Storage) m1initialize(ctx context.Context, mlink *neo.NodeLink) (req
 		lastTid, zerr1 := stor.zstor.LastTid(ctx)
 		lastOid, zerr2 := stor.zstor.LastOid(ctx)
 		if zerr := xerr.First(zerr1, zerr2); zerr != nil {
-				return nil, zerr	// XXX send the error to M
+			return zerr	// XXX send the error to M
 		}
 
 		err = req.Reply(&neo.AnswerLastIDs{LastTid: lastTid, LastOid: lastOid})
@@ -292,12 +308,7 @@ func (stor *Storage) m1initialize(ctx context.Context, mlink *neo.NodeLink) (req
 	}
 
 	// XXX move req.Reply here and ^^^ only prepare reply
-		if err != nil {
-			return nil, err
-		}
-
-		req.Close() // XXX err?
-	}
+	return err
 }
 
 // m1serve drives storage by master messages during service phase.
@@ -324,6 +335,7 @@ func (stor *Storage) m1serve(ctx context.Context, reqStart *neo.Request) (err er
 	// reply M we are ready
 	// XXX according to current neo/py this is separate send - not reply - and so we do here
 	err = reqStart.Reply(&neo.NotifyReady{})
+	reqStart.Close()
 	if err != nil {
 		return err
 	}
@@ -334,9 +346,16 @@ func (stor *Storage) m1serve(ctx context.Context, reqStart *neo.Request) (err er
 		if err != nil {
 			return err
 		}
+		err = stor.m1serve1(ctx, req)
+		req.Close()
+		if err != nil {
+			return err
+		}
+	}
+}
 
-		req.Close() // XXX stub, err
-
+// m1serve1 handles one message from master under m1serve
+func (stor *Storage) m1serve1(ctx context.Context, req neo.Request) error {
 	switch msg := req.Msg.(type) {
 	default:
 		return fmt.Errorf("unexpected message: %T", msg)
@@ -355,7 +374,8 @@ func (stor *Storage) m1serve(ctx context.Context, reqStart *neo.Request) (err er
 
 	// TODO commit related messages
 	}
-	}
+
+	return nil
 }
 
 // --- serve incoming connections from other nodes ---
@@ -466,6 +486,7 @@ func (stor *Storage) serveClient(ctx context.Context, req neo.Request) {
 	for {
 		resp := stor.serveClient1(ctx, req.Msg)
 		err := req.Reply(resp)
+		req.Close()
 		if err != nil {
 			log.Error(ctx, err)
 			return