go/neo/proto/RowInfo: Fix representation on the wire

Some NEO protocol packets have the field 'RowList'. This field contains
information about each row of a partition table. In NEO/go the information
of each row is represented with the 'RowInfo' type [1]. This type is defined
as a struct with the field ‘CellList’. ‘CellList’ is defined as a list of
'CellInfo' [1] (e.g. an entry for each cell).

NEO/go {en,de}codes struct types with ‘genStructHead’ (structures in
golang are encoded as arrays in msgpack) [2]. From the 'RowList'
definition, the msgpack decoder currently expects the following msgpack
array structure:

    ArrayHeader (RowList)

        ArrayHeader (RowInfo)

            ArrayHeader (CellList)

                ArrayHeader (CellInfo)

                    int32 (NID)
                    enum (State)

However NEO/py actually sends:

    ArrayHeader (RowList)

        ArrayHeader (CellList)

            ArrayHeader (CellInfo)

                int32 (NID)
                enum (State)

In other words, currently the NEO/go msgpack encoder expects one more
nesting, which NEO/py doesn’t provide (and which also doesn’t seem to
be necessary as the outer nesting would always only contain one
element). We could adjust the msgpack {en,de}coder to introduce an
exception for the 'RowInfo' type, however as the protocol definition in
'proto.go' aims to transparently reflect the structure of the packets on
the wire, it seems to be more appropriate to fix this straight in the
protocol definition. This is also less error-prone as we don't have to
fix all the different positions of the encoder, decoder & sizer and it's
less code (particularly if 'RowInfo' doesn't stay the only case for such
an issue).

[1] https://lab.nexedi.com/kirr/neo/-/blob/1ad088c8/go/neo/proto/proto.go#L391-394
[2] https://lab.nexedi.com/kirr/neo/-/blob/1ad088c8/go/neo/proto/protogen.go#L1770-1775

go/neo/neo_test.go

@@ -100,7 +100,7 @@ func _TestMasterStorage(t0 *tEnv) {
 		PTid:		1,
 		NumReplicas:    0,
 		RowList:	[]proto.RowInfo{
-			{[]proto.CellInfo{{proto.NID(proto.STORAGE, 1), proto.UP_TO_DATE}}},
+			proto.RowInfo{proto.CellInfo{proto.NID(proto.STORAGE, 1), proto.UP_TO_DATE}},
 		},
 	}))
 
@@ -173,7 +173,7 @@ func _TestMasterStorage(t0 *tEnv) {
 		PTid:		1,
 		NumReplicas:	0,
 		RowList:	[]proto.RowInfo{
-			{[]proto.CellInfo{{proto.NID(proto.STORAGE, 1), proto.UP_TO_DATE}}},
+			proto.RowInfo{proto.CellInfo{proto.NID(proto.STORAGE, 1), proto.UP_TO_DATE}},
 		},
 	}))
 

go/neo/proto/proto.go

@@ -389,9 +389,7 @@ type CellInfo struct {
 }
 
 //neo:proto typeonly
-type RowInfo struct {
-	CellList []CellInfo
-}
+type RowInfo []CellInfo
 
 
 

go/neo/proto/zproto-marshal.go

@@ -1585,7 +1585,7 @@ func (p *AnswerPartitionTable) neoMsgEncodedLenN() int {
 	var size int
 	for i := 0; i < len(p.RowList); i++ {
 		a := &p.RowList[i]
-		size += len((*a).CellList) * 5
+		size += len((*a)) * 5
 	}
 	return 16 + len(p.RowList)*4 + size
 }
@@ -1600,11 +1600,11 @@ func (p *AnswerPartitionTable) neoMsgEncodeN(data []byte) {
 		for i := 0; i < l; i++ {
 			a := &p.RowList[i]
 			{
-				l := len((*a).CellList)
+				l := len((*a))
 				binary.BigEndian.PutUint32(data[0:], uint32(l))
 				data = data[4:]
 				for i := 0; i < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					binary.BigEndian.PutUint32(data[0:], uint32(int32((*a).NID)))
 					(data[4:])[0] = uint8(int8((*a).State))
 					data = data[5:]
@@ -1637,9 +1637,9 @@ func (p *AnswerPartitionTable) neoMsgDecodeN(data []byte) (int, error) {
 					goto overflow
 				}
 				nread += uint64(l) * 5
-				(*a).CellList = make([]CellInfo, l)
+				(*a) = make([]CellInfo, l)
 				for i := 0; uint32(i) < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					(*a).NID = NodeID(int32(binary.BigEndian.Uint32(data[0 : 0+4])))
 					(*a).State = CellState(int8((data[4 : 4+1])[0]))
 					data = data[5:]
@@ -1658,13 +1658,13 @@ func (p *AnswerPartitionTable) neoMsgEncodedLenM() int {
 	var size int
 	for i := 0; i < len(p.RowList); i++ {
 		a := &p.RowList[i]
-		for i := 0; i < len((*a).CellList); i++ {
-			a := &(*a).CellList[i]
+		for i := 0; i < len((*a)); i++ {
+			a := &(*a)[i]
 			size += msgpack.Int32Size(int32((*a).NID))
 		}
-		size += msgpack.ArrayHeadSize(len((*a).CellList)) + len((*a).CellList)*4
+		size += msgpack.ArrayHeadSize(len((*a))) + len((*a))*4
 	}
-	return 1 + msgpack.Uint64Size(uint64(p.PTid)) + msgpack.Uint32Size(p.NumReplicas) + msgpack.ArrayHeadSize(len(p.RowList)) + len(p.RowList)*1 + size
+	return 1 + msgpack.Uint64Size(uint64(p.PTid)) + msgpack.Uint32Size(p.NumReplicas) + msgpack.ArrayHeadSize(len(p.RowList)) + size
 }
 
 func (p *AnswerPartitionTable) neoMsgEncodeM(data []byte) {
@@ -1683,13 +1683,12 @@ func (p *AnswerPartitionTable) neoMsgEncodeM(data []byte) {
 		data = data[0+n:]
 		for i := 0; i < l; i++ {
 			a := &p.RowList[i]
-			data[0] = byte(msgpack.FixArray_4 | 1)
 			{
-				l := len((*a).CellList)
-				n := msgpack.PutArrayHead(data[1:], l)
-				data = data[1+n:]
+				l := len((*a))
+				n := msgpack.PutArrayHead(data[0:], l)
+				data = data[0+n:]
 				for i := 0; i < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					data[0] = byte(msgpack.FixArray_4 | 2)
 					{
 						n := msgpack.PutInt32(data[1:], int32((*a).NID))
@@ -1749,27 +1748,16 @@ func (p *AnswerPartitionTable) neoMsgDecodeM(data []byte) (int, error) {
 		p.RowList = make([]RowInfo, l)
 		for i := 0; uint32(i) < l; i++ {
 			a := &p.RowList[i]
-			if op, opOk := msgpack.Op(data[0]), msgpack.FixArray_4|1; op != opOk {
-				return 0, &mstructDecodeError{"AnswerPartitionTable", op, opOk}
-			}
 			{
 				l, tail, err := msgp.ReadArrayHeaderBytes(data)
 				if err != nil {
 					return 0, mdecodeErr("AnswerPartitionTable", err)
 				}
-				data = tail
-				nread += uint64(l)
-			}
-			{
-				l, tail, err := msgp.ReadArrayHeaderBytes(data)
-				if err != nil {
-					return 0, mdecodeErr("AnswerPartitionTable.CellList", err)
-				}
 				nread += uint64(len(data) - len(tail))
 				data = tail
-				(*a).CellList = make([]CellInfo, l)
+				(*a) = make([]CellInfo, l)
 				for i := 0; uint32(i) < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					if op, opOk := msgpack.Op(data[0]), msgpack.FixArray_4|2; op != opOk {
 						return 0, &mstructDecodeError{"AnswerPartitionTable", op, opOk}
 					}
@@ -1828,7 +1816,7 @@ func (p *SendPartitionTable) neoMsgEncodedLenN() int {
 	var size int
 	for i := 0; i < len(p.RowList); i++ {
 		a := &p.RowList[i]
-		size += len((*a).CellList) * 5
+		size += len((*a)) * 5
 	}
 	return 16 + len(p.RowList)*4 + size
 }
@@ -1843,11 +1831,11 @@ func (p *SendPartitionTable) neoMsgEncodeN(data []byte) {
 		for i := 0; i < l; i++ {
 			a := &p.RowList[i]
 			{
-				l := len((*a).CellList)
+				l := len((*a))
 				binary.BigEndian.PutUint32(data[0:], uint32(l))
 				data = data[4:]
 				for i := 0; i < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					binary.BigEndian.PutUint32(data[0:], uint32(int32((*a).NID)))
 					(data[4:])[0] = uint8(int8((*a).State))
 					data = data[5:]
@@ -1880,9 +1868,9 @@ func (p *SendPartitionTable) neoMsgDecodeN(data []byte) (int, error) {
 					goto overflow
 				}
 				nread += uint64(l) * 5
-				(*a).CellList = make([]CellInfo, l)
+				(*a) = make([]CellInfo, l)
 				for i := 0; uint32(i) < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					(*a).NID = NodeID(int32(binary.BigEndian.Uint32(data[0 : 0+4])))
 					(*a).State = CellState(int8((data[4 : 4+1])[0]))
 					data = data[5:]
@@ -1901,13 +1889,13 @@ func (p *SendPartitionTable) neoMsgEncodedLenM() int {
 	var size int
 	for i := 0; i < len(p.RowList); i++ {
 		a := &p.RowList[i]
-		for i := 0; i < len((*a).CellList); i++ {
-			a := &(*a).CellList[i]
+		for i := 0; i < len((*a)); i++ {
+			a := &(*a)[i]
 			size += msgpack.Int32Size(int32((*a).NID))
 		}
-		size += msgpack.ArrayHeadSize(len((*a).CellList)) + len((*a).CellList)*4
+		size += msgpack.ArrayHeadSize(len((*a))) + len((*a))*4
 	}
-	return 1 + msgpack.Uint64Size(uint64(p.PTid)) + msgpack.Uint32Size(p.NumReplicas) + msgpack.ArrayHeadSize(len(p.RowList)) + len(p.RowList)*1 + size
+	return 1 + msgpack.Uint64Size(uint64(p.PTid)) + msgpack.Uint32Size(p.NumReplicas) + msgpack.ArrayHeadSize(len(p.RowList)) + size
 }
 
 func (p *SendPartitionTable) neoMsgEncodeM(data []byte) {
@@ -1926,13 +1914,12 @@ func (p *SendPartitionTable) neoMsgEncodeM(data []byte) {
 		data = data[0+n:]
 		for i := 0; i < l; i++ {
 			a := &p.RowList[i]
-			data[0] = byte(msgpack.FixArray_4 | 1)
 			{
-				l := len((*a).CellList)
-				n := msgpack.PutArrayHead(data[1:], l)
-				data = data[1+n:]
+				l := len((*a))
+				n := msgpack.PutArrayHead(data[0:], l)
+				data = data[0+n:]
 				for i := 0; i < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					data[0] = byte(msgpack.FixArray_4 | 2)
 					{
 						n := msgpack.PutInt32(data[1:], int32((*a).NID))
@@ -1992,27 +1979,16 @@ func (p *SendPartitionTable) neoMsgDecodeM(data []byte) (int, error) {
 		p.RowList = make([]RowInfo, l)
 		for i := 0; uint32(i) < l; i++ {
 			a := &p.RowList[i]
-			if op, opOk := msgpack.Op(data[0]), msgpack.FixArray_4|1; op != opOk {
-				return 0, &mstructDecodeError{"SendPartitionTable", op, opOk}
-			}
 			{
 				l, tail, err := msgp.ReadArrayHeaderBytes(data)
 				if err != nil {
 					return 0, mdecodeErr("SendPartitionTable", err)
 				}
-				data = tail
-				nread += uint64(l)
-			}
-			{
-				l, tail, err := msgp.ReadArrayHeaderBytes(data)
-				if err != nil {
-					return 0, mdecodeErr("SendPartitionTable.CellList", err)
-				}
 				nread += uint64(len(data) - len(tail))
 				data = tail
-				(*a).CellList = make([]CellInfo, l)
+				(*a) = make([]CellInfo, l)
 				for i := 0; uint32(i) < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					if op, opOk := msgpack.Op(data[0]), msgpack.FixArray_4|2; op != opOk {
 						return 0, &mstructDecodeError{"SendPartitionTable", op, opOk}
 					}
@@ -6387,7 +6363,7 @@ func (p *AnswerPartitionList) neoMsgEncodedLenN() int {
 	var size int
 	for i := 0; i < len(p.RowList); i++ {
 		a := &p.RowList[i]
-		size += len((*a).CellList) * 5
+		size += len((*a)) * 5
 	}
 	return 16 + len(p.RowList)*4 + size
 }
@@ -6402,11 +6378,11 @@ func (p *AnswerPartitionList) neoMsgEncodeN(data []byte) {
 		for i := 0; i < l; i++ {
 			a := &p.RowList[i]
 			{
-				l := len((*a).CellList)
+				l := len((*a))
 				binary.BigEndian.PutUint32(data[0:], uint32(l))
 				data = data[4:]
 				for i := 0; i < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					binary.BigEndian.PutUint32(data[0:], uint32(int32((*a).NID)))
 					(data[4:])[0] = uint8(int8((*a).State))
 					data = data[5:]
@@ -6439,9 +6415,9 @@ func (p *AnswerPartitionList) neoMsgDecodeN(data []byte) (int, error) {
 					goto overflow
 				}
 				nread += uint64(l) * 5
-				(*a).CellList = make([]CellInfo, l)
+				(*a) = make([]CellInfo, l)
 				for i := 0; uint32(i) < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					(*a).NID = NodeID(int32(binary.BigEndian.Uint32(data[0 : 0+4])))
 					(*a).State = CellState(int8((data[4 : 4+1])[0]))
 					data = data[5:]
@@ -6460,13 +6436,13 @@ func (p *AnswerPartitionList) neoMsgEncodedLenM() int {
 	var size int
 	for i := 0; i < len(p.RowList); i++ {
 		a := &p.RowList[i]
-		for i := 0; i < len((*a).CellList); i++ {
-			a := &(*a).CellList[i]
+		for i := 0; i < len((*a)); i++ {
+			a := &(*a)[i]
 			size += msgpack.Int32Size(int32((*a).NID))
 		}
-		size += msgpack.ArrayHeadSize(len((*a).CellList)) + len((*a).CellList)*4
+		size += msgpack.ArrayHeadSize(len((*a))) + len((*a))*4
 	}
-	return 1 + msgpack.Uint64Size(uint64(p.PTid)) + msgpack.Uint32Size(p.NumReplicas) + msgpack.ArrayHeadSize(len(p.RowList)) + len(p.RowList)*1 + size
+	return 1 + msgpack.Uint64Size(uint64(p.PTid)) + msgpack.Uint32Size(p.NumReplicas) + msgpack.ArrayHeadSize(len(p.RowList)) + size
 }
 
 func (p *AnswerPartitionList) neoMsgEncodeM(data []byte) {
@@ -6485,13 +6461,12 @@ func (p *AnswerPartitionList) neoMsgEncodeM(data []byte) {
 		data = data[0+n:]
 		for i := 0; i < l; i++ {
 			a := &p.RowList[i]
-			data[0] = byte(msgpack.FixArray_4 | 1)
 			{
-				l := len((*a).CellList)
-				n := msgpack.PutArrayHead(data[1:], l)
-				data = data[1+n:]
+				l := len((*a))
+				n := msgpack.PutArrayHead(data[0:], l)
+				data = data[0+n:]
 				for i := 0; i < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					data[0] = byte(msgpack.FixArray_4 | 2)
 					{
 						n := msgpack.PutInt32(data[1:], int32((*a).NID))
@@ -6551,27 +6526,16 @@ func (p *AnswerPartitionList) neoMsgDecodeM(data []byte) (int, error) {
 		p.RowList = make([]RowInfo, l)
 		for i := 0; uint32(i) < l; i++ {
 			a := &p.RowList[i]
-			if op, opOk := msgpack.Op(data[0]), msgpack.FixArray_4|1; op != opOk {
-				return 0, &mstructDecodeError{"AnswerPartitionList", op, opOk}
-			}
 			{
 				l, tail, err := msgp.ReadArrayHeaderBytes(data)
 				if err != nil {
 					return 0, mdecodeErr("AnswerPartitionList", err)
 				}
-				data = tail
-				nread += uint64(l)
-			}
-			{
-				l, tail, err := msgp.ReadArrayHeaderBytes(data)
-				if err != nil {
-					return 0, mdecodeErr("AnswerPartitionList.CellList", err)
-				}
 				nread += uint64(len(data) - len(tail))
 				data = tail
-				(*a).CellList = make([]CellInfo, l)
+				(*a) = make([]CellInfo, l)
 				for i := 0; uint32(i) < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					if op, opOk := msgpack.Op(data[0]), msgpack.FixArray_4|2; op != opOk {
 						return 0, &mstructDecodeError{"AnswerPartitionList", op, opOk}
 					}
@@ -7296,7 +7260,7 @@ func (p *AnswerTweakPartitionTable) neoMsgEncodedLenN() int {
 	var size int
 	for i := 0; i < len(p.RowList); i++ {
 		a := &p.RowList[i]
-		size += len((*a).CellList) * 5
+		size += len((*a)) * 5
 	}
 	return 5 + len(p.RowList)*4 + size
 }
@@ -7310,11 +7274,11 @@ func (p *AnswerTweakPartitionTable) neoMsgEncodeN(data []byte) {
 		for i := 0; i < l; i++ {
 			a := &p.RowList[i]
 			{
-				l := len((*a).CellList)
+				l := len((*a))
 				binary.BigEndian.PutUint32(data[0:], uint32(l))
 				data = data[4:]
 				for i := 0; i < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					binary.BigEndian.PutUint32(data[0:], uint32(int32((*a).NID)))
 					(data[4:])[0] = uint8(int8((*a).State))
 					data = data[5:]
@@ -7346,9 +7310,9 @@ func (p *AnswerTweakPartitionTable) neoMsgDecodeN(data []byte) (int, error) {
 					goto overflow
 				}
 				nread += uint64(l) * 5
-				(*a).CellList = make([]CellInfo, l)
+				(*a) = make([]CellInfo, l)
 				for i := 0; uint32(i) < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					(*a).NID = NodeID(int32(binary.BigEndian.Uint32(data[0 : 0+4])))
 					(*a).State = CellState(int8((data[4 : 4+1])[0]))
 					data = data[5:]
@@ -7367,13 +7331,13 @@ func (p *AnswerTweakPartitionTable) neoMsgEncodedLenM() int {
 	var size int
 	for i := 0; i < len(p.RowList); i++ {
 		a := &p.RowList[i]
-		for i := 0; i < len((*a).CellList); i++ {
-			a := &(*a).CellList[i]
+		for i := 0; i < len((*a)); i++ {
+			a := &(*a)[i]
 			size += msgpack.Int32Size(int32((*a).NID))
 		}
-		size += msgpack.ArrayHeadSize(len((*a).CellList)) + len((*a).CellList)*4
+		size += msgpack.ArrayHeadSize(len((*a))) + len((*a))*4
 	}
-	return 2 + msgpack.ArrayHeadSize(len(p.RowList)) + len(p.RowList)*1 + size
+	return 2 + msgpack.ArrayHeadSize(len(p.RowList)) + size
 }
 
 func (p *AnswerTweakPartitionTable) neoMsgEncodeM(data []byte) {
@@ -7385,13 +7349,12 @@ func (p *AnswerTweakPartitionTable) neoMsgEncodeM(data []byte) {
 		data = data[2+n:]
 		for i := 0; i < l; i++ {
 			a := &p.RowList[i]
-			data[0] = byte(msgpack.FixArray_4 | 1)
 			{
-				l := len((*a).CellList)
-				n := msgpack.PutArrayHead(data[1:], l)
-				data = data[1+n:]
+				l := len((*a))
+				n := msgpack.PutArrayHead(data[0:], l)
+				data = data[0+n:]
 				for i := 0; i < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					data[0] = byte(msgpack.FixArray_4 | 2)
 					{
 						n := msgpack.PutInt32(data[1:], int32((*a).NID))
@@ -7445,27 +7408,16 @@ func (p *AnswerTweakPartitionTable) neoMsgDecodeM(data []byte) (int, error) {
 		p.RowList = make([]RowInfo, l)
 		for i := 0; uint32(i) < l; i++ {
 			a := &p.RowList[i]
-			if op, opOk := msgpack.Op(data[0]), msgpack.FixArray_4|1; op != opOk {
-				return 0, &mstructDecodeError{"AnswerTweakPartitionTable", op, opOk}
-			}
 			{
 				l, tail, err := msgp.ReadArrayHeaderBytes(data)
 				if err != nil {
 					return 0, mdecodeErr("AnswerTweakPartitionTable", err)
 				}
-				data = tail
-				nread += uint64(l)
-			}
-			{
-				l, tail, err := msgp.ReadArrayHeaderBytes(data)
-				if err != nil {
-					return 0, mdecodeErr("AnswerTweakPartitionTable.CellList", err)
-				}
 				nread += uint64(len(data) - len(tail))
 				data = tail
-				(*a).CellList = make([]CellInfo, l)
+				(*a) = make([]CellInfo, l)
 				for i := 0; uint32(i) < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					if op, opOk := msgpack.Op(data[0]), msgpack.FixArray_4|2; op != opOk {
 						return 0, &mstructDecodeError{"AnswerTweakPartitionTable", op, opOk}
 					}

go/neo/xneo/parttab.go

@@ -260,7 +260,7 @@ func (pt *PartitionTable) ToMsg() *proto.SendPartitionTable {
 			cellv[j] = cell.CellInfo
 		}
 
-		rowv[i] = proto.RowInfo{CellList: cellv}
+		rowv[i] = cellv
 	}
 	msg.RowList = rowv
 
@@ -280,7 +280,7 @@ func PartTabFromMsg(msg *proto.SendPartitionTable) *PartitionTable {
 		}
 
 		//pt.tab[i] = append(pt.tab[i], row.CellList...)
-		for _, cell := range row.CellList {
+		for _, cell := range row {
 			pt.tab[i] = append(pt.tab[i], Cell{cell})
 		}
 	}