go/neo/proto/RowInfo: Fix representation on the wire

Some NEO protocol packets have the field 'RowList'. This field contains
information about each row of a partition table. In NEO/go the information
of each row is represented with the 'RowInfo' type [1]. This type is defined
as a struct with the field ‘CellList’. ‘CellList’ is defined as a list of
'CellInfo' [1] (e.g. an entry for each cell).

NEO/go {en,de}codes struct types with ‘genStructHead’ (structures in
golang are encoded as arrays in msgpack) [2]. From the 'RowList'
definition, the msgpack decoder currently expects the following msgpack
array structure:

    ArrayHeader (RowList)

        ArrayHeader (RowInfo)

            ArrayHeader (CellList)

                ArrayHeader (CellInfo)

                    int32 (NID)
                    enum (State)

However NEO/py actually sends:

    ArrayHeader (RowList)

        ArrayHeader (CellList)

            ArrayHeader (CellInfo)

                int32 (NID)
                enum (State)

In other words, currently the NEO/go msgpack encoder expects one more
nesting, which NEO/py doesn’t provide (and which also doesn’t seem to
be necessary as the outer nesting would always only contain one
element). We could adjust the msgpack {en,de}coder to introduce an
exception for the 'RowInfo' type, however as the protocol definition in
'proto.go' aims to transparently reflect the structure of the packets on
the wire, it seems to be more appropriate to fix this straight in the
protocol definition. This is also less error-prone as we don't have to
fix all the different positions of the encoder, decoder & sizer and it's
less code (particularly if 'RowInfo' doesn't stay the only case for such
an issue).

[1] https://lab.nexedi.com/kirr/neo/-/blob/1ad088c8/go/neo/proto/proto.go#L391-394
[2] https://lab.nexedi.com/kirr/neo/-/blob/1ad088c8/go/neo/proto/protogen.go#L1770-1775

go/neo/neo_test.go

@@ -100,7 +100,7 @@ func _TestMasterStorage(t0 *tEnv) {
 		PTid:		1,
 		NumReplicas:    0,
 		RowList:	[]proto.RowInfo{
-			{[]proto.CellInfo{{proto.NID(proto.STORAGE, 1), proto.UP_TO_DATE}}},
+			proto.RowInfo{proto.CellInfo{proto.NID(proto.STORAGE, 1), proto.UP_TO_DATE}},
 		},
 	}))
 
@@ -173,7 +173,7 @@ func _TestMasterStorage(t0 *tEnv) {
 		PTid:		1,
 		NumReplicas:	0,
 		RowList:	[]proto.RowInfo{
-			{[]proto.CellInfo{{proto.NID(proto.STORAGE, 1), proto.UP_TO_DATE}}},
+			proto.RowInfo{proto.CellInfo{proto.NID(proto.STORAGE, 1), proto.UP_TO_DATE}},
 		},
 	}))
 

go/neo/proto/proto.go

@@ -389,9 +389,7 @@ type CellInfo struct {
 }
 
 //neo:proto typeonly
-type RowInfo struct {
-	CellList []CellInfo
-}
+type RowInfo []CellInfo
 
 
 

go/neo/proto/proto_test.go

@@ -210,9 +210,9 @@ func TestMsgMarshal(t *testing.T) {
 			PTid:        0x0102030405060708,
 			NumReplicas: 34,
 			RowList: []RowInfo{
-				{[]CellInfo{{11, UP_TO_DATE}, {17, OUT_OF_DATE}}},
-				{[]CellInfo{{11, FEEDING}}},
-				{[]CellInfo{{11, CORRUPTED}, {15, DISCARDED}, {23, UP_TO_DATE}}},
+				{CellInfo{11, UP_TO_DATE}, CellInfo{17, OUT_OF_DATE}},
+				{CellInfo{11, FEEDING}},
+				{CellInfo{11, CORRUPTED}, CellInfo{15, DISCARDED}, CellInfo{23, UP_TO_DATE}},
 			},
 			},
 
@@ -229,9 +229,9 @@ func TestMsgMarshal(t *testing.T) {
 			hex("cf0102030405060708") +
 			hex("22") +
 			hex("93") +
-				hex("91"+"92"+"920bd40001"+"9211d40000") +
-				hex("91"+"91"+"920bd40002") +
-				hex("91"+"93"+"920bd40003"+"920fd40004"+"9217d40001"),
+				hex("92"+"920bd40001"+"9211d40000") +
+				hex("91"+"920bd40002") +
+				hex("93"+"920bd40003"+"920fd40004"+"9217d40001"),
 		},
 
 		// map[Oid]struct {Tid,Tid,bool}

go/neo/proto/zproto-marshal.go

@@ -1525,7 +1525,7 @@ func (p *AnswerPartitionTable) neoMsgEncodedLenN() int {
 	var size int
 	for i := 0; i < len(p.RowList); i++ {
 		a := &p.RowList[i]
-		size += len((*a).CellList) * 5
+		size += len((*a)) * 5
 	}
 	return 16 + len(p.RowList)*4 + size
 }
@@ -1540,11 +1540,11 @@ func (p *AnswerPartitionTable) neoMsgEncodeN(data []byte) {
 		for i := 0; i < l; i++ {
 			a := &p.RowList[i]
 			{
-				l := len((*a).CellList)
+				l := len((*a))
 				binary.BigEndian.PutUint32(data[0:], uint32(l))
 				data = data[4:]
 				for i := 0; i < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					binary.BigEndian.PutUint32(data[0:], uint32(int32((*a).NID)))
 					(data[4:])[0] = uint8(int8((*a).State))
 					data = data[5:]
@@ -1577,9 +1577,9 @@ func (p *AnswerPartitionTable) neoMsgDecodeN(data []byte) (int, error) {
 					goto overflow
 				}
 				nread += uint64(l) * 5
-				(*a).CellList = make([]CellInfo, l)
+				(*a) = make([]CellInfo, l)
 				for i := 0; uint32(i) < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					(*a).NID = NodeID(int32(binary.BigEndian.Uint32(data[0 : 0+4])))
 					(*a).State = CellState(int8((data[4 : 4+1])[0]))
 					data = data[5:]
@@ -1598,11 +1598,11 @@ func (p *AnswerPartitionTable) neoMsgEncodedLenM() int {
 	var size int
 	for i := 0; i < len(p.RowList); i++ {
 		a := &p.RowList[i]
-		for i := 0; i < len((*a).CellList); i++ {
-			a := &(*a).CellList[i]
+		for i := 0; i < len((*a)); i++ {
+			a := &(*a)[i]
 			size += msgpack.ArrayHeadSize(reflect.TypeOf((*a)).NumField()) + msgpack.Int32Size(int32((*a).NID))
 		}
-		size += msgpack.ArrayHeadSize(reflect.TypeOf((*a)).NumField()) + msgpack.ArrayHeadSize(len((*a).CellList)) + len((*a).CellList)*3
+		size += msgpack.ArrayHeadSize(len((*a))) + len((*a))*3
 	}
 	return msgpack.ArrayHeadSize(reflect.TypeOf((*p)).NumField()) + msgpack.Uint64Size(uint64(p.PTid)) + msgpack.Uint32Size(p.NumReplicas) + msgpack.ArrayHeadSize(len(p.RowList)) + size
 }
@@ -1623,13 +1623,12 @@ func (p *AnswerPartitionTable) neoMsgEncodeM(data []byte) {
 		data = data[0+n:]
 		for i := 0; i < l; i++ {
 			a := &p.RowList[i]
-			data[0] = byte(msgpack.FixArray_4 | 1)
 			{
-				l := len((*a).CellList)
-				n := msgpack.PutArrayHead(data[1:], l)
-				data = data[1+n:]
+				l := len((*a))
+				n := msgpack.PutArrayHead(data[0:], l)
+				data = data[0+n:]
 				for i := 0; i < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					data[0] = byte(msgpack.FixArray_4 | 2)
 					{
 						n := msgpack.PutInt32(data[1:], int32((*a).NID))
@@ -1686,24 +1685,16 @@ func (p *AnswerPartitionTable) neoMsgDecodeM(data []byte) (int, error) {
 		p.RowList = make([]RowInfo, l)
 		for i := 0; uint32(i) < l; i++ {
 			a := &p.RowList[i]
-			{
-				_, tail, err := msgp.ReadArrayHeaderBytes(data)
-				if err != nil {
-					return 0, mdecodeErr("AnswerPartitionTable", err)
-				}
-				nread += uint64(len(data) - len(tail))
-				data = tail
-			}
 			{
 				l, tail, err := msgp.ReadArrayHeaderBytes(data)
 				if err != nil {
-					return 0, mdecodeErr("AnswerPartitionTable.CellList", err)
+					return 0, mdecodeErr("AnswerPartitionTable", err)
 				}
 				nread += uint64(len(data) - len(tail))
 				data = tail
-				(*a).CellList = make([]CellInfo, l)
+				(*a) = make([]CellInfo, l)
 				for i := 0; uint32(i) < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					{
 						_, tail, err := msgp.ReadArrayHeaderBytes(data)
 						if err != nil {
@@ -1759,7 +1750,7 @@ func (p *SendPartitionTable) neoMsgEncodedLenN() int {
 	var size int
 	for i := 0; i < len(p.RowList); i++ {
 		a := &p.RowList[i]
-		size += len((*a).CellList) * 5
+		size += len((*a)) * 5
 	}
 	return 16 + len(p.RowList)*4 + size
 }
@@ -1774,11 +1765,11 @@ func (p *SendPartitionTable) neoMsgEncodeN(data []byte) {
 		for i := 0; i < l; i++ {
 			a := &p.RowList[i]
 			{
-				l := len((*a).CellList)
+				l := len((*a))
 				binary.BigEndian.PutUint32(data[0:], uint32(l))
 				data = data[4:]
 				for i := 0; i < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					binary.BigEndian.PutUint32(data[0:], uint32(int32((*a).NID)))
 					(data[4:])[0] = uint8(int8((*a).State))
 					data = data[5:]
@@ -1811,9 +1802,9 @@ func (p *SendPartitionTable) neoMsgDecodeN(data []byte) (int, error) {
 					goto overflow
 				}
 				nread += uint64(l) * 5
-				(*a).CellList = make([]CellInfo, l)
+				(*a) = make([]CellInfo, l)
 				for i := 0; uint32(i) < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					(*a).NID = NodeID(int32(binary.BigEndian.Uint32(data[0 : 0+4])))
 					(*a).State = CellState(int8((data[4 : 4+1])[0]))
 					data = data[5:]
@@ -1832,11 +1823,11 @@ func (p *SendPartitionTable) neoMsgEncodedLenM() int {
 	var size int
 	for i := 0; i < len(p.RowList); i++ {
 		a := &p.RowList[i]
-		for i := 0; i < len((*a).CellList); i++ {
-			a := &(*a).CellList[i]
+		for i := 0; i < len((*a)); i++ {
+			a := &(*a)[i]
 			size += msgpack.ArrayHeadSize(reflect.TypeOf((*a)).NumField()) + msgpack.Int32Size(int32((*a).NID))
 		}
-		size += msgpack.ArrayHeadSize(reflect.TypeOf((*a)).NumField()) + msgpack.ArrayHeadSize(len((*a).CellList)) + len((*a).CellList)*3
+		size += msgpack.ArrayHeadSize(len((*a))) + len((*a))*3
 	}
 	return msgpack.ArrayHeadSize(reflect.TypeOf((*p)).NumField()) + msgpack.Uint64Size(uint64(p.PTid)) + msgpack.Uint32Size(p.NumReplicas) + msgpack.ArrayHeadSize(len(p.RowList)) + size
 }
@@ -1857,13 +1848,12 @@ func (p *SendPartitionTable) neoMsgEncodeM(data []byte) {
 		data = data[0+n:]
 		for i := 0; i < l; i++ {
 			a := &p.RowList[i]
-			data[0] = byte(msgpack.FixArray_4 | 1)
 			{
-				l := len((*a).CellList)
-				n := msgpack.PutArrayHead(data[1:], l)
-				data = data[1+n:]
+				l := len((*a))
+				n := msgpack.PutArrayHead(data[0:], l)
+				data = data[0+n:]
 				for i := 0; i < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					data[0] = byte(msgpack.FixArray_4 | 2)
 					{
 						n := msgpack.PutInt32(data[1:], int32((*a).NID))
@@ -1920,24 +1910,16 @@ func (p *SendPartitionTable) neoMsgDecodeM(data []byte) (int, error) {
 		p.RowList = make([]RowInfo, l)
 		for i := 0; uint32(i) < l; i++ {
 			a := &p.RowList[i]
-			{
-				_, tail, err := msgp.ReadArrayHeaderBytes(data)
-				if err != nil {
-					return 0, mdecodeErr("SendPartitionTable", err)
-				}
-				nread += uint64(len(data) - len(tail))
-				data = tail
-			}
 			{
 				l, tail, err := msgp.ReadArrayHeaderBytes(data)
 				if err != nil {
-					return 0, mdecodeErr("SendPartitionTable.CellList", err)
+					return 0, mdecodeErr("SendPartitionTable", err)
 				}
 				nread += uint64(len(data) - len(tail))
 				data = tail
-				(*a).CellList = make([]CellInfo, l)
+				(*a) = make([]CellInfo, l)
 				for i := 0; uint32(i) < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					{
 						_, tail, err := msgp.ReadArrayHeaderBytes(data)
 						if err != nil {
@@ -6173,7 +6155,7 @@ func (p *AnswerPartitionList) neoMsgEncodedLenN() int {
 	var size int
 	for i := 0; i < len(p.RowList); i++ {
 		a := &p.RowList[i]
-		size += len((*a).CellList) * 5
+		size += len((*a)) * 5
 	}
 	return 16 + len(p.RowList)*4 + size
 }
@@ -6188,11 +6170,11 @@ func (p *AnswerPartitionList) neoMsgEncodeN(data []byte) {
 		for i := 0; i < l; i++ {
 			a := &p.RowList[i]
 			{
-				l := len((*a).CellList)
+				l := len((*a))
 				binary.BigEndian.PutUint32(data[0:], uint32(l))
 				data = data[4:]
 				for i := 0; i < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					binary.BigEndian.PutUint32(data[0:], uint32(int32((*a).NID)))
 					(data[4:])[0] = uint8(int8((*a).State))
 					data = data[5:]
@@ -6225,9 +6207,9 @@ func (p *AnswerPartitionList) neoMsgDecodeN(data []byte) (int, error) {
 					goto overflow
 				}
 				nread += uint64(l) * 5
-				(*a).CellList = make([]CellInfo, l)
+				(*a) = make([]CellInfo, l)
 				for i := 0; uint32(i) < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					(*a).NID = NodeID(int32(binary.BigEndian.Uint32(data[0 : 0+4])))
 					(*a).State = CellState(int8((data[4 : 4+1])[0]))
 					data = data[5:]
@@ -6246,11 +6228,11 @@ func (p *AnswerPartitionList) neoMsgEncodedLenM() int {
 	var size int
 	for i := 0; i < len(p.RowList); i++ {
 		a := &p.RowList[i]
-		for i := 0; i < len((*a).CellList); i++ {
-			a := &(*a).CellList[i]
+		for i := 0; i < len((*a)); i++ {
+			a := &(*a)[i]
 			size += msgpack.ArrayHeadSize(reflect.TypeOf((*a)).NumField()) + msgpack.Int32Size(int32((*a).NID))
 		}
-		size += msgpack.ArrayHeadSize(reflect.TypeOf((*a)).NumField()) + msgpack.ArrayHeadSize(len((*a).CellList)) + len((*a).CellList)*3
+		size += msgpack.ArrayHeadSize(len((*a))) + len((*a))*3
 	}
 	return msgpack.ArrayHeadSize(reflect.TypeOf((*p)).NumField()) + msgpack.Uint64Size(uint64(p.PTid)) + msgpack.Uint32Size(p.NumReplicas) + msgpack.ArrayHeadSize(len(p.RowList)) + size
 }
@@ -6271,13 +6253,12 @@ func (p *AnswerPartitionList) neoMsgEncodeM(data []byte) {
 		data = data[0+n:]
 		for i := 0; i < l; i++ {
 			a := &p.RowList[i]
-			data[0] = byte(msgpack.FixArray_4 | 1)
 			{
-				l := len((*a).CellList)
-				n := msgpack.PutArrayHead(data[1:], l)
-				data = data[1+n:]
+				l := len((*a))
+				n := msgpack.PutArrayHead(data[0:], l)
+				data = data[0+n:]
 				for i := 0; i < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					data[0] = byte(msgpack.FixArray_4 | 2)
 					{
 						n := msgpack.PutInt32(data[1:], int32((*a).NID))
@@ -6334,24 +6315,16 @@ func (p *AnswerPartitionList) neoMsgDecodeM(data []byte) (int, error) {
 		p.RowList = make([]RowInfo, l)
 		for i := 0; uint32(i) < l; i++ {
 			a := &p.RowList[i]
-			{
-				_, tail, err := msgp.ReadArrayHeaderBytes(data)
-				if err != nil {
-					return 0, mdecodeErr("AnswerPartitionList", err)
-				}
-				nread += uint64(len(data) - len(tail))
-				data = tail
-			}
 			{
 				l, tail, err := msgp.ReadArrayHeaderBytes(data)
 				if err != nil {
-					return 0, mdecodeErr("AnswerPartitionList.CellList", err)
+					return 0, mdecodeErr("AnswerPartitionList", err)
 				}
 				nread += uint64(len(data) - len(tail))
 				data = tail
-				(*a).CellList = make([]CellInfo, l)
+				(*a) = make([]CellInfo, l)
 				for i := 0; uint32(i) < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					{
 						_, tail, err := msgp.ReadArrayHeaderBytes(data)
 						if err != nil {
@@ -7052,7 +7025,7 @@ func (p *AnswerTweakPartitionTable) neoMsgEncodedLenN() int {
 	var size int
 	for i := 0; i < len(p.RowList); i++ {
 		a := &p.RowList[i]
-		size += len((*a).CellList) * 5
+		size += len((*a)) * 5
 	}
 	return 5 + len(p.RowList)*4 + size
 }
@@ -7066,11 +7039,11 @@ func (p *AnswerTweakPartitionTable) neoMsgEncodeN(data []byte) {
 		for i := 0; i < l; i++ {
 			a := &p.RowList[i]
 			{
-				l := len((*a).CellList)
+				l := len((*a))
 				binary.BigEndian.PutUint32(data[0:], uint32(l))
 				data = data[4:]
 				for i := 0; i < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					binary.BigEndian.PutUint32(data[0:], uint32(int32((*a).NID)))
 					(data[4:])[0] = uint8(int8((*a).State))
 					data = data[5:]
@@ -7102,9 +7075,9 @@ func (p *AnswerTweakPartitionTable) neoMsgDecodeN(data []byte) (int, error) {
 					goto overflow
 				}
 				nread += uint64(l) * 5
-				(*a).CellList = make([]CellInfo, l)
+				(*a) = make([]CellInfo, l)
 				for i := 0; uint32(i) < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					(*a).NID = NodeID(int32(binary.BigEndian.Uint32(data[0 : 0+4])))
 					(*a).State = CellState(int8((data[4 : 4+1])[0]))
 					data = data[5:]
@@ -7123,11 +7096,11 @@ func (p *AnswerTweakPartitionTable) neoMsgEncodedLenM() int {
 	var size int
 	for i := 0; i < len(p.RowList); i++ {
 		a := &p.RowList[i]
-		for i := 0; i < len((*a).CellList); i++ {
-			a := &(*a).CellList[i]
+		for i := 0; i < len((*a)); i++ {
+			a := &(*a)[i]
 			size += msgpack.ArrayHeadSize(reflect.TypeOf((*a)).NumField()) + msgpack.Int32Size(int32((*a).NID))
 		}
-		size += msgpack.ArrayHeadSize(reflect.TypeOf((*a)).NumField()) + msgpack.ArrayHeadSize(len((*a).CellList)) + len((*a).CellList)*3
+		size += msgpack.ArrayHeadSize(len((*a))) + len((*a))*3
 	}
 	return 1 + msgpack.ArrayHeadSize(reflect.TypeOf((*p)).NumField()) + msgpack.ArrayHeadSize(len(p.RowList)) + size
 }
@@ -7141,13 +7114,12 @@ func (p *AnswerTweakPartitionTable) neoMsgEncodeM(data []byte) {
 		data = data[2+n:]
 		for i := 0; i < l; i++ {
 			a := &p.RowList[i]
-			data[0] = byte(msgpack.FixArray_4 | 1)
 			{
-				l := len((*a).CellList)
-				n := msgpack.PutArrayHead(data[1:], l)
-				data = data[1+n:]
+				l := len((*a))
+				n := msgpack.PutArrayHead(data[0:], l)
+				data = data[0+n:]
 				for i := 0; i < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					data[0] = byte(msgpack.FixArray_4 | 2)
 					{
 						n := msgpack.PutInt32(data[1:], int32((*a).NID))
@@ -7198,24 +7170,16 @@ func (p *AnswerTweakPartitionTable) neoMsgDecodeM(data []byte) (int, error) {
 		p.RowList = make([]RowInfo, l)
 		for i := 0; uint32(i) < l; i++ {
 			a := &p.RowList[i]
-			{
-				_, tail, err := msgp.ReadArrayHeaderBytes(data)
-				if err != nil {
-					return 0, mdecodeErr("AnswerTweakPartitionTable", err)
-				}
-				nread += uint64(len(data) - len(tail))
-				data = tail
-			}
 			{
 				l, tail, err := msgp.ReadArrayHeaderBytes(data)
 				if err != nil {
-					return 0, mdecodeErr("AnswerTweakPartitionTable.CellList", err)
+					return 0, mdecodeErr("AnswerTweakPartitionTable", err)
 				}
 				nread += uint64(len(data) - len(tail))
 				data = tail
-				(*a).CellList = make([]CellInfo, l)
+				(*a) = make([]CellInfo, l)
 				for i := 0; uint32(i) < l; i++ {
-					a := &(*a).CellList[i]
+					a := &(*a)[i]
 					{
 						_, tail, err := msgp.ReadArrayHeaderBytes(data)
 						if err != nil {

go/neo/xneo/parttab.go

@@ -260,7 +260,7 @@ func (pt *PartitionTable) ToMsg() *proto.SendPartitionTable {
 			cellv[j] = cell.CellInfo
 		}
 
-		rowv[i] = proto.RowInfo{CellList: cellv}
+		rowv[i] = cellv
 	}
 	msg.RowList = rowv
 
@@ -280,7 +280,7 @@ func PartTabFromMsg(msg *proto.SendPartitionTable) *PartitionTable {
 		}
 
 		//pt.tab[i] = append(pt.tab[i], row.CellList...)
-		for _, cell := range row.CellList {
+		for _, cell := range row {
 			pt.tab[i] = append(pt.tab[i], Cell{cell})
 		}
 	}