go/neo/internal/msgpack: New package with runtime support for MessagePack

It complements github.com/tinylib/msgp with things that neo/proto needs
to encode/decode messages.

go/neo/internal/msgpack/msgpack.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package msgpack complements tinylib/msgp in providing runtime support for MessagePack.
+//
+// https://github.com/msgpack/msgpack/blob/master/spec.md
+package msgpack
+
+import (
+	"encoding/binary"
+	"math"
+)
+
+// Op represents a MessagePack opcode.
+type Op byte
+const (
+	FixMap_4   Op  = 0b1000_0000 // 1000_XXXX
+	FixArray_4 Op  = 0b1001_0000 // 1001_XXXX
+
+	False	 Op  = 0xc2
+	True	 Op  = 0xc3
+
+	Bin8     Op  = 0xc4
+	Bin16    Op  = 0xc5
+	Bin32    Op  = 0xc6
+
+	Float32  Op  = 0xca
+	Float64  Op  = 0xcb
+
+	Uint8    Op  = 0xcc
+	Uint16   Op  = 0xcd
+	Uint32   Op  = 0xce
+	Uint64   Op  = 0xcf
+
+	Int8     Op  = 0xd0
+	Int16    Op  = 0xd1
+	Int32    Op  = 0xd2
+	Int64    Op  = 0xd3
+
+	FixExt1  Op  = 0xd4
+	FixExt2  Op  = 0xd5
+	FixExt4  Op  = 0xd6
+
+	Array16  Op  = 0xdc
+	Array32  Op  = 0xdd
+	Map16    Op  = 0xde
+	Map32    Op  = 0xdf
+)
+
+// op converts Op into byte.
+// it is used internally to make sure that only Op is put into encoded data.
+func op(x Op) byte {
+	return byte(x)
+}
+
+// Bool returns op corresponding to bool value v.
+func Bool(v bool) Op {
+	if v {
+		return True
+	} else {
+		return False
+	}
+}
+
+// u?intXSize(i) returns size needed to encode i.
+func Int8Size (i int8) int	{ return Int64Size(int64(i)) }
+func Int16Size(i int16) int	{ return Int64Size(int64(i)) }
+func Int32Size(i int32) int	{ return Int64Size(int64(i)) }
+func Uint8Size (i uint8) int	{ return Uint64Size(uint64(i)) }
+func Uint16Size(i uint16) int	{ return Uint64Size(uint64(i)) }
+func Uint32Size(i uint32) int	{ return Uint64Size(uint64(i)) }
+
+// Putu?intX(data, i X) encodes i into data and returns encoded size.
+func PutInt8 (data []byte, i int8)  int    { return PutInt64(data, int64(i)) }
+func PutInt16(data []byte, i int16) int    { return PutInt64(data, int64(i)) }
+func PutInt32(data []byte, i int32) int    { return PutInt64(data, int64(i)) }
+func PutUint8 (data []byte, i uint8)  int  { return PutUint64(data, uint64(i)) }
+func PutUint16(data []byte, i uint16) int  { return PutUint64(data, uint64(i)) }
+func PutUint32(data []byte, i uint32) int  { return PutUint64(data, uint64(i)) }
+
+func Int64Size(i int64) int {
+	switch {
+	case -32 <= i && i <= 0b0_1111111:	return 1   // posfixint | negfixint
+	case int64(int8(i))  == i:		return 1+1 // int8  + i8
+	case int64(int16(i)) == i:		return 1+2 // int16 + i16
+	case int64(int32(i)) == i:		return 1+4 // int32 + i32
+	default:				return 1+8 // int64 + u64
+	}
+}
+
+func PutInt64(data []byte, i int64) (n int) {
+	switch {
+	// posfixint | negfixint
+	case -32 <= i && i <= 0b0_1111111:
+		data[0] = uint8(i)
+		return 1
+
+	// int8 + s8
+	case int64(int8(i)) == i:
+		data[0] = op(Int8)
+		data[1] = uint8(i)
+		return 1+1
+
+	// int16 + s16
+	case int64(int16(i)) == i:
+		data[0] = op(Int16)
+		binary.BigEndian.PutUint16(data[1:], uint16(i))
+		return 1+2
+
+	// int32 + s32
+	case int64(int32(i)) == i:
+		data[0] = op(Int32)
+		binary.BigEndian.PutUint32(data[1:], uint32(i))
+		return 1+4
+
+	// int64 + s64
+	default:
+		data[0] = op(Int64)
+		binary.BigEndian.PutUint64(data[1:], uint64(i))
+		return 1+8
+	}
+}
+
+func Uint64Size(i uint64) int {
+	switch {
+	case i <= 0x7f:		return 1   // posfixint
+	case i <= 0xff:		return 1+1 // uint8  + u8
+	case i <= 0xffff:	return 1+2 // uint16 + u16
+	case i <= 0xffffffff:	return 1+4 // uint32 + u32
+	default:		return 1+8 // uint64 + u64
+	}
+}
+
+func PutUint64(data []byte, i uint64) (n int) {
+	switch {
+	// posfixint
+	case i <= 0x7f:
+		data[0] = uint8(i)
+		return 1
+
+	// uint8  + u8
+	case i <= math.MaxUint8:
+		data[0] = op(Uint8)
+		data[1] = uint8(i)
+		return 1+1
+
+	// uint16 + be16
+	case i <= math.MaxUint16:
+		data[0] = op(Uint16)
+		binary.BigEndian.PutUint16(data[1:], uint16(i))
+		return 1+2
+
+	// uint32 + be32
+	case i <= math.MaxUint32:
+		data[0] = op(Uint32)
+		binary.BigEndian.PutUint32(data[1:], uint32(i))
+		return 1+4
+
+	// uint64 + be64
+	default:
+		data[0] = op(Uint64)
+		binary.BigEndian.PutUint64(data[1:], i)
+		return 1+8
+	}
+}
+
+// BinHeadSize return number of bytes needed to encode header for [l]bin.
+func BinHeadSize(l int) int {
+	switch {
+	case l < 0: panic("len < 0")
+	case l <= math.MaxUint8:  return 1+1 // bin8  + len8
+	case l <= math.MaxUint16: return 1+2 // bin16 + len16
+	case l <= math.MaxUint32: return 1+4 // bin32 + len32
+	default: panic("len overflows uint32")
+	}
+}
+
+// PutBinHead puts binary header for [size]bin.
+func PutBinHead(data []byte, l int) (n int) {
+	switch {
+	case l < 0: panic("len < 0")
+
+	// bin8 + len8
+	case l <= 0xff:
+		data[0] = op(Bin8)
+		data[1] = uint8(l)
+		return 1+1
+
+	// bin16 + len16
+	case l <= math.MaxUint16:
+		data[0] = op(Bin16)
+		binary.BigEndian.PutUint16(data[1:], uint16(l))
+		return 1+2
+
+	// bin32 + len32
+	case l <= math.MaxUint32:
+		data[0] = op(Bin32)
+		binary.BigEndian.PutUint32(data[1:], uint32(l))
+		return 1+4
+
+	default: panic("len overflows uint32")
+	}
+}
+
+// ArrayHeadSize returns size for array header for [size]array.
+func ArrayHeadSize(l int) int {
+	switch {
+	case l <  0:               panic("len < 0")
+	case l <= 0x0f:            return 1   // fixarray
+	case l <= math.MaxUint16:  return 1+2 // array16 + len16
+	case l <= math.MaxUint32:  return 1+4 // array32 + len32
+	default: panic("len overflows uint32")
+	}
+}
+
+// PutArrayHead puts array header for [size]array.
+func PutArrayHead(data []byte, l int) (n int) {
+	switch {
+	case l < 0: panic("len < 0")
+
+	// fixarray
+	case l <= 0x0f:
+		data[0] = op(FixArray_4 | Op(l))
+		return 1
+
+	// array16 + len16
+	case l <= math.MaxUint16:
+		data[0] = op(Array16)
+		binary.BigEndian.PutUint16(data[1:], uint16(l))
+		return 1+2
+
+	// array32 + len32
+	case l <= math.MaxUint32:
+		data[0] = op(Array32)
+		binary.BigEndian.PutUint32(data[1:], uint32(l))
+		return 1+4
+
+	default: panic("len overflows uint32")
+	}
+}
+
+// MapHeadSize returns size for map header for [size]map.
+func MapHeadSize(l int) int {
+	return ArrayHeadSize(l) // the same 0x0f/len16/len32 scheme
+}
+
+// PutMapHead puts map header for [size]map.
+func PutMapHead(data []byte, l int) (n int) {
+	switch {
+	case l < 0: panic("len < 0")
+
+	// fixmap
+	case l <= 0x0f:
+		data[0] = op(FixMap_4 | Op(l))
+		return 1
+
+	// map16 + len16
+	case l <= math.MaxUint16:
+		data[0] = op(Map16)
+		binary.BigEndian.PutUint16(data[1:], uint16(l))
+		return 1+2
+
+	// map32 + len32
+	case l <= math.MaxUint32:
+		data[0] = op(Map32)
+		binary.BigEndian.PutUint32(data[1:], uint32(l))
+		return 1+4
+
+	default: panic("len overflows uint32")
+	}
+}

go/neo/internal/msgpack/msgpack_test.go

+// Copyright (C) 2020-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package msgpack
+
+import (
+	hexpkg "encoding/hex"
+	"testing"
+)
+
+// hex decodes string as hex; panics on error.
+func hex(s string) string {
+	b, err := hexpkg.DecodeString(s)
+	if err != nil {
+		panic(err)
+	}
+	return string(b)
+}
+
+
+
+// tGetPutSize is interface with Get/Put/Size methods, e.g. with
+// GetBinHead/PutBinHead/BinHeadSize.
+type tGetPutSize interface {
+	// TODO Get(data []byte) (n int, ret interface{})
+	Size(arg interface{}) int
+	Put(data []byte, arg interface{}) int
+}
+
+// test1 verifies enc functions on one argument.
+func test1(t *testing.T, enc tGetPutSize, arg interface{}, encoded string) {
+	t.Helper()
+	data := make([]byte, 16)
+	n := enc.Put(data, arg)
+	got := string(data[:n])
+	if got != encoded {
+		t.Errorf("%v -> %x  ; want %x", arg, got, encoded)
+	}
+	if sz := enc.Size(arg); sz != n {
+		t.Errorf("size(%v) -> %d  ; len(data)=%d", arg, sz, n)
+	}
+	// TODO decode == arg, n
+	// TODO decode([:n-1]) -> overflow
+}
+
+type tEncUint64 struct{}
+func (_ *tEncUint64) Size(xi interface{}) int              { return Uint64Size(xi.(uint64)) }
+func (_ *tEncUint64) Put(data []byte, xi interface{}) int  { return PutUint64(data, xi.(uint64)) }
+func TestUint(t *testing.T) {
+	h := hex
+	testv := []struct{i uint64; encoded string}{
+		{0,    h("00")}, // posfixint
+		{1,    h("01")},
+		{0x7f, h("7f")},
+		{0x80,  h("cc80")}, // uint8
+		{0xff,  h("ccff")},
+		{0x100,  h("cd0100")}, // uint16
+		{0xffff, h("cdffff")},
+		{0x10000,    h("ce00010000")}, // uint32
+		{0xffffffff, h("ceffffffff")},
+		{0x100000000,        h("cf0000000100000000")}, // uint64
+		{0xffffffffffffffff, h("cfffffffffffffffff")},
+	}
+
+	for _, tt := range testv {
+		test1(t, &tEncUint64{}, tt.i, tt.encoded)
+	}
+}
+
+type tEncInt64 struct{}
+func (_ *tEncInt64) Size(xi interface{}) int              { return Int64Size(xi.(int64)) }
+func (_ *tEncInt64) Put(data []byte, xi interface{}) int  { return PutInt64(data, xi.(int64)) }
+func TestInt(t *testing.T) {
+	h := hex
+	testv := []struct{i int64; encoded string}{
+		{0,    h("00")}, // posfixint
+		{1,    h("01")},
+		{0x7f, h("7f")},
+		{-1,   h("ff")}, // negfixint
+		{-2,   h("fe")},
+		{-31,  h("e1")},
+		{-32,  h("e0")},
+		{-33,   h("d0df")}, // int8
+		{-0x7f, h("d081")},
+		{-0x80, h("d080")},
+		{0x80,    h("d10080")}, // int16
+		{0x7fff,  h("d17fff")},
+		{-0x7fff, h("d18001")},
+		{-0x8000, h("d18000")},
+		{0x8000,      h("d200008000")}, // int32
+		{0x7fffffff,  h("d27fffffff")},
+		{-0x8001,     h("d2ffff7fff")},
+		{-0x7fffffff, h("d280000001")},
+		{-0x80000000, h("d280000000")},
+		{0x80000000,          h("d30000000080000000")}, // int64
+		{0x7fffffffffffffff,  h("d37fffffffffffffff")},
+		{-0x80000001,         h("d3ffffffff7fffffff")},
+		{-0x7fffffffffffffff, h("d38000000000000001")},
+		{-0x8000000000000000, h("d38000000000000000")},
+	}
+
+	for _, tt := range testv {
+		test1(t, &tEncInt64{}, tt.i, tt.encoded)
+	}
+}
+
+type tEncBinHead struct{}
+func (_ *tEncBinHead) Size(xl interface{}) int              { return BinHeadSize(xl.(int)) }
+func (_ *tEncBinHead) Put(data []byte, xl interface{}) int  { return PutBinHead(data, xl.(int)) }
+func TestBin(t *testing.T) {
+	h := hex
+	testv := []struct{l int; encoded string}{
+		{0,    h("c400")}, // bin8
+		{1,    h("c401")},
+		{0xff, h("c4ff")},
+		{0x100,  h("c50100")}, // bin16
+		{0xffff, h("c5ffff")},
+		{0x10000,    h("c600010000")}, // bin32
+		{0xffffffff, h("c6ffffffff")},
+	}
+
+	for _, tt := range testv {
+		test1(t, &tEncBinHead{}, tt.l, tt.encoded)
+	}
+}
+
+type tEncArrayHead struct{}
+func (_ *tEncArrayHead) Size(xl interface{}) int              { return ArrayHeadSize(xl.(int)) }
+func (_ *tEncArrayHead) Put(data []byte, xl interface{}) int  { return PutArrayHead(data, xl.(int)) }
+func TestArray(t *testing.T) {
+	h := hex
+	testv := []struct{l int; encoded string}{
+		{0,    h("90")}, // fixarray
+		{1,    h("91")},
+		{14,   h("9e")},
+		{15,   h("9f")},
+		{0x10,   h("dc0010")}, // array16
+		{0x11,   h("dc0011")},
+		{0x100,  h("dc0100")},
+		{0xffff, h("dcffff")},
+		{0x10000,    h("dd00010000")}, // array32
+		{0xffffffff, h("ddffffffff")},
+	}
+
+	for _, tt := range testv {
+		test1(t, &tEncArrayHead{}, tt.l, tt.encoded)
+	}
+}
+
+type tEncMapHead struct{}
+func (_ *tEncMapHead) Size(xl interface{}) int              { return MapHeadSize(xl.(int)) }
+func (_ *tEncMapHead) Put(data []byte, xl interface{}) int  { return PutMapHead(data, xl.(int)) }
+func TestMap(t *testing.T) {
+	h := hex
+	testv := []struct{l int; encoded string}{
+		{0,    h("80")}, // fixmap
+		{1,    h("81")},
+		{14,   h("8e")},
+		{15,   h("8f")},
+		{0x10,   h("de0010")}, // map16
+		{0x11,   h("de0011")},
+		{0x100,  h("de0100")},
+		{0xffff, h("deffff")},
+		{0x10000,    h("df00010000")}, // map32
+		{0xffffffff, h("dfffffffff")},
+	}
+
+	for _, tt := range testv {
+		test1(t, &tEncMapHead{}, tt.l, tt.encoded)
+	}
+}