Skip to content

N
neo
Commit e8143557 authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

go/neo/proto: MessagePack support for messages (draft) 

This patch adds proto.Encoding('M') and teaches it to encode/decode
messages via MessagePack by the rules defined in

	nexedi/neoppod@9d0bf97a
	( nexedi/neoppod!11 )

It only adds support for messages serialization, without changing
proto.go to match changes in e.g. enums reordering, and without adding
support for MessagePack at link layer in neonet.

M-encoding was only tested for NEO/go-NEO/go, and was not yet tested for
NEO/go-NEO/py interoperation. There will be likely small mistakes
discovered on my side that should be hopefully easy to fix step by step
once we get to that phase.
parent a8249bd9
Hide whitespace changes
Inline Side-by-side
Showing with 7403 additions and 101 deletions
go/neo/proto/msgpack.go 0 → 100644
View file @ e8143557
// Copyright (C) 2020-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package proto
// runtime glue for msgpack support
import (
"fmt"
"github.com/tinylib/msgp/msgp"
"lab.nexedi.com/kirr/neo/go/neo/internal/msgpack"
)
// mstructDecodeError represents decode error when decoder was expecting
// tuple<nfield> for structure named path.
type mstructDecodeError struct {
path string // "Type.field.field"
op msgpack.Op // op we got
opOk msgpack.Op // op expected
}
func (e *mstructDecodeError) Error() string {
return fmt.Sprintf("decode: M: struct %s: got opcode %02x; expect %02x", e.path, e.op, e.opOk)
}
// mdecodeErr is called to normilize error when msgp.ReadXXX returns err when decoding path.
func mdecodeErr(path string, err error) error {
if err == msgp.ErrShortBytes {
return ErrDecodeOverflow
}
return &mdecodeError{path, err}
}
type mdecodeError struct {
path string // "Type.field.field"
err error
}
func (e *mdecodeError) Error() string {
return fmt.Sprintf("decode: M: %s: %s", e.path, e.err)
}
// mOpError represents decode error when decoder faces unexpected operation.
type mOpError struct {
op msgpack.Op // op we got
expectedv []msgpack.Op // expected was any op from expectedv
}
func (e *mOpError) Error() string {
return fmt.Sprintf("got opcode %02x; expected any from %02x", e.op, e.expectedv)
}
func mdecodeOpErr(path string, op msgpack.Op, expectedv ...msgpack.Op) error {
return mdecodeErr(path+"/op", &mOpError{op, expectedv})
}
// mLen8Error represents decode error when decoder faces unexpected length in Bin8.
type mLen8Error struct {
l, lOk byte // len we got and expected
}
func (e *mLen8Error) Error() string {
return fmt.Sprintf("expected length %d; got %d", e.lOk, e.l)
}
func mdecodeLen8Err(path string, l, lOk uint8) error {
return mdecodeErr(path+"/len", &mLen8Error{l, lOk})
}
func mdecodeEnumTypeErr(path string, enumType, enumTypeOk byte) error {
return mdecodeErr(path+"/enumType",
fmt.Errorf("expected %d; got %d", enumTypeOk, enumType))
}
func mdecodeEnumValueErr(path string, v byte) error {
return mdecodeErr(path, fmt.Errorf("invalid enum payload %02x", v))
}
go/neo/proto/proto.go
View file @ e8143557
......@@ -129,6 +129,10 @@ type Msg interface {
neoMsgEncodeN(buf []byte)
neoMsgDecodeN(data []byte) (nread int, err error)
// M encoding (via MessagePack)
neoMsgEncodedLenM() int
neoMsgEncodeM(buf []byte)
neoMsgDecodeM(data []byte) (nread int, err error)
}
// Encoding represents messages encoding.
......@@ -139,6 +143,7 @@ func (e Encoding) MsgEncodedLen(msg Msg) int {
switch e {
default: panic("bug")
case 'N': return msg.neoMsgEncodedLenN()
case 'M': return msg.neoMsgEncodedLenM()
}
}
......@@ -149,6 +154,7 @@ func (e Encoding) MsgEncode(msg Msg, buf []byte) {
switch e {
default: panic("bug")
case 'N': msg.neoMsgEncodeN(buf)
case 'M': msg.neoMsgEncodeM(buf)
}
}
......@@ -157,6 +163,7 @@ func (e Encoding) MsgDecode(msg Msg, data []byte) (nread int, err error) {
switch e {
default: panic("bug")
case 'N': return msg.neoMsgDecodeN(data)
case 'M': return msg.neoMsgDecodeM(data)
}
}
......@@ -166,6 +173,7 @@ var ErrDecodeOverflow = errors.New("decode: buffer overflow")
// ---- messages ----
//neo:proto enum
type ErrorCode uint32
const (
ACK ErrorCode = iota
......@@ -183,6 +191,7 @@ const (
INCOMPLETE_TRANSACTION
)
//neo:proto enum
type ClusterState int8
const (
// The cluster is initially in the RECOVERING state, and it goes back to
......@@ -216,6 +225,7 @@ const (
STOPPING_BACKUP
)
//neo:proto enum
type NodeType int8
const (
MASTER NodeType = iota
......@@ -224,6 +234,7 @@ const (
ADMIN
)
//neo:proto enum
type NodeState int8
const (
UNKNOWN NodeState = iota //short: U // XXX tag prefix name ?
......@@ -232,6 +243,7 @@ const (
PENDING //short: P
)
//neo:proto enum
type CellState int8
const (
// Write-only cell. Last transactions are missing because storage is/was down
......
go/neo/proto/proto_test.go
View file @ e8143557
......@@ -164,12 +164,19 @@ func TestMsgMarshal(t *testing.T) {
var testv = []struct {
msg Msg
encodedN string // []byte
encodedM string // []byte
}{
// empty
{&Ping{}, ""},
{&Ping{},
"",
"\x90",
},
// uint32, string
{&Error{Code: 0x01020304, Message: "hello"}, "\x01\x02\x03\x04\x00\x00\x00\x05hello"},
// uint32(N)/enum(M), string
{&Error{Code: 0x00000045, Message: "hello"},
"\x00\x00\x00\x45\x00\x00\x00\x05hello",
hex("92") + hex("d40045") + "\xc4\x05hello",
},
// Oid, Tid, bool, Checksum, []byte
{&StoreObject{
......@@ -185,7 +192,18 @@ func TestMsgMarshal(t *testing.T) {
hex("01020304050607080a0b0c0d0e0f010200") +
hex("0102030405060708090a0b0c0d0e0f1011121314") +
hex("0000000b") + "hello world" +
hex("0a0b0c0d0e0f01030a0b0c0d0e0f0104")},
hex("0a0b0c0d0e0f01030a0b0c0d0e0f0104"),
// M
hex("97") +
hex("c408") + hex("0102030405060708") +
hex("c408") + hex("0a0b0c0d0e0f0102") +
hex("c2") +
hex("c414") + hex("0102030405060708090a0b0c0d0e0f1011121314") +
hex("c40b") + "hello world" +
hex("c408") + hex("0a0b0c0d0e0f0103") +
hex("c408") + hex("0a0b0c0d0e0f0104"),
},
// PTid, [] (of [] of {UUID, CellState})
{&AnswerPartitionTable{
......@@ -205,6 +223,15 @@ func TestMsgMarshal(t *testing.T) {
hex("000000020000000b010000001100") +
hex("000000010000000b02") +
hex("000000030000000b030000000f040000001701"),
// M
hex("93") +
hex("cf0102030405060708") +
hex("22") +
hex("93") +
hex("91"+"92"+"920bd40401"+"9211d40400") +
hex("91"+"91"+"920bd40402") +
hex("91"+"93"+"920bd40403"+"920fd40404"+"9217d40401"),
},
// map[Oid]struct {Tid,Tid,bool}
......@@ -226,6 +253,14 @@ func TestMsgMarshal(t *testing.T) {
u64(2) + u64(7) + u64(1) + hex("01") +
u64(5) + u64(4) + u64(3) + hex("01") +
u64(8) + u64(7) + u64(1) + hex("00"),
// M
hex("91") +
hex("84") +
hex("c408")+u64(1) + hex("93") + hex("c408")+u64(1) + hex("c408")+u64(0) + hex("c2") +
hex("c408")+u64(2) + hex("93") + hex("c408")+u64(7) + hex("c408")+u64(1) + hex("c3") +
hex("c408")+u64(5) + hex("93") + hex("c408")+u64(4) + hex("c408")+u64(3) + hex("c3") +
hex("c408")+u64(8) + hex("93") + hex("c408")+u64(7) + hex("c408")+u64(1) + hex("c2"),
},
// map[uint32]UUID + trailing ...
......@@ -247,12 +282,28 @@ func TestMsgMarshal(t *testing.T) {
u32(4) + u32(17) +
u32(7) + u32(3) +
u64(23) + u64(128),
// M
hex("93") +
hex("84") +
hex("01" + "07") +
hex("02" + "09") +
hex("04" + "11") +
hex("07" + "03") +
hex("c408") + u64(23) +
hex("c408") + u64(128),
},
// uint32, []uint32
{&PartitionCorrupted{7, []NodeUUID{1, 3, 9, 4}},
// N
u32(7) + u32(4) + u32(1) + u32(3) + u32(9) + u32(4),
// M
hex("92") +
hex("07") +
hex("94") +
hex("01030904"),
},
// uint32, Address, string, IdTime
......@@ -264,6 +315,16 @@ func TestMsgMarshal(t *testing.T) {
hex("3fbf9add1091c895") +
u32(2) + u32(5)+"room1" + u32(7)+"rack234" +
u32(3) + u32(3)+u32(4)+u32(5),
// M
hex("97") +
hex("d40202") +
hex("11") +
hex("92") + hex("c409")+"localhost" + hex("cd")+u16(7777) +
hex("c406")+"myname" +
hex("cb" + "3fbf9add1091c895") +
hex("92") + hex("c405")+"room1" + hex("c407")+"rack234" +
hex("93") + hex("030405"),
},
// IdTime, empty Address, int32
......@@ -273,12 +334,26 @@ func TestMsgMarshal(t *testing.T) {
hex("41d66b15517b469d") + u32(1) +
u8(2) + u32(0) /* <- ø Address */ + hex("e0000001") + u8(2) +
hex("41d66b15517b3d04"),
// M
hex("92") +
hex("cb" + "41d66b15517b469d") +
hex("91") +
hex("95") +
hex("d40202") +
hex("92" + "c400"+"" + "00") +
hex("d2" + "e0000001") +
hex("d40302") +
hex("cb" + "41d66b15517b3d04"),
},
// empty IdTime
{&NotifyNodeInformation{IdTimeNone, []NodeInfo{}},
// N
hex("ffffffffffffffff") + hex("00000000"),
// M
hex("92") +
hex("cb" + "fff0000000000000") + // XXX nan/-inf not handled yet
hex("90"),
},
// TODO we need tests for:
......@@ -288,6 +363,7 @@ func TestMsgMarshal(t *testing.T) {
for _, tt := range testv {
testMsgMarshal(t, 'N', tt.msg, tt.encodedN)
testMsgMarshal(t, 'M', tt.msg, tt.encodedM)
}
}
......@@ -295,11 +371,14 @@ func TestMsgMarshal(t *testing.T) {
// this way we additionally lightly check encode / decode overflow behaviour for all types.
func TestMsgMarshalAllOverflowLightly(t *testing.T) {
for _, typ := range msgTypeRegistry {
for _, enc := range []Encoding{'N'} {
for _, enc := range []Encoding{'N', 'M'} {
// zero-value for a type
msg := reflect.New(typ).Interface().(Msg)
l := enc.MsgEncodedLen(msg)
zerol := make([]byte, l)
if enc != 'N' { // M-encoding of zero-value is not all zeros
enc.MsgEncode(msg, zerol)
}
// decoding will turn nil slice & map into empty allocated ones.
// we need it so that reflect.DeepEqual works for msg encode/decode comparison
n, err := enc.MsgDecode(msg, zerol)
......@@ -313,6 +392,7 @@ func TestMsgMarshalAllOverflowLightly(t *testing.T) {
}
// Verify overflow handling on decodeN len checks
// TODO + M-variants with big len too?
func TestMsgDecodeLenOverflowN(t *testing.T) {
enc := Encoding('N')
......
go/neo/proto/protogen.go
View file @ e8143557
......@@ -27,7 +27,7 @@ For every type the following methods are generated in accordance with neo.Msg in
neoMsgCode() uint16
; E stands for 'N' encoding
; E stands for 'N' and 'M' encodings
neoMsgEncodedLen<E>() int
neoMsgEncode<E>(buf []byte)
neoMsgDecode<E>(data []byte) (nread int, err error)
......@@ -80,6 +80,8 @@ import (
"os"
"sort"
"strings"
"lab.nexedi.com/kirr/neo/go/neo/internal/msgpack"
)
// parsed & typechecked input
......@@ -126,6 +128,10 @@ var zodbOid types.Type
var neo_customCodecN *types.Interface // type of neo.customCodecN
var memBuf types.Type // type of mem.Buf
// registry of enums
var enumRegistry = map[types.Type]int{} // type -> enum type serial
// bytes.Buffer + bell & whistles
type Buffer struct {
bytes.Buffer
......@@ -188,6 +194,7 @@ func loadPkg(pkgPath string, sources ...string) *types.Package {
type Annotation struct {
typeonly bool
answer bool
enum bool
}
// parse checks doc for specific comment annotations and, if present, loads them.
......@@ -218,6 +225,12 @@ func (a *Annotation) parse(doc *ast.CommentGroup) {
}
a.answer = true
case "enum":
if a.enum {
log.Fatalf("%v: duplicate `enum`", cpos)
}
a.enum = true
default:
log.Fatalf("%v: unknown neo:proto directive %q", cpos, arg)
}
......@@ -303,6 +316,9 @@ import (
"reflect"
"sort"
"github.com/tinylib/msgp/msgp"
"lab.nexedi.com/kirr/neo/go/neo/internal/msgpack"
"lab.nexedi.com/kirr/go123/mem"
"lab.nexedi.com/kirr/neo/go/zodb"
)`)
......@@ -312,6 +328,7 @@ import (
// go over message types declaration and generate marshal code for them
buf.emit("// messages marshalling\n")
msgSerial := 0
enumSerial := 0
for _, decl := range f.Decls {
// we look for types (which can be only under GenDecl)
gendecl, ok := decl.(*ast.GenDecl)
......@@ -332,16 +349,21 @@ import (
typespec := spec.(*ast.TypeSpec) // must be because tok = TYPE
typename := typespec.Name.Name
// we are only interested in struct types
if _, ok := typespec.Type.(*ast.StructType); !ok {
continue
}
// `//neo:proto ...` annotation for this particular type
specAnnotation := declAnnotation // inheriting from decl
specAnnotation.parse(typespec.Doc)
// type only -> don't generate message interface for it
// remember enum types
if specAnnotation.enum {
typ := typeInfo.Defs[typespec.Name].Type()
enumRegistry[typ]= enumSerial
enumSerial++
}
// messages are only struct types without typeonly annotation
if _, ok := typespec.Type.(*ast.StructType); !ok {
continue
}
if specAnnotation.typeonly {
continue
}
......@@ -366,6 +388,11 @@ import (
buf.WriteString(generateCodecCode(typespec, &encoderN{}))
buf.WriteString(generateCodecCode(typespec, &decoderN{}))
// TODO keep all M routines separate from N for code locality ?
buf.WriteString(generateCodecCode(typespec, &sizerM{}))
buf.WriteString(generateCodecCode(typespec, &encoderM{}))
buf.WriteString(generateCodecCode(typespec, &decoderM{}))
msgTypeRegistry[msgCode] = typename
msgSerial++
}
......@@ -431,6 +458,11 @@ func typeSizeFixed(encoding byte, typ types.Type) (wireSize int, ok bool) {
var size SymSize
switch encoding {
case 'M':
s := &sizerM{}
codegenType("x", typ, nil, s)
size = s.size
case 'N':
s := &sizerN{}
codegenType("x", typ, nil, s)
......@@ -493,6 +525,7 @@ type CodeGenCustomize interface {
// X reports encoding=X
type N struct{}; func (_ *N) encoding() byte { return 'N' }
type M struct{}; func (_ *M) encoding() byte { return 'M' }
// common part of codegenerators
type commonCodeGen struct {
......@@ -526,6 +559,12 @@ func (c *commonCodeGen) var_(varname string) string {
return varname
}
// pathName returns name representing path or assignto.
func (c *commonCodeGen) pathName(path string) string {
// Type, p.f1.f2 -> Type.f1.f2
return strings.Join(append([]string{c.typeName}, strings.Split(path, ".")[1:]...), ".")
}
// symbolic size
// consists of numeric & symbolic expression parts
// size is num + expr1 + expr2 + ...
......@@ -631,6 +670,7 @@ type sizerCommon struct {
size SymSize // currently accumulated size
}
type sizerN struct { sizerCommon; N }
type sizerM struct { sizerCommon; M }
// encoderX generates code to X-encode a message.
//
......@@ -652,6 +692,7 @@ type encoderCommon struct {
n int // current write position in data
}
type encoderN struct { encoderCommon; N }
type encoderM struct { encoderCommon; M }
// decoderX generates code to X-decode a message.
//
......@@ -686,12 +727,17 @@ type decoderCommon struct {
overflow OverflowCheck
}
type decoderN struct { decoderCommon; N }
type decoderM struct { decoderCommon; M }
var _ CodeGenerator = (*sizerN)(nil)
var _ CodeGenerator = (*encoderN)(nil)
var _ CodeGenerator = (*decoderN)(nil)
var _ CodeGenerator = (*sizerM)(nil)
var _ CodeGenerator = (*encoderM)(nil)
var _ CodeGenerator = (*decoderM)(nil)
func (s *sizerCommon) generatedCode() string {
code := Buffer{}
......@@ -849,7 +895,7 @@ func (d *decoderCommon) generatedCode() string {
// `goto overflow` is not used only for empty structs
// NOTE for >0 check actual X in StdSizes{X} does not particularly matter
if (&types.StdSizes{8, 8}).Sizeof(d.typ) > 0 {
if (&types.StdSizes{8, 8}).Sizeof(d.typ) > 0 || d.enc != 'N' {
code.emit("\noverflow:")
code.emit("return 0, ErrDecodeOverflow")
}
......@@ -897,6 +943,228 @@ func (d *decoderN) genBasic(assignto string, typ *types.Basic, userType types.Ty
d.overflow.Add(basic.wireSize)
}
// M: emit code to size/encode/decode basic fixed type
func (s *sizerM) genBasic(path string, typ *types.Basic, userType types.Type) {
// upath casts path into basic type if needed
// e.g. p.x -> int32(p.x) if p.x is custom type with underlying int32
upath := path
if userType.Underlying() != userType {
upath = fmt.Sprintf("%s(%s)", typ.Name(), upath)
}
// zodb.Tid and zodb.Oid are encoded as [8]bin
if userType == zodbTid || userType == zodbOid {
s.size.Add(1+1+8) // mbin8 + 8 + [8]data
return
}
// enums are encoded as extensions
if _, isEnum := enumRegistry[userType]; isEnum {
s.size.Add(1+1+1) // fixext1 enumType value
return
}
switch typ.Kind() {
case types.Bool: s.size.Add(1) // mfalse|mtrue
case types.Int8: s.size.AddExpr("msgpack.Int8Size(%s)", upath)
case types.Int16: s.size.AddExpr("msgpack.Int16Size(%s)", upath)
case types.Int32: s.size.AddExpr("msgpack.Int32Size(%s)", upath)
case types.Int64: s.size.AddExpr("msgpack.Int64Size(%s)", upath)
case types.Uint8: s.size.AddExpr("msgpack.Uint8Size(%s)", upath)
case types.Uint16: s.size.AddExpr("msgpack.Uint16Size(%s)", upath)
case types.Uint32: s.size.AddExpr("msgpack.Uint32Size(%s)", upath)
case types.Uint64: s.size.AddExpr("msgpack.Uint64Size(%s)", upath)
case types.Float64: s.size.Add(1+8) // mfloat64 + <value64>
}
}
func (e *encoderM) genBasic(path string, typ *types.Basic, userType types.Type) {
// upath casts path into basic type if needed
// e.g. p.x -> int32(p.x) if p.x is custom type with underlying int32
upath := path
if userType.Underlying() != userType {
upath = fmt.Sprintf("%s(%s)", typ.Name(), upath)
}
// zodb.Tid and zodb.Oid are encoded as [8]bin
if userType == zodbTid || userType == zodbOid {
e.emit("data[%v] = byte(msgpack.Bin8)", e.n); e.n++
e.emit("data[%v] = 8", e.n); e.n++
e.emit("binary.BigEndian.PutUint64(data[%v:], uint64(%s))", e.n, path)
e.n += 8
return
}
// enums are encoded as `fixext1 enumType fixint<value>`
if enum, ok := enumRegistry[userType]; ok {
e.emit("data[%v] = byte(msgpack.FixExt1)", e.n); e.n++
e.emit("data[%v] = %d", e.n, enum); e.n++
e.emit("if !(0 <= %s && %s <= 0x7f) {", path, path) // mposfixint
e.emit(` panic("%s: invalid %s enum value)")`, path, typeName(userType))
e.emit("}")
e.emit("data[%v] = byte(%s)", e.n, path); e.n++
return
}
// mputint emits mput<kind>int<size>(path)
mputint := func(kind string, size int) {
KI := "I" // I or <Kind>i
if kind != "" {
KI = strings.ToUpper(kind) + "i"
}
e.emit("{")
e.emit("n := msgpack.Put%snt%d(data[%v:], %s)", KI, size, e.n, upath)
e.emit("data = data[%v+n:]", e.n)
e.emit("}")
e.n = 0
}
switch typ.Kind() {
case types.Bool:
e.emit("data[%v] = byte(msgpack.Bool(%s))", e.n, path)
e.n += 1
case types.Int8: mputint("", 8)
case types.Int16: mputint("", 16)
case types.Int32: mputint("", 32)
case types.Int64: mputint("", 64)
case types.Uint8: mputint("u", 8)
case types.Uint16: mputint("u", 16)
case types.Uint32: mputint("u", 32)
case types.Uint64: mputint("u", 64)
case types.Float64:
// mfloat64 f64
e.emit("data[%v] = byte(msgpack.Float64)", e.n); e.n++
e.emit("float64_neoEncode(data[%v:], %s)", e.n, upath); e.n += 8
}
}
// decoder expects <op>
func (d *decoderM) expectOp(assignto string, op string) {
d.emit("if op := msgpack.Op(data[%v]); op != %s {", d.n, op); d.n++
d.emit(" return 0, mdecodeOpErr(%q, op, %s)", d.pathName(assignto), op)
d.emit("}")
d.overflow.Add(1)
}
// decoder expects mbin8 l
func (d *decoderM) expectBin8Fix(assignto string, l int) {
d.expectOp(assignto, "msgpack.Bin8")
d.emit("if l := data[%v]; l != %d {", d.n, l); d.n++
d.emit(" return 0, mdecodeLen8Err(%q, l, %d)", d.pathName(assignto), l)
d.emit("}")
d.overflow.Add(1)
}
// decoder expects mfixext1 <enumType>
func (d *decoderM) expectEnum(assignto string, enumType int) {
d.expectOp(assignto, "msgpack.FixExt1")
d.emit("if enumType := data[%v]; enumType != %d {", d.n, enumType); d.n++
d.emit(" return 0, mdecodeEnumTypeErr(%q, enumType, %d)", d.pathName(assignto), enumType)
d.emit("}")
d.overflow.Add(1)
}
func (d *decoderM) genBasic(assignto string, typ *types.Basic, userType types.Type) {
// zodb.Tid and zodb.Oid are encoded as [8]bin
if userType == zodbTid || userType == zodbOid {
d.expectBin8Fix(assignto, 8)
d.emit("%s= %s(binary.BigEndian.Uint64(data[%v:]))", assignto, typeName(userType), d.n)
d.n += 8
d.overflow.Add(8)
return
}
// enums are encoded as `fixext1 enumType fixint<value>`
if enum, ok := enumRegistry[userType]; ok {
d.expectEnum(assignto, enum)
d.emit("{")
d.emit("v := data[%v]", d.n); d.n++
d.emit("if !(0 <= v && v <= 0x7f) {") // mposfixint
d.emit(" return 0, mdecodeEnumValueErr(%q, v)", d.pathName(assignto))
d.emit("}")
d.emit("%s= %s(v)", assignto, typeName(userType))
d.emit("}")
d.overflow.Add(1)
return
}
// v represents basic decoded value casted to user type if needed
v := "v"
if userType.Underlying() != userType {
v = fmt.Sprintf("%s(v)", typeName(userType))
}
// mgetint emits assignto = mget<kind>int<size>()
mgetint := func(kind string, size int) {
// we are going to go into msgp - flush previously queued
// overflow checks; put place for next overflow check after
// msgp is done.
d.overflowCheck()
d.resetPos()
defer d.overflowCheck()
KI := "I" // I or <Kind>i
if kind != "" {
KI = strings.ToUpper(kind) + "i"
}
d.emit("{")
d.emit("v, tail, err := msgp.Read%snt%dBytes(data)", KI, size)
d.emit("if err != nil {")
d.emit(" return 0, mdecodeErr(%q, err)", d.pathName(assignto))
d.emit("}")
d.emit("%s= %s", assignto, v)
d.emit("%v += uint64(len(data) - len(tail))", d.var_("nread"))
d.emit("data = tail")
d.emit("}")
}
// mgetfloat emits mgetfloat<size>
mgetfloat := func(size int) {
// delving into msgp - flush/prepare next site for overflow check
d.overflowCheck()
d.resetPos()
defer d.overflowCheck()
d.emit("{")
d.emit("v, tail, err := msgp.ReadFloat%dBytes(data)", size)
d.emit("if err != nil {")
d.emit(" return 0, mdecodeErr(%q, err)", d.pathName(assignto))
d.emit("}")
d.emit("%s= %s", assignto, v)
d.emit("%v += uint64(len(data) - len(tail))", d.var_("nread"))
d.emit("data = tail")
d.emit("}")
}
switch typ.Kind() {
case types.Bool:
d.emit("switch op := msgpack.Op(data[%v]); op {", d.n)
d.emit("default: return 0, mdecodeOpErr(%q, op, msgpack.True, msgpack.False)", d.pathName(assignto))
d.emit("case msgpack.True: %s = true", assignto)
d.emit("case msgpack.False: %s = false", assignto)
d.emit("}")
d.n++
d.overflow.Add(1)
case types.Int8: mgetint("", 8)
case types.Int16: mgetint("", 16)
case types.Int32: mgetint("", 32)
case types.Int64: mgetint("", 64)
case types.Uint8: mgetint("u", 8)
case types.Uint16: mgetint("u", 16)
case types.Uint32: mgetint("u", 32)
case types.Uint64: mgetint("u", 64)
case types.Float64: mgetfloat(64)
}
}
// emit code to size/encode/decode array with sizeof(elem)==1
// [len(A)]byte
func (s *sizerN) genArray1(path string, typ *types.Array) {
......@@ -915,6 +1183,39 @@ func (d *decoderN) genArray1(assignto string, typ *types.Array) {
d.overflow.Add(typLen)
}
// binX+lenX
// [len(A)]byte
func (s *sizerM) genArray1(path string, typ *types.Array) {
l := int(typ.Len())
s.size.Add(msgpack.BinHeadSize(l))
s.size.Add(l)
}
func (e *encoderM) genArray1(path string, typ *types.Array) {
l := int(typ.Len())
if l > 0xff {
panic("TODO: array1 with > 255 elements")
}
e.emit("data[%v] = byte(msgpack.Bin8)", e.n); e.n++
e.emit("data[%v] = %d", e.n, l); e.n++
e.emit("copy(data[%v:], %v[:])", e.n, path)
e.n += l
}
func (d *decoderM) genArray1(assignto string, typ *types.Array) {
l := int(typ.Len())
if l > 0xff {
panic("TODO: array1 with > 255 elements")
}
d.expectBin8Fix(assignto, l)
d.emit("copy(%v[:], data[%v:%v])", assignto, d.n, d.n+l)
d.n += l
d.overflow.Add(l)
}
// emit code to size/encode/decode string or []byte
// len u32
// [len]byte
......@@ -963,15 +1264,71 @@ func (d *decoderN) genSlice1(assignto string, typ types.Type) {
d.emit("}")
}
// bin8+len8|bin16+len16|bin32+len32
// [len]byte
func (s *sizerM) genSlice1(path string, typ types.Type) {
s.size.AddExpr("msgpack.BinHeadSize(len(%s))", path)
s.size.AddExpr("len(%s)", path)
}
func (e *encoderM) genSlice1(path string, typ types.Type) {
e.emit("{")
e.emit("l := len(%s)", path)
e.emit("n := msgpack.PutBinHead(data[%v:], l)", e.n)
e.emit("data = data[%v+n:]", e.n)
e.emit("copy(data, %v)", path)
e.emit("data = data[l:]")
e.emit("}")
e.n = 0
}
func (d *decoderM) genSlice1(assignto string, typ types.Type) {
// -> msgp: flush queued overflow checks; put place for next overflow
// checks after msgp is done.
d.overflowCheck()
d.resetPos()
defer d.overflowCheck()
d.emit("{")
d.emit("b, tail, err := msgp.ReadBytesZC(data)")
d.emit("if err != nil {")
d.emit(" return 0, mdecodeErr(%q, err)", d.pathName(assignto))
d.emit("}")
switch t := typ.(type) {
case *types.Basic:
if t.Kind() != types.String {
log.Panicf("bad basic type in slice1: %v", t)
}
d.emit("%v= string(b)", assignto)
case *types.Slice:
// TODO eventually do not copy, but reference data from original
d.emit("%v= make(%v, len(b))", assignto, typeName(typ))
d.emit("copy(%v, b)", assignto)
default:
log.Panicf("bad type in slice1: %v", typ)
}
d.emit("%v += uint64(len(data) - len(tail))", d.var_("nread"))
d.emit("data = tail")
d.emit("}")
}
// emit code to size/encode/decode mem.Buf
// same as slice1 but buffer is allocated via mem.BufAlloc
func (s *sizerN) genBuf(path string) {
s.genSlice1(path+".XData()", nil /* typ unused */)
}
func (s *sizerM) genBuf(path string) {
s.genSlice1(path+".XData()", nil /* typ unused */)
}
func (e *encoderN) genBuf(path string) {
e.genSlice1(path+".XData()", nil /* typ unused */)
}
func (e *encoderM) genBuf(path string) {
e.genSlice1(path+".XData()", nil /* typ unused */)
}
func (d *decoderN) genBuf(assignto string) {
d.emit("{")
......@@ -990,6 +1347,29 @@ func (d *decoderN) genBuf(assignto string) {
d.emit("}")
}
func (d *decoderM) genBuf(assignto string) {
// -> msgp: flush queued overflow checks; put place for next overflow
// checks after msgp is done.
d.overflowCheck()
d.resetPos()
defer d.overflowCheck()
d.emit("{")
d.emit("b, tail, err := msgp.ReadBytesZC(data)")
d.emit("if err != nil {")
d.emit(" return 0, mdecodeErr(%q, err)", d.pathName(assignto))
d.emit("}")
// TODO eventually do not copy but reference original
d.emit("%v= mem.BufAlloc(len(b))", assignto)
d.emit("copy(%v.Data, b)", assignto)
d.emit("%v += uint64(len(data) - len(tail))", d.var_("nread"))
d.emit("data = tail")
d.emit("}")
}
// emit code to size/encode/decode slice
// len u32
// [len]item
......@@ -1087,6 +1467,44 @@ func (d *decoderCommon) genSliceCommon(xd CodeGenCustomize, assignto string, typ
d.emit("}")
}
// fixarray|array16+YYYY|array32+ZZZZ
// [len]item
func (s *sizerM) genSliceHead(path string, typ *types.Slice, obj types.Object) {
s.size.AddExpr("msgpack.ArrayHeadSize(len(%s))", path)
}
func (s *sizerM) genSlice(path string, typ *types.Slice, obj types.Object) {
s.genSliceCommon(s, path, typ, obj)
}
func (e *encoderM) genSliceHead(path string, typ *types.Slice, obj types.Object) {
e.emit("l := len(%s)", path)
e.emit("n := msgpack.PutArrayHead(data[%v:], l)", e.n)
e.emit("data = data[%v+n:]", e.n)
e.n = 0
}
func (e *encoderM) genSlice(path string, typ *types.Slice, obj types.Object) {
e.genSliceCommon(e, path, typ, obj)
}
func (d *decoderM) genSliceHead(assignto string, typ *types.Slice, obj types.Object) {
// -> msgp: flush queued overflow checks; put place for next overflow
// checks after msgp is done.
d.overflowCheck()
d.resetPos()
defer d.overflowCheck()
d.emit("l, tail, err := msgp.ReadArrayHeaderBytes(data)")
d.emit("if err != nil {")
d.emit(" return 0, mdecodeErr(%q, err)", d.pathName(assignto))
d.emit("}")
d.emit("%v += uint64(len(data) - len(tail))", d.var_("nread"))
d.emit("data = tail")
}
func (d *decoderM) genSlice(assignto string, typ *types.Slice, obj types.Object) {
d.genSliceCommon(d, assignto, typ, obj)
}
// generate code to encode/decode map
// len u32
// [len](key, value)
......@@ -1208,6 +1626,44 @@ func (d *decoderCommon) genMapCommon(xd CodeGenCustomize, assignto string, typ *
d.emit("}")
}
// fixmap|map16+YYYY|map32+ZZZZ
// [len]key/value
func (s *sizerM) genMapHead(path string, typ *types.Map, obj types.Object) {
s.size.AddExpr("msgpack.MapHeadSize(len(%s))", path)
}
func (s *sizerM) genMap(path string, typ *types.Map, obj types.Object) {
s.genMapCommon(s, path, typ, obj)
}
func (e *encoderM) genMapHead(path string, typ *types.Map, obj types.Object) {
e.emit("l := len(%s)", path)
e.emit("n := msgpack.PutMapHead(data[%v:], l)", e.n)
e.emit("data = data[%v+n:]", e.n)
e.n = 0
}
func (e *encoderM) genMap(path string, typ *types.Map, obj types.Object) {
e.genMapCommon(e, path, typ, obj)
}
func (d *decoderM) genMapHead(assignto string, typ *types.Map, obj types.Object) {
// -> msgp: flush queued overflow checks; put place for next overflow
// checks after msgp is done.
d.overflowCheck()
d.resetPos()
defer d.overflowCheck()
d.emit("l, tail, err := msgp.ReadMapHeaderBytes(data)")
d.emit("if err != nil {")
d.emit(" return 0, mdecodeErr(%q, err)", d.pathName(assignto))
d.emit("}")
d.emit("%v += uint64(len(data) - len(tail))", d.var_("nread"))
d.emit("data = tail")
}
func (d *decoderM) genMap(assignto string, typ *types.Map, obj types.Object) {
d.genMapCommon(d, assignto, typ, obj)
}
// emit code to size/encode/decode custom type
func (s *sizerN) genCustomN(path string) {
s.size.AddExpr("%s.neoEncodedLenN()", path)
......@@ -1246,6 +1702,35 @@ func (s *sizerN) genStructHead(path string, typ *types.Struct, userType types.
func (e *encoderN) genStructHead(path string, typ *types.Struct, userType types.Type) {}
func (d *decoderN) genStructHead(path string, typ *types.Struct, userType types.Type) {}
// M: array<nfields>
func (s *sizerM) genStructHead(path string, typ *types.Struct, userType types.Type) {
s.size.Add(1) // mfixarray|marray16|marray32
if typ.NumFields() > 0x0f {
panic("TODO: struct with > 15 elements")
}
}
func (e *encoderM) genStructHead(path string, typ *types.Struct, userType types.Type) {
if typ.NumFields() > 0x0f {
panic("TODO: struct with > 15 elements")
}
e.emit("data[%v] = byte(msgpack.FixArray_4 | %d)", e.n, typ.NumFields())
e.n += 1
}
func (d *decoderM) genStructHead(path string, typ *types.Struct, userType types.Type) {
if typ.NumFields() > 0x0f {
panic("TODO: struct with > 15 elements")
}
d.emit("if op, opOk := msgpack.Op(data[%v]), msgpack.FixArray_4 | %d ; op != opOk {", d.n, typ.NumFields())
d.emit("return 0, &mstructDecodeError{%q, op, opOk}", d.pathName(path))
d.emit("}")
d.n += 1
d.overflow.Add(1)
}
// top-level driver for emitting size/encode/decode code for a type
//
......@@ -1274,7 +1759,7 @@ func codegenType(path string, typ types.Type, obj types.Object, codegen CodeGene
break
}
_, ok := basicTypesN[u.Kind()] // ok to check N to see if supported for any encoding
_, ok := basicTypesN[u.Kind()] // ok to check N to see if supported for both N and M
if !ok {
log.Fatalf("%v: %v: basic type %v not supported", pos(obj), obj.Name(), u)
}
......
go/neo/proto/zproto-marshal.go
View file @ e8143557
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7