// Copyright (C) 2017-2021  Nexedi SA and Contributors.
//                          Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.

package neo
// test interaction between nodes

import (
	"bytes"
	"context"
	"crypto/sha1"
	"io"
	"reflect"
	"testing"

	"github.com/kylelemons/godebug/pretty"

	"lab.nexedi.com/kirr/neo/go/neo/proto"
	"lab.nexedi.com/kirr/neo/go/zodb"
	"lab.nexedi.com/kirr/neo/go/internal/xzodb"

	"lab.nexedi.com/kirr/go123/exc"
	"lab.nexedi.com/kirr/go123/tracing/tracetest"
	"lab.nexedi.com/kirr/go123/xsync"
)

// Verify basic M+S recovery.
func TestClusterRecoveryBasic(t *testing.T) {
	Verify(t, func(t0 *tEnv) {
		zback := xfs1back("../zodb/storage/fs1/testdata/1.fs")
		t := t0.NewCluster_MS("abc1", zback)
		defer t.Stop()
	})
}

// M drives cluster with 1 S & C through recovery -> verification -> service -> shutdown.
func TestMasterStorage(t *testing.T) {
	Verify(t, _TestMasterStorage)
}
func _TestMasterStorage(t0 *tEnv) {
	X := exc.Raiseif
	bg := context.Background()

	zback := xfs1back("../zodb/storage/fs1/testdata/1.fs")
	zstor := xfs1stor("../zodb/storage/fs1/testdata/1.fs")
	defer func() {
		err := zstor.Close();  X(err)
	}()

	lastTid, err := zstor.Sync(bg);  X(err)

	// create the cluster
	t := t0.NewCluster_MS("abc1", zback)
	defer t.Stop()
	M := t.Master("m")
	C := t.NewClient("c", "m:1")

	// TODO create C; C tries connect to master - rejected ("not yet operational")
	// TODO test ID rejects (nid already registered, ...)

	// M <- start cmd
	wg := xsync.NewWorkGroup(bg)
	gox(wg, func(ctx context.Context) {
		err := M.Start();  X(err)
	})

	// trace

	t.Expect("m",   δnode("m", "s:1", proto.STORAGE, 1, proto.RUNNING, 0.01))
	t.Expect("m-s", conntx("m:2", "s:2", 6, &proto.NotifyNodeInformation{
		IdTime:		proto.IdTimeNone,	// XXX ?
		NodeList:	[]proto.NodeInfo{
			nodei("s:1", proto.STORAGE, 1, proto.RUNNING, 0.01),
		},
	}))
	t.Expect("s",   δnode("s", "s:1", proto.STORAGE, 1, proto.RUNNING, 0.01))
	xwait(wg)

	// XXX M.partTab <- S1

	// M starts verification
	t.Expect("m", clusterState("m", proto.ClusterVerifying))

	t.Expect("m-s", conntx("m:2", "s:2", 8, &proto.SendPartitionTable{
		PTid:		1,
		NumReplicas:    0,
		RowList:	[]proto.RowInfo{
			{[]proto.CellInfo{{proto.NID(proto.STORAGE, 1), proto.UP_TO_DATE}}},
		},
	}))

	t.Expect("m-s", conntx("m:2", "s:2", 10, &proto.LockedTransactions{}))
	t.Expect("m-s", conntx("s:2", "m:2", 10, &proto.AnswerLockedTransactions{
		TidDict: nil,	// map[zodb.Tid]zodb.Tid{},
	}))

	lastOid, err := zstor.LastOid(bg);  X(err)
	t.Expect("m-s", conntx("m:2", "s:2", 12, &proto.LastIDs{}))
	t.Expect("m-s", conntx("s:2", "m:2", 12, &proto.AnswerLastIDs{
		LastOid: lastOid,
		LastTid: lastTid,
	}))

	// XXX M -> S ClusterInformation(VERIFICATION) ?

	// TODO there is actually txn to finish
	// TODO S leave at verify
	// TODO S join at verify
	// TODO M.Stop() while verify

	// verification ok; M start service
	t.Expect("m", clusterState("m", proto.ClusterRunning))
	// TODO ^^^ should be sent to S

	t.Expect("m-s", conntx("m:2", "s:2", 14, &proto.StartOperation{Backup: false}))
	t.Expect("m-s", conntx("s:2", "m:2", 14, &proto.NotifyReady{}))

	// TODO S leave while service
	// TODO S join while service
	// TODO M.Stop while service

	// ----------------------------------------

	// trace of client start

	// C connects M
	t.Expect("c-m", netdial("c", "m:1"))
	t.Expect("c-m", netconnect("c:1", "m:3",  "m:1"))
	t.Expect("c-m", conntx("c:1", "m:3", 1, &proto.RequestIdentification{
		NodeType:	proto.CLIENT,
		NID:		proto.NID(proto.CLIENT, 0),
		Address:	xnaddr(""),
		ClusterName:	"abc1",
		IdTime:		proto.IdTimeNone,
		DevPath:        nil,
		NewNID:         nil,
	}))

	t.Expect("m", δnode("m", "", proto.CLIENT, 1, proto.RUNNING, 0.02))

	t.Expect("c-m", conntx("m:3", "c:1", 1, &proto.AcceptIdentification{
		NodeType:	proto.MASTER,
		MyNID:		proto.NID(proto.MASTER, 1),
		YourNID:	proto.NID(proto.CLIENT, 1),
	}))

	// C <- M NotifyNodeInformation M1,S1,C1
	t.Expect("m-c", conntx("m:3", "c:1", 0, &proto.NotifyNodeInformation{
		IdTime:		proto.IdTimeNone,	// XXX ?
		NodeList:	[]proto.NodeInfo{
			nodei("m:1", proto.MASTER,  1, proto.RUNNING, proto.IdTimeNone),
			nodei("s:1", proto.STORAGE, 1, proto.RUNNING, 0.01),
			nodei("",    proto.CLIENT,  1, proto.RUNNING, 0.02),
		},
	}))

	t.Expect("m-c", conntx("m:3", "c:1", 2, &proto.SendPartitionTable{
		PTid:		1,
		NumReplicas:	0,
		RowList:	[]proto.RowInfo{
			{[]proto.CellInfo{{proto.NID(proto.STORAGE, 1), proto.UP_TO_DATE}}},
		},
	}))

	t.Expect("c", δnode("c", "m:1", proto.MASTER,  1, proto.RUNNING, proto.IdTimeNone))
	t.Expect("c", δnode("c", "s:1", proto.STORAGE, 1, proto.RUNNING, 0.01))
	t.Expect("c", δnode("c", "",    proto.CLIENT,  1, proto.RUNNING, 0.02))

	// S <- M NotifyNodeInformation C1
	t.Expect("m-s", conntx("m:2", "s:2", 16, &proto.NotifyNodeInformation{
		IdTime:		proto.IdTimeNone,	// XXX ?
		NodeList:	[]proto.NodeInfo{
			nodei("",    proto.CLIENT,  1, proto.RUNNING, 0.02),
		},
	}))
	t.Expect("s", δnode("s", "",    proto.CLIENT,  1, proto.RUNNING, 0.02))

	// C asks M last_tid
	t.Expect("c-m", conntx("c:1", "m:3", 3, &proto.LastTransaction{}))
	t.Expect("c-m", conntx("m:3", "c:1", 3, &proto.AnswerLastTransaction{lastTid}))

	// ----------------------------------------

	// C asks M about last tid
	wg = xsync.NewWorkGroup(bg)
	gox(wg, func(ctx context.Context) {
		cLastTid, err := C.Sync(ctx);  X(err)

		if cLastTid != lastTid {
			exc.Raisef("C.Sync -> %v  ; want %v", cLastTid, lastTid)
		}
	})

	t.Expect("c-m", conntx("c:1", "m:3", 5, &proto.LastTransaction{}))
	t.Expect("c-m", conntx("m:3", "c:1", 5, &proto.AnswerLastTransaction{lastTid}))
	xwait(wg)


	// ----------------------------------------

	// C starts loading first object ...
	wg = xsync.NewWorkGroup(bg)
	xid1 := zodb.Xid{Oid: 1, At: zodb.TidMax}
	buf1, serial1, err := zstor.Load(bg, xid1);  X(err)
	gox(wg, func(ctx context.Context) {
		buf, serial, err := C.Load(ctx, xid1);  X(err)

		if !(bytes.Equal(buf.Data, buf1.Data) && serial==serial1) {
			exc.Raisef("C.Load(%v) ->\ndata:\n%s\nserial:\n%s\n", xid1,
				pretty.Compare(buf1.Data, buf.Data), pretty.Compare(serial1, serial))
		}
	})

	// trace

	// ... -> connects to S
	t.Expect("c-s", netdial("c", "s:1"))
	t.Expect("c-s", netconnect("c:2", "s:3",  "s:1"))
	t.Expect("c-s", conntx("c:2", "s:3", 1, &proto.RequestIdentification{
		NodeType:	proto.CLIENT,
		NID:		proto.NID(proto.CLIENT, 1),
		Address:	xnaddr(""),
		ClusterName:	"abc1",
		IdTime:		0.02,
		DevPath:        nil,
		NewNID:         nil,
	}))

	t.Expect("c-s", conntx("s:3", "c:2", 1, &proto.AcceptIdentification{
		NodeType:	proto.STORAGE,
		MyNID:		proto.NID(proto.STORAGE, 1),
		YourNID:	proto.NID(proto.CLIENT, 1),
	}))

	// ... -> GetObject(xid1)
	t.Expect("c-s", conntx("c:2", "s:3", 3, &proto.GetObject{
		Oid:	xid1.Oid,
		Before:	xzodb.At2Before(xid1.At),
		At:     proto.INVALID_TID,
	}))
	t.Expect("c-s", conntx("s:3", "c:2", 3, &proto.AnswerObject{
		Oid:		xid1.Oid,
		Serial:		serial1,
		NextSerial:	proto.INVALID_TID,
		Compression:	false,
		Data:		buf1,
		DataSerial:	0,		// XXX
		Checksum:	sha1.Sum(buf1.Data),
	}))


	xwait(wg)

	// ----------------------------------------

	// verify NextSerial is properly returned in AnswerObject via trace-loading prev. revision of obj1
	// (XXX we currently need NextSerial for neo/py client cache)
	wg = xsync.NewWorkGroup(bg)
	xid1prev := zodb.Xid{Oid: 1, At: serial1 - 1}
	buf1prev, serial1prev, err := zstor.Load(bg, xid1prev);  X(err)
	gox(wg, func(ctx context.Context) {
		buf, serial, err := C.Load(ctx, xid1prev);  X(err)

		if !(bytes.Equal(buf.Data, buf1prev.Data) && serial==serial1prev) {
			exc.Raisef("C.Load(%v) ->\ndata:\n%s\nserial:\n%s\n", xid1prev,
				pretty.Compare(buf1prev.Data, buf.Data), pretty.Compare(serial1prev, serial))
		}
	})

	// ... -> GetObject(xid1prev)
	t.Expect("c-s", conntx("c:2", "s:3", 5, &proto.GetObject{
		Oid:	xid1prev.Oid,
		Before:	serial1,
		At:     proto.INVALID_TID,
	}))
	t.Expect("c-s", conntx("s:3", "c:2", 5, &proto.AnswerObject{
		Oid:		xid1prev.Oid,
		Serial:		serial1prev,
		NextSerial:	serial1,
		Compression:	false,
		Data:		buf1prev,
		DataSerial:	0,		// XXX
		Checksum:	sha1.Sum(buf1prev.Data),
	}))

	xwait(wg)


	// C loads every other {<,=}serial:oid - established link is reused
	ziter := zstor.Iterate(bg, 0, zodb.TidMax)

	// XXX hack: disable tracing early so that C.Load() calls do not deadlock	NOTE
	// TODO refactor cluster creation into func
	// TODO move client all loading tests into separate test where tracing will be off
	t.TraceOff()

	for {
		_, dataIter, err := ziter.NextTxn(bg)
		if err == io.EOF {
			break
		}
		if err != nil {
			t.Fatalf("ziter.NextTxn: %v", err)
		}

		for {
			datai, err := dataIter.NextData(bg)
			if err == io.EOF {
				break
			}
			if err != nil {
				t.Fatalf("ziter.NextData: %v", err)
			}

			// TODO also test GetObject(tid=ø, serial=...) which originate from loadSerial on py side
			xid := zodb.Xid{Oid: datai.Oid, At: datai.Tid}

			buf, serial, err := C.Load(bg, xid)
			if datai.Data != nil {
				if !(bytes.Equal(buf.Data, datai.Data) && serial == datai.Tid && err == nil) {
					t.Fatalf("load: %v:\nhave: %v %v %q\nwant: %v nil %q",
						xid, serial, err, buf.Data, datai.Tid, datai.Data)
				}
			} else {
				// deleted
				errWant := &zodb.OpError{
					URL:  C.URL(),
					Op:   "load",
					Args: xid,
					Err:  &zodb.NoDataError{Oid: xid.Oid, DeletedAt: datai.Tid},
				}
				if !(buf == nil && serial == 0 && reflect.DeepEqual(err, errWant)) {
					t.Fatalf("load: %v ->\nbuf:\n%s\nserial:\n%s\nerr:\n%s\n", xid,
						pretty.Compare(nil, buf),
						pretty.Compare(0, serial),
						pretty.Compare(errWant, err))
				}
			}
		}
	}


	// TODO S.Stop() or Scancel()
	// expect:
	// M.nodeTab -= S
	// M.clusterState	<- RECOVERY
	// ...

	// TODO Scancel -> S down - test how M behaves

	// TODO test M.recovery starting back from verification/service
	// (M needs to resend to all storages recovery messages just from start)
}


func benchmarkGetObject(b *testing.B, opt tClusterOptions, benchit func(xcload1 func())) {
	X := exc.Raiseif

	tracetest.Run(b, func(t0 *tracetest.T) {
		tenv := &tEnv{t0, opt}
		// create test cluster
		zback := xfs1back("../zodb/storage/fs1/testdata/1.fs")
		t := tenv.NewCluster_MS("abc1", zback)
		defer t.Stop()
		M := t.Master("m")

		t.TraceOff()
		err := M.Start();  X(err)

		C := t.NewClient("c", "m:1")

		xid1 := zodb.Xid{Oid: 1, At: zodb.TidMax}

		obj1, err := zback.Load(t.runCtx, xid1) // XXX -> t.Ctx ?
		if err != nil {
			t.Fatal(err)
		}
		buf1, serial1 := obj1.Data, obj1.Serial

		// C.Load(xid1)
		xcload1 := func() {
			cbuf1, cserial1, err := C.Load(t.runCtx, xid1) // XXX -> t.Ctx ?
			if err != nil {
				t.Fatal(err)
			}

			if !(bytes.Equal(cbuf1.Data, buf1.Data) && cserial1 == serial1) {
				t.Fatalf("C.Load first -> %q %v  ; want %q %v", cbuf1.Data, cserial1, buf1.Data, serial1)
			}

			cbuf1.Release()
		}

		// do first C.Load - this also implicitly waits for M & S to come up
		// and C to connect to M and S.
		xcload1()

		// now start the benchmark
		b.ResetTimer()

		benchit(xcload1)
	})
}

func benchmarkGetObjectSerial(b *testing.B, opt tClusterOptions) {
	benchmarkGetObject(b, opt, func(xcload1 func()) {
		for i := 0; i < b.N; i++ {
			xcload1()
		}
	})
}

func benchmarkGetObjectParallel(b *testing.B, opt tClusterOptions) {
	benchmarkGetObject(b, opt, func(xcload1 func()) {
		//println()
		b.SetParallelism(32) // we are io-bound
		b.RunParallel(func(pb *testing.PB) {
			//print(".")
			for pb.Next() {
				xcload1()
			}
		})
	})
}

func BenchmarkGetObjectNetPipe(b *testing.B) {
	benchmarkGetObjectSerial(b, tClusterOptions{Network: "pipenet"})
}

func BenchmarkGetObjectNetPipeParallel(b *testing.B) {
	benchmarkGetObjectParallel(b, tClusterOptions{Network: "pipenet"})
}

func BenchmarkGetObjectTCPlo(b *testing.B) {
	benchmarkGetObjectSerial(b, tClusterOptions{Network: "lonet"})
}

func BenchmarkGetObjectTCPloParallel(b *testing.B) {
	benchmarkGetObjectParallel(b, tClusterOptions{Network: "lonet"})
}
