// Copyright (C) 2017-2021  Nexedi SA and Contributors.
//                          Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.

// Package xtesting provides infrastructure for ZODB testing.
package xtesting

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"net/url"
	"os"
	"os/exec"
	"reflect"
	"strings"
	"sync"
	"testing"
	"time"

	"lab.nexedi.com/kirr/go123/xerr"
	"lab.nexedi.com/kirr/go123/xstrings"
	"lab.nexedi.com/kirr/neo/go/zodb"
)

var (
	pyMu   sync.Mutex
	pyHave = map[string]bool{} // {} pymod -> y/n  ; ".python" indicates python presence
)

// NeedPy skips current test if python and specified modules are not available.
//
// For example
//
//	xtesting.NeedPy(t)
//
// would check for python presence, and
//
//	xtesting.NeedPy(t, "ZODB", "golang.strconv")
//
// would check if all python and ZODB and golang.strconv python modules are available.
func NeedPy(t testing.TB, modules ...string) {
	pyMu.Lock()
	defer pyMu.Unlock()

	// verify if python is present
	havePy, know := pyHave[".python"]
	if !know {
		cmd := exec.Command("python", "-c", "0")
		err := cmd.Run()
		havePy = (err == nil)
		pyHave[".python"] = havePy
	}

	if !havePy {
		t.Skipf("skipping: python is not availble")
	}

	var donthave []string
	for _, pymod := range modules {
		have, know := pyHave[pymod]
		if !know {
			cmd := exec.Command("python", "-c", "import "+pymod)
			err := cmd.Run()
			have = (err == nil)
			pyHave[pymod] = have
		}
		if !have {
			donthave = append(donthave, pymod)
		}
	}

	if len(donthave) != 0 {
		t.Skipf("skipping: the following python modules are not available: %s", donthave)
	}

	// we verified everything - now it is ok not to skip.
	return
}

// ZRawObject represents raw ZODB object state.
type ZRawObject struct { // keep in sync with zodb(test).ZRawObject
	Oid  zodb.Oid
	Data []byte // raw serialized zodb data
}

// ZPyCommitRaw commits new transaction into database @ zurl with raw data specified by objv.
//
// Nil data means "delete object".
// The commit is performed via zodbtools/py.
func ZPyCommitRaw(zurl string, at zodb.Tid, objv ...ZRawObject) (_ zodb.Tid, err error) {
	defer xerr.Contextf(&err, "%s: zpycommit @%s", zurl, at)

	// prepare text input for `zodb commit`
	zin := &bytes.Buffer{}
	fmt.Fprintf(zin, "user %q\n", "author")
	fmt.Fprintf(zin, "description %q\n", fmt.Sprintf("test commit; at=%s", at))
	fmt.Fprintf(zin, "extension %q\n", "")
	for _, obj := range objv {
		// !data -> delete
		if obj.Data == nil {
			fmt.Fprintf(zin, "obj %s delete\n", obj.Oid)
		} else {
			fmt.Fprintf(zin, "obj %s %d null:00\n", obj.Oid, len(obj.Data))
			zin.Write(obj.Data)
			zin.WriteString("\n")
		}
	}
	zin.WriteString("\n")

	// run py `zodb commit`
	cmd:= exec.Command("python", "-m", "zodbtools.zodb", "commit", zurl, at.String())
	cmd.Stdin  = zin
	cmd.Stderr = os.Stderr
	out, err := cmd.Output()
	if err != nil {
		return zodb.InvalidTid, err
	}

	out = bytes.TrimSuffix(out, []byte("\n"))
	tid, err := zodb.ParseTid(string(out))
	if err != nil {
		return zodb.InvalidTid, fmt.Errorf("committed, but invalid output: %s", err)
	}

	return tid, nil
}

// XXX + ZPyCommitSrv ?

// ZPyRestore restores transactions specified by zin in zodbdump format.
//
// The restore is performed via zodbtools/py.
func ZPyRestore(zurl string, zin string) (tidv []zodb.Tid, err error) {
	defer xerr.Contextf(&err, "%s: zpyrestore", zurl)

	// run py `zodb restore`
	cmd:= exec.Command("python", "-m", "zodbtools.zodb", "restore", zurl)
	cmd.Stdin  = strings.NewReader(zin)
	cmd.Stderr = os.Stderr
	out, err := cmd.Output()
	if err != nil {
		return nil, err
	}

	for _, line := range xstrings.SplitLines(string(out), "\n") {
		tid, err := zodb.ParseTid(line)
		if err != nil {
			return nil, fmt.Errorf("restored, but invalid output: %s", err)
		}

		tidv = append(tidv, tid)
	}

	return tidv, nil
}


// ---- tests for storage drivers ----

// DrvTestEmptyDB verifies that zdrv works correctly on empty database.
func DrvTestEmptyDB(t *testing.T, zdrv zodb.IStorageDriver) {
	X := FatalIf(t)
	ctx := context.Background()
	head, err := zdrv.Sync(ctx); X(err)
	headOk := zodb.Tid(0)
	if head != headOk {
		t.Errorf("head=%s  ; expected %s", head, headOk)
	}
}

// state of an object in the database for some particular revision
type objState struct {
	tid  zodb.Tid
	data []byte // nil if obj was deleted
}

// Txn represents one transaction.
type Txn struct {
	Header *zodb.TxnInfo
	Data   []*zodb.DataInfo
}

// LoadDBHistory loads whole content of a ZODB database.
//
// it returns full history of all transactions with committed data.
func LoadDBHistory(zurl string) (_ []Txn, err error) {
	xerr.Contextf(&err, "loadDBHistory %s", zurl)
	ctx := context.Background()
	zstor, err := zodb.Open(ctx, zurl, &zodb.OpenOptions{ReadOnly: true})
	if err != nil {
		return nil, err
	}
	defer func() {
		err2 := zstor.Close()
		if err == nil {
			err = err2
		}
	}()

	txnv := []Txn{}
	it := zstor.Iterate(ctx, 0, zodb.TidMax)
	for {
		txni, dataIt, err := it.NextTxn(ctx)
		if err != nil {
			if err == io.EOF {
				break
			}
			return nil, err
		}
		txn := Txn{Header: &zodb.TxnInfo{
			// clone (txni stays valid only until next iteration)
			Tid:         txni.Tid,
			Status:      txni.Status,
			User:        bcopy(txni.User),
			Description: bcopy(txni.Description),
			Extension:   bcopy(txni.Extension),
		}}

		for {
			datai, err := dataIt.NextData(ctx)
			if err != nil {
				if err == io.EOF {
					break
				}
				return nil, err
			}

			txn.Data = append(txn.Data, &zodb.DataInfo{
				// clone (datai stays valid only until next iteration)
				Oid:         datai.Oid,
				Tid:         datai.Tid,
				Data:        bcopy(datai.Data),
				DataTidHint: datai.DataTidHint,
			})
		}

		txnv = append(txnv, txn)
	}

	return txnv, nil
}

// checkLoad verifies that zdrv.Load(xid) returns expected result.
func checkLoad(t *testing.T, zdrv zodb.IStorageDriver, xid zodb.Xid, expect objState, bugs map[string]bool) {
	t.Helper()
	buf, tid, err := zdrv.Load(context.Background(), xid)

	// deleted obj - it should load with "no data"
	if expect.data == nil {
		var eNoData error = &zodb.NoDataError{Oid: xid.Oid, DeletedAt: expect.tid}
		if bugs["load:noserial-after-deleted"] {
			// https://github.com/zopefoundation/ZODB/issues/318
			eNoData = &zodb.NoObjectError{Oid: xid.Oid}
		}
		errOk := &zodb.OpError{
			URL:  zdrv.URL(),
			Op:   "load",
			Args: xid,
			Err:  eNoData,
		}
		if !reflect.DeepEqual(err, errOk) {
			t.Errorf("load %v: returned err unexpected:\nhave: %v\nwant: %v", xid, err, errOk)
		}

		if tid != 0 {
			t.Errorf("load %v: returned tid unexpected: %v  ; want: %v", xid, tid, expect.tid)
		}

		if buf != nil {
			t.Errorf("load %v: returned buf != nil", xid)
		}

	// regular load
	} else {
		if err != nil {
			t.Errorf("load %v: returned err unexpected:\nhave: %v\nwant: nil", xid, err)
		}

		if tid != expect.tid {
			t.Errorf("load %v: returned tid unexpected: %v  ; want: %v", xid, tid, expect.tid)
		}

		switch {
		case buf == nil:
			t.Errorf("load %v: returned buf = nil", xid)

		case !reflect.DeepEqual(buf.Data, expect.data): // NOTE reflect to catch nil != ""
			t.Errorf("load %v: different data:\nhave: %q\nwant: %q", xid, buf.Data, expect.data)
		}
	}
}

// DrvTestLoad verifies that zdrv implements Load correctly.
//
// txnvOk is what data to expect to be in the database.
func DrvTestLoad(t *testing.T, zdrv zodb.IStorageDriver, txnvOk []Txn, bugv ...string) {
	bugs := argv2map(bugv)

	// current knowledge of what was "before" for an oid as we scan over
	// data base entries
	before := map[zodb.Oid]objState{}

	for _, txn := range txnvOk {
		for _, obj := range txn.Data {
			txh := txn.Header

			// XXX assert obj.Tid == txn.Tid

			// XXX check Load finds data at correct .Pos / etc ?

			// ~ loadSerial
			xid := zodb.Xid{txh.Tid, obj.Oid}
			checkLoad(t, zdrv, xid, objState{txh.Tid, obj.Data}, bugs)

			// ~ loadBefore
			xid = zodb.Xid{txh.Tid - 1, obj.Oid}
			expect, exists := before[obj.Oid]
			if exists {
				checkLoad(t, zdrv, xid, expect, bugs)
			} else {
				// verify load of not-yet-created object
				checkLoad(t, zdrv, xid, objState{0, nil}, bugs)
			}

			before[obj.Oid] = objState{txh.Tid, obj.Data}

		}
	}

	// load at ∞ with TidMax
	// XXX should we get "no such transaction" with at > head? - yes
	for oid, expect := range before {
		xid := zodb.Xid{zodb.TidMax, oid}
		checkLoad(t, zdrv, xid, expect, bugs)
	}

	// verify load of non-existing object
	var oid zodb.Oid
	for oid = 0;; oid++ {
		_, exists := before[oid]
		if !exists {
			break
		}
	}
	xid := zodb.Xid{zodb.TidMax, oid} // XXX TidMax -> head
	bugs["load:noserial-after-deleted"] = true // force NoObjectError
	checkLoad(t, zdrv, xid, objState{0, nil}, bugs)
}

// DrvTestWatch verifies that storage driver watcher can observe commits done from outside.
func DrvTestWatch(t *testing.T, zurl string, zdrvOpen zodb.DriverOpener, bugv ...string) {
	X := FatalIf(t)
	bugs := argv2map(bugv)

	if bugs["nocommit"] {
		t.Skip("skipping: server does not implement commit")
	}
	NeedPy(t, "zodbtools")

	u, err := url.Parse(zurl); X(err)

	// xcommit commits new transaction into zurl with raw data specified by objv.
	xcommit := func(at zodb.Tid, objv ...ZRawObject) zodb.Tid {
		t.Helper()
		tid, err := ZPyCommitRaw(zurl, at, objv...); X(err)
		return tid
	}

	// force database creation & open zurl at go side
	at := xcommit(0, ZRawObject{0, b("data0")})

	ctx := context.Background()
	watchq := make(chan zodb.Event)
	zdrv, at0, err := zdrvOpen(ctx, u, &zodb.DriverOptions{ReadOnly: true, Watchq: watchq}); X(err)
	if at0 != at {
		t.Fatalf("opened @ %s  ; want %s", at0, at)
	}

	checkHead := func(headOk zodb.Tid) {
		t.Helper()
		head, err := zdrv.Sync(ctx); X(err)
		if head != headOk {
			t.Fatalf("check head: got %s;  want %s", head, headOk)
		}
	}

	checkHead(at)

	checkLoad := func(at zodb.Tid, oid zodb.Oid, dataOk string, serialOk zodb.Tid) {
		t.Helper()
		xid := zodb.Xid{at, oid}
		buf, serial, err := zdrv.Load(ctx, xid); X(err)
		data := string(buf.XData())
		if !(data == dataOk && serial == serialOk) {
			t.Fatalf("check load %s:\nhave: %q %s\nwant: %q %s",
				xid, data, serial, dataOk, serialOk)
		}
	}

	// commit -> check watcher observes what we committed.
	//
	// XXX python `import pkg_resources` takes ~ 300ms.
	// https://github.com/pypa/setuptools/issues/510
	//
	// Since pkg_resources are used everywhere (e.g. in zodburi to find all
	// uri resolvers) this import slowness becomes the major part of time to
	// run py `zodb commit`.
	//
	// if one day it is either fixed, or worked around, we could ↑ 10 to 100.
	data0 := ""
	lastDataAt := zodb.Tid(0)
	for i := zodb.Oid(1); i <= 10; i++ {
		empty := (5 <= i && i <= 7) // this transactions are committed as empty

		objv := []ZRawObject{}
		datai := ""
		if !empty {
			data0 = fmt.Sprintf("data0.%d", i)
			datai = fmt.Sprintf("data%d", i)
			objv = append(objv, ZRawObject{0, b(data0)})
			objv = append(objv, ZRawObject{i, b(datai)})
		}
		at = xcommit(at, objv...)
		if !empty {
			lastDataAt = at
		}

		// TODO also test for watcher errors
		event := <-watchq

		var δ *zodb.EventCommit
		switch event := event.(type) {
		default:
			panic(fmt.Sprintf("unexpected event: %T", event))

		case *zodb.EventError:
			t.Fatal(event.Err)

		case *zodb.EventCommit:
			δ = event
		}

		objvWant := []zodb.Oid{}
		if !empty {
			objvWant = []zodb.Oid{0, i}
		}
		if !(δ.Tid == at && reflect.DeepEqual(δ.Changev, objvWant)) {
			t.Fatalf("watch:\nhave: %s %s\nwant: %s %s", δ.Tid, δ.Changev, at, objvWant)
		}

		checkHead(at)

		// make sure we can load what was committed.
		checkLoad(at, 0, data0, lastDataAt)
		if !empty {
			checkLoad(at, i, datai, at)
		}
	}

	// commit something more and wait a bit to raise chances the driver enqueues to watchq<- .
	_ = xcommit(at, ZRawObject{0, b("at the end")})
	time.Sleep(1*time.Second)

	// the driver must handle Close and cancel that watchq<-
	err = zdrv.Close(); X(err)

	e, ok := <-watchq
	if ok {
		t.Fatalf("watch after close -> %v;  want closed", e)
	}
}


// FatalIf(t) returns function f(err), which call t.Fatal if err != nil.
func FatalIf(t *testing.T) func(error) {
	return func(err error) {
		t.Helper()
		if err != nil {
			t.Fatal(err)
		}
	}
}

// argv2map converts ...string to map[string]bool.
// it is handy to use e.g. to convert ...bugv to bugs {}.
func argv2map(argv []string) map[string]bool {
	dict := map[string]bool{}
	for _, arg := range argv {
		dict[arg] = true
	}
	return dict
}


// b is syntactic sugar for byte literals.
//
// e.g.
//
//	b("hello")
func b(data string) []byte {
	return []byte(data)
}

// bcopy makes a clone of byte slice.
func bcopy(data []byte) []byte {
	if data == nil {
		return nil
	}
	d2 := make([]byte, len(data))
	copy(d2, data)
	return d2
}
