.

go/zodb/storage/fs1/fs1tools/dump.go

@@ -51,6 +51,9 @@ type Dumper interface {
 	//
 	// If dumper return io.EOF the whole dumping process finishes.
 	DumpTxn(buf *xfmt.Buffer, it *fs1.Iter) error
+
+	// DumpEndOK is called at the end of successfull dump.
+	DumpEndOK(buf *xfmt.Buffer)
 }
 
 // Dump dumps content of a FileStorage file @ path.
@@ -101,7 +104,7 @@ func Dump(w io.Writer, path string, dir fs1.IterDir, d Dumper) (err error) {
 		err = it.NextTxn(fs1.LoadAll)
 		if err != nil {
 			if err == io.EOF {
-				err = nil
+				break
 			}
 			return err
 		}
@@ -109,7 +112,7 @@ func Dump(w io.Writer, path string, dir fs1.IterDir, d Dumper) (err error) {
 		err = d.DumpTxn(buf, it)
 		if err != nil {
 			if err == io.EOF {
-				err = nil
+				break
 			}
 			return err
 		}
@@ -119,6 +122,9 @@ func Dump(w io.Writer, path string, dir fs1.IterDir, d Dumper) (err error) {
 			return err
 		}
 	}
+
+	d.DumpEndOK(buf)
+	return nil
 }
 
 // ----------------------------------------
@@ -198,6 +204,9 @@ func (d *DumperFsDump) DumpTxn(buf *xfmt.Buffer, it *fs1.Iter) error {
 	}
 }
 
+func (d *DumperFsDump) DumpEndOK(buf *xfmt.Buffer) {
+}
+
 // DumperFsDumpVerbose implements a very verbose dumper with output identical
 // to fsdump.Dumper in zodb/py originally written by Jeremy Hylton:
 //
@@ -281,6 +290,9 @@ func (d *DumperFsDumpVerbose) dumpData(buf *xfmt.Buffer, it *fs1.Iter) error {
 	return nil
 }
 
+func (d *DumperFsDumpVerbose) DumpEndOK(buf *xfmt.Buffer) {
+}
+
 const dumpSummary = "dump database transactions"
 
 func dumpUsage(w io.Writer) {
@@ -383,6 +395,9 @@ func (d *DumperFsTail) DumpTxn(buf *xfmt.Buffer, it *fs1.Iter) error {
 	return nil
 }
 
+func (d *DumperFsTail) DumpEndOK(buf *xfmt.Buffer) {
+}
+
 const tailSummary = "dump last few transactions of a database"
 const ntxnDefault = 10
 

go/zodb/storage/fs1/fs1tools/main.go

-// Copyright (C) 2017  Nexedi SA and Contributors.
+// Copyright (C) 2017-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -30,7 +30,7 @@ var commands = prog.CommandRegistry{
 	{"reindex", reindexSummary, reindexUsage, reindexMain},
 	{"verify-index", verifyIdxSummary, verifyIdxUsage, verifyIdxMain},
 	// recover		(fsrecover.py)
-	// verify		(fstest.py)
+	{"verify",  verifySummary,  verifyUsage,  verifyMain},
 
 	// XXX repozo ?
 }

go/zodb/storage/fs1/fs1tools/verify.go

+// Copyright (C) 2021  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package fs1tools
+// verify subcommand
+//
+// verification output mimics fstest from ZODB/py as originally written by Jeremy Hylton:
+// https://github.com/zopefoundation/ZODB/blob/5.6.0-35-g1fb097b41/src/ZODB/scripts/fstest.py
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+
+	"lab.nexedi.com/kirr/neo/go/zodb/storage/fs1"
+
+	"lab.nexedi.com/kirr/go123/prog"
+	"lab.nexedi.com/kirr/go123/xflag"
+	"lab.nexedi.com/kirr/go123/xfmt"
+)
+
+// Verify verifies content of a FileStorage file @ path.
+//
+// Only data part of the data base is verified (the *.fs file). Use
+// VerifyIndexFor to verify the index part (*.fs.index).
+func Verify(w io.Writer, path string, verbose int) (err error) {
+	// just iterate through the file and emit progress.
+	// the FileStorage driver implements all consistency checks by itself.
+	v := &Verifier{verbose: verbose}
+	return Dump(w, path, fs1.IterForward, v)
+}
+
+// XXX
+type Verifier struct {
+	ntxn    int // current transaction record #
+	verbose int // >=1 (print txn)   >=2 (print objects)
+
+	// for loading data
+	dhLoading fs1.DataHeader
+}
+
+func (v *Verifier) DumperName() string {
+	return "fsverify"
+}
+
+func (v *Verifier) DumpFileHeader(buf *xfmt.Buffer, fh *fs1.FileHeader) error {
+	return nil
+}
+
+func (v *Verifier) DumpTxn(buf *xfmt.Buffer, it *fs1.Iter) error {
+	txnh := &it.Txnh
+
+	for i := 0; ; i++ {
+		err := it.NextData()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return err
+		}
+
+		dh := &it.Datah
+
+		// load data
+		v.dhLoading = *dh
+		dbuf, err := v.dhLoading.LoadData(it.R)
+		if err != nil {
+			return err
+		}
+
+		if v.verbose >= 2 {
+			buf .S(fmt.Sprintf("%10d: object oid %s #%d\n", dh.Pos, dh.Oid, i))
+		}
+
+		dbuf.Release()
+	}
+
+	if v.verbose >= 1 {
+		buf .S(fmt.Sprintf("%10d: transaction tid %s #%d \n", txnh.Pos, txnh.Tid, v.ntxn))
+	}
+
+	v.ntxn++
+	return nil
+}
+
+func (v *Verifier) DumpEndOK(buf *xfmt.Buffer) {
+	if v.verbose >= 1 {
+		buf .S("no errors detected\n")
+	}
+}
+
+
+// ----------------------------------------
+
+const verifySummary = "verify database content"
+
+func verifyUsage(w io.Writer) {
+	fmt.Fprintf(w,
+`Usage: fs1 verify [options] <storage>
+Dump transactions from a FileStorage		XXX
+
+<storage> is a path to FileStorage
+
+  options:
+
+	-h --help       this help text.
+	-v		increase verbosity.
+`)
+}
+
+func verifyMain(argv []string) {
+	verbose := 0
+	flags := flag.FlagSet{Usage: func() { verifyUsage(os.Stderr) }}
+	flags.Init("", flag.ExitOnError)
+	flags.Var((*xflag.Count)(&verbose), "v", "verbosity level")
+	flags.Parse(argv[1:])
+
+	argv = flags.Args()
+	if len(argv) < 1 {
+		flags.Usage()
+		prog.Exit(2)
+	}
+	storPath := argv[0]
+
+	err := Verify(os.Stdout, storPath, verbose)
+	if err != nil {
+		prog.Fatal(err)
+	}
+}