.

go/zodb/storage/fs1/filestorage_test.go

@@ -284,25 +284,6 @@ func TestIterate(t *testing.T) {
 	testIterate(t, fs, 0, zodb.TidMax, _1fs_dbEntryv[:])
 }
 
-func TestComputeIndex(t *testing.T) {
-	fs := xfsopen(t, "testdata/1.fs")	// TODO open ro
-	defer exc.XRun(fs.Close)
-
-	index, err := fs.computeIndex(context.Background())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	indexOk, err := LoadIndexFile("testdata/1.fs.index")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if !index.Equal(indexOk) {
-		t.Fatal("computed index differ from expected")
-	}
-}
-
 func BenchmarkIterate(b *testing.B) {
 	fs := xfsopen(b, "testdata/1.fs")	// TODO open ro
 	defer exc.XRun(fs.Close)

go/zodb/storage/fs1/index.go

@@ -55,7 +55,7 @@ type Index struct {
 
 // IndexNew creates new empty index
 func IndexNew() *Index {
-	return &Index{Tree: fsb.TreeNew()}
+	return &Index{TopPos: txnValidFrom, Tree: fsb.TreeNew()}
 }
 
 // NOTE Get/Set/... are taken as-is from fsb.Tree
@@ -413,7 +413,7 @@ func treeEqual(a, b *fsb.Tree) bool {
 //
 // On success returned error is nil and index.TopPos is set to either:
 // - topPos (if it is != -1), or
-// - r's position at which read got EOF (if topPos=-1)
+// - r's position at which read got EOF (if topPos=-1).
 func (index *Index) Update(ctx context.Context, r io.ReaderAt, topPos int64) (err error) {
 	defer xerr.Contextf(&err, "%s: reindex %v..%v", xio.Name(r), index.TopPos, topPos)
 
@@ -488,16 +488,37 @@ func (index *Index) Update(ctx context.Context, r io.ReaderAt, topPos int64) (er
 }
 
 // BuildIndex builds new in-memory index for data in r
-func BuildIndex(ctx context.Context, r io.ReaderAt) (index *Index, err error) {
-	index = IndexNew()
-	index.TopPos = txnValidFrom
+//
+// non-nil valid and consistent index is always returned - even in case of error
+// the index will describe data till top-position of highest transaction that
+// could be read without error.
+//
+// In such cases the index building could be retried to be finished with
+// index.Update().
+func BuildIndex(ctx context.Context, r io.ReaderAt) (*Index, error) {
+	index := IndexNew()
+	err := index.Update(ctx, r, -1)
+	return index, err
+}
 
-	err = index.Update(ctx, r, -1)
+// BuildIndexForFile builds new in-memory index for data in file @ path
+//
+// See BuildIndex for semantic description.
+func BuildIndexForFile(ctx context.Context, path string) (index *Index, err error) {
+	f, err := os.Open(path)
 	if err != nil {
-		return nil, err
+		return IndexNew(), err
 	}
 
-	return index, nil
+	defer func() {
+		err2 := f.Close()
+		err = xerr.First(err, err2)
+	}()
+
+	// use IO optimized for sequential access when building index
+	fSeq := xbufio.NewSeqReaderAt(f)
+
+	return BuildIndex(ctx, fSeq)
 }
 
 // VerifyNTxn checks index correctness against several transactions of FileStorage data in r.

go/zodb/storage/fs1/index_test.go

@@ -22,6 +22,7 @@ package fs1
 //go:generate ./py/gen-testdata
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"log"
@@ -182,6 +183,13 @@ func TestIndexSaveLoad(t *testing.T) {
 }
 
 
+var _1fs_index = func () *Index {
+	idx := IndexNew()
+	idx.TopPos = _1fs_indexTopPos
+	setIndex(idx, _1fs_indexEntryv[:])
+	return idx
+}()
+
 // test that we can correctly load index data as saved by zodb/py
 func TestIndexLoadFromPy(t *testing.T) {
 	fsiPy, err := LoadIndexFile("testdata/1.fs.index")
@@ -189,11 +197,7 @@ func TestIndexLoadFromPy(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fsiExpect := IndexNew()
-	fsiExpect.TopPos = _1fs_indexTopPos
-	setIndex(fsiExpect, _1fs_indexEntryv[:])
-
-	checkIndexEqual(t, "index load", fsiPy, fsiExpect)
+	checkIndexEqual(t, "index load", fsiPy, _1fs_index)
 }
 
 // test zodb/py can read index data as saved by us
@@ -201,11 +205,7 @@ func TestIndexSaveToPy(t *testing.T) {
 	needZODBPy(t)
 	workdir := xworkdir(t)
 
-	fsi := IndexNew()
-	fsi.TopPos = _1fs_indexTopPos
-	setIndex(fsi, _1fs_indexEntryv[:])
-
-	err := fsi.SaveFile(workdir + "/1.fs.index")
+	err := _1fs_index.SaveFile(workdir + "/1.fs.index")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -220,6 +220,17 @@ func TestIndexSaveToPy(t *testing.T) {
 	}
 }
 
+func TestIndexBuild(t *testing.T) {
+	index, err := BuildIndexForFile(context.Background(), "testdata/1.fs")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !index.Equal(_1fs_index) {
+		t.Fatal("computed index differ from expected")
+	}
+}
+
 
 func BenchmarkIndexLoad(b *testing.B) {
 	// FIXME small testdata/1.fs is not representative for benchmarks