Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
N
neo
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
2
Merge Requests
2
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Jobs
Commits
Open sidebar
Kirill Smelkov
neo
Commits
fe751a15
Commit
fe751a15
authored
Dec 04, 2018
by
Kirill Smelkov
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
go/zodb/fs1: Make it render more well in godoc
- Put dot after the subject, - Indent lists, - ...
parent
4eed282c
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
37 additions
and
33 deletions
+37
-33
go/zodb/storage/fs1/filestorage.go
go/zodb/storage/fs1/filestorage.go
+5
-5
go/zodb/storage/fs1/format.go
go/zodb/storage/fs1/format.go
+17
-15
go/zodb/storage/fs1/index.go
go/zodb/storage/fs1/index.go
+15
-13
No files found.
go/zodb/storage/fs1/filestorage.go
View file @
fe751a15
// Copyright (C) 2017 Nexedi SA and Contributors.
// Copyright (C) 2017
-2018
Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
Kirill Smelkov <kirr@nexedi.com>
//
//
// This program is free software: you can Use, Study, Modify and Redistribute
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// it under the terms of the GNU General Public License version 3, or (at your
...
@@ -218,7 +218,7 @@ const (
...
@@ -218,7 +218,7 @@ const (
zIterPreloaded
// data for this iteration was already preloaded
zIterPreloaded
// data for this iteration was already preloaded
)
)
// NextTxn iterates to next/previous transaction record according to iteration direction
// NextTxn iterates to next/previous transaction record according to iteration direction
.
func
(
zi
*
zIter
)
NextTxn
(
_
context
.
Context
)
(
*
zodb
.
TxnInfo
,
zodb
.
IDataIterator
,
error
)
{
func
(
zi
*
zIter
)
NextTxn
(
_
context
.
Context
)
(
*
zodb
.
TxnInfo
,
zodb
.
IDataIterator
,
error
)
{
// TODO err -> OpError("iter", tidmin..tidmax)
// TODO err -> OpError("iter", tidmin..tidmax)
switch
{
switch
{
...
@@ -246,7 +246,7 @@ func (zi *zIter) NextTxn(_ context.Context) (*zodb.TxnInfo, zodb.IDataIterator,
...
@@ -246,7 +246,7 @@ func (zi *zIter) NextTxn(_ context.Context) (*zodb.TxnInfo, zodb.IDataIterator,
return
&
zi
.
iter
.
Txnh
.
TxnInfo
,
zi
,
nil
return
&
zi
.
iter
.
Txnh
.
TxnInfo
,
zi
,
nil
}
}
// NextData iterates to next data record and loads data content
// NextData iterates to next data record and loads data content
.
func
(
zi
*
zIter
)
NextData
(
_
context
.
Context
)
(
*
zodb
.
DataInfo
,
error
)
{
func
(
zi
*
zIter
)
NextData
(
_
context
.
Context
)
(
*
zodb
.
DataInfo
,
error
)
{
// TODO err -> OpError("iter", tidmin..tidmax)
// TODO err -> OpError("iter", tidmin..tidmax)
err
:=
zi
.
iter
.
NextData
()
err
:=
zi
.
iter
.
NextData
()
...
@@ -370,7 +370,7 @@ func (fs *FileStorage) findTxnRecord(r io.ReaderAt, tid zodb.Tid) (TxnHeader, er
...
@@ -370,7 +370,7 @@ func (fs *FileStorage) findTxnRecord(r io.ReaderAt, tid zodb.Tid) (TxnHeader, er
return
txnhFound
,
nil
return
txnhFound
,
nil
}
}
// Iterate creates zodb-level iterator for tidMin..tidMax range
// Iterate creates zodb-level iterator for tidMin..tidMax range
.
func
(
fs
*
FileStorage
)
Iterate
(
_
context
.
Context
,
tidMin
,
tidMax
zodb
.
Tid
)
zodb
.
ITxnIterator
{
func
(
fs
*
FileStorage
)
Iterate
(
_
context
.
Context
,
tidMin
,
tidMax
zodb
.
Tid
)
zodb
.
ITxnIterator
{
// when iterating use IO optimized for sequential access
// when iterating use IO optimized for sequential access
fsSeq
:=
seqReadAt
(
fs
.
file
)
fsSeq
:=
seqReadAt
(
fs
.
file
)
...
...
go/zodb/storage/fs1/format.go
View file @
fe751a15
// Copyright (C) 2017 Nexedi SA and Contributors.
// Copyright (C) 2017
-2018
Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
Kirill Smelkov <kirr@nexedi.com>
//
//
// This program is free software: you can Use, Study, Modify and Redistribute
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// it under the terms of the GNU General Public License version 3, or (at your
...
@@ -32,12 +32,12 @@ import (
...
@@ -32,12 +32,12 @@ import (
"lab.nexedi.com/kirr/go123/xbytes"
"lab.nexedi.com/kirr/go123/xbytes"
)
)
// FileHeader represents header of whole data file
// FileHeader represents header of whole data file
.
type
FileHeader
struct
{
type
FileHeader
struct
{
Magic
[
4
]
byte
Magic
[
4
]
byte
}
}
// TxnHeader represents header of a transaction record
// TxnHeader represents header of a transaction record
.
type
TxnHeader
struct
{
type
TxnHeader
struct
{
Pos
int64
// position of transaction start
Pos
int64
// position of transaction start
LenPrev
int64
// whole previous transaction record length (see EOF/error rules in Load)
LenPrev
int64
// whole previous transaction record length (see EOF/error rules in Load)
...
@@ -52,7 +52,7 @@ type TxnHeader struct {
...
@@ -52,7 +52,7 @@ type TxnHeader struct {
workMem
[]
byte
workMem
[]
byte
}
}
// DataHeader represents header of a data record
// DataHeader represents header of a data record
.
type
DataHeader
struct
{
type
DataHeader
struct
{
Pos
int64
// position of data record start
Pos
int64
// position of data record start
Oid
zodb
.
Oid
Oid
zodb
.
Oid
...
@@ -110,25 +110,26 @@ func (e *RecordError) Error() string {
...
@@ -110,25 +110,26 @@ func (e *RecordError) Error() string {
}
}
// err creates RecordError for transaction located at txnh.Pos in f
// err creates RecordError for transaction located at txnh.Pos in f
.
func
(
txnh
*
TxnHeader
)
err
(
f
interface
{},
subj
string
,
err
error
)
error
{
func
(
txnh
*
TxnHeader
)
err
(
f
interface
{},
subj
string
,
err
error
)
error
{
return
&
RecordError
{
ioname
(
f
),
"transaction record"
,
txnh
.
Pos
,
subj
,
err
}
return
&
RecordError
{
ioname
(
f
),
"transaction record"
,
txnh
.
Pos
,
subj
,
err
}
}
}
// err creates RecordError for data record located at dh.Pos in f
// err creates RecordError for data record located at dh.Pos in f
.
func
(
dh
*
DataHeader
)
err
(
f
interface
{},
subj
string
,
err
error
)
error
{
func
(
dh
*
DataHeader
)
err
(
f
interface
{},
subj
string
,
err
error
)
error
{
return
&
RecordError
{
ioname
(
f
),
"data record"
,
dh
.
Pos
,
subj
,
err
}
return
&
RecordError
{
ioname
(
f
),
"data record"
,
dh
.
Pos
,
subj
,
err
}
}
}
// ierr is an interface for something which can create errors.
// ierr is an interface for something which can create errors.
//
// it is used by TxnHeader and DataHeader to create appropriate errors with their context.
// it is used by TxnHeader and DataHeader to create appropriate errors with their context.
type
ierr
interface
{
type
ierr
interface
{
err
(
f
interface
{},
subj
string
,
err
error
)
error
err
(
f
interface
{},
subj
string
,
err
error
)
error
}
}
// errf is syntactic shortcut for err and fmt.Errorf
// errf is syntactic shortcut for err and fmt.Errorf
.
func
errf
(
f
interface
{},
e
ierr
,
subj
,
format
string
,
a
...
interface
{})
error
{
func
errf
(
f
interface
{},
e
ierr
,
subj
,
format
string
,
a
...
interface
{})
error
{
return
e
.
err
(
f
,
subj
,
fmt
.
Errorf
(
format
,
a
...
))
return
e
.
err
(
f
,
subj
,
fmt
.
Errorf
(
format
,
a
...
))
}
}
...
@@ -169,12 +170,12 @@ func (txnh *TxnHeader) HeaderLen() int64 {
...
@@ -169,12 +170,12 @@ func (txnh *TxnHeader) HeaderLen() int64 {
return
TxnHeaderFixSize
+
int64
(
len
(
txnh
.
workMem
))
return
TxnHeaderFixSize
+
int64
(
len
(
txnh
.
workMem
))
}
}
// DataPos returns start position of data inside transaction record
// DataPos returns start position of data inside transaction record
.
func
(
txnh
*
TxnHeader
)
DataPos
()
int64
{
func
(
txnh
*
TxnHeader
)
DataPos
()
int64
{
return
txnh
.
Pos
+
txnh
.
HeaderLen
()
return
txnh
.
Pos
+
txnh
.
HeaderLen
()
}
}
// DataLen returns length of all data inside transaction record container
// DataLen returns length of all data inside transaction record container
.
func
(
txnh
*
TxnHeader
)
DataLen
()
int64
{
func
(
txnh
*
TxnHeader
)
DataLen
()
int64
{
return
txnh
.
Len
-
txnh
.
HeaderLen
()
-
8
/* trailer redundant length */
return
txnh
.
Len
-
txnh
.
HeaderLen
()
-
8
/* trailer redundant length */
}
}
...
@@ -329,7 +330,7 @@ func (txnh *TxnHeader) Load(r io.ReaderAt, pos int64, flags TxnLoadFlags) error
...
@@ -329,7 +330,7 @@ func (txnh *TxnHeader) Load(r io.ReaderAt, pos int64, flags TxnLoadFlags) error
return
err
return
err
}
}
// loadStrings makes sure strings that are part of transaction header are loaded
// loadStrings makes sure strings that are part of transaction header are loaded
.
func
(
txnh
*
TxnHeader
)
loadStrings
(
r
io
.
ReaderAt
)
error
{
func
(
txnh
*
TxnHeader
)
loadStrings
(
r
io
.
ReaderAt
)
error
{
// XXX make it no-op if strings are already loaded?
// XXX make it no-op if strings are already loaded?
...
@@ -659,7 +660,8 @@ func (dh *DataHeader) loadNext(r io.ReaderAt, txnh *TxnHeader) error {
...
@@ -659,7 +660,8 @@ func (dh *DataHeader) loadNext(r io.ReaderAt, txnh *TxnHeader) error {
// LoadData loads data for the data record taking backpointers into account.
// LoadData loads data for the data record taking backpointers into account.
//
//
// NOTE on success dh state is changed to data header of original data transaction.
// On success dh state is changed to data header of original data transaction.
//
// NOTE "deleted" records are indicated via returning buf with .Data=nil without error.
// NOTE "deleted" records are indicated via returning buf with .Data=nil without error.
func
(
dh
*
DataHeader
)
LoadData
(
r
io
.
ReaderAt
)
(
*
mem
.
Buf
,
error
)
{
func
(
dh
*
DataHeader
)
LoadData
(
r
io
.
ReaderAt
)
(
*
mem
.
Buf
,
error
)
{
// scan via backpointers
// scan via backpointers
...
@@ -686,7 +688,7 @@ func (dh *DataHeader) LoadData(r io.ReaderAt) (*mem.Buf, error) {
...
@@ -686,7 +688,7 @@ func (dh *DataHeader) LoadData(r io.ReaderAt) (*mem.Buf, error) {
// --- raw iteration ---
// --- raw iteration ---
// Iter is combined 2-level iterator over transaction and data records
// Iter is combined 2-level iterator over transaction and data records
.
type
Iter
struct
{
type
Iter
struct
{
R
io
.
ReaderAt
R
io
.
ReaderAt
Dir
IterDir
Dir
IterDir
...
@@ -728,13 +730,13 @@ func (it *Iter) NextTxn(flags TxnLoadFlags) error {
...
@@ -728,13 +730,13 @@ func (it *Iter) NextTxn(flags TxnLoadFlags) error {
return
err
return
err
}
}
// NextData iterates to next data record header inside current transaction
// NextData iterates to next data record header inside current transaction
.
func
(
it
*
Iter
)
NextData
()
error
{
func
(
it
*
Iter
)
NextData
()
error
{
return
it
.
Datah
.
LoadNext
(
it
.
R
,
&
it
.
Txnh
)
return
it
.
Datah
.
LoadNext
(
it
.
R
,
&
it
.
Txnh
)
}
}
// Iterate creates Iter to iterate over r starting from posStart in direction dir
// Iterate creates Iter to iterate over r starting from posStart in direction dir
.
func
Iterate
(
r
io
.
ReaderAt
,
posStart
int64
,
dir
IterDir
)
*
Iter
{
func
Iterate
(
r
io
.
ReaderAt
,
posStart
int64
,
dir
IterDir
)
*
Iter
{
it
:=
&
Iter
{
R
:
r
,
Dir
:
dir
,
Txnh
:
TxnHeader
{
Pos
:
posStart
}}
it
:=
&
Iter
{
R
:
r
,
Dir
:
dir
,
Txnh
:
TxnHeader
{
Pos
:
posStart
}}
switch
dir
{
switch
dir
{
...
...
go/zodb/storage/fs1/index.go
View file @
fe751a15
// Copyright (C) 2017 Nexedi SA and Contributors.
// Copyright (C) 2017
-2018
Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
Kirill Smelkov <kirr@nexedi.com>
//
//
// This program is free software: you can Use, Study, Modify and Redistribute
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// it under the terms of the GNU General Public License version 3, or (at your
...
@@ -83,7 +83,7 @@ const (
...
@@ -83,7 +83,7 @@ const (
posValidMask
uint64
=
1
<<
48
-
1
// 0x0000ffffffffffff
posValidMask
uint64
=
1
<<
48
-
1
// 0x0000ffffffffffff
)
)
// IndexSaveError is the error type returned by index save routines
// IndexSaveError is the error type returned by index save routines
.
type
IndexSaveError
struct
{
type
IndexSaveError
struct
{
Err
error
// error that occurred during the operation
Err
error
// error that occurred during the operation
}
}
...
@@ -92,7 +92,7 @@ func (e *IndexSaveError) Error() string {
...
@@ -92,7 +92,7 @@ func (e *IndexSaveError) Error() string {
return
"index save: "
+
e
.
Err
.
Error
()
return
"index save: "
+
e
.
Err
.
Error
()
}
}
// Save saves index to a writer
// Save saves index to a writer
.
func
(
fsi
*
Index
)
Save
(
w
io
.
Writer
)
(
err
error
)
{
func
(
fsi
*
Index
)
Save
(
w
io
.
Writer
)
(
err
error
)
{
defer
func
()
{
defer
func
()
{
if
err
==
nil
{
if
err
==
nil
{
...
@@ -201,7 +201,7 @@ func (fsi *Index) SaveFile(path string) error {
...
@@ -201,7 +201,7 @@ func (fsi *Index) SaveFile(path string) error {
return
nil
return
nil
}
}
// IndexLoadError is the error type returned by index load routines
// IndexLoadError is the error type returned by index load routines
.
type
IndexLoadError
struct
{
type
IndexLoadError
struct
{
Filename
string
// present if used IO object was with .Name()
Filename
string
// present if used IO object was with .Name()
Pos
int64
Pos
int64
...
@@ -220,7 +220,7 @@ func (e *IndexLoadError) Error() string {
...
@@ -220,7 +220,7 @@ func (e *IndexLoadError) Error() string {
return
s
return
s
}
}
// LoadIndex loads index from a reader
// LoadIndex loads index from a reader
.
func
LoadIndex
(
r
io
.
Reader
)
(
fsi
*
Index
,
err
error
)
{
func
LoadIndex
(
r
io
.
Reader
)
(
fsi
*
Index
,
err
error
)
{
var
picklePos
int64
var
picklePos
int64
defer
func
()
{
defer
func
()
{
...
@@ -388,7 +388,7 @@ func treeEqual(a, b *fsb.Tree) bool {
...
@@ -388,7 +388,7 @@ func treeEqual(a, b *fsb.Tree) bool {
// --- build index from FileStorage data ---
// --- build index from FileStorage data ---
// IndexUpdateProgress is data sent by Index.Update to notify about progress
// IndexUpdateProgress is data sent by Index.Update to notify about progress
.
type
IndexUpdateProgress
struct
{
type
IndexUpdateProgress
struct
{
TopPos
int64
// data range to update to; if = -1 -- till EOF
TopPos
int64
// data range to update to; if = -1 -- till EOF
TxnIndexed
int
// # transactions read/indexed so far
TxnIndexed
int
// # transactions read/indexed so far
...
@@ -410,8 +410,9 @@ type IndexUpdateProgress struct {
...
@@ -410,8 +410,9 @@ type IndexUpdateProgress struct {
// which position in data the index could be updated.
// which position in data the index could be updated.
//
//
// On success returned error is nil and index.TopPos is set to either:
// On success returned error is nil and index.TopPos is set to either:
// - topPos (if it is != -1), or
//
// - r's position at which read got EOF (if topPos=-1).
// - topPos (if it is != -1), or
// - r's position at which read got EOF (if topPos=-1).
func
(
index
*
Index
)
Update
(
ctx
context
.
Context
,
r
io
.
ReaderAt
,
topPos
int64
,
progress
func
(
*
IndexUpdateProgress
))
(
err
error
)
{
func
(
index
*
Index
)
Update
(
ctx
context
.
Context
,
r
io
.
ReaderAt
,
topPos
int64
,
progress
func
(
*
IndexUpdateProgress
))
(
err
error
)
{
defer
xerr
.
Contextf
(
&
err
,
"%sreindex %v..%v"
,
ioprefix
(
r
),
index
.
TopPos
,
topPos
)
defer
xerr
.
Contextf
(
&
err
,
"%sreindex %v..%v"
,
ioprefix
(
r
),
index
.
TopPos
,
topPos
)
...
@@ -553,7 +554,7 @@ func indexCorrupt(f interface{}, format string, argv ...interface{}) *IndexCorru
...
@@ -553,7 +554,7 @@ func indexCorrupt(f interface{}, format string, argv ...interface{}) *IndexCorru
return
&
IndexCorruptError
{
DataFileName
:
ioname
(
f
),
Detail
:
fmt
.
Sprintf
(
format
,
argv
...
)}
return
&
IndexCorruptError
{
DataFileName
:
ioname
(
f
),
Detail
:
fmt
.
Sprintf
(
format
,
argv
...
)}
}
}
// IndexVerifyProgress is data sent by Index.Verify to notify about progress
// IndexVerifyProgress is data sent by Index.Verify to notify about progress
.
type
IndexVerifyProgress
struct
{
type
IndexVerifyProgress
struct
{
TxnTotal
int
// total # of transactions to verify; if = -1 -- whole data
TxnTotal
int
// total # of transactions to verify; if = -1 -- whole data
TxnChecked
int
TxnChecked
int
...
@@ -575,8 +576,9 @@ type IndexVerifyProgress struct {
...
@@ -575,8 +576,9 @@ type IndexVerifyProgress struct {
// exactly the same as if it was build anew for data in range ..index.TopPos .
// exactly the same as if it was build anew for data in range ..index.TopPos .
//
//
// Returned error is either:
// Returned error is either:
// - of type *IndexCorruptError, when data in index was found not to match original data, or
//
// - any other error type representing e.g. IO error when reading original data or something else.
// - of type *IndexCorruptError, when data in index was found not to match original data, or
// - any other error type representing e.g. IO error when reading original data or something else.
func
(
index
*
Index
)
Verify
(
ctx
context
.
Context
,
r
io
.
ReaderAt
,
ntxn
int
,
progress
func
(
*
IndexVerifyProgress
))
(
oidChecked
map
[
zodb
.
Oid
]
struct
{},
err
error
)
{
func
(
index
*
Index
)
Verify
(
ctx
context
.
Context
,
r
io
.
ReaderAt
,
ntxn
int
,
progress
func
(
*
IndexVerifyProgress
))
(
oidChecked
map
[
zodb
.
Oid
]
struct
{},
err
error
)
{
defer
func
()
{
defer
func
()
{
if
_
,
ok
:=
err
.
(
*
IndexCorruptError
);
ok
{
if
_
,
ok
:=
err
.
(
*
IndexCorruptError
);
ok
{
...
@@ -672,7 +674,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int, progres
...
@@ -672,7 +674,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int, progres
return
oidChecked
,
nil
return
oidChecked
,
nil
}
}
// VerifyForFile checks index correctness against FileStorage data in file @ path
// VerifyForFile checks index correctness against FileStorage data in file @ path
.
//
//
// See Verify for semantic description.
// See Verify for semantic description.
func
(
index
*
Index
)
VerifyForFile
(
ctx
context
.
Context
,
path
string
,
ntxn
int
,
progress
func
(
*
IndexVerifyProgress
))
(
oidChecked
map
[
zodb
.
Oid
]
struct
{},
err
error
)
{
func
(
index
*
Index
)
VerifyForFile
(
ctx
context
.
Context
,
path
string
,
ntxn
int
,
progress
func
(
*
IndexVerifyProgress
))
(
oidChecked
map
[
zodb
.
Oid
]
struct
{},
err
error
)
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment