Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
N
neoppod
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Levin Zimmermann
neoppod
Commits
3e0610ce
Commit
3e0610ce
authored
Nov 22, 2017
by
Kirill Smelkov
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
.
parent
428eb133
Changes
7
Show whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
15 additions
and
17 deletions
+15
-17
go/zodb/storage/fs1/cmd/fs1/fs1.go
go/zodb/storage/fs1/cmd/fs1/fs1.go
+1
-1
go/zodb/storage/fs1/filestorage.go
go/zodb/storage/fs1/filestorage.go
+7
-6
go/zodb/storage/fs1/format.go
go/zodb/storage/fs1/format.go
+2
-2
go/zodb/storage/fs1/fs1tools/dump.go
go/zodb/storage/fs1/fs1tools/dump.go
+2
-4
go/zodb/storage/fs1/fs1tools/dump_test.go
go/zodb/storage/fs1/fs1tools/dump_test.go
+0
-1
go/zodb/storage/fs1/fsb/fsb.go
go/zodb/storage/fs1/fsb/fsb.go
+2
-2
go/zodb/storage/fs1/index.go
go/zodb/storage/fs1/index.go
+1
-1
No files found.
go/zodb/storage/fs1/cmd/fs1/fs1.go
View file @
3e0610ce
...
@@ -17,7 +17,7 @@
...
@@ -17,7 +17,7 @@
// See COPYING file for full licensing terms.
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// See https://www.nexedi.com/licensing for rationale and options.
// fs1 is a driver program for running
&
invoking fs1 subcommands.
// fs1 is a driver program for running
and
invoking fs1 subcommands.
package
main
package
main
import
(
import
(
...
...
go/zodb/storage/fs1/filestorage.go
View file @
3e0610ce
...
@@ -36,7 +36,8 @@
...
@@ -36,7 +36,8 @@
//
//
// In addition to append-only transaction/data log, an index is automatically
// In addition to append-only transaction/data log, an index is automatically
// maintained mapping oid -> latest data record which modified this oid. The
// maintained mapping oid -> latest data record which modified this oid. The
// index is used to implement zodb.IStorage.Load without linear scan.
// index is used to implement zodb.IStorage.Load for latest data without linear
// scan.
//
//
// The data format is bit-to-bit identical to FileStorage format implemented in ZODB/py.
// The data format is bit-to-bit identical to FileStorage format implemented in ZODB/py.
// Please see the following links for original FileStorage format definition:
// Please see the following links for original FileStorage format definition:
...
@@ -88,7 +89,6 @@ type FileStorage struct {
...
@@ -88,7 +89,6 @@ type FileStorage struct {
// transaction headers for min/max transactions committed
// transaction headers for min/max transactions committed
// (both with .Len=0 & .Tid=0 if database is empty)
// (both with .Len=0 & .Tid=0 if database is empty)
// XXX keep loaded with LoadNoStrings ?
txnhMin
TxnHeader
txnhMin
TxnHeader
txnhMax
TxnHeader
txnhMax
TxnHeader
...
@@ -114,12 +114,12 @@ func (fs *FileStorage) LastTid(_ context.Context) (zodb.Tid, error) {
...
@@ -114,12 +114,12 @@ func (fs *FileStorage) LastTid(_ context.Context) (zodb.Tid, error) {
func
(
fs
*
FileStorage
)
LastOid
(
_
context
.
Context
)
(
zodb
.
Oid
,
error
)
{
func
(
fs
*
FileStorage
)
LastOid
(
_
context
.
Context
)
(
zodb
.
Oid
,
error
)
{
// XXX must be under lock
// XXX must be under lock
// XXX what if an oid was deleted?
lastOid
,
_
:=
fs
.
index
.
Last
()
// returns zero-value, if empty
lastOid
,
_
:=
fs
.
index
.
Last
()
// returns zero-value, if empty
return
lastOid
,
nil
return
lastOid
,
nil
}
}
// ErrXidLoad is returned when there is an error while loading xid
// ErrXidLoad is returned when there is an error while loading xid
// XXX -> zodb (common bits)
type
ErrXidLoad
struct
{
type
ErrXidLoad
struct
{
Xid
zodb
.
Xid
Xid
zodb
.
Xid
Err
error
Err
error
...
@@ -129,6 +129,8 @@ func (e *ErrXidLoad) Error() string {
...
@@ -129,6 +129,8 @@ func (e *ErrXidLoad) Error() string {
return
fmt
.
Sprintf
(
"loading %v: %v"
,
e
.
Xid
,
e
.
Err
)
return
fmt
.
Sprintf
(
"loading %v: %v"
,
e
.
Xid
,
e
.
Err
)
}
}
// XXX +Cause
// freelist(DataHeader)
// freelist(DataHeader)
var
dhPool
=
sync
.
Pool
{
New
:
func
()
interface
{}
{
return
&
DataHeader
{}
}}
var
dhPool
=
sync
.
Pool
{
New
:
func
()
interface
{}
{
return
&
DataHeader
{}
}}
...
@@ -145,13 +147,13 @@ func (dh *DataHeader) Free() {
...
@@ -145,13 +147,13 @@ func (dh *DataHeader) Free() {
}
}
func
(
fs
*
FileStorage
)
Load
(
_
context
.
Context
,
xid
zodb
.
Xid
)
(
buf
*
zodb
.
Buf
,
tid
zodb
.
Tid
,
err
error
)
{
func
(
fs
*
FileStorage
)
Load
(
_
context
.
Context
,
xid
zodb
.
Xid
)
(
buf
*
zodb
.
Buf
,
tid
zodb
.
Tid
,
err
error
)
{
// lookup in index position of oid data record within latest transaction wh
o
changed this oid
// lookup in index position of oid data record within latest transaction wh
ich
changed this oid
dataPos
,
ok
:=
fs
.
index
.
Get
(
xid
.
Oid
)
dataPos
,
ok
:=
fs
.
index
.
Get
(
xid
.
Oid
)
if
!
ok
{
if
!
ok
{
return
nil
,
0
,
&
zodb
.
ErrOidMissing
{
Oid
:
xid
.
Oid
}
return
nil
,
0
,
&
zodb
.
ErrOidMissing
{
Oid
:
xid
.
Oid
}
}
}
// FIXME zodb.TidMax is only 7fff... tid from outside can be ffff...
// FIXME zodb.TidMax is only 7fff... tid from outside can be ffff...
-> TODO reject tid out of range
// XXX go compiler cannot deduce dh should be on stack here
// XXX go compiler cannot deduce dh should be on stack here
//dh := DataHeader{Oid: xid.Oid, Tid: zodb.TidMax, PrevRevPos: dataPos}
//dh := DataHeader{Oid: xid.Oid, Tid: zodb.TidMax, PrevRevPos: dataPos}
...
@@ -201,7 +203,6 @@ func (fs *FileStorage) _Load(dh *DataHeader, xid zodb.Xid) (*zodb.Buf, zodb.Tid,
...
@@ -201,7 +203,6 @@ func (fs *FileStorage) _Load(dh *DataHeader, xid zodb.Xid) (*zodb.Buf, zodb.Tid,
}
}
if
buf
.
Data
==
nil
{
if
buf
.
Data
==
nil
{
// data was deleted
// data was deleted
// XXX or allow this and return via buf.Data=nil ?
return
nil
,
0
,
&
zodb
.
ErrXidMissing
{
Xid
:
xid
}
return
nil
,
0
,
&
zodb
.
ErrXidMissing
{
Xid
:
xid
}
}
}
...
...
go/zodb/storage/fs1/format.go
View file @
3e0610ce
...
@@ -93,7 +93,7 @@ type RecordError struct {
...
@@ -93,7 +93,7 @@ type RecordError struct {
Path
string
// path of the data file
Path
string
// path of the data file
Record
string
// record kind - "file header", "transaction record", "data record", ...
Record
string
// record kind - "file header", "transaction record", "data record", ...
Pos
int64
// position of record
Pos
int64
// position of record
Subj
string
// subject context for the error - e.g. "read"
, "check" or "bug
"
Subj
string
// subject context for the error - e.g. "read"
or "check
"
Err
error
// actual error
Err
error
// actual error
}
}
...
@@ -282,7 +282,7 @@ func (txnh *TxnHeader) Load(r io.ReaderAt, pos int64, flags TxnLoadFlags) error
...
@@ -282,7 +282,7 @@ func (txnh *TxnHeader) Load(r io.ReaderAt, pos int64, flags TxnLoadFlags) error
if
tlen
<
TxnHeaderFixSize
{
if
tlen
<
TxnHeaderFixSize
{
return
checkErr
(
r
,
txnh
,
"invalid txn record length: %v"
,
tlen
)
return
checkErr
(
r
,
txnh
,
"invalid txn record length: %v"
,
tlen
)
}
}
//
XXX also check tlen to not go beyond file size ?
//
NOTE no need to check tlen does not go beyon file size - loadStrings/LoadData or LoadNext will catch it.
// txnh.Len will be set =tlen at last - after checking other fields for correctness.
// txnh.Len will be set =tlen at last - after checking other fields for correctness.
txnh
.
Status
=
zodb
.
TxnStatus
(
work
[
8
+
16
])
txnh
.
Status
=
zodb
.
TxnStatus
(
work
[
8
+
16
])
...
...
go/zodb/storage/fs1/fs1tools/dump.go
View file @
3e0610ce
...
@@ -57,7 +57,7 @@ type Dumper interface {
...
@@ -57,7 +57,7 @@ type Dumper interface {
// Dump dumps content of a FileStorage file @ path.
// Dump dumps content of a FileStorage file @ path.
//
//
// To do so it reads file header and then iterates over all transactions in the file.
// To do so it reads file header and then iterates over all transactions in the file.
// The logic to actually output information and, if needed read/process data,
is
implemented by Dumper d.
// The logic to actually output information and, if needed read/process data,
should be
implemented by Dumper d.
func
Dump
(
w
io
.
Writer
,
path
string
,
dir
fs1
.
IterDir
,
d
Dumper
)
(
err
error
)
{
func
Dump
(
w
io
.
Writer
,
path
string
,
dir
fs1
.
IterDir
,
d
Dumper
)
(
err
error
)
{
defer
xerr
.
Contextf
(
&
err
,
"%s: %s"
,
d
.
DumperName
(),
path
)
defer
xerr
.
Contextf
(
&
err
,
"%s: %s"
,
d
.
DumperName
(),
path
)
...
@@ -163,7 +163,7 @@ func (d *DumperFsDump) DumpTxn(buf *xfmt.Buffer, it *fs1.Iter) error {
...
@@ -163,7 +163,7 @@ func (d *DumperFsDump) DumpTxn(buf *xfmt.Buffer, it *fs1.Iter) error {
err
:=
it
.
NextData
()
err
:=
it
.
NextData
()
if
err
!=
nil
{
if
err
!=
nil
{
if
err
==
io
.
EOF
{
if
err
==
io
.
EOF
{
break
err
=
nil
// XXX -> okEOF(err)
}
}
return
err
return
err
}
}
...
@@ -197,8 +197,6 @@ func (d *DumperFsDump) DumpTxn(buf *xfmt.Buffer, it *fs1.Iter) error {
...
@@ -197,8 +197,6 @@ func (d *DumperFsDump) DumpTxn(buf *xfmt.Buffer, it *fs1.Iter) error {
buf
.
S
(
"
\n
"
)
buf
.
S
(
"
\n
"
)
dbuf
.
Release
()
dbuf
.
Release
()
}
}
return
nil
}
}
// DumperFsDumpVerbose implements a very verbose dumper with output identical
// DumperFsDumpVerbose implements a very verbose dumper with output identical
...
...
go/zodb/storage/fs1/fs1tools/dump_test.go
View file @
3e0610ce
...
@@ -40,7 +40,6 @@ import (
...
@@ -40,7 +40,6 @@ import (
"github.com/kylelemons/godebug/diff"
"github.com/kylelemons/godebug/diff"
)
)
// XXX -> xtesting ?
func
loadFile
(
t
*
testing
.
T
,
path
string
)
string
{
func
loadFile
(
t
*
testing
.
T
,
path
string
)
string
{
data
,
err
:=
ioutil
.
ReadFile
(
path
)
data
,
err
:=
ioutil
.
ReadFile
(
path
)
if
err
!=
nil
{
if
err
!=
nil
{
...
...
go/zodb/storage/fs1/fsb/fsb.go
View file @
3e0610ce
...
@@ -26,8 +26,8 @@ package fsb
...
@@ -26,8 +26,8 @@ package fsb
import
"lab.nexedi.com/kirr/neo/go/zodb"
import
"lab.nexedi.com/kirr/neo/go/zodb"
// comparison function for fsbTree
// comparison function for fsbTree
.
// kept short & inlineable
// kept short & inlineable
.
func
oidCmp
(
a
,
b
zodb
.
Oid
)
int
{
func
oidCmp
(
a
,
b
zodb
.
Oid
)
int
{
if
a
<
b
{
if
a
<
b
{
return
-
1
return
-
1
...
...
go/zodb/storage/fs1/index.go
View file @
3e0610ce
...
@@ -18,7 +18,7 @@
...
@@ -18,7 +18,7 @@
// See https://www.nexedi.com/licensing for rationale and options.
// See https://www.nexedi.com/licensing for rationale and options.
package
fs1
package
fs1
// index for quickly finding oid -> latest data record
// index for quickly finding oid ->
oid's
latest data record
import
(
import
(
"bufio"
"bufio"
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment