Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
N
neoppod
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Levin Zimmermann
neoppod
Commits
81dbf1d6
Commit
81dbf1d6
authored
Sep 11, 2017
by
Kirill Smelkov
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
.
parent
814a1fa8
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
39 additions
and
11 deletions
+39
-11
go/neo/t/zsha1.go
go/neo/t/zsha1.go
+4
-3
go/zodb/storage/fs1/filestorage.go
go/zodb/storage/fs1/filestorage.go
+22
-1
go/zodb/storage/fs1/format.go
go/zodb/storage/fs1/format.go
+6
-0
go/zodb/storage/fs1/index.go
go/zodb/storage/fs1/index.go
+7
-7
No files found.
go/neo/t/zsha1.go
View file @
81dbf1d6
...
@@ -103,9 +103,10 @@ func zsha1(ctx context.Context, url string, useprefetch bool) (err error) {
...
@@ -103,9 +103,10 @@ func zsha1(ctx context.Context, url string, useprefetch bool) (err error) {
}
}
before
:=
lastTid
+
1
// XXX overflow ?
before
:=
lastTid
+
1
// XXX overflow ?
if
tru
e
{
if
fals
e
{
//defer profile.Start(profile.TraceProfile).Stop()
//defer profile.Start(profile.TraceProfile).Stop()
defer
profile
.
Start
()
.
Stop
()
//defer profile.Start(profile.MemProfile).Stop()
defer
profile
.
Start
(
profile
.
CPUProfile
)
.
Stop
()
}
}
for
qqq
:=
0
;
qqq
<
10
;
qqq
++
{
for
qqq
:=
0
;
qqq
<
10
;
qqq
++
{
...
@@ -142,7 +143,7 @@ loop:
...
@@ -142,7 +143,7 @@ loop:
return
err
return
err
}
}
m
.
Write
(
data
)
//
m.Write(data)
//fmt.Fprintf(os.Stderr, "%d @%s\tsha1: %x\n", uint(oid), serial, m.Sum(nil))
//fmt.Fprintf(os.Stderr, "%d @%s\tsha1: %x\n", uint(oid), serial, m.Sum(nil))
//fmt.Fprintf(os.Stderr, "\tdata: %x\n", data)
//fmt.Fprintf(os.Stderr, "\tdata: %x\n", data)
...
...
go/zodb/storage/fs1/filestorage.go
View file @
81dbf1d6
...
@@ -68,6 +68,7 @@ import (
...
@@ -68,6 +68,7 @@ import (
"fmt"
"fmt"
"io"
"io"
"os"
"os"
"sync"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/kirr/neo/go/xcommon/xbufio"
"lab.nexedi.com/kirr/neo/go/xcommon/xbufio"
...
@@ -224,6 +225,19 @@ func (e *ErrXidLoad) Error() string {
...
@@ -224,6 +225,19 @@ func (e *ErrXidLoad) Error() string {
return
fmt
.
Sprintf
(
"loading %v: %v"
,
e
.
Xid
,
e
.
Err
)
return
fmt
.
Sprintf
(
"loading %v: %v"
,
e
.
Xid
,
e
.
Err
)
}
}
// freelist(DataHeader)
var
dhPool
=
sync
.
Pool
{
New
:
func
()
interface
{}
{
return
&
DataHeader
{}
}}
// DataHeaderAlloc allocates DataHeader from freelist.
func
DataHeaderAlloc
()
*
DataHeader
{
return
dhPool
.
Get
()
.
(
*
DataHeader
)
}
// Free puts dh back into DataHeader freelist.
func
(
dh
*
DataHeader
)
Free
()
{
dhPool
.
Put
(
dh
)
}
func
(
fs
*
FileStorage
)
Load
(
_
context
.
Context
,
xid
zodb
.
Xid
)
(
data
[]
byte
,
tid
zodb
.
Tid
,
err
error
)
{
func
(
fs
*
FileStorage
)
Load
(
_
context
.
Context
,
xid
zodb
.
Xid
)
(
data
[]
byte
,
tid
zodb
.
Tid
,
err
error
)
{
// lookup in index position of oid data record within latest transaction who changed this oid
// lookup in index position of oid data record within latest transaction who changed this oid
dataPos
,
ok
:=
fs
.
index
.
Get
(
xid
.
Oid
)
dataPos
,
ok
:=
fs
.
index
.
Get
(
xid
.
Oid
)
...
@@ -232,7 +246,14 @@ func (fs *FileStorage) Load(_ context.Context, xid zodb.Xid) (data []byte, tid z
...
@@ -232,7 +246,14 @@ func (fs *FileStorage) Load(_ context.Context, xid zodb.Xid) (data []byte, tid z
}
}
// FIXME zodb.TidMax is only 7fff... tid from outside can be ffff...
// FIXME zodb.TidMax is only 7fff... tid from outside can be ffff...
dh
:=
DataHeader
{
Oid
:
xid
.
Oid
,
Tid
:
zodb
.
TidMax
,
PrevRevPos
:
dataPos
}
// XXX go compiler cannot deduce dh should be on stack here
//dh := DataHeader{Oid: xid.Oid, Tid: zodb.TidMax, PrevRevPos: dataPos}
dh
:=
DataHeaderAlloc
()
dh
.
Oid
=
xid
.
Oid
dh
.
Tid
=
zodb
.
TidMax
dh
.
PrevRevPos
=
dataPos
defer
dh
.
Free
()
tidBefore
:=
xid
.
XTid
.
Tid
tidBefore
:=
xid
.
XTid
.
Tid
if
!
xid
.
XTid
.
TidBefore
{
if
!
xid
.
XTid
.
TidBefore
{
tidBefore
++
// XXX recheck this is ok wrt overflow
tidBefore
++
// XXX recheck this is ok wrt overflow
...
...
go/zodb/storage/fs1/format.go
View file @
81dbf1d6
...
@@ -401,6 +401,7 @@ func (txnh *TxnHeader) LoadPrev(r io.ReaderAt, flags TxnLoadFlags) error {
...
@@ -401,6 +401,7 @@ func (txnh *TxnHeader) LoadPrev(r io.ReaderAt, flags TxnLoadFlags) error {
}
}
// LoadNext reads and decodes next transaction record header.
// LoadNext reads and decodes next transaction record header.
//
// prerequisite: txnh .Pos and .Len should be already initialized by: XXX also .Tid
// prerequisite: txnh .Pos and .Len should be already initialized by: XXX also .Tid
// - previous successful call to Load() initially XXX ^^^
// - previous successful call to Load() initially XXX ^^^
// - TODO
// - TODO
...
@@ -512,6 +513,7 @@ func (dh *DataHeader) Load(r io.ReaderAt, pos int64) error {
...
@@ -512,6 +513,7 @@ func (dh *DataHeader) Load(r io.ReaderAt, pos int64) error {
}
}
// LoadPrevRev reads and decodes previous revision data record header.
// LoadPrevRev reads and decodes previous revision data record header.
//
// prerequisite: dh .Oid .Tid .PrevRevPos are initialized:
// prerequisite: dh .Oid .Tid .PrevRevPos are initialized:
// - TODO describe how
// - TODO describe how
// when there is no previous revision: io.EOF is returned
// when there is no previous revision: io.EOF is returned
...
@@ -553,6 +555,7 @@ func (dh *DataHeader) loadPrevRev(r io.ReaderAt) error {
...
@@ -553,6 +555,7 @@ func (dh *DataHeader) loadPrevRev(r io.ReaderAt) error {
}
}
// LoadBackRef reads data for the data record and decodes it as backpointer reference.
// LoadBackRef reads data for the data record and decodes it as backpointer reference.
//
// prerequisite: dh loaded and .LenData == 0 (data record with back-pointer)
// prerequisite: dh loaded and .LenData == 0 (data record with back-pointer)
// XXX return backPos=-1 if err?
// XXX return backPos=-1 if err?
// XXX unused?
// XXX unused?
...
@@ -578,6 +581,7 @@ func (dh *DataHeader) LoadBackRef(r io.ReaderAt) (backPos int64, err error) {
...
@@ -578,6 +581,7 @@ func (dh *DataHeader) LoadBackRef(r io.ReaderAt) (backPos int64, err error) {
}
}
// LoadBack reads and decodes data header for revision linked via back-pointer.
// LoadBack reads and decodes data header for revision linked via back-pointer.
//
// prerequisite: dh XXX .DataLen == 0
// prerequisite: dh XXX .DataLen == 0
// if link is to zero (means deleted record) io.EOF is returned
// if link is to zero (means deleted record) io.EOF is returned
func
(
dh
*
DataHeader
)
LoadBack
(
r
io
.
ReaderAt
)
error
{
func
(
dh
*
DataHeader
)
LoadBack
(
r
io
.
ReaderAt
)
error
{
...
@@ -617,6 +621,7 @@ func (dh *DataHeader) LoadBack(r io.ReaderAt) error {
...
@@ -617,6 +621,7 @@ func (dh *DataHeader) LoadBack(r io.ReaderAt) error {
}
}
// LoadNext reads and decodes data header for next data record in the same transaction.
// LoadNext reads and decodes data header for next data record in the same transaction.
//
// prerequisite: dh .Pos .DataLen are initialized
// prerequisite: dh .Pos .DataLen are initialized
// when there is no more data records: io.EOF is returned
// when there is no more data records: io.EOF is returned
func
(
dh
*
DataHeader
)
LoadNext
(
r
io
.
ReaderAt
,
txnh
*
TxnHeader
)
error
{
func
(
dh
*
DataHeader
)
LoadNext
(
r
io
.
ReaderAt
,
txnh
*
TxnHeader
)
error
{
...
@@ -661,6 +666,7 @@ func (dh *DataHeader) loadNext(r io.ReaderAt, txnh *TxnHeader) error {
...
@@ -661,6 +666,7 @@ func (dh *DataHeader) loadNext(r io.ReaderAt, txnh *TxnHeader) error {
}
}
// LoadData loads data for the data record taking backpointers into account.
// LoadData loads data for the data record taking backpointers into account.
//
// Data is loaded into *buf, which, if needed, is reallocated to hold whole loading data size.
// Data is loaded into *buf, which, if needed, is reallocated to hold whole loading data size.
// NOTE on success dh state is changed to data header of original data transaction
// NOTE on success dh state is changed to data header of original data transaction
// NOTE "deleted" records are indicated via returning *buf=nil
// NOTE "deleted" records are indicated via returning *buf=nil
...
...
go/zodb/storage/fs1/index.go
View file @
81dbf1d6
...
@@ -56,7 +56,7 @@ type Index struct {
...
@@ -56,7 +56,7 @@ type Index struct {
*
fsb
.
Tree
*
fsb
.
Tree
}
}
// IndexNew creates new empty index
// IndexNew creates new empty index
.
func
IndexNew
()
*
Index
{
func
IndexNew
()
*
Index
{
return
&
Index
{
TopPos
:
txnValidFrom
,
Tree
:
fsb
.
TreeNew
()}
return
&
Index
{
TopPos
:
txnValidFrom
,
Tree
:
fsb
.
TreeNew
()}
}
}
...
@@ -171,7 +171,7 @@ out:
...
@@ -171,7 +171,7 @@ out:
return
&
IndexSaveError
{
err
}
return
&
IndexSaveError
{
err
}
}
}
// SaveFile saves index to a file @ path
// SaveFile saves index to a file @ path
.
//
//
// Index data is first saved to a temporary file and when complete the
// Index data is first saved to a temporary file and when complete the
// temporary is renamed to be at requested path. This way file @ path will be
// temporary is renamed to be at requested path. This way file @ path will be
...
@@ -349,7 +349,7 @@ out:
...
@@ -349,7 +349,7 @@ out:
return
nil
,
&
IndexLoadError
{
xio
.
Name
(
r
),
picklePos
,
err
}
return
nil
,
&
IndexLoadError
{
xio
.
Name
(
r
),
picklePos
,
err
}
}
}
// LoadIndexFile loads index from a file @ path
// LoadIndexFile loads index from a file @ path
.
func
LoadIndexFile
(
path
string
)
(
fsi
*
Index
,
err
error
)
{
func
LoadIndexFile
(
path
string
)
(
fsi
*
Index
,
err
error
)
{
f
,
err
:=
os
.
Open
(
path
)
f
,
err
:=
os
.
Open
(
path
)
if
err
!=
nil
{
if
err
!=
nil
{
...
@@ -370,7 +370,7 @@ func LoadIndexFile(path string) (fsi *Index, err error) {
...
@@ -370,7 +370,7 @@ func LoadIndexFile(path string) (fsi *Index, err error) {
// ----------------------------------------
// ----------------------------------------
// Equal returns whether two indices are the same
// Equal returns whether two indices are the same
.
func
(
a
*
Index
)
Equal
(
b
*
Index
)
bool
{
func
(
a
*
Index
)
Equal
(
b
*
Index
)
bool
{
if
a
.
TopPos
!=
b
.
TopPos
{
if
a
.
TopPos
!=
b
.
TopPos
{
return
false
return
false
...
@@ -379,7 +379,7 @@ func (a *Index) Equal(b *Index) bool {
...
@@ -379,7 +379,7 @@ func (a *Index) Equal(b *Index) bool {
return
treeEqual
(
a
.
Tree
,
b
.
Tree
)
return
treeEqual
(
a
.
Tree
,
b
.
Tree
)
}
}
// treeEqual returns whether two fsb.Tree are the same
// treeEqual returns whether two fsb.Tree are the same
.
func
treeEqual
(
a
,
b
*
fsb
.
Tree
)
bool
{
func
treeEqual
(
a
,
b
*
fsb
.
Tree
)
bool
{
if
a
.
Len
()
!=
b
.
Len
()
{
if
a
.
Len
()
!=
b
.
Len
()
{
return
false
return
false
...
@@ -527,7 +527,7 @@ func (index *Index) Update(ctx context.Context, r io.ReaderAt, topPos int64, pro
...
@@ -527,7 +527,7 @@ func (index *Index) Update(ctx context.Context, r io.ReaderAt, topPos int64, pro
return
nil
return
nil
}
}
// BuildIndex builds new in-memory index for data in r
// BuildIndex builds new in-memory index for data in r
.
//
//
// non-nil valid and consistent index is always returned - even in case of error
// non-nil valid and consistent index is always returned - even in case of error
// the index will describe data till top-position of highest transaction that
// the index will describe data till top-position of highest transaction that
...
@@ -541,7 +541,7 @@ func BuildIndex(ctx context.Context, r io.ReaderAt, progress func(*IndexUpdatePr
...
@@ -541,7 +541,7 @@ func BuildIndex(ctx context.Context, r io.ReaderAt, progress func(*IndexUpdatePr
return
index
,
err
return
index
,
err
}
}
// BuildIndexForFile builds new in-memory index for data in file @ path
// BuildIndexForFile builds new in-memory index for data in file @ path
.
//
//
// See BuildIndex for semantic description.
// See BuildIndex for semantic description.
func
BuildIndexForFile
(
ctx
context
.
Context
,
path
string
,
progress
func
(
*
IndexUpdateProgress
))
(
index
*
Index
,
err
error
)
{
func
BuildIndexForFile
(
ctx
context
.
Context
,
path
string
,
progress
func
(
*
IndexUpdateProgress
))
(
index
*
Index
,
err
error
)
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment