Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
N
neoppod
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
1
Issues
1
List
Boards
Labels
Milestones
Merge Requests
2
Merge Requests
2
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
nexedi
neoppod
Commits
15af0624
Commit
15af0624
authored
Nov 08, 2017
by
Kirill Smelkov
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
.
parent
f71eebf7
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
157 additions
and
170 deletions
+157
-170
go/zodb/storage/fs1/index.go
go/zodb/storage/fs1/index.go
+157
-170
No files found.
go/zodb/storage/fs1/index.go
View file @
15af0624
...
@@ -80,8 +80,8 @@ func IndexNew() *Index {
...
@@ -80,8 +80,8 @@ func IndexNew() *Index {
// oid[6:8]oid[6:8]oid[6:8]...pos[2:8]pos[2:8]pos[2:8]...
// oid[6:8]oid[6:8]oid[6:8]...pos[2:8]pos[2:8]pos[2:8]...
const
(
const
(
oidPrefixMask
zodb
.
Oid
=
(
1
<<
64
-
1
)
^
(
1
<<
16
-
1
)
// 0xffffffffffff0000
oidPrefixMask
zodb
.
Oid
=
(
1
<<
64
-
1
)
^
(
1
<<
16
-
1
)
// 0xffffffffffff0000
posInvalidMask
uint64
=
(
1
<<
64
-
1
)
^
(
1
<<
48
-
1
)
// 0xffff000000000000
posInvalidMask
uint64
=
(
1
<<
64
-
1
)
^
(
1
<<
48
-
1
)
// 0xffff000000000000
posValidMask
uint64
=
1
<<
48
-
1
// 0x0000ffffffffffff
posValidMask
uint64
=
1
<<
48
-
1
// 0x0000ffffffffffff
)
)
...
@@ -95,15 +95,26 @@ func (e *IndexSaveError) Error() string {
...
@@ -95,15 +95,26 @@ func (e *IndexSaveError) Error() string {
}
}
// Save saves index to a writer
// Save saves index to a writer
func
(
fsi
*
Index
)
Save
(
w
io
.
Writer
)
error
{
func
(
fsi
*
Index
)
Save
(
w
io
.
Writer
)
(
err
error
)
{
var
err
error
defer
func
()
{
if
err
==
nil
{
return
}
if
_
,
ok
:=
err
.
(
*
pickle
.
TypeError
);
ok
{
panic
(
err
)
// all our types are expected to be supported by pickle
}
// otherwise it is an error returned by writer, which should already
// have filename & op as context.
err
=
&
IndexSaveError
{
err
}
}()
{
p
:=
pickle
.
NewEncoder
(
w
)
p
:=
pickle
.
NewEncoder
(
w
)
err
=
p
.
Encode
(
fsi
.
TopPos
)
err
=
p
.
Encode
(
fsi
.
TopPos
)
if
err
!=
nil
{
if
err
!=
nil
{
goto
out
return
err
}
}
var
oidb
[
8
]
byte
var
oidb
[
8
]
byte
...
@@ -128,7 +139,7 @@ func (fsi *Index) Save(w io.Writer) error {
...
@@ -128,7 +139,7 @@ func (fsi *Index) Save(w io.Writer) error {
t
[
1
]
=
bytes
.
Join
([][]
byte
{
oidBuf
,
posBuf
},
nil
)
t
[
1
]
=
bytes
.
Join
([][]
byte
{
oidBuf
,
posBuf
},
nil
)
err
=
p
.
Encode
(
pickle
.
Tuple
(
t
[
:
]))
err
=
p
.
Encode
(
pickle
.
Tuple
(
t
[
:
]))
if
err
!=
nil
{
if
err
!=
nil
{
goto
out
return
err
}
}
oidPrefixCur
=
oidPrefix
oidPrefixCur
=
oidPrefix
...
@@ -141,9 +152,8 @@ func (fsi *Index) Save(w io.Writer) error {
...
@@ -141,9 +152,8 @@ func (fsi *Index) Save(w io.Writer) error {
}
}
// check pos does not overflow 6 bytes
// check pos does not overflow 6 bytes
if
uint64
(
pos
)
&
posInvalidMask
!=
0
{
if
uint64
(
pos
)
&
posInvalidMask
!=
0
{
err
=
fmt
.
Errorf
(
"entry position too large: 0x%x"
,
pos
)
return
fmt
.
Errorf
(
"entry position too large: 0x%x"
,
pos
)
goto
out
}
}
binary
.
BigEndian
.
PutUint64
(
oidb
[
:
],
uint64
(
oid
))
binary
.
BigEndian
.
PutUint64
(
oidb
[
:
],
uint64
(
oid
))
...
@@ -155,20 +165,7 @@ func (fsi *Index) Save(w io.Writer) error {
...
@@ -155,20 +165,7 @@ func (fsi *Index) Save(w io.Writer) error {
}
}
err
=
p
.
Encode
(
pickle
.
None
{})
err
=
p
.
Encode
(
pickle
.
None
{})
}
out
:
if
err
==
nil
{
return
err
return
err
}
if
_
,
ok
:=
err
.
(
*
pickle
.
TypeError
);
ok
{
panic
(
err
)
// all our types are expected to be supported by pickle
}
// otherwise it is an error returned by writer, which should already
// have filename & op as context.
return
&
IndexSaveError
{
err
}
}
}
// SaveFile saves index to a file @ path.
// SaveFile saves index to a file @ path.
...
@@ -178,9 +175,9 @@ out:
...
@@ -178,9 +175,9 @@ out:
// updated only with complete index data.
// updated only with complete index data.
func
(
fsi
*
Index
)
SaveFile
(
path
string
)
error
{
func
(
fsi
*
Index
)
SaveFile
(
path
string
)
error
{
dir
,
name
:=
filepath
.
Dir
(
path
),
filepath
.
Base
(
path
)
dir
,
name
:=
filepath
.
Dir
(
path
),
filepath
.
Base
(
path
)
f
,
err
:=
ioutil
.
TempFile
(
dir
,
name
+
".tmp"
)
f
,
err
:=
ioutil
.
TempFile
(
dir
,
name
+
".tmp"
)
if
err
!=
nil
{
if
err
!=
nil
{
return
&
IndexSaveError
{
err
}
// XXX needed?
return
&
IndexSaveError
{
err
}
}
}
// use buffering for f (ogórek does not buffer itself on encoding)
// use buffering for f (ogórek does not buffer itself on encoding)
...
@@ -193,14 +190,14 @@ func (fsi *Index) SaveFile(path string) error {
...
@@ -193,14 +190,14 @@ func (fsi *Index) SaveFile(path string) error {
os
.
Remove
(
f
.
Name
())
os
.
Remove
(
f
.
Name
())
err
=
err1
err
=
err1
if
err
==
nil
{
if
err
==
nil
{
err
=
&
IndexSaveError
{
xerr
.
First
(
err2
,
err3
)}
// XXX needed?
err
=
&
IndexSaveError
{
xerr
.
First
(
err2
,
err3
)}
}
}
return
err
return
err
}
}
err
=
os
.
Rename
(
f
.
Name
(),
path
)
err
=
os
.
Rename
(
f
.
Name
(),
path
)
if
err
!=
nil
{
if
err
!=
nil
{
return
&
IndexSaveError
{
err
}
// XXX needed?
return
&
IndexSaveError
{
err
}
}
}
return
nil
return
nil
...
@@ -242,8 +239,12 @@ func xint64(xv interface{}) (v int64, ok bool) {
...
@@ -242,8 +239,12 @@ func xint64(xv interface{}) (v int64, ok bool) {
// LoadIndex loads index from a reader
// LoadIndex loads index from a reader
func
LoadIndex
(
r
io
.
Reader
)
(
fsi
*
Index
,
err
error
)
{
func
LoadIndex
(
r
io
.
Reader
)
(
fsi
*
Index
,
err
error
)
{
var
picklePos
int64
var
picklePos
int64
defer
func
()
{
if
err
!=
nil
{
err
=
&
IndexLoadError
{
xio
.
Name
(
r
),
picklePos
,
err
}
}
}()
{
var
ok
bool
var
ok
bool
var
xtopPos
,
xv
interface
{}
var
xtopPos
,
xv
interface
{}
...
@@ -254,38 +255,35 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
...
@@ -254,38 +255,35 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
picklePos
=
xr
.
InputOffset
()
picklePos
=
xr
.
InputOffset
()
xtopPos
,
err
=
p
.
Decode
()
xtopPos
,
err
=
p
.
Decode
()
if
err
!=
nil
{
if
err
!=
nil
{
goto
out
return
nil
,
err
}
}
topPos
,
ok
:=
xint64
(
xtopPos
)
topPos
,
ok
:=
xint64
(
xtopPos
)
if
!
ok
{
if
!
ok
{
err
=
fmt
.
Errorf
(
"topPos is %T:%v (expected int64)"
,
xtopPos
,
xtopPos
)
return
nil
,
fmt
.
Errorf
(
"topPos is %T:%v (expected int64)"
,
xtopPos
,
xtopPos
)
goto
out
}
}
fsi
=
IndexNew
()
fsi
=
IndexNew
()
fsi
.
TopPos
=
topPos
fsi
.
TopPos
=
topPos
var
oidb
[
8
]
byte
var
oidb
[
8
]
byte
loop
:
loop
:
for
{
for
{
// load/decode next entry
// load/decode next entry
var
v
pickle
.
Tuple
var
v
pickle
.
Tuple
picklePos
=
xr
.
InputOffset
()
picklePos
=
xr
.
InputOffset
()
xv
,
err
=
p
.
Decode
()
xv
,
err
=
p
.
Decode
()
if
err
!=
nil
{
if
err
!=
nil
{
goto
out
return
nil
,
err
}
}
switch
xv
:=
xv
.
(
type
)
{
switch
xv
:=
xv
.
(
type
)
{
default
:
default
:
err
=
fmt
.
Errorf
(
"invalid entry: type %T"
,
xv
)
return
nil
,
fmt
.
Errorf
(
"invalid entry: type %T"
,
xv
)
goto
out
case
pickle
.
None
:
case
pickle
.
None
:
break
loop
break
loop
// we accept tuple or list
// we accept tuple or list
// XXX accept only tuple ?
case
pickle
.
Tuple
:
case
pickle
.
Tuple
:
v
=
xv
v
=
xv
case
[]
interface
{}
:
case
[]
interface
{}
:
...
@@ -294,20 +292,17 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
...
@@ -294,20 +292,17 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
// unpack entry tuple -> oidPrefix, fsBucket
// unpack entry tuple -> oidPrefix, fsBucket
if
len
(
v
)
!=
2
{
if
len
(
v
)
!=
2
{
err
=
fmt
.
Errorf
(
"invalid entry: len = %i"
,
len
(
v
))
return
nil
,
fmt
.
Errorf
(
"invalid entry: len = %i"
,
len
(
v
))
goto
out
}
}
// decode oidPrefix
// decode oidPrefix
xoidPrefixStr
:=
v
[
0
]
xoidPrefixStr
:=
v
[
0
]
oidPrefixStr
,
ok
:=
xoidPrefixStr
.
(
string
)
oidPrefixStr
,
ok
:=
xoidPrefixStr
.
(
string
)
if
!
ok
{
if
!
ok
{
err
=
fmt
.
Errorf
(
"invalid oidPrefix: type %T"
,
xoidPrefixStr
)
return
nil
,
fmt
.
Errorf
(
"invalid oidPrefix: type %T"
,
xoidPrefixStr
)
goto
out
}
}
if
l
:=
len
(
oidPrefixStr
);
l
!=
6
{
if
l
:=
len
(
oidPrefixStr
);
l
!=
6
{
err
=
fmt
.
Errorf
(
"invalid oidPrefix: len = %i"
,
l
)
return
nil
,
fmt
.
Errorf
(
"invalid oidPrefix: len = %i"
,
l
)
goto
out
}
}
copy
(
oidb
[
:
],
oidPrefixStr
)
copy
(
oidb
[
:
],
oidPrefixStr
)
oidPrefix
:=
zodb
.
Oid
(
binary
.
BigEndian
.
Uint64
(
oidb
[
:
]))
oidPrefix
:=
zodb
.
Oid
(
binary
.
BigEndian
.
Uint64
(
oidb
[
:
]))
...
@@ -316,12 +311,10 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
...
@@ -316,12 +311,10 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
xkvStr
:=
v
[
1
]
xkvStr
:=
v
[
1
]
kvStr
,
ok
:=
xkvStr
.
(
string
)
kvStr
,
ok
:=
xkvStr
.
(
string
)
if
!
ok
{
if
!
ok
{
err
=
fmt
.
Errorf
(
"invalid fsBucket: type %T"
,
xkvStr
)
return
nil
,
fmt
.
Errorf
(
"invalid fsBucket: type %T"
,
xkvStr
)
goto
out
}
}
if
l
:=
len
(
kvStr
);
l
%
8
!=
0
{
if
l
:=
len
(
kvStr
);
l
%
8
!=
0
{
err
=
fmt
.
Errorf
(
"invalid fsBucket: len = %i"
,
l
)
return
nil
,
fmt
.
Errorf
(
"invalid fsBucket: len = %i"
,
l
)
goto
out
}
}
// load btree from fsBucket entries
// load btree from fsBucket entries
...
@@ -331,7 +324,7 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
...
@@ -331,7 +324,7 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
oidBuf
:=
kvBuf
[
:
n
*
2
]
oidBuf
:=
kvBuf
[
:
n
*
2
]
posBuf
:=
kvBuf
[
n
*
2
-
2
:
]
// NOTE starting 2 bytes behind
posBuf
:=
kvBuf
[
n
*
2
-
2
:
]
// NOTE starting 2 bytes behind
for
i
:=
0
;
i
<
n
;
i
++
{
for
i
:=
0
;
i
<
n
;
i
++
{
oid
:=
zodb
.
Oid
(
binary
.
BigEndian
.
Uint16
(
oidBuf
[
i
*
2
:
]))
oid
:=
zodb
.
Oid
(
binary
.
BigEndian
.
Uint16
(
oidBuf
[
i
*
2
:
]))
oid
|=
oidPrefix
oid
|=
oidPrefix
pos
:=
int64
(
binary
.
BigEndian
.
Uint64
(
posBuf
[
i
*
6
:
])
&
posValidMask
)
pos
:=
int64
(
binary
.
BigEndian
.
Uint64
(
posBuf
[
i
*
6
:
])
&
posValidMask
)
...
@@ -339,14 +332,8 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
...
@@ -339,14 +332,8 @@ func LoadIndex(r io.Reader) (fsi *Index, err error) {
fsi
.
Set
(
oid
,
pos
)
fsi
.
Set
(
oid
,
pos
)
}
}
}
}
}
out
:
if
err
==
nil
{
return
fsi
,
err
}
return
nil
,
&
IndexLoadError
{
xio
.
Name
(
r
),
picklePos
,
err
}
return
fsi
,
nil
}
}
// LoadIndexFile loads index from a file @ path.
// LoadIndexFile loads index from a file @ path.
...
@@ -441,7 +428,7 @@ type IndexUpdateProgress struct {
...
@@ -441,7 +428,7 @@ type IndexUpdateProgress struct {
// On success returned error is nil and index.TopPos is set to either:
// On success returned error is nil and index.TopPos is set to either:
// - topPos (if it is != -1), or
// - topPos (if it is != -1), or
// - r's position at which read got EOF (if topPos=-1).
// - r's position at which read got EOF (if topPos=-1).
func
(
index
*
Index
)
Update
(
ctx
context
.
Context
,
r
io
.
ReaderAt
,
topPos
int64
,
progress
func
(
*
IndexUpdateProgress
))
(
err
error
)
{
func
(
index
*
Index
)
Update
(
ctx
context
.
Context
,
r
io
.
ReaderAt
,
topPos
int64
,
progress
func
(
*
IndexUpdateProgress
))
(
err
error
)
{
defer
xerr
.
Contextf
(
&
err
,
"%s: reindex %v..%v"
,
xio
.
Name
(
r
),
index
.
TopPos
,
topPos
)
defer
xerr
.
Contextf
(
&
err
,
"%s: reindex %v..%v"
,
xio
.
Name
(
r
),
index
.
TopPos
,
topPos
)
if
topPos
>=
0
&&
index
.
TopPos
>
topPos
{
if
topPos
>=
0
&&
index
.
TopPos
>
topPos
{
...
@@ -485,7 +472,7 @@ func (index *Index) Update(ctx context.Context, r io.ReaderAt, topPos int64, pro
...
@@ -485,7 +472,7 @@ func (index *Index) Update(ctx context.Context, r io.ReaderAt, topPos int64, pro
// check for topPos overlapping txn & whether we are done.
// check for topPos overlapping txn & whether we are done.
// topPos=-1 will never match here
// topPos=-1 will never match here
if
it
.
Txnh
.
Pos
<
topPos
&&
(
it
.
Txnh
.
Pos
+
it
.
Txnh
.
Len
)
>
topPos
{
if
it
.
Txnh
.
Pos
<
topPos
&&
(
it
.
Txnh
.
Pos
+
it
.
Txnh
.
Len
)
>
topPos
{
return
fmt
.
Errorf
(
"transaction %v @%v overlaps topPos boundary"
,
return
fmt
.
Errorf
(
"transaction %v @%v overlaps topPos boundary"
,
it
.
Txnh
.
Tid
,
it
.
Txnh
.
Pos
)
it
.
Txnh
.
Tid
,
it
.
Txnh
.
Pos
)
}
}
...
@@ -547,7 +534,7 @@ func BuildIndex(ctx context.Context, r io.ReaderAt, progress func(*IndexUpdatePr
...
@@ -547,7 +534,7 @@ func BuildIndex(ctx context.Context, r io.ReaderAt, progress func(*IndexUpdatePr
func
BuildIndexForFile
(
ctx
context
.
Context
,
path
string
,
progress
func
(
*
IndexUpdateProgress
))
(
index
*
Index
,
err
error
)
{
func
BuildIndexForFile
(
ctx
context
.
Context
,
path
string
,
progress
func
(
*
IndexUpdateProgress
))
(
index
*
Index
,
err
error
)
{
f
,
err
:=
os
.
Open
(
path
)
f
,
err
:=
os
.
Open
(
path
)
if
err
!=
nil
{
if
err
!=
nil
{
return
IndexNew
(),
err
// XXX add err ctx?
return
IndexNew
(),
err
}
}
defer
func
()
{
defer
func
()
{
...
@@ -637,7 +624,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int, progres
...
@@ -637,7 +624,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int, progres
wholeData
=
true
wholeData
=
true
break
break
}
}
return
oidChecked
,
err
// XXX err ctx
return
oidChecked
,
err
}
}
for
{
for
{
...
@@ -646,7 +633,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int, progres
...
@@ -646,7 +633,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int, progres
if
err
==
io
.
EOF
{
if
err
==
io
.
EOF
{
break
break
}
}
return
oidChecked
,
err
// XXX err ctx
return
oidChecked
,
err
}
}
// if oid was already checked - do not check index anymore
// if oid was already checked - do not check index anymore
...
@@ -700,7 +687,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int, progres
...
@@ -700,7 +687,7 @@ func (index *Index) Verify(ctx context.Context, r io.ReaderAt, ntxn int, progres
// VerifyForFile checks index correctness against FileStorage data in file @ path
// VerifyForFile checks index correctness against FileStorage data in file @ path
//
//
// See Verify for semantic description.
// See Verify for semantic description.
func
(
index
*
Index
)
VerifyForFile
(
ctx
context
.
Context
,
path
string
,
ntxn
int
,
progress
func
(
*
IndexVerifyProgress
))
(
oidChecked
map
[
zodb
.
Oid
]
struct
{},
err
error
)
{
func
(
index
*
Index
)
VerifyForFile
(
ctx
context
.
Context
,
path
string
,
ntxn
int
,
progress
func
(
*
IndexVerifyProgress
))
(
oidChecked
map
[
zodb
.
Oid
]
struct
{},
err
error
)
{
f
,
err
:=
os
.
Open
(
path
)
f
,
err
:=
os
.
Open
(
path
)
if
err
!=
nil
{
if
err
!=
nil
{
return
nil
,
err
return
nil
,
err
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment