Commit 51e0d65d authored by Kirill Smelkov's avatar Kirill Smelkov

X fix demo not to deadlock δ while it is verified

ZEO and NEO are not yet adjusted to new "watchq vs Close" requirement.
parent 2888bbbc
...@@ -32,6 +32,7 @@ import ( ...@@ -32,6 +32,7 @@ import (
"strings" "strings"
"sync" "sync"
"testing" "testing"
"time"
"lab.nexedi.com/kirr/go123/xerr" "lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/go123/xstrings" "lab.nexedi.com/kirr/go123/xstrings"
...@@ -467,6 +468,11 @@ func DrvTestWatch(t *testing.T, zurl string, zdrvOpen zodb.DriverOpener, bugv .. ...@@ -467,6 +468,11 @@ func DrvTestWatch(t *testing.T, zurl string, zdrvOpen zodb.DriverOpener, bugv ..
} }
} }
// commit something more and wait a bit to raise chances the driver enqueues to watchq<- .
_ = xcommit(at, ZRawObject{0, b("at the end")})
time.Sleep(1*time.Second)
// the driver must handle Close and cancel that watchq<-
err = zdrv.Close(); X(err) err = zdrv.Close(); X(err)
e, ok := <-watchq e, ok := <-watchq
......
// Copyright (C) 2017-2019 Nexedi SA and Contributors. // Copyright (C) 2017-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -46,12 +46,30 @@ type DriverOptions struct { ...@@ -46,12 +46,30 @@ type DriverOptions struct {
// //
// Watchq can be nil to ignore such events. However if Watchq != nil, the events // Watchq can be nil to ignore such events. However if Watchq != nil, the events
// have to be consumed or else the storage driver will misbehave - e.g. // have to be consumed or else the storage driver will misbehave - e.g.
// it can get out of sync with the on-disk database file. // it can get out of sync with the on-disk database file, or deadlock
// on any user-called operation.
// //
// The storage driver closes !nil Watchq when the driver is closed. // The storage driver closes !nil Watchq when the driver is closed.
// //
// The storage driver will send only and all events in (at₀, +∞] range, // The storage driver will send only and all events in (at₀, +∞] range,
// where at₀ is at returned by driver open. // where at₀ is at returned by driver open.
//
// The storage driver will stop sending events after call to Close.
// In particular the following example is valid and safe from deadlock:
//
// watchq := make(chan zodb.Event)
// stor, at0, err := zodb.OpenDriver(..., &DriverOptions{Watchq: watchq})
// defer stor.Close()
//
// for {
// select {
// case <-ctx.Done():
// return ctx.Err()
//
// case <-watchq:
// ...
// }
// }
Watchq chan<- Event Watchq chan<- Event
} }
...@@ -73,7 +91,10 @@ func RegisterDriver(scheme string, opener DriverOpener) { ...@@ -73,7 +91,10 @@ func RegisterDriver(scheme string, opener DriverOpener) {
driverRegistry[scheme] = opener driverRegistry[scheme] = opener
} }
// XXX ... // OpenDriver opens ZODB storage driver by URL.
//
// It is similar to Open but returns low-level IStorageDriver instead of IStorage.
// Most users should use Open.
func OpenDriver(ctx context.Context, zurl string, opt *DriverOptions) (_ IStorageDriver, at0 Tid, _ error) { func OpenDriver(ctx context.Context, zurl string, opt *DriverOptions) (_ IStorageDriver, at0 Tid, _ error) {
// no scheme -> file:// // no scheme -> file://
if !strings.Contains(zurl, ":") { if !strings.Contains(zurl, ":") {
...@@ -108,7 +129,7 @@ func OpenDriver(ctx context.Context, zurl string, opt *DriverOptions) (_ IStorag ...@@ -108,7 +129,7 @@ func OpenDriver(ctx context.Context, zurl string, opt *DriverOptions) (_ IStorag
// Users should import in storage packages they use or zodb/wks package to // Users should import in storage packages they use or zodb/wks package to
// get support for well-known storages. // get support for well-known storages.
// //
// Storage authors should register their storages with RegisterStorage. // Storage authors should register their storages with RegisterDriver.
func Open(ctx context.Context, zurl string, opt *OpenOptions) (IStorage, error) { func Open(ctx context.Context, zurl string, opt *OpenOptions) (IStorage, error) {
drvWatchq := make(chan Event) drvWatchq := make(chan Event)
drvOpt := &DriverOptions{ drvOpt := &DriverOptions{
...@@ -369,7 +390,7 @@ func (s *storage) _watcher() error { ...@@ -369,7 +390,7 @@ func (s *storage) _watcher() error {
func (s *storage) AddWatch(watchq chan<- Event) (at0 Tid) { func (s *storage) AddWatch(watchq chan<- Event) (at0 Tid) {
ack := make(chan Tid) ack := make(chan Tid)
select { select {
// no longer operational: behave if watchq was registered before that // no longer operational: behave as if watchq was registered before that
// and then seen down/close events. Interact with DelWatch directly. // and then seen down/close events. Interact with DelWatch directly.
case <-s.down: case <-s.down:
s.headMu.Lock() // shutdown may be due to Close call and watcher might be s.headMu.Lock() // shutdown may be due to Close call and watcher might be
......
...@@ -51,6 +51,7 @@ type Storage struct { ...@@ -51,6 +51,7 @@ type Storage struct {
baseAt0 zodb.Tid baseAt0 zodb.Tid
baseWatchq <-chan zodb.Event baseWatchq <-chan zodb.Event
δWatchq <-chan zodb.Event // nil if demo is opened with watchq=nil δWatchq <-chan zodb.Event // nil if demo is opened with watchq=nil
δWatchq0 <-chan zodb.Event // buffer for δ events queued while δ was initially verified
watchq chan<- zodb.Event // user requested to deliver events here watchq chan<- zodb.Event // user requested to deliver events here
watchWG *xsync.WorkGroup watchWG *xsync.WorkGroup
...@@ -59,6 +60,9 @@ type Storage struct { ...@@ -59,6 +60,9 @@ type Storage struct {
downOnce sync.Once downOnce sync.Once
down chan struct{} // ready when storage is down down chan struct{} // ready when storage is down
downErr error // reason for shutdown downErr error // reason for shutdown
closeOnce sync.Once
closed chan struct{} // ready when storage is Closed
} }
// baseMutatedError is reported when Storage.base is detected to change. // baseMutatedError is reported when Storage.base is detected to change.
...@@ -91,6 +95,13 @@ func (d *Storage) watcher(ctx context.Context) error { ...@@ -91,6 +95,13 @@ func (d *Storage) watcher(ctx context.Context) error {
} }
for { for {
var δWatchq <-chan zodb.Event
if len(d.δWatchq0) != 0 {
δWatchq = d.δWatchq0
} else {
δWatchq = d.δWatchq
}
select { select {
// Close requests to stop watching // Close requests to stop watching
case <-ctx.Done(): case <-ctx.Done():
...@@ -117,22 +128,33 @@ func (d *Storage) watcher(ctx context.Context) error { ...@@ -117,22 +128,33 @@ func (d *Storage) watcher(ctx context.Context) error {
} }
d.shutdown(edown) d.shutdown(edown)
ev := &zodb.EventError{&zodb.OpError{URL: d.URL(), Op: "watch", Err: edown}} ev := &zodb.EventError{&zodb.OpError{URL: d.URL(), Op: "watcher", Err: edown}}
if d.watchq != nil { if d.watchq != nil {
d.watchq <- ev // XXX + select on close ? select {
case <-d.closed:
// wakeup to return edown
case d.watchq <- ev:
// ok
}
} }
return edown return edown
// event on δ -> proxy to user // event on δ -> proxy to user
case event, ok := <-d.δWatchq: case event, ok := <-δWatchq:
if !ok { if !ok {
// δ closed // δ closed
d.δWatchq = nil d.δWatchq = nil
continue continue
} }
// XXX +select on close select {
d.watchq <- event // !nil because d.δWatchq != nil case <-d.closed:
return nil
case d.watchq <- event: // !nil because d.δWatchq != nil
// ok
}
} }
} }
} }
...@@ -156,6 +178,9 @@ func (d *Storage) Close() (err error) { ...@@ -156,6 +178,9 @@ func (d *Storage) Close() (err error) {
}() }()
d.shutdown(errClosed) d.shutdown(errClosed)
d.closeOnce.Do(func() {
close(d.closed)
})
errδ := d.δ.Close() errδ := d.δ.Close()
errBase := d.base.Close() errBase := d.base.Close()
...@@ -343,34 +368,68 @@ func openByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (_ zodb ...@@ -343,34 +368,68 @@ func openByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (_ zodb
} }
}() }()
// XXX read watchqs while we verify? (not to deadlock)
// verify that either // verify that either
// - δ is all empty (just created), or // - δ is all empty (just created), or
// - all δ transactions come strictly after base. // - all δ transactions come strictly after base.
at0 := baseAt0 at0 := baseAt0
var δEventq0 []zodb.Event
if δAt0 != 0 { if δAt0 != 0 {
if δAt0 < baseAt0 { if δAt0 < baseAt0 {
return nil, zodb.InvalidTid, fmt.Errorf("base is ahead of δ: base.head=%s δ.head=%s", baseAt0, δAt0) return nil, zodb.InvalidTid, fmt.Errorf("base is ahead of δ: base.head=%s δ.head=%s", baseAt0, δAt0)
} }
:= δ.Iterate(ctx, 0, baseAt0) // read and queue data from δWatchq while we verify δ (not to deadlock δ driver)
δq0Stop := make(chan struct{}) // reader <- main "stop"
δq0Done := make(chan struct{}) // reader -> main "done"
go func() {
defer close(δq0Done)
for { for {
select {
case <-δq0Stop:
return
case δevent, ok := <-δWatchq:
if !ok {
return
}
δEventq0 = append(δEventq0, δevent)
}
}
}()
// verify δ
:= δ.Iterate(ctx, 0, baseAt0)
δtxni, _, err := .NextTxn(ctx) δtxni, _, err := .NextTxn(ctx)
if err == io.EOF { // TODO iδ.Close()
break // all ok - nothing in δ switch {
case err == io.EOF:
err = nil // ok - nothing in δ
case err == nil:
// there is a δ transaction ∈ [δAt0, baseAt0)
err = fmt.Errorf("base overlaps with δ: base.head=%s δ.tail=%s", baseAt0, δtxni.Tid)
} }
close(δq0Stop)
<-δq0Done
if err != nil { if err != nil {
return nil, zodb.InvalidTid, err return nil, zodb.InvalidTid, err
} }
// there is a δ transaction ∈ [δAt0, baseAt0)
// TODO iδ.Close()
return nil, zodb.InvalidTid, fmt.Errorf("base overlaps with δ: base.head=%s δ.tail=%s", baseAt0, δtxni.Tid)
}
at0 = δAt0 at0 = δAt0
} }
// requeue δWatchq0 <- δEventq0
var δWatchq0 chan zodb.Event
if l := len(δEventq0); l != 0 {
δWatchq0 = make(chan zodb.Event, l)
for _, ev := range δEventq0 {
δWatchq0 <- ev
}
}
d := &Storage{ d := &Storage{
base: base, base: base,
δ : δ, δ : δ,
...@@ -378,9 +437,11 @@ func openByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (_ zodb ...@@ -378,9 +437,11 @@ func openByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (_ zodb
baseAt0: baseAt0, baseAt0: baseAt0,
baseWatchq: baseWatchq, baseWatchq: baseWatchq,
δWatchq: δWatchq, δWatchq: δWatchq,
δWatchq0: δWatchq0,
watchq: opt.Watchq, watchq: opt.Watchq,
down: make(chan struct{}), down: make(chan struct{}),
closed: make(chan struct{}),
} }
// spawn watcher to listen on baseWatchq and shutdown storage if base changes. // spawn watcher to listen on baseWatchq and shutdown storage if base changes.
......
...@@ -228,17 +228,20 @@ func TestWatchLoad_vs_BaseMutate(t *testing.T) { ...@@ -228,17 +228,20 @@ func TestWatchLoad_vs_BaseMutate(t *testing.T) {
tid, err := ddat.MutateBase(); X(err) tid, err := ddat.MutateBase(); X(err)
// first wait for error from watchq // first wait for error from watchq
event := <-watchq event, ok := <-watchq
if !ok {
t.Fatal("after base mutate: premature watchq close")
}
evErr, ok := event.(*zodb.EventError) evErr, ok := event.(*zodb.EventError)
if !ok { if !ok {
t.Fatalf("unexpected event: %T", event) t.Fatalf("after base mutate: unexpected event: %T", event)
} }
errBaseMutated := &baseMutatedError{ errBaseMutated := &baseMutatedError{
baseAt0: 0, baseAt0: 0,
baseHead: tid, baseHead: tid,
} }
evErrOk := &zodb.EventError{&zodb.OpError{URL: ddrv.URL(), Op: "watch", Err: errBaseMutated}} evErrOk := &zodb.EventError{&zodb.OpError{URL: ddrv.URL(), Op: "watcher", Err: errBaseMutated}}
if !reflect.DeepEqual(evErr, evErrOk) { if !reflect.DeepEqual(evErr, evErrOk) {
t.Fatalf("after base mutate: unexpected event:\nhave: %s\nwant: %s", evErr, evErrOk) t.Fatalf("after base mutate: unexpected event:\nhave: %s\nwant: %s", evErr, evErrOk)
} }
......
...@@ -109,6 +109,9 @@ type FileStorage struct { ...@@ -109,6 +109,9 @@ type FileStorage struct {
downOnce sync.Once // shutdown may be due to both Close and IO error in watcher downOnce sync.Once // shutdown may be due to both Close and IO error in watcher
errClose error // error from .file.Close() errClose error // error from .file.Close()
watchWg sync.WaitGroup // to wait for watcher finish watchWg sync.WaitGroup // to wait for watcher finish
closed chan struct{} // ready when storage was Closed
closeOnce sync.Once
} }
// IStorageDriver // IStorageDriver
...@@ -468,7 +471,7 @@ func (fs *FileStorage) watcher(w *fsnotify.Watcher, errFirstRead chan<- error) { ...@@ -468,7 +471,7 @@ func (fs *FileStorage) watcher(w *fsnotify.Watcher, errFirstRead chan<- error) {
// XXX it can also be internal.poll.ErrFileClosing // XXX it can also be internal.poll.ErrFileClosing
e.Err.Error() == "use of closed file") { e.Err.Error() == "use of closed file") {
select { select {
case <-fs.down: case <-fs.closed:
err = nil err = nil
default: default:
} }
...@@ -485,7 +488,13 @@ func (fs *FileStorage) watcher(w *fsnotify.Watcher, errFirstRead chan<- error) { ...@@ -485,7 +488,13 @@ func (fs *FileStorage) watcher(w *fsnotify.Watcher, errFirstRead chan<- error) {
if fs.watchq != nil { if fs.watchq != nil {
if err != nil { if err != nil {
fs.watchq <- &zodb.EventError{err} select {
case <-fs.closed:
// closed - skip send to watchq
case fs.watchq <- &zodb.EventError{err}:
// ok
}
} }
close(fs.watchq) close(fs.watchq)
} }
...@@ -535,9 +544,9 @@ mainloop: ...@@ -535,9 +544,9 @@ mainloop:
if !first { if !first {
traceWatch("select ...") traceWatch("select ...")
select { select {
case <-fs.down: case <-fs.closed:
// closed // closed
traceWatch("down") traceWatch("closed")
return nil return nil
case err := <-w.Errors: case err := <-w.Errors:
...@@ -694,7 +703,7 @@ mainloop: ...@@ -694,7 +703,7 @@ mainloop:
// notify client // notify client
if fs.watchq != nil { if fs.watchq != nil {
select { select {
case <-fs.down: case <-fs.closed:
return nil return nil
case fs.watchq <- &zodb.EventCommit{it.Txnh.Tid, δoid}: case fs.watchq <- &zodb.EventCommit{it.Txnh.Tid, δoid}:
...@@ -772,6 +781,9 @@ func (fs *FileStorage) shutdown(reason error) { ...@@ -772,6 +781,9 @@ func (fs *FileStorage) shutdown(reason error) {
} }
func (fs *FileStorage) Close() error { func (fs *FileStorage) Close() error {
fs.closeOnce.Do(func() {
close(fs.closed)
})
fs.shutdown(fmt.Errorf("closed")) fs.shutdown(fmt.Errorf("closed"))
fs.watchWg.Wait() fs.watchWg.Wait()
...@@ -794,6 +806,7 @@ func Open(ctx context.Context, path string, opt *zodb.DriverOptions) (_ *FileSto ...@@ -794,6 +806,7 @@ func Open(ctx context.Context, path string, opt *zodb.DriverOptions) (_ *FileSto
fs := &FileStorage{ fs := &FileStorage{
watchq: opt.Watchq, watchq: opt.Watchq,
down: make(chan struct{}), down: make(chan struct{}),
closed: make(chan struct{}),
} }
f, err := os.Open(path) f, err := os.Open(path)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment