Skip to content

W
wendelin.core
Commit e7d161d1 authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

Restore Python3 support 

Since wendelin.core beginning both py2 and py3 were supported. But
during WCFS development I focused on WCFS itself instead of trying to
support two major py versions, and so py2 remained to work while py3
became broken a bit.

-> Finally fix that, mostly around code in wcfs.

Now all wendelin.core tests pass with both Python2 and Python3 except
test_zstor_2zurl which on py3 fails on NEO-related part because NEO is
not yet ported to Python3.

Hereby changes were partly co-authored with and motivated by preliminary
work by Carlos Ramos Carreño on nexedi/wendelin.core!27.

/proposed-for-review-at nexedi/wendelin.core!27 (comment 217042)
parents db6fea3d 96b26c2a
Hide whitespace changes
Inline Side-by-side
Showing with 284 additions and 165 deletions
bigfile/file_zodb.py
View file @ e7d161d1
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Wendelin.bigfile | BigFile ZODB backend # Wendelin.bigfile | BigFile ZODB backend
# Copyright (C) 2014-2024 Nexedi SA and Contributors. # Copyright (C) 2014-2025 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
...@@ -201,6 +201,8 @@ class ZBlkBase(Persistent): ...@@ -201,6 +201,8 @@ class ZBlkBase(Persistent):
# client requests us to set blkdata to be later saved to DB # client requests us to set blkdata to be later saved to DB
# (DB <- ) .blkdata <- memory-page # (DB <- ) .blkdata <- memory-page
# #
# buf: buffer object, or, for convenience anything with buffer interface
#
# return: blkchanged=(True|False) - whether blk should be considered changed # return: blkchanged=(True|False) - whether blk should be considered changed
# after setting its data. # after setting its data.
# #
...@@ -291,7 +293,8 @@ class ZBlk0(ZBlkBase): ...@@ -291,7 +293,8 @@ class ZBlk0(ZBlkBase):
# (DB <- ) ._v_blkdata <- memory-page # (DB <- ) ._v_blkdata <- memory-page
def setblkdata(self, buf): def setblkdata(self, buf):
blkdata = bytes(buf) # FIXME does memcpy buf = memoryview(buf)
blkdata = buf.tobytes() # FIXME does memcpy
# trim trailing \0 # trim trailing \0
self._v_blkdata = blkdata.rstrip(b'\0') # FIXME copy self._v_blkdata = blkdata.rstrip(b'\0') # FIXME copy
...@@ -407,6 +410,7 @@ class ZBlk1(ZBlkBase): ...@@ -407,6 +410,7 @@ class ZBlk1(ZBlkBase):
# (DB <- ) .chunktab <- memory-page # (DB <- ) .chunktab <- memory-page
def setblkdata(self, buf): def setblkdata(self, buf):
buf = memoryview(buf)
chunktab = self.chunktab chunktab = self.chunktab
CHUNKSIZE = self.CHUNKSIZE CHUNKSIZE = self.CHUNKSIZE
blkchanged= False blkchanged= False
...@@ -420,11 +424,10 @@ class ZBlk1(ZBlkBase): ...@@ -420,11 +424,10 @@ class ZBlk1(ZBlkBase):
# scan over buf and update/delete changed chunks # scan over buf and update/delete changed chunks
for start in range(0, len(buf), CHUNKSIZE): for start in range(0, len(buf), CHUNKSIZE):
data = buf[start:start+CHUNKSIZE] # FIXME copy on py2 data = buf[start:start+CHUNKSIZE]
# make sure data is bytes # make sure data is bytes
# (else we cannot .rstrip() it below) # (else we cannot .rstrip() it below)
if not isinstance(data, bytes): data = data.tobytes() # FIXME copy
data = bytes(data) # FIXME copy on py3
# trim trailing \0 # trim trailing \0
data = data.rstrip(b'\0') # FIXME copy data = data.rstrip(b'\0') # FIXME copy
chunk = chunktab.get(start) chunk = chunktab.get(start)
......
lib/testing.py
View file @ e7d161d1
# Wendelin. Testing utilities # Wendelin. Testing utilities
# Copyright (C) 2014-2021 Nexedi SA and Contributors. # Copyright (C) 2014-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
...@@ -249,16 +249,11 @@ class TestDB_ZEO(TestDB_Base): ...@@ -249,16 +249,11 @@ class TestDB_ZEO(TestDB_Base):
self.zeo_forker = forker self.zeo_forker = forker
# .z5 represents whether we are running with ZEO5 or earlier # .z5 represents whether we are running with ZEO5 or earlier
dzodb3 = pkg_resources.working_set.find(pkg_resources.Requirement.parse('ZODB3'))
dzeo = pkg_resources.working_set.find(pkg_resources.Requirement.parse('ZEO')) dzeo = pkg_resources.working_set.find(pkg_resources.Requirement.parse('ZEO'))
v5 = pkg_resources.parse_version('5.0dev') v5 = pkg_resources.parse_version('5.0dev')
v311 = pkg_resources.parse_version('3.11dev')
if dzodb3 is not None and dzodb3.parsed_version < v311: assert dzeo is not None
self.z5 = False # ZODB 3.11 just requires latest ZODB & ZEO self.z5 = (dzeo.parsed_version >= v5)
else:
assert dzeo is not None
self.z5 = (dzeo.parsed_version >= v5)
def setup(self): def setup(self):
......
wcfs/client/_wcfs.pyx
View file @ e7d161d1
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2018-2021 Nexedi SA and Contributors. # Copyright (C) 2018-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
...@@ -26,6 +26,7 @@ ...@@ -26,6 +26,7 @@
# See _wcfs.pxd for package overview. # See _wcfs.pxd for package overview.
from golang cimport pychan, pyerror, nil from golang cimport pychan, pyerror, nil
from golang import b as pyb # TODO cimport directly after https://lab.nexedi.com/nexedi/pygolang/-/merge_requests/21 is merged
from golang cimport io from golang cimport io
cdef extern from *: cdef extern from *:
...@@ -40,9 +41,9 @@ cdef class PyWCFS: ...@@ -40,9 +41,9 @@ cdef class PyWCFS:
property mountpoint: property mountpoint:
def __get__(PyWCFS pywc): def __get__(PyWCFS pywc):
return pywc.wc.mountpoint return str(pyb(pywc.wc.mountpoint)) # TODO remove str(·) after bstr can be mixed with unicode in os.path.join
def __set__(PyWCFS pywc, string v): def __set__(PyWCFS pywc, v):
pywc.wc.mountpoint = v pywc.wc.mountpoint = pyb(v)
def connect(PyWCFS pywc, pyat): # -> PyConn def connect(PyWCFS pywc, pyat): # -> PyConn
cdef Tid at = u64(pyat) cdef Tid at = u64(pyat)
...@@ -197,7 +198,8 @@ cdef class PyWatchLink: ...@@ -197,7 +198,8 @@ cdef class PyWatchLink:
raise pyerr(err) raise pyerr(err)
def sendReq(PyWatchLink pywlink, context.PyContext pyctx, string req): # -> reply(string) def sendReq(PyWatchLink pywlink, context.PyContext pyctx, pyreq): # -> reply(bstr)
cdef string req = pyb(pyreq)
with nogil: with nogil:
_ = wlink_sendReq_pyexc(pywlink.wlink, pyctx.ctx, req) _ = wlink_sendReq_pyexc(pywlink.wlink, pyctx.ctx, req)
reply = _.first reply = _.first
...@@ -206,7 +208,7 @@ cdef class PyWatchLink: ...@@ -206,7 +208,7 @@ cdef class PyWatchLink:
if err != nil: if err != nil:
raise pyerr(err) raise pyerr(err)
return reply return pyb(reply)
def recvReq(PyWatchLink pywlink, context.PyContext pyctx): # -> PinReq | None when EOF def recvReq(PyWatchLink pywlink, context.PyContext pyctx): # -> PinReq | None when EOF
cdef PyPinReq pyreq = PyPinReq.__new__(PyPinReq) cdef PyPinReq pyreq = PyPinReq.__new__(PyPinReq)
...@@ -220,7 +222,8 @@ cdef class PyWatchLink: ...@@ -220,7 +222,8 @@ cdef class PyWatchLink:
return pyreq return pyreq
def replyReq(PyWatchLink pywlink, context.PyContext pyctx, PyPinReq pyreq, string reply): def replyReq(PyWatchLink pywlink, context.PyContext pyctx, PyPinReq pyreq, pyreply):
cdef string reply = pyb(pyreply)
with nogil: with nogil:
err = wlink_replyReq_pyexc(pywlink.wlink, pyctx.ctx, &pyreq.pinreq, reply) err = wlink_replyReq_pyexc(pywlink.wlink, pyctx.ctx, &pyreq.pinreq, reply)
...@@ -259,11 +262,11 @@ cdef class PyPinReq: ...@@ -259,11 +262,11 @@ cdef class PyPinReq:
# wcfs_test.py uses req.msg in several places # wcfs_test.py uses req.msg in several places
property msg: property msg:
def __get__(PyPinReq pypin): def __get__(PyPinReq pypin):
return pypin.pinreq.msg return pyb(pypin.pinreq.msg)
def _tpywlinkwrite(PyWatchLink pywlink, bytes pypkt): def _tpywlinkwrite(PyWatchLink pywlink, pypkt):
cdef string pkt = pypkt cdef string pkt = pyb(pypkt)
with nogil: with nogil:
err = _twlinkwrite_pyexc(pywlink.wlink, pkt) err = _twlinkwrite_pyexc(pywlink.wlink, pkt)
if err != nil: if err != nil:
......
wcfs/go.mod
View file @ e7d161d1
...@@ -6,32 +6,35 @@ require ( ...@@ -6,32 +6,35 @@ require (
github.com/golang/glog v1.2.2 github.com/golang/glog v1.2.2
github.com/hanwen/go-fuse/v2 v2.4.2 // replaced to -> kirr/go-fuse@y/nodefs-cancel github.com/hanwen/go-fuse/v2 v2.4.2 // replaced to -> kirr/go-fuse@y/nodefs-cancel
github.com/johncgriffin/overflow v0.0.0-20211019200055-46fa312c352c github.com/johncgriffin/overflow v0.0.0-20211019200055-46fa312c352c
github.com/kisielk/og-rek v1.2.0 github.com/kisielk/og-rek v1.2.1-0.20240923165241-e691997e3596
github.com/pkg/errors v0.9.1 github.com/pkg/errors v0.9.1
github.com/shirou/gopsutil/v4 v4.24.8 github.com/shirou/gopsutil/v4 v4.24.8
github.com/stretchr/testify v1.9.0 github.com/stretchr/testify v1.9.0
lab.nexedi.com/kirr/go123 v0.0.0-20230822135329-95433de34faf lab.nexedi.com/kirr/go123 v0.0.0-20240626173136-48920809d24c
lab.nexedi.com/kirr/neo/go v0.0.0-20240918161556-efde5253dc81 lab.nexedi.com/kirr/neo/go v0.0.0-20240924102820-6235fb602308
) )
require ( require (
crawshaw.io/sqlite v0.3.2 // indirect crawshaw.io/sqlite v0.3.2 // indirect
github.com/DataDog/czlib v0.0.0-20210322182103-8087f4e14ae7 // indirect github.com/DataDog/czlib v0.0.0-20210322182103-8087f4e14ae7 // indirect
github.com/aristanetworks/gomap v0.0.0-20230726210543-f4e41046dced // indirect
github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 // indirect github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-ole/go-ole v1.2.6 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/philhofer/fwd v1.1.1 // indirect github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/shamaton/msgpack v1.2.1 // indirect github.com/shamaton/msgpack v1.2.1 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/someonegg/gocontainer v1.0.0 // indirect github.com/someonegg/gocontainer v1.0.0 // indirect
github.com/tinylib/msgp v1.1.6 // indirect github.com/tinylib/msgp v1.2.0 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect github.com/tklauser/numcpus v0.6.1 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect
go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6 // indirect
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 // indirect
golang.org/x/sys v0.24.0 // indirect golang.org/x/sys v0.24.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect
) )
......
wcfs/go.sum
View file @ e7d161d1
...@@ -7,6 +7,8 @@ crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4= ...@@ -7,6 +7,8 @@ crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/DataDog/czlib v0.0.0-20210322182103-8087f4e14ae7 h1:6ZJZdzkbvKb6HRXmZ12ICZ0IbqfR+0Cd2C0IutWHHIA= github.com/DataDog/czlib v0.0.0-20210322182103-8087f4e14ae7 h1:6ZJZdzkbvKb6HRXmZ12ICZ0IbqfR+0Cd2C0IutWHHIA=
github.com/DataDog/czlib v0.0.0-20210322182103-8087f4e14ae7/go.mod h1:ROY4muaTWpoeQAx/oUkvxe9zKCmgU5xDGXsfEbA+omc= github.com/DataDog/czlib v0.0.0-20210322182103-8087f4e14ae7/go.mod h1:ROY4muaTWpoeQAx/oUkvxe9zKCmgU5xDGXsfEbA+omc=
github.com/aristanetworks/gomap v0.0.0-20230726210543-f4e41046dced h1:HxlRMDx/VeRqzj3nvqX9k4tjeBcEIkoNHDJPsS389hs=
github.com/aristanetworks/gomap v0.0.0-20230726210543-f4e41046dced/go.mod h1:p7lmI+ecoe1RTyD11SPXWsSQ3H+pJ4cp5y7vtKW4QdM=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
...@@ -27,6 +29,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m ...@@ -27,6 +29,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
...@@ -64,6 +68,8 @@ github.com/johncgriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:2n/HCxBM7 ...@@ -64,6 +68,8 @@ github.com/johncgriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:2n/HCxBM7
github.com/johncgriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:B9OPZOhZ3FIi6bu54lAgCMzXLh11Z7ilr3rOr/ClP+E= github.com/johncgriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:B9OPZOhZ3FIi6bu54lAgCMzXLh11Z7ilr3rOr/ClP+E=
github.com/kisielk/og-rek v1.2.0 h1:CTvDIin+YnetsSQAYbe+QNAxXU3B50C5hseEz8xEoJw= github.com/kisielk/og-rek v1.2.0 h1:CTvDIin+YnetsSQAYbe+QNAxXU3B50C5hseEz8xEoJw=
github.com/kisielk/og-rek v1.2.0/go.mod h1:6ihsOSzSAxR/65S3Bn9zNihoEqRquhDQZ2c6I2+MG3c= github.com/kisielk/og-rek v1.2.0/go.mod h1:6ihsOSzSAxR/65S3Bn9zNihoEqRquhDQZ2c6I2+MG3c=
github.com/kisielk/og-rek v1.2.1-0.20240923165241-e691997e3596 h1:m4FSNNEFnhXcUfaMmPphtrIGPhuGdxmVS744izSg9uI=
github.com/kisielk/og-rek v1.2.1-0.20240923165241-e691997e3596/go.mod h1:4at7oxyfBTDilURhNCf7irHWtosJlJl9uyqUqAkrP4w=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
...@@ -80,6 +86,8 @@ github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vyg ...@@ -80,6 +86,8 @@ github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vyg
github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ= github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ=
github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 h1:jYi87L8j62qkXzaYHAQAhEapgukhenIMZRBKTNRLHJ4=
github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
...@@ -107,6 +115,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT ...@@ -107,6 +115,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw= github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw=
github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw= github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
github.com/tinylib/msgp v1.2.0 h1:0uKB/662twsVBpYUPbokj4sTSKhWFKB7LopO2kWK8lY=
github.com/tinylib/msgp v1.2.0/go.mod h1:2vIGs3lcUo8izAATNobrCHevYZC/LMsJtw4JPiYPHro=
github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
...@@ -115,10 +125,14 @@ github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+l ...@@ -115,10 +125,14 @@ github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+l
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6 h1:lGdhQUN/cnWdSH3291CUuxSEqc+AsGTiDxPP3r2J0l4=
go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
...@@ -217,9 +231,13 @@ lab.nexedi.com/kirr/go-fuse/v2 v2.4.2-0.20231211215333-9f9ad4a1c7cc/go.mod h1:xK ...@@ -217,9 +231,13 @@ lab.nexedi.com/kirr/go-fuse/v2 v2.4.2-0.20231211215333-9f9ad4a1c7cc/go.mod h1:xK
lab.nexedi.com/kirr/go123 v0.0.0-20211124154638-01e8697d1901/go.mod h1:pwDpdCuvtz0QxisDzV/z9eUb9zc/rMQec520h4i8VWQ= lab.nexedi.com/kirr/go123 v0.0.0-20211124154638-01e8697d1901/go.mod h1:pwDpdCuvtz0QxisDzV/z9eUb9zc/rMQec520h4i8VWQ=
lab.nexedi.com/kirr/go123 v0.0.0-20230822135329-95433de34faf h1:UwAEraoydFHxDqudGYtxBpyXbt9SqInI657OKf9VMJc= lab.nexedi.com/kirr/go123 v0.0.0-20230822135329-95433de34faf h1:UwAEraoydFHxDqudGYtxBpyXbt9SqInI657OKf9VMJc=
lab.nexedi.com/kirr/go123 v0.0.0-20230822135329-95433de34faf/go.mod h1:pwDpdCuvtz0QxisDzV/z9eUb9zc/rMQec520h4i8VWQ= lab.nexedi.com/kirr/go123 v0.0.0-20230822135329-95433de34faf/go.mod h1:pwDpdCuvtz0QxisDzV/z9eUb9zc/rMQec520h4i8VWQ=
lab.nexedi.com/kirr/go123 v0.0.0-20240626173136-48920809d24c h1:GboU09uDjU09xORiCgCco5hPI4pmMOw6Qye2hJNal70=
lab.nexedi.com/kirr/go123 v0.0.0-20240626173136-48920809d24c/go.mod h1:pwDpdCuvtz0QxisDzV/z9eUb9zc/rMQec520h4i8VWQ=
lab.nexedi.com/kirr/neo/go v0.0.0-20240723085959-839ee634bd66 h1:xAQcab3p0CiJ36aLqNGB8kH6hpsalgOnEL9D5CZgoN0= lab.nexedi.com/kirr/neo/go v0.0.0-20240723085959-839ee634bd66 h1:xAQcab3p0CiJ36aLqNGB8kH6hpsalgOnEL9D5CZgoN0=
lab.nexedi.com/kirr/neo/go v0.0.0-20240723085959-839ee634bd66/go.mod h1:0Wk1qKrdjMWr8njsuBIbgPMBNjPlNFozuBDe15Lqq6A= lab.nexedi.com/kirr/neo/go v0.0.0-20240723085959-839ee634bd66/go.mod h1:0Wk1qKrdjMWr8njsuBIbgPMBNjPlNFozuBDe15Lqq6A=
lab.nexedi.com/kirr/neo/go v0.0.0-20240806095154-6fb93a602cbe h1:E8qMqhZJlVtsNTzVxdj2zsDq8XJx/QOBODGhid9gqxM= lab.nexedi.com/kirr/neo/go v0.0.0-20240806095154-6fb93a602cbe h1:E8qMqhZJlVtsNTzVxdj2zsDq8XJx/QOBODGhid9gqxM=
lab.nexedi.com/kirr/neo/go v0.0.0-20240806095154-6fb93a602cbe/go.mod h1:0Wk1qKrdjMWr8njsuBIbgPMBNjPlNFozuBDe15Lqq6A= lab.nexedi.com/kirr/neo/go v0.0.0-20240806095154-6fb93a602cbe/go.mod h1:0Wk1qKrdjMWr8njsuBIbgPMBNjPlNFozuBDe15Lqq6A=
lab.nexedi.com/kirr/neo/go v0.0.0-20240918161556-efde5253dc81 h1:eleDgtjbfZ2jz89tyyJCQAjFC1d6lGtzYLREpCORfrI= lab.nexedi.com/kirr/neo/go v0.0.0-20240918161556-efde5253dc81 h1:eleDgtjbfZ2jz89tyyJCQAjFC1d6lGtzYLREpCORfrI=
lab.nexedi.com/kirr/neo/go v0.0.0-20240918161556-efde5253dc81/go.mod h1:YzPDKaJIPNRCvRrpLDHP54ggZbLKlH3lErGX8NILtCQ= lab.nexedi.com/kirr/neo/go v0.0.0-20240918161556-efde5253dc81/go.mod h1:YzPDKaJIPNRCvRrpLDHP54ggZbLKlH3lErGX8NILtCQ=
lab.nexedi.com/kirr/neo/go v0.0.0-20240924102820-6235fb602308 h1:KMS9m8EbI05yKK/482Q1+DrzJyzD1pTSMHsZWwXczPA=
lab.nexedi.com/kirr/neo/go v0.0.0-20240924102820-6235fb602308/go.mod h1:P7HdxcOKE4FUf2/KI7UyfjRmQ97YYcXaoYQog7rsJNs=
wcfs/internal/pycompat/pycompat.go deleted 100644 → 0
View file @ db6fea3d
// XXX dup from neo/go/zodb/internal/pickletools
package pycompat
import (
"math/big"
)
// Int64 tries to convert unpickled Python value to int64.
//
// (ogórek decodes python long as big.Int)
//
// XXX + support for float?
func Int64(xv interface{}) (v int64, ok bool) {
switch v := xv.(type) {
case int64:
return v, true
case *big.Int:
if v.IsInt64() {
return v.Int64(), true
}
}
return 0, false
}
wcfs/internal/wcfs_test.pyx
View file @ e7d161d1
...@@ -54,7 +54,7 @@ cdef class _tWCFS: ...@@ -54,7 +54,7 @@ cdef class _tWCFS:
# but, if _abort_ontimeout uses GIL, won't continue to run trying to lock # but, if _abort_ontimeout uses GIL, won't continue to run trying to lock
# GIL -> deadlock. # GIL -> deadlock.
def _abort_ontimeout(_tWCFS t, int fdabort, double dt, pychan timeoutch not None, pychan nogilready not None): def _abort_ontimeout(_tWCFS t, int fdabort, double dt, pychan timeoutch not None, pychan nogilready not None):
emsg1 = "\nC: test timed out after %.1fs\n" % (dt / time.second) emsg1 = b"\nC: test timed out after %.1fs\n" % (dt / time.second)
cdef char *_emsg1 = emsg1 cdef char *_emsg1 = emsg1
with nogil: with nogil:
# tell main thread that we entered nogil world # tell main thread that we entered nogil world
......
wcfs/internal/xbtree/xbtreetest/treeenv.go
View file @ e7d161d1
// Copyright (C) 2020-2021 Nexedi SA and Contributors. // Copyright (C) 2020-2024 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -26,6 +26,7 @@ import ( ...@@ -26,6 +26,7 @@ import (
"sort" "sort"
"testing" "testing"
pickle "github.com/kisielk/og-rek"
"lab.nexedi.com/kirr/go123/exc" "lab.nexedi.com/kirr/go123/exc"
"lab.nexedi.com/kirr/neo/go/transaction" "lab.nexedi.com/kirr/neo/go/transaction"
"lab.nexedi.com/kirr/neo/go/zodb" "lab.nexedi.com/kirr/neo/go/zodb"
...@@ -289,7 +290,7 @@ func xGetBlkTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]ZBlkInfo { ...@@ -289,7 +290,7 @@ func xGetBlkTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]ZBlkInfo {
err = zroot.PActivate(ctx); X(err) err = zroot.PActivate(ctx); X(err)
defer zroot.PDeactivate() defer zroot.PDeactivate()
xzblkdir, ok := zroot.Data["treegen/values"] xzblkdir, ok := zroot.Get_("treegen/values")
if !ok { if !ok {
exc.Raisef("root['treegen/values'] missing") exc.Raisef("root['treegen/values'] missing")
} }
...@@ -301,10 +302,10 @@ func xGetBlkTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]ZBlkInfo { ...@@ -301,10 +302,10 @@ func xGetBlkTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]ZBlkInfo {
err = zblkdir.PActivate(ctx); X(err) err = zblkdir.PActivate(ctx); X(err)
defer zblkdir.PDeactivate() defer zblkdir.PDeactivate()
for xname, xzblk := range zblkdir.Data { zblkdir.Iter()(func(xname, xzblk any) bool {
name, ok := xname.(string) name, err := pickle.AsString(xname)
if !ok { if err != nil {
exc.Raisef("root['treegen/values']: key [%q]: expected str, got %T", xname, xname) exc.Raisef("root['treegen/values']: key [%q]: %s", xname, err)
} }
zblk, ok := xzblk.(zodb.IPersistent) zblk, ok := xzblk.(zodb.IPersistent)
...@@ -315,7 +316,9 @@ func xGetBlkTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]ZBlkInfo { ...@@ -315,7 +316,9 @@ func xGetBlkTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]ZBlkInfo {
oid := zblk.POid() oid := zblk.POid()
data := xzgetBlkData(ctx, zconn, oid) data := xzgetBlkData(ctx, zconn, oid)
blkTab[oid] = ZBlkInfo{name, data} blkTab[oid] = ZBlkInfo{name, data}
}
return true
})
return blkTab return blkTab
} }
......
wcfs/internal/xbtree/xbtreetest/treegen.py
View file @ e7d161d1
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2020-2021 Nexedi SA and Contributors. # Copyright (C) 2020-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
...@@ -133,7 +133,7 @@ by `treegen trees`: ...@@ -133,7 +133,7 @@ by `treegen trees`:
from __future__ import print_function, absolute_import from __future__ import print_function, absolute_import
import sys import sys
from golang import func, defer, panic from golang import func, defer, panic, b
from golang import time from golang import time
from ZODB import DB from ZODB import DB
from ZODB.Connection import Connection from ZODB.Connection import Connection
...@@ -174,6 +174,21 @@ BTrees.LOBTree.LOBTree = XLOTree ...@@ -174,6 +174,21 @@ BTrees.LOBTree.LOBTree = XLOTree
from BTrees.LOBTree import LOBTree from BTrees.LOBTree import LOBTree
# Treegen works in strings domain. To help this:
#
# - loadblkstr loads ZBlk data as string.
# - setblkstr sets ZBlk data from string.
#
# We and ΔBtail/ΔFtail tests store into ZBlks only small set of letters with
# optional digit suffix (e.g. "c" and "d4"), so using strings to handle ZBlk
# data is ok and convenient. Besides ZBlk everything else in treegen uses
# strings natively.
def loadblkstr(zblk): # -> bstr
return b(zblk.loadblkdata())
def setblkstr(zblk, strdata):
zblk.setblkdata(b(strdata))
# ZCtx represent treegen-level connection to ZODB. # ZCtx represent treegen-level connection to ZODB.
# It wraps zconn + provides treegen-specif integration. # It wraps zconn + provides treegen-specif integration.
class ZCtx(object): class ZCtx(object):
...@@ -191,13 +206,13 @@ class ZCtx(object): ...@@ -191,13 +206,13 @@ class ZCtx(object):
valdict = zctx.root.get('treegen/values', None) valdict = zctx.root.get('treegen/values', None)
if valdict is None: if valdict is None:
valdict = zctx.root['treegen/values'] = PersistentMapping() valdict = zctx.root['treegen/values'] = PersistentMapping()
valv = b'abcdefghij' valv = 'abcdefghij'
for v in valv: for v in valv:
zblk = valdict.get(v, None) zblk = valdict.get(v, None)
if zblk is not None and zblk.loadblkdata() == v: if zblk is not None and loadblkstr(zblk) == v:
continue continue
zblk = ZBlk() zblk = ZBlk()
zblk.setblkdata(v) setblkstr(zblk, v)
valdict[v] = zblk valdict[v] = zblk
zctx.valdict = valdict zctx.valdict = valdict
commit('treegen/values: init %r' % valv, skipIfEmpty=True) commit('treegen/values: init %r' % valv, skipIfEmpty=True)
...@@ -270,14 +285,14 @@ def TreesSrv(zstor, r): ...@@ -270,14 +285,14 @@ def TreesSrv(zstor, r):
zblk = valdict.get(k) zblk = valdict.get(k)
v1 = None v1 = None
if zblk is not None: if zblk is not None:
v1 = zblk.loadblkdata() v1 = loadblkstr(zblk)
v2 = zv.get(k) v2 = zv.get(k)
if v1 != v2: if v1 != v2:
if v1 is None: if v1 is None:
zblk = ZBlk() zblk = ZBlk()
valdict[k] = zblk valdict[k] = zblk
if v2 is not None: if v2 is not None:
zblk.setblkdata(v2) setblkstr(zblk, v2)
zblk._p_changed = True zblk._p_changed = True
elif v2 is None: elif v2 is None:
del valdict[k] del valdict[k]
......
wcfs/internal/zdata/testdata/.gitignore
View file @ e7d161d1
/*.lock *.lock
/*.tmp *.tmp
/*.tr[0-9] *.tr[0-9]
wcfs/internal/zdata/testdata/zblk_test_gen.py
View file @ e7d161d1
#!/usr/bin/env python2 #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2018-2021 Nexedi SA and Contributors. # Copyright (C) 2018-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
...@@ -20,25 +20,33 @@ ...@@ -20,25 +20,33 @@
# See https://www.nexedi.com/licensing for rationale and options. # See https://www.nexedi.com/licensing for rationale and options.
"""zblk_test_gen.py generates test data for zblk_test.go""" """zblk_test_gen.py generates test data for zblk_test.go"""
from __future__ import print_function
from ZODB.DB import DB from ZODB.DB import DB
from ZODB.utils import u64 from ZODB.utils import u64
from wendelin.bigfile.file_zodb import ZBlk0, ZBlk1, ZBigFile from wendelin.bigfile.file_zodb import ZBlk0, ZBlk1, ZBigFile
from BTrees.IOBTree import IOBTree, IOBucket from BTrees.IOBTree import IOBTree, IOBucket
from numpy import arange from numpy import arange
import os, os.path, transaction import os, os.path, shutil, transaction
import zodbtools.test.gen_testdata # to make time predictable import zodbtools.test.gen_testdata # to make time predictable
from zodbtools.test.gen_testdata import run_with_zodb4py2_compat from zodbtools.test.gen_testdata import run_with_all_zodb_pickle_kinds, current_zkind, xtime_reset
K = 1024 K = 1024
def main(): def main():
run_with_zodb4py2_compat(main2) run_with_all_zodb_pickle_kinds(main2)
def main2(): def main2():
outfs = "testdata/zblk.fs" xtime_reset()
rm_f(outfs)
rm_f(outfs + ".index") zkind = current_zkind()
prefix = "testdata/%s" % zkind
if os.path.exists(prefix):
shutil.rmtree(prefix)
os.makedirs(prefix)
outfs = "%s/zblk.fs" % prefix
db = DB(outfs) db = DB(outfs)
conn = db.open() conn = db.open()
root = conn.root() root = conn.root()
...@@ -67,22 +75,27 @@ def main2(): ...@@ -67,22 +75,27 @@ def main2():
transaction.commit() transaction.commit()
with open("ztestdata_zblk_test.go", "w") as f: with open("ztestdata_zblk_%s_test.go" % zkind, "w") as f:
def emit(v): def emit(v):
print >>f, v print(v, file=f)
emit("// Code generated by %s; DO NOT EDIT." % __file__) emit("// Code generated by %s; DO NOT EDIT." % os.path.relpath(__file__))
emit("package zdata\n") emit("package zdata\n")
emit('import "lab.nexedi.com/kirr/neo/go/zodb"\n') emit('import "lab.nexedi.com/kirr/neo/go/zodb"\n')
emit("const zf_blksize = %d" % zf.blksize) emit("func init() {")
emit("const zf_size = %d" % ((zf.blktab.maxKey()+1)*zf.blksize)) emit('\tzblkTestDataRegistry["%s"] = ZBlkTestData{' % zkind)
emit("const z0_oid = zodb.Oid(%d)" % u64(z0._p_oid)) emit('\t\tdata_fs : "%s",' % outfs)
emit("const z1_oid = zodb.Oid(%d)" % u64(z1._p_oid)) emit("\t\tzf_blksize : %d," % zf.blksize)
emit("const zf_oid = zodb.Oid(%d)" % u64(zf._p_oid)) emit("\t\tzf_size : %d," % ((zf.blktab.maxKey()+1)*zf.blksize))
emit("const z0_rev = zodb.Tid(0x%x)" % u64(z0._p_serial)) emit("\t\tz0_oid : zodb.Oid(%d)," % u64(z0._p_oid))
emit("const z1_rev = zodb.Tid(0x%x)" % u64(z1._p_serial)) emit("\t\tz1_oid : zodb.Oid(%d)," % u64(z1._p_oid))
emit("const z0_len = %d" % len(z0.loadblkdata())) emit("\t\tzf_oid : zodb.Oid(%d)," % u64(zf._p_oid))
emit("const z1_htlen = %d" % z1ht) emit("\t\tz0_rev : zodb.Tid(0x%x)," % u64(z0._p_serial))
emit("\t\tz1_rev : zodb.Tid(0x%x)," % u64(z1._p_serial))
emit("\t\tz0_len : %d," % len(z0.loadblkdata()))
emit("\t\tz1_htlen : %d," % z1ht)
emit("\t}")
emit("}")
conn.close() conn.close()
...@@ -119,11 +132,6 @@ def assertIOBTreeHas2Buckets(t): ...@@ -119,11 +132,6 @@ def assertIOBTreeHas2Buckets(t):
assert isinstance(_[0][2], IOBucket), _[0][2] # bucket1 assert isinstance(_[0][2], IOBucket), _[0][2] # bucket1
assert isinstance(_[1], IOBucket), _[1] # .firstbucket assert isinstance(_[1], IOBucket), _[1] # .firstbucket
# rm_f is like `rm -f` in shell.
def rm_f(path):
if os.path.exists(path):
os.remove(path)
if __name__ == '__main__': if __name__ == '__main__':
main() main()
wcfs/internal/zdata/zblk.go
View file @ e7d161d1
// Copyright (C) 2018-2021 Nexedi SA and Contributors. // Copyright (C) 2018-2024 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -29,7 +29,7 @@ package zdata ...@@ -29,7 +29,7 @@ package zdata
// .blktab LOBtree{} blk -> ZBlk*(blkdata) // .blktab LOBtree{} blk -> ZBlk*(blkdata)
// //
// ZBlk0 (aliased as ZBlk) // ZBlk0 (aliased as ZBlk)
// str with trailing '\0' removed. // bytes|bytestr with trailing '\0' removed.
// //
// ZBlk1 // ZBlk1
// .chunktab IOBtree{} offset -> ZData(chunk) // .chunktab IOBtree{} offset -> ZData(chunk)
...@@ -53,7 +53,6 @@ import ( ...@@ -53,7 +53,6 @@ import (
"lab.nexedi.com/kirr/neo/go/zodb" "lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/kirr/neo/go/zodb/btree" "lab.nexedi.com/kirr/neo/go/zodb/btree"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/pycompat"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb" "lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
) )
...@@ -80,7 +79,7 @@ type ZBlk0 struct { ...@@ -80,7 +79,7 @@ type ZBlk0 struct {
zodb.Persistent zodb.Persistent
// NOTE py source uses bytes(buf) but on python2 it still results in str // NOTE py source uses bytes(buf) but on python2 it still results in str
blkdata string blkdata pickle.Bytes
} }
type zBlk0State ZBlk0 // hide state methods from public API type zBlk0State ZBlk0 // hide state methods from public API
...@@ -97,9 +96,9 @@ func (zb *zBlk0State) PyGetState() interface{} { ...@@ -97,9 +96,9 @@ func (zb *zBlk0State) PyGetState() interface{} {
// PySetState implements zodb.PyStateful. // PySetState implements zodb.PyStateful.
func (zb *zBlk0State) PySetState(pystate interface{}) error { func (zb *zBlk0State) PySetState(pystate interface{}) error {
blkdata, ok := pystate.(string) blkdata, err := pickle.AsBytes(pystate)
if !ok { if err != nil {
return fmt.Errorf("expect str; got %s", xzodb.TypeOf(pystate)) return err
} }
zb.blkdata = blkdata zb.blkdata = blkdata
...@@ -116,7 +115,7 @@ func (zb *ZBlk0) LoadBlkData(ctx context.Context) (_ []byte, _ zodb.Tid, err err ...@@ -116,7 +115,7 @@ func (zb *ZBlk0) LoadBlkData(ctx context.Context) (_ []byte, _ zodb.Tid, err err
} }
defer zb.PDeactivate() defer zb.PDeactivate()
return mem.Bytes(zb.blkdata), zb.PSerial(), nil return mem.Bytes(string(zb.blkdata)), zb.PSerial(), nil
} }
// ---- ZBlk1 --- // ---- ZBlk1 ---
...@@ -126,7 +125,7 @@ type ZData struct { ...@@ -126,7 +125,7 @@ type ZData struct {
zodb.Persistent zodb.Persistent
// NOTE py source uses bytes(buf) but on python2 it still results in str // NOTE py source uses bytes(buf) but on python2 it still results in str
data string data pickle.Bytes
} }
type zDataState ZData // hide state methods from public API type zDataState ZData // hide state methods from public API
...@@ -143,9 +142,9 @@ func (zd *zDataState) PyGetState() interface{} { ...@@ -143,9 +142,9 @@ func (zd *zDataState) PyGetState() interface{} {
// PySetState implements zodb.PyStateful. // PySetState implements zodb.PyStateful.
func (zd *zDataState) PySetState(pystate interface{}) error { func (zd *zDataState) PySetState(pystate interface{}) error {
data, ok := pystate.(string) data, err := pickle.AsBytes(pystate)
if !ok { if err != nil {
return fmt.Errorf("expect str; got %s", xzodb.TypeOf(pystate)) return err
} }
zd.data = data zd.data = data
...@@ -382,9 +381,9 @@ func (bf *zBigFileState) PySetState(pystate interface{}) (err error) { ...@@ -382,9 +381,9 @@ func (bf *zBigFileState) PySetState(pystate interface{}) (err error) {
return fmt.Errorf("expect [2](); got [%d]()", len(t)) return fmt.Errorf("expect [2](); got [%d]()", len(t))
} }
blksize, ok := pycompat.Int64(t[0]) blksize, err := pickle.AsInt64(t[0])
if !ok { if err != nil {
return fmt.Errorf("blksize: expect integer; got %s", xzodb.TypeOf(t[0])) return fmt.Errorf("blksize: %s", err)
} }
if blksize <= 0 { if blksize <= 0 {
return fmt.Errorf("blksize: must be > 0; got %d", blksize) return fmt.Errorf("blksize: must be > 0; got %d", blksize)
......
wcfs/internal/zdata/zblk_test.go
View file @ e7d161d1
// Copyright (C) 2018-2021 Nexedi SA and Contributors. // Copyright (C) 2018-2024 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -19,12 +19,14 @@ ...@@ -19,12 +19,14 @@
package zdata package zdata
//go:generate ./testdata/zblk_test_gen.py //go:generate python2 ./testdata/zblk_test_gen.py
//go:generate python3 ./testdata/zblk_test_gen.py
import ( import (
"bytes" "bytes"
"context" "context"
"encoding/binary" "encoding/binary"
"sort"
"testing" "testing"
"lab.nexedi.com/kirr/go123/exc" "lab.nexedi.com/kirr/go123/exc"
...@@ -35,13 +37,46 @@ import ( ...@@ -35,13 +37,46 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
// ZBlkTestData describes one ZBlk test data generated by zblk_test_gen.py for particular zkind .
type ZBlkTestData struct {
data_fs string // e.g. "testdata/zblk-py2-pickle3.fs"
zf_blksize int64 // root.zbigf.blksize
zf_size int64 // size of zbigf data
z0_oid zodb.Oid // oid of root.zblk0
z1_oid zodb.Oid // oid of root.zblk1
zf_oid zodb.Oid // oid of root.zbigf
z0_rev zodb.Tid // serial of zblk0
z1_rev zodb.Tid // serial of zblk1
z0_len int // len of zblk0 data
z1_htlen int // len of head and tail in zblk1
}
// each generated testdata kind is registered in zblkTestDataRegistry.
var zblkTestDataRegistry = map[/*zkind*/string]ZBlkTestData{}
// TestZBlk verifies that ZBlk* and ZBigFile saved by Python can be read correctly by Go. // TestZBlk verifies that ZBlk* and ZBigFile saved by Python can be read correctly by Go.
// TODO also test with data saved by Python3.
func TestZBlk(t *testing.T) { func TestZBlk(t *testing.T) {
// run the test for all generated testdata kinds
zkindv := []string{}
for zkind := range zblkTestDataRegistry {
zkindv = append(zkindv, zkind)
}
sort.Strings(zkindv)
for _, zkind := range zkindv {
z := zblkTestDataRegistry[zkind]
t.Run(zkind, func(t *testing.T) {
_TestZBlk(t, z)
})
}
}
func _TestZBlk(t *testing.T, z ZBlkTestData) {
X := exc.Raiseif X := exc.Raiseif
assert := require.New(t) assert := require.New(t)
ctx := context.Background() ctx := context.Background()
stor, err := zodb.Open(ctx, "testdata/zblk.fs", &zodb.OpenOptions{ReadOnly: true}); X(err) stor, err := zodb.Open(ctx, z.data_fs, &zodb.OpenOptions{ReadOnly: true}); X(err)
db := zodb.NewDB(stor, &zodb.DBOptions{}) db := zodb.NewDB(stor, &zodb.DBOptions{})
defer func() { defer func() {
err := db.Close(); X(err) err := db.Close(); X(err)
...@@ -53,9 +88,9 @@ func TestZBlk(t *testing.T) { ...@@ -53,9 +88,9 @@ func TestZBlk(t *testing.T) {
conn, err := db.Open(ctx, &zodb.ConnOptions{}); X(err) conn, err := db.Open(ctx, &zodb.ConnOptions{}); X(err)
xz0, err := conn.Get(ctx, z0_oid); X(err) xz0, err := conn.Get(ctx, z.z0_oid); X(err)
xz1, err := conn.Get(ctx, z1_oid); X(err) xz1, err := conn.Get(ctx, z.z1_oid); X(err)
xzf, err := conn.Get(ctx, zf_oid); X(err) xzf, err := conn.Get(ctx, z.zf_oid); X(err)
z0, ok := xz0.(*ZBlk0) z0, ok := xz0.(*ZBlk0)
if !ok { if !ok {
...@@ -82,22 +117,22 @@ func TestZBlk(t *testing.T) { ...@@ -82,22 +117,22 @@ func TestZBlk(t *testing.T) {
z0Data, z0Rev, err := z0.LoadBlkData(ctx); X(err) z0Data, z0Rev, err := z0.LoadBlkData(ctx); X(err)
z0DataOK := brange32(z0_len) z0DataOK := brange32(z.z0_len)
assert.Equal(z0Data, z0DataOK, "ZBlk0 data wrong") assert.Equal(z0Data, z0DataOK, "ZBlk0 data wrong")
assert.Equal(z0Rev, z0_rev, "ZBlk0 rev wrong") assert.Equal(z0Rev, z.z0_rev, "ZBlk0 rev wrong")
z1Data, z1Rev, err := z1.LoadBlkData(ctx); X(err) z1Data, z1Rev, err := z1.LoadBlkData(ctx); X(err)
z1DataOK := make([]byte, zf_blksize) // zeros z1DataOK := make([]byte, z.zf_blksize) // zeros
copy(z1DataOK[0:], brange32(z1_htlen)) // head copy(z1DataOK[0:], brange32(z.z1_htlen)) // head
copy(z1DataOK[len(z1DataOK)-z1_htlen:], breverse(brange32(z1_htlen))) // tail copy(z1DataOK[len(z1DataOK)-z.z1_htlen:], breverse(brange32(z.z1_htlen))) // tail
z1DataOK = bytes.TrimRight(z1DataOK, "\x00") // trailing 0 are not persisted z1DataOK = bytes.TrimRight(z1DataOK, "\x00") // trailing 0 are not persisted
assert.Equal(z1Data, z1DataOK, "ZBlk1 data wrong") assert.Equal(z1Data, z1DataOK, "ZBlk1 data wrong")
assert.Equal(z1Rev, z1_rev, "ZBlk1 rev wrong") assert.Equal(z1Rev, z.z1_rev, "ZBlk1 rev wrong")
xactivate(zf) xactivate(zf)
if zf.blksize != zf_blksize { if zf.blksize != z.zf_blksize {
t.Fatalf("zf: blksize=%d; want %d", zf.blksize, zf_blksize) t.Fatalf("zf: blksize=%d; want %d", zf.blksize, z.zf_blksize)
} }
z0_, ok, err := zf.blktab.Get(ctx, 1); X(err) z0_, ok, err := zf.blktab.Get(ctx, 1); X(err)
...@@ -111,7 +146,7 @@ func TestZBlk(t *testing.T) { ...@@ -111,7 +146,7 @@ func TestZBlk(t *testing.T) {
} }
size, _, _, err := zf.Size(ctx); X(err) size, _, _, err := zf.Size(ctx); X(err)
assert.Equal(size, int64(zf_size), "ZBigFile size wrong") assert.Equal(size, z.zf_size, "ZBigFile size wrong")
// LoadBlk // LoadBlk
......
wcfs/internal/zdata/ztestdata_zblk_py2_pickle1_test.go 0 → 100644
View file @ e7d161d1
// Code generated by testdata/zblk_test_gen.py; DO NOT EDIT.
package zdata
import "lab.nexedi.com/kirr/neo/go/zodb"
func init() {
zblkTestDataRegistry["py2_pickle1"] = ZBlkTestData{
data_fs : "testdata/py2_pickle1/zblk.fs",
zf_blksize : 2097152,
zf_size : 8388608,
z0_oid : zodb.Oid(2),
z1_oid : zodb.Oid(3),
zf_oid : zodb.Oid(1),
z0_rev : zodb.Tid(0x285cbac3851eb99),
z1_rev : zodb.Tid(0x285cbac3851eb99),
z0_len : 16384,
z1_htlen : 131072,
}
}
wcfs/internal/zdata/ztestdata_zblk_py2_pickle2_test.go 0 → 100644
View file @ e7d161d1
// Code generated by testdata/zblk_test_gen.py; DO NOT EDIT.
package zdata
import "lab.nexedi.com/kirr/neo/go/zodb"
func init() {
zblkTestDataRegistry["py2_pickle2"] = ZBlkTestData{
data_fs : "testdata/py2_pickle2/zblk.fs",
zf_blksize : 2097152,
zf_size : 8388608,
z0_oid : zodb.Oid(2),
z1_oid : zodb.Oid(3),
zf_oid : zodb.Oid(1),
z0_rev : zodb.Tid(0x285cbac3851eb99),
z1_rev : zodb.Tid(0x285cbac3851eb99),
z0_len : 16384,
z1_htlen : 131072,
}
}
wcfs/internal/zdata/ztestdata_zblk_py2_pickle3_test.go 0 → 100644
View file @ e7d161d1
// Code generated by testdata/zblk_test_gen.py; DO NOT EDIT.
package zdata
import "lab.nexedi.com/kirr/neo/go/zodb"
func init() {
zblkTestDataRegistry["py2_pickle3"] = ZBlkTestData{
data_fs : "testdata/py2_pickle3/zblk.fs",
zf_blksize : 2097152,
zf_size : 8388608,
z0_oid : zodb.Oid(2),
z1_oid : zodb.Oid(3),
zf_oid : zodb.Oid(1),
z0_rev : zodb.Tid(0x285cbac3851eb99),
z1_rev : zodb.Tid(0x285cbac3851eb99),
z0_len : 16384,
z1_htlen : 131072,
}
}
wcfs/internal/zdata/ztestdata_zblk_py3_pickle3_test.go 0 → 100644
View file @ e7d161d1
// Code generated by testdata/zblk_test_gen.py; DO NOT EDIT.
package zdata
import "lab.nexedi.com/kirr/neo/go/zodb"
func init() {
zblkTestDataRegistry["py3_pickle3"] = ZBlkTestData{
data_fs : "testdata/py3_pickle3/zblk.fs",
zf_blksize : 2097152,
zf_size : 8388608,
z0_oid : zodb.Oid(2),
z1_oid : zodb.Oid(3),
zf_oid : zodb.Oid(1),
z0_rev : zodb.Tid(0x285cbac3851eb99),
z1_rev : zodb.Tid(0x285cbac3851eb99),
z0_len : 16384,
z1_htlen : 131072,
}
}
wcfs/internal/zdata/ztestdata_zblk_test.go deleted 100644 → 0
View file @ db6fea3d
// Code generated by ./testdata/zblk_test_gen.py; DO NOT EDIT.
package zdata
import "lab.nexedi.com/kirr/neo/go/zodb"
const zf_blksize = 2097152
const zf_size = 8388608
const z0_oid = zodb.Oid(2)
const z1_oid = zodb.Oid(3)
const zf_oid = zodb.Oid(1)
const z0_rev = zodb.Tid(0x285cbac3851eb99)
const z1_rev = zodb.Tid(0x285cbac3851eb99)
const z0_len = 16384
const z1_htlen = 131072
wcfs/internal/zdata/δftail_test.go
View file @ e7d161d1
// Copyright (C) 2019-2022 Nexedi SA and Contributors. // Copyright (C) 2019-2024 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -734,7 +734,7 @@ func (t *T) XLoadZFile(ctx context.Context, zconn *zodb.Connection) (zfile *ZBig ...@@ -734,7 +734,7 @@ func (t *T) XLoadZFile(ctx context.Context, zconn *zodb.Connection) (zfile *ZBig
xzroot, err := zconn.Get(ctx, 0); X(err) xzroot, err := zconn.Get(ctx, 0); X(err)
zroot := xzroot.(*zodb.Map) zroot := xzroot.(*zodb.Map)
err = zroot.PActivate(ctx); X(err) err = zroot.PActivate(ctx); X(err)
zfile = zroot.Data["treegen/file"].(*ZBigFile) zfile = zroot.Get("treegen/file").(*ZBigFile)
zroot.PDeactivate() zroot.PDeactivate()
err = zfile.PActivate(ctx); X(err) err = zfile.PActivate(ctx); X(err)
blksize = zfile.blksize blksize = zfile.blksize
......
wcfs/wcfs_test.py
View file @ e7d161d1
...@@ -58,6 +58,8 @@ from wendelin.wcfs.internal.wcfs_test import _tWCFS, read_exfault_nogil, Segment ...@@ -58,6 +58,8 @@ from wendelin.wcfs.internal.wcfs_test import _tWCFS, read_exfault_nogil, Segment
from wendelin.wcfs.client._wcfs import _tpywlinkwrite as _twlinkwrite from wendelin.wcfs.client._wcfs import _tpywlinkwrite as _twlinkwrite
from wendelin.wcfs import _is_mountpoint as is_mountpoint, _procwait as procwait, _waitfor as waitfor, _ready as ready, _rmdir_ifexists as rmdir_ifexists from wendelin.wcfs import _is_mountpoint as is_mountpoint, _procwait as procwait, _waitfor as waitfor, _ready as ready, _rmdir_ifexists as rmdir_ifexists
bstr = type(b('')) # TODO import directly after https://lab.nexedi.com/nexedi/pygolang/-/merge_requests/21 is merged
# setup: # setup:
# - create test database, compute zurl and mountpoint for wcfs # - create test database, compute zurl and mountpoint for wcfs
...@@ -466,14 +468,14 @@ class tWCFS(_tWCFS): ...@@ -466,14 +468,14 @@ class tWCFS(_tWCFS):
assert kv == kvok, "stats did not stay at expected state" assert kv == kvok, "stats did not stay at expected state"
# _loadStats loads content of .wcfs/stats . # _loadStats loads content of .wcfs/stats .
def _loadStats(t): # -> {} def _loadStats(t): # -> {} bstr -> int
stats = {} stats = {}
for l in t.wc._read(".wcfs/stats").splitlines(): for l in t.wc._read(".wcfs/stats").splitlines():
# key : value # key : value
k, v = l.split(':') k, v = l.split(b':')
k = k.strip() k = k.strip()
v = v.strip() v = v.strip()
stats[k] = int(v) stats[b(k)] = int(v)
# verify that keys remains the same and that cumulative counters do not decrease # verify that keys remains the same and that cumulative counters do not decrease
if t._stats_prev is not None: if t._stats_prev is not None:
...@@ -824,6 +826,7 @@ class tFile: ...@@ -824,6 +826,7 @@ class tFile:
@func @func
def _assertBlk(t, blk, dataok, pinokByWLink=None, pinfunc=None, timeout=None): def _assertBlk(t, blk, dataok, pinokByWLink=None, pinfunc=None, timeout=None):
assert isinstance(dataok, bstr)
assert len(dataok) <= t.blksize assert len(dataok) <= t.blksize
dataok += b'\0'*(t.blksize - len(dataok)) # tailing zeros dataok += b'\0'*(t.blksize - len(dataok)) # tailing zeros
assert blk < t._sizeinblk() assert blk < t._sizeinblk()
...@@ -897,11 +900,11 @@ class tFile: ...@@ -897,11 +900,11 @@ class tFile:
have_read = chan(1) have_read = chan(1)
def _(): def _():
try: try:
b = read_exfault_nogil(blkview[0:1]) got = read_exfault_nogil(blkview[0:1])
except SegmentationFault: except SegmentationFault:
b = 'FAULT' got = 'FAULT'
t._blkaccess(blk) t._blkaccess(blk)
have_read.send(b) have_read.send(got)
go(_) go(_)
_, _rx = select( _, _rx = select(
ctx.done().recv, # 0 ctx.done().recv, # 0
...@@ -909,9 +912,9 @@ class tFile: ...@@ -909,9 +912,9 @@ class tFile:
) )
if _ == 0: if _ == 0:
raise ctx.err() raise ctx.err()
b = _rx got = _rx
ev.append('read ' + b) ev.append('read ' + b(got))
ev = doCheckingPin(ctx, _, pinokByWLink, pinfunc) ev = doCheckingPin(ctx, _, pinokByWLink, pinfunc)
# XXX hack - wlinks are notified and emit events simultaneously - we # XXX hack - wlinks are notified and emit events simultaneously - we
...@@ -949,7 +952,10 @@ class tFile: ...@@ -949,7 +952,10 @@ class tFile:
assert st.st_blksize == t.blksize assert st.st_blksize == t.blksize
assert st.st_size == len(dataokv)*t.blksize assert st.st_size == len(dataokv)*t.blksize
if mtime is not None: if mtime is not None:
assert st.st_mtime == tidtime(mtime) # st_mtime comes from wcfs via tidtime/go
# ZODB/py vs ZODB/go time resolution is not better than 1µs
# see e.g. https://lab.nexedi.com/kirr/neo/commit/9112f21e
assert abs(st.st_mtime - tidtime(mtime)) <= 1e-6
cachev = t.cached() cachev = t.cached()
for blk, dataok in enumerate(dataokv): for blk, dataok in enumerate(dataokv):
...@@ -1212,7 +1218,7 @@ def doCheckingPin(ctx, f, pinokByWLink, pinfunc=None): # -> []event(str) ...@@ -1212,7 +1218,7 @@ def doCheckingPin(ctx, f, pinokByWLink, pinfunc=None): # -> []event(str)
def _expectPin(twlink, ctx, zf, expect): # -> []SrvReq def _expectPin(twlink, ctx, zf, expect): # -> []SrvReq
expected = set() # of expected pin messages expected = set() # of expected pin messages
for blk, at in expect.items(): for blk, at in expect.items():
hat = h(at) if at is not None else 'head' hat = h(at) if at is not None else b'head'
msg = b"pin %s #%d @%s" % (h(zf._p_oid), blk, hat) msg = b"pin %s #%d @%s" % (h(zf._p_oid), blk, hat)
assert msg not in expected assert msg not in expected
expected.add(msg) expected.add(msg)
...@@ -1806,7 +1812,7 @@ def test_wcfs_remmap_on_pin(): ...@@ -1806,7 +1812,7 @@ def test_wcfs_remmap_on_pin():
assert at == at1 assert at == at1
mm.map_into_ro(f._blk(blk), f1.f.fileno(), blk*f.blksize) mm.map_into_ro(f._blk(blk), f1.f.fileno(), blk*f.blksize)
f._assertBlk(2, 'hello', {wl: {2:at1}}, pinfunc=_) # NOTE not world f._assertBlk(2, b('hello'), {wl: {2:at1}}, pinfunc=_) # NOTE not world
# verify that pin message is not sent for the same blk@at twice. # verify that pin message is not sent for the same blk@at twice.
...@@ -2004,13 +2010,7 @@ def writefile(path, data): ...@@ -2004,13 +2010,7 @@ def writefile(path, data):
# tidtime converts tid to transaction commit time. # tidtime converts tid to transaction commit time.
def tidtime(tid): def tidtime(tid):
t = TimeStamp(tid).timeTime() return TimeStamp(tid).timeTime()
# ZODB/py vs ZODB/go time resolution is not better than 1µs
# see e.g. https://lab.nexedi.com/kirr/neo/commit/9112f21e
#
# NOTE pytest.approx supports only ==, not e.g. <, so we use plain round.
return round(t, 6)
# tidfromtime converts time into corresponding transaction ID. # tidfromtime converts time into corresponding transaction ID.
def tidfromtime(t): def tidfromtime(t):
...@@ -2022,8 +2022,8 @@ def tidfromtime(t): ...@@ -2022,8 +2022,8 @@ def tidfromtime(t):
ts = TimeStamp(_.tm_year, _.tm_mon, _.tm_mday, _.tm_hour, _.tm_min, s) ts = TimeStamp(_.tm_year, _.tm_mon, _.tm_mday, _.tm_hour, _.tm_min, s)
return ts.raw() return ts.raw()
# verify that tidtime is precise enough to show difference in between transactions. # verify that tidtime is precise to show difference in between transactions.
# verify that tidtime -> tidfromtime is identity within rounding tolerance. # verify that tidtime -> tidfromtime is identity.
@func @func
def test_tidtime(): def test_tidtime():
t = tDB() t = tDB()
...@@ -2041,7 +2041,7 @@ def test_tidtime(): ...@@ -2041,7 +2041,7 @@ def test_tidtime():
tat = tidtime(at) tat = tidtime(at)
at_ = tidfromtime(tat) at_ = tidfromtime(tat)
tat_ = tidtime(at_) tat_ = tidtime(at_)
assert abs(tat_ - tat) <= 2E-6 assert tat_ == tat
# tAt is bytes whose repr returns human readable string considering it as `at` under tDB. # tAt is bytes whose repr returns human readable string considering it as `at` under tDB.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7