# -*- coding: utf-8 -*- # Wendelin.bigfile | WCFS part of BigFile ZODB backend # Copyright (C) 2014-2020 Nexedi SA and Contributors. # Kirill Smelkov <kirr@nexedi.com> # # This program is free software: you can Use, Study, Modify and Redistribute # it under the terms of the GNU General Public License version 3, or (at your # option) any later version, as published by the Free Software Foundation. # # You can also Link and Combine this program with other software covered by # the terms of any of the Free Software licenses or any of the Open Source # Initiative approved licenses and Convey the resulting work. Corresponding # source of such a combination shall include the source code for all other # software used. # # This program is distributed WITHOUT ANY WARRANTY; without even the implied # warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # # See COPYING file for full licensing terms. # See https://www.nexedi.com/licensing for rationale and options. # cython: language_level=2 # distutils: language=c++ """XXX""" from __future__ import print_function, absolute_import cdef extern from "wcfs/client/wcfs.h": pass cdef extern from "bigfile/_bigfile.h": struct PyBigFile: pass ctypedef extern class wendelin.bigfile._bigfile.BigFile[object PyBigFile]: pass # ZBigFile_mmap_ops is virtmem mmap functions for _ZBigFile. cdef extern from "<wendelin/bigfile/file.h>" nogil: struct bigfile_ops: pass cdef extern from * nogil: """ extern const bigfile_ops ZBigFile_mmap_ops; """ const bigfile_ops ZBigFile_mmap_ops import wcfs as pywcfs from wcfs.client cimport _wcfs as wcfs from golang cimport nil from cpython cimport PyCapsule_New from ZODB.Connection import Connection as ZConnection from ZODB.utils import u64 from wendelin.lib.zodb import zconn_at # _ZBigFile is base class for ZBigFile that provides BigFile-line base. # # The other base line is from Persistent. It is not possible to inherit from # both Persistent and BigFile at the same time since both are C types and their # layouts conflict. # # _ZBigFile: # # - redirects loadblk/storeblk calls to ZBigFile. # - provides blkmmapper with WCFS integration. cdef public class _ZBigFile(BigFile) [object _ZBigFile, type _ZBigFile_Type]: cdef object zself # reference to ZBigFile cdef wcfs.FileH wfileh # WCFS file handle. Initially nil, opened by blkmmapper # _new creates new _ZBigFile associated with ZBigFile zself. # XXX Cython does not allow __new__ nor to change arguments passed to __cinit__ / __init__ @staticmethod def _new(zself, blksize): cdef _ZBigFile obj = _ZBigFile.__new__(_ZBigFile, blksize) obj.zself = zself obj.wfileh = nil return obj def __dealloc__(_ZBigFile zf): zf.wfileh = nil # redirect load/store to main class def loadblk(self, blk, buf): return self.zself.loadblk(blk, buf) def storeblk(self, blk, buf): return self.zself.storeblk(blk, buf) # blkmmapper is pycapsule with virtmem mmap functions for _ZBigFile. # The functions rely on .wfileh being initialized by .fileh_open() blkmmapper = PyCapsule_New(<void*>&ZBigFile_mmap_ops, "wendelin.bigfile.IBlkMMapper", NULL) # fileh_open wraps BigFile.fileh_open and makes sure that WCFS file handle # corresponding to ZBigFile is opened if use_wcfs=True. def fileh_open(_ZBigFile zf, bint use_wcfs): mmap_overlay = False cdef wcfs.PyFileH pywfileh if use_wcfs: mmap_overlay = True if zf.wfileh == nil: zconn = zf.zself._p_jar assert zconn is not None # XXX locking? or rely on that ZODB objects for zconn must be used from under 1 thread only? # join zconn to wconn; link to wconn from _ZBigFile pywconn = pywconnOf(zconn) pywfileh = pywconn.open(zf.zself._p_oid) zf.wfileh = pywfileh.wfileh return super(_ZBigFile, zf).fileh_open(mmap_overlay) # pywconnOf establishes and returns (py) wcfs.Conn associated with zconn. # XXX wconn is maintained and kept in sync with zconn. # XXX simultaneous call? # XXX move -> .py ? cdef wcfs.PyConn pywconnOf(zconn): assert isinstance(zconn, ZConnection) assert zconn.opened # XXX needed # XXX locking wconn = getattr(zconn, '_wcfs_wconn', None) if wconn is not None: return wconn # zconn is not yet associated with wconn zstor = zconn.db().storage zurl = pywcfs.zstor_2zurl(zstor) wc = pywcfs.join(zurl) wconn = wc.connect(zconn_at(zconn)) zconn._wcfs_wconn = wconn # keep wconn view of the database in sync with zconn zconn._wcfs_wconn_zsync = ZSync(zconn, wconn) return wconn # ZSync keeps wconn in sync with zconn. # XXX -> make generic and use for _ZBigFileH too? # -> y: ZSync base + ZSyncWConn, ZSyncBigFileH # XXX naming? -> ZKeepInSync ? class ZSync: # .zconn # .wconn def __init__(zsync, zconn, wconn): print('ZSync %r %r' % (zconn, wconn)) assert zconn.open zsync.zconn = zconn # XXX -> weakref zsync.wconn = wconn # NOTE zconn.onOpenCallback is not enough: zconn.at can change even # without zconn.close/zconn.open, e.g.: # zconn = DB.open(transaction_manager=tm) # tm.commit() # zconn.at updated (zconn.afterCompletion -> zconn.newTransaction) # tm.commit() # zconn.at updated again # # TODO test for that. #zconn.onOpenCallback(zsync) zconn.onResyncCallback(zsync) """ # DB.open() pops .zconn from connection pool and "opens" for usage. # -> resync .wconn to new database view of ZODB connection. # def on_connection_open(zsync): print('ZSync.resync %r %r' % (zsync.zconn, zsync.wconn)) zsync.wconn.resync(zconn_at(zsync.zconn)) """ def on_connection_resync(zsync): #print('\nZZZSync.resync %r %r' % (zsync.zconn, zsync.wconn)) #import traceback; traceback.print_stack() zsync.wconn.resync(zconn_at(zsync.zconn)) # TODO zconn dealloc -> wconn.close