bigfile/zodb: Prepare to have several ZBlk formats

- current ZBlk becomes format 0
- write format can be selected via WENDELIN_CORE_ZBLK_FMT env var
- upon writing a block we always make sure we write it in current write
  format - so if a block was previously written in one format, it could
  be changed on the next write.
- tox is prepared to test all write formats (so far only ZBlk0 there).

The reason is - in the next patch we'll introduce another format for
blocks which is optimized for small changes.

bigfile/file_zodb.py

@@ -37,6 +37,7 @@ from BTrees.LOBTree import LOBTree
 from zope.interface import implementer
 from ZODB.Connection import Connection
 from weakref import WeakSet
+import os
 
 # TODO document that first data access must be either after commit or Connection.add
 
@@ -98,8 +99,13 @@ class ZBlkBase(Persistent):
         Persistent._p_invalidate(self)
 
 
-# data of 1 file block as stored in ZODB
-class ZBlk(ZBlkBase):
+
+# ZBlk storage formats
+# NOTE once established formats do not change on disk
+
+
+# ZBlk format 0: raw bytes
+class ZBlk0(ZBlkBase):
     # ._v_blkdata   - bytes
     __slots__ = ('_v_blkdata',)
 
@@ -158,7 +164,7 @@ class ZBlk(ZBlkBase):
     # DB (through pickle) loads data to memory
     # DB -> ._v_blkdata  (-> memory-page)
     def __setstate__(self, state):
-        super(ZBlk, self).__init__()
+        super(ZBlk0, self).__init__()
         self._v_blkdata = state
 
     # ZBlk as initially created (empty placeholder)
@@ -166,6 +172,23 @@ class ZBlk(ZBlkBase):
         self.__setstate__(None)
 
 
+# backward compatibility (early versions wrote ZBlk0 named as ZBlk)
+ZBlk = ZBlk0
+
+# format-name -> blk format type
+ZBlk_fmt_registry = {
+    'ZBlk0':    ZBlk0,
+}
+
+# format for updated blocks
+ZBlk_fmt_write = os.environ.get('WENDELIN_CORE_ZBLK_FMT', 'ZBlk0')
+if ZBlk_fmt_write not in ZBlk_fmt_registry:
+    raise RuntimeError('E: Unknown ZBlk format %r' % ZBlk_fmt_write)
+
+
+# ----------------------------------------
+
+
 # helper for ZBigFile - just redirect loadblk/storeblk back
 # (because it is not possible to inherit from both Persistent and BigFile at
 #  the same time - see below)
@@ -218,7 +241,7 @@ class ZBigFile(LivePersistent):
     # file is split into blocks; each block is stored as separate object in the DB
     #
     #   .blksize
-    #   .blktab       {} blk -> ZBlk(blkdata)
+    #   .blktab       {} blk -> ZBlk*(blkdata)
 
     # ._v_file      _ZBigFile helper
     # ._v_filehset  weakset( _ZBigFileH ) that we created
@@ -263,8 +286,11 @@ class ZBigFile(LivePersistent):
     # store data    dirty page -> ZODB obj
     def storeblk(self, blk, buf):
         zblk = self.blktab.get(blk)
-        if zblk is None:
-            zblk = self.blktab[blk] = ZBlk()
+        zblk_type_write = ZBlk_fmt_registry[ZBlk_fmt_write]
+        # if zblk was absent or of different type - we (re-)create it anew
+        if zblk is None  or \
+           type(zblk) is not zblk_type_write:
+            zblk = self.blktab[blk] = zblk_type_write()
 
         zblk.setblkdata(buf)
         zblk._p_changed = True          # if zblk was already in DB: _p_state -> CHANGED

tox.ini

 # wendelin.core | tox setup
 [tox]
-envlist = py27-ZODB3-{fs,zeo,neo}-{numpy18,numpy19}, {py27,py34}-ZODB4-{fs,zeo}-{numpy18,numpy19}
+envlist = py27-ZODB3-{zblk0}-{fs,zeo,neo}-{numpy18,numpy19}, {py27,py34}-ZODB4-{zblk0}-{fs,zeo}-{numpy18,numpy19}
 # (NOTE ZODB3 does not work on python3)
 # (NOTE NEO does not work on ZODB4)
 
@@ -31,6 +31,8 @@ setenv =
     zeo: WENDELIN_CORE_TEST_DB=<zeo>
     neo: WENDELIN_CORE_TEST_DB=<neo>
 
+    zblk0:  WENDELIN_CORE_ZBLK_FMT=ZBlk0
+
 commands= {envpython} setup.py test
 # XXX setenv = TMPDIR = ... ?  (so that /tmp is not on tmpfs and we don't run out of memory on bench)
 # + {envpython} setup.py bench  (?)