Commit f845ad4f authored by Kirill Smelkov's avatar Kirill Smelkov

fixup! ZBigFile: Add ZBlk format option 'h' (heuristic) (2)

Rename 'h' to 'auto' and update module documentation accoring to
introduced heuristic. The rename is done because 'h' is not
descriptive, while 'auto' is more descriptive.
parent b72a79e6
...@@ -83,13 +83,18 @@ changes. "Small" here means something like 1-10000 bytes per transaction as ...@@ -83,13 +83,18 @@ changes. "Small" here means something like 1-10000 bytes per transaction as
larger changes become comparable to 2M block size and are handled efficiently larger changes become comparable to 2M block size and are handled efficiently
out of the box. Until the problem is fixed on ZODB server side, wendelin.core out of the box. Until the problem is fixed on ZODB server side, wendelin.core
provides on-client workaround in the form of specialized block format, and provides on-client workaround in the form of specialized block format, and
users have to explicitly indicate via environment variable that their workload users can explicitly indicate via environment variable that their workload is
is "small changes" if they prefer to prioritize database size over access either "big changes", if they prefer to prioritize access speed, or "small
speed:: changes" if they prefer to prioritize database size over access speed. There is
also "auto" mode that tries to heuristically use both ZBlk0 and ZBlk1 depending
on change pattern and works relatively good regrding both access speed and
database size for append-like workloads::
$WENDELIN_CORE_ZBLK_FMT $WENDELIN_CORE_ZBLK_FMT
ZBlk0 fast reads (default) ZBlk0 fast reads
ZBlk1 small changes ZBlk1 small changes
auto (default) heuristically use either ZBlk0 or ZBlk1
depending on change pattern
Description of block formats follow: Description of block formats follow:
...@@ -483,8 +488,8 @@ ZBlk_fmt_registry = { ...@@ -483,8 +488,8 @@ ZBlk_fmt_registry = {
} }
# format for updated blocks # format for updated blocks
ZBlk_fmt_write = os.environ.get('WENDELIN_CORE_ZBLK_FMT', 'h') ZBlk_fmt_write = os.environ.get('WENDELIN_CORE_ZBLK_FMT', 'auto')
if ZBlk_fmt_write != "h" and ZBlk_fmt_write not in ZBlk_fmt_registry: if ZBlk_fmt_write != "auto" and ZBlk_fmt_write not in ZBlk_fmt_registry:
raise RuntimeError('E: Unknown ZBlk format %r' % ZBlk_fmt_write) raise RuntimeError('E: Unknown ZBlk format %r' % ZBlk_fmt_write)
...@@ -547,7 +552,7 @@ class ZBigFile(LivePersistent): ...@@ -547,7 +552,7 @@ class ZBigFile(LivePersistent):
def storeblk(self, blk, buf): def storeblk(self, blk, buf):
zblk = self.blktab.get(blk) zblk = self.blktab.get(blk)
zblk_fmt = ZBlk_fmt_write zblk_fmt = ZBlk_fmt_write
if zblk_fmt == "h": # apply heuristic if zblk_fmt == "auto": # apply heuristic
zblk_fmt = self._zblk_fmt_heuristic(zblk, blk, buf) zblk_fmt = self._zblk_fmt_heuristic(zblk, blk, buf)
self._setzblk(blk, zblk, buf, zblk_fmt) self._setzblk(blk, zblk, buf, zblk_fmt)
......
...@@ -20,7 +20,7 @@ ...@@ -20,7 +20,7 @@
# #
# - ZBlk0 # - ZBlk0
# - ZBlk1 # - ZBlk1
# - h # - auto
import os import os
import random import random
......
...@@ -22,9 +22,9 @@ ...@@ -22,9 +22,9 @@
# #
# - ZBlk0 # - ZBlk0
# - ZBlk1 # - ZBlk1
# - h # - auto
# #
# The heuristic 'h' should behave as good as ZBlk0 in case of wide changes # The heuristic 'auto' should behave as good as ZBlk0 in case of wide changes
# and as good as ZBlk1 in case of small changes. # and as good as ZBlk1 in case of small changes.
function test { function test {
...@@ -58,7 +58,7 @@ function test { ...@@ -58,7 +58,7 @@ function test {
export arrsize=$arrsize export arrsize=$arrsize
export change_type=$change_type export change_type=$change_type
t h t auto
t ZBlk0 t ZBlk0
t ZBlk1 t ZBlk1
......
...@@ -692,16 +692,15 @@ def test_bigfile_zblk1_zdata_reuse(): ...@@ -692,16 +692,15 @@ def test_bigfile_zblk1_zdata_reuse():
assert zdata_v1[i] is zdata_v2[i] assert zdata_v1[i] is zdata_v2[i]
# Minimal test to ensure normal operations work as expected # Minimal test to ensure normal operations work as expected with zblk format 'auto'.
# with zblk fmt 'h'
@func @func
def test_bigfile_zblk_fmt_heuristic(): def test_bigfile_zblk_fmt_auto():
root = dbopen() root = dbopen()
defer(lambda: dbclose(root)) defer(lambda: dbclose(root))
# set ZBlk_fmt_write to 'h' for this test # set ZBlk_fmt_write to 'h' for this test
fmt_write_save = file_zodb.ZBlk_fmt_write fmt_write_save = file_zodb.ZBlk_fmt_write
file_zodb.ZBlk_fmt_write = 'h' file_zodb.ZBlk_fmt_write = 'auto'
def _(): def _():
file_zodb.ZBlk_fmt_write = fmt_write_save file_zodb.ZBlk_fmt_write = fmt_write_save
defer(_) defer(_)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment