.

wcfs/wcfs_test.py

@@ -197,10 +197,12 @@ class tDB:
     # open opens file corresponding to obj on wcfs.
     def open(t, obj, at=None):
         path = t.path(obj, at=at)
-        return open(path)
+        return open(path, 'rb', 0)  # unbuffered
 
     # track starts to track wcfs file corresponding to zf@at.
     # see returned tFile for details.
+    #
+    # XXX -> openfile? testopen?
     def track(t, zf, at=None):
         tf = tFile(t, zf, at=at)
         t.tracked.add(tf)
@@ -208,13 +210,27 @@ class tDB:
 
 # tFile is testing environment for one bigfile on wcfs.
 class tFile:
+    # maximum number of pages we mmap for 1 file.
+    # this should be not big not to exceed mlock limit.
+    _max_tracked = 10
+
     def __init__(t, tdb, zf, at=None):
         assert isinstance(zf, ZBigFile)
         t.tdb = tdb
         t.f   = tdb.open(zf, at=at)
         t.blksize = zf.blksize
 
+        # mmap the file past the end up to XXX pages and lock the pages with
+        # MLOCK_ONFAULT. This way when a page is read by mmap access we have
+        # the guarantee from kernel that the page will stay in pagecache. We
+        # rely on this to verify OS cache state.
+        assert t.blksize % mm.PAGE_SIZE == 0
+        t.fmmap = mm.map_ro(t.f.fileno(), 0, t._max_tracked*t.blksize)
+        mm.lock(t.fmmap, mm.MLOCK_ONFAULT)
+
     def close(t):
+        # XXX remove from t.tdb.tracked
+        mm.unmap(t.fmmap)
         t.f.close()
 
     """
@@ -245,6 +261,26 @@ class tFile:
         pass    # TODO
 
 
+    # blk returns bytearray connected to view of file[blk].
+    def blk(t, blk):
+        assert blk <= t._max_tracked
+        return bytearray(t.fmmap[blk*t.blksize:(blk+1)*t.blksize])
+
+    # assertBlk asserts that file block #blk has data as expected.
+    #
+    # Expected data may be given with size < t.blksize. In such case the data
+    # is implicitly appended with trailing zeros.
+    def assertBlk(t, blk, data):
+        assert len(data) <= t.blksize
+        data += b'\0'*(t.blksize - len(data))   # tailing zeros
+        st = os.fstat(t.f.fileno())
+        assert st.st_size % t.blksize == 0
+        assert blk < (st.st_size // t.blksize)
+        assert st.st_size // t.blksize <= t._max_tracked
+
+        assert t.blk(blk) == data, ("#blk: %d" % blk)
+
+
     # assertData asserts that file has data blocks as specified.
     #
     # Expected blocks may be given with size < zf.blksize. In such case they
@@ -253,12 +289,16 @@ class tFile:
     # It also check file size and optionally mtime.
     #
     # XXX also check pagecache state?
-    def assertData(t, blkv, mtime=None):
+    def assertData(t, datav, mtime=None):
         st = os.fstat(t.f.fileno())
-        assert st.st_size == len(blkv)*t.blksize
+        assert st.st_size == len(datav)*t.blksize
         if mtime is not None:
             assert st.st_mtime == tidtime(mtime)
 
+        for blk, data in enumerate(datav):
+            t.assertBlk(blk, data)
+
+        """
         # XXX hack -> access mapped page
         t.f.seek(0)
         data = t.f.read()
@@ -267,6 +307,7 @@ class tFile:
             assert len(blk) <= t.blksize
             blk += b'\0'*(t.blksize - len(blk)) # trailing zeros
             assert data[i*t.blksize:(i+1)*t.blksize] == blk, ("#blk: %d" % i)
+        """
 
         # XXX assertCache for read blocks to be 1
 
@@ -291,7 +332,7 @@ def test_wcfs():
         t.stat(nonfile)
     assert exc.value.errno == EINVAL
 
-    f = t.track(zf)
+    f = t.track(zf) # XXX -> testopen, fileopen?
 
     # file initially empty
     f.assertCache([])
@@ -299,17 +340,16 @@ def test_wcfs():
 
     # commit data to zf and make sure we can see it on wcfs
     # use !wcfs mode so that we prepare data independently of wcfs code paths.
-    hole = 10
     zfh = zf.fileh_open(_use_wcfs=False)
-    vma = zfh.mmap(hole, 1)     # 1 page at offset=10
+    vma = zfh.mmap(2, 1)     # 1 page at offset=2
     s   = b"hello world"
     memcpy(vma, s)
 
     t.commit()
     t.wcsync()    # sync wcfs to ZODB
 
-    f.assertCache([0]*(hole+1))  # initially not cached
-    f.assertData ([b'']*hole + [s], mtime=t.head)
+    f.assertCache([0,0,0])  # initially not cached
+    f.assertData ([b'',b'',s], mtime=t.head)
     # XXX assertCache all present?
 
 
@@ -317,8 +357,8 @@ def test_wcfs():
     tcommit1 = t.head
 
     zfh  = zf.fileh_open(_use_wcfs=False)
-    vma1 = zfh.mmap(hole, 1)
-    vma2 = zfh.mmap(hole+1, 1)
+    vma1 = zfh.mmap(2,   1)
+    vma2 = zfh.mmap(2+1, 1)
     s1  = b"hello 123"
     s2  = b"alpha"
     memcpy(vma1,s1)
@@ -328,13 +368,13 @@ def test_wcfs():
     t.wcsync()
 
     # f @head
-    f.assertCache([1]*hole + [0,0])
-    f.assertData ([b'']*hole + [s1+b'ld', s2], mtime=t.head)
+    f.assertCache([1,1,0,0])
+    f.assertData ([b'',b'', s1+b'ld', s2], mtime=t.head)
 
     # f @tcommit1
     f1 = t.track(zf, at=tcommit1)
-    f1.assertCache([0]*hole + [1])
-    f1.assertData ([b'']*hole + [s])  # XXX + mtime=tcommit1?
+    f1.assertCache([0,0,1])
+    f1.assertData ([b'',b'',s])  # XXX + mtime=tcommit1?
 
 
 # TODO pagecache state after loading (via mincore)