Add PersistentList.py and fsIndex.py and their tests (merge)

trunk/src/Persistence/PersistentList.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+# 
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+# 
+##############################################################################
+
+"""Python implementation of persistent list.
+
+$Id: PersistentList.py,v 1.2 2002/02/11 23:49:07 gvanrossum Exp $"""
+
+__version__='$Revision: 1.2 $'[11:-2]
+
+import Persistence
+from UserList import UserList
+
+class PersistentList(UserList, Persistence.Persistent):
+    __super_setitem = UserList.__setitem__
+    __super_delitem = UserList.__delitem__
+    __super_setslice = UserList.__setslice__
+    __super_delslice = UserList.__delslice__
+    __super_iadd = UserList.__iadd__
+    __super_imul = UserList.__imul__
+    __super_append = UserList.append
+    __super_insert = UserList.insert
+    __super_pop = UserList.pop
+    __super_remove = UserList.remove
+    __super_reverse = UserList.reverse
+    __super_sort = UserList.sort
+    __super_extend = UserList.extend
+
+    def __setitem__(self, i, item):
+        self.__super_setitem(i, item)
+        self._p_changed = 1
+
+    def __delitem__(self, i):
+        self.__super_delitem(i)
+        self._p_changed = 1
+
+    def __setslice__(self, i, j, other):
+        self.__super_setslice(i, j, other)
+        self._p_changed = 1
+
+    def __delslice__(self, i, j):
+        self.__super_delslice(i, j)
+        self._p_changed = 1
+    
+    def __iadd__(self, other):
+        self.__super_iadd(other)
+        self._p_changed = 1
+
+    def __imul__(self, n):
+        self.__super_imul(n)
+        self._p_changed = 1
+
+    def append(self, item):
+        self.__super_append(item)
+        self._p_changed = 1
+        
+    def insert(self, i, item):
+        self.__super_insert(i, item)
+        self._p_changed = 1
+
+    def pop(self, i=-1):
+        rtn = self.__super_pop(i)
+        self._p_changed = 1
+        return rtn
+
+    def remove(self, item):
+        self.__super_remove(item)
+        self._p_changed = 1
+        
+    def reverse(self):
+        self.__super_reverse()
+        self._p_changed = 1
+        
+    def sort(self, *args):
+        self.__super_sort(*args)
+        self._p_changed = 1
+
+    def extend(self, other):
+        self.__super_extend(other)
+        self._p_changed = 1
+
+    # This works around a bug in Python 2.1.x (up to 2.1.2 at least) where the
+    # __cmp__ bogusly raises a RuntimeError, and because this is an extension
+    # class, none of the rich comparison stuff works anyway.
+    def __cmp__(self, other):
+        return cmp(self.data, self._UserList__cast(other))

trunk/src/ZODB/PersistentList.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+# 
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+# 
+##############################################################################
+
+"""Python implementation of persistent list.
+
+$Id: PersistentList.py,v 1.2 2002/02/11 23:49:07 gvanrossum Exp $"""
+
+__version__='$Revision: 1.2 $'[11:-2]
+
+import Persistence
+from UserList import UserList
+
+class PersistentList(UserList, Persistence.Persistent):
+    __super_setitem = UserList.__setitem__
+    __super_delitem = UserList.__delitem__
+    __super_setslice = UserList.__setslice__
+    __super_delslice = UserList.__delslice__
+    __super_iadd = UserList.__iadd__
+    __super_imul = UserList.__imul__
+    __super_append = UserList.append
+    __super_insert = UserList.insert
+    __super_pop = UserList.pop
+    __super_remove = UserList.remove
+    __super_reverse = UserList.reverse
+    __super_sort = UserList.sort
+    __super_extend = UserList.extend
+
+    def __setitem__(self, i, item):
+        self.__super_setitem(i, item)
+        self._p_changed = 1
+
+    def __delitem__(self, i):
+        self.__super_delitem(i)
+        self._p_changed = 1
+
+    def __setslice__(self, i, j, other):
+        self.__super_setslice(i, j, other)
+        self._p_changed = 1
+
+    def __delslice__(self, i, j):
+        self.__super_delslice(i, j)
+        self._p_changed = 1
+    
+    def __iadd__(self, other):
+        self.__super_iadd(other)
+        self._p_changed = 1
+
+    def __imul__(self, n):
+        self.__super_imul(n)
+        self._p_changed = 1
+
+    def append(self, item):
+        self.__super_append(item)
+        self._p_changed = 1
+        
+    def insert(self, i, item):
+        self.__super_insert(i, item)
+        self._p_changed = 1
+
+    def pop(self, i=-1):
+        rtn = self.__super_pop(i)
+        self._p_changed = 1
+        return rtn
+
+    def remove(self, item):
+        self.__super_remove(item)
+        self._p_changed = 1
+        
+    def reverse(self):
+        self.__super_reverse()
+        self._p_changed = 1
+        
+    def sort(self, *args):
+        self.__super_sort(*args)
+        self._p_changed = 1
+
+    def extend(self, other):
+        self.__super_extend(other)
+        self._p_changed = 1
+
+    # This works around a bug in Python 2.1.x (up to 2.1.2 at least) where the
+    # __cmp__ bogusly raises a RuntimeError, and because this is an extension
+    # class, none of the rich comparison stuff works anyway.
+    def __cmp__(self, other):
+        return cmp(self.data, self._UserList__cast(other))

trunk/src/ZODB/fsIndex.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+# 
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+# 
+##############################################################################
+"""Implement an OID to File-position (long integer) mapping
+"""
+# 
+# To save space, we do two things:
+# 
+#     1. We split the keys (OIDS) into 6-byte prefixes and 2-byte suffixes.
+#        We use the prefixes as keys in a mapping from prefix to mappings
+#        of suffix to data:
+# 
+#           data is  {prefix -> {suffix -> data}}
+# 
+#     2. We limit the data size to 48 bits. This should allow databases
+#        as large as 256 terabytes.
+# 
+# Mostof the space is consumed by items in the mappings from 2-byte
+# suffix to 6-byte data. This should reduce the overall memory usage to
+# 8-16 bytes per OID.
+# 
+# We use p64 to convert integers to 8-byte strings and lop off the two
+# high-order bytes when saving. On loading data, we add the leading
+# bytes back before using U64 to convert the data back to (long)
+# integers.
+
+from BTrees._fsBTree import fsBTree as _fsBTree
+
+import struct
+
+# convert between numbers and six-byte strings
+
+_t32 = 1L<< 32
+
+def num2str(n):
+    h, l = divmod(long(n), _t32)
+    return struct.pack(">HI", h, l)
+
+def str2num(s):
+    h, l = struct.unpack(">HI", s)
+    if h:
+        return (long(h) << 32) + l
+    else:
+       return l
+
+class fsIndex:
+
+    def __init__(self):
+        self._data = {}
+
+    def __getitem__(self, key):
+        return str2num(self._data[key[:6]][key[6:]])
+
+    def get(self, key, default=None):
+        tree = self._data.get(key[:6], default)
+        if tree is default:
+            return default
+        v = tree.get(key[6:], default)
+        if v is default:
+            return default
+        return str2num(v)
+
+    def __setitem__(self, key, value):
+        value = num2str(value)
+        treekey = key[:6]
+        tree = self._data.get(treekey)
+        if tree is None:
+            tree = _fsBTree()
+            self._data[treekey] = tree
+        tree[key[6:]] = value
+
+    def __len__(self):
+        r = 0
+        for tree in self._data.values():
+            r += len(tree)
+        return r
+
+    def update(self, mapping):
+        for k, v in mapping.items():
+            self[k] = v
+
+    def has_key(self, key):
+        v=self.get(key, self)
+        return v is not self
+
+    def clear(self):
+        self._data.clear()
+
+    def keys(self):
+        r = []
+        for prefix, tree in self._data.items():
+            for suffix in tree.keys():
+                r.append(prefix + suffix)
+        return r
+
+    def items(self):
+        r = []
+        for prefix, tree in self._data.items():
+            for suffix, v in tree.items():
+                r.append(((prefix + suffix), str2num(v)))
+        return r
+
+    def values(self):
+        r = []
+        for prefix, tree in self._data.items():
+            for v in tree.values():
+                r.append(str2num(v))
+        return r

trunk/src/ZODB/tests/testPersistentList.py

+"""Test the list interface to PersistentList
+"""
+
+import unittest
+from ZODB.PersistentList import PersistentList
+
+l0 = []
+l1 = [0]
+l2 = [0, 1]
+
+class TestPList(unittest.TestCase):
+    def checkTheWorld(self):
+        # Test constructors
+        u = PersistentList()
+        u0 = PersistentList(l0)
+        u1 = PersistentList(l1)
+        u2 = PersistentList(l2)
+
+        uu = PersistentList(u)
+        uu0 = PersistentList(u0)
+        uu1 = PersistentList(u1)
+        uu2 = PersistentList(u2)
+
+        v = PersistentList(tuple(u))
+        class OtherList:
+            def __init__(self, initlist):
+                self.__data = initlist
+            def __len__(self):
+                return len(self.__data)
+            def __getitem__(self, i):
+                return self.__data[i]
+        v0 = PersistentList(OtherList(u0))
+        vv = PersistentList("this is also a sequence")
+
+        # Test __repr__
+        eq = self.assertEqual
+
+        eq(str(u0), str(l0), "str(u0) == str(l0)")
+        eq(repr(u1), repr(l1), "repr(u1) == repr(l1)")
+        eq(`u2`, `l2`, "`u2` == `l2`")
+
+        # Test __cmp__ and __len__
+
+        def mycmp(a, b):
+            r = cmp(a, b)
+            if r < 0: return -1
+            if r > 0: return 1
+            return r
+
+        all = [l0, l1, l2, u, u0, u1, u2, uu, uu0, uu1, uu2]
+        for a in all:
+            for b in all:
+                eq(mycmp(a, b), mycmp(len(a), len(b)),
+                      "mycmp(a, b) == mycmp(len(a), len(b))")
+
+        # Test __getitem__
+
+        for i in range(len(u2)):
+            eq(u2[i], i, "u2[i] == i")
+
+        # Test __setitem__
+
+        uu2[0] = 0
+        uu2[1] = 100
+        try:
+            uu2[2] = 200
+        except IndexError:
+            pass
+        else:
+            raise TestFailed("uu2[2] shouldn't be assignable")
+
+        # Test __delitem__
+
+        del uu2[1]
+        del uu2[0]
+        try:
+            del uu2[0]
+        except IndexError:
+            pass
+        else:
+            raise TestFailed("uu2[0] shouldn't be deletable")
+
+        # Test __getslice__
+
+        for i in range(-3, 4):
+            eq(u2[:i], l2[:i], "u2[:i] == l2[:i]")
+            eq(u2[i:], l2[i:], "u2[i:] == l2[i:]")
+            for j in range(-3, 4):
+                eq(u2[i:j], l2[i:j], "u2[i:j] == l2[i:j]")
+
+        # Test __setslice__
+
+        for i in range(-3, 4):
+            u2[:i] = l2[:i]
+            eq(u2, l2, "u2 == l2")
+            u2[i:] = l2[i:]
+            eq(u2, l2, "u2 == l2")
+            for j in range(-3, 4):
+                u2[i:j] = l2[i:j]
+                eq(u2, l2, "u2 == l2")
+
+        uu2 = u2[:]
+        uu2[:0] = [-2, -1]
+        eq(uu2, [-2, -1, 0, 1], "uu2 == [-2, -1, 0, 1]")
+        uu2[0:] = []
+        eq(uu2, [], "uu2 == []")
+
+        # Test __contains__
+        for i in u2:
+            self.failUnless(i in u2, "i in u2")
+        for i in min(u2)-1, max(u2)+1:
+            self.failUnless(i not in u2, "i not in u2")
+
+        # Test __delslice__
+
+        uu2 = u2[:]
+        del uu2[1:2]
+        del uu2[0:1]
+        eq(uu2, [], "uu2 == []")
+
+        uu2 = u2[:]
+        del uu2[1:]
+        del uu2[:1]
+        eq(uu2, [], "uu2 == []")
+
+        # Test __add__, __radd__, __mul__ and __rmul__
+
+        #self.failUnless(u1 + [] == [] + u1 == u1, "u1 + [] == [] + u1 == u1")
+        self.failUnless(u1 + [1] == u2, "u1 + [1] == u2")
+        #self.failUnless([-1] + u1 == [-1, 0], "[-1] + u1 == [-1, 0]")
+        self.failUnless(u2 == u2*1 == 1*u2, "u2 == u2*1 == 1*u2")
+        self.failUnless(u2+u2 == u2*2 == 2*u2, "u2+u2 == u2*2 == 2*u2")
+        self.failUnless(u2+u2+u2 == u2*3 == 3*u2, "u2+u2+u2 == u2*3 == 3*u2")
+
+        # Test append
+
+        u = u1[:]
+        u.append(1)
+        eq(u, u2, "u == u2")
+
+        # Test insert
+
+        u = u2[:]
+        u.insert(0, -1)
+        eq(u, [-1, 0, 1], "u == [-1, 0, 1]")
+
+        # Test pop
+
+        u = PersistentList([0, -1, 1])
+        u.pop()
+        eq(u, [0, -1], "u == [0, -1]")
+        u.pop(0)
+        eq(u, [-1], "u == [-1]")
+
+        # Test remove
+
+        u = u2[:]
+        u.remove(1)
+        eq(u, u1, "u == u1")
+
+        # Test count
+        u = u2*3
+        eq(u.count(0), 3, "u.count(0) == 3")
+        eq(u.count(1), 3, "u.count(1) == 3")
+        eq(u.count(2), 0, "u.count(2) == 0")
+
+
+        # Test index
+
+        eq(u2.index(0), 0, "u2.index(0) == 0")
+        eq(u2.index(1), 1, "u2.index(1) == 1")
+        try:
+            u2.index(2)
+        except ValueError:
+            pass
+        else:
+            raise TestFailed("expected ValueError")
+
+        # Test reverse
+
+        u = u2[:]
+        u.reverse()
+        eq(u, [1, 0], "u == [1, 0]")
+        u.reverse()
+        eq(u, u2, "u == u2")
+
+        # Test sort
+
+        u = PersistentList([1, 0])
+        u.sort()
+        eq(u, u2, "u == u2")
+
+        # Test extend
+
+        u = u1[:]
+        u.extend(u2)
+        eq(u, u1 + u2, "u == u1 + u2")
+
+
+def test_suite():
+    return unittest.makeSuite(TestPList, 'check')
+
+if __name__ == "__main__":
+    loader = unittest.TestLoader()
+    loader.testMethodPrefix = "check"
+    unittest.main(testLoader=loader)

trunk/src/ZODB/tests/testfsIndex.py

+
+import unittest, sys
+from ZODB.fsIndex import fsIndex
+from ZODB.utils import p64
+
+
+class Test(unittest.TestCase):
+
+    def testInserts(self):
+        index=fsIndex()
+
+        for i in range(200):
+            index[p64(i*1000)]=(i*1000L+1)
+
+        for i in range(0,200):
+            self.assertEqual((i,index[p64(i*1000)]), (i,(i*1000L+1)))
+            
+        self.assertEqual(len(index), 200)
+
+        key=p64(2000)
+
+        self.assertEqual(index.get(key), 2001)
+
+        key=p64(2001)
+        self.assertEqual(index.get(key), None)
+        self.assertEqual(index.get(key, ''), '')
+
+        # self.failUnless(len(index._data) > 1)
+
+    def testUpdate(self):
+        index=fsIndex()
+        d={}
+
+        for i in range(200):
+            d[p64(i*1000)]=(i*1000L+1)
+
+        index.update(d)
+
+        for i in range(400,600):
+            d[p64(i*1000)]=(i*1000L+1)
+        
+        index.update(d)
+
+        for i in range(100, 500):
+            d[p64(i*1000)]=(i*1000L+2)
+            
+        index.update(d)
+
+        self.assertEqual(index.get(p64(2000)), 2001)
+        self.assertEqual(index.get(p64(599000)), 599001)
+        self.assertEqual(index.get(p64(399000)), 399002)
+        self.assertEqual(len(index), 600)
+
+
+def test_suite():
+    loader=unittest.TestLoader()
+    return loader.loadTestsFromTestCase(Test)
+
+if __name__=='__main__':
+    unittest.TextTestRunner().run(test_suite())

trunk/src/persistent/list.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+# 
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+# 
+##############################################################################
+
+"""Python implementation of persistent list.
+
+$Id: list.py,v 1.2 2002/02/11 23:49:07 gvanrossum Exp $"""
+
+__version__='$Revision: 1.2 $'[11:-2]
+
+import Persistence
+from UserList import UserList
+
+class PersistentList(UserList, Persistence.Persistent):
+    __super_setitem = UserList.__setitem__
+    __super_delitem = UserList.__delitem__
+    __super_setslice = UserList.__setslice__
+    __super_delslice = UserList.__delslice__
+    __super_iadd = UserList.__iadd__
+    __super_imul = UserList.__imul__
+    __super_append = UserList.append
+    __super_insert = UserList.insert
+    __super_pop = UserList.pop
+    __super_remove = UserList.remove
+    __super_reverse = UserList.reverse
+    __super_sort = UserList.sort
+    __super_extend = UserList.extend
+
+    def __setitem__(self, i, item):
+        self.__super_setitem(i, item)
+        self._p_changed = 1
+
+    def __delitem__(self, i):
+        self.__super_delitem(i)
+        self._p_changed = 1
+
+    def __setslice__(self, i, j, other):
+        self.__super_setslice(i, j, other)
+        self._p_changed = 1
+
+    def __delslice__(self, i, j):
+        self.__super_delslice(i, j)
+        self._p_changed = 1
+    
+    def __iadd__(self, other):
+        self.__super_iadd(other)
+        self._p_changed = 1
+
+    def __imul__(self, n):
+        self.__super_imul(n)
+        self._p_changed = 1
+
+    def append(self, item):
+        self.__super_append(item)
+        self._p_changed = 1
+        
+    def insert(self, i, item):
+        self.__super_insert(i, item)
+        self._p_changed = 1
+
+    def pop(self, i=-1):
+        rtn = self.__super_pop(i)
+        self._p_changed = 1
+        return rtn
+
+    def remove(self, item):
+        self.__super_remove(item)
+        self._p_changed = 1
+        
+    def reverse(self):
+        self.__super_reverse()
+        self._p_changed = 1
+        
+    def sort(self, *args):
+        self.__super_sort(*args)
+        self._p_changed = 1
+
+    def extend(self, other):
+        self.__super_extend(other)
+        self._p_changed = 1
+
+    # This works around a bug in Python 2.1.x (up to 2.1.2 at least) where the
+    # __cmp__ bogusly raises a RuntimeError, and because this is an extension
+    # class, none of the rich comparison stuff works anyway.
+    def __cmp__(self, other):
+        return cmp(self.data, self._UserList__cast(other))