HBTreeFolder2 fixes

There's no magic in this patch series: it is known that HBTreeFolder2 has
limitations about the ids that can be set without conflict, and this can't be
fixed without causing compatibility issues with existing data.

The patches contain:
- some optimization
- bug fixes
- detection of id conflicts before causing data loss

This will also allow us to use a newer version of ZODB. Recent BTrees failed
on the following line of `_setOb`:

        if len(id_list) == 1 and not htree.has_key(None):

(None is not valid key for comparison since
 ZODB commit bb5aac21277f43333d6450064dc6670c8c280e40)

The long story about id conflicts is that a HBTreeFolder2 can't store 2 objects
<A> and <A>-<B> where <A> does not contain '-', and that's the rule followed by
_getOb/_setOb/_delOb. However:
- Conflicts are detected by testing the type of the value, which means
  HBTreeFolder2 can't store values of the same type as the one it uses
  internally (i.e. OOBTree).
- For performance reasons, _htree_iteritems and getTreeIdList use a stricter
  rule: they assume there can't be 2 objects <A> and <A>-<B>, regardless of the
  presence of a separator in <A>. Maybe this rule should be enforced in _setOb.

/reviewed-on nexedi/erp5!112

product/ERP5Type/Core/Folder.py

@@ -904,30 +904,6 @@ class Folder(CopyContainer, CMFBTreeFolder, CMFHBTreeFolder, Base, FolderMixIn):
         return False
       return CMFBTreeFolder.has_key(self, id)
 
-  def treeIds(self, base_id=None):
-    """ Return a list of subtree ids
-    """
-    if self._folder_handler == HBTREE_HANDLER:
-      return CMFHBTreeFolder.treeIds(self, base_id)
-    else:
-      return CMFBTreeFolder.treeIds(self, base_id)
-
-  def _getTree(self, base_id):
-    """ Return the tree wich has the base_id
-    """
-    if self._folder_handler == HBTREE_HANDLER:
-      return CMFHBTreeFolder._getTree(self, base_id)
-    else:
-      return CMFBTreeFolder._getTree(self, base_id)
-
-  def _getTreeIdList(self, htree=None):
-    """ recursively build a list of btree ids
-    """
-    if self._folder_handler == HBTREE_HANDLER:
-      return CMFHBTreeFolder._getTreeIdList(self, htree)
-    else:
-      return CMFBTreeFolder._getTreeIdList(self, htree)
-
   def getTreeIdList(self, htree=None):
     """ recursively build a list of btree ids
     """
@@ -936,38 +912,6 @@ class Folder(CopyContainer, CMFBTreeFolder, CMFHBTreeFolder, Base, FolderMixIn):
     else:
       return CMFBTreeFolder.getTreeIdList(self, htree)
 
-  def _treeObjectValues(self, base_id=None):
-    """ return object values for a given btree
-    """
-    if self._folder_handler == HBTREE_HANDLER:
-      return CMFHBTreeFolder._treeObjectValues(self, base_id)
-    else:
-      return CMFBTreeFolder._treeObjectValues(self, base_id)
-
-  def _treeObjectIds(self, base_id=None):
-    """ return object ids for a given btree
-    """
-    if self._folder_handler == HBTREE_HANDLER:
-      return CMFHBTreeFolder._treeObjectIds(self, base_id)
-    else:
-      return CMFBTreeFolder._treeObjectIds(self, base_id)
-
-  def _isNotBTree(self, obj):
-    """ test object is not a btree
-    """
-    if self._folder_handler == HBTREE_HANDLER:
-      return CMFHBTreeFolder._isNotBTree(self, obj)
-    else:
-      return CMFBTreeFolder._isNotBTree(self, obj)
-
-  def _checkObjectId(self, id):
-    """ test id is not in btree id list
-    """
-    if self._folder_handler == HBTREE_HANDLER:
-      return CMFHBTreeFolder._checkObjectId(self, id)
-    else:
-      return CMFBTreeFolder._checkObjectId(self, id)
-
   def objectIds(self, spec=None, **kw):
     if self._folder_handler == HBTREE_HANDLER:
       if self._htree is None:

product/HBTreeFolder2/tests/testHBTreeFolder2.py

@@ -39,6 +39,28 @@ class HBTreeFolder2Tests(ERP5TypeTestCase):
         self.f._setOb(ff.id, ff)
         self.ff = self.f._getOb(ff.id)
 
+    def testKey(self):
+        f = self.f
+        ff = f.item
+        ok = "a", "b-a", "b-b", "c-a-b", "c-a-d"
+        for id in ok:
+          f._setOb(id, ff)
+          f._getOb(id)
+        for id in "a-a", "b", "c-a":
+          if id != "c-a":
+            self.assertRaises(KeyError, f._setOb, id, ff)
+          self.assertRaises(KeyError, f._getOb, id)
+          self.assertRaises(KeyError, f._delOb, id)
+        self.assertEqual(len(f), 1 + len(ok))
+        self.assertEqual(f.getTreeIdList(), [None, "b", "c-a"])
+        self.assertEqual(len(f._htree), 4)
+        for id in ok:
+          f._delOb(id)
+        self.assertEqual(len(f._htree), 1)
+        self.assertEqual(len(f), 1)
+        self.assertEqual(f.getTreeIdList(), [None])
+        self.assertEqual(ff.getTreeIdList(), [])
+
     def testAdded(self):
         self.assertEqual(self.ff.id, 'item')
 
@@ -99,11 +121,12 @@ class HBTreeFolder2Tests(ERP5TypeTestCase):
 
     def testWrapped(self):
         # Verify that the folder returns wrapped versions of objects.
-        base = self.getBase(self.f._getOb('item'))
-        self.assert_(self.f._getOb('item') is not base)
-        self.assert_(self.f['item'] is not base)
-        self.assert_(self.f.get('item') is not base)
-        self.assert_(self.getBase(self.f._getOb('item')) is base)
+        base = self.f.item
+        for x in (self.f._getOb('item'),
+                  self.f['item'],
+                  self.f.get('item')):
+            self.assertIsNot(base, x)
+            self.assertIs(base, self.getBase(x))
 
     def testGenerateId(self):
         ids = {}
@@ -173,23 +196,6 @@ class HBTreeFolder2Tests(ERP5TypeTestCase):
         expect = '<option value="%s">%s</option>' % (name, name)
         self.assert_(info['formatted_list'].find(expect) > 0)
 
-    def testCleanup(self):
-        self.assert_(self.f._cleanup())
-        key = TrojanKey('a')
-        self.f._htree[key] = 'b'
-        self.assert_(self.f._cleanup())
-        key.value = 'z'
-
-        # With a key in the wrong place, there should now be damage.
-        self.assert_(not self.f._cleanup())
-        # Now it's fixed.
-        self.assert_(self.f._cleanup())
-        # Verify the management interface also works,
-        # but don't test return values.
-        self.f.manage_cleanup()
-        key.value = 'a'
-        self.f.manage_cleanup()
-
     def testIterItems(self):
         h = HBTreeFolder2()
         id_list = ['a-b', 'a-cd',
@@ -328,21 +334,6 @@ class HBTreeFolder2Tests(ERP5TypeTestCase):
         self.assertTrue(t3_2 > t3_3)
 
 
-class TrojanKey:
-    """Pretends to be a consistent, immutable, humble citizen...
-
-    then sweeps the rug out from under the HBTree.
-    """
-    def __init__(self, value):
-        self.value = value
-
-    def __cmp__(self, other):
-        return cmp(self.value, other)
-
-    def __hash__(self):
-        return hash(self.value)
-
-
 def test_suite():
     return unittest.TestSuite((
         unittest.makeSuite(HBTreeFolder2Tests),