[IdTool] Use OOBTree instead of PersistentMapping in IdTool.

Use OOBTree instead of PersistentMapping. Because it is more disk-efficent,
it will reduce the number of conflicts, improve the performance.

product/ERP5/Document/SQLNonContinuousIncreasingIdGenerator.py

@@ -29,13 +29,13 @@
 import zope.interface
 from Acquisition import aq_base
 from AccessControl import ClassSecurityInfo
-from Products.ERP5Type.Globals import PersistentMapping
 from Products.ERP5Type import Permissions, PropertySheet, interfaces
 from Products.ERP5Type.Utils import ScalarMaxConflictResolver
 from Products.ERP5.Document.IdGenerator import IdGenerator
 from _mysql_exceptions import ProgrammingError
 from MySQLdb.constants.ER import NO_SUCH_TABLE
 from zLOG import LOG, INFO
+from BTrees.OOBTree import OOBTree
 
 class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
   """
@@ -135,7 +135,7 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
       id_group_done.append(id_group)
    
     # save the last ids which not exist in sql
-    for id_group in (set(self.last_max_id_dict) - set(id_group_done)):
+    for id_group in (set(self.last_max_id_dict.keys()) - set(id_group_done)):
       set_last_id_method(id_group=id_group,
           last_id=self.last_max_id_dict[id_group].value)
 
@@ -168,7 +168,7 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
     LOG('initialize SQL Generator', INFO, 'Id Generator: %s' % (self,))
     # Check the dictionnary
     if self.last_max_id_dict is None:
-      self.last_max_id_dict = PersistentMapping()
+      self.last_max_id_dict = OOBTree()
     # Create table portal_ids if not exists
     portal = self.getPortalObject()
     try:
@@ -219,7 +219,7 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
       added here)
     """
     # Remove dictionary
-    self.last_max_id_dict = PersistentMapping()
+    self.last_max_id_dict = OOBTree()
     # Remove and recreate portal_ids table
     portal = self.getPortalObject()
     portal.IdTool_zDropTable()
@@ -263,9 +263,24 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
     # Update persistent dict
     if self.getStoredInZodb():
       if self.last_max_id_dict is None:
-        self.last_max_id_dict = PersistentMapping()
+        self.last_max_id_dict = OOBTree()
       self.last_max_id_dict.update(new_id_dict)
 
+  security.declareProtected(Permissions.ModifyPortalContent,
+       'rebuildGeneratorIdDict')
+  def rebuildGeneratorIdDict(self):
+    """
+      Rebuild generator id dict from SQL table.
+
+      This is usefull when we are migrating the dict structure, or cleanly
+      rebuild the dict from sql table. This method is opposite of
+      rebuildSqlTable().
+    """
+    if not self.getStoredInZodb():
+      raise RuntimeError('Please set \"stored in zodb\" flag before rebuild.')
+    id_dict = self.exportGeneratorIdDict()
+    self.importGeneratorIdDict(id_dict=id_dict, clear=True)
+
   security.declareProtected(Permissions.ModifyPortalContent,
       'rebuildSqlTable')
   def rebuildSqlTable(self):
@@ -319,7 +334,7 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
     portal = self.getPortalObject()
     last_max_id_dict = self.last_max_id_dict
     if last_max_id_dict is None:
-      self.last_max_id_dict = last_max_id_dict = PersistentMapping()
+      self.last_max_id_dict = last_max_id_dict = OOBTree()
     last_id_group = None
     for line in portal.IdTool_zGetValueList(id_group=id_group):
       last_id_group = id_group = line[0]

product/ERP5/Document/ZODBContinuousIncreasingIdGenerator.py

@@ -28,9 +28,9 @@
 
 import zope.interface
 from AccessControl import ClassSecurityInfo
-from Products.ERP5Type.Globals import PersistentMapping
 from Products.ERP5Type import Permissions, interfaces
 from Products.ERP5.Document.IdGenerator import IdGenerator
+from BTrees.OOBTree import OOBTree
 
 from zLOG import LOG, INFO
 
@@ -97,7 +97,7 @@ class ZODBContinuousIncreasingIdGenerator(IdGenerator):
     """
     LOG('initialize ZODB Generator', INFO, 'Id Generator: %s' % (self,))
     if getattr(self, 'last_id_dict', None) is None:
-      self.last_id_dict = PersistentMapping()
+      self.last_id_dict = OOBTree() 
 
     # XXX compatiblity code below, dump the old dictionnaries
     portal_ids = getattr(self, 'portal_ids', None)
@@ -124,7 +124,7 @@ class ZODBContinuousIncreasingIdGenerator(IdGenerator):
       added here)
     """
     # Remove dictionary
-    self.last_id_dict = PersistentMapping()
+    self.last_id_dict = OOBTree()
 
   security.declareProtected(Permissions.ModifyPortalContent,
       'exportGeneratorIdDict')
@@ -149,3 +149,17 @@ class ZODBContinuousIncreasingIdGenerator(IdGenerator):
       if not isinstance(value, int):
         raise TypeError, 'the value given in dictionary is not a integer'
     self.last_id_dict.update(id_dict)
+
+  security.declareProtected(Permissions.ModifyPortalContent,
+       'rebuildGeneratorIdDict')
+  def rebuildGeneratorIdDict(self):
+    """
+      Rebuild generator id dict.
+      In fact, export it, clear it and import it into new dict.
+      This is mostly intendted to use when we are migrating the id dict
+      structure.
+    """
+    id_dict = self.exportGeneratorIdDict()
+    self.importGeneratorIdDict(id_dict=id_dict, clear=True)
+
+

product/ERP5/tests/testIdTool.py

@@ -33,6 +33,7 @@ import unittest
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
 from Products.ERP5Type.tests.utils import createZODBPythonScript
 from _mysql_exceptions import ProgrammingError
+from BTrees.OOBTree import OOBTree
 
 class TestIdTool(ERP5TypeTestCase):
 
@@ -154,7 +155,7 @@ class TestIdTool(ERP5TypeTestCase):
     zodb_generator = self.getLastGenerator('test_application_zodb')
     zodb_portal_type = 'ZODB Continuous Increasing Id Generator'
     self.assertEquals(zodb_generator.getPortalType(), zodb_portal_type)
-    self.assertEqual(getattr(zodb_generator, 'last_id_dict', {}), {})
+    self.assertEqual(len(zodb_generator.last_id_dict), 0)
     # generate ids
     self.checkGenerateNewId('test_application_zodb')
     # check zodb dict
@@ -171,7 +172,11 @@ class TestIdTool(ERP5TypeTestCase):
     sql_generator = self.getLastGenerator('test_application_sql')
     sql_portal_type = 'SQL Non Continuous Increasing Id Generator'
     self.assertEquals(sql_generator.getPortalType(), sql_portal_type)
-    self.assertEquals(getattr(sql_generator, 'last_max_id_dict', {}), {})
+    # This assertEquals() make sure that last_max_id_dict property is empty.
+    # Note that keys(), values() and items() methods of OOBTree do not return
+    # a list of all the items. The methods return a lazy evaluated object.
+    # len() method on OOBTree can handle properly even in the situation.
+    self.assertEquals(len(sql_generator.last_max_id_dict), 0)
     # retrieve method to recovery the last id in the database
     last_id_method = getattr(self.portal, 'IdTool_zGetLastId', None)
     self.assertNotEquals(last_id_method, None)
@@ -189,7 +194,7 @@ class TestIdTool(ERP5TypeTestCase):
       self.assertEquals(sql_generator.last_max_id_dict['c02'].value, 0)
       self.assertEquals(sql_generator.last_max_id_dict['d02'].value, 21)
     else:
-      self.assertEquals(getattr(sql_generator, 'last_max_id_dict', {}), {})
+      self.assertEquals(len(sql_generator.last_max_id_dict), 0)
 
   def test_02b_generateNewIdWithSQLGeneratorWithoutStorageZODB(self):
     """

product/ERP5/tests/testIdToolUpgrade.py

@@ -27,12 +27,17 @@
 #
 ##############################################################################
 
+import unittest
+
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
 from Products.ERP5Type.Globals import PersistentMapping
+from Products.ERP5Type.Utils import ScalarMaxConflictResolver
 from BTrees.Length import Length
+from BTrees.OOBTree import OOBTree
 from zLOG import LOG
 
-class TestIdTool(ERP5TypeTestCase):
+
+class TestIdToolUpgrade(ERP5TypeTestCase):
   """
   Automatic upgrade of id tool is really sensible to any change. Therefore,
   make sure that the upgrade is still working even if there changes.
@@ -46,6 +51,66 @@ class TestIdTool(ERP5TypeTestCase):
     """
     return "Test Id Tool Upgrade"
 
+  def afterSetUp(self):
+    self.login()
+    self.portal = self.getPortal()
+    self.id_tool = self.portal.portal_ids
+    self.id_tool.initializeGenerator(all=True)
+    self.createGenerators()
+    self.tic()
+
+  def beforeTearDown(self):
+    self.id_tool.clearGenerator(all=True)
+
+  def createGenerators(self):
+    """
+      Initialize some generators for the tests
+    """
+    self.application_sql_generator = self.id_tool.newContent(\
+                            portal_type='Application Id Generator',
+                            reference='test_application_sql',
+                            version='001')
+    self.conceptual_sql_generator = self.id_tool.newContent(\
+                           portal_type='Conceptual Id Generator',
+                           reference='test_non_continuous_increasing',
+                           version='001')
+    self.sql_generator = self.id_tool.newContent(\
+                    portal_type='SQL Non Continuous Increasing Id Generator',
+                    reference='test_sql_non_continuous_increasing',
+                    version='001')
+    self.application_sql_generator.setSpecialiseValue(\
+                                   self.conceptual_sql_generator)
+    self.conceptual_sql_generator.setSpecialiseValue(self.sql_generator)
+
+    self.application_zodb_generator = self.id_tool.newContent(\
+                            portal_type='Application Id Generator',
+                            reference='test_application_zodb',
+                            version='001')
+    self.conceptual_zodb_generator = self.id_tool.newContent(\
+                           portal_type='Conceptual Id Generator',
+                           reference='test_continuous_increasing',
+                           version='001')
+    self.zodb_generator = self.id_tool.newContent(\
+                    portal_type='ZODB Continuous Increasing Id Generator',
+                    reference='test_zodb_continuous_increasing',
+                    version='001')
+    self.application_zodb_generator.setSpecialiseValue(\
+                                    self.conceptual_zodb_generator)
+    self.conceptual_zodb_generator.setSpecialiseValue(self.zodb_generator)
+
+  def getLastGenerator(self, id_generator):
+    """
+      Return Last Id Generator
+    """
+    document_generator = self.id_tool.searchFolder(reference=id_generator)[0]
+    application_generator = document_generator.getLatestVersionValue()
+    conceptual_generator = application_generator.getSpecialiseValue()\
+                           .getLatestVersionValue()
+    last_generator = conceptual_generator.getSpecialiseValue()\
+                     .getLatestVersionValue()
+    return last_generator
+
+
   def testUpgradeIdToolDicts(self):
     # With old erp5_core, we have no generators, no IdTool_* zsql methods,
     # and we have a dictionary stored on id tool
@@ -123,3 +188,132 @@ class TestIdTool(ERP5TypeTestCase):
     generator = generator_list[0]
     self.assertEquals(generator.last_id_dict['foo'], 4)
     self.assertEquals(generator.last_id_dict["('bar', 'baz')"], 3)
+
+
+  def _setUpLastMaxIdDict(self, id_generator_reference):
+    def countup(id_generator, id_group, until):
+      for i in xrange(until + 1):
+        self.id_tool.generateNewId(id_generator=id_generator_reference,
+                                   id_group=id_group)
+
+    countup(id_generator_reference, 'A-01', 2)
+    countup(id_generator_reference, 'B-01', 1)
+    var_id = 'C-%04d'
+    for x in xrange(self.a_lot_of_key):
+      countup(id_generator_reference, var_id % x, 0)
+
+  def _getLastIdDictName(self, id_generator):
+    portal_type = id_generator.getPortalType()
+    if portal_type == 'SQL Non Continuous Increasing Id Generator':
+      return 'last_max_id_dict'
+    elif portal_type == 'ZODB Continuous Increasing Id Generator':
+      return 'last_id_dict'
+    else:
+      raise RuntimeError("not expected to test the generator :%s" % portal_type)
+
+  def _getLastIdDict(self, id_generator):
+    last_id_dict_name = self._getLastIdDictName(id_generator)
+    return getattr(id_generator, last_id_dict_name)
+
+  def _setLastIdDict(self, id_generator, value):
+    last_id_dict_name = self._getLastIdDictName(id_generator)
+    setattr(id_generator, last_id_dict_name, value)
+
+  def _getValueFromLastIdDict(self, last_id_dict, key):
+    value = last_id_dict[key]
+    if isinstance(value, int):
+      # in ZODB Id Generator it is stored in int
+      return value
+    elif isinstance(value, ScalarMaxConflictResolver):
+      return value.value
+    else:
+      raise RuntimeError('not expected to test the value: %s' % value)
+
+  def _assertIdGeneratorLastMaxIdDict(self, id_generator):
+    last_id_dict = self._getLastIdDict(id_generator)
+    self.assertEquals(2, self._getValueFromLastIdDict(last_id_dict, 'A-01'))
+    self.assertEquals(1, self._getValueFromLastIdDict(last_id_dict, 'B-01'))
+    for x in xrange(self.a_lot_of_key):
+      key = 'C-%04d' % x
+      self.assertEquals(0, self._getValueFromLastIdDict(last_id_dict, key))
+
+    # 1(A-01) + 1(B-01) + a_lot_of_key(C-*)
+    number_of_group_id = self.a_lot_of_key + 2
+    self.assertEqual(number_of_group_id,
+                     len(id_generator.exportGeneratorIdDict()))
+    self.assertEqual(number_of_group_id, len(last_id_dict))
+
+
+  def _checkDataStructureMigration(self, id_generator):
+    """ First, simulate previous data structure which is using
+    PersisntentMapping as the storage, then migrate to OOBTree.
+    Then, migrate the id generator again from OOBTree to OOBtree
+    just to be sure."""
+    id_generator_reference  = id_generator.getReference()
+
+    reference_portal_type_dict = {
+      'test_sql_non_continuous_increasing':'SQL Non Continuous ' \
+                                           'Increasing Id Generator',
+      'test_zodb_continuous_increasing':'ZODB Continuous ' \
+                                        'Increasing Id Generator'
+    }
+    try:
+      portal_type = reference_portal_type_dict[id_generator_reference]
+      self.assertEquals(id_generator.getPortalType(), portal_type)
+    except:
+      raise ValueError("reference is not valid: %s" % id_generator_reference)
+
+    self._setLastIdDict(id_generator, PersistentMapping()) # simulate previous
+    last_id_dict = self._getLastIdDict(id_generator)
+
+    # setUp the data for migration test
+    self._setUpLastMaxIdDict(id_generator_reference)
+
+    # test migration: PersistentMapping to OOBTree
+    self.assertTrue(isinstance(last_id_dict, PersistentMapping))
+    self._assertIdGeneratorLastMaxIdDict(id_generator)
+    id_generator.rebuildGeneratorIdDict() # migrate the dict
+    self._assertIdGeneratorLastMaxIdDict(id_generator)
+
+    # test migration: OOBTree to OOBTree. this changes nothing, just to be sure
+    last_id_dict = self._getLastIdDict(id_generator)
+    self.assertTrue(isinstance(last_id_dict, OOBTree))
+    self._assertIdGeneratorLastMaxIdDict(id_generator)
+    id_generator.rebuildGeneratorIdDict() # migrate the dict
+    self._assertIdGeneratorLastMaxIdDict(id_generator)
+
+    # test migration: SQL to OOBTree
+    if id_generator.getPortalType() == \
+      'SQL Non Continuous Increasing Id Generator':
+      self._setLastIdDict(id_generator, OOBTree()) # set empty one
+      last_id_dict = self._getLastIdDict(id_generator)
+      assert(len(last_id_dict), 0) # 0 because it is empty
+      self.assertTrue(isinstance(last_id_dict, OOBTree))
+      # migrate the dict totally from sql table in this case
+      id_generator.rebuildGeneratorIdDict()
+      self._assertIdGeneratorLastMaxIdDict(id_generator)
+
+
+  def testRebuildIdDictFromPersistentMappingToOOBTree(self):
+    """
+      Check migration is working
+    """
+    # this is the amount of keys that is creating in this test
+    self.a_lot_of_key = 1010
+    # check sql id generator migration
+    id_generator_reference = 'test_application_sql'
+    id_generator = self.getLastGenerator(id_generator_reference)
+    id_generator.setStoredInZodb(True)
+    id_generator.clearGenerator() # clear stored data
+    self._checkDataStructureMigration(id_generator)
+
+    # check zodb id generator migration
+    id_generator_reference = 'test_application_zodb'
+    id_generator = self.getLastGenerator(id_generator_reference)
+    id_generator.clearGenerator() # clear stored data
+    self._checkDataStructureMigration(id_generator)
+
+def test_suite():
+  suite = unittest.TestSuite()
+  suite.addTest(unittest.makeSuite(TestIdToolUpgrade))
+  return suite