Skip to content

erp5
erp5
Commit db6a43cd authored by Danièle Vanbaelinghem's avatar Danièle Vanbaelinghem
Browse files
Options

Add store interval on SQLNonContinuousIncreasingIdGenerator 

Add a export and import dictionary for the generators
Add a method to update the persistent dictionary in the SQL table


git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@36283 20353a03-c40f-0410-a6d1-a30d3c3de9de
parent e4a18449
Hide whitespace changes
Inline Side-by-side
Showing with 317 additions and 196 deletions
product/ERP5/Document/IdGenerator.py
View file @ db6a43cd
......@@ -131,3 +131,28 @@ class IdGenerator(Base):
self.getReference()
specialise.getLatestVersionValue().clearGenerator()
security.declareProtected(Permissions.ModifyPortalContent,
'exportGeneratorIdDict')
def exportGeneratorIdDict(self):
"""
Export last id values in a dictionnary in the form { group_id : last_id }
"""
specialise = self.getSpecialiseValue()
if specialise is None:
raise ValueError, "the id generator %s doesn't have specialise value" %\
self.getReference()
return specialise.getLatestVersionValue().exportGeneratorIdDict()
security.declareProtected(Permissions.ModifyPortalContent,
'importGeneratorIdDict')
def importGeneratorIdDict(self, id_dict, clear=False):
"""
Import data, this is usefull if we want to replace a generator by
another one.
"""
specialise = self.getSpecialiseValue()
if specialise is None:
raise ValueError, "the id generator %s doesn't have specialise value" %\
self.getReference()
specialise.getLatestVersionValue().importGeneratorIdDict(id_dict=id_dict,
clear=clear)
product/ERP5/Document/SQLNonContinuousIncreasingIdGenerator.py
View file @ db6a43cd
......@@ -90,7 +90,7 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
except ProgrammingError, error:
if error[0] != NO_SUCH_TABLE:
raise
# If the database not exist, initialise the generator
# If the database not exist, initialize the generator
self.initializeGenerator()
if self.getStoredInZodb():
# Store the new_id on ZODB if the checkbox storedInZodb is enabled
......@@ -100,12 +100,46 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
# If the dictionary not exist, initialize the generator
self.initializeGenerator()
last_max_id_dict = getattr(aq_base(self), 'last_max_id_dict')
# Store the new value id
if last_max_id_dict.get(id_group, None) is not None and \
last_max_id_dict[id_group].value > new_id:
raise ValueError, 'The last_id %s stored in zodb dictionary is ' \
'higher than the new id %s generated' % \
(last_max_id_dict[id_group].value, new_id)
# Check the store interval to store the data
store_interval = self.getStoreInterval()
if not store_interval:
store_interval = 1
# Store the new id
if last_max_id_dict.get(id_group, None) is None:
last_max_id_dict[id_group] = ScalarMaxConflictResolver(new_id)
last_max_id_dict[id_group].set(new_id)
elif last_max_id_dict[id_group].value <= (new_id - store_interval):
last_max_id_dict[id_group].set(new_id)
return new_id
def _updateSqlTable(self):
"""
Update the portal ids table with the data of persistent dictionary
"""
portal = self.getPortalObject()
get_value_list = getattr(portal, 'IdTool_zGetValueList')
set_last_id_method = getattr(portal, 'IdTool_zSetLastId')
id_group_done = []
# Save the last id of persistent dict if it is higher that
# the last id stored in the sql table
for line in get_value_list().dictionaries():
id_group = line['id_group']
last_id = line['last_id']
if self.last_max_id_dict.has_key(id_group) and \
self.last_max_id_dict[id_group].value > last_id:
set_last_id_method(id_group=id_group,
last_id=self.last_max_id_dict[id_group].value)
id_group_done.append(id_group)
# save the last ids which not exist in sql
for id_group in (set(self.last_max_id_dict) - set(id_group_done)):
set_last_id_method(id_group=id_group,
last_id=self.last_max_id_dict[id_group].value)
security.declareProtected(Permissions.AccessContentsInformation,
'generateNewId')
def generateNewId(self, id_group=None, default=None):
......@@ -178,13 +212,15 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
if storage and (not self.last_max_id_dict.has_key(id_group) or \
self.last_max_id_dict[id_group].value < last_insert_id):
self.last_max_id_dict[id_group] = ScalarMaxConflictResolver(last_insert_id)
self.last_max_id_dict[id_group].set(last_insert_id)
continue
last_id = int(last_id.value)
set_last_id_method(id_group=id_group, last_id=last_id)
if storage:
self.last_max_id_dict[id_group] = ScalarMaxConflictResolver(last_id)
self.last_max_id_dict[id_group].set(last_id)
# Store last_max_id_dict in mysql
if self.getStoredInZodb():
self._updateSqlTable()
security.declareProtected(Permissions.AccessContentsInformation,
'clearGenerator')
......@@ -210,7 +246,49 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
drop_method()
create_method()
security.declareProtected(Permissions.AccessContentsInformation,
security.declareProtected(Permissions.ModifyPortalContent,
'exportGeneratorIdDict')
def exportGeneratorIdDict(self):
"""
Export last id values in a dictionnary in the form { group_id : last_id }
"""
portal = self.getPortalObject()
# Store last_max_id_dict in mysql
if self.getStoredInZodb():
self._updateSqlTable()
# Return values from sql
get_value_list = getattr(portal, 'IdTool_zGetValueList')
return dict([(line['id_group'],int(line['last_id'])) for line in
get_value_list().dictionaries()])
security.declareProtected(Permissions.ModifyPortalContent,
'importGeneratorIdDict')
def importGeneratorIdDict(self, id_dict=None, clear=False):
"""
Import data, this is usefull if we want to replace a generator by
another one.
"""
if clear:
self.clearGenerator()
portal = self.getPortalObject()
set_last_id_method = getattr(portal, 'IdTool_zSetLastId')
if not isinstance(id_dict, dict):
raise TypeError, 'the argument given is not a dictionary'
new_id_dict = dict()
for key, value in id_dict.items():
if isinstance(value, int):
set_last_id_method(id_group=key, last_id=value)
# The id must be a ScalarMaxConflictResolver object for the persistent dict
new_id_dict[key] = ScalarMaxConflictResolver(value)
else:
raise TypeError, 'the value in the dictionary given is not a integer'
# Update persistent dict
if self.getStoredInZodb():
if getattr(self, 'last_max_id_dict', None) is None:
self.last_max_id_dict = PersistentMapping()
self.last_max_id_dict.update(new_id_dict)
security.declareProtected(Permissions.ModifyPortalContent,
'rebuildSqlTable')
def rebuildSqlTable(self):
"""
......@@ -221,14 +299,8 @@ class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
generation
"""
portal = self.getPortalObject()
getattr(portal, 'IdTool_zDropTable')()
getattr(self, 'SQLNonContinuousIncreasingIdGenerator_zCreateTable')()
security.declareProtected(Permissions.AccessContentsInformation,
'rebuildSqlTable')
def getPersistentIdDict(self):
"""
Return all data stored in zodb
"""
return dict([(x[0],x[1].value) for x in
getattr(self, 'last_max_id_dict', {}).iteritems()])
drop_method = getattr(portal, 'IdTool_zDropTable')
create_method = getattr(portal, 'IdTool_zCreateEmptyTable')
drop_method()
create_method()
self._updateSqlTable()
product/ERP5/Document/ZODBContinuousIncreasingIdGenerator.py
View file @ db6a43cd
......@@ -133,3 +133,26 @@ class ZODBContinuousIncreasingIdGenerator(IdGenerator):
# Remove dictionary
self.last_id_dict = PersistentMapping()
security.declareProtected(Permissions.ModifyPortalContent,
'exportGeneratorIdDict')
def exportGeneratorIdDict(self):
"""
Export last id values in a dictionnary in the form { group_id : last_id }
"""
return dict(self.last_id_dict)
security.declareProtected(Permissions.ModifyPortalContent,
'importGeneratorIdDict')
def importGeneratorIdDict(self, id_dict, clear=False):
"""
Import data, this is usefull if we want to replace a generator by
another one.
"""
if clear:
self.clearGenerator()
if not isinstance(id_dict, dict):
raise TypeError, 'the argument given is not a dictionary'
for value in id_dict.values():
if not isinstance(value, int):
raise TypeError, 'the value given in dictionary is not a integer'
self.last_id_dict.update(id_dict)
product/ERP5/PropertySheet/SQLIdGenerator.py
View file @ db6a43cd
......@@ -40,4 +40,10 @@ class SQLIdGenerator:
'type' :'boolean',
'mode' :'w'
},
{'id' :'store_interval',
'label' :'The generator store the last id in the zodb \
every store interval',
'type' :'int',
'mode' :'w'
},
)
product/ERP5/bootstrap/erp5_core/PathTemplateItem/portal_ids/6.xml
View file @ db6a43cd
......@@ -33,6 +33,10 @@
<none/>
</value>
</item>
<item>
<key> <string>store_interval</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>stored_in_zodb</string> </key>
<value> <int>1</int> </value>
......
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/SQLNonContinuousIncreasingIdGenerator_view.xml
View file @ db6a43cd
......@@ -92,14 +92,16 @@
<string>my_portal_type</string>
<string>my_reference</string>
<string>my_version</string>
<string>my_stored_in_zodb</string>
</list>
</value>
</item>
<item>
<key> <string>right</string> </key>
<value>
<list/>
<list>
<string>my_stored_in_zodb</string>
<string>my_store_interval</string>
</list>
</value>
</item>
</dictionary>
......
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/SQLNonContinuousIncreasingIdGenerator_view/my_store_interval.xml 0 → 100644
View file @ db6a43cd
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<tuple>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
<tuple/>
</tuple>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list>
<string>title</string>
</list>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>my_store_interval</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_integer_field</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string>Click to edit the target</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Store Interval</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/SQLNonContinuousIncreasingIdGenerator_zCreateTable.xml deleted 100644 → 0
View file @ e4a18449
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<tuple>
<global name="SQL" module="Products.ZSQLMethods.SQL"/>
<tuple/>
</tuple>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_arg</string> </key>
<value>
<object>
<klass>
<global name="Args" module="Shared.DC.ZRDB.Aqueduct"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_data</string> </key>
<value>
<dictionary/>
</value>
</item>
<item>
<key> <string>_keys</string> </key>
<value>
<list/>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_col</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_transactionless_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SQLNonContinuousIncreasingIdGenerator_zCreateTable</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value>
</item>
<item>
<key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[
CREATE TABLE `portal_ids` (\n
`id_group` VARBINARY(255),\n
`last_id` BIGINT UNSIGNED,\n
PRIMARY KEY (`id_group`)\n
) TYPE=InnoDB\n
<dtml-var sql_delimiter>\n
<dtml-in expr="getPersistentIdDict().items()">\n
INSERT INTO `portal_ids` (`id_group`, `last_id`) VALUES (<dtml-sqlvar sequence-key type="string">, <dtml-sqlvar sequence-item type="int">)\n
<dtml-var sql_delimiter>\n
</dtml-in>\n
COMMIT
]]></string> </value>
</item>
<item>
<key> <string>template</string> </key>
<value>
<object>
<klass>
<global name="__newobj__" module="copy_reg"/>
</klass>
<tuple>
<global name="SQL" module="Shared.DC.ZRDB.DA"/>
</tuple>
<state>
<dictionary>
<item>
<key> <string>__name__</string> </key>
<value> <string encoding="cdata"><![CDATA[
<string>
]]></string> </value>
</item>
<item>
<key> <string>_vars</string> </key>
<value>
<dictionary/>
</value>
</item>
<item>
<key> <string>globals</string> </key>
<value>
<dictionary/>
</value>
</item>
<item>
<key> <string>raw</string> </key>
<value> <string encoding="cdata"><![CDATA[
CREATE TABLE `portal_ids` (\n
`id_group` VARBINARY(255),\n
`last_id` BIGINT UNSIGNED,\n
PRIMARY KEY (`id_group`)\n
) TYPE=InnoDB\n
<dtml-var sql_delimiter>\n
<dtml-in expr="getPersistentIdDict().items()">\n
INSERT INTO `portal_ids` (`id_group`, `last_id`) VALUES (<dtml-sqlvar sequence-key type="string">, <dtml-sqlvar sequence-item type="int">)\n
<dtml-var sql_delimiter>\n
</dtml-in>\n
COMMIT
]]></string> </value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/interfaces/id_generator.py
View file @ db6a43cd
......@@ -104,7 +104,7 @@ class IIdGenerator(Interface):
added here)
"""
def importGeneratorIdDict(id_dict):
def importGeneratorIdDict(id_dict, clear=False):
"""
Import data, this is usefull if we want to replace a generator by
another one. It will allows to make the new generator starting from
......@@ -114,6 +114,9 @@ class IIdGenerator(Interface):
id_dict (dict)
A dictionnary in the form { group_id : last_id }
clear(bool)
A boolean to clear the generator before import the data
This can be incompatible with some particular generator implementation,
in this case a particular error will be raised (to be determined and
......
product/ERP5/tests/testIdTool.py
View file @ db6a43cd
......@@ -41,13 +41,13 @@ class TestIdTool(ERP5TypeTestCase):
self.login()
self.portal = self.getPortal()
self.id_tool = self.portal.portal_ids
self.id_tool.initializeGenerator(all=True)
self.createGenerators()
transaction.commit()
self.tic()
def beforeTearDown(self):
self.id_tool.clearGenerator(all=True)
self.id_tool.initializeGenerator(all=True)
def getTitle(self):
"""
......@@ -156,7 +156,7 @@ class TestIdTool(ERP5TypeTestCase):
zodb_generator = self.getLastGenerator('test_application_zodb')
zodb_portal_type = 'ZODB Continuous Increasing Id Generator'
self.assertEquals(zodb_generator.getPortalType(), zodb_portal_type)
self.assertEqual(getattr(zodb_generator, 'last_id_dict', None), None)
self.assertEqual(getattr(zodb_generator, 'last_id_dict', {}), {})
# generate ids
self.checkGenerateNewId('test_application_zodb')
# check zodb dict
......@@ -173,13 +173,14 @@ class TestIdTool(ERP5TypeTestCase):
sql_generator = self.getLastGenerator('test_application_sql')
sql_portal_type = 'SQL Non Continuous Increasing Id Generator'
self.assertEquals(sql_generator.getPortalType(), sql_portal_type)
self.assertEqual(getattr(sql_generator, 'last_max_id_dict', None), None)
self.assertEquals(getattr(sql_generator, 'last_max_id_dict', {}), {})
# retrieve method to recovery the last id in the database
last_id_method = getattr(self.portal, 'IdTool_zGetLastId', None)
self.assertNotEquals(last_id_method, None)
# store the ids in zodb
if store:
sql_generator.setStoredInZodb(True)
sql_generator.setStoreInterval(1)
# generate ids
self.checkGenerateNewId('test_application_sql')
# check last_id in sql
......@@ -187,10 +188,10 @@ class TestIdTool(ERP5TypeTestCase):
self.assertEquals(last_id_method(id_group='d02')[0]['LAST_INSERT_ID()'], 21)
# check zodb dict
if store:
self.assertEqual(sql_generator.last_max_id_dict['c02'].value, 0)
self.assertEqual(sql_generator.last_max_id_dict['d02'].value, 21)
self.assertEquals(sql_generator.last_max_id_dict['c02'].value, 0)
self.assertEquals(sql_generator.last_max_id_dict['d02'].value, 21)
else:
self.assertEqual(getattr(sql_generator, 'last_max_id_dict', None), None)
self.assertEquals(getattr(sql_generator, 'last_max_id_dict', {}), {})
def test_02b_generateNewIdWithSQLGeneratorWithoutStorageZODB(self):
"""
......@@ -279,6 +280,57 @@ class TestIdTool(ERP5TypeTestCase):
result = sql_connection.manage_test(query)
self.assertEqual(result[0].last_id, 4)
def checkExportImportDict(self, id_generator):
"""
Check export import on id generator
"""
generator = self.getLastGenerator(id_generator)
self.assertEquals(0, self.id_tool.generateNewId(id_generator=id_generator,
id_group='06'))
id_dict = generator.exportGeneratorIdDict()
self.assertEquals(0, id_dict['06'])
generator.importGeneratorIdDict(id_dict={'06':6})
self.assertEquals(7, self.id_tool.generateNewId(id_generator=id_generator,
id_group='06'))
def test_06_ExportImportDict(self):
"""
Check export import dict for generator sql and zodb
"""
self.checkExportImportDict(id_generator='test_application_zodb')
self.checkExportImportDict(id_generator='test_application_sql')
def test_07_checkImportValueAndStoreInterval(self):
"""
Check that the store_interval store the last_id every N increments
store_interval is only on SQL
"""
id_generator = 'test_application_sql'
sql_generator = self.getLastGenerator(id_generator)
sql_generator.setStoredInZodb(True)
sql_generator.setStoreInterval(2)
#sql_generator.setStoreInterval(2)
self.assertEquals(0, self.id_tool.generateNewId(id_generator=id_generator,
id_group='07'))
self.assertEquals(sql_generator.last_max_id_dict['07'].value, 0)
self.assertEquals(1, self.id_tool.generateNewId(id_generator=id_generator,
id_group='07'))
# last_id isn't stored because 1 < last_id (0) + store_interval
self.assertEquals(sql_generator.last_max_id_dict['07'].value, 0)
self.assertEquals(2, self.id_tool.generateNewId(id_generator=id_generator,
id_group='07'))
self.assertEquals(sql_generator.last_max_id_dict['07'].value, 2)
self.getLastGenerator(id_generator).\
importGeneratorIdDict(id_dict = {'07':5})
self.assertEquals(6, self.id_tool.generateNewId(id_generator=id_generator,
id_group='07'))
# last_id stored because 6 < last_id (5) + store_interval
self.assertEquals(sql_generator.last_max_id_dict['07'].value, 5)
# the sql value is higher that zodb value so the export return the sql
# value
id_dict = self.getLastGenerator(id_generator).exportGeneratorIdDict()
self.assertEquals(id_dict['07'], 6)
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestIdTool))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7