Commit babbf56f authored by Vincent Pelletier's avatar Vincent Pelletier

Folder: Make recursiveReindexObject scalable by calling _recurseCallMethod.

Should make Folder_reindexAll and most custom indexation methods obsolete.
Remaining valid reindexation methods are:
- reindexObject: for a single document, which may contain subdocuments
  which indexation is not necessary
- recursiveReindexobject: for any subtree of documents
- ERP5Site_reindexAll: for site-wide reindexations, as there is a semantic-
  dependent indexation order.

Also, uniformise and factorise spawning immediateReindexObject.

Also:
- testSupply: Drop check for the previous magic threshold.
  _recurseCallMethod takes care of it all now.
- testXMLMatrix: Let activities execute before changing cell id.
  This works only because recursiveReindexObject on the matrix spawns a
  single recursiveImmediateReindexObject activity on that context. Now,
  up to 1k immediateReindexObject activities (for the first 1k sub-objects)
  are spawned immediately, preventing their renaming immediately after
  commit.
  So let test wait for indexation before trying to rename.
- testERP5Security: More activities are now spawned immediately, adapt.
parent 028c9ad7
...@@ -605,77 +605,6 @@ class TestSaleSupply(TestSupplyMixin, SubcontentReindexingWrapper, ...@@ -605,77 +605,6 @@ class TestSaleSupply(TestSupplyMixin, SubcontentReindexingWrapper,
self.assertEqual(10, movement.getPrice()) self.assertEqual(10, movement.getPrice())
self.abort() self.abort()
def _createTwoHundredSupplyLineInASupply(self):
supply = self._makeSupply(
start_date_range_min='2014/01/01',
start_date_range_max='2014/01/31',
)
for i in range(200):
resource_value = self.portal.product_module['%s_%d' % (self.id(), i)]
supply_line = self._makeSupplyLine(supply)
supply_line.edit(resource_value=resource_value,
base_price=100)
return supply
def _createTwoHundredResource(self):
for i in range(200):
self._makeResource('%s_%d' % (self.id(), i))
def testReindexOnLargeSupply(self):
"""
Make sure that recursiveImmediateReindexObject is not called on the root
document when the document has more than 100 sub objects.
"""
self._makeSections()
self._createTwoHundredResource()
supply = self._createTwoHundredSupplyLineInASupply()
# First, clear activities just in case.
self.tic()
# Editing triggers reindexObject(active_kw={}) through DCWorkflowDefinition.
# Not only edit(), but also all the workflow transitions can trigger it.
# Likewise, recursiveReindexObject(active_kw={}) is triggered, for instance
# in Supply.py, Delivery.py and etc,.
# This is because of the reindexObject() method definition on the Documents.
supply.edit(title='xx')
transaction.commit()
sql_connection = self.getSQLConnection()
supply_path = supply.getPath()
sql = """SELECT
count(*)
FROM
message
WHERE
path like '%s'
AND
method_id = 'recursiveImmediateReindexObject'
""" % (supply_path.replace('_', r'\_') + '/%')
result = sql_connection.manage_test(sql)
all_the_spply_line_activities_count = result[0]['COUNT(*)']
# supply line reindex activity count must be 200 since created 200 lines
self.assertEqual(200, all_the_spply_line_activities_count)
sql_connection = self.getSQLConnection()
sql = "SELECT count(*) FROM message WHERE path='%s'" % supply_path
result = sql_connection.manage_test(sql)
supply_document_reindex_count = result[0]['COUNT(*)']
# reindex activity with the same supply must be only one in this case.
self.assertEqual(1, supply_document_reindex_count)
sql = "SELECT count(*) FROM message WHERE path='%s' AND method_id='%s'" \
% (supply_path, 'recursiveImmediateReindexObject')
result = sql_connection.manage_test(sql)
supply_recursive_immediate_reindex_count = result[0]['COUNT(*)']
# the count of recursiveImmediateReindex on Supply document must be zero
# because the supply contains >100 sub objects. And the the suply lines
# reindex are already triggerred. Thus if recursiveImmediateReindex
# is also triggered on Supply, the reindex will be duplicated. Moreover,
# recursiveImmediateReindex in a single node is less efficient comparing
# to use all nodes for the reindex, in such a >100 case.
self.assertEqual(0, supply_recursive_immediate_reindex_count)
class TestPurchaseSupply(TestSaleSupply): class TestPurchaseSupply(TestSaleSupply):
""" """
Test Purchase Supplies usage Test Purchase Supplies usage
......
...@@ -1270,7 +1270,8 @@ class TestLocalRoleManagement(ERP5TypeTestCase): ...@@ -1270,7 +1270,8 @@ class TestLocalRoleManagement(ERP5TypeTestCase):
self.assertTrue(len(person.objectIds())) self.assertTrue(len(person.objectIds()))
person.reindexObjectSecurity() person.reindexObjectSecurity()
self.commit() self.commit()
check(['recursiveImmediateReindexObject']) # One reindexation activity per subobject, and one on the person itself.
check(['immediateReindexObject'] * (len(person) + 1))
self.tic() self.tic()
def test_suite(): def test_suite():
......
...@@ -2811,30 +2811,38 @@ class Base( CopyContainer, ...@@ -2811,30 +2811,38 @@ class Base( CopyContainer,
# Do not check if root is indexable, it is done into catalogObjectList, # Do not check if root is indexable, it is done into catalogObjectList,
# so we will save time # so we will save time
if self.isIndexable: if self.isIndexable:
if activate_kw is None: kw, activate_kw = self._getReindexAndActivateParameterDict(
activate_kw = {} kw,
activate_kw,
reindex_kw = self.getDefaultReindexParameterDict() )
if reindex_kw is not None: activate_kw['serialization_tag'] = self.getRootDocumentPath()
reindex_kw = reindex_kw.copy() self.activate(**activate_kw).immediateReindexObject(**kw)
reindex_activate_kw = reindex_kw.pop('activate_kw', None) or {}
reindex_activate_kw.update(activate_kw) def _getReindexAndActivateParameterDict(self, kw, activate_kw):
reindex_kw.update(kw) if activate_kw is None:
kw = reindex_kw activate_kw = ()
activate_kw = reindex_activate_kw reindex_kw = self.getDefaultReindexParameterDict()
if reindex_kw is not None:
group_id_list = [] reindex_kw = reindex_kw.copy()
if kw.get("group_id") not in ('', None): reindex_activate_kw = reindex_kw.pop('activate_kw', None) or {}
group_id_list.append(kw["group_id"]) reindex_activate_kw.update(activate_kw)
if kw.get("sql_catalog_id") not in ('', None): reindex_kw.update(kw)
group_id_list.append(kw["sql_catalog_id"]) kw = reindex_kw
group_id = ' '.join(group_id_list) activate_kw = reindex_activate_kw
else:
self.activate(group_method_id='portal_catalog/catalogObjectList', activate_kw = dict(activate_kw)
alternate_method_id='alternateReindexObject', group_id_list = []
group_id=group_id, if kw.get("group_id") not in ('', None):
serialization_tag=self.getRootDocumentPath(), group_id_list.append(kw["group_id"])
**activate_kw).immediateReindexObject(**kw) if kw.get("sql_catalog_id") not in ('', None):
group_id_list.append(kw["sql_catalog_id"])
if activate_kw.get('group_id') not in ('', None):
group_id_list.append(activate_kw['group_id'])
activate_kw['group_id'] = ' '.join(group_id_list)
activate_kw['group_method_id'] = 'portal_catalog/catalogObjectList'
activate_kw['alternate_method_id'] = 'alternateReindexObject'
activate_kw['activity'] = 'SQLDict'
return kw, activate_kw
security.declarePublic('recursiveReindexObject') security.declarePublic('recursiveReindexObject')
recursiveReindexObject = reindexObject recursiveReindexObject = reindexObject
......
...@@ -74,7 +74,6 @@ import os ...@@ -74,7 +74,6 @@ import os
from zLOG import LOG, WARNING from zLOG import LOG, WARNING
import warnings import warnings
from urlparse import urlparse from urlparse import urlparse
REINDEX_SPLIT_COUNT = 100 # if folder containes more than this, reindexing should be splitted.
from Products.ERP5Type.Message import translateString from Products.ERP5Type.Message import translateString
# Dummy Functions for update / upgrade # Dummy Functions for update / upgrade
...@@ -1231,49 +1230,26 @@ class Folder(CopyContainer, OFSFolder2, CMFBTreeFolder, CMFHBTreeFolder, Base, F ...@@ -1231,49 +1230,26 @@ class Folder(CopyContainer, OFSFolder2, CMFBTreeFolder, CMFHBTreeFolder, Base, F
security.declarePublic('recursiveReindexObject') security.declarePublic('recursiveReindexObject')
def recursiveReindexObject(self, activate_kw=None, **kw): def recursiveReindexObject(self, activate_kw=None, **kw):
if self.isIndexable: if self.isIndexable:
if not activate_kw and self.objectCount() > REINDEX_SPLIT_COUNT: kw, activate_kw = self._getReindexAndActivateParameterDict(
# If the number of objects to reindex is too high kw,
# we should try to split reindexing in order to be more efficient activate_kw,
# NOTE: this heuristic will fail for example with orders which )
# contain > REINDEX_SPLIT_COUNT order lines. activate_kw['group_method_cost'] = 0.01
# It will be less efficient in this case. We also do not self._recurseCallMethod(
# use this heuristic whenever activate_kw is defined 'immediateReindexObject',
self._reindexObject(**kw) method_kw=kw,
# XXX-JPS: Here, we could invoke Folder_reindexAll instead, like this: activate_kw=activate_kw,
# self.Folder_reindexAll() get_activate_kw_method_id='_updateActivateKwWithSerialisationTag',
# return max_depth=None,
# this shows that both methods should be merged. skip_method_id='_isDocumentNonIndexable',
for c in self.objectValues(): )
if getattr(aq_base(c),
'recursiveReindexObject', None) is not None: def _isDocumentNonIndexable(self, document):
c.recursiveReindexObject(**kw) return not document.isIndexable
return
def _updateActivateKwWithSerialisationTag(self, document, activate_kw):
if activate_kw is None: activate_kw['serialization_tag'] = document.getRootDocumentPath()
activate_kw = {} return activate_kw
reindex_kw = self.getDefaultReindexParameterDict()
if reindex_kw is not None:
reindex_kw = reindex_kw.copy()
reindex_activate_kw = reindex_kw.pop('activate_kw', None) or {}
reindex_activate_kw.update(activate_kw)
reindex_kw.update(kw)
kw = reindex_kw
activate_kw = reindex_activate_kw
group_id_list = []
if kw.get("group_id") not in ('', None):
group_id_list.append(kw["group_id"])
if kw.get("sql_catalog_id") not in ('', None):
group_id_list.append(kw["sql_catalog_id"])
group_id = ' '.join(group_id_list)
self.activate(group_method_id='portal_catalog/catalogObjectList',
expand_method_id='getIndexableChildValueList',
alternate_method_id='alternateReindexObject',
group_id=group_id,
serialization_tag=self.getRootDocumentPath(),
**activate_kw).recursiveImmediateReindexObject(**kw)
security.declareProtected( Permissions.AccessContentsInformation, security.declareProtected( Permissions.AccessContentsInformation,
'getIndexableChildValueList' ) 'getIndexableChildValueList' )
......
...@@ -451,7 +451,7 @@ class TestXMLMatrix(ERP5TypeTestCase, LogInterceptor): ...@@ -451,7 +451,7 @@ class TestXMLMatrix(ERP5TypeTestCase, LogInterceptor):
cell_range = [['2', ], ['b',]] cell_range = [['2', ], ['b',]]
matrix.setCellRange(*cell_range, **kwd) matrix.setCellRange(*cell_range, **kwd)
self.commit() self.tic()
self.assertEquals(set(["quantity_1_1"]), set([ self.assertEquals(set(["quantity_1_1"]), set([
x.getId() for x in matrix.objectValues()])) x.getId() for x in matrix.objectValues()]))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment