Commit 9d05fd02 authored by Vincent Pelletier's avatar Vincent Pelletier

all: Get rid of Folder_reindex{All,Objectlist,TreeObjectList}

These are now superseded by recursiveReindexObject.
parent 67bb6b90
""" This script reindex all the objects created before updating local roles """ """ This script reindex all the objects created before updating local roles """
portal = context.getPortalObject() portal = context.getPortalObject()
portal.portal_types.Folder_reindexAll() portal.portal_types.recursiveReindexObject()
stack = [ stack = [
portal.document_module, portal.document_module,
portal.image_module, portal.image_module,
......
...@@ -5,7 +5,7 @@ portal.portal_templates.updateRepositoryBusinessTemplateList( ...@@ -5,7 +5,7 @@ portal.portal_templates.updateRepositoryBusinessTemplateList(
['http://www.erp5.org/dists/snapshot/bt5/']) ['http://www.erp5.org/dists/snapshot/bt5/'])
business_template_list = context.Base_getDemoSMBBusinessTemplateList() business_template_list = context.Base_getDemoSMBBusinessTemplateList()
kw = dict(tag="start", after_method_id = ["immediateReindexObject", "Folder_reindexAll"]) kw = {'tag': "start", 'after_method_id': ["immediateReindexObject"]}
for business_template_id in business_template_list: for business_template_id in business_template_list:
portal.portal_templates.activate(**kw).installBusinessTemplatesFromRepositories((business_template_id,)) portal.portal_templates.activate(**kw).installBusinessTemplatesFromRepositories((business_template_id,))
kw["after_tag"] = kw["tag"] kw["after_tag"] = kw["tag"]
......
...@@ -1940,16 +1940,17 @@ class ERP5Generator(PortalGenerator): ...@@ -1940,16 +1940,17 @@ class ERP5Generator(PortalGenerator):
p._v_bootstrapping = False p._v_bootstrapping = False
# XXX: Is it useful to wait for indexing before using upgradeSite ? reindex_all_tag = 'ERP5Site_reindexAll'
after_method_id = 'immediateReindexObject' upgrade_tag = 'updgradeSite'
preference_tag = 'initSystemPreference'
if bt5_repository_url: if bt5_repository_url:
p.portal_templates.repository_dict = dict.fromkeys( p.portal_templates.repository_dict = dict.fromkeys(bt5_repository_url.split())
bt5_repository_url.split())
if bt5: if bt5:
method_id = 'upgradeSite' p.portal_templates.activate(
getattr(p.portal_templates.activate(after_method_id=after_method_id), # XXX: Is it useful to wait for indexing ?
method_id)(bt5.split(), update_catalog=True) after_tag=reindex_all_tag,
after_method_id = method_id tag=upgrade_tag,
).upgradeSite(bt5.split(), update_catalog=True)
if id_store_interval != '': if id_store_interval != '':
id_store_interval = int(id_store_interval) id_store_interval = int(id_store_interval)
if id_store_interval < 0: if id_store_interval < 0:
...@@ -1961,22 +1962,17 @@ class ERP5Generator(PortalGenerator): ...@@ -1961,22 +1962,17 @@ class ERP5Generator(PortalGenerator):
else: else:
ob._setStoredInZodb(0) ob._setStoredInZodb(0)
if cloudooo_url: if cloudooo_url:
method_id = '_initSystemPreference' p.portal_activities.activateObject(
getattr(p.portal_activities.activateObject(p, p,
after_method_id=after_method_id), method_id)(cloudooo_url=cloudooo_url) after_tag=(reindex_all_tag, upgrade_tag),
after_method_id = method_id tag=preference_tag,
)._initSystemPreference(cloudooo_url=cloudooo_url)
id_ = 'isPortalBeingCreated' id_ = 'isPortalBeingCreated'
setattr(p, id_, ConstantGetter(id_, value=True)) setattr(p, id_, ConstantGetter(id_, value=True))
# XXX: ERP5Site_reindexAll should be reviewed so that one can depend on a p.portal_activities.activateObject(
# final tag. A more general approach is to have an activity dependency p,
# to anything, so that _delPropValue is called as soon as activity after_tag=(reindex_all_tag, upgrade_tag, preference_tag),
# nodes have nothing else to do.
after_method_id = tuple({after_method_id}.union(('Folder_reindexAll',
'Folder_reindexObjectList', 'InventoryModule_reindexMovementList',
'immediateReindexObject', 'SQLCatalog_deferFullTextIndexActivity')))
p.portal_activities.activateObject(p, after_method_id=after_method_id,
)._delPropValue(id_) )._delPropValue(id_)
return p return p
@classmethod @classmethod
......
...@@ -50,7 +50,7 @@ ...@@ -50,7 +50,7 @@
</item> </item>
<item> <item>
<key> <string>_params</string> </key> <key> <string>_params</string> </key>
<value> <string>sql_catalog_id=None, additional_priority=0, clear_catalog=0, final_activity_tag=None, start_tree=None, stop_tree=None</string> </value> <value> <string>sql_catalog_id=None, clear_catalog=False, additional_priority=1, final_activity_tag="ERP5Site_reindexAll"</string> </value>
</item> </item>
<item> <item>
<key> <string>id</string> </key> <key> <string>id</string> </key>
......
folder = context
activate_kw = {
'tag': object_tag,
'after_tag': folder_after_tag,
}
for key, value in activate_kw.items():
if value is None:
activate_kw.pop(key)
# Reindex folder immediately
folder.reindexObject(sql_catalog_id=sql_catalog_id)
folder_id = folder.getId()
if folder_after_tag is None:
folder_after_tag = ()
bundle_tag = "%s_bundle_reindex" % folder_id
bundle_object_tag = "%s_reindex" % folder_id
# Spawn activities for bundles of content objects.
# Bundle size, in object count
BUNDLE_ITEM_COUNT=1000
def Folder_reindexObjectList(id_list_list):
"""
Create an activity calling Folder_reindexObjectList.
"""
folder.activate(activity='SQLQueue', priority=object_priority,
**activate_kw).Folder_reindexObjectList(
id_list=None,
id_list_list=id_list_list,
object_priority=object_priority,
object_tag=object_tag,
object_after_tag=object_after_tag,
folder_tag=bundle_tag,
folder_after_tag=bundle_object_tag,
sql_catalog_id=sql_catalog_id,
)
archive_test_script = getattr(context.getPortalObject(), "Archive_test", None)
try:
tree_id_list = folder.getTreeIdList()
except (NotImplementedError, AttributeError):
# Build a list of list, like this we parse ids only one time,
# and then Folder_reinexObjectList will work with one list at
# a time and remove it from the list of list
# This id_list_list can be quite big and generate quite big
# activities, but the effect is limited, because if we have too
# much objects (like millions), we should use HBTree Folders, and
# then the work will be splitted
id_list = [x for x in folder.objectIds()]
id_list_list = []
for bundle_index in xrange(len(id_list) / BUNDLE_ITEM_COUNT):
id_list_list.append(id_list[bundle_index * BUNDLE_ITEM_COUNT:((bundle_index + 1) * BUNDLE_ITEM_COUNT)])
remaining_object_id_count = len(id_list) % BUNDLE_ITEM_COUNT
if remaining_object_id_count > 0:
id_list_list.append(id_list[-remaining_object_id_count:])
Folder_reindexObjectList(id_list_list)
else:
if archive_test_script is not None:
new_tree_id_list = []
for tree_id in tree_id_list:
if folder.Archive_test(tree_id=tree_id, start_tree=start_tree, stop_tree=stop_tree):
new_tree_id_list.append(tree_id)
tree_id_list = new_tree_id_list
else:
if start_tree is not None:
new_tree_id_list = []
for tree_id in tree_id_list:
if tree_id >= start_tree:
new_tree_id_list.append(tree_id)
tree_id_list = new_tree_id_list
if stop_tree is not None:
new_tree_id_list = []
for tree_id in tree_id_list:
if tree_id < stop_tree:
new_tree_id_list.append(tree_id)
tree_id_list = new_tree_id_list
if len(tree_id_list) == 0:
return
i = 0
tree_tag = "%s_tree" % folder_id
# Say to Folder_reindexTreeObjectList to call himself again and
# again until all tree_id_list are parsed. Also, make sure that
# the work of the previous Folder_reindexTreeObjectList is completely
# done
tree_after_tag = folder_after_tag + (bundle_tag, bundle_object_tag)
folder.activate(activity='SQLQueue', priority=object_priority,
after_tag=tree_after_tag, tag=tree_tag,
).Folder_reindexTreeObjectList(
tree_id=None,
tree_id_list=tree_id_list,
folder_tag=bundle_tag,
folder_after_tag=bundle_object_tag,
object_priority=object_priority,
sql_catalog_id=sql_catalog_id,
object_tag=bundle_object_tag,
tree_after_tag=tree_after_tag,
tree_tag=tree_tag,
)
# Start an activity wich will wait the end of the module
folder_id_after_tag = folder_after_tag + (tree_tag, bundle_tag, bundle_object_tag)
id_activate_kw = {}
if object_tag is not None:
id_activate_kw['tag'] = object_tag
folder.activate(activity='SQLDict', priority=object_priority,
after_tag=folder_id_after_tag, **id_activate_kw).getId()
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>folder_tag=None, folder_after_tag=None, object_tag=None, object_after_tag=None, object_priority=1, sql_catalog_id=None, start_tree=None, stop_tree=None</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Folder_reindexAll</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
folder = context
# Keep compatibility with id_list
if id_list_list is not None:
if id_list is not None:
raise ValueError, "both id_list and id_list_list can be defined"
if len(id_list_list) == 0:
return None
id_list = id_list_list.pop()
activate_kw = {
'tag': object_tag,
'after_tag': object_after_tag,
'priority': object_priority
}
for key, value in activate_kw.items():
if value is None:
activate_kw.pop(key)
for id in id_list:
obj = getattr(folder, id, None)
if obj is not None:
obj.recursiveReindexObject(activate_kw=activate_kw,
sql_catalog_id=sql_catalog_id)
if id_list_list is not None:
if len(id_list_list) > 0:
if count is None:
count = 1
new_activity_kw = {}
# We do not want to wait until there is enough activities
# So add BUNDLE_ITEM_COUNT * node_len activities before waiting
node_len = len(context.portal_activities.getProcessingNodeList())
if count % node_len == 0:
new_activity_kw['after_tag'] = folder_after_tag
count = 0
count += 1
# By calling again and again, we improve performance and we have
# less activities by the same time
folder.activate(activity='SQLQueue',
priority=object_priority,
tag=folder_tag, **new_activity_kw).Folder_reindexObjectList(
None,
id_list_list=id_list_list,
object_priority=object_priority,
object_tag=object_tag,
sql_catalog_id=sql_catalog_id,
folder_tag=folder_tag,
folder_after_tag=folder_after_tag,
count=count,
)
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>id_list, id_list_list=None, folder_tag=None, folder_after_tag=None, object_tag=None, object_after_tag=None, object_priority=1, sql_catalog_id=None, count=None</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Folder_reindexObjectList</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
from Products.ERP5Type.Log import log
folder = context
# Keep compatibility with tree_id
if tree_id_list is not None:
log('tree_id', tree_id)
log('tree_id_list', tree_id_list)
if tree_id is not None:
raise ValueError, "both tree and tree_id_list should not be defined"
tree_id = tree_id_list.pop()
# Spawn activities for bundles of content objects.
# Bundle size, in object count
BUNDLE_ITEM_COUNT = 1000
folder_id = folder.getId()
def Folder_reindexObjectList(id_list_list):
"""
Create an activity calling Folder_reindexObjectList.
"""
folder.activate(activity='SQLQueue', priority=object_priority,
after_tag=object_tag,
tag=folder_tag).Folder_reindexObjectList(
None,
id_list_list=id_list_list,
object_priority=object_priority,
object_tag=object_tag,
sql_catalog_id=sql_catalog_id,
folder_tag=folder_tag,
folder_after_tag=folder_after_tag,
)
# HBTree folder
id_list = [x for x in folder.objectIds(base_id=tree_id)]
# Build a list of list, like this we parse ids only one time,
# and then Folder_reinexObjectList will work with one list at
# a time and remove it from the list of list
# This id_list_list can be quite big and generate quite big
# activities, but the effect is limited, because the work is
# splitted for each base_id of the HBTree.
id_list_list = []
for bundle_index in xrange(len(id_list) / BUNDLE_ITEM_COUNT):
id_list_list.append(id_list[bundle_index * BUNDLE_ITEM_COUNT:((bundle_index + 1) * BUNDLE_ITEM_COUNT)])
remaining_object_id_count = len(id_list) % BUNDLE_ITEM_COUNT
if remaining_object_id_count > 0:
id_list_list.append(id_list[-remaining_object_id_count:])
Folder_reindexObjectList(id_list_list=id_list_list)
if tree_id_list is not None:
if len(tree_id_list) > 0:
# Calling again and again the same script allow to decrease the
# number of activities by the same time and increase performance.
folder.activate(activity='SQLQueue', priority=object_priority,
after_tag=tree_after_tag,
tag=tree_tag).Folder_reindexTreeObjectList(
tree_id=None,
tree_id_list=tree_id_list,
folder_tag=folder_tag,
folder_after_tag=folder_after_tag,
object_priority=object_priority,
sql_catalog_id=sql_catalog_id,
object_tag=object_tag,
tree_after_tag=tree_after_tag,
tree_tag=tree_tag,
)
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>tree_id=None, tree_id_list=None, folder_tag=None, folder_after_tag=None, object_tag=None, object_after_tag=None, object_priority=1, sql_catalog_id=None, tree_tag=None, tree_after_tag=None</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Folder_reindexTreeObjectList</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
...@@ -2768,9 +2768,6 @@ class BusinessTemplateMixin(ERP5TypeTestCase, LogInterceptor): ...@@ -2768,9 +2768,6 @@ class BusinessTemplateMixin(ERP5TypeTestCase, LogInterceptor):
self.assertEqual(len(message_list), 0) self.assertEqual(len(message_list), 0)
def stepCheckHasClearedCatalog(self, sequence=None, **kw): def stepCheckHasClearedCatalog(self, sequence=None, **kw):
"""
Check if we have activity for Folder_reindexAll.
"""
self.assertTrue(self._has_cleared_catalog) self.assertTrue(self._has_cleared_catalog)
del self._has_cleared_catalog[:] del self._has_cleared_catalog[:]
......
...@@ -419,7 +419,7 @@ class TestDomainTool(TestPredicateMixIn): ...@@ -419,7 +419,7 @@ class TestDomainTool(TestPredicateMixIn):
# Enable system preference and reindex relevant predicates # Enable system preference and reindex relevant predicates
system_preference.enable() system_preference.enable()
self.tic() self.tic()
supply_module.Folder_reindexAll() supply_module.recursiveReindexObject()
self.tic() self.tic()
# if document has relations using base categories which are not present in the preference, then left join mode is still used. # if document has relations using base categories which are not present in the preference, then left join mode is still used.
assertUsesLeftJoinAndPredicateItemsMatchingOrderLineEqual(True, [supply1_line1]) assertUsesLeftJoinAndPredicateItemsMatchingOrderLineEqual(True, [supply1_line1])
...@@ -432,7 +432,7 @@ class TestDomainTool(TestPredicateMixIn): ...@@ -432,7 +432,7 @@ class TestDomainTool(TestPredicateMixIn):
system_preference.setPreferredPredicateCategoryList(['source_section', 'destination_section', 'price_currency', 'resource']) system_preference.setPreferredPredicateCategoryList(['source_section', 'destination_section', 'price_currency', 'resource'])
self.portal.portal_caches.clearAllCache() self.portal.portal_caches.clearAllCache()
self.tic() self.tic()
supply_module.Folder_reindexAll() supply_module.recursiveReindexObject()
self.tic() self.tic()
# resource is not in preferred predicate category list, so only inner join is used # resource is not in preferred predicate category list, so only inner join is used
assertUsesLeftJoinAndPredicateItemsMatchingOrderLineEqual(False, [supply1_line1], tested_base_category_list=['source_section', 'destination_section', 'price_currency', 'resource']) assertUsesLeftJoinAndPredicateItemsMatchingOrderLineEqual(False, [supply1_line1], tested_base_category_list=['source_section', 'destination_section', 'price_currency', 'resource'])
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment