Commit 9d05fd02 authored by Vincent Pelletier's avatar Vincent Pelletier

all: Get rid of Folder_reindex{All,Objectlist,TreeObjectList}

These are now superseded by recursiveReindexObject.
parent 67bb6b90
""" This script reindex all the objects created before updating local roles """ """ This script reindex all the objects created before updating local roles """
portal = context.getPortalObject() portal = context.getPortalObject()
portal.portal_types.Folder_reindexAll() portal.portal_types.recursiveReindexObject()
stack = [ stack = [
portal.document_module, portal.document_module,
portal.image_module, portal.image_module,
......
...@@ -5,7 +5,7 @@ portal.portal_templates.updateRepositoryBusinessTemplateList( ...@@ -5,7 +5,7 @@ portal.portal_templates.updateRepositoryBusinessTemplateList(
['http://www.erp5.org/dists/snapshot/bt5/']) ['http://www.erp5.org/dists/snapshot/bt5/'])
business_template_list = context.Base_getDemoSMBBusinessTemplateList() business_template_list = context.Base_getDemoSMBBusinessTemplateList()
kw = dict(tag="start", after_method_id = ["immediateReindexObject", "Folder_reindexAll"]) kw = {'tag': "start", 'after_method_id': ["immediateReindexObject"]}
for business_template_id in business_template_list: for business_template_id in business_template_list:
portal.portal_templates.activate(**kw).installBusinessTemplatesFromRepositories((business_template_id,)) portal.portal_templates.activate(**kw).installBusinessTemplatesFromRepositories((business_template_id,))
kw["after_tag"] = kw["tag"] kw["after_tag"] = kw["tag"]
......
...@@ -1940,16 +1940,17 @@ class ERP5Generator(PortalGenerator): ...@@ -1940,16 +1940,17 @@ class ERP5Generator(PortalGenerator):
p._v_bootstrapping = False p._v_bootstrapping = False
# XXX: Is it useful to wait for indexing before using upgradeSite ? reindex_all_tag = 'ERP5Site_reindexAll'
after_method_id = 'immediateReindexObject' upgrade_tag = 'updgradeSite'
preference_tag = 'initSystemPreference'
if bt5_repository_url: if bt5_repository_url:
p.portal_templates.repository_dict = dict.fromkeys( p.portal_templates.repository_dict = dict.fromkeys(bt5_repository_url.split())
bt5_repository_url.split())
if bt5: if bt5:
method_id = 'upgradeSite' p.portal_templates.activate(
getattr(p.portal_templates.activate(after_method_id=after_method_id), # XXX: Is it useful to wait for indexing ?
method_id)(bt5.split(), update_catalog=True) after_tag=reindex_all_tag,
after_method_id = method_id tag=upgrade_tag,
).upgradeSite(bt5.split(), update_catalog=True)
if id_store_interval != '': if id_store_interval != '':
id_store_interval = int(id_store_interval) id_store_interval = int(id_store_interval)
if id_store_interval < 0: if id_store_interval < 0:
...@@ -1961,22 +1962,17 @@ class ERP5Generator(PortalGenerator): ...@@ -1961,22 +1962,17 @@ class ERP5Generator(PortalGenerator):
else: else:
ob._setStoredInZodb(0) ob._setStoredInZodb(0)
if cloudooo_url: if cloudooo_url:
method_id = '_initSystemPreference' p.portal_activities.activateObject(
getattr(p.portal_activities.activateObject(p, p,
after_method_id=after_method_id), method_id)(cloudooo_url=cloudooo_url) after_tag=(reindex_all_tag, upgrade_tag),
after_method_id = method_id tag=preference_tag,
)._initSystemPreference(cloudooo_url=cloudooo_url)
id_ = 'isPortalBeingCreated' id_ = 'isPortalBeingCreated'
setattr(p, id_, ConstantGetter(id_, value=True)) setattr(p, id_, ConstantGetter(id_, value=True))
# XXX: ERP5Site_reindexAll should be reviewed so that one can depend on a p.portal_activities.activateObject(
# final tag. A more general approach is to have an activity dependency p,
# to anything, so that _delPropValue is called as soon as activity after_tag=(reindex_all_tag, upgrade_tag, preference_tag),
# nodes have nothing else to do.
after_method_id = tuple({after_method_id}.union(('Folder_reindexAll',
'Folder_reindexObjectList', 'InventoryModule_reindexMovementList',
'immediateReindexObject', 'SQLCatalog_deferFullTextIndexActivity')))
p.portal_activities.activateObject(p, after_method_id=after_method_id,
)._delPropValue(id_) )._delPropValue(id_)
return p return p
@classmethod @classmethod
......
from random import getrandbits
portal = context.getPortalObject() portal = context.getPortalObject()
if clear_catalog: if clear_catalog:
# clear the catalog before reindexing portal.portal_catalog.getSQLCatalog(sql_catalog_id).manage_catalogClear()
catalog = portal.portal_catalog.getSQLCatalog(sql_catalog_id)
catalog.manage_catalogClear()
# disable alarms while we are reindexing # disable alarms while we are reindexing
is_subscribed = portal.portal_alarms.isSubscribed() is_subscribed = portal.portal_alarms.isSubscribed()
if clear_catalog: if clear_catalog and is_subscribed:
if is_subscribed:
portal.portal_alarms.unsubscribe() portal.portal_alarms.unsubscribe()
# Reindex person module base_tag = '%s_%x_' % (script.id, getrandbits(32))
user_tag = base_tag + 'person_stage_1'
category_tag = base_tag + 'category'
document_tag = base_tag + 'document'
preference_tag = base_tag + 'portal_preferences'
simulation_tag = base_tag + 'simulation'
inventory_tag = base_tag + 'inventory'
last_inventory_tag = base_tag + 'last_inventory_activity'
def reindex(document_list, tag, after_tag):
for document in document_list:
print '#### Indexing', document.id, '####'
document.activate(
priority=additional_priority,
tag=tag,
after_tag=after_tag,
).recursiveReindexObject(
activate_kw={
'tag': tag,
'priority': additional_priority,
},
sql_catalog_id=sql_catalog_id,
)
return printed
# XXX: Must be replaced by an in-ZODB mapping from user_id to any type of user
# documents (not just Persons), otherwise catalog risks undergoing a
# security_uid explosion if many users (ex: persons) have local roles on
# documents (ex: persons) granting them View permission but the user is not
# indexed before corresponding document is.
print "#### Indexing person_module, stage 1 ####" print "#### Indexing person_module, stage 1 ####"
person_module=getattr(portal, 'person_module', None) person_module = getattr(portal, 'person_module', None)
higher_priority = 1 + additional_priority if person_module is not None:
if person_module is not None :
tag = 'person_stage_1'
person_module.recurseCallMethod( person_module.recurseCallMethod(
method_id='immediateReindexObject', method_id='immediateReindexObject',
group_method_id='portal_catalog/catalogObjectList',
method_kw={ method_kw={
'sql_catalog_id': sql_catalog_id, 'sql_catalog_id': sql_catalog_id,
}, },
activate_kw={ activate_kw={
'tag': tag, 'group_method_id': 'portal_catalog/catalogObjectList',
'priority': higher_priority, 'tag': user_tag,
}, },
max_depth=1, # Do not reindex Person's subobjects max_depth=1, # Do not reindex Person's subobjects
) )
print "#### Indexing translations ####" print "#### Indexing translations ####"
context.ERP5Site_updateTranslationTable(sql_catalog_id=sql_catalog_id) portal.ERP5Site_updateTranslationTable(sql_catalog_id=sql_catalog_id)
print reindex(
# Reindex categories [portal.portal_categories],
print "#### Indexing categories ####" tag=category_tag,
folder_tag = 'module' after_tag=user_tag,
folder_after_tag = ('person_stage_1', 'group_person_stage_1') ),
object_tag = 'category' print reindex(
object_after_tag = folder_after_tag [portal.portal_alarms, portal.portal_activities],
tag=document_tag,
context.portal_categories.activate( after_tag=(user_tag, category_tag),
tag=folder_tag, ),
priority=higher_priority, print reindex(
after_tag=folder_after_tag).Folder_reindexAll( [portal.portal_preferences],
folder_tag=folder_tag,
folder_after_tag=folder_after_tag,
object_tag=object_tag,
object_after_tag=object_after_tag,
object_priority=higher_priority,
sql_catalog_id=sql_catalog_id,
start_tree=start_tree,
stop_tree=stop_tree,)
print "#### Indexing alarms ####"
print "#### Indexing active results ####"
folder_tag = 'module'
folder_after_tag = ('category', 'person_stage_1', 'group_person_stage_1')
object_tag = 'document'
object_after_tag = folder_after_tag
object_priority = 2 + additional_priority
for folder in [context.portal_alarms, context.portal_activities]:
folder.activate(
tag=folder_tag,
priority=object_priority,
after_tag=folder_after_tag).Folder_reindexAll(
folder_tag=folder_tag,
folder_after_tag=folder_after_tag,
object_tag=object_tag,
object_after_tag=object_after_tag,
object_priority=object_priority,
sql_catalog_id=sql_catalog_id,
start_tree=start_tree,
stop_tree=stop_tree,)
print "#### Indexing preferences ####"
preference_tag = 'portal_preferences'
context.portal_preferences.activate(
tag=preference_tag, tag=preference_tag,
after_tag='category', after_tag=(user_tag, category_tag),
priority=additional_priority).Folder_reindexAll( ),
folder_tag=preference_tag, # Simulation is needed to calculate tests (ie. related quantity)
object_tag=preference_tag, print reindex(
object_priority=additional_priority, [portal.portal_simulation],
sql_catalog_id=sql_catalog_id, tag=simulation_tag,
start_tree=start_tree, after_tag=(user_tag, category_tag, document_tag, preference_tag),
stop_tree=stop_tree,) ),
print reindex(
# We index simulation first to make sure we can calculate tests [
# (ie. related quantity) x for x in portal.objectValues()
print "#### Indexing simulation ####" if x.getUid != portal.getUid and
folder_tag = 'module' x.id not in (
folder_after_tag = ('category', 'document', 'person_stage_1', 'group_person_stage_1', preference_tag) 'portal_alarms',
object_tag = 'simulation' 'portal_activities',
object_after_tag = folder_after_tag 'portal_categories',
object_priority = 3 + additional_priority 'portal_classes',
context.portal_simulation.activate( 'portal_preferences',
tag=folder_tag, 'portal_simulation',
priority=higher_priority, 'portal_uidhandler',
after_tag=folder_after_tag).Folder_reindexAll( ) and
folder_tag=folder_tag, 'inventory' not in x.id
folder_after_tag=folder_after_tag, ],
object_tag=object_tag, tag=document_tag,
object_after_tag=object_after_tag, after_tag=(user_tag, category_tag, preference_tag),
object_priority=higher_priority, ),
sql_catalog_id=sql_catalog_id, # Then we index ERP5 Python Scripts and ERP5 Form - this is fundamentally broken and will go away, do not depend on it !
start_tree=start_tree, skin_activate_kw = {
stop_tree=stop_tree,) 'tag': document_tag,
'priority': additional_priority,
# We index tools secondly 'after_tag': (user_tag, category_tag, preference_tag),
print "#### Indexing tools ####" }
folder_tag = 'module'
folder_after_tag = ('category', 'person_stage_1', 'group_person_stage_1', preference_tag)
object_tag = 'document'
object_after_tag = folder_after_tag
object_priority = 2 + additional_priority
tool_list = [x for x in portal.objectValues() if \
x.getUid != portal.getUid and \
x.meta_type != 'ERP5 Folder' and \
x.id not in ('portal_alarms', 'portal_activities', 'portal_classes', 'portal_preferences', 'portal_simulation', 'portal_uidhandler')]
for folder in tool_list:
folder.activate(
tag=folder_tag,
priority=object_priority,
after_tag=folder_after_tag).Folder_reindexAll(
folder_tag=folder_tag,
folder_after_tag=folder_after_tag,
object_tag=object_tag,
object_after_tag=object_after_tag,
object_priority=object_priority,
sql_catalog_id=sql_catalog_id,
start_tree=start_tree,
stop_tree=stop_tree,)
# Then we index ERP5 Python Scripts and ERP5 Form
print "#### Indexing ERP5 Python Scripts, ERP5 Forms and ERP5 Reports ####"
skin_activate_kw = {'tag': folder_tag, 'priority': object_priority, 'after_tag': folder_after_tag}
for _, obj in portal.portal_skins.ZopeFind(portal.portal_skins, obj_metatypes=('ERP5 Python Script', 'ERP5 Form', 'ERP5 Report'), search_sub=1): for _, obj in portal.portal_skins.ZopeFind(portal.portal_skins, obj_metatypes=('ERP5 Python Script', 'ERP5 Form', 'ERP5 Report'), search_sub=1):
obj.recursiveReindexObject(activate_kw=skin_activate_kw, obj.recursiveReindexObject(activate_kw=skin_activate_kw,
sql_catalog_id=sql_catalog_id) sql_catalog_id=sql_catalog_id)
print reindex(
# Then we index everything except inventories [
for folder in portal.objectValues(("ERP5 Folder",)): x for x in portal.objectValues(("ERP5 Folder", ))
if folder.getId().find('inventory') < 0: if 'inventory' in x.id
print "#### Indexing contents inside folder %s ####" % folder.id ],
folder.activate( tag=inventory_tag,
tag=folder_tag, after_tag=(user_tag, category_tag, document_tag, preference_tag),
priority=object_priority, ),
after_tag=folder_after_tag).Folder_reindexAll(
folder_tag=folder_tag, portal.portal_activities.activate(
folder_after_tag=folder_after_tag, after_tag=(user_tag, category_tag, document_tag, preference_tag, inventory_tag, simulation_tag),
object_tag=object_tag, ).InventoryModule_reindexMovementList(
object_after_tag=object_after_tag,
object_priority=object_priority,
sql_catalog_id=sql_catalog_id,
start_tree=start_tree,
stop_tree=stop_tree,)
# Then we index inventories
object_tag = 'inventory'
object_after_tag = ('module', 'category', 'person_stage_1', 'document', 'group_person_stage_1')
for folder in portal.objectValues(("ERP5 Folder",)):
if folder.getId().find('inventory') >= 0:
print "#### Indexing contents inside folder %s ####" % folder.id
folder.activate(
tag=folder_tag,
priority=object_priority,
after_tag=folder_after_tag).Folder_reindexAll(
folder_tag=folder_tag,
folder_after_tag=folder_after_tag,
object_tag=object_tag,
object_after_tag=object_after_tag,
object_priority=object_priority,
sql_catalog_id=sql_catalog_id, sql_catalog_id=sql_catalog_id,
start_tree=start_tree, final_activity_tag=last_inventory_tag,
stop_tree=stop_tree,) )
# start activty from simulation because the erp5site is not an active object
context.portal_simulation.activate(
after_tag=('inventory', 'simulation', 'person_stage_1', 'group_person_stage_1'),
priority=3 + additional_priority
).InventoryModule_reindexMovementList(
sql_catalog_id=sql_catalog_id,
final_activity_tag='last_inventory_activity')
# restore alarm node # restore alarm node
if clear_catalog and is_subscribed: if clear_catalog and is_subscribed:
portal.portal_alarms.activate(after_tag=('inventory', 'module', 'inventory', 'simulation', 'person_stage_1', portal.portal_alarms.activate(
'group_person_stage_1', 'last_inventory_activity', 'document')).subscribe() after_tag=(user_tag, category_tag, document_tag, preference_tag, inventory_tag, simulation_tag, last_inventory_tag),
).subscribe()
if final_activity_tag is not None:
# Start a dummy activity which will get discarded when all started activities portal.portal_activities.activate(
# (and all activities they trigger) are over. tag=final_activity_tag,
# Started on portal_simulation because activate does not work on portal object... after_tag=(user_tag, category_tag, document_tag, preference_tag, inventory_tag, simulation_tag, last_inventory_tag),
# No idea if there is a better place. ).getId()
context.portal_simulation.activate(tag=final_activity_tag,
priority=3 + additional_priority,
after_tag=('module', 'inventory', 'simulation', 'person_stage_1',
'group_person_stage_1', 'last_inventory_activity', 'document')
).getId()
return printed return printed
...@@ -50,7 +50,7 @@ ...@@ -50,7 +50,7 @@
</item> </item>
<item> <item>
<key> <string>_params</string> </key> <key> <string>_params</string> </key>
<value> <string>sql_catalog_id=None, additional_priority=0, clear_catalog=0, final_activity_tag=None, start_tree=None, stop_tree=None</string> </value> <value> <string>sql_catalog_id=None, clear_catalog=False, additional_priority=1, final_activity_tag="ERP5Site_reindexAll"</string> </value>
</item> </item>
<item> <item>
<key> <string>id</string> </key> <key> <string>id</string> </key>
......
folder = context
activate_kw = {
'tag': object_tag,
'after_tag': folder_after_tag,
}
for key, value in activate_kw.items():
if value is None:
activate_kw.pop(key)
# Reindex folder immediately
folder.reindexObject(sql_catalog_id=sql_catalog_id)
folder_id = folder.getId()
if folder_after_tag is None:
folder_after_tag = ()
bundle_tag = "%s_bundle_reindex" % folder_id
bundle_object_tag = "%s_reindex" % folder_id
# Spawn activities for bundles of content objects.
# Bundle size, in object count
BUNDLE_ITEM_COUNT=1000
def Folder_reindexObjectList(id_list_list):
"""
Create an activity calling Folder_reindexObjectList.
"""
folder.activate(activity='SQLQueue', priority=object_priority,
**activate_kw).Folder_reindexObjectList(
id_list=None,
id_list_list=id_list_list,
object_priority=object_priority,
object_tag=object_tag,
object_after_tag=object_after_tag,
folder_tag=bundle_tag,
folder_after_tag=bundle_object_tag,
sql_catalog_id=sql_catalog_id,
)
archive_test_script = getattr(context.getPortalObject(), "Archive_test", None)
try:
tree_id_list = folder.getTreeIdList()
except (NotImplementedError, AttributeError):
# Build a list of list, like this we parse ids only one time,
# and then Folder_reinexObjectList will work with one list at
# a time and remove it from the list of list
# This id_list_list can be quite big and generate quite big
# activities, but the effect is limited, because if we have too
# much objects (like millions), we should use HBTree Folders, and
# then the work will be splitted
id_list = [x for x in folder.objectIds()]
id_list_list = []
for bundle_index in xrange(len(id_list) / BUNDLE_ITEM_COUNT):
id_list_list.append(id_list[bundle_index * BUNDLE_ITEM_COUNT:((bundle_index + 1) * BUNDLE_ITEM_COUNT)])
remaining_object_id_count = len(id_list) % BUNDLE_ITEM_COUNT
if remaining_object_id_count > 0:
id_list_list.append(id_list[-remaining_object_id_count:])
Folder_reindexObjectList(id_list_list)
else:
if archive_test_script is not None:
new_tree_id_list = []
for tree_id in tree_id_list:
if folder.Archive_test(tree_id=tree_id, start_tree=start_tree, stop_tree=stop_tree):
new_tree_id_list.append(tree_id)
tree_id_list = new_tree_id_list
else:
if start_tree is not None:
new_tree_id_list = []
for tree_id in tree_id_list:
if tree_id >= start_tree:
new_tree_id_list.append(tree_id)
tree_id_list = new_tree_id_list
if stop_tree is not None:
new_tree_id_list = []
for tree_id in tree_id_list:
if tree_id < stop_tree:
new_tree_id_list.append(tree_id)
tree_id_list = new_tree_id_list
if len(tree_id_list) == 0:
return
i = 0
tree_tag = "%s_tree" % folder_id
# Say to Folder_reindexTreeObjectList to call himself again and
# again until all tree_id_list are parsed. Also, make sure that
# the work of the previous Folder_reindexTreeObjectList is completely
# done
tree_after_tag = folder_after_tag + (bundle_tag, bundle_object_tag)
folder.activate(activity='SQLQueue', priority=object_priority,
after_tag=tree_after_tag, tag=tree_tag,
).Folder_reindexTreeObjectList(
tree_id=None,
tree_id_list=tree_id_list,
folder_tag=bundle_tag,
folder_after_tag=bundle_object_tag,
object_priority=object_priority,
sql_catalog_id=sql_catalog_id,
object_tag=bundle_object_tag,
tree_after_tag=tree_after_tag,
tree_tag=tree_tag,
)
# Start an activity wich will wait the end of the module
folder_id_after_tag = folder_after_tag + (tree_tag, bundle_tag, bundle_object_tag)
id_activate_kw = {}
if object_tag is not None:
id_activate_kw['tag'] = object_tag
folder.activate(activity='SQLDict', priority=object_priority,
after_tag=folder_id_after_tag, **id_activate_kw).getId()
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>folder_tag=None, folder_after_tag=None, object_tag=None, object_after_tag=None, object_priority=1, sql_catalog_id=None, start_tree=None, stop_tree=None</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Folder_reindexAll</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
folder = context
# Keep compatibility with id_list
if id_list_list is not None:
if id_list is not None:
raise ValueError, "both id_list and id_list_list can be defined"
if len(id_list_list) == 0:
return None
id_list = id_list_list.pop()
activate_kw = {
'tag': object_tag,
'after_tag': object_after_tag,
'priority': object_priority
}
for key, value in activate_kw.items():
if value is None:
activate_kw.pop(key)
for id in id_list:
obj = getattr(folder, id, None)
if obj is not None:
obj.recursiveReindexObject(activate_kw=activate_kw,
sql_catalog_id=sql_catalog_id)
if id_list_list is not None:
if len(id_list_list) > 0:
if count is None:
count = 1
new_activity_kw = {}
# We do not want to wait until there is enough activities
# So add BUNDLE_ITEM_COUNT * node_len activities before waiting
node_len = len(context.portal_activities.getProcessingNodeList())
if count % node_len == 0:
new_activity_kw['after_tag'] = folder_after_tag
count = 0
count += 1
# By calling again and again, we improve performance and we have
# less activities by the same time
folder.activate(activity='SQLQueue',
priority=object_priority,
tag=folder_tag, **new_activity_kw).Folder_reindexObjectList(
None,
id_list_list=id_list_list,
object_priority=object_priority,
object_tag=object_tag,
sql_catalog_id=sql_catalog_id,
folder_tag=folder_tag,
folder_after_tag=folder_after_tag,
count=count,
)
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>id_list, id_list_list=None, folder_tag=None, folder_after_tag=None, object_tag=None, object_after_tag=None, object_priority=1, sql_catalog_id=None, count=None</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Folder_reindexObjectList</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
from Products.ERP5Type.Log import log
folder = context
# Keep compatibility with tree_id
if tree_id_list is not None:
log('tree_id', tree_id)
log('tree_id_list', tree_id_list)
if tree_id is not None:
raise ValueError, "both tree and tree_id_list should not be defined"
tree_id = tree_id_list.pop()
# Spawn activities for bundles of content objects.
# Bundle size, in object count
BUNDLE_ITEM_COUNT = 1000
folder_id = folder.getId()
def Folder_reindexObjectList(id_list_list):
"""
Create an activity calling Folder_reindexObjectList.
"""
folder.activate(activity='SQLQueue', priority=object_priority,
after_tag=object_tag,
tag=folder_tag).Folder_reindexObjectList(
None,
id_list_list=id_list_list,
object_priority=object_priority,
object_tag=object_tag,
sql_catalog_id=sql_catalog_id,
folder_tag=folder_tag,
folder_after_tag=folder_after_tag,
)
# HBTree folder
id_list = [x for x in folder.objectIds(base_id=tree_id)]
# Build a list of list, like this we parse ids only one time,
# and then Folder_reinexObjectList will work with one list at
# a time and remove it from the list of list
# This id_list_list can be quite big and generate quite big
# activities, but the effect is limited, because the work is
# splitted for each base_id of the HBTree.
id_list_list = []
for bundle_index in xrange(len(id_list) / BUNDLE_ITEM_COUNT):
id_list_list.append(id_list[bundle_index * BUNDLE_ITEM_COUNT:((bundle_index + 1) * BUNDLE_ITEM_COUNT)])
remaining_object_id_count = len(id_list) % BUNDLE_ITEM_COUNT
if remaining_object_id_count > 0:
id_list_list.append(id_list[-remaining_object_id_count:])
Folder_reindexObjectList(id_list_list=id_list_list)
if tree_id_list is not None:
if len(tree_id_list) > 0:
# Calling again and again the same script allow to decrease the
# number of activities by the same time and increase performance.
folder.activate(activity='SQLQueue', priority=object_priority,
after_tag=tree_after_tag,
tag=tree_tag).Folder_reindexTreeObjectList(
tree_id=None,
tree_id_list=tree_id_list,
folder_tag=folder_tag,
folder_after_tag=folder_after_tag,
object_priority=object_priority,
sql_catalog_id=sql_catalog_id,
object_tag=object_tag,
tree_after_tag=tree_after_tag,
tree_tag=tree_tag,
)
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>tree_id=None, tree_id_list=None, folder_tag=None, folder_after_tag=None, object_tag=None, object_after_tag=None, object_priority=1, sql_catalog_id=None, tree_tag=None, tree_after_tag=None</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Folder_reindexTreeObjectList</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
...@@ -2768,9 +2768,6 @@ class BusinessTemplateMixin(ERP5TypeTestCase, LogInterceptor): ...@@ -2768,9 +2768,6 @@ class BusinessTemplateMixin(ERP5TypeTestCase, LogInterceptor):
self.assertEqual(len(message_list), 0) self.assertEqual(len(message_list), 0)
def stepCheckHasClearedCatalog(self, sequence=None, **kw): def stepCheckHasClearedCatalog(self, sequence=None, **kw):
"""
Check if we have activity for Folder_reindexAll.
"""
self.assertTrue(self._has_cleared_catalog) self.assertTrue(self._has_cleared_catalog)
del self._has_cleared_catalog[:] del self._has_cleared_catalog[:]
......
...@@ -419,7 +419,7 @@ class TestDomainTool(TestPredicateMixIn): ...@@ -419,7 +419,7 @@ class TestDomainTool(TestPredicateMixIn):
# Enable system preference and reindex relevant predicates # Enable system preference and reindex relevant predicates
system_preference.enable() system_preference.enable()
self.tic() self.tic()
supply_module.Folder_reindexAll() supply_module.recursiveReindexObject()
self.tic() self.tic()
# if document has relations using base categories which are not present in the preference, then left join mode is still used. # if document has relations using base categories which are not present in the preference, then left join mode is still used.
assertUsesLeftJoinAndPredicateItemsMatchingOrderLineEqual(True, [supply1_line1]) assertUsesLeftJoinAndPredicateItemsMatchingOrderLineEqual(True, [supply1_line1])
...@@ -432,7 +432,7 @@ class TestDomainTool(TestPredicateMixIn): ...@@ -432,7 +432,7 @@ class TestDomainTool(TestPredicateMixIn):
system_preference.setPreferredPredicateCategoryList(['source_section', 'destination_section', 'price_currency', 'resource']) system_preference.setPreferredPredicateCategoryList(['source_section', 'destination_section', 'price_currency', 'resource'])
self.portal.portal_caches.clearAllCache() self.portal.portal_caches.clearAllCache()
self.tic() self.tic()
supply_module.Folder_reindexAll() supply_module.recursiveReindexObject()
self.tic() self.tic()
# resource is not in preferred predicate category list, so only inner join is used # resource is not in preferred predicate category list, so only inner join is used
assertUsesLeftJoinAndPredicateItemsMatchingOrderLineEqual(False, [supply1_line1], tested_base_category_list=['source_section', 'destination_section', 'price_currency', 'resource']) assertUsesLeftJoinAndPredicateItemsMatchingOrderLineEqual(False, [supply1_line1], tested_base_category_list=['source_section', 'destination_section', 'price_currency', 'resource'])
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment