Skip to content

erp5
erp5
Commit 9d05fd02 authored by Vincent Pelletier's avatar Vincent Pelletier
Browse files
Options

all: Get rid of Folder_reindex{All,Objectlist,TreeObjectList} 

These are now superseded by recursiveReindexObject.
parent 67bb6b90
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 130 additions and 630 deletions
bt5/erp5_configurator_run_my_doc/SkinTemplateItem/portal_skins/erp5_configurator_run_my_doc/BusinessConfiguration_reindexRunMyDocObjectsAndRoles.py
View file @ 9d05fd02
""" This script reindex all the objects created before updating local roles """ """ This script reindex all the objects created before updating local roles """
portal = context.getPortalObject() portal = context.getPortalObject()
portal.portal_types.Folder_reindexAll() portal.portal_types.recursiveReindexObject()
stack = [ stack = [
portal.document_module, portal.document_module,
portal.image_module, portal.image_module,
......
bt5/erp5_demo_smb/SkinTemplateItem/portal_skins/erp5_demo_smb/Alarm_installBusinessTemplateList.py
View file @ 9d05fd02
...@@ -5,7 +5,7 @@ portal.portal_templates.updateRepositoryBusinessTemplateList( ...@@ -5,7 +5,7 @@ portal.portal_templates.updateRepositoryBusinessTemplateList(
['http://www.erp5.org/dists/snapshot/bt5/']) ['http://www.erp5.org/dists/snapshot/bt5/'])
business_template_list = context.Base_getDemoSMBBusinessTemplateList() business_template_list = context.Base_getDemoSMBBusinessTemplateList()
kw = dict(tag="start", after_method_id = ["immediateReindexObject", "Folder_reindexAll"]) kw = {'tag': "start", 'after_method_id': ["immediateReindexObject"]}
for business_template_id in business_template_list: for business_template_id in business_template_list:
portal.portal_templates.activate(**kw).installBusinessTemplatesFromRepositories((business_template_id,)) portal.portal_templates.activate(**kw).installBusinessTemplatesFromRepositories((business_template_id,))
kw["after_tag"] = kw["tag"] kw["after_tag"] = kw["tag"]
......
product/ERP5/ERP5Site.py
View file @ 9d05fd02
...@@ -1940,16 +1940,17 @@ class ERP5Generator(PortalGenerator): ...@@ -1940,16 +1940,17 @@ class ERP5Generator(PortalGenerator):
p._v_bootstrapping = False p._v_bootstrapping = False
# XXX: Is it useful to wait for indexing before using upgradeSite ? reindex_all_tag = 'ERP5Site_reindexAll'
after_method_id = 'immediateReindexObject' upgrade_tag = 'updgradeSite'
preference_tag = 'initSystemPreference'
if bt5_repository_url: if bt5_repository_url:
p.portal_templates.repository_dict = dict.fromkeys( p.portal_templates.repository_dict = dict.fromkeys(bt5_repository_url.split())
bt5_repository_url.split())
if bt5: if bt5:
method_id = 'upgradeSite' p.portal_templates.activate(
getattr(p.portal_templates.activate(after_method_id=after_method_id), # XXX: Is it useful to wait for indexing ?
method_id)(bt5.split(), update_catalog=True) after_tag=reindex_all_tag,
after_method_id = method_id tag=upgrade_tag,
).upgradeSite(bt5.split(), update_catalog=True)
if id_store_interval != '': if id_store_interval != '':
id_store_interval = int(id_store_interval) id_store_interval = int(id_store_interval)
if id_store_interval < 0: if id_store_interval < 0:
...@@ -1961,22 +1962,17 @@ class ERP5Generator(PortalGenerator): ...@@ -1961,22 +1962,17 @@ class ERP5Generator(PortalGenerator):
else: else:
ob._setStoredInZodb(0) ob._setStoredInZodb(0)
if cloudooo_url: if cloudooo_url:
method_id = '_initSystemPreference' p.portal_activities.activateObject(
getattr(p.portal_activities.activateObject(p, p,
after_method_id=after_method_id), method_id)(cloudooo_url=cloudooo_url) after_tag=(reindex_all_tag, upgrade_tag),
after_method_id = method_id tag=preference_tag,
)._initSystemPreference(cloudooo_url=cloudooo_url)
id_ = 'isPortalBeingCreated' id_ = 'isPortalBeingCreated'
setattr(p, id_, ConstantGetter(id_, value=True)) setattr(p, id_, ConstantGetter(id_, value=True))
# XXX: ERP5Site_reindexAll should be reviewed so that one can depend on a p.portal_activities.activateObject(
# final tag. A more general approach is to have an activity dependency p,
# to anything, so that _delPropValue is called as soon as activity after_tag=(reindex_all_tag, upgrade_tag, preference_tag),
# nodes have nothing else to do. )._delPropValue(id_)
after_method_id = tuple({after_method_id}.union(('Folder_reindexAll',
'Folder_reindexObjectList', 'InventoryModule_reindexMovementList',
'immediateReindexObject', 'SQLCatalog_deferFullTextIndexActivity')))
p.portal_activities.activateObject(p, after_method_id=after_method_id,
)._delPropValue(id_)
return p return p
@classmethod @classmethod
......
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/ERP5Site_reindexAll.xml
View file @ 9d05fd02
...@@ -50,7 +50,7 @@ ...@@ -50,7 +50,7 @@
</item> </item>
<item> <item>
<key> <string>_params</string> </key> <key> <string>_params</string> </key>
<value> <string>sql_catalog_id=None, additional_priority=0, clear_catalog=0, final_activity_tag=None, start_tree=None, stop_tree=None</string> </value> <value> <string>sql_catalog_id=None, clear_catalog=False, additional_priority=1, final_activity_tag="ERP5Site_reindexAll"</string> </value>
</item> </item>
<item> <item>
<key> <string>id</string> </key> <key> <string>id</string> </key>
......
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexAll.py deleted 100644 → 0
View file @ 67bb6b90
folder = context
activate_kw = {
'tag': object_tag,
'after_tag': folder_after_tag,
}
for key, value in activate_kw.items():
if value is None:
activate_kw.pop(key)
# Reindex folder immediately
folder.reindexObject(sql_catalog_id=sql_catalog_id)
folder_id = folder.getId()
if folder_after_tag is None:
folder_after_tag = ()
bundle_tag = "%s_bundle_reindex" % folder_id
bundle_object_tag = "%s_reindex" % folder_id
# Spawn activities for bundles of content objects.
# Bundle size, in object count
BUNDLE_ITEM_COUNT=1000
def Folder_reindexObjectList(id_list_list):
"""
Create an activity calling Folder_reindexObjectList.
"""
folder.activate(activity='SQLQueue', priority=object_priority,
**activate_kw).Folder_reindexObjectList(
id_list=None,
id_list_list=id_list_list,
object_priority=object_priority,
object_tag=object_tag,
object_after_tag=object_after_tag,
folder_tag=bundle_tag,
folder_after_tag=bundle_object_tag,
sql_catalog_id=sql_catalog_id,
)
archive_test_script = getattr(context.getPortalObject(), "Archive_test", None)
try:
tree_id_list = folder.getTreeIdList()
except (NotImplementedError, AttributeError):
# Build a list of list, like this we parse ids only one time,
# and then Folder_reinexObjectList will work with one list at
# a time and remove it from the list of list
# This id_list_list can be quite big and generate quite big
# activities, but the effect is limited, because if we have too
# much objects (like millions), we should use HBTree Folders, and
# then the work will be splitted
id_list = [x for x in folder.objectIds()]
id_list_list = []
for bundle_index in xrange(len(id_list) / BUNDLE_ITEM_COUNT):
id_list_list.append(id_list[bundle_index * BUNDLE_ITEM_COUNT:((bundle_index + 1) * BUNDLE_ITEM_COUNT)])
remaining_object_id_count = len(id_list) % BUNDLE_ITEM_COUNT
if remaining_object_id_count > 0:
id_list_list.append(id_list[-remaining_object_id_count:])
Folder_reindexObjectList(id_list_list)
else:
if archive_test_script is not None:
new_tree_id_list = []
for tree_id in tree_id_list:
if folder.Archive_test(tree_id=tree_id, start_tree=start_tree, stop_tree=stop_tree):
new_tree_id_list.append(tree_id)
tree_id_list = new_tree_id_list
else:
if start_tree is not None:
new_tree_id_list = []
for tree_id in tree_id_list:
if tree_id >= start_tree:
new_tree_id_list.append(tree_id)
tree_id_list = new_tree_id_list
if stop_tree is not None:
new_tree_id_list = []
for tree_id in tree_id_list:
if tree_id < stop_tree:
new_tree_id_list.append(tree_id)
tree_id_list = new_tree_id_list
if len(tree_id_list) == 0:
return
i = 0
tree_tag = "%s_tree" % folder_id
# Say to Folder_reindexTreeObjectList to call himself again and
# again until all tree_id_list are parsed. Also, make sure that
# the work of the previous Folder_reindexTreeObjectList is completely
# done
tree_after_tag = folder_after_tag + (bundle_tag, bundle_object_tag)
folder.activate(activity='SQLQueue', priority=object_priority,
after_tag=tree_after_tag, tag=tree_tag,
).Folder_reindexTreeObjectList(
tree_id=None,
tree_id_list=tree_id_list,
folder_tag=bundle_tag,
folder_after_tag=bundle_object_tag,
object_priority=object_priority,
sql_catalog_id=sql_catalog_id,
object_tag=bundle_object_tag,
tree_after_tag=tree_after_tag,
tree_tag=tree_tag,
)
# Start an activity wich will wait the end of the module
folder_id_after_tag = folder_after_tag + (tree_tag, bundle_tag, bundle_object_tag)
id_activate_kw = {}
if object_tag is not None:
id_activate_kw['tag'] = object_tag
folder.activate(activity='SQLDict', priority=object_priority,
after_tag=folder_id_after_tag, **id_activate_kw).getId()
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexAll.xml deleted 100644 → 0
View file @ 67bb6b90
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>folder_tag=None, folder_after_tag=None, object_tag=None, object_after_tag=None, object_priority=1, sql_catalog_id=None, start_tree=None, stop_tree=None</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Folder_reindexAll</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexObjectList.py deleted 100644 → 0
View file @ 67bb6b90
folder = context
# Keep compatibility with id_list
if id_list_list is not None:
if id_list is not None:
raise ValueError, "both id_list and id_list_list can be defined"
if len(id_list_list) == 0:
return None
id_list = id_list_list.pop()
activate_kw = {
'tag': object_tag,
'after_tag': object_after_tag,
'priority': object_priority
}
for key, value in activate_kw.items():
if value is None:
activate_kw.pop(key)
for id in id_list:
obj = getattr(folder, id, None)
if obj is not None:
obj.recursiveReindexObject(activate_kw=activate_kw,
sql_catalog_id=sql_catalog_id)
if id_list_list is not None:
if len(id_list_list) > 0:
if count is None:
count = 1
new_activity_kw = {}
# We do not want to wait until there is enough activities
# So add BUNDLE_ITEM_COUNT * node_len activities before waiting
node_len = len(context.portal_activities.getProcessingNodeList())
if count % node_len == 0:
new_activity_kw['after_tag'] = folder_after_tag
count = 0
count += 1
# By calling again and again, we improve performance and we have
# less activities by the same time
folder.activate(activity='SQLQueue',
priority=object_priority,
tag=folder_tag, **new_activity_kw).Folder_reindexObjectList(
None,
id_list_list=id_list_list,
object_priority=object_priority,
object_tag=object_tag,
sql_catalog_id=sql_catalog_id,
folder_tag=folder_tag,
folder_after_tag=folder_after_tag,
count=count,
)
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexObjectList.xml deleted 100644 → 0
View file @ 67bb6b90
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>id_list, id_list_list=None, folder_tag=None, folder_after_tag=None, object_tag=None, object_after_tag=None, object_priority=1, sql_catalog_id=None, count=None</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Folder_reindexObjectList</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexTreeObjectList.py deleted 100644 → 0
View file @ 67bb6b90
from Products.ERP5Type.Log import log
folder = context
# Keep compatibility with tree_id
if tree_id_list is not None:
log('tree_id', tree_id)
log('tree_id_list', tree_id_list)
if tree_id is not None:
raise ValueError, "both tree and tree_id_list should not be defined"
tree_id = tree_id_list.pop()
# Spawn activities for bundles of content objects.
# Bundle size, in object count
BUNDLE_ITEM_COUNT = 1000
folder_id = folder.getId()
def Folder_reindexObjectList(id_list_list):
"""
Create an activity calling Folder_reindexObjectList.
"""
folder.activate(activity='SQLQueue', priority=object_priority,
after_tag=object_tag,
tag=folder_tag).Folder_reindexObjectList(
None,
id_list_list=id_list_list,
object_priority=object_priority,
object_tag=object_tag,
sql_catalog_id=sql_catalog_id,
folder_tag=folder_tag,
folder_after_tag=folder_after_tag,
)
# HBTree folder
id_list = [x for x in folder.objectIds(base_id=tree_id)]
# Build a list of list, like this we parse ids only one time,
# and then Folder_reinexObjectList will work with one list at
# a time and remove it from the list of list
# This id_list_list can be quite big and generate quite big
# activities, but the effect is limited, because the work is
# splitted for each base_id of the HBTree.
id_list_list = []
for bundle_index in xrange(len(id_list) / BUNDLE_ITEM_COUNT):
id_list_list.append(id_list[bundle_index * BUNDLE_ITEM_COUNT:((bundle_index + 1) * BUNDLE_ITEM_COUNT)])
remaining_object_id_count = len(id_list) % BUNDLE_ITEM_COUNT
if remaining_object_id_count > 0:
id_list_list.append(id_list[-remaining_object_id_count:])
Folder_reindexObjectList(id_list_list=id_list_list)
if tree_id_list is not None:
if len(tree_id_list) > 0:
# Calling again and again the same script allow to decrease the
# number of activities by the same time and increase performance.
folder.activate(activity='SQLQueue', priority=object_priority,
after_tag=tree_after_tag,
tag=tree_tag).Folder_reindexTreeObjectList(
tree_id=None,
tree_id_list=tree_id_list,
folder_tag=folder_tag,
folder_after_tag=folder_after_tag,
object_priority=object_priority,
sql_catalog_id=sql_catalog_id,
object_tag=object_tag,
tree_after_tag=tree_after_tag,
tree_tag=tree_tag,
)
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexTreeObjectList.xml deleted 100644 → 0
View file @ 67bb6b90
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>tree_id=None, tree_id_list=None, folder_tag=None, folder_after_tag=None, object_tag=None, object_after_tag=None, object_priority=1, sql_catalog_id=None, tree_tag=None, tree_after_tag=None</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Folder_reindexTreeObjectList</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/tests/testBusinessTemplate.py
View file @ 9d05fd02
...@@ -2768,9 +2768,6 @@ class BusinessTemplateMixin(ERP5TypeTestCase, LogInterceptor): ...@@ -2768,9 +2768,6 @@ class BusinessTemplateMixin(ERP5TypeTestCase, LogInterceptor):
self.assertEqual(len(message_list), 0) self.assertEqual(len(message_list), 0)
def stepCheckHasClearedCatalog(self, sequence=None, **kw): def stepCheckHasClearedCatalog(self, sequence=None, **kw):
"""
Check if we have activity for Folder_reindexAll.
"""
self.assertTrue(self._has_cleared_catalog) self.assertTrue(self._has_cleared_catalog)
del self._has_cleared_catalog[:] del self._has_cleared_catalog[:]
......
product/ERP5/tests/testDomainTool.py
View file @ 9d05fd02
...@@ -419,7 +419,7 @@ class TestDomainTool(TestPredicateMixIn): ...@@ -419,7 +419,7 @@ class TestDomainTool(TestPredicateMixIn):
# Enable system preference and reindex relevant predicates # Enable system preference and reindex relevant predicates
system_preference.enable() system_preference.enable()
self.tic() self.tic()
supply_module.Folder_reindexAll() supply_module.recursiveReindexObject()
self.tic() self.tic()
# if document has relations using base categories which are not present in the preference, then left join mode is still used. # if document has relations using base categories which are not present in the preference, then left join mode is still used.
assertUsesLeftJoinAndPredicateItemsMatchingOrderLineEqual(True, [supply1_line1]) assertUsesLeftJoinAndPredicateItemsMatchingOrderLineEqual(True, [supply1_line1])
...@@ -432,7 +432,7 @@ class TestDomainTool(TestPredicateMixIn): ...@@ -432,7 +432,7 @@ class TestDomainTool(TestPredicateMixIn):
system_preference.setPreferredPredicateCategoryList(['source_section', 'destination_section', 'price_currency', 'resource']) system_preference.setPreferredPredicateCategoryList(['source_section', 'destination_section', 'price_currency', 'resource'])
self.portal.portal_caches.clearAllCache() self.portal.portal_caches.clearAllCache()
self.tic() self.tic()
supply_module.Folder_reindexAll() supply_module.recursiveReindexObject()
self.tic() self.tic()
# resource is not in preferred predicate category list, so only inner join is used # resource is not in preferred predicate category list, so only inner join is used
assertUsesLeftJoinAndPredicateItemsMatchingOrderLineEqual(False, [supply1_line1], tested_base_category_list=['source_section', 'destination_section', 'price_currency', 'resource']) assertUsesLeftJoinAndPredicateItemsMatchingOrderLineEqual(False, [supply1_line1], tested_base_category_list=['source_section', 'destination_section', 'price_currency', 'resource'])
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7