all: Avoid most direct calls to {recursiveI,i}mmediateReindexObject

These methods must not be called synchronously:
- they can break catalog by not being careful enough about other
  reindexations which may happen in parallel. See the serialization_tag
  mechanism for more.
- indexation gets executed in the security context of the user causing the
  call, which may lead to an indexation result different from what happens
  when indexation happens with an all-accesses user.

Also, simplify a few scripts while doing so.

bt5/erp5_configurator_run_my_doc/SkinTemplateItem/portal_skins/erp5_configurator_run_my_doc/BusinessConfiguration_reindexRunMyDocObjectsAndRoles.py

 """ This script reindex all the objects created before updating local roles """
-module_list = ['document_module',
-               'image_module',
-               'knowledge_pad_module',
-               'organisation_module',
-               'person_module',
-               'review_module',
-               'test_page_module',
-               'web_page_module',
-               'web_site_module']
-
-context.portal_types.recursiveImmediateReindexObject()
 portal = context.getPortalObject()
-for module_id in module_list:
-  module = getattr(portal, module_id)
-  module.recursiveImmediateReindexObject()
-  stack = [module]
-  for obj in stack:
-    for child in obj.objectValues():
-      stack.append(child)
-    obj.updateLocalRolesOnSecurityGroups()
-    obj.reindexObjectSecurity()
+portal.portal_types.Folder_reindexAll()
+stack = [
+  portal.document_module,
+  portal.image_module,
+  portal.knowledge_pad_module,
+  portal.organisation_module,
+  portal.person_module,
+  portal.review_module,
+  portal.test_page_module,
+  portal.web_page_module,
+  portal.web_site_module,
+]
+for obj in stack:
+  stack.extend(obj.objectValues())
+  obj.updateLocalRolesOnSecurityGroups()

bt5/erp5_egov/WorkflowTemplateItem/portal_workflow/egov_interaction_workflow/scripts/SubscriptionForm_createAccount.py

+from Products.ERP5Type.ImmediateReindexContextManager import ImmediateReindexContextManager
 changed_object = state_change['object']
 
 portal = changed_object.getPortalObject()
@@ -28,29 +29,29 @@ else:
 
 # create the person wich represent the company
 person_module = portal.getDefaultModule(portal_type='Person')
-accountant = person_module.newContent(portal_type='Person',
-                         title=changed_object.getAccountantName(),
-                         default_telephone_text=changed_object.getAccountantTelNumber(),
-                         default_fax_text=changed_object.getAccountantFax(),
-                         default_email_text=changed_object.getAccountantEmail(),
-                         address_street_address=changed_object.getAccountantAddress(),
-                         address_city=changed_object.getAccountantCity(),
-                         career_subordination_value=organisation)
-
- 
-# create an assignment to be able to login :
-from DateTime import DateTime
-assignment = accountant.newContent(portal_type='Assignment')
-assignment.setStartDate(DateTime())
-assignment.setStopDate(DateTime()+365)
-assignment.setCareerFunction(changed_object.getAccountantFunction())
-assignment.open()
-
-# set the login and password required a manager role, so a script with a 
-# proxy role is used
-login = context.generateNewLogin(text=changed_object.getAccountantName())
-password = changed_object.Person_generatePassword()
-context.EGov_setLoginAndPasswordAsManager(accountant, login, password)
-accountant.immediateReindexObject()
+with ImmediateReindexContextManager() as immediate_reindex_context_manager:
+  accountant = person_module.newContent(portal_type='Person',
+                           immediate_reindex=immediate_reindex_context_manager,
+                           title=changed_object.getAccountantName(),
+                           default_telephone_text=changed_object.getAccountantTelNumber(),
+                           default_fax_text=changed_object.getAccountantFax(),
+                           default_email_text=changed_object.getAccountantEmail(),
+                           address_street_address=changed_object.getAccountantAddress(),
+                           address_city=changed_object.getAccountantCity(),
+                           career_subordination_value=organisation)
+
+
+  # create an assignment to be able to login :
+  assignment = accountant.newContent(portal_type='Assignment')
+  assignment.setStartDate(DateTime())
+  assignment.setStopDate(DateTime()+365)
+  assignment.setCareerFunction(changed_object.getAccountantFunction())
+  assignment.open()
+
+  # set the login and password required a manager role, so a script with a
+  # proxy role is used
+  login = context.generateNewLogin(text=changed_object.getAccountantName())
+  password = changed_object.Person_generatePassword()
+  context.EGov_setLoginAndPasswordAsManager(accountant, login, password)
 
 accountant.Person_sendCrendentialsByEMail()

bt5/erp5_knowledge_pad/SkinTemplateItem/portal_skins/erp5_knowledge_pad/ERP5Site_addNewKnowledgePad.py

-pad = context.knowledge_pad_module.newContent(portal_type='Knowledge Pad',
-                                              title = pad_title)
-# for web mode
-if mode in ('web_front', 'web_section',):
-  # in Web Mode we can have a temporary Web Site objects created based on current language
-  real_context = context.Base_getRealContext()
-  pad.setPublicationSectionValue(real_context)
+from Products.ERP5Type.ImmediateReindexContextManager import ImmediateReindexContextManager
+with ImmediateReindexContextManager() as immediate_reindex_context_manager:
+  pad = context.knowledge_pad_module.newContent(portal_type='Knowledge Pad',
+                                                immediate_reindex=immediate_reindex_context_manager,
+                                                title = pad_title)
+  # for web mode
+  if mode in ('web_front', 'web_section',):
+    # in Web Mode we can have a temporary Web Site objects created based on current language
+    real_context = context.Base_getRealContext()
+    pad.setPublicationSectionValue(real_context)
 
-# set it as active
-context.ERP5Site_toggleActiveKnowledgePad(pad, mode=mode, redirect=False)
-
-# See ERP5Site_createDefaultKnowledgePadListForUser
-pad.immediateReindexObject()
+  # set it as active
+  context.ERP5Site_toggleActiveKnowledgePad(pad, mode=mode, redirect=False)
 
 if redirect_url:
   return context.REQUEST.RESPONSE.redirect(redirect_url)

bt5/erp5_knowledge_pad/SkinTemplateItem/portal_skins/erp5_knowledge_pad/ERP5Site_createDefaultKnowledgePadListForUser.py

+from Products.ERP5Type.ImmediateReindexContextManager import ImmediateReindexContextManager
 knowledge_pad = None
 portal = context.getPortalObject()
 system_pref = context.portal_preferences.getActiveSystemPreference()
@@ -13,61 +14,58 @@ else:
   # find from preferences for the same context(site, section, page)
   filter_pad = lambda x: context in x.getPublicationSectionValueList()
 
-# try to find template KnowledgePad from System Preference (and user Preference
-# for backward compatibility only).
-for pref in (system_pref, user_pref):
-  if pref is not None:
-    # use template from preferences
-    for pref_pad in pref.objectValues(portal_type='Knowledge Pad'):
-      if filter_pad(pref_pad):
-        break
-    else:
-      continue
-    cp = pref.manage_copyObjects(ids=[pref_pad.getId()])
-    new_id = context.knowledge_pad_module.manage_pasteObjects(
-                                   cb_copy_data=cp)[0]['new_id']
-    knowledge_pad = context.knowledge_pad_module[new_id]
-    knowledge_pad.makeTemplateInstance()
-    # set each contaned box's state manually to visible
-    # by default their state as well pads would be invisible (default state)
-    # pad's visibility is fixed in ERP5Site_toggleActiveKnowledgePad()
-    for box in knowledge_pad.contentValues(portal_type='Knowledge Box'):
-      box.visible()
-    break
-else:
-  # created empty one because no template found
-  knowledge_pad = context.knowledge_pad_module.newContent(
-                            portal_type = 'Knowledge Pad',
-                            title = context.Base_translateString('Tab 1'))
-if is_web_mode:
-  # in Web Mode we can have a temporary Web Site objects created based on current language
-  real_context = context.Base_getRealContext()
-  if real_context.getPortalType() == 'Web Site' and not default_pad_group:
-    # script is called within Front Page Gadgets view
-    knowledge_pad.setPublicationSectionValue(real_context)
-
-  # create a default pad for user belonging to respective pad group
-  # this pad will be available globally for other contexes using the same
-  # layout definition
-  knowledge_pad.setGroup(default_pad_group)
+with ImmediateReindexContextManager() as immediate_reindex_context_manager:
+  # try to find template KnowledgePad from System Preference (and user Preference
+  # for backward compatibility only).
+  for pref in (system_pref, user_pref):
+    if pref is not None:
+      # use template from preferences
+      for pref_pad in pref.objectValues(portal_type='Knowledge Pad'):
+        if filter_pad(pref_pad):
+          break
+      else:
+        continue
+      new_id = context.knowledge_pad_module.manage_pasteObjects(
+        cb_copy_data=pref.manage_copyObjects(ids=[pref_pad.getId()]),
+        immediate_reindex=immediate_reindex_context_manager,
+      )[0]['new_id']
+      knowledge_pad = context.knowledge_pad_module[new_id]
+      knowledge_pad.makeTemplateInstance()
+      # set each contaned box's state manually to visible
+      # by default their state as well pads would be invisible (default state)
+      # pad's visibility is fixed in ERP5Site_toggleActiveKnowledgePad()
+      for box in knowledge_pad.contentValues(portal_type='Knowledge Box'):
+        box.visible()
+      break
+  else:
+    # created empty one because no template found
+    knowledge_pad = context.knowledge_pad_module.newContent(
+      immediate_reindex=immediate_reindex_context_manager,
+      portal_type = 'Knowledge Pad',
+      title = context.Base_translateString('Tab 1'),
+    )
+  if is_web_mode:
+    # in Web Mode we can have a temporary Web Site objects created based on current language
+    real_context = context.Base_getRealContext()
+    if real_context.getPortalType() == 'Web Site' and not default_pad_group:
+      # script is called within Front Page Gadgets view
+      knowledge_pad.setPublicationSectionValue(real_context)
 
-knowledge_pad.visible()
-# set owner
-if owner is not None:
-  current_user = context.portal_membership.getAuthenticatedMember()
-  knowledge_pad.manage_setLocalRoles(userid=owner, roles=['Owner'])
-  knowledge_pad.manage_delLocalRoles([current_user.getIdOrUserName()])
-  knowledge_pad.reindexObject()
+    # create a default pad for user belonging to respective pad group
+    # this pad will be available globally for other contexes using the same
+    # layout definition
+    knowledge_pad.setGroup(default_pad_group)
 
-# set default gadgets
-context.ERP5Site_createDefaultKnowledgeBox(knowledge_pad)
+  knowledge_pad.visible()
+  # set owner
+  if owner is not None:
+    current_user = context.portal_membership.getAuthenticatedMember()
+    knowledge_pad.manage_setLocalRoles(userid=owner, roles=['Owner'])
+    knowledge_pad.manage_delLocalRoles([current_user.getIdOrUserName()])
+    knowledge_pad.reindexObject()
 
-# Calling immediateReindexObject explicitly is a coding crime.
-# But it's safe for newly created objects and this script should
-# be called rarely enough to not cause any performance issue.
-# Any other solution would be more complicated.
-# See also ERP5Site_addNewKnowledgePad
-knowledge_pad.immediateReindexObject()
+  # set default gadgets
+  context.ERP5Site_createDefaultKnowledgeBox(knowledge_pad)
 
 if REQUEST is None:
   return knowledge_pad

bt5/erp5_officejs_support_request_ui/SkinTemplateItem/portal_skins/erp5_officejs_support_request/PostModule_createHTMLPostForSupportRequest.py

+from Products.ERP5Type.ImmediateReindexContextManager import ImmediateReindexContextManager
 follow_up_value = context.getPortalObject().restrictedTraverse(follow_up)
 assert follow_up_value.getPortalType() == "Support Request"
 follow_up_value.edit()  # update modification date
 
-post = context.PostModule_createHTMLPostFromText(
-  follow_up=follow_up,
-  data=data,
-)
+with ImmediateReindexContextManager() as immediate_reindex_context_manager:
+  post = context.PostModule_createHTMLPostFromText(
+    follow_up=follow_up,
+    data=data,
+    immediate_reindex_context_manager=immediate_reindex_context_manager,
+  )
 
-if file not in ("undefined", None):  # XXX "undefined" ? should also be fixed in javascript side
-  document_kw = {'batch_mode': True,
-                 'redirect_to_document': False,
-                 'file': file}
-  document = context.Base_contribute(**document_kw)
-  # set relation between post and document
-  # XXX successor is used as a way to put a relation between the attachment and the post,
-  #     the actual way should be to use a proper container like an Event that will have
-  #     one or several posts and one or several attachments.
-  post.setSuccessorValueList([document])
-  # XXX depending on security model this should be changed accordingly
-  document.publish()
+  if file not in ("undefined", None):  # XXX "undefined" ? should also be fixed in javascript side
+    document_kw = {'batch_mode': True,
+                   'redirect_to_document': False,
+                   'file': file}
+    document = context.Base_contribute(**document_kw)
+    # set relation between post and document
+    # XXX successor is used as a way to put a relation between the attachment and the post,
+    #     the actual way should be to use a proper container like an Event that will have
+    #     one or several posts and one or several attachments.
+    post.setSuccessorValueList([document])
+    # XXX depending on security model this should be changed accordingly
+    document.publish()
 
-post.publish()
-# XXX in support request web app interface, discussable page reloads right after
-#     adding a post, searching for new post hoping it is already indexed.
-post.immediateReindexObject()
+  post.publish()

bt5/erp5_officejs_support_request_ui/SkinTemplateItem/portal_skins/erp5_officejs_support_request/SupportRequestModule_createSupportRequest.py

+from Products.ERP5Type.ImmediateReindexContextManager import ImmediateReindexContextManager
 portal = context.getPortalObject()
 logged_in_user_value = portal.portal_membership.getAuthenticatedMember().getUserValue()
 
@@ -7,7 +8,13 @@ now = DateTime()
 project_list = portal.portal_catalog(portal_type="Project", id=project) # with id keyword, this function will return a sequence data type which contains one element.
 project_object = project_list[0].getObject()
 
-# support_request.Base_getRelatedPostList
+with ImmediateReindexContextManager() as immediate_reindex_context_manager:
+  support_request = portal.support_request_module.newContent(
+    immediate_reindex=immediate_reindex_context_manager,
+    portal_type='Support Request',
+    title=title,
+    resource="service_module/" + resource,
+  )
 
 #    - Reference = automatically generated - already implemented
 #    - Requester = current user person
@@ -17,21 +24,20 @@ project_object = project_list[0].getObject()
 #    - Location = Project related Location
 #    - Supervisor = Project related Supervisor
 
-support_request = portal.support_request_module.newContent(
-  portal_type='Support Request',
-  title=title,
-  resource="service_module/" + resource,
-  destination_decision_value=logged_in_user_value,
-  source_decision_value = project_object.getSourceDecisionValue(),
-  source_section_value = project_object.getSourceSectionValue(),
-  source_project_value = project_object,
-  destination_value = project_object.getDestinationValue(),
-  start_date=now,
-)
+  support_request = portal.support_request_module.newContent(
+    portal_type='Support Request',
+    title=title,
+    resource="service_module/" + resource,
+    destination_decision_value=logged_in_user_value,
+    source_decision_value = project_object.getSourceDecisionValue(),
+    source_section_value = project_object.getSourceSectionValue(),
+    source_project_value = project_object,
+    destination_value = project_object.getDestinationValue(),
+    start_date=now,
+  )
 
 
-support_request.submit()
-support_request.immediateReindexObject()
+  support_request.submit()
 
 if description is not None or file is not None:
   portal.post_module.PostModule_createHTMLPostForSupportRequest(

bt5/erp5_post/SkinTemplateItem/portal_skins/erp5_post/PostModule_createHTMLPost.py

@@ -13,6 +13,7 @@ if predecessor not in (None, ""):
   predecessor_value, = portal.portal_catalog(relative_url=predecessor, limit=2)
   post_edit_kw["predecessor_value"] = predecessor_value.getObject()
 post = post_module.newContent(
+  immediate_reindex=immediate_reindex_context_manager,
   portal_type='HTML Post',
   **post_edit_kw
 )

bt5/erp5_post/SkinTemplateItem/portal_skins/erp5_post/PostModule_createHTMLPost.xml

@@ -50,7 +50,7 @@
         </item>
         <item>
             <key> <string>_params</string> </key>
-            <value> <string>follow_up, data, predecessor=None</string> </value>
+            <value> <string>follow_up, data, predecessor=None, immediate_reindex_context_manager=None</string> </value>
         </item>
         <item>
             <key> <string>_proxy_roles</string> </key>

bt5/erp5_post/SkinTemplateItem/portal_skins/erp5_post/PostModule_createHTMLPostFromText.py

@@ -2,4 +2,5 @@ return context.PostModule_createHTMLPost(
   follow_up=follow_up,
   predecessor=predecessor,
   data="<p>" + data.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace("  ", " &nbsp;").replace("\n", "<br/>") + "</p>",
+  immediate_reindex_context_manager=immediate_reindex_context_manager,
 )

bt5/erp5_post/SkinTemplateItem/portal_skins/erp5_post/PostModule_createHTMLPostFromText.xml

@@ -50,7 +50,7 @@
         </item>
         <item>
             <key> <string>_params</string> </key>
-            <value> <string>follow_up, data, predecessor=None</string> </value>
+            <value> <string>follow_up, data, predecessor=None, immediate_reindex_context_manager=None</string> </value>
         </item>
         <item>
             <key> <string>_proxy_roles</string> </key>

bt5/erp5_trade/SkinTemplateItem/portal_skins/erp5_trade/SalePackingList_fastInput.py

@@ -2,6 +2,7 @@
 # key of dict : id of resource
 # item of dict : tuples (resource_value, variation_category_list, quantity)
 from ZTUtils import make_query
+from Products.ERP5Type.ImmediateReindexContextManager import ImmediateReindexContextManager
 
 delivery = context
 next_container_number = next_container_int_index
@@ -28,75 +29,77 @@ for listitem in listbox :
                                   (movement.getVariationCategoryList(),
                                   container_quantity)]
 
-# we build 'container_count' containers 
-for container_number in range(container_count) :
-
-  new_container_id = 'c'+str(next_container_number)
-  # we use container_type to know which are the resource (and variation) 
-  # of the container
-  container = delivery.newContent(
-                          portal_type="Container",
-                          title=new_container_id,
-                          int_index=next_container_number,
-                          serial_number="%06d%04d" % (int(delivery.getId()),
-                                                      next_container_number),
-                          resource = container_type,
-                          gross_weight = gross_weight,
-  )
-
-  next_container_number += 1
-
-  # now build container_lines
-  for resource_url in desired_lines.keys():
-
-    # compute variation_base_category_list and variation_category_list for this line
-    line_variation_base_category_dict = {}
-    line_variation_category_list = []
-
-    for variation_category_list, quantity in desired_lines[resource_url]:
-
-      for variation_item in variation_category_list:
-        if not variation_item in line_variation_category_list :
-          line_variation_category_list.append(variation_item)
-          variation_base_category_items = variation_item.split('/')
-          if len(variation_base_category_items) > 0 :
-            line_variation_base_category_dict[variation_base_category_items[0]] = 1
-
-      line_variation_base_category_list = line_variation_base_category_dict.keys()
-
-    # construct new content (container_line)
-    resource_url = resource_url
-    new_container_line_id = str(container.generateNewId())
-    container_line = container.newContent(
-                portal_type="Container Line",
-                title=new_container_line_id,
-                resource=resource_url,
-                quantity_unit=quantity_unit_dict[resource_url],
-                variation_category_list=line_variation_category_list
+with ImmediateReindexContextManager() as immediate_reindex_context_manager:
+  # we build 'container_count' containers
+  for container_number in range(container_count):
+
+    new_container_id = 'c'+str(next_container_number)
+    # we use container_type to know which are the resource (and variation)
+    # of the container
+    container = delivery.newContent(
+      # Container must be immediately reindexed,
+      # in order to see good packed quantity in fast input form
+      immediate_reindex=immediate_reindex_context_manager,
+      portal_type="Container",
+      title=new_container_id,
+      int_index=next_container_number,
+      serial_number="%06d%04d" % (int(delivery.getId()),
+                                  next_container_number),
+      resource = container_type,
+      gross_weight = gross_weight,
     )
 
-    for cell_key, quantity in desired_lines[resource_url]:
-      if variation_category_list == []:
-        container_line.edit(quantity=quantity)
-      else:
-        # create a new cell
-        base_id = 'movement'
-        if not container_line.hasCell(base_id=base_id, *cell_key):
-          cell = container_line.newCell(
-                     base_id=base_id,
-                     portal_type="Container Cell",
-                     *cell_key
-          )
-          cell.setCategoryList(cell_key)
-          cell.setMappedValuePropertyList(['quantity'])
-          cell.setMembershipCriterionCategoryList(cell_key)
-          cell.setMembershipCriterionBaseCategoryList(
-                                          line_variation_base_category_list)
-          cell.edit(quantity=quantity)
-
-# Container must be immediately reindexed, 
-# in order to see good packed quantity in fast input form
-container.recursiveImmediateReindexObject()
+    next_container_number += 1
+
+    # now build container_lines
+    for resource_url in desired_lines.keys():
+
+      # compute variation_base_category_list and variation_category_list for this line
+      line_variation_base_category_dict = {}
+      line_variation_category_list = []
+
+      for variation_category_list, quantity in desired_lines[resource_url]:
+
+        for variation_item in variation_category_list:
+          if not variation_item in line_variation_category_list:
+            line_variation_category_list.append(variation_item)
+            variation_base_category_items = variation_item.split('/')
+            if len(variation_base_category_items) > 0:
+              line_variation_base_category_dict[variation_base_category_items[0]] = 1
+
+        line_variation_base_category_list = line_variation_base_category_dict.keys()
+
+      # construct new content (container_line)
+      resource_url = resource_url
+      new_container_line_id = str(container.generateNewId())
+      container_line = container.newContent(
+        immediate_reindex=immediate_reindex_context_manager,
+        portal_type="Container Line",
+        title=new_container_line_id,
+        resource=resource_url,
+        quantity_unit=quantity_unit_dict[resource_url],
+        variation_category_list=line_variation_category_list
+      )
+
+      for cell_key, quantity in desired_lines[resource_url]:
+        if variation_category_list == []:
+          container_line.edit(quantity=quantity)
+        else:
+          # create a new cell
+          base_id = 'movement'
+          if not container_line.hasCell(base_id=base_id, *cell_key):
+            cell = container_line.newCell(
+              immediate_reindex=immediate_reindex_context_manager,
+              base_id=base_id,
+              portal_type="Container Cell",
+              *cell_key
+            )
+            cell.setCategoryList(cell_key)
+            cell.setMappedValuePropertyList(['quantity'])
+            cell.setMembershipCriterionCategoryList(cell_key)
+            cell.setMembershipCriterionBaseCategoryList(
+                                            line_variation_base_category_list)
+            cell.edit(quantity=quantity)
 
 url_params = make_query(selection_name=selection_name, 
                         dialog_category=dialog_category, 

product/ERP5Type/tests/testERP5Type.py

@@ -3262,14 +3262,8 @@ class TestAccessControl(ERP5TypeTestCase):
     method.setFiltered(1)
     method.setExpression(self.expression)
 
-    createZODBPythonScript(self.getSkinsTool().custom,
-                           'Base_immediateReindexObject',
-                           '',
-                           'context.immediateReindexObject()'
-                          ).manage_proxy(('Manager',))
-
   def test(self):
-    self.portal.person_module.newContent().Base_immediateReindexObject()
+    self.portal.person_module.newContent(immediate_reindex=True)
 
 
 def test_suite():