From 998daccdc4827b253a6b6ff250b9b56c24e4e221 Mon Sep 17 00:00:00 2001 From: Sebastien Robin <seb@nexedi.com> Date: Sun, 21 Oct 2007 23:15:16 +0000 Subject: [PATCH] =?UTF-8?q?-=20Added=20Folder=5FreindexTreeObject=20made?= =?UTF-8?q?=20by=20Aur=C3=A9lien.=20I=20improved=20it=20so=20that=20it=20s?= =?UTF-8?q?tarts=20once=20and=20then=20call=20himself=20until=20all=20sub?= =?UTF-8?q?=20trees=20are=20parsed.=20-=20Increased=20from=20100=20to=2010?= =?UTF-8?q?00=20the=20number=20of=20objects=20in=20a=20bundle,=20this=20is?= =?UTF-8?q?=20not=20that=20much=20because=201000=20can=20be=20parsed=20in?= =?UTF-8?q?=20less=20than=2020=20seconds.=20-=20Make=20Folder=5FreindexObj?= =?UTF-8?q?ectList=20starts=20once=20and=20then=20call=20himself=20until?= =?UTF-8?q?=20it=20finishes=20a=20list=20of=20lists=20of=20ids.=20-=20make?= =?UTF-8?q?=20sure=20that=20ERP5Site=5FreindexAll=20still=20reindex=20obje?= =?UTF-8?q?cts=20in=20several=20modules=20by=20the=20same=20time.=20-=20Ad?= =?UTF-8?q?ded=20additionnal=5Fpriority=20in=20order=20to=20increase=20def?= =?UTF-8?q?ault=20priority=20values,=20this=20is=20very=20nice=20while=20d?= =?UTF-8?q?oing=20hot=20reindexing.=20-=20Added=20simple=20tag=20dependenc?= =?UTF-8?q?y=20in=20order=20to=20make=20sure=20the=20amount=20of=20reindex?= =?UTF-8?q?ed=20objects=20stay=20limited.=20Also=20decrease=20a=20lot=20th?= =?UTF-8?q?e=20number=20of=20Folder=5FreindexObjectList=20activities=20by?= =?UTF-8?q?=20the=20same=20time,=20like=20this=20performance=20is=20increa?= =?UTF-8?q?sed.=20-=20Add=20a=20condition=20when=20adding=20after=5Ftag=20?= =?UTF-8?q?parameter=20while=20calling=20Folder=5FriendexObjectList,=20so?= =?UTF-8?q?=20we=20wait=20that=20the=20previous=20work=20was=20finished=20?= =?UTF-8?q?only=20after=20adding=201000=20*=20nb=5Fnodes=20objects=20reind?= =?UTF-8?q?ex=20activities.=20-=20explain=20the=20most=20important=20point?= =?UTF-8?q?s=20with=20comments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@17070 20353a03-c40f-0410-a6d1-a30d3c3de9de --- .../erp5_core/ERP5Site_reindexAll.xml | 48 ++-- .../erp5_core/Folder_reindexAll.xml | 106 +++++-- .../erp5_core/Folder_reindexObjectList.xml | 76 ++++- .../Folder_reindexTreeObjectList.xml | 261 ++++++++++++++++++ product/ERP5/bootstrap/erp5_core/bt/revision | 2 +- 5 files changed, 430 insertions(+), 63 deletions(-) create mode 100644 product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexTreeObjectList.xml diff --git a/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/ERP5Site_reindexAll.xml b/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/ERP5Site_reindexAll.xml index 56c59974e3..26d60cd08a 100644 --- a/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/ERP5Site_reindexAll.xml +++ b/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/ERP5Site_reindexAll.xml @@ -3,11 +3,8 @@ <record id="1" aka="AAAAAAAAAAE="> <pickle> <tuple> - <tuple> - <string>Products.PythonScripts.PythonScript</string> - <string>PythonScript</string> - </tuple> - <none/> + <global name="PythonScript" module="Products.PythonScripts.PythonScript"/> + <tuple/> </tuple> </pickle> <pickle> @@ -77,11 +74,14 @@ if clear_catalog:\n # Reindex person module\n print "#### Indexing person_module, stage 1 ####"\n person_module=getattr(portal, \'person_module\', None)\n +higher_priority = 1 + additional_priority\n if person_module is not None :\n tag = \'person_stage_1\'\n - person_module.activate(tag=tag).Folder_reindexAll(\n + person_module.activate(tag=tag,\n + priority=higher_priority).Folder_reindexAll(\n folder_tag=tag,\n object_tag=tag,\n + object_priority=higher_priority,\n sql_catalog_id=sql_catalog_id,\n passive_commit=passive_commit)\n \n @@ -97,11 +97,13 @@ object_after_tag = folder_after_tag\n \n context.portal_categories.activate(\n tag=folder_tag,\n + priority=higher_priority,\n after_tag=folder_after_tag).Folder_reindexAll(\n folder_tag=folder_tag,\n folder_after_tag=folder_after_tag,\n object_tag=object_tag,\n object_after_tag=object_after_tag,\n + object_priority=higher_priority,\n sql_catalog_id=sql_catalog_id,\n passive_commit=passive_commit)\n \n @@ -111,10 +113,11 @@ folder_tag = \'module\'\n folder_after_tag = (\'category\', \'person_stage_1\', \'group_person_stage_1\')\n object_tag = \'document\'\n object_after_tag = folder_after_tag\n -object_priority = 2\n +object_priority = 2 + additional_priority\n for folder in [context.portal_alarms, context.portal_preferences]:\n folder.activate(\n tag=folder_tag,\n + priority=object_priority,\n after_tag=folder_after_tag).Folder_reindexAll(\n folder_tag=folder_tag,\n folder_after_tag=folder_after_tag,\n @@ -131,14 +134,16 @@ folder_tag = \'module\'\n folder_after_tag = (\'category\', \'document\', \'person_stage_1\', \'group_person_stage_1\')\n object_tag = \'simulation\'\n object_after_tag = folder_after_tag\n -object_priority = 3\n +object_priority = 3 + additional_priority\n context.portal_simulation.activate(\n tag=folder_tag,\n + priority=higher_priority,\n after_tag=folder_after_tag).Folder_reindexAll(\n folder_tag=folder_tag,\n folder_after_tag=folder_after_tag,\n object_tag=object_tag,\n object_after_tag=object_after_tag,\n + object_priority=higher_priority,\n sql_catalog_id=sql_catalog_id,\n passive_commit=passive_commit)\n \n @@ -159,12 +164,13 @@ folder_tag = \'module\'\n folder_after_tag = (\'category\', \'person_stage_1\', \'group_person_stage_1\')\n object_tag = \'document\'\n object_after_tag = folder_after_tag\n -object_priority = 2\n +object_priority = 2 + additional_priority\n for folder in [context.portal_templates, context.portal_trash,\n context.portal_domains, context.portal_rules,\n context.portal_deliveries, context.portal_orders]:\n folder.activate(\n tag=folder_tag,\n + priority=object_priority,\n after_tag=folder_after_tag).Folder_reindexAll(\n folder_tag=folder_tag,\n folder_after_tag=folder_after_tag,\n @@ -181,18 +187,7 @@ for folder in portal.objectValues(("ERP5 Folder",)):\n \n folder.activate(\n tag=folder_tag,\n - after_tag=folder_after_tag).Folder_reindexAll(\n - folder_tag=folder_tag,\n - folder_after_tag=folder_after_tag,\n - object_tag=object_tag,\n - object_after_tag=object_after_tag,\n - object_priority=object_priority,\n - sql_catalog_id=sql_catalog_id,\n - passive_commit=passive_commit)\n -# reindex portal_caches plugins\n -print "#### Indexing portal_caches tool ####"\n -context.portal_caches.activate(\n - tag=folder_tag,\n + priority=object_priority,\n after_tag=folder_after_tag).Folder_reindexAll(\n folder_tag=folder_tag,\n folder_after_tag=folder_after_tag,\n @@ -210,6 +205,7 @@ for folder in portal.objectValues(("ERP5 Folder",)):\n print "#### Indexing contents inside folder %s ####" % folder.id\n folder.activate(\n tag=folder_tag,\n + priority=object_priority,\n after_tag=folder_after_tag).Folder_reindexAll(\n folder_tag=folder_tag,\n folder_after_tag=folder_after_tag,\n @@ -222,7 +218,7 @@ for folder in portal.objectValues(("ERP5 Folder",)):\n # start activty from simulation because the erp5site is not an active object\n context.portal_simulation.activate(\n after_tag=(\'inventory\', \'simulation\', \'person_stage_1\', \'group_person_stage_1\'),\n - priority=3\n + priority=3 + additional_priority\n ).InventoryModule_reindexMovementList(\n sql_catalog_id=sql_catalog_id,\n passive_commit=passive_commit,\n @@ -234,6 +230,7 @@ if final_activity_tag is not None:\n # Started on portal_simulation because activate does not work on portal object...\n # No idea if there is a better place.\n context.portal_simulation.activate(tag=final_activity_tag,\n + priority=3 + additional_priority,\n after_tag=(\'module\', \'inventory\', \'simulation\', \'person_stage_1\',\n \'group_person_stage_1\', \'last_inventory_activity\', \'document\')\n ).getId()\n @@ -261,7 +258,7 @@ return printed\n </item> <item> <key> <string>_params</string> </key> - <value> <string>sql_catalog_id=None, passive_commit=1, clear_catalog=0, final_activity_tag=None</string> </value> + <value> <string>sql_catalog_id=None, additional_priority=0, passive_commit=1, clear_catalog=0, final_activity_tag=None</string> </value> </item> <item> <key> <string>errors</string> </key> @@ -281,13 +278,14 @@ return printed\n <dictionary> <item> <key> <string>co_argcount</string> </key> - <value> <int>4</int> </value> + <value> <int>5</int> </value> </item> <item> <key> <string>co_varnames</string> </key> <value> <tuple> <string>sql_catalog_id</string> + <string>additional_priority</string> <string>passive_commit</string> <string>clear_catalog</string> <string>final_activity_tag</string> @@ -300,6 +298,7 @@ return printed\n <string>getattr</string> <string>None</string> <string>person_module</string> + <string>higher_priority</string> <string>tag</string> <string>folder_tag</string> <string>folder_after_tag</string> @@ -321,6 +320,7 @@ return printed\n <value> <tuple> <none/> + <int>0</int> <int>1</int> <int>0</int> <none/> diff --git a/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexAll.xml b/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexAll.xml index af6593335e..6a30bbc2bd 100644 --- a/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexAll.xml +++ b/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexAll.xml @@ -3,11 +3,8 @@ <record id="1" aka="AAAAAAAAAAE="> <pickle> <tuple> - <tuple> - <string>Products.PythonScripts.PythonScript</string> - <string>PythonScript</string> - </tuple> - <none/> + <global name="PythonScript" module="Products.PythonScripts.PythonScript"/> + <tuple/> </tuple> </pickle> <pickle> @@ -73,7 +70,7 @@ folder = context\n \n activate_kw = {\n - \'tag\': folder_tag,\n + \'tag\': object_tag,\n \'after_tag\': folder_after_tag,\n \'passive_commit\': passive_commit\n }\n @@ -84,25 +81,18 @@ for key, value in activate_kw.items():\n # Reindex folder immediately\n folder.reindexObject(sql_catalog_id=sql_catalog_id)\n \n -group_tag = None\n -group_after_tag = object_after_tag\n -if object_tag is not None:\n - group_tag = "group_%s" % object_tag\n - if object_after_tag is not None:\n - group_after_tag = object_after_tag + (object_tag,)\n - else:\n - group_after_tag = (object_tag, )\n -\n # Spawn activities for bundles of content objects.\n # Bundle size, in object count\n -BUNDLE_ITEM_COUNT=100\n +BUNDLE_ITEM_COUNT=1000\n \n -def Folder_reindexObjectList(id_list):\n +def Folder_reindexObjectList(id_list_list):\n """\n Create an activity calling Folder_reindexObjectList.\n """\n - folder.activate(activity=\'SQLQueue\', priority=object_priority, **activate_kw).Folder_reindexObjectList(\n - id_list=id_list,\n + folder.activate(activity=\'SQLQueue\', priority=object_priority, \n + round_robin_scheduling=1,**activate_kw).Folder_reindexObjectList(\n + id_list=None,\n + id_list_list=id_list_list,\n object_priority=object_priority,\n object_tag=object_tag,\n object_after_tag=object_after_tag,\n @@ -110,13 +100,64 @@ def Folder_reindexObjectList(id_list):\n passive_commit=passive_commit,\n )\n \n -id_list = [x for x in folder.objectIds()]\n -for bundle_index in xrange(len(id_list) / BUNDLE_ITEM_COUNT):\n - Folder_reindexObjectList(id_list=id_list[bundle_index * BUNDLE_ITEM_COUNT:((bundle_index + 1) * BUNDLE_ITEM_COUNT)])\n +try:\n + tree_id_list = folder.getTreeIdList()\n +except NotImplementedError:\n + # Build a list of list, like this we parse ids only one time,\n + # and then Folder_reinexObjectList will work with one list at\n + # a time and remove it from the list of list\n + # This id_list_list can be quite big and generate quite big\n + # activities, but the effect is limited, because if we have too\n + # much objects (like millions), we should use HBTree Folders, and\n + # then the work will be splitted\n + id_list = [x for x in folder.objectIds()]\n + id_list_list = []\n + for bundle_index in xrange(len(id_list) / BUNDLE_ITEM_COUNT):\n + Folder_reindexObjectList(id_list=id_list[bundle_index * BUNDLE_ITEM_COUNT:((bundle_index + 1) * BUNDLE_ITEM_COUNT)])\n + id_list_list.append(id_list[bundle_index * BUNDLE_ITEM_COUNT:((bundle_index + 1) * BUNDLE_ITEM_COUNT)])\n +\n + remaining_object_id_count = len(id_list) % BUNDLE_ITEM_COUNT\n + if remaining_object_id_count > 0:\n + id_list_list.append(id_list[-remaining_object_id_count:])\n + Folder_reindexObjectList(id_list_list)\n +else:\n + if folder_after_tag is None:\n + folder_after_tag = ()\n + i = 0\n + folder_id = folder.getId()\n +\n + tree_tag = "%s_tree" % folder_id\n + bundle_tag = "%s_bundle_reindex" % folder_id\n + bundle_object_tag = "%s_reindex" % folder_id\n +\n + # Say to Folder_reindexTreeObjectList to call himself again and\n + # again until all tree_id_list are parsed. Also, make sure that\n + # the work of the previous Folder_reindexTreeObjectList is completely\n + # done\n + tree_after_tag = folder_after_tag + (bundle_tag, bundle_object_tag)\n + folder.activate(activity=\'SQLQueue\', priority=object_priority, \n + after_tag=tree_after_tag, tag=tree_tag,\n + ).Folder_reindexTreeObjectList(\n + tree_id=None,\n + tree_id_list=tree_id_list,\n + folder_tag=bundle_tag,\n + folder_after_tag=bundle_object_tag,\n + object_priority=object_priority,\n + sql_catalog_id=sql_catalog_id,\n + passive_commit=passive_commit,\n + object_tag=bundle_object_tag,\n + tree_after_tag=tree_after_tag,\n + tree_tag=tree_tag,\n + )\n + \n + # Start an activity wich will wait the end of the module\n + folder_id_after_tag = folder_after_tag + (tree_tag, bundle_tag, bundle_object_tag)\n + id_activate_kw = {}\n + if object_tag is not None:\n + id_activate_kw[\'tag\'] = object_tag\n \n -remaining_object_id_count = len(id_list) % BUNDLE_ITEM_COUNT\n -if remaining_object_id_count > 0:\n - Folder_reindexObjectList(id_list=id_list[-remaining_object_id_count:])\n + folder.activate(activity=\'SQLDict\', priority=object_priority, \n + after_tag=folder_id_after_tag, **id_activate_kw).getId()\n ]]></string> </value> @@ -182,19 +223,30 @@ if remaining_object_id_count > 0:\n <string>key</string> <string>value</string> <string>None</string> - <string>group_tag</string> - <string>group_after_tag</string> <string>BUNDLE_ITEM_COUNT</string> <string>Folder_reindexObjectList</string> + <string>tree_id_list</string> + <string>NotImplementedError</string> <string>append</string> <string>$append0</string> <string>x</string> <string>id_list</string> + <string>id_list_list</string> <string>xrange</string> <string>len</string> <string>bundle_index</string> <string>_getitem_</string> <string>remaining_object_id_count</string> + <string>i</string> + <string>folder_id</string> + <string>tree_tag</string> + <string>bundle_tag</string> + <string>bundle_object_tag</string> + <string>tree_after_tag</string> + <string>folder_id_after_tag</string> + <string>id_activate_kw</string> + <string>_write_</string> + <string>_apply_</string> </tuple> </value> </item> diff --git a/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexObjectList.xml b/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexObjectList.xml index 0308590e20..95ae4e40a6 100644 --- a/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexObjectList.xml +++ b/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexObjectList.xml @@ -3,11 +3,8 @@ <record id="1" aka="AAAAAAAAAAE="> <pickle> <tuple> - <tuple> - <string>Products.PythonScripts.PythonScript</string> - <string>PythonScript</string> - </tuple> - <none/> + <global name="PythonScript" module="Products.PythonScripts.PythonScript"/> + <tuple/> </tuple> </pickle> <pickle> @@ -68,7 +65,17 @@ </item> <item> <key> <string>_body</string> </key> - <value> <string>folder = context\n + <value> <string encoding="cdata"><![CDATA[ + +folder = context\n +\n +# Keep compatibility with id_list\n +if id_list_list is not None:\n + if id_list is not None:\n + raise ValueError, "both id_list and id_list_list can be defined"\n + if len(id_list_list) == 0:\n + return None\n + id_list = id_list_list.pop()\n \n activate_kw = {\n \'tag\': object_tag,\n @@ -85,7 +92,39 @@ for id in id_list:\n if obj is not None:\n obj.recursiveReindexObject(activate_kw=activate_kw,\n sql_catalog_id=sql_catalog_id)\n -</string> </value> +\n +\n +if id_list_list is not None:\n + if len(id_list_list) > 0:\n + if count is None:\n + count = 1\n + new_activity_kw = {}\n + # We do not want to wait until there is enough activities\n + # So add BUNDLE_ITEM_COUNT * node_len activities before waiting\n + node_len = len(context.portal_activities.getProcessingNodeList())\n + if count % node_len == 0:\n + new_activity_kw[\'after_tag\'] = folder_after_tag\n + count = 0\n + count += 1\n +\n + # By calling again and again, we improve performance and we have\n + # less activities by the same time\n + folder.activate(activity=\'SQLQueue\',\n + priority=object_priority,\n + tag=folder_tag, **new_activity_kw).Folder_reindexObjectList(\n + None,\n + id_list_list=id_list_list,\n + object_priority=object_priority,\n + object_tag=object_tag,\n + sql_catalog_id=sql_catalog_id,\n + passive_commit=passive_commit,\n + folder_tag=folder_tag,\n + folder_after_tag=folder_after_tag,\n + count=count,\n + )\n + + +]]></string> </value> </item> <item> <key> <string>_code</string> </key> @@ -107,7 +146,7 @@ for id in id_list:\n </item> <item> <key> <string>_params</string> </key> - <value> <string>id_list, object_tag=None, object_after_tag=None, object_priority=1, sql_catalog_id=None, passive_commit=1</string> </value> + <value> <string>id_list, id_list_list=None, folder_tag=None, folder_after_tag=None, object_tag=None, object_after_tag=None, object_priority=1, sql_catalog_id=None, passive_commit=1, count=None</string> </value> </item> <item> <key> <string>errors</string> </key> @@ -127,29 +166,40 @@ for id in id_list:\n <dictionary> <item> <key> <string>co_argcount</string> </key> - <value> <int>6</int> </value> + <value> <int>10</int> </value> </item> <item> <key> <string>co_varnames</string> </key> <value> <tuple> <string>id_list</string> + <string>id_list_list</string> + <string>folder_tag</string> + <string>folder_after_tag</string> <string>object_tag</string> <string>object_after_tag</string> <string>object_priority</string> <string>sql_catalog_id</string> <string>passive_commit</string> + <string>count</string> <string>context</string> <string>folder</string> + <string>None</string> + <string>ValueError</string> + <string>len</string> + <string>_getattr_</string> <string>activate_kw</string> <string>_getiter_</string> - <string>_getattr_</string> <string>key</string> <string>value</string> - <string>None</string> <string>id</string> <string>getattr</string> <string>obj</string> + <string>new_activity_kw</string> + <string>node_len</string> + <string>_write_</string> + <string>_inplacevar_</string> + <string>_apply_</string> </tuple> </value> </item> @@ -162,11 +212,15 @@ for id in id_list:\n <key> <string>func_defaults</string> </key> <value> <tuple> + <none/> + <none/> + <none/> <none/> <none/> <int>1</int> <none/> <int>1</int> + <none/> </tuple> </value> </item> diff --git a/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexTreeObjectList.xml b/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexTreeObjectList.xml new file mode 100644 index 0000000000..ca1ac65a50 --- /dev/null +++ b/product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Folder_reindexTreeObjectList.xml @@ -0,0 +1,261 @@ +<?xml version="1.0"?> +<ZopeData> + <record id="1" aka="AAAAAAAAAAE="> + <pickle> + <tuple> + <global name="PythonScript" module="Products.PythonScripts.PythonScript"/> + <tuple/> + </tuple> + </pickle> + <pickle> + <dictionary> + <item> + <key> <string>Python_magic</string> </key> + <value> + <none/> + </value> + </item> + <item> + <key> <string>Script_magic</string> </key> + <value> <int>3</int> </value> + </item> + <item> + <key> <string>__ac_local_roles__</string> </key> + <value> + <none/> + </value> + </item> + <item> + <key> <string>_bind_names</string> </key> + <value> + <object> + <klass> + <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/> + </klass> + <tuple/> + <state> + <dictionary> + <item> + <key> <string>_asgns</string> </key> + <value> + <dictionary> + <item> + <key> <string>name_container</string> </key> + <value> <string>container</string> </value> + </item> + <item> + <key> <string>name_context</string> </key> + <value> <string>context</string> </value> + </item> + <item> + <key> <string>name_m_self</string> </key> + <value> <string>script</string> </value> + </item> + <item> + <key> <string>name_subpath</string> </key> + <value> <string>traverse_subpath</string> </value> + </item> + </dictionary> + </value> + </item> + </dictionary> + </state> + </object> + </value> + </item> + <item> + <key> <string>_body</string> </key> + <value> <string encoding="cdata"><![CDATA[ + +folder = context\n +\n +# Keep compatibility with tree_id\n +if tree_id_list is not None:\n + context.log(\'tree_id\', tree_id)\n + context.log(\'tree_id_list\', tree_id_list)\n + if tree_id is not None:\n + raise ValueError, "both tree and tree_id_list should not be defined"\n + tree_id = tree_id_list.pop()\n +\n +activate_kw = {\n + \'passive_commit\': passive_commit\n +}\n +for key, value in activate_kw.items():\n + if value is None:\n + activate_kw.pop(key)\n +\n +\n +# Spawn activities for bundles of content objects.\n +# Bundle size, in object count\n +BUNDLE_ITEM_COUNT = 1000\n +\n +folder_id = folder.getId()\n +def Folder_reindexObjectList(id_list_list):\n + """\n + Create an activity calling Folder_reindexObjectList.\n + """\n + folder.activate(activity=\'SQLQueue\', priority=object_priority, \n + after_tag=object_tag,\n + tag=folder_tag, **activate_kw).Folder_reindexObjectList(\n + None,\n + id_list_list=id_list_list,\n + object_priority=object_priority,\n + object_tag=object_tag,\n + sql_catalog_id=sql_catalog_id,\n + passive_commit=passive_commit,\n + folder_tag=folder_tag,\n + folder_after_tag=folder_after_tag,\n + )\n +\n +# HBTree folder\n +id_list = [x for x in folder.objectIds(base_id=tree_id)]\n +# Build a list of list, like this we parse ids only one time,\n +# and then Folder_reinexObjectList will work with one list at\n +# a time and remove it from the list of list\n +# This id_list_list can be quite big and generate quite big\n +# activities, but the effect is limited, because the work is\n +# splitted for each base_id of the HBTree.\n +id_list_list = []\n +for bundle_index in xrange(len(id_list) / BUNDLE_ITEM_COUNT):\n + id_list_list.append(id_list[bundle_index * BUNDLE_ITEM_COUNT:((bundle_index + 1) * BUNDLE_ITEM_COUNT)])\n +\n +remaining_object_id_count = len(id_list) % BUNDLE_ITEM_COUNT\n +if remaining_object_id_count > 0:\n + id_list_list.append(id_list[-remaining_object_id_count:])\n +Folder_reindexObjectList(id_list_list=id_list_list)\n +\n +if tree_id_list is not None:\n + if len(tree_id_list) > 0:\n + # Calling again and again the same script allow to decrease the\n + # number of activities by the same time and increase performance.\n + folder.activate(activity=\'SQLQueue\', priority=object_priority,\n + after_tag=tree_after_tag, \n + tag=tree_tag).Folder_reindexTreeObjectList(\n + tree_id=None,\n + tree_id_list=tree_id_list,\n + folder_tag=folder_tag,\n + folder_after_tag=folder_after_tag,\n + object_priority=object_priority,\n + sql_catalog_id=sql_catalog_id,\n + object_tag=object_tag,\n + passive_commit=passive_commit,\n + tree_after_tag=tree_after_tag,\n + tree_tag=tree_tag,\n + )\n + + +]]></string> </value> + </item> + <item> + <key> <string>_code</string> </key> + <value> + <none/> + </value> + </item> + <item> + <key> <string>_filepath</string> </key> + <value> + <none/> + </value> + </item> + <item> + <key> <string>_params</string> </key> + <value> <string>tree_id=None, tree_id_list=None, folder_tag=None, folder_after_tag=None, object_tag=None, object_after_tag=None, object_priority=1, sql_catalog_id=None, passive_commit=1, tree_tag=None, tree_after_tag=None</string> </value> + </item> + <item> + <key> <string>errors</string> </key> + <value> + <tuple/> + </value> + </item> + <item> + <key> <string>func_code</string> </key> + <value> + <object> + <klass> + <global name="FuncCode" module="Shared.DC.Scripts.Signature"/> + </klass> + <tuple/> + <state> + <dictionary> + <item> + <key> <string>co_argcount</string> </key> + <value> <int>11</int> </value> + </item> + <item> + <key> <string>co_varnames</string> </key> + <value> + <tuple> + <string>tree_id</string> + <string>tree_id_list</string> + <string>folder_tag</string> + <string>folder_after_tag</string> + <string>object_tag</string> + <string>object_after_tag</string> + <string>object_priority</string> + <string>sql_catalog_id</string> + <string>passive_commit</string> + <string>tree_tag</string> + <string>tree_after_tag</string> + <string>context</string> + <string>folder</string> + <string>None</string> + <string>_getattr_</string> + <string>ValueError</string> + <string>activate_kw</string> + <string>_getiter_</string> + <string>key</string> + <string>value</string> + <string>BUNDLE_ITEM_COUNT</string> + <string>folder_id</string> + <string>Folder_reindexObjectList</string> + <string>append</string> + <string>$append0</string> + <string>x</string> + <string>id_list</string> + <string>id_list_list</string> + <string>xrange</string> + <string>len</string> + <string>bundle_index</string> + <string>_getitem_</string> + <string>remaining_object_id_count</string> + </tuple> + </value> + </item> + </dictionary> + </state> + </object> + </value> + </item> + <item> + <key> <string>func_defaults</string> </key> + <value> + <tuple> + <none/> + <none/> + <none/> + <none/> + <none/> + <none/> + <int>1</int> + <none/> + <int>1</int> + <none/> + <none/> + </tuple> + </value> + </item> + <item> + <key> <string>id</string> </key> + <value> <string>Folder_reindexTreeObjectList</string> </value> + </item> + <item> + <key> <string>warnings</string> </key> + <value> + <tuple/> + </value> + </item> + </dictionary> + </pickle> + </record> +</ZopeData> diff --git a/product/ERP5/bootstrap/erp5_core/bt/revision b/product/ERP5/bootstrap/erp5_core/bt/revision index 19afd4ebd9..db3a73389d 100644 --- a/product/ERP5/bootstrap/erp5_core/bt/revision +++ b/product/ERP5/bootstrap/erp5_core/bt/revision @@ -1 +1 @@ -527 \ No newline at end of file +536 \ No newline at end of file -- 2.30.9