Skip to content

erp5
erp5
Commit 4e6c6d8b authored by Nicolas Dumazet's avatar Nicolas Dumazet
Browse files
Options

group transformation indexation to avoid using many (too small) transactions 


git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@33897 20353a03-c40f-0410-a6d1-a30d3c3de9de
parent 3552e3c8
Hide whitespace changes
Inline Side-by-side
Showing with 111 additions and 57 deletions
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_catalogTransformation.xml
View file @ 4e6c6d8b
......@@ -53,41 +53,51 @@
</item>
<item>
<key> <string>_body</string> </key>
<value> <string encoding="cdata"><![CDATA[
# This script indexes the preferred Transformation to produce\n
<value> <string># This script indexes the preferred Transformation to produce\n
# a variation of a product.\n
# Only the transformation is passed as a parameter, the resource being\n
# transformation.getResourceValue()\n
# transformation_item_list is a list of:\n
# transformation, variation_list_list\n
# transformation is a transformation to index, while variation_list_list\n
# is a list of variation categories that are relevant for the produced resource\n
\n
\n
from Products.ERP5Type.Document import newTempMovement\n
\n
transformation = context.restrictedTraverse(transformation_relative_url)\n
resource = transformation.getResourceValue()\n
# List of dictionaries:\n
# { id:resource_id,\n
# variation_text: resource_variation_text,\n
# row_dict_list: list of rows to insert; each row is represented as a dict.}\n
row_dict_dict_list = []\n
\n
for transformation_relative_url, variation_list_list in transformation_item_list:\n
transformation = context.restrictedTraverse(transformation_relative_url)\n
resource = transformation.getResourceValue()\n
\n
row_dict_list = []\n
movement = newTempMovement(resource, \'temp\',\n
variation_category_list=variation_list,\n
resource=resource.getRelativeUrl(),\n
quantity=1.0)\n
if resource is None:\n
continue\n
for variation_list in variation_list_list:\n
movement = newTempMovement(resource, \'temp\',\n
variation_category_list=variation_list,\n
resource=resource.getRelativeUrl(),\n
quantity=1.0)\n
base_row = dict(uid=resource.getUid(), variation_text=movement.getVariationText())\n
\n
base_row = dict(uid=resource.getUid(), variation_text=movement.getVariationText())\n
row_dict_list = []\n
for amount in transformation.getAggregatedAmountList(movement):\n
transformed_resource_uid = amount.getResourceUid()\n
quantity = amount.getQuantity()\n
if transformed_resource_uid is not None and quantity is not None:\n
row_dict = base_row.copy()\n
row_dict.update(transformed_uid=transformed_resource_uid,\n
transformed_variation_text=amount.getVariationText(),\n
quantity=quantity)\n
row_dict_list.append(row_dict)\n
\n
for amount in transformation.getAggregatedAmountList(movement):\n
transformed_resource_uid = amount.getResourceUid()\n
quantity = amount.getQuantity()\n
if transformed_resource_uid is not None and quantity is not None:\n
row_dict = base_row.copy()\n
row_dict.update(transformed_uid=transformed_resource_uid,\n
transformed_variation_text=amount.getVariationText(),\n
quantity=quantity)\n
row_dict_list.append(row_dict)\n
base_row[\'row_dict_list\'] = row_dict_list\n
row_dict_dict_list.append(base_row)\n
\n
if len(row_dict_list) > 0:\n
context.z_catalog_transformation(row_dict_list=row_dict_list)\n
]]></string> </value>
context.z_catalog_transformation_list(row_dict_dict_list=row_dict_dict_list)\n
</string> </value>
</item>
<item>
<key> <string>_code</string> </key>
......@@ -97,7 +107,7 @@ if len(row_dict_list) > 0:\n
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>transformation_relative_url, variation_list</string> </value>
<value> <string>transformation_item_list</string> </value>
</item>
<item>
<key> <string>errors</string> </key>
......@@ -117,31 +127,34 @@ if len(row_dict_list) > 0:\n
<dictionary>
<item>
<key> <string>co_argcount</string> </key>
<value> <int>2</int> </value>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>co_varnames</string> </key>
<value>
<tuple>
<string>transformation_relative_url</string>
<string>variation_list</string>
<string>transformation_item_list</string>
<string>Products.ERP5Type.Document</string>
<string>newTempMovement</string>
<string>row_dict_dict_list</string>
<string>_getiter_</string>
<string>transformation_relative_url</string>
<string>variation_list_list</string>
<string>_getattr_</string>
<string>context</string>
<string>transformation</string>
<string>resource</string>
<string>row_dict_list</string>
<string>None</string>
<string>variation_list</string>
<string>movement</string>
<string>dict</string>
<string>base_row</string>
<string>_getiter_</string>
<string>row_dict_list</string>
<string>amount</string>
<string>transformed_resource_uid</string>
<string>quantity</string>
<string>None</string>
<string>row_dict</string>
<string>len</string>
<string>_write_</string>
</tuple>
</value>
</item>
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_catalogTransformationList.xml
View file @ 4e6c6d8b
......@@ -53,19 +53,36 @@
</item>
<item>
<key> <string>_body</string> </key>
<value> <string># Indexing all transformation lines for all possible variations of a Resource can be very costly.\n
<value> <string encoding="cdata"><![CDATA[
# Indexing all transformation lines for all possible variations of a Resource can be very costly.\n
# Avoid doing this in a single transaction, and split the operation.\n
#\n
# One activity is generated per (resource, variation_category_list) possibility\n
\n
batch_size = 100\n
current_batch = []\n
current_size = 0\n
\n
for i, transformation in enumerate(getDefaultConversionTransformationValue):\n
if transformation is None:\n
continue\n
transformation_relative_url = transformation.getRelativeUrl()\n
for variation_category_list in getTransformationVariationCategoryCartesianProduct[i]:\n
context.activate(activity=\'SQLQueue\').SQLCatalog_catalogTransformation(transformation_relative_url,\n
variation_category_list)\n
</string> </value>
variation_list_list = getTransformationVariationCategoryCartesianProduct[i]\n
size = len(transformation)*len(variation_list_list)\n
\n
if size + current_size < batch_size:\n
current_batch.append((transformation_relative_url, variation_list_list))\n
current_size += size\n
else:\n
context.activate(activity=\'SQLQueue\').SQLCatalog_catalogTransformation(current_batch)\n
current_batch = (transformation_relative_url, variation_list_list),\n
current_size = size\n
\n
\n
if current_batch:\n
context.activate(activity=\'SQLQueue\').SQLCatalog_catalogTransformation(current_batch)\n
]]></string> </value>
</item>
<item>
<key> <string>_code</string> </key>
......@@ -103,6 +120,9 @@ for i, transformation in enumerate(getDefaultConversionTransformationValue):\n
<tuple>
<string>getDefaultConversionTransformationValue</string>
<string>getTransformationVariationCategoryCartesianProduct</string>
<string>batch_size</string>
<string>current_batch</string>
<string>current_size</string>
<string>_getiter_</string>
<string>enumerate</string>
<string>i</string>
......@@ -111,7 +131,10 @@ for i, transformation in enumerate(getDefaultConversionTransformationValue):\n
<string>_getattr_</string>
<string>transformation_relative_url</string>
<string>_getitem_</string>
<string>variation_category_list</string>
<string>variation_list_list</string>
<string>len</string>
<string>size</string>
<string>_inplacevar_</string>
<string>context</string>
</tuple>
</value>
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_transformation.xml → product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_transformation_list.xml
View file @ 4e6c6d8b
......@@ -24,7 +24,7 @@
<value>
<dictionary>
<item>
<key> <string>row_dict_list</string> </key>
<key> <string>row_dict_dict_list</string> </key>
<value>
<dictionary/>
</value>
......@@ -36,7 +36,7 @@
<key> <string>_keys</string> </key>
<value>
<list>
<string>row_dict_list</string>
<string>row_dict_dict_list</string>
</list>
</value>
</item>
......@@ -53,7 +53,7 @@
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>row_dict_list</string> </value>
<value> <string>row_dict_dict_list</string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
......@@ -79,7 +79,7 @@
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z_catalog_transformation</string> </value>
<value> <string>z_catalog_transformation_list</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
......@@ -93,15 +93,22 @@
<key> <string>src</string> </key>
<value> <string encoding="cdata"><![CDATA[
<dtml-let row_list="[]">\n
<dtml-in row_dict_dict_list prefix="outer">\n
DELETE FROM `transformation` WHERE\n
<dtml-sqltest expr="row_dict_list[0][\'uid\']" column="uid" type="int">\n
<dtml-sqltest expr="outer_item[\'uid\']" column="uid" type="int">\n
AND\n
<dtml-sqltest expr="row_dict_list[0][\'variation_text\']" column="variation_text" type="string">;\n
<dtml-sqltest expr="outer_item[\'variation_text\']" column="variation_text" type="string">;\n
<dtml-var sql_delimiter>\n
<dtml-call "row_list.extend(outer_item[\'row_dict_list\'])">\n
</dtml-in>\n
\n
<dtml-if "len(row_list)>0">\n
<dtml-var sql_delimiter>\n
\n
INSERT INTO `transformation`\n
VALUES\n
<dtml-in row_dict_list prefix="loop">\n
<dtml-in row_list prefix="loop">\n
(\n
<dtml-sqlvar expr="loop_item[\'uid\']" type="int">,\n
<dtml-sqlvar expr="loop_item[\'variation_text\']" type="string">,\n
......@@ -110,7 +117,9 @@ VALUES\n
<dtml-sqlvar expr="loop_item[\'quantity\']" type="float">\n
)\n
<dtml-unless sequence-end>,</dtml-unless>\n
</dtml-in>
</dtml-in>\n
</dtml-if>\n
</dtml-let>
]]></string> </value>
</item>
......@@ -150,15 +159,22 @@ VALUES\n
<key> <string>raw</string> </key>
<value> <string encoding="cdata"><![CDATA[
<dtml-let row_list="[]">\n
<dtml-in row_dict_dict_list prefix="outer">\n
DELETE FROM `transformation` WHERE\n
<dtml-sqltest expr="row_dict_list[0][\'uid\']" column="uid" type="int">\n
<dtml-sqltest expr="outer_item[\'uid\']" column="uid" type="int">\n
AND\n
<dtml-sqltest expr="row_dict_list[0][\'variation_text\']" column="variation_text" type="string">;\n
<dtml-sqltest expr="outer_item[\'variation_text\']" column="variation_text" type="string">;\n
<dtml-var sql_delimiter>\n
<dtml-call "row_list.extend(outer_item[\'row_dict_list\'])">\n
</dtml-in>\n
\n
<dtml-if "len(row_list)>0">\n
<dtml-var sql_delimiter>\n
\n
INSERT INTO `transformation`\n
VALUES\n
<dtml-in row_dict_list prefix="loop">\n
<dtml-in row_list prefix="loop">\n
(\n
<dtml-sqlvar expr="loop_item[\'uid\']" type="int">,\n
<dtml-sqlvar expr="loop_item[\'variation_text\']" type="string">,\n
......@@ -167,7 +183,9 @@ VALUES\n
<dtml-sqlvar expr="loop_item[\'quantity\']" type="float">\n
)\n
<dtml-unless sequence-end>,</dtml-unless>\n
</dtml-in>
</dtml-in>\n
</dtml-if>\n
</dtml-let>
]]></string> </value>
</item>
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/bt/revision
View file @ 4e6c6d8b
183
\ No newline at end of file
184
\ No newline at end of file
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/bt/template_catalog_method_id_list
View file @ 4e6c6d8b
......@@ -48,7 +48,7 @@ erp5_mysql_innodb/z_catalog_predicate_category_list
erp5_mysql_innodb/z_catalog_predicate_list
erp5_mysql_innodb/z_catalog_roles_and_users_list
erp5_mysql_innodb/z_catalog_stock_list
erp5_mysql_innodb/z_catalog_transformation
erp5_mysql_innodb/z_catalog_transformation_list
erp5_mysql_innodb/z_catalog_translation_list
erp5_mysql_innodb/z_catalog_versioning_list
erp5_mysql_innodb/z_clear_reserved
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7