erp5_mysql_innodb_catalog: record deleted object paths.

so that we can identify recently deleted objects, that are important to resynchronise catalog and ZODB in case of a catastrophe.

bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testERP5Core.py

@@ -35,6 +35,7 @@ import urllib
 import lxml.html
 
 from AccessControl.SecurityManagement import newSecurityManager
+from DateTime import DateTime
 from Testing import ZopeTestCase
 
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
@@ -770,3 +771,20 @@ class TestERP5Core(ERP5TypeTestCase, ZopeTestCase.Functional):
           uid=old.getUid(),
         )
       ], [old_indexation_timestamp])
+
+  def test_deleted_catalog(self):
+    query = self.portal.erp5_sql_connection().query
+    query('DELETE FROM deleted_catalog')
+    person_list = [
+      self.portal.person_module.newContent(portal_type='Person')
+      for _ in range(10)
+    ]
+    self.tic()
+    self.portal.person_module.manage_delObjects(
+      ids=[e.getId() for e in person_list],
+    )
+    # check if deleted_catalog is immediately updated after deletion.
+    self.assertEqual(
+      len(self.portal.z_get_deleted_path_list(timestamp=DateTime() - 1)),
+      len(person_list),
+    )

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/z_get_deleted_path_list.sql

+SELECT uid, path FROM deleted_catalog WHERE <dtml-sqltest column="deletion_timestamp" expr="timestamp" op="ge" type="datetime">

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/z_get_deleted_path_list.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="SQL Method" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>allow_simple_one_argument_traversal</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>arguments_src</string> </key>
+            <value> <string>timestamp</string> </value>
+        </item>
+        <item>
+            <key> <string>cache_time_</string> </key>
+            <value> <int>0</int> </value>
+        </item>
+        <item>
+            <key> <string>class_file_</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>class_name_</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>connection_hook</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>connection_id</string> </key>
+            <value> <string>erp5_sql_connection</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>z_get_deleted_path_list</string> </value>
+        </item>
+        <item>
+            <key> <string>max_cache_</string> </key>
+            <value> <int>0</int> </value>
+        </item>
+        <item>
+            <key> <string>max_rows_</string> </key>
+            <value> <int>0</int> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>SQL Method</string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string></string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_drop_deleted_catalog.catalog_keys.xml

+<catalog_method>
+ <item key="sql_clear_catalog" type="int">
+  <value>1</value>
+ </item>
+</catalog_method>

product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_drop_deleted_catalog.sql

+DROP TABLE IF EXISTS deleted_catalog
\ No newline at end of file

product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_drop_deleted_catalog.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="SQL Method" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>allow_simple_one_argument_traversal</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>arguments_src</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>cache_time_</string> </key>
+            <value> <int>0</int> </value>
+        </item>
+        <item>
+            <key> <string>class_file_</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>class_name_</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>connection_hook</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>connection_id</string> </key>
+            <value> <string>erp5_sql_connection</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>z0_drop_deleted_catalog</string> </value>
+        </item>
+        <item>
+            <key> <string>max_cache_</string> </key>
+            <value> <int>100</int> </value>
+        </item>
+        <item>
+            <key> <string>max_rows_</string> </key>
+            <value> <int>1000</int> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>SQL Method</string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string></string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_create_deleted_catalog.catalog_keys.xml

+<catalog_method>
+ <item key="sql_clear_catalog" type="int">
+  <value>1</value>
+ </item>
+</catalog_method>

product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_create_deleted_catalog.sql

+<dtml-comment>
+* deletion_timestanmp is needed both to know when a deletion happened (when restoring consistency) and to forget old-enough entries.
+* path is needed to locate the document once the ZODB has been truncated before the deletion transaction.
+* uid is needed to detect if the document we do retrieve is the one which was deleted, or another one.
+* an index with deletion_timestanmp as first column is needed for good query performance.
+* a primary key is good for good deletion performance.
+* there is no natural primary key in this table, because a given path may have multiple uids / a given uid may have multiple paths over time.
+</dtml-comment>
+CREATE TABLE `deleted_catalog` (
+  `uid` BIGINT UNSIGNED NOT NULL,
+  `path` varchar(255) NOT NULL,
+  `deletion_timestamp` TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (`deletion_timestamp`, `path`, `uid`)
+) ENGINE=InnoDB;

product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_create_deleted_catalog.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="SQL Method" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_col</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>allow_simple_one_argument_traversal</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>arguments_src</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>cache_time_</string> </key>
+            <value> <int>0</int> </value>
+        </item>
+        <item>
+            <key> <string>class_file_</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>class_name_</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>connection_hook</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>connection_id</string> </key>
+            <value> <string>erp5_sql_connection</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>z_create_deleted_catalog</string> </value>
+        </item>
+        <item>
+            <key> <string>max_cache_</string> </key>
+            <value> <int>100</int> </value>
+        </item>
+        <item>
+            <key> <string>max_rows_</string> </key>
+            <value> <int>1000</int> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>SQL Method</string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string></string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_delete_uid.sql

@@ -3,6 +3,16 @@ REPLACE INTO
 VALUES
   (<dtml-sqlvar uid type="int">, 'deleted','',NULL,'',NULL)
 <dtml-var sql_delimiter>
+<dtml-if expr="'deleted_catalog' in portal_catalog.getSQLCatalog().getTableIds()">
+<dtml-comment>
+  Here we use REPLACE instead of INSERT to avoid duplicate entries.
+</dtml-comment>
+REPLACE INTO
+  deleted_catalog (uid, path)
+VALUES
+  (<dtml-sqlvar uid type="int">, <dtml-sqlvar path type="string">)
+<dtml-var sql_delimiter>
+</dtml-if>
 <dtml-comment>
   Note on "UPDATE stock" query: this query preserve transactionality of movement
   deletion: all movements deleted in a single transaction will be either

product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_delete_uid.xml

@@ -14,7 +14,8 @@
         </item>
         <item>
             <key> <string>arguments_src</string> </key>
-            <value> <string>uid</string> </value>
+            <value> <string>uid\n
+path</string> </value>
         </item>
         <item>
             <key> <string>cache_time_</string> </key>

product/ERP5Type/CopySupport.py

@@ -354,7 +354,7 @@ class CopyContainer:
               if s is None: ob._p_deactivate()
 
   security.declareProtected(Permissions.ModifyPortalContent, 'unindexObject')
-  def unindexObject(self, path=None):
+  def unindexObject(self):
       """
           Unindex the object from the portal catalog.
       """
@@ -372,7 +372,7 @@ class CopyContainer:
           # Set the path as deleted, sql wich generate no locks
           # Set also many columns in order to make sure lines
           # marked as deleted will not be selected
-          catalog.beforeUnindexObject(None,path=path,uid=uid)
+          catalog.beforeUnindexObject(None, path=self.getPath(), uid=uid)
           # Then start activity in order to remove lines in catalog,
           # sql wich generate locks
           # - serialization_tag is used in order to prevent unindexation to

product/ZSQLCatalog/SQLCatalog.py

@@ -1488,7 +1488,10 @@ class Catalog(Folder,
       LOG('ZSQLCatalog.beforeUncatalogObject', INFO, 'The sql_catalog_delete_uid method is not defined')
       return self.uncatalogObject(path=path,uid=uid)
     method = self._getOb(method_name)
-    method(uid = uid)
+    try:
+      method(uid=uid, path=path)
+    except ValueError: # BBB
+      method(uid=uid)
 
   def getSqlUncatalogObjectList(self):
     return self.sql_uncatalog_object