erp5_catalog: Migration script to move catalog to new ERP5Catalog object.

What we expect during migration:
1. Copy objects as well as properties from SQLCatalog to ERP5Catalog
2. Migration of SQL Methods and Python Scripts as ERP5 objects
3. Update migration with filter_dict for methods

Also, update BusinessTemplate fucntions according to changes made after migration

product/ERP5/Document/BusinessTemplate.py

@@ -152,12 +152,12 @@ SEPARATELY_EXPORTED_PROPERTY_DICT = {
 
 def _getCatalog(acquisition_context):
   """
-    Return the id of the SQLCatalog which correspond to the current BT.
+    Return the id of the Catalog which correspond to the current BT.
   """
   catalog_method_id_list = acquisition_context.getTemplateCatalogMethodIdList()
   if len(catalog_method_id_list) == 0:
     try:
-      return acquisition_context.getPortalObject().portal_catalog.objectIds('SQLCatalog')[0]
+      return acquisition_context.getPortalObject().portal_catalog.objectIds()[0]
     except IndexError:
       return None
   catalog_method_id = catalog_method_id_list[0]
@@ -1461,7 +1461,7 @@ class ObjectTemplateItem(BaseTemplateItem):
               # an object which cannot (e.g. External Method).
               LOG('BusinessTemplate', WARNING,
                   'could not restore %r in %r' % (subobject_id, obj))
-        if obj.meta_type in ('Z SQL Method',):
+        if obj.meta_type in ('Z SQL Method', 'ERP5 SQL Method'):
           fixZSQLMethod(portal, obj)
         # portal transforms specific initialization
         elif obj.meta_type in ('Transform', 'TransformsChain'):
@@ -1953,7 +1953,7 @@ class SkinTemplateItem(ObjectTemplateItem):
       if not force and update_dict.get(relative_url)  == 'nothing':
         continue
       folder = self.unrestrictedResolveValue(p, relative_url)
-      for obj in folder.objectValues(spec=('Z SQL Method',)):
+      for obj in folder.objectValues(spec=('Z SQL Method', 'ERP5 SQL Method')):
         fixZSQLMethod(p, obj)
       if folder.aq_parent.meta_type == 'CMF Skins Tool':
         registerSkinFolder(skin_tool, folder)
@@ -2893,14 +2893,17 @@ class CatalogMethodTemplateItem(ObjectTemplateItem):
     """Extracts properties for a given method in the catalog.
     Returns a mapping of property name -> boolean """
     method_properties = PersistentMapping()
-    for prop in catalog._properties:
+    property_list = list(catalog._properties)
+    if catalog.meta_type == 'ERP5 Catalog':
+      property_list = list(catalog.propertyMap())
+    for prop in property_list:
       if prop.get('select_variable') == 'getCatalogMethodIds':
         if prop['type'] == 'selection' and \
-            getattr(catalog, prop['id']) == method_id:
-          method_properties[prop['id']] = 1
+            getattr(catalog, prop['base_id']) == method_id:
+          method_properties[prop['base_id']] = 1
         elif prop['type'] == 'multiple selection' and \
-            method_id in getattr(catalog, prop['id']):
-          method_properties[prop['id']] = 1
+            method_id in getattr(catalog, prop['base_id']):
+          method_properties[prop['base_id']] = 1
     return method_properties
 
   def build(self, context, **kw):
@@ -2919,7 +2922,10 @@ class CatalogMethodTemplateItem(ObjectTemplateItem):
       method_id = obj.id
       self._method_properties[method_id] = self._extractMethodProperties(
                                                           catalog, method_id)
-      filter = catalog.filter_dict.get(method_id, {})
+      filter_dict = catalog.filter_dict
+      if catalog.meta_type == 'ERP5 Catalog':
+        filter_dict = catalog.getFilterDict()
+      filter = filter_dict.get(method_id, {})
       self._is_filtered_archive[method_id] = filter.get('filtered', 0)
       for method in catalog_method_filter_list:
         property = method[8:-8]
@@ -2978,6 +2984,26 @@ class CatalogMethodTemplateItem(ObjectTemplateItem):
     force = kw.get('force')
     values = []
 
+    # When the default catalog is of 'ERP5 Catalog' meta_type, its better to ..
+    # convert all the CatalogMethodTemplateItems in the current BT to the
+    # allowed types for ERP5 Catalog, i.e, to ERP5 SQLMethod and ERP5 Python Script
+    # and update the self._objects dict accordingly
+    if catalog.meta_type == 'ERP5 Catalog':
+      portal = self.getPortalObject()
+      # Will be modifying dict, so better to use .items()
+      # XXX: In python3 it should be .copy.items().
+      for path, obj in self._objects.items():
+        method = self.unrestrictedResolveValue(portal, path)
+        if method.meta_type == 'Z SQL Method':
+          self.convertSQLMethodToERP5SQLMethod(method)
+        if method.meta_type == 'Script (Python)':
+          self.convertPythonScriptToERP5PythonScript(method)
+        # Check what is better option to get the method object here,
+        # path resolving or getting object from erp5 catalog
+        method = self.unrestrictedResolveValue(portal, path)
+        new_obj  = method.aq_base
+        self._objects[path] = new_obj
+
     if force: # get all objects
       values = self._objects.values()
     else: # get only selected object
@@ -3005,23 +3031,43 @@ class CatalogMethodTemplateItem(ObjectTemplateItem):
               setattr(catalog, key, tuple(new_value))
 
       # Restore filter
-      if self._is_filtered_archive.get(method_id, 0):
-        expression = self._filter_expression_archive[method_id]
-        if expression and expression.strip():
-          # only compile non-empty expressions
-          expr_instance = Expression(expression)
-        else:
-          expr_instance = None
-        catalog.filter_dict[method_id] = PersistentMapping()
-        catalog.filter_dict[method_id]['filtered'] = 1
-        catalog.filter_dict[method_id]['expression'] = expression
-        catalog.filter_dict[method_id]['expression_instance'] = expr_instance
-        catalog.filter_dict[method_id]['expression_cache_key'] = \
-          self._filter_expression_cache_key_archive.get(method_id, ())
-        catalog.filter_dict[method_id]['type'] = \
-          self._filter_type_archive.get(method_id, ())
-      elif method_id in catalog.filter_dict.keys():
-        catalog.filter_dict[method_id]['filtered'] = 0
+      if catalog.meta_type == 'ERP5 Catalog':
+        method = catalog._getOb(method_id)
+        if self._is_filtered_archive.get(method_id, 0):
+          expression = self._filter_expression_archive[method_id]
+          if expression and expression.strip():
+            # only compile non-empty expressions
+            expr_instance = Expression(expression)
+          else:
+            expr_instance = None
+          method.setFiltered(1)
+          method.setExpression(expression)
+          method.setExpressionInstance(expr_instance)
+          method.setTypeList( \
+            self._filter_type_archive.get(method_id, ()))
+          method.setExpressionCacheKeyList( \
+            self._filter_expression_cache_key_archive.get(method_id, ()))
+        elif method_id in catalog.getFilterDict().keys():
+          method.setFiltered(0)
+
+      elif catalog.meta_type == 'SQLCatalog':
+        if self._is_filtered_archive.get(method_id, 0):
+          expression = self._filter_expression_archive[method_id]
+          if expression and expression.strip():
+            # only compile non-empty expressions
+            expr_instance = Expression(expression)
+          else:
+            expr_instance = None
+          catalog.filter_dict[method_id] = PersistentMapping()
+          catalog.filter_dict[method_id]['filtered'] = 1
+          catalog.filter_dict[method_id]['expression'] = expression
+          catalog.filter_dict[method_id]['expression_instance'] = expr_instance
+          catalog.filter_dict[method_id]['expression_cache_key'] = \
+            self._filter_expression_cache_key_archive.get(method_id, ())
+          catalog.filter_dict[method_id]['type'] = \
+            self._filter_type_archive.get(method_id, ())
+        elif method_id in catalog.filter_dict.keys():
+          catalog.filter_dict[method_id]['filtered'] = 0
 
       # backward compatibility
       if hasattr(self, '_is_catalog_list_method_archive'):
@@ -3077,15 +3123,20 @@ class CatalogMethodTemplateItem(ObjectTemplateItem):
         values.append(value)
     for obj in values:
       method_id = obj.id
+      property_list = list(catalog._properties)
+      if catalog.meta_type == 'ERP5 Catalog':
+        property_list = list(catalog.propertyMap())
       # remove method references in portal_catalog
-      for catalog_prop in catalog._properties:
+      for catalog_prop in property_list:
         if catalog_prop.get('select_variable') == 'getCatalogMethodIds'\
             and catalog_prop['type'] == 'multiple selection':
-          old_value = getattr(catalog, catalog_prop['id'], ())
+          old_value = getattr(catalog, catalog_prop['base_id'], ())
           if method_id in old_value:
             new_value = list(old_value)
             new_value.remove(method_id)
-            setattr(catalog, catalog_prop['id'], new_value)
+            # Better to set the attribute value as tuple as it would be consistent
+            # with both SQL Catalog and ERP5 Catalog.
+            setattr(catalog, catalog_prop['base_id'], tuple(new_value))
 
       if catalog.filter_dict.has_key(method_id):
         del catalog.filter_dict[method_id]

product/ERP5/ERP5Site.py

@@ -34,6 +34,7 @@ from Products.ERP5Type.Cache import caching_instance_method
 from Products.ERP5Type.Cache import CachingMethod, CacheCookieMixin
 from Products.ERP5Type.ERP5Type import ERP5TypeInformation
 from Products.ERP5Type.Log import log as unrestrictedLog
+from Products.ERP5Type.Utils import UpperCase
 from Products.CMFActivity.Errors import ActivityPendingError
 import ERP5Defaults
 from Products.ERP5Type.TransactionalVariable import getTransactionalVariable
@@ -1685,6 +1686,271 @@ class ERP5Site(FolderMixIn, CMFSite, CacheCookieMixin):
             for obj in tool.objectValues():
               obj.migrateToPortalTypeClass()
 
+  def convertPythonScriptToERP5PythonScript(self, python_script):
+    """
+    Converts object with meta_type 'Script (Python)' to ERP5 Python Script
+    and moves all the properties and id to the new object.
+    Also, deletes the initial object.
+
+    Params:
+    python_script:  Should be a python script inside ERP5 Catalog.
+                    It is forced for the aq_parent to be an ERP5 Catalog.
+    """
+    erp5_catalog = python_script.aq_parent
+    if not python_script: return None
+    # Filter only the objects which are of 'Script (Python)' meta_type
+    if python_script.meta_type == "Script (Python)":
+        script_id = python_script.id
+        title       = python_script.title
+        code        = python_script._body
+        parameter   = python_script._params
+        _v_change   = python_script._v_change
+        # Delete the python_script object from ther erp5_catalog
+        erp5_catalog.manage_delObjects(ids=script_id)
+        # Create new Python Script object inside erp5_catalog from the info data
+        # of the zope python script object.
+        erp5_python_script = erp5_catalog.newContent(portal_type='Python Script', \
+                          id = script_id )
+        erp5_python_script.title   = title
+        # Calling setter function for body would also compile the code body
+        erp5_python_script._setBody(code)
+        erp5_python_script._setParameterSignature(parameter)
+        erp5_python_script._v_change = _v_change
+
+        # Update filter properies only if SQL Catalog exists, in other case the
+        # updating of filter dict for the current object would be handled while
+        # installation itself.
+        try:
+          catalog = self.portal_catalog.getSQLCatalog()
+          if not catalog:
+            return
+          if catalog.meta_type != 'SQLCatalog':
+            return
+          filter_dict = catalog.filter_dict
+        except AttributeError:
+          return
+        else:
+          if script_id in filter_dict.keys():
+            filter_ = filter_dict[script_id]
+            erp5_python_script.setFiltered(filter_['filtered'])
+            erp5_python_script.setTypeList(filter_['type'])
+            erp5_python_script.setExpressionCacheKeyList(filter_['expression_cache_key'])
+            erp5_python_script.setExpression(filter_['expression'])
+            erp5_python_script.setExpressionInstance(filter_['expression_instance'])
+    else:
+      return
+
+  def convertSQLMethodToERP5SQLMethod(self, method):
+    """
+    Converts the SQL Method(SQL) objects with all its properties and id to an
+    ERP5 SQL Method object.
+
+    Params:
+      method: Should be an sql method inside ERP5 Catalog.
+              It is forced for the aq_parent to be an ERP5 Catalog.
+    """
+    try:
+      erp5_catalog = method.aq_parent
+    except Exception:
+      raise
+    if not method: return
+    if method.meta_type == "Z SQL Method":
+        sql_method = method
+        method_id = sql_method.id
+        title = sql_method.title
+        connection_id = sql_method.connection_id
+        connection_hook = sql_method.connection_hook
+        max_rows_ = sql_method.max_rows_
+        max_cache_ = sql_method.max_cache_
+        cache_time_ = sql_method.cache_time_
+        allow_simple_one_argument_traversal = sql_method.allow_simple_one_argument_traversal
+        class_file_ = sql_method.class_file_
+        class_name_ = sql_method.class_name_
+        arguments_src = sql_method.arguments_src
+        template = sql_method.src
+
+        # Delete old object and create a new object with same id
+        erp5_catalog.manage_delObjects(ids=method_id)
+        erp5_sql_method = erp5_catalog.newContent(portal_type='SQL Method',\
+                    id = method_id)
+
+        # Update properties and attributes of erp5_sql_catalog with the info
+        # data of same attributes of sql_method
+        erp5_sql_method.title = title
+        erp5_sql_method.setConnectionId(connection_id)
+        erp5_sql_method.setSrc(template)
+
+        # Update advanced attributes for the SQL Method
+        erp5_sql_method.setConnectionHook(connection_hook)
+        erp5_sql_method.setMaxRows(max_rows_)
+        erp5_sql_method.setMaxCache(max_cache_)
+        erp5_sql_method.setCacheTime(cache_time_)
+        erp5_sql_method.setAllowSimpleOneArgumentTraversal(\
+                                            allow_simple_one_argument_traversal)
+        erp5_sql_method.setClassFile(class_file_)
+        erp5_sql_method.setClassName(class_name_)
+
+        # Update argument at last cause this will update other attributes by
+        # calling manage_edit for SQLMethod in ZRDB.DA
+        erp5_sql_method._setArgument(arguments_src)
+
+        # Delete sql_method object from erp5_catalog and update the Id of
+        # erp5_catalog_object with the Id of sql_method
+        #erp5_catalog._delOb(id=method_id)
+        #erp5_sql_method.manage_renameObject(method_id)
+
+        # Update filter properies only if SQL Catalog exists, in other case the
+        # updating of filter dict for the current object would be handled while
+        # installation itself.
+        try:
+          catalog = self.portal_catalog.getSQLCatalog()
+          if not catalog:
+            return
+          # Only carry out updation of filter_dict when SQL Catalog still exists
+          if catalog.meta_type != 'SQLCatalog':
+            return
+          filter_dict = catalog.filter_dict
+        except AttributeError:
+          return
+        else:
+          if method_id in filter_dict.keys():
+            filter_ = filter_dict[method_id]
+            erp5_sql_method.setFiltered(filter_['filtered'])
+            erp5_sql_method.setTypeList(filter_['type'])
+            erp5_sql_method.setExpressionCacheKeyList(filter_['expression_cache_key'])
+            erp5_sql_method.setExpression(filter_['expression'])
+            erp5_sql_method.setExpressionInstance(filter_['expression_instance'])
+          else:
+            pass
+    else:
+      return
+
+  def migrateSQLCatalogToERP5Catalog(self):
+    """
+      Migrate SQLCatalog objects to ERP5Catalog objects which is a Folder object
+      holding indexes of objects inside ERP5 Catalog Tool.
+
+      Create new ERP5Catalog object and copy the Catalog data from the Default
+      SQLCatalog object to this new object.
+      This function should be run at the end of ERP5 Site installation so as to
+      maintain consistency for the objects.
+
+      Now, we do shift the default_sql_catalog_id to the new ERP5 Catalog installed
+    """
+    # We'll be migrating Persistent Objects
+    from Products.ERP5Type.dynamic.persistent_migration import PickleUpdater
+    from Products.ERP5Catalog.ERP5Catalog import ERP5Catalog
+    PickleUpdater(self)
+
+    # Getting ERP5 Catalog Tool
+    catalog_tool = self.portal_catalog
+    sql_catalog = catalog_tool.getSQLCatalog()
+
+    # Return in case this portal is trying to migrate even after the default ..
+    # catalog to be ERP5 catalog
+    if sql_catalog.meta_type == 'ERP5 Catalog':
+      message = 'You already have ERP5 Catalog as your default.'
+      return message
+
+    # Now, let's rename the sql_catalog, so as we can create a new erp5_catalog
+    # with its id. Also update default_sql_catalog_id so as not to break compatibility
+    catalog_tool.manage_renameObject(id='erp5_mysql_innodb', new_id='mysql_innodb')
+    catalog_tool.default_sql_catalog_id = 'mysql_innodb'
+
+    erp5_catalog_id = 'erp5_mysql_innodb'
+    # If there is no default SQL Catalog installed already, add a warning log
+    # in ERP5Site and break the migration
+    if not sql_catalog:
+      from Products.ERP5Type.Log import log
+      warning_message = 'No SQLCatlaog found out. No migration going to happen'
+      log(warning_message)
+      return
+
+    if not isinstance(sql_catalog, ERP5Catalog):
+      if erp5_catalog_id not in catalog_tool.objectIds():
+        # Create new ERP5Catalog object if it doesn't exist
+        catalog_tool.newContent(portal_type='Catalog',
+                                id = erp5_catalog_id,
+                                title='')
+        # Update default erp5 catalog id, the first time we run create ..
+        # ERP5Catalog object.
+        catalog_tool.default_erp5_catalog_id = erp5_catalog_id
+      erp5_catalog = catalog_tool._getOb(erp5_catalog_id)
+
+      # No need to migrate if the erp5_catalog is not an object of ERP5Catlog
+      if not isinstance(erp5_catalog, ERP5Catalog):
+        return
+
+      # Copy-paste objects from SQLCatalog to ERP5Catalog
+      for id in sql_catalog.objectIds():
+        ob = sql_catalog._getOb(id)
+        # Check if the object is copyable and that it doesn't already exist
+        # The copying attributes are from OFS.CopySupport
+        if hasattr(ob, '_canCopy') and not erp5_catalog.has_key(id):
+          # Get a copy of object for new conainer, i.e. erp5_catalog
+          ob = ob._getCopy(erp5_catalog)
+          ob._setId(id)
+          # Set the copied object in the erp5_catalog
+          erp5_catalog._setObject(id, ob)
+          ob = erp5_catalog._getOb(id)
+          ob._postCopy(erp5_catalog, op=0)
+
+      # Update properties list for the new catalog from the SQLCatalog
+      # Not an efficient way to carry this
+      for property_id in sql_catalog.propertyIds():
+        # This is needed because there have been problems with setting property
+        # for title, see this docstring for explanations:
+        # https://lab.nexedi.com/nexedi/erp5/blob/198df7021a17725e81ab930015c3c618e94263db/product/ERP5Type/Base.py#L3531
+        if property_id == 'title':
+          # Ignore case when property_id is title
+          continue
+        value = getattr(sql_catalog, property_id)
+        property_type = sql_catalog.getPropertyType(property_id)
+        erp5_catalog.setProperty(key=property_id, value=value, type=property_type)
+
+      # This step is required as we don't have consistency between the properties
+      # of new ERP5Catalog object and SQLCatalog.Catalog objects, i.e,
+      # for example: Trying to get property `sql_catalog_clear_reserved` for
+      # `erp5_catalog` would give us a tuple cause it is basically calling the
+      # getter function getSqlCatalogClearReserved, which when called for
+      # `sql_catalog.sql_catalog_clear_reserved` would give a string.
+      # This inconsistency arised due to the generation of getter functions where
+      # 'selection' type is taken as list_types which leads to creating getters
+      # and setters accordingly to create list and tuples instead of just a
+      # string. Well, imo(Ayush), this might not be good way to solve this
+      # problem. Other way would be to leave the proeprties as it is for ERP5Catalog
+      # objects and change the function calls in ERP5Catalog accordignly instead.
+      for p in sql_catalog.propertyMap():
+        if p['type'] in ['selection', 'multiple selection', 'lines']:
+          accessor_name = 'get' + UpperCase(p['id'])
+          value = getattr(sql_catalog, p['id'])
+          setattr(erp5_catalog, p['id'], value)
+
+      object_id_list = list(erp5_catalog.objectIds())
+      # Copy the list and make a set before iterating, cause we would be making
+      # change in it
+      catalog_object_list = set(object_id_list[:])
+
+      for method_id in catalog_object_list:
+        try:
+          obj = getattr(erp5_catalog, method_id, None)
+        except Exception as e:
+          continue
+        else:
+          # Call conversion function for both type of allowed objects for ..
+          # erp5_catalog.
+          if obj.meta_type == 'Z SQL Method':
+            self.convertSQLMethodToERP5SQLMethod(obj)
+            continue
+          if obj.meta_type == 'Script (Python)':
+            self.convertPythonScriptToERP5PythonScript(obj)
+            continue
+
+      # Update default_sql_catalog_id with the erp5_catalog id
+      catalog_tool.default_sql_catalog_id = erp5_catalog_id
+      # We don't need sql_catalog now, delete it
+      catalog_tool._delObject(id='mysql_innodb', suppress_events=True)
+
 Globals.InitializeClass(ERP5Site)
 
 def getBootstrapDirectory():
@@ -2234,6 +2500,7 @@ class ERP5Generator(PortalGenerator):
       self.setupWorkflow(p)
       self.setupERP5Core(p,**kw)
       self.setupERP5Promise(p,**kw)
+      p.migrateSQLCatalogToERP5Catalog()
 
     # Make sure the cache is initialized
     p.portal_caches.updateCache()