ZODB Components: Products Documents for a given bt5 can now be migrated from filesystem.

Until now, only bt5 Extension/Test/Document could be migrated from
filesystem.

From migration dialog, allow to select any Products.ERP5.Documents.* (only,
for now) to be migrated. By default, automatically select Products Documents
used by the current bt5 Portal Types (by looking at the mro() of its
erp5.portal_type classes).

XXX-BEFORE-MERGE: Add Unit Test.

product/ERP5/Document/BusinessTemplate.py

@@ -6293,15 +6293,67 @@ Business Template is a set of definitions, such as skins, portal types and categ
       setattr(self, 'template_portal_type_base_category', ())
       return
 
+    @staticmethod
+    def _getAllFilesystemModuleFromPortalTypeIdList(portal_type_id_list):
+      import erp5.portal_type
+      import inspect
+      import Products.ERP5Type
+
+      product_base_path = inspect.getfile(Products.ERP5Type).rsplit('/', 2)[0]
+      seen_cls_set = set()
+      for portal_type in portal_type_id_list:
+        portal_type_cls = getattr(erp5.portal_type, portal_type)
+        # Calling mro() would not load the class...
+        portal_type_cls.loadClass()
+
+        for cls in portal_type_cls.mro():
+          if (not cls.__module__.startswith('erp5.') and
+              cls not in seen_cls_set):
+            seen_cls_set.add(cls)
+            try:
+              cls_path = inspect.getfile(cls)
+            except TypeError:
+              pass
+            else:
+              if cls_path.startswith(product_base_path):
+                cls_name = cls.__name__
+                cls_module = cls.__module__
+                yield cls_name, cls_module, cls_path
+
     security.declareProtected(Permissions.ManagePortal,
                               'getMigratableSourceCodeFromFilesystemList')
-    def getMigratableSourceCodeFromFilesystemList(self, *args, **kwargs):
+    def getMigratableSourceCodeFromFilesystemList(self,
+                                                  current_bt_only=False,
+                                                  *args,
+                                                  **kwargs):
       """
       Return the list of Business Template {Extension, Document, Test} Documents
       and Products Documents which can be migrated to ZODB Components.
       """
+      import inspect
+
+      bt_migratable_uid_list = []
       migratable_component_list = []
-      component_tool = self.getPortalObject().portal_components
+      portal = self.getPortalObject()
+      component_tool = portal.portal_components
+
+      from base64 import b64encode
+      import cPickle
+      def __newTempComponent(portal_type, reference, source_reference, migrate=False):
+        uid = b64encode("%s|%s|%s" % (portal_type, reference, source_reference))
+        if migrate:
+          bt_migratable_uid_list.append(uid)
+
+        obj = component_tool.newContent(temp_object=1,
+                                        id="temp_" + uid,
+                                        uid=uid,
+                                        portal_type=portal_type,
+                                        reference=reference,
+                                        source_reference=source_reference)
+
+        migratable_component_list.append(obj)
+
+        return obj
 
       for portal_type, id_list in (
           ('Document Component', self.getTemplateDocumentIdList()),
@@ -6310,14 +6362,56 @@ Business Template is a set of definitions, such as skins, portal types and categ
         for id_ in id_list:
           existing_component = getattr(component_tool, id_, None)
           if existing_component is None:
-            obj = component_tool.newContent(id="tmp_source_code_migration_%s" % id_,
-                                            portal_type=portal_type,
-                                            reference=id_,
-                                            temp_object=1)
-
-            migratable_component_list.append(obj)
+            obj = __newTempComponent(portal_type=portal_type,
+                                     reference=id_,
+                                     source_reference="%s:%s" % (self.getTitle(), id_),
+                                     migrate=True)
+
+      # Inspect Portal Types classes mro() of this Business Template to find
+      # Products Documents to migrate by default
+      portal_type_module_set = set(
+        self._getAllFilesystemModuleFromPortalTypeIdList(
+          self.getTemplatePortalTypeIdList()))
+
+      # XXX: Only migrate Documents in ERP5 for the moment...
+      import Products.ERP5.Document
+      for name, obj in Products.ERP5.Document.__dict__.iteritems():
+        if not name.startswith('_') and inspect.ismodule(obj):
+          source_reference = obj.__name__
+
+          migrate = ((name, source_reference, inspect.getfile(obj))
+                     in portal_type_module_set)
+          if current_bt_only and not migrate:
+            continue
 
-      return sorted(migratable_component_list, key=lambda o: o.getReference())
+          obj = __newTempComponent(portal_type='Document Component',
+                                   reference=name,
+                                   source_reference=source_reference,
+                                   migrate=migrate)
+
+      if not current_bt_only:
+        import Products.ERP5.tests
+        from glob import iglob
+        for test_path in iglob("%s/test*.py" %
+                               inspect.getfile(Products.ERP5.tests).rsplit('/', 1)[0]):
+          reference = test_path.rsplit('/', 1)[1][:-3]
+          obj = __newTempComponent(portal_type='Test Component',
+                                   reference=reference,
+                                   source_reference="Products.ERP5.tests." + reference)
+
+      # Automatically select ZODB Components to be migrated in Migration Dialog
+      selection_name = kwargs.get('selection_name')
+      if (selection_name is not None and
+          # XXX: Do not set uids on {check,uncheck}All, better way?
+          self.REQUEST.get('listbox_uncheckAll') is None and
+          self.REQUEST.get('listbox_checkAll') is None):
+        portal.portal_selections.setSelectionCheckedUidsFor(selection_name,
+                                                            bt_migratable_uid_list)
+
+      return sorted(migratable_component_list,
+                    key=lambda o: (not o.getProperty('migrate', False),
+                                   o.getPortalType(),
+                                   o.getReference()))
 
     security.declareProtected(Permissions.ManagePortal,
                               'migrateSourceCodeFromFilesystem')
@@ -6333,16 +6427,46 @@ Business Template is a set of definitions, such as skins, portal types and categ
       component_tool = portal.portal_components
       failed_import_dict = {}
       list_selection_name = kw.get('list_selection_name')
+      migrated_product_module_set = set()
 
       template_document_id_set = set(self.getTemplateDocumentIdList())
       template_extension_id_set = set(self.getTemplateExtensionIdList())
       template_test_id_set = set(self.getTemplateTestIdList())
 
-      for temp_obj in self.getMigratableSourceCodeFromFilesystemList():
+      if list_selection_name is None:
+        temp_obj_list = self.getMigratableSourceCodeFromFilesystemList(
+          current_bt_only=True)
+      else:
+        from base64 import b64decode
+        import cPickle
+        temp_obj_list = []
+        for uid in portal.portal_selections.getSelectionCheckedUidsFor(
+            list_selection_name):
+          portal_type, reference, source_reference = b64decode(uid).split('|')
+          obj = component_tool.newContent(temp_object=1,
+                                          id="temp_" + uid,
+                                          uid=uid,
+                                          portal_type=portal_type,
+                                          reference=reference,
+                                          source_reference=source_reference)
+
+          temp_obj_list.append(obj)
+
+      if not temp_obj_list:
+        if list_selection_name is not None:
+          return self.Base_redirect(
+            'view',
+            keep_items={'portal_status_message': 'Nothing Selected.'})
+
+        return
+
+      for temp_obj in temp_obj_list:
+        source_reference = temp_obj.getSourceReference()
         try:
           obj = temp_obj.importFromFilesystem(component_tool,
                                               temp_obj.getReference(),
-                                              version)
+                                              version,
+                                              source_reference)
         except Exception, e:
           LOG("BusinessTemplate", WARNING,
               "Could not import component '%s' ('%s') from the filesystem" %
@@ -6361,7 +6485,11 @@ Business Template is a set of definitions, such as skins, portal types and categ
           else:
             id_set = template_document_id_set
 
-          id_set.discard(temp_obj.getReference())
+          if source_reference.startswith('Products'):
+            migrated_product_module_set.add(source_reference)
+          else:
+            id_set.discard(temp_obj.getReference())
+
           id_set.add(obj.getId())
 
       if failed_import_dict:
@@ -6382,6 +6510,27 @@ Business Template is a set of definitions, such as skins, portal types and categ
       self.setTemplateExtensionIdList(sorted(template_extension_id_set))
       self.setTemplateTestIdList(sorted(template_test_id_set))
 
+      # This will trigger a reset so that Portal Types mro() can be checked
+      # after migration for filesystem Products modules still being used
+      transaction.commit()
+
+      still_used_list_dict = {}
+      for _, cls_module, _ in self._getAllFilesystemModuleFromPortalTypeIdList(
+          portal.portal_types.objectIds()):
+        if cls_module in migrated_product_module_set:
+          package, module = cls_module.rsplit('.', 1)
+          still_used_list_dict.setdefault(package, []).append(module)
+
+      if still_used_list_dict:
+        module_still_used_message = ', '.join(
+            [ "%s.{%s}" % (package, ','.join(sorted(module_list)))
+              for package, module_list in still_used_list_dict.iteritems() ])
+
+        LOG('BusinessTemplate',
+            WARNING,
+            "The following Documents are still being imported so code need to "
+            "be updated: " + module_still_used_message)
+
       if list_selection_name is not None:
         message = (
           "All components were successfully imported from filesystem to ZODB. "

product/ERP5/bootstrap/erp5_core/ActionTemplateItem/portal_types/Business%20Template/migrate_component_from_filesystem.xml

@@ -92,7 +92,7 @@
       <dictionary>
         <item>
             <key> <string>text</string> </key>
-            <value> <string>python: portal.Base_checkPermission(\'portal_components\', \'Add portal content\') and context.getInstallationState() != \'installed\' and [id_ for id_ in (context.getTemplateExtensionIdList() + context.getTemplateDocumentIdList() + context.getTemplateTestIdList()) if not id_.startswith(\'extension.\') and not id_.startswith(\'document.\') and not id_.startswith(\'test.\') ]</string> </value>
+            <value> <string>python: portal.Base_checkPermission(\'portal_components\', \'Add portal content\') and context.getInstallationState() != \'installed\'</string> </value>
         </item>
       </dictionary>
     </pickle>

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/BusinessTemplate_viewMigrateSourceCodeFromFilesystemDialog/listbox.xml

@@ -403,6 +403,10 @@
                           <string>reference</string>
                           <string>Name</string>
                         </tuple>
+                        <tuple>
+                          <string>source_reference</string>
+                          <string>Module</string>
+                        </tuple>
                         <tuple>
                           <string>portal_type</string>
                           <string>Destination Portal Type</string>
@@ -549,7 +553,7 @@
                 </item>
                 <item>
                     <key> <string>select</string> </key>
-                    <value> <int>0</int> </value>
+                    <value> <int>1</int> </value>
                 </item>
                 <item>
                     <key> <string>selection_name</string> </key>

product/ERP5Type/interfaces/component.py

@@ -68,7 +68,7 @@ class IComponent(Interface):
     Return the ID prefix for Component objects
     """
 
-  def importFromFilesystem(cls, context, reference, version):
+  def importFromFilesystem(cls, context, reference, version, source_reference=None):
     """
     Import a Component from the filesystem into ZODB after checking that the
     source code is valid

product/ERP5Type/mixin/component.py

@@ -345,14 +345,21 @@ class ComponentMixin(PropertyRecordableMixin, Base):
   security.declareProtected(Permissions.ModifyPortalContent,
                             'importFromFilesystem')
   @classmethod
-  def importFromFilesystem(cls, context, reference, version):
+  def importFromFilesystem(cls, context, reference, version, source_reference=None):
     """
     Import a Component from the filesystem into ZODB and validate it so it can
     be loaded straightaway provided validate() does not raise any error of
     course
     """
     import os.path
-    path = os.path.join(cls._getFilesystemPath(), reference + '.py')
+    if source_reference is None or not source_reference.startswith('Products'):
+      path = os.path.join(cls._getFilesystemPath(), reference + '.py')
+    else:
+      import inspect
+      module_obj = __import__(source_reference, globals(), {},
+                              level=0, fromlist=[source_reference])
+      path = inspect.getsourcefile(module_obj)
+
     with open(path) as f:
       source_code = f.read()
 
@@ -363,6 +370,7 @@ class ComponentMixin(PropertyRecordableMixin, Base):
     object_id = '%s.%s.%s' % (cls.getIdPrefix(), version, reference)
     new_component = context.newContent(id=object_id,
                                        reference=reference,
+                                       source_reference=source_reference,
                                        version=version,
                                        text_content=source_code,
                                        portal_type=cls.portal_type)