Commit c94dd3f5 authored by Arnaud Fontaine's avatar Arnaud Fontaine

ZODB Components: Products Documents for a given bt5 can now be migrated from filesystem.

Until now, only bt5 Extension/Test/Document could be migrated from
filesystem.

From migration dialog, allow to select any Products.ERP5.Documents.* (only,
for now) to be migrated. By default, automatically select Products Documents
used by the current bt5 Portal Types (by looking at the mro() of its
erp5.portal_type classes).

XXX-BEFORE-MERGE: Add Unit Test.
parent d5f978df
...@@ -6293,15 +6293,66 @@ Business Template is a set of definitions, such as skins, portal types and categ ...@@ -6293,15 +6293,66 @@ Business Template is a set of definitions, such as skins, portal types and categ
setattr(self, 'template_portal_type_base_category', ()) setattr(self, 'template_portal_type_base_category', ())
return return
@staticmethod
def _getAllFilesystemModuleFromPortalTypeIdList(portal_type_id_list):
import erp5.portal_type
import inspect
import Products.ERP5Type
product_base_path = inspect.getfile(Products.ERP5Type).rsplit('/', 2)[0]
seen_cls_set = set()
for portal_type in portal_type_id_list:
portal_type_cls = getattr(erp5.portal_type, portal_type)
# Calling mro() would not load the class...
portal_type_cls.loadClass()
for cls in portal_type_cls.mro():
if (not cls.__module__.startswith('erp5.') and
cls not in seen_cls_set):
seen_cls_set.add(cls)
try:
cls_path = inspect.getfile(cls)
except TypeError:
pass
else:
if cls_path.startswith(product_base_path):
cls_name = cls.__name__
cls_module = cls.__module__
yield cls_name, cls_module, cls_path
security.declareProtected(Permissions.ManagePortal, security.declareProtected(Permissions.ManagePortal,
'getMigratableSourceCodeFromFilesystemList') 'getMigratableSourceCodeFromFilesystemList')
def getMigratableSourceCodeFromFilesystemList(self, *args, **kwargs): def getMigratableSourceCodeFromFilesystemList(self,
current_bt_only=False,
*args,
**kwargs):
""" """
Return the list of Business Template {Extension, Document, Test} Documents Return the list of Business Template {Extension, Document, Test} Documents
and Products Documents which can be migrated to ZODB Components. and Products Documents which can be migrated to ZODB Components.
""" """
import inspect
bt_migratable_uid_list = []
migratable_component_list = [] migratable_component_list = []
component_tool = self.getPortalObject().portal_components portal = self.getPortalObject()
component_tool = portal.portal_components
from base64 import b64encode
import cPickle
def __newTempComponent(migrate=False, **kw):
uid = b64encode(cPickle.dumps(kw))
if migrate:
bt_migratable_uid_list.append(uid)
obj = component_tool.newContent(temp_object=1,
id="temp_" + uid,
uid=uid,
migrate=migrate,
**kw)
migratable_component_list.append(obj)
return obj
for portal_type, id_list in ( for portal_type, id_list in (
('Document Component', self.getTemplateDocumentIdList()), ('Document Component', self.getTemplateDocumentIdList()),
...@@ -6310,14 +6361,58 @@ Business Template is a set of definitions, such as skins, portal types and categ ...@@ -6310,14 +6361,58 @@ Business Template is a set of definitions, such as skins, portal types and categ
for id_ in id_list: for id_ in id_list:
existing_component = getattr(component_tool, id_, None) existing_component = getattr(component_tool, id_, None)
if existing_component is None: if existing_component is None:
obj = component_tool.newContent(id="tmp_source_code_migration_%s" % id_, obj = __newTempComponent(migrate=True,
portal_type=portal_type, portal_type=portal_type,
reference=id_, reference=id_,
temp_object=1) source_reference="%s:%s" % (self.getTitle(), id_))
migratable_component_list.append(obj) # Inspect Portal Types classes mro() of this Business Template to find
# Products Documents to migrate by default
portal_type_module_set = set(
self._getAllFilesystemModuleFromPortalTypeIdList(
self.getTemplatePortalTypeIdList()))
# XXX: Only migrate Documents in ERP5 for the moment...
import Products.ERP5.Document
for name, obj in Products.ERP5.Document.__dict__.iteritems():
if not name.startswith('_') and inspect.ismodule(obj):
source_reference = obj.__name__
migrate = ((name, source_reference, inspect.getfile(obj))
in portal_type_module_set)
if current_bt_only and not migrate:
continue
return sorted(migratable_component_list, key=lambda o: o.getReference()) obj = __newTempComponent(
migrate=migrate,
portal_type='Document Component',
reference=name,
source_reference=source_reference)
if not current_bt_only:
import Products.ERP5.tests
from glob import iglob
for test_path in iglob("%s/test*.py" %
inspect.getfile(Products.ERP5.tests).rsplit('/', 1)[0]):
reference = test_path.rsplit('/', 1)[1][:-3]
obj = __newTempComponent(
portal_type='Test Component',
reference=reference,
source_reference="Products.ERP5.tests." + reference)
# Automatically select ZODB Components to be migrated in Migration Dialog
selection_name = kwargs.get('selection_name')
if (selection_name is not None and
# XXX: Do not set uids on {check,uncheck}All, better way?
self.REQUEST.get('listbox_uncheckAll') is None and
self.REQUEST.get('listbox_checkAll') is None):
portal.portal_selections.setSelectionCheckedUidsFor(selection_name,
bt_migratable_uid_list)
return sorted(migratable_component_list,
key=lambda o: (not o.getProperty('migrate', False),
o.getPortalType(),
o.getReference()))
security.declareProtected(Permissions.ManagePortal, security.declareProtected(Permissions.ManagePortal,
'migrateSourceCodeFromFilesystem') 'migrateSourceCodeFromFilesystem')
...@@ -6333,16 +6428,44 @@ Business Template is a set of definitions, such as skins, portal types and categ ...@@ -6333,16 +6428,44 @@ Business Template is a set of definitions, such as skins, portal types and categ
component_tool = portal.portal_components component_tool = portal.portal_components
failed_import_dict = {} failed_import_dict = {}
list_selection_name = kw.get('list_selection_name') list_selection_name = kw.get('list_selection_name')
migrated_product_module_set = set()
template_document_id_set = set(self.getTemplateDocumentIdList()) template_document_id_set = set(self.getTemplateDocumentIdList())
template_extension_id_set = set(self.getTemplateExtensionIdList()) template_extension_id_set = set(self.getTemplateExtensionIdList())
template_test_id_set = set(self.getTemplateTestIdList()) template_test_id_set = set(self.getTemplateTestIdList())
for temp_obj in self.getMigratableSourceCodeFromFilesystemList(): if list_selection_name is None:
temp_obj_list = self.getMigratableSourceCodeFromFilesystemList(
current_bt_only=True)
else:
from base64 import b64decode
import cPickle
temp_obj_list = []
for uid in portal.portal_selections.getSelectionCheckedUidsFor(
list_selection_name):
property_dict = cPickle.loads(b64decode(uid))
obj = component_tool.newContent(temp_object=1,
id="temp_" + uid,
uid=uid,
**property_dict)
temp_obj_list.append(obj)
if not temp_obj_list:
if list_selection_name is not None:
return self.Base_redirect(
'view',
keep_items={'portal_status_message': 'Nothing Selected.'})
return
for temp_obj in temp_obj_list:
source_reference = temp_obj.getSourceReference()
try: try:
obj = temp_obj.importFromFilesystem(component_tool, obj = temp_obj.importFromFilesystem(component_tool,
temp_obj.getReference(), temp_obj.getReference(),
version) version,
source_reference)
except Exception, e: except Exception, e:
import traceback import traceback
LOG("BusinessTemplate", WARNING, LOG("BusinessTemplate", WARNING,
...@@ -6362,7 +6485,11 @@ Business Template is a set of definitions, such as skins, portal types and categ ...@@ -6362,7 +6485,11 @@ Business Template is a set of definitions, such as skins, portal types and categ
else: else:
id_set = template_document_id_set id_set = template_document_id_set
if source_reference.startswith('Products'):
migrated_product_module_set.add(source_reference)
else:
id_set.discard(temp_obj.getReference()) id_set.discard(temp_obj.getReference())
id_set.add(obj.getId()) id_set.add(obj.getId())
if failed_import_dict: if failed_import_dict:
...@@ -6383,6 +6510,27 @@ Business Template is a set of definitions, such as skins, portal types and categ ...@@ -6383,6 +6510,27 @@ Business Template is a set of definitions, such as skins, portal types and categ
self.setTemplateExtensionIdList(sorted(template_extension_id_set)) self.setTemplateExtensionIdList(sorted(template_extension_id_set))
self.setTemplateTestIdList(sorted(template_test_id_set)) self.setTemplateTestIdList(sorted(template_test_id_set))
# This will trigger a reset so that Portal Types mro() can be checked
# after migration for filesystem Products modules still being used
transaction.commit()
still_used_list_dict = {}
for _, cls_module, _ in self._getAllFilesystemModuleFromPortalTypeIdList(
portal.portal_types.objectIds()):
if cls_module in migrated_product_module_set:
package, module = cls_module.rsplit('.', 1)
still_used_list_dict.setdefault(package, []).append(module)
if still_used_list_dict:
module_still_used_message = ', '.join(
[ "%s.{%s}" % (package, ','.join(sorted(module_list)))
for package, module_list in still_used_list_dict.iteritems() ])
LOG('BusinessTemplate',
WARNING,
"The following Documents are still being imported so code need to "
"be updated: " + module_still_used_message)
if list_selection_name is not None: if list_selection_name is not None:
message = ( message = (
"All components were successfully imported from filesystem to ZODB. " "All components were successfully imported from filesystem to ZODB. "
......
...@@ -92,7 +92,7 @@ ...@@ -92,7 +92,7 @@
<dictionary> <dictionary>
<item> <item>
<key> <string>text</string> </key> <key> <string>text</string> </key>
<value> <string>python: portal.Base_checkPermission(\'portal_components\', \'Add portal content\') and context.getInstallationState() != \'installed\' and [id_ for id_ in (context.getTemplateExtensionIdList() + context.getTemplateDocumentIdList() + context.getTemplateTestIdList()) if not id_.startswith(\'extension.\') and not id_.startswith(\'document.\') and not id_.startswith(\'test.\') ]</string> </value> <value> <string>python: portal.Base_checkPermission(\'portal_components\', \'Add portal content\') and context.getInstallationState() != \'installed\'</string> </value>
</item> </item>
</dictionary> </dictionary>
</pickle> </pickle>
......
...@@ -404,6 +404,8 @@ ...@@ -404,6 +404,8 @@
<string>Name</string> <string>Name</string>
</tuple> </tuple>
<tuple> <tuple>
<string>source_reference</string>
<string>Module</string>
</tuple> </tuple>
<tuple> <tuple>
<string>portal_type</string> <string>portal_type</string>
...@@ -546,7 +548,7 @@ ...@@ -546,7 +548,7 @@
</item> </item>
<item> <item>
<key> <string>select</string> </key> <key> <string>select</string> </key>
<value> <int>0</int> </value> <value> <int>1</int> </value>
</item> </item>
<item> <item>
<key> <string>selection_name</string> </key> <key> <string>selection_name</string> </key>
...@@ -561,7 +563,12 @@ ...@@ -561,7 +563,12 @@
<item> <item>
<key> <string>sort_columns</string> </key> <key> <string>sort_columns</string> </key>
<value> <value>
<list/> <list>
<tuple>
<string>None</string>
<string>None</string>
</tuple>
</list>
</value> </value>
</item> </item>
<item> <item>
......
...@@ -68,7 +68,7 @@ class IComponent(Interface): ...@@ -68,7 +68,7 @@ class IComponent(Interface):
Return the ID prefix for Component objects Return the ID prefix for Component objects
""" """
def importFromFilesystem(cls, context, reference, version): def importFromFilesystem(cls, context, reference, version, source_reference=None):
""" """
Import a Component from the filesystem into ZODB after checking that the Import a Component from the filesystem into ZODB after checking that the
source code is valid source code is valid
......
...@@ -345,14 +345,21 @@ class ComponentMixin(PropertyRecordableMixin, Base): ...@@ -345,14 +345,21 @@ class ComponentMixin(PropertyRecordableMixin, Base):
security.declareProtected(Permissions.ModifyPortalContent, security.declareProtected(Permissions.ModifyPortalContent,
'importFromFilesystem') 'importFromFilesystem')
@classmethod @classmethod
def importFromFilesystem(cls, context, reference, version): def importFromFilesystem(cls, context, reference, version, source_reference=None):
""" """
Import a Component from the filesystem into ZODB and validate it so it can Import a Component from the filesystem into ZODB and validate it so it can
be loaded straightaway provided validate() does not raise any error of be loaded straightaway provided validate() does not raise any error of
course course
""" """
import os.path import os.path
if source_reference is None or not source_reference.startswith('Products'):
path = os.path.join(cls._getFilesystemPath(), reference + '.py') path = os.path.join(cls._getFilesystemPath(), reference + '.py')
else:
import inspect
module_obj = __import__(source_reference, globals(), {},
level=0, fromlist=[source_reference])
path = inspect.getsourcefile(module_obj)
with open(path) as f: with open(path) as f:
source_code = f.read() source_code = f.read()
...@@ -363,6 +370,7 @@ class ComponentMixin(PropertyRecordableMixin, Base): ...@@ -363,6 +370,7 @@ class ComponentMixin(PropertyRecordableMixin, Base):
object_id = '%s.%s.%s' % (cls._getIdPrefix(), version, reference) object_id = '%s.%s.%s' % (cls._getIdPrefix(), version, reference)
new_component = context.newContent(id=object_id, new_component = context.newContent(id=object_id,
reference=reference, reference=reference,
source_reference=source_reference,
version=version, version=version,
text_content=source_code, text_content=source_code,
portal_type=cls.portal_type) portal_type=cls.portal_type)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment