Commit b33264b6 authored by Arnaud Fontaine's avatar Arnaud Fontaine Committed by Cédric Le Ninivin

ZODB Components: Products Documents for a given bt5 can now be migrated from filesystem.

Until now, only bt5 Extension/Test/Document could be migrated from
filesystem.

From migration dialog, allow to select any Products.ERP5.Document.* (only,
for now) to be migrated. By default, automatically select Products Documents
used by the current bt5 Portal Types (by looking at the mro() of its
erp5.portal_type classes).

Also, to easily identified where it was migrated from, source_reference
is set to 'bt.getTitle():ID' for bt5 Extension/Test/Document and
'Products.ERP5.Document.XXX' for filesystem Products.
parent c4e467d6
......@@ -6344,15 +6344,79 @@ Business Template is a set of definitions, such as skins, portal types and categ
setattr(self, 'template_portal_type_base_category', ())
return
@staticmethod
def _getAllFilesystemModuleFromPortalTypeIdList(portal_type_id_list):
import erp5.portal_type
import inspect
import Products.ERP5Type
product_base_path = inspect.getfile(Products.ERP5Type).rsplit('/', 2)[0]
seen_cls_set = set()
for portal_type in portal_type_id_list:
# According to ObjectTemplateItem.__init__, this could happen (see
# stepAddPortalTypeToBusinessTemplate)
if portal_type == '':
continue
portal_type_cls = getattr(erp5.portal_type, portal_type)
# Calling mro() would not load the class...
try:
portal_type_cls.loadClass()
except Exception:
LOG("BusinessTemplate", WARNING,
"Could not load Portal Type Class %s, ignored for migration..." %
portal_type,
error=True)
continue
for cls in portal_type_cls.mro():
if (not cls.__module__.startswith('erp5.') and
cls not in seen_cls_set):
seen_cls_set.add(cls)
try:
cls_path = inspect.getfile(cls)
except TypeError:
pass
else:
if cls_path.startswith(product_base_path):
cls_name = cls.__name__
cls_module = cls.__module__
yield cls_name, cls_module, cls_path
security.declareProtected(Permissions.ManagePortal,
'getMigratableSourceCodeFromFilesystemList')
def getMigratableSourceCodeFromFilesystemList(self, *args, **kwargs):
def getMigratableSourceCodeFromFilesystemList(self,
current_bt_only=False,
*args,
**kwargs):
"""
Return the list of Business Template {Extension, Document, Test} Documents
and Products Documents which can be migrated to ZODB Components.
"""
import inspect
bt_migratable_uid_list = []
migratable_component_list = []
component_tool = self.getPortalObject().portal_components
portal = self.getPortalObject()
component_tool = portal.portal_components
from base64 import b64encode
import cPickle
def __newTempComponent(portal_type, reference, source_reference, migrate=False):
uid = b64encode("%s|%s|%s" % (portal_type, reference, source_reference))
if migrate:
bt_migratable_uid_list.append(uid)
obj = component_tool.newContent(temp_object=1,
id="temp_" + uid,
uid=uid,
portal_type=portal_type,
reference=reference,
source_reference=source_reference)
migratable_component_list.append(obj)
return obj
for portal_type, id_list in (
('Document Component', self.getTemplateDocumentIdList()),
......@@ -6361,14 +6425,56 @@ Business Template is a set of definitions, such as skins, portal types and categ
for id_ in id_list:
existing_component = getattr(component_tool, id_, None)
if existing_component is None:
obj = component_tool.newContent(id="tmp_source_code_migration_%s" % id_,
portal_type=portal_type,
reference=id_,
temp_object=1)
migratable_component_list.append(obj)
obj = __newTempComponent(portal_type=portal_type,
reference=id_,
source_reference="%s:%s" % (self.getTitle(), id_),
migrate=True)
# Inspect Portal Types classes mro() of this Business Template to find
# Products Documents to migrate by default
portal_type_module_set = set(
self._getAllFilesystemModuleFromPortalTypeIdList(
self.getTemplatePortalTypeIdList()))
# XXX: Only migrate Documents in ERP5 for the moment...
import Products.ERP5.Document
for name, obj in Products.ERP5.Document.__dict__.iteritems():
if not name.startswith('_') and inspect.ismodule(obj):
source_reference = obj.__name__
migrate = ((name, source_reference, inspect.getfile(obj))
in portal_type_module_set)
if current_bt_only and not migrate:
continue
return sorted(migratable_component_list, key=lambda o: o.getReference())
obj = __newTempComponent(portal_type='Document Component',
reference=name,
source_reference=source_reference,
migrate=migrate)
if not current_bt_only:
import Products.ERP5.tests
from glob import iglob
for test_path in iglob("%s/test*.py" %
inspect.getfile(Products.ERP5.tests).rsplit('/', 1)[0]):
reference = test_path.rsplit('/', 1)[1][:-3]
obj = __newTempComponent(portal_type='Test Component',
reference=reference,
source_reference="Products.ERP5.tests." + reference)
# Automatically select ZODB Components to be migrated in Migration Dialog
selection_name = kwargs.get('selection_name')
if (selection_name is not None and
# XXX: Do not set uids on {check,uncheck}All, better way?
self.REQUEST.get('listbox_uncheckAll') is None and
self.REQUEST.get('listbox_checkAll') is None):
portal.portal_selections.setSelectionCheckedUidsFor(selection_name,
bt_migratable_uid_list)
return sorted(migratable_component_list,
key=lambda o: (not o.getProperty('migrate', False),
o.getPortalType(),
o.getReference()))
security.declareProtected(Permissions.ManagePortal,
'migrateSourceCodeFromFilesystem')
......@@ -6384,16 +6490,46 @@ Business Template is a set of definitions, such as skins, portal types and categ
component_tool = portal.portal_components
failed_import_dict = {}
list_selection_name = kw.get('list_selection_name')
migrated_product_module_set = set()
template_document_id_set = set(self.getTemplateDocumentIdList())
template_extension_id_set = set(self.getTemplateExtensionIdList())
template_test_id_set = set(self.getTemplateTestIdList())
for temp_obj in self.getMigratableSourceCodeFromFilesystemList():
if list_selection_name is None:
temp_obj_list = self.getMigratableSourceCodeFromFilesystemList(
current_bt_only=True)
else:
from base64 import b64decode
import cPickle
temp_obj_list = []
for uid in portal.portal_selections.getSelectionCheckedUidsFor(
list_selection_name):
portal_type, reference, source_reference = b64decode(uid).split('|')
obj = component_tool.newContent(temp_object=1,
id="temp_" + uid,
uid=uid,
portal_type=portal_type,
reference=reference,
source_reference=source_reference)
temp_obj_list.append(obj)
if not temp_obj_list:
if list_selection_name is not None:
return self.Base_redirect(
'view',
keep_items={'portal_status_message': 'Nothing Selected.'})
return
for temp_obj in temp_obj_list:
source_reference = temp_obj.getSourceReference()
try:
obj = temp_obj.importFromFilesystem(component_tool,
temp_obj.getReference(),
version)
version,
source_reference)
except Exception, e:
LOG("BusinessTemplate", WARNING,
"Could not import component '%s' ('%s') from the filesystem" %
......@@ -6412,7 +6548,11 @@ Business Template is a set of definitions, such as skins, portal types and categ
else:
id_set = template_document_id_set
id_set.discard(temp_obj.getReference())
if source_reference.startswith('Products'):
migrated_product_module_set.add(source_reference)
else:
id_set.discard(temp_obj.getReference())
id_set.add(obj.getId())
if failed_import_dict:
......@@ -6433,6 +6573,27 @@ Business Template is a set of definitions, such as skins, portal types and categ
self.setTemplateExtensionIdList(sorted(template_extension_id_set))
self.setTemplateTestIdList(sorted(template_test_id_set))
# This will trigger a reset so that Portal Types mro() can be checked
# after migration for filesystem Products modules still being used
transaction.commit()
still_used_list_dict = {}
for _, cls_module, _ in self._getAllFilesystemModuleFromPortalTypeIdList(
portal.portal_types.objectIds()):
if cls_module in migrated_product_module_set:
package, module = cls_module.rsplit('.', 1)
still_used_list_dict.setdefault(package, []).append(module)
if still_used_list_dict:
module_still_used_message = ', '.join(
[ "%s.{%s}" % (package, ','.join(sorted(module_list)))
for package, module_list in still_used_list_dict.iteritems() ])
LOG('BusinessTemplate',
WARNING,
"The following Documents are still being imported so code need to "
"be updated: " + module_still_used_message)
if list_selection_name is not None:
message = (
"All components were successfully imported from filesystem to ZODB. "
......
......@@ -92,7 +92,7 @@
<dictionary>
<item>
<key> <string>text</string> </key>
<value> <string>python: portal.Base_checkPermission(\'portal_components\', \'Add portal content\') and context.getInstallationState() != \'installed\' and [id_ for id_ in (context.getTemplateExtensionIdList() + context.getTemplateDocumentIdList() + context.getTemplateTestIdList()) if not id_.startswith(\'extension.\') and not id_.startswith(\'document.\') and not id_.startswith(\'test.\') ]</string> </value>
<value> <string>python: portal.Base_checkPermission(\'portal_components\', \'Add portal content\') and context.getInstallationState() != \'installed\'</string> </value>
</item>
</dictionary>
</pickle>
......
......@@ -403,6 +403,10 @@
<string>reference</string>
<string>Name</string>
</tuple>
<tuple>
<string>source_reference</string>
<string>Module</string>
</tuple>
<tuple>
<string>portal_type</string>
<string>Destination Portal Type</string>
......@@ -549,7 +553,7 @@
</item>
<item>
<key> <string>select</string> </key>
<value> <int>0</int> </value>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>selection_name</string> </key>
......
......@@ -7758,6 +7758,144 @@ class _ZodbComponentTemplateItemMixin(BusinessTemplateMixin):
sequence_list.addSequenceString(sequence_string)
sequence_list.play(self)
class _ProductMigrationTemplateItemMixin:
"""
Test Cases specific to migration of Products from filesystem
(Products.XXX.{Document,mixin,interfaces}.YYY) to ZODB Components
(erp5.component.{document,mixin,interface}.YYY).
"""
def stepCreateProductDocumentAndPortalType(self, sequence=None, **_):
# Delete the bt5 Document created by previous tests (same name)
file_path = os.path.join(self.document_base_path, self.document_title + '.py')
if os.path.exists(file_path):
os.remove(file_path)
# getMigratableSourceCodeFromFilesystemList()
document_source_reference = 'Products.ERP5.Document.%s' % self.document_title
import Products.ERP5.Document
file_path = os.path.join(Products.ERP5.Document.__path__[0],
self.document_title + '.py')
with open(file_path, 'w') as f:
f.write(self.document_data)
self.assertTrue(os.path.exists(file_path))
from Products.ERP5Type.Utils import importLocalDocument
# This is normally called at Zope startup to register Products Documents
# into document_class_registry then used in generatePortalTypeClass().
#
# Another reason to do that is because bt5 Document may already be there
# from a previous tests execution and thus would be used instead of this
# one...
importLocalDocument(
self.document_title,
class_path="%s.%s" % (document_source_reference, self.document_title))
sequence.edit(document_title=self.document_title,
document_path=file_path,
document_data=self.document_data,
document_source_reference=document_source_reference)
object_type = self.getTypeTool().newContent(self.document_portal_type,
'Base Type',
type_class=self.document_title)
self.assertTrue(object_type is not None)
sequence.edit(object_ptype_id=self.document_portal_type)
import erp5.portal_type
portal_type_class = getattr(erp5.portal_type, sequence['object_ptype_id'])
# Accessing __mro__ would not load the class...
portal_type_class.loadClass()
from importlib import import_module
filesystem_module = import_module(document_source_reference)
filesystem_class = getattr(filesystem_module, self.document_title)
self.assertTrue(filesystem_class in portal_type_class.__mro__)
def stepCheckProductDocumentMigration(self, sequence=None, **kw):
self.stepCheckDocumentMigration(sequence, **kw)
from importlib import import_module
module_name = sequence['document_title']
import erp5.portal_type
portal_type_class = getattr(erp5.portal_type, sequence['object_ptype_id'])
# Accessing __mro__ would not load the class...
portal_type_class.loadClass()
filesystem_module = import_module('Products.ERP5.Document.%s' % module_name)
filesystem_class = getattr(filesystem_module, module_name)
self.assertFalse(filesystem_class in portal_type_class.__mro__)
component_module_name = 'erp5.component.document.' + module_name
try:
component_module = import_module(component_module_name)
except ImportError:
raise AssertionError("%s should be importable" % component_module_name)
else:
component_class = getattr(component_module, module_name)
self.assertTrue(component_class in portal_type_class.__mro__)
def test_BusinessTemplateUpgradeProductDocumentFromFilesystemToZodb(self):
"""
Same as test_BusinessTemplateUpgradeDocumentFromFilesystemToZodb above,
but for Product Document rather than bt5 Document. Also, Products Document
are not going to be removed automatically (safer).
"""
sequence_list = SequenceList()
sequence_string = """
CreateProductDocumentAndPortalType
CreateNewBusinessTemplate
UseExportBusinessTemplate
AddPortalTypeToBusinessTemplate
UseExportBusinessTemplate
CheckModifiedBuildingState
CheckNotInstalledInstallationState
BuildBusinessTemplate
CheckBuiltBuildingState
CheckNotInstalledInstallationState
CheckObjectPropertiesInBusinessTemplate
UseCurrentBusinessTemplateForInstall
InstallWithoutForceBusinessTemplate
Tic
CheckInstalledInstallationState
CheckBuiltBuildingState
CheckSkinsLayers
CheckDocumentExists
CopyAndMigrateDocumentBusinessTemplate
CheckProductDocumentMigration
BuildBusinessTemplate
CheckBuiltBuildingState
CheckNotInstalledInstallationState
SaveBusinessTemplate
RemoveBusinessTemplate
CheckZodbDocumentWorkflowHistoryUnchanged
RemoveZodbDocument
CheckDocumentExists
CheckZodbDocumentRemoved
ImportBusinessTemplate
UseImportBusinessTemplate
CheckBuiltBuildingState
CheckNotInstalledInstallationState
InstallWithoutForceBusinessTemplate
Tic
CheckInstalledInstallationState
CheckBuiltBuildingState
CheckSkinsLayers
CheckDocumentExists
CheckZodbDocumentExistsAndValidated
UseExportBusinessTemplate
CheckReplacedInstallationState
UseImportBusinessTemplate
UninstallBusinessTemplate
RemoveAllTrashBins
CheckBuiltBuildingState
CheckNotInstalledInstallationState
CheckZodbDocumentRemoved
"""
sequence_list.addSequenceString(sequence_string)
sequence_list.play(self)
# bt5 (instancehome and ZODB Component)
class _LocalTemplateItemMixin:
"""
......@@ -7772,8 +7910,9 @@ class _LocalTemplateItemMixin:
f.write(self.document_data)
f.close()
self.assertTrue(os.path.exists(file_path))
sequence.edit(document_title=self.document_title, document_path=file_path,
document_data=self.document_data)
sequence.edit(document_title=self.document_title,
document_path=file_path,
document_data=self.document_data)
def stepCreateUpdatedDocument(self, sequence=None, **kw):
file_path = os.path.join(self.document_base_path, self.document_title+'.py')
......@@ -7788,7 +7927,11 @@ class _LocalTemplateItemMixin:
def stepAddDocumentToBusinessTemplate(self, sequence=None, **kw):
bt = sequence['current_bt']
bt.edit(**{self.template_property: [sequence['document_title']]})
document_title = sequence['document_title']
bt.edit(**{self.template_property: [document_title]})
# getMigratableSourceCodeFromFilesystemList()
sequence.edit(document_source_reference='%s:%s' % (bt.getTitle(),
document_title))
def stepRemoveDocument(self, sequence=None, **kw):
document_path = sequence['document_path']
......@@ -8081,6 +8224,8 @@ class _LocalTemplateItemMixin:
self.assertEqual(component.getReference(), sequence['document_title'])
self.assertEqual(component.getTextContent(), sequence['document_data'])
self.assertEqual(component.getPortalType(), self.component_portal_type)
self.assertEqual(component.getSourceReference(), sequence['document_source_reference'])
self.assertEqual(component.getValidationState(), 'validated')
sequence.edit(document_id=component_id)
def test_BusinessTemplateWithZodbDocumentMigrated(self):
......@@ -8308,16 +8453,35 @@ class _LocalTemplateItemMixin:
from Products.ERP5Type.Core.DocumentComponent import DocumentComponent
# bt5 (instancehome and ZODB Component) and Products
class TestDocumentTemplateItem(_LocalTemplateItemMixin,
_ZodbComponentTemplateItemMixin):
_ZodbComponentTemplateItemMixin,
_ProductMigrationTemplateItemMixin):
document_title = 'UnitTest'
document_data = """class UnitTest:
document_portal_type = 'Unit Test'
document_data = """from Products.ERP5Type.XMLObject import XMLObject
from Products.ERP5Type.Globals import InitializeClass
from AccessControl import ClassSecurityInfo
class UnitTest(XMLObject):
meta_type = 'ERP5 Unit Test'
portal_type = 'Unit Test'"""
document_data_updated = """class UnitTest:
portal_type = 'Unit Test'
security = ClassSecurityInfo()
InitializeClass(UnitTest)
"""
document_data_updated = """from Products.ERP5Type.XMLObject import XMLObject
from Products.ERP5Type.Globals import InitializeClass
from AccessControl import ClassSecurityInfo
class UnitTest(XMLObject):
meta_type = 'ERP5 Unit Test'
portal_type = 'Unit Test'
security = ClassSecurityInfo()
def updated(self):
pass"""
pass
InitializeClass(UnitTest)
"""
document_base_path = os.path.join(getConfiguration().instancehome, 'Document')
template_property = 'template_document_id_list'
component_id_prefix = DocumentComponent.getIdPrefix()
......@@ -8325,7 +8489,8 @@ class TestDocumentTemplateItem(_LocalTemplateItemMixin,
from Products.ERP5Type.Core.InterfaceComponent import InterfaceComponent
# bt5 (ZODB Component only) and Products
class TestInterfaceTemplateItem(_ZodbComponentTemplateItemMixin):
class TestInterfaceTemplateItem(_ZodbComponentTemplateItemMixin,
_ProductMigrationTemplateItemMixin):
document_title = 'IUnitTest'
document_data = '''from zope.interface import Interface
......@@ -8341,7 +8506,8 @@ class IUnitTest(Interface):
from Products.ERP5Type.Core.MixinComponent import MixinComponent
# bt5 (ZODB Component only) and Products
class TestMixinTemplateItem(_ZodbComponentTemplateItemMixin):
class TestMixinTemplateItem(_ZodbComponentTemplateItemMixin,
_ProductMigrationTemplateItemMixin):
document_title = 'UnitTestMixin'
document_data = '''class UnitTestMixin:
def foo(self):
......@@ -8356,7 +8522,8 @@ class TestMixinTemplateItem(_ZodbComponentTemplateItemMixin):
# bt5 (instancehome (legacy) and ZODB Component) and Products
class TestConstraintTemplateItem(_LocalTemplateItemMixin,
_ZodbComponentTemplateItemMixin):
_ZodbComponentTemplateItemMixin,
_ProductMigrationTemplateItemMixin):
document_title = 'UnitTest'
document_data = ' \nclass UnitTest: \n """ \n Fake constraint for unit test \n \
""" \n _properties = ( \n ) \n _categories = ( \n ) \n\n'
......@@ -8388,7 +8555,8 @@ class TestExtensionTemplateItem(_LocalTemplateItemMixin,
from Products.ERP5Type.Core.TestComponent import TestComponent
# bt5 (instancehome (legacy) and ZODB Component) and Products
class TestTestTemplateItem(_LocalTemplateItemMixin,
_ZodbComponentTemplateItemMixin):
_ZodbComponentTemplateItemMixin,
_ProductMigrationTemplateItemMixin):
document_title = 'UnitTest'
document_data = """class UnitTest:
meta_type = 'ERP5 Unit Test'
......@@ -8508,6 +8676,19 @@ TestConstraintTemplateItem.test_BusinessTemplateUpgradeDocumentFromFilesystemToZ
TestConstraintTemplateItem.test_BusinessTemplateUpgradeDocumentFromFilesystemToZodbWithSyntaxError = \
skip('Not implemented yet')(TestConstraintTemplateItem.test_BusinessTemplateUpgradeDocumentFromFilesystemToZodbWithSyntaxError)
TestConstraintTemplateItem.test_BusinessTemplateUpgradeProductDocumentFromFilesystemToZodb = \
skip('Not implemented yet')(TestConstraintTemplateItem.test_BusinessTemplateUpgradeProductDocumentFromFilesystemToZodb)
# TODO-arnau
TestInterfaceTemplateItem.test_BusinessTemplateUpgradeProductDocumentFromFilesystemToZodb = \
skip('Not implemented yet')(TestInterfaceTemplateItem.test_BusinessTemplateUpgradeProductDocumentFromFilesystemToZodb)
TestTestTemplateItem.test_BusinessTemplateUpgradeProductDocumentFromFilesystemToZodb = \
skip('Not implemented yet')(TestTestTemplateItem.test_BusinessTemplateUpgradeProductDocumentFromFilesystemToZodb)
TestMixinTemplateItem.test_BusinessTemplateUpgradeProductDocumentFromFilesystemToZodb = \
skip('Not implemented yet')(TestMixinTemplateItem.test_BusinessTemplateUpgradeProductDocumentFromFilesystemToZodb)
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestBusinessTemplate))
......
......@@ -68,7 +68,7 @@ class IComponent(Interface):
Return the ID prefix for Component objects
"""
def importFromFilesystem(cls, context, reference, version):
def importFromFilesystem(cls, context, reference, version, source_reference=None):
"""
Import a Component from the filesystem into ZODB after checking that the
source code is valid
......
......@@ -345,14 +345,21 @@ class ComponentMixin(PropertyRecordableMixin, Base):
security.declareProtected(Permissions.ModifyPortalContent,
'importFromFilesystem')
@classmethod
def importFromFilesystem(cls, context, reference, version):
def importFromFilesystem(cls, context, reference, version, source_reference=None):
"""
Import a Component from the filesystem into ZODB and validate it so it can
be loaded straightaway provided validate() does not raise any error of
course
"""
import os.path
path = os.path.join(cls._getFilesystemPath(), reference + '.py')
if source_reference is None or not source_reference.startswith('Products'):
path = os.path.join(cls._getFilesystemPath(), reference + '.py')
else:
import inspect
module_obj = __import__(source_reference, globals(), {},
level=0, fromlist=[source_reference])
path = inspect.getsourcefile(module_obj)
with open(path) as f:
source_code = f.read()
......@@ -363,6 +370,7 @@ class ComponentMixin(PropertyRecordableMixin, Base):
object_id = '%s.%s.%s' % (cls.getIdPrefix(), version, reference)
new_component = context.newContent(id=object_id,
reference=reference,
source_reference=source_reference,
version=version,
text_content=source_code,
portal_type=cls.portal_type)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment