Commit d50901ba authored by Arnaud Fontaine's avatar Arnaud Fontaine

Import Components from filesystem manually and add a test to check migration.

When versions will be introduced, there will be an import action which will
take the required version. Moreover, there are not so many Extensions and
Documents to migrate after all and Products will have to be migrated manually
as well.
parent f0f5284a
......@@ -3767,6 +3767,13 @@ class DocumentTemplateItem(FilesystemToZodbTemplateItem):
Extensions are now stored in ZODB rather than on the filesystem. However,
some Business Templates may still have filesystem Extensions which need to
be migrated to the ZODB.
The migration is performed in two steps:
1/ Copy the Business Template to be migrated;
2/ Run the migration script which will update properly the Document IDs in
the Business Template.
"""
_tool_id = 'portal_components'
......@@ -3815,13 +3822,41 @@ class DocumentTemplateItem(FilesystemToZodbTemplateItem):
XMLExportImport.exportXML(obj._p_jar, obj._p_oid, f)
bta.addObject(f, key, path=path)
@staticmethod
def _migrateFromFilesystem(tool,
filesystem_path,
filesystem_file,
class_id):
return DocumentComponentDocument.importFromFilesystem(tool,
filesystem_path)
def getTemplateIdList(self):
return self.getTemplateDocumentIdList()
def build(self, context, **kw):
if not self._archive:
return
if not self._is_already_migrated(self._archive.keys()):
document_id_list = self.getTemplateIdList()
try:
context.getPortalObject().unrestrictedTraverse(
'portal_components/' + document_id_list[0])
except (IndexError, KeyError):
return FilesystemDocumentTemplateItem.build(self, context, **kw)
else:
self._archive.clear()
for name in document_id_list:
self._archive['portal_components/' + name] = None
return ObjectTemplateItem.build(self, context, **kw)
def install(self, context, **kw):
"""
In contrary to ZODB Property Sheets, Components are not migrated
automatically as the version must be set manually. This should not be an
issue as there are not so many Documents in bt5...
"""
object_list = list((context.getTemplateFormatVersion() == 1 and
self._objects or self._archive))
if self._is_already_migrated(object_list):
return ObjectTemplateItem.install(self, context, **kw)
else:
return FilesystemDocumentTemplateItem.install(self, context, **kw)
from Products.ERP5Type.Core.ExtensionComponent import ExtensionComponent as \
ExtensionComponentDocument
......@@ -3843,20 +3878,8 @@ class ExtensionTemplateItem(DocumentTemplateItem):
def _getZodbObjectId(id):
return 'erp5.component.extension.%s' % id
@staticmethod
def _getFilesystemPath(class_id):
from App.config import getConfiguration
return os.path.join(getConfiguration().instancehome,
"Extensions",
"%s.py" % class_id)
@staticmethod
def _migrateFromFilesystem(tool,
filesystem_path,
filesystem_file,
class_id):
return ExtensionComponentDocument.importFromFilesystem(tool,
filesystem_path)
def getTemplateIdList(self):
return self.getTemplateExtensionIdList()
class TestTemplateItem(FilesystemDocumentTemplateItem):
local_file_reader_name = staticmethod(readLocalTest)
......
......@@ -6662,64 +6662,7 @@ class TestDocumentTemplateItem(BusinessTemplateMixin):
sequence_list.addSequenceString(sequence_string)
sequence_list.play(self)
component_module = DocumentComponent._getDynamicModuleNamespace()
component_portal_type = DocumentComponent.portal_type
def getBusinessTemplateList(self):
return (super(TestDocumentTemplateItem, self).getBusinessTemplateList() +
('erp5_core_component',))
def stepCheckZodbDocumentRemoved(self, sequence=None, **kw):
component_tool = self.getPortalObject().portal_components
component_id = '%s.%s' % (self.component_module, sequence['document_title'])
self.failIf(component_id in component_tool.objectIds())
def stepRemoveZodbDocument(self, sequence=None, **kw):
"""
Remove Property Sheet, but only from ZODB
"""
component_id = '%s.%s' % (self.component_module, sequence['document_title'])
self.portal.portal_components.manage_delObjects([component_id])
def stepCheckDocumentMigration(self, sequence=None, **kw):
"""
Check migration of Document from the Filesystem to ZODB
"""
component_id = '%s.%s' % (self.component_module, sequence['document_title'])
component_tool = self.getPortalObject().portal_components
self.failUnless(component_id in component_tool.objectIds())
component = getattr(component_tool, component_id)
self.assertEquals(component.getReference(), self.document_title)
self.assertEquals(component.getTextContent(), self.document_data)
self.assertEquals(component.getPortalType(), self.component_portal_type)
def stepCheckForkedMigrationExport(self, sequence=None, **kw):
component_bt_tool_path = os.path.join(sequence['template_path'],
self.__class__.__name__.replace('Test', ''),
'portal_components')
self.assertTrue(os.path.exists(component_bt_tool_path))
component_id = '%s.%s' % (self.component_module, sequence['document_title'])
base_path = os.path.join(component_bt_tool_path, component_id)
python_source_code_path = base_path + '.py'
self.assertTrue(os.path.exists(python_source_code_path))
source_code = sequence['document_data']
with open(python_source_code_path) as f:
self.assertEquals(f.read(), source_code)
xml_path = base_path + '.xml'
self.assertTrue(os.path.exists(xml_path))
first_line = source_code.split('\n', 1)[0]
with open(xml_path) as f:
for line in f:
self.failIf(first_line in line)
def test_BusinessTemplateWithDocumentMigration(self):
def test_BusinessTemplateUpdateWithDocument(self):
sequence_list = SequenceList()
sequence_string = '\
CreateDocument \
......@@ -6748,29 +6691,66 @@ class TestDocumentTemplateItem(BusinessTemplateMixin):
CheckBuiltBuildingState \
CheckNoTrashBin \
CheckSkinsLayers \
CheckDocumentMigration \
CheckDocumentRemoved \
UninstallBusinessTemplate \
CheckDocumentExists \
RemoveDocument \
CreateUpdatedDocument \
CreateNewBusinessTemplate \
UseExportBusinessTemplate \
AddDocumentToBusinessTemplate \
CheckModifiedBuildingState \
CheckNotInstalledInstallationState \
BuildBusinessTemplate \
CheckBuiltBuildingState \
CheckNotInstalledInstallationState \
CheckZodbDocumentRemoved \
CheckObjectPropertiesInBusinessTemplate \
SaveBusinessTemplate \
CheckForkedMigrationExport \
CheckBuiltBuildingState \
CheckNotInstalledInstallationState \
CreateDocument \
RemoveBusinessTemplate \
RemoveAllTrashBins \
ImportBusinessTemplate \
UseImportBusinessTemplate \
CheckBuiltBuildingState \
CheckNotInstalledInstallationState \
InstallWithoutForceBusinessTemplate \
Tic \
CheckUpdatedDocumentExists \
UninstallBusinessTemplate \
CheckBuiltBuildingState \
CheckNotInstalledInstallationState \
CheckDocumentRemoved \
'
sequence_list.addSequenceString(sequence_string)
sequence_list.play(self)
# XXX-arnau: Temporary until _perform_migration is set to True by default
from Products.ERP5.Document.BusinessTemplate import DocumentTemplateItem
DocumentTemplateItem._perform_migration = True
try:
sequence_list.play(self)
finally:
DocumentTemplateItem._perform_migration = False
def test_BusinessTemplateUpdateWithDocument(self):
def test_BusinessTemplateWithDocumentNonExistingBefore(self):
sequence_list = SequenceList()
sequence_string = '\
CreateNewBusinessTemplate \
UseExportBusinessTemplate \
CheckModifiedBuildingState \
CheckNotInstalledInstallationState \
BuildBusinessTemplate \
CheckBuiltBuildingState \
CheckNotInstalledInstallationState \
CheckObjectPropertiesInBusinessTemplate \
SaveBusinessTemplate \
CheckBuiltBuildingState \
CheckNotInstalledInstallationState \
RemoveBusinessTemplate \
RemoveAllTrashBins \
ImportBusinessTemplate \
UseImportBusinessTemplate \
CheckBuiltBuildingState \
CheckNotInstalledInstallationState \
InstallWithoutForceBusinessTemplate \
Tic \
CheckInstalledInstallationState \
CheckBuiltBuildingState \
CheckNoTrashBin \
CheckSkinsLayers \
\
CreateDocument \
CreateNewBusinessTemplate \
UseExportBusinessTemplate \
......@@ -6798,11 +6778,27 @@ class TestDocumentTemplateItem(BusinessTemplateMixin):
CheckNoTrashBin \
CheckSkinsLayers \
CheckDocumentExists \
RemoveDocument \
CreateUpdatedDocument \
UninstallBusinessTemplate \
CheckBuiltBuildingState \
CheckNotInstalledInstallationState \
CheckDocumentRemoved \
'
sequence_list.addSequenceString(sequence_string)
sequence_list.play(self)
def test_BusinessTemplateWithDocumentPropertySheetMigrated(self):
"""Checks that if Business Template defines Document and PropertySheet
Document is not removed after Property Sheet was migrated and Business Template
was updated"""
sequence_list = SequenceList()
sequence_string = '\
CreateDocument \
CreatePropertySheet \
CheckDocumentPropertySheetSameName \
CreateNewBusinessTemplate \
UseExportBusinessTemplate \
AddDocumentToBusinessTemplate \
AddPropertySheetToBusinessTemplate \
CheckModifiedBuildingState \
CheckNotInstalledInstallationState \
BuildBusinessTemplate \
......@@ -6812,7 +6808,8 @@ class TestDocumentTemplateItem(BusinessTemplateMixin):
SaveBusinessTemplate \
CheckBuiltBuildingState \
CheckNotInstalledInstallationState \
CreateDocument \
RemoveDocument \
RemovePropertySheet \
RemoveBusinessTemplate \
RemoveAllTrashBins \
ImportBusinessTemplate \
......@@ -6821,16 +6818,161 @@ class TestDocumentTemplateItem(BusinessTemplateMixin):
CheckNotInstalledInstallationState \
InstallWithoutForceBusinessTemplate \
Tic \
CheckUpdatedDocumentExists \
CheckInstalledInstallationState \
CheckBuiltBuildingState \
CheckNoTrashBin \
CheckDocumentExists \
CheckPropertySheetExists \
\
SimulateAndCopyPrePropertySheetMigrationBusinessTemplate \
Tic \
\
CreateAllPropertySheetsFromFilesystem \
Tic \
CheckPropertySheetRemoved \
CheckPropertySheetMigration \
\
CheckDraftBuildingState \
CheckNotInstalledInstallationState \
BuildBusinessTemplate \
CheckBuiltBuildingState \
CheckNotInstalledInstallationState \
CheckObjectPropertiesInBusinessTemplate \
SaveBusinessTemplate \
CheckBuiltBuildingState \
CheckNotInstalledInstallationState \
RemoveBusinessTemplate \
Tic \
CreatePropertySheet \
RemovePropertySheetZodbOnly \
Tic \
ImportBusinessTemplate \
UseImportBusinessTemplate \
CheckBuiltBuildingState \
CheckNotInstalledInstallationState \
InstallWithoutForceBusinessTemplate \
Tic \
\
CheckPropertySheetMigration \
CheckInstalledInstallationState \
CheckBuiltBuildingState \
CheckDocumentExists \
CheckPropertySheetRemoved \
'
sequence_list.addSequenceString(sequence_string)
sequence_list.play(self)
# Specific to ZODB Components *only*
def getBusinessTemplateList(self):
return (super(TestDocumentTemplateItem, self).getBusinessTemplateList() +
('erp5_core_component',))
component_module = DocumentComponent._getDynamicModuleNamespace()
component_portal_type = DocumentComponent.portal_type
def stepCreateZodbDocument(self, sequence=None, **kw):
document_id = '%s.%s' % (self.component_module, self.document_title)
self.getPortalObject().portal_components.newContent(
id=document_id,
version='erp5',
reference=self.document_title,
text_content=self.document_data,
portal_type=self.component_portal_type)
sequence.edit(document_title=self.document_title,
document_id=document_id,
document_data=self.document_data)
def stepAddZodbDocumentToBusinessTemplate(self, sequence=None, **kw):
sequence['current_bt'].setTemplateDocumentIdList(sequence['document_id'])
def stepRemoveZodbDocument(self, sequence=None, **kw):
self.getPortalObject().portal_components.deleteContent(
sequence['document_id'])
def stepCheckZodbDocumentExists(self, sequence=None, **kw):
self.assertHasAttribute(self.getPortalObject().portal_components,
sequence['document_id'])
def stepCheckZodbDocumentRemoved(self, sequence=None, **kw):
component_tool = self.getPortalObject().portal_components
self.failIf(sequence['document_id'] in component_tool.objectIds())
def stepRemoveZodbDocument(self, sequence=None, **kw):
self.portal.portal_components.manage_delObjects([sequence['document_id']])
def stepCheckForkedMigrationExport(self, sequence=None, **kw):
"""
After saving a Business Template, two files should have been created for
each Component, one is the Python source code (ending with '.py') and the
other one is the metadata (ending with '.xml')
"""
component_bt_tool_path = os.path.join(sequence['template_path'],
self.__class__.__name__.replace('Test', ''),
'portal_components')
self.assertTrue(os.path.exists(component_bt_tool_path))
component_id = '%s.%s' % (self.component_module, sequence['document_title'])
base_path = os.path.join(component_bt_tool_path, component_id)
python_source_code_path = base_path + '.py'
self.assertTrue(os.path.exists(python_source_code_path))
source_code = sequence['document_data']
with open(python_source_code_path) as f:
self.assertEquals(f.read(), source_code)
xml_path = base_path + '.xml'
self.assertTrue(os.path.exists(xml_path))
first_line = source_code.split('\n', 1)[0]
with open(xml_path) as f:
for line in f:
self.failIf(first_line in line)
def test_BusinessTemplateWithZodbDocument(self):
sequence_list = SequenceList()
sequence_string = '\
CreateZodbDocument \
CreateNewBusinessTemplate \
UseExportBusinessTemplate \
AddZodbDocumentToBusinessTemplate \
CheckModifiedBuildingState \
CheckNotInstalledInstallationState \
BuildBusinessTemplate \
CheckBuiltBuildingState \
CheckNotInstalledInstallationState \
CheckObjectPropertiesInBusinessTemplate \
SaveBusinessTemplate \
CheckForkedMigrationExport \
CheckBuiltBuildingState \
CheckNotInstalledInstallationState \
RemoveZodbDocument \
RemoveBusinessTemplate \
RemoveAllTrashBins \
ImportBusinessTemplate \
UseImportBusinessTemplate \
CheckBuiltBuildingState \
CheckNotInstalledInstallationState \
InstallWithoutForceBusinessTemplate \
Tic \
CheckInstalledInstallationState \
CheckBuiltBuildingState \
CheckNoTrashBin \
CheckSkinsLayers \
CheckZodbDocumentExists \
UninstallBusinessTemplate \
CheckBuiltBuildingState \
CheckNotInstalledInstallationState \
CheckDocumentRemoved \
CheckZodbDocumentRemoved \
SaveBusinessTemplate \
CheckForkedMigrationExport \
'
sequence_list.addSequenceString(sequence_string)
sequence_list.play(self)
def test_BusinessTemplateWithDocumentNonExistingBefore(self):
def test_BusinessTemplateWithZodbDocumentNonExistingBefore(self):
sequence_list = SequenceList()
sequence_string = '\
CreateNewBusinessTemplate \
......@@ -6857,10 +6999,10 @@ class TestDocumentTemplateItem(BusinessTemplateMixin):
CheckNoTrashBin \
CheckSkinsLayers \
\
CreateDocument \
CreateZodbDocument \
CreateNewBusinessTemplate \
UseExportBusinessTemplate \
AddDocumentToBusinessTemplate \
AddZodbDocumentToBusinessTemplate \
CheckModifiedBuildingState \
CheckNotInstalledInstallationState \
BuildBusinessTemplate \
......@@ -6868,9 +7010,10 @@ class TestDocumentTemplateItem(BusinessTemplateMixin):
CheckNotInstalledInstallationState \
CheckObjectPropertiesInBusinessTemplate \
SaveBusinessTemplate \
CheckForkedMigrationExport \
CheckBuiltBuildingState \
CheckNotInstalledInstallationState \
RemoveDocument \
RemoveZodbDocument \
RemoveBusinessTemplate \
RemoveAllTrashBins \
ImportBusinessTemplate \
......@@ -6883,28 +7026,70 @@ class TestDocumentTemplateItem(BusinessTemplateMixin):
CheckBuiltBuildingState \
CheckNoTrashBin \
CheckSkinsLayers \
CheckDocumentExists \
CheckZodbDocumentExists \
UninstallBusinessTemplate \
CheckBuiltBuildingState \
CheckNotInstalledInstallationState \
CheckDocumentRemoved \
CheckZodbDocumentRemoved \
'
sequence_list.addSequenceString(sequence_string)
sequence_list.play(self)
def test_BusinessTemplateWithDocumentPropertySheetMigrated(self):
set_template_id_method_name = 'setTemplateDocumentId'
def stepCopyAndMigrateDocumentBusinessTemplate(self, sequence=None, **kw):
"""
Simulate migration from filesystem to ZODB
XXX-arnau: implement importer at BusinessTemplate level
"""
portal = self.getPortalObject()
template_tool = portal.portal_templates
current_bt = sequence['current_bt']
cb_data = template_tool.manage_copyObjects([current_bt.getId()])
copied, = template_tool.manage_pasteObjects(cb_data)
copied_bt = template_tool._getOb(copied['new_id'])
getattr(copied_bt, self.set_template_id_method_name)(sequence['document_id'])
transaction.commit()
sequence.edit(current_bt=copied_bt)
importFromFilesystem = DocumentComponent.importFromFilesystem
def stepImportDocumentFromFilesystem(self, sequence=None, **kw):
"""
Import a Component from Filesystem to ZODB
"""
component_tool = self.getPortalObject().portal_components
document_object = self.importFromFilesystem(
component_tool,
os.path.join(sequence['document_path']))
sequence.edit(document_id=document_object.getId())
def stepCheckDocumentMigration(self, sequence=None, **kw):
"""
Check migration of Document from the Filesystem to ZODB
"""
component_id = sequence['document_id']
component_tool = self.getPortalObject().portal_components
self.failUnless(component_id in component_tool.objectIds())
component = getattr(component_tool, component_id)
self.assertEquals(component.getReference(), sequence['document_title'])
self.assertEquals(component.getTextContent(), sequence['document_data'])
self.assertEquals(component.getPortalType(), self.component_portal_type)
def test_BusinessTemplateWithZodbDocumentMigrated(self):
"""Checks that if Business Template defines Document and PropertySheet
Document is not removed after Property Sheet was migrated and Business Template
was updated"""
sequence_list = SequenceList()
sequence_string = '\
CreateDocument \
CreatePropertySheet \
CheckDocumentPropertySheetSameName \
CreateNewBusinessTemplate \
UseExportBusinessTemplate \
AddDocumentToBusinessTemplate \
AddPropertySheetToBusinessTemplate \
CheckModifiedBuildingState \
CheckNotInstalledInstallationState \
BuildBusinessTemplate \
......@@ -6915,7 +7100,6 @@ class TestDocumentTemplateItem(BusinessTemplateMixin):
CheckBuiltBuildingState \
CheckNotInstalledInstallationState \
RemoveDocument \
RemovePropertySheet \
RemoveBusinessTemplate \
RemoveAllTrashBins \
ImportBusinessTemplate \
......@@ -6928,15 +7112,13 @@ class TestDocumentTemplateItem(BusinessTemplateMixin):
CheckBuiltBuildingState \
CheckNoTrashBin \
CheckDocumentExists \
CheckPropertySheetExists \
\
SimulateAndCopyPrePropertySheetMigrationBusinessTemplate \
ImportDocumentFromFilesystem \
Tic \
\
CreateAllPropertySheetsFromFilesystem \
CheckDocumentRemoved \
CheckDocumentMigration \
CopyAndMigrateDocumentBusinessTemplate \
Tic \
CheckPropertySheetRemoved \
CheckPropertySheetMigration \
\
CheckDraftBuildingState \
CheckNotInstalledInstallationState \
......@@ -6945,12 +7127,11 @@ class TestDocumentTemplateItem(BusinessTemplateMixin):
CheckNotInstalledInstallationState \
CheckObjectPropertiesInBusinessTemplate \
SaveBusinessTemplate \
CheckForkedMigrationExport \
CheckBuiltBuildingState \
CheckNotInstalledInstallationState \
RemoveBusinessTemplate \
Tic \
CreatePropertySheet \
RemovePropertySheetZodbOnly \
RemoveZodbDocument \
Tic \
ImportBusinessTemplate \
UseImportBusinessTemplate \
......@@ -6959,11 +7140,9 @@ class TestDocumentTemplateItem(BusinessTemplateMixin):
InstallWithoutForceBusinessTemplate \
Tic \
\
CheckPropertySheetMigration \
CheckZodbDocumentExists \
CheckInstalledInstallationState \
CheckBuiltBuildingState \
CheckDocumentExists \
CheckPropertySheetRemoved \
'
sequence_list.addSequenceString(sequence_string)
sequence_list.play(self)
......@@ -6992,8 +7171,11 @@ class TestExtensionTemplateItem(TestDocumentTemplateItem):
document_base_path = os.path.join(getConfiguration().instancehome, 'Extensions')
template_property = 'template_extension_id_list'
# Specific to ZODB Extension Component
component_module = ExtensionComponent._getDynamicModuleNamespace()
component_portal_type = ExtensionComponent.portal_type
importFromFilesystem = ExtensionComponent.importFromFilesystem
set_template_id_method_name = 'setTemplateExtensionId'
class TestTestTemplateItem(TestDocumentTemplateItem):
document_title = 'UnitTest'
......
......@@ -270,7 +270,14 @@ class Component(Base):
namespace_dict = {}
exec source_code in namespace_dict
return context.newContent(id=id,
reference=class_name,
text_content=source_code,
portal_type=cls.portal_type)
new_component = context.newContent(id=id,
reference=class_name,
text_content=source_code,
portal_type=cls.portal_type)
new_component.validate()
# XXX-arnau: is it really safe?
os.remove(path)
return new_component
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment