ZODB Components not following ID=(document|extension|test).VERSION.REFERENCE...

ZODB Components not following ID=(document|extension|test).VERSION.REFERENCE convention are not importable anymore.

Import mechanism now relies on this convention to find the document in
portal_components.

This is necessary to get rid of _registry_dict (ZODB Components equivalent of
document_class_registry), along with several locks. This used to be a mapping
between (reference, version) to ID stored outside of Zope Transactions as a
dict on global erp5.component.XXX modules.

This did not work for long transactions such as installing many bt5s:
  1. Transaction1: Install a bt5 with ZODB Component "Foo".
     => Trigger a reset and _registry_dict is cleared.
  2. Transaction2: TimerServer kicks in.
     => Generate _registry_dict without "Foo".
  3. Transaction1: Install another bt5 using "Foo".
     => "Foo" not in _registry_dict and thus considered not present.

product/ERP5/Document/BusinessTemplate.py

@@ -4234,7 +4234,7 @@ class DocumentTemplateItem(FilesystemToZodbTemplateItem):
 
   @staticmethod
   def _getZodbObjectId(id):
-    return DocumentComponent._getIdPrefix() + '.' + id
+    return DocumentComponent.getIdPrefix() + '.' + id
 
   @staticmethod
   def _getFilesystemPath(class_id):
@@ -4342,7 +4342,7 @@ class ExtensionTemplateItem(DocumentTemplateItem):
 
   @staticmethod
   def _getZodbObjectId(id):
-    return ExtensionComponent._getIdPrefix() + '.' + id
+    return ExtensionComponent.getIdPrefix() + '.' + id
 
   def getTemplateIdList(self):
     return self.getTemplateExtensionIdList()
@@ -4363,7 +4363,7 @@ class TestTemplateItem(DocumentTemplateItem):
 
   @staticmethod
   def _getZodbObjectId(id):
-    return TestComponent._getIdPrefix() + '.' + id
+    return TestComponent.getIdPrefix() + '.' + id
 
   def getTemplateIdList(self):
     return self.getTemplateTestIdList()

product/ERP5/tests/testBusinessTemplate.py

@@ -7349,7 +7349,7 @@ class TestBusinessTemplate(BusinessTemplateMixin):
         Folder._setOb(self, id, object)
     """)
 
-    component_id_prefix = DocumentComponent._getIdPrefix()
+    component_id_prefix = DocumentComponent.getIdPrefix()
     component_portal_type = DocumentComponent.portal_type
     tool_type = 'My Tool'
     tool_class = 'MyTool'
@@ -7859,7 +7859,7 @@ class TestDocumentTemplateItem(BusinessTemplateMixin):
     sequence_list.addSequenceString(sequence_string)
     sequence_list.play(self)
 
-  component_id_prefix = DocumentComponent._getIdPrefix()
+  component_id_prefix = DocumentComponent.getIdPrefix()
   component_portal_type = DocumentComponent.portal_type
 
   def stepCreateZodbDocument(self, sequence=None, **kw):
@@ -8345,7 +8345,7 @@ class TestExtensionTemplateItem(TestDocumentTemplateItem):
   template_property = 'template_extension_id_list'
 
   # Specific to ZODB Extension Component
-  component_id_prefix = ExtensionComponent._getIdPrefix()
+  component_id_prefix = ExtensionComponent.getIdPrefix()
   component_portal_type = ExtensionComponent.portal_type
 
 from Products.ERP5Type.Core.TestComponent import TestComponent
@@ -8364,7 +8364,7 @@ class TestTestTemplateItem(TestDocumentTemplateItem):
   template_property = 'template_test_id_list'
 
   # Specific to ZODB Extension Component
-  component_id_prefix = TestComponent._getIdPrefix()
+  component_id_prefix = TestComponent.getIdPrefix()
   component_portal_type = TestComponent.portal_type
 
   def stepAddTestToBusinessTemplate(self, sequence=None, **kw):

product/ERP5Type/Core/DocumentComponent.py

@@ -60,5 +60,5 @@ class DocumentComponent(ComponentMixin, TextContentHistoryMixin):
     return 'erp5.component.document'
 
   @staticmethod
-  def _getIdPrefix():
+  def getIdPrefix():
     return 'document'

product/ERP5Type/Core/ExtensionComponent.py

@@ -59,5 +59,5 @@ class ExtensionComponent(ComponentMixin, TextContentHistoryMixin):
     return 'erp5.component.extension'
 
   @staticmethod
-  def _getIdPrefix():
+  def getIdPrefix():
     return 'extension'

product/ERP5Type/Core/TestComponent.py

@@ -60,5 +60,5 @@ class TestComponent(ComponentMixin, TextContentHistoryMixin):
     return 'erp5.component.test'
 
   @staticmethod
-  def _getIdPrefix():
+  def getIdPrefix():
     return 'test'

product/ERP5Type/Tool/TypesTool.py

@@ -210,7 +210,23 @@ class TypesTool(TypeProvider):
     document_type_set = set(document_class_registry)
 
     import erp5.component.document
-    document_type_set.update(erp5.component.document._registry_dict)
+    portal = self.getPortalObject()
+    version_priority_set = set(portal.getVersionPriorityNameList())
+
+    # objectValues should not be used for a large number of objects, but
+    # this is only done upon reset, moreover using the Catalog is too risky
+    # as it lags behind and depends upon objects being reindexed
+    for component in portal.portal_components.objectValues(portal_type='Document Component'):
+      # Only consider modified or validated states as state transition will
+      # be handled by component_validation_workflow which will take care of
+      # updating the registry
+      validation_state_tuple = component.getValidationState()
+      if validation_state_tuple in ('modified', 'validated'):
+        version = component.getVersion(validated_only=True)
+        # The versions should have always been set on ERP5Site property
+        # beforehand
+        if version in version_priority_set:
+          document_type_set.add(component.getReference(validated_only=True))
 
     return sorted(document_type_set)
 

product/ERP5Type/dynamic/component_package.py

@@ -39,6 +39,7 @@ from Products.ERP5Type.Globals import get_request
 from . import aq_method_lock
 from types import ModuleType
 from zLOG import LOG, BLATHER, WARNING
+from Acquisition import aq_base
 
 class ComponentVersionPackage(ModuleType):
   """
@@ -73,9 +74,9 @@ class ComponentDynamicPackage(ModuleType):
 
     self._namespace = namespace
     self._namespace_prefix = namespace + '.'
+    self._id_prefix = namespace.rsplit('.', 1)[1]
     self._portal_type = portal_type
     self.__version_suffix_len = len('_version')
-    self.__registry_dict = collections.defaultdict(dict)
     self.__fullname_source_code_dict = {}
 
     # Add this module to sys.path for future imports
@@ -84,48 +85,6 @@ class ComponentDynamicPackage(ModuleType):
     # Add the import hook
     sys.meta_path.append(self)
 
-  @property
-  def _registry_dict(self):
-    """
-    Create the component registry, this is very similar to
-    Products.ERP5Type.document_class_registry and avoids checking whether a
-    Component exists at each call at the expense of being slower when being
-    re-generated after a reset. Moreover, it allows to handle reference
-    easily.
-    """
-    if not self.__registry_dict:
-      portal = getSite()
-
-      try:
-        component_tool = portal.portal_components
-      # When installing ERP5 site, erp5_core_components has not been installed
-      # yet, thus this will obviously failed...
-      #
-      # XXX-arnau: Is this needed as it is now done in synchronizeDynamicModules?
-      except AttributeError:
-        return {}
-
-      version_priority_set = set(portal.getVersionPriorityNameList())
-
-      # objectValues should not be used for a large number of objects, but
-      # this is only done upon reset, moreover using the Catalog is too risky
-      # as it lags behind and depends upon objects being reindexed
-      for component in component_tool.objectValues(portal_type=self._portal_type):
-        # Only consider modified or validated states as state transition will
-        # be handled by component_validation_workflow which will take care of
-        # updating the registry
-        validation_state_tuple = component.getValidationState()
-        if validation_state_tuple in ('modified', 'validated'):
-          version = component.getVersion(validated_only=True)
-          # The versions should have always been set on ERP5Site property
-          # beforehand
-          if version in version_priority_set:
-            reference = component.getReference(validated_only=True)
-            self.__registry_dict[reference][version] = (component.getId(),
-                                                        component._p_oid)
-
-    return self.__registry_dict
-
   def get_source(self, fullname):
     """
     PEP-302 function to get the source code, used mainly by linecache for
@@ -160,12 +119,6 @@ class ComponentDynamicPackage(ModuleType):
     except RuntimeError:
       import_lock_held = False
 
-    # The import lock has been released, but as _registry_dict may be
-    # initialized or cleared, no other Components should access this critical
-    # region
-    #
-    # TODO-arnau: Too coarse-grain?
-    aq_method_lock.acquire()
     try:
       site = getSite()
 
@@ -173,6 +126,7 @@ class ComponentDynamicPackage(ModuleType):
       # erp5.component.XXX.YYY.ZZZ where erp5.component.XXX.YYY is the current
       # Component where an import is done
       name = fullname[len(self._namespace_prefix):]
+      # name=VERSION_version.REFERENCE
       if '.' in name:
         try:
           version, name = name.split('.')
@@ -180,23 +134,38 @@ class ComponentDynamicPackage(ModuleType):
         except ValueError:
           return None
 
-        try:
-          self._registry_dict[name][version]
-        except KeyError:
+        id_ = "%s.%s.%s" % (self._id_prefix, version, name)
+        # aq_base() because this should not go up to ERP5Site and trigger
+        # side-effects, after all this only check for existence...
+        component = getattr(aq_base(site.portal_components), id_, None)
+        if component is None or component.getValidationState() not in ('modified',
+                                                                       'validated'):
           return None
 
       # Skip unavailable components, otherwise Products for example could be
       # wrongly considered as importable and thus the actual filesystem class
       # ignored
-      elif (name not in self._registry_dict and
-            name[:-self.__version_suffix_len] not in site.getVersionPriorityNameList()):
-        return None
+      #
+      # name=VERSION_version
+      elif name.endswith('_version'):
+        if name[:-self.__version_suffix_len] not in site.getVersionPriorityNameList():
+          return None
+
+      # name=REFERENCE
+      else:
+        component_tool = aq_base(site.portal_components)
+        for version in site.getVersionPriorityNameList():
+          id_ = "%s.%s.%s" % (self._id_prefix, version, name)
+          component = getattr(component_tool, id_, None)
+          if component is not None and component.getValidationState() in ('modified',
+                                                                          'validated'):
+            break
+        else:
+          return None
 
       return self
 
     finally:
-      aq_method_lock.release()
-
       # Internal release of import lock at the end of import machinery will
       # fail if the hook is not acquired
       if import_lock_held:
@@ -258,25 +227,17 @@ class ComponentDynamicPackage(ModuleType):
         raise ImportError("%s: should be %s.VERSION.COMPONENT_REFERENCE (%s)" % \
                             (fullname, self._namespace, error))
 
-      try:
-        component_id = self._registry_dict[name][version][0]
-      except KeyError:
-        raise ImportError("%s: version %s of Component %s could not be found" % \
-                            (fullname, version, name))
+      component_id = "%s.%s.%s" % (self._id_prefix, version, name)
 
     # Otherwise, find the Component with the highest version priority
     else:
-      try:
-        component_version_dict = self._registry_dict[name]
-      except KeyError:
-        raise ImportError("%s: Component %s could not be found" % (fullname,
-                                                                   name))
-
+      component_tool = aq_base(site.portal_components)
       # Version priority name list is ordered in descending order
       for version in site.getVersionPriorityNameList():
-        component_id_uid_tuple = component_version_dict.get(version)
-        if component_id_uid_tuple is not None:
-          component_id = component_id_uid_tuple[0]
+        component_id = "%s.%s.%s" % (self._id_prefix, version, name)
+        component = getattr(component_tool, component_id, None)
+        if component is not None and component.getValidationState() in ('modified',
+                                                                        'validated'):
           break
       else:
         raise ImportError("%s: no version of Component %s in Site priority" % \
@@ -433,8 +394,7 @@ class ComponentDynamicPackage(ModuleType):
     if sub_package:
       package = sub_package
     else:
-      # Clear the Component registry and source code dict only once
-      self.__registry_dict.clear()
+      # Clear the source code dict only once
       self.__fullname_source_code_dict.clear()
       package = self
 

product/ERP5Type/interfaces/component.py

@@ -63,7 +63,7 @@ class IComponent(Interface):
     Return the module name where Component module are loaded into
     """
 
-  def _getIdPrefix():
+  def getIdPrefix():
     """
     Return the ID prefix for Component objects
     """

product/ERP5Type/mixin/component.py

@@ -180,14 +180,14 @@ class ComponentMixin(PropertyRecordableMixin, Base):
                                        default='')
     }
 
+  _message_invalid_id = "ID is invalid, should be '${id_prefix}.VERSION.REFERENCE'"
+
   _message_reference_not_set = "Reference must be set"
   _message_invalid_reference = "Reference cannot end with '_version' or "\
       "start with '_' or be equal to find_module, load_module or reset"
 
   _message_version_not_set = "Version must be set"
   _message_invalid_version = "Version cannot start with '_'"
-  _message_duplicated_version_reference = "${id} is validated has the same "\
-       "Reference and Version"
 
   _message_text_content_not_set = "No source code"
   _message_text_content_error = "Error in Source Code: ${error_message}"
@@ -216,10 +216,27 @@ class ComponentMixin(PropertyRecordableMixin, Base):
     """
     error_list = super(ComponentMixin, self).checkConsistency(*args, **kw)
     object_relative_url = self.getRelativeUrl()
+
+    is_id_invalid = False
+    try:
+      prefix, version, reference = self.getId().split('.')
+    except ValueError:
+      is_id_invalid = True
+    else:
+      if (prefix != self.getIdPrefix() or
+          version != self.getVersion() or
+          reference != self.getReference()):
+        is_id_invalid = True
+
+    if is_id_invalid:
+      error_list.append(
+        ConsistencyMessage(self,
+                           object_relative_url,
+                           message=self._message_invalid_id,
+                           mapping={'id_prefix': self.getIdPrefix()}))
+
     reference = self.getReference()
-    reference_has_error = False
     if not reference:
-      reference_has_error = True
       error_list.append(
         ConsistencyMessage(self,
                            object_relative_url,
@@ -229,7 +246,6 @@ class ComponentMixin(PropertyRecordableMixin, Base):
     elif (reference.endswith('_version') or
           reference[0] == '_' or
           reference in ('find_module', 'load_module', 'reset')):
-      reference_has_error = True
       error_list.append(
         ConsistencyMessage(self,
                            object_relative_url,
@@ -247,26 +263,6 @@ class ComponentMixin(PropertyRecordableMixin, Base):
                                            object_relative_url,
                                            message=self._message_invalid_version,
                                            mapping={}))
-    else:
-      package = __import__(self._getDynamicModuleNamespace(), globals(),
-                           fromlist=[self._getDynamicModuleNamespace()], level=0)
-      component_id = None
-      component_uid = None
-      from Products.ERP5Type.dynamic import aq_method_lock
-      with aq_method_lock:
-        component_id_uid_tuple = package._registry_dict.get(
-          self.getReference(), {}).get(self.getVersion(), None)
-        if component_id_uid_tuple:
-          component_id, component_uid = component_id_uid_tuple
-
-      if (component_id is not None and component_uid is not None and
-          not reference_has_error and
-          component_uid != self._p_oid and component_id != self.getId()):
-        error_list.append(
-          ConsistencyMessage(self,
-                             object_relative_url,
-                             message=self._message_duplicated_version_reference,
-                             mapping=dict(id=component_id)))
 
     text_content = self.getTextContent()
     if not text_content:
@@ -360,7 +356,7 @@ class ComponentMixin(PropertyRecordableMixin, Base):
     # needed when importing from filesystem, moreover errors may occur
     # if in the same transaction a Component is created and another
     # one depending upon the former...
-    object_id = '%s.%s.%s' % (cls._getIdPrefix(), version, reference)
+    object_id = '%s.%s.%s' % (cls.getIdPrefix(), version, reference)
     new_component = context.newContent(id=object_id,
                                        reference=reference,
                                        version=version,