*: replace erp5_site_global_id by a mechanism in test framework

erp5_site_global_id was from the tiolive time, when we were ERP5 with
mariadb and memcached instances shared by many zope instances, this was
used as a way to implement some namespace in memcached, to prevent
conflicts such as two different zope using the same cache keys.

Nowadays, we no longer share memcached, an ERP5 instance has its own
memcached and this prefixing is no longer needed, but there is still one
exception, when we run test using runTestSuite with --node_quantity
higher than 1, then we have multiple running test instances sharing the
same mariadb and the same memcached. In that case, each test instance
uses a different mariadb database, so use this information to use as
a prefix for memcached.
This was more or less the current implementation, but we were setting
erp5_site_global_id on the wrong object, so it was not working since
4889d523 (Define property's default value at class level., 2012-12-31)
and because it was not working we did not realize that using such a
long prefix was not practical, because memcached keys are limited in
size, to overcome this, we don't use the full connection string (which
is around 70 characters), but a short hash of the connection string.

bt5/erp5_configurator_standard/TestTemplateItem/portal_components/test.erp5.testOfficeJSSDKConfigurator.py

@@ -67,11 +67,6 @@ class TestOfficeJSSDKConfigurator(SecurityTestCase):
     # Execute the business configuration if not installed
     business_configuration = self.getBusinessConfiguration()
     if (business_configuration.getSimulationState() != 'installed'):
-      self.portal.portal_caches.erp5_site_global_id = '%s' % random.random()
-      self.portal.portal_caches._p_changed = 1
-      self.commit()
-      self.portal.portal_caches.updateCache()
-
       self.bootstrapSite()
       self.commit()
 

bt5/erp5_user_tutorial/TestTemplateItem/portal_components/test.erp5.testFunctionalStandaloneUserTutorial.py

@@ -91,11 +91,6 @@ class TestZeleniumStandaloneUserTutorial(ERP5TypeFunctionalTestCase):
    # Execute the business configuration if not installed
     business_configuration = self.getBusinessConfiguration()
     if (business_configuration.getSimulationState() != 'installed'):
-      self.portal.portal_caches.erp5_site_global_id = '%s' % random.random()
-      self.portal.portal_caches._p_changed = 1
-      self.commit()
-      self.portal.portal_caches.updateCache()
-
       self.bootstrapSite()
       self.commit()
 

product/ERP5Type/Tool/MemcachedTool.py

@@ -300,9 +300,6 @@ if memcache is not None:
       if memcached_plugin is None:
         raise ValueError('Memcached Plugin does not exists: %r' % (
           plugin_path, ))
-      global_prefix = self.erp5_site_global_id
-      if global_prefix:
-        key_prefix = global_prefix + '_' + key_prefix
       return SharedDict(memcached_plugin.getConnection(), prefix=key_prefix)
 
   InitializeClass(MemcachedTool)

product/ERP5Type/tests/ERP5TypeLiveTestCase.py

@@ -162,6 +162,7 @@ class ERP5TypeLiveTestCase(ERP5TypeTestCaseMixin):
                 .portal_activities.isSubscribed()
         self.portal.portal_activities.unsubscribe()
         self._setUpDummyMailHost()
+        self._setMemcachedKeyPrefix()
 
     setUp = PortalTestCase.setUp
 

product/ERP5Type/tests/ERP5TypeTestCase.py

@@ -7,7 +7,6 @@
 
 __version__ = '0.3.0'
 
-import base64
 import errno
 import os
 import random
@@ -1026,6 +1025,11 @@ class ERP5TypeCommandLineTestCase(ERP5TypeTestCaseMixin):
          use the hooks instead.
       '''
       from Products.CMFActivity.ActivityRuntimeEnvironment import BaseMessage
+      # Activities in unit tests shall never fail.
+      # Let's be a little tolerant for the moment.
+      BaseMessage.max_retry = property(lambda self:
+        self.activity_kw.get('max_retry', 1))
+
       self.__connector_set = set()
       onConnect = self.__onConnect
       self.__original_ZMySQLDA_connect = original_ZMySQLDA_connect = ZMySQLDA_Connection.connect
@@ -1033,10 +1037,8 @@ class ERP5TypeCommandLineTestCase(ERP5TypeTestCaseMixin):
         onConnect(self)
         return original_ZMySQLDA_connect(self, *args, **kw)
       ZMySQLDA_Connection.connect = connect
-      # Activities in unit tests shall never fail.
-      # Let's be a litte tolerant for the moment.
-      BaseMessage.max_retry = property(lambda self:
-        self.activity_kw.get('max_retry', 1))
+
+      self._setMemcachedKeyPrefix()
 
       template_list = list(self.getBusinessTemplateList())
       erp5_catalog_storage = os.environ.get('erp5_catalog_storage',
@@ -1323,10 +1325,6 @@ class ERP5TypeCommandLineTestCase(ERP5TypeTestCaseMixin):
                                        reindex=reindex,
                                        create_activities=create_activities,
                                        **kw)
-              sql = kw.get('erp5_sql_connection_string')
-              if sql:
-                app[portal_name]._setProperty('erp5_site_global_id',
-                                              base64.standard_b64encode(str2bytes(sql)))
               if not quiet:
                 ZopeTestCase._print('done (%.3fs)\n' % (time.time() - _start))
               # Release locks
@@ -1541,6 +1539,7 @@ class ZEOServerTestCase(ERP5TypeTestCase):
       os.write(zeo_client, str2bytes(repr(host_port)))
       os.close(zeo_client)
     ZopeTestCase._print("\nZEO Storage started at %s:%s ... " % host_port)
+    self._setMemcachedKeyPrefix()
 
   def asyncore_loop(self):
     try:

product/ERP5Type/tests/ProcessingNodeTestCase.py

 # -*- coding: utf-8 -*-
 import abc
 import errno, logging, mock, os, socket, time
+import hashlib
 import itertools
 from threading import Thread
 import six
@@ -18,6 +19,7 @@ from ExtensionClass import pmc_init_of
 from Products.ERP5Type.tests.utils import \
   addUserToDeveloperRole, DummyMailHostMixin, parseListeningAddress
 from Products.CMFActivity.ActivityTool import getCurrentNode
+from Products.ERP5Type.Utils import str2bytes
 
 
 class DictPersistentWrapperMetaClass(abc.ABCMeta):
@@ -353,6 +355,7 @@ class ProcessingNodeTestCase(ZopeTestCase.TestCase):
     from Zope2.custom_zodb import cluster
     self._registerNode(distributing=not cluster, processing=1)
     self.commit()
+    self._setMemcachedKeyPrefix()
 
   def _setUpDummyMailHost(self):
     """Replace Original Mail Host by Dummy Mail Host in a non-persistent way
@@ -371,6 +374,22 @@ class ProcessingNodeTestCase(ZopeTestCase.TestCase):
         cls.__bases__ = cls.__bases__[1:]
         pmc_init_of(cls)
 
+  def _setMemcachedKeyPrefix(self):
+    """Inject a prefix to keys used by memcached, so that when we are
+    running multiple runUnitTest process using the same memcached
+    instance they don't have conflicting keys.
+    """
+    from Products.ERP5Type.Tool.MemcachedTool import MemcachedTool
+    erp5_sql_connection_string = os.environ.get('erp5_sql_connection_string')
+    if erp5_sql_connection_string:
+      test_prefix = hashlib.md5(str2bytes(erp5_sql_connection_string)).hexdigest()[:6]
+      original_getMemcachedDict =  MemcachedTool.getMemcachedDict
+      def getMemcachedDict(self, key_prefix, plugin_path):
+        return original_getMemcachedDict(self, test_prefix + key_prefix, plugin_path)
+      patcher = mock.patch.object(MemcachedTool, 'getMemcachedDict', getMemcachedDict)
+      patcher.start()
+      self.addCleanup(patcher.stop)
+
   def processing_node(self):
     """Main loop for nodes that process activities"""
     setRequest(self.app.REQUEST)

product/ERP5Type/tests/runUnitTest.py

@@ -358,6 +358,7 @@ class ERP5TypeTestLoader(unittest.TestLoader):
     # ZODB Test Components requires bootstrap to install BTs before running the
     # actual test
     test_list_len = len(test_list)
+    cleanup = None
     if test_list_len > 0 and ':' in test_list[0]:
       # TODO-arnau: Does anyone specifies multiple test file on command line, at
       # least test bot does not...
@@ -391,6 +392,14 @@ class ERP5TypeTestLoader(unittest.TestLoader):
           self._test_list = test_list
           self._bt_already_installed_list = []
 
+          # so that we can use addCleanup during setUp
+          self._cleanups = []
+          self._outcome = None
+          class ResultForDoCleanup:
+            def addError(self, testcase, exc_info):
+              raise exc_info[1]
+          self._resultForDoCleanups = ResultForDoCleanup()
+
         def getBusinessTemplateList(self):
           """
           Only return the Business Template specifies on the command line, its
@@ -445,9 +454,15 @@ class ERP5TypeTestLoader(unittest.TestLoader):
                        self)._installBusinessTemplateList(url_bt_tuple_list,
                                                           *args, **kwargs)
 
-      _ZodbTestComponentBootstrapOnly(test_list).setUp()
+      zodb_test_component_bootstrap = _ZodbTestComponentBootstrapOnly(test_list)
+      zodb_test_component_bootstrap.setUp()
+      cleanup  = zodb_test_component_bootstrap.doCleanups
 
-    return super(ERP5TypeTestLoader, self).loadTestsFromNames(test_list)
+    try:
+      return super(ERP5TypeTestLoader, self).loadTestsFromNames(test_list)
+    finally:
+      if cleanup is not None:
+        cleanup()
 
   def getTestCaseNames(self, testCaseClass):
     """Return a sorted sequence of method names found within testCaseClass