Add scalability test support

component/cloudooo/buildout.cfg

@@ -17,7 +17,6 @@ command = grep parts ${buildout:develop-eggs-directory}/cloudooo.egg-link
 [cloudooo-repository]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/cloudooo.git
-branch = master
 git-executable = ${git:location}/bin/git
 
 [cloudooo]

setup.py

@@ -108,6 +108,7 @@ setup(name=name,
           'erp5.update = slapos.recipe.erp5_update:Recipe',
           'erp5scalabilitytestbed = slapos.recipe.erp5scalabilitytestbed:Recipe',
           'erp5testnode = slapos.recipe.erp5testnode:Recipe',
+          'scalability = slapos.recipe.scalability:Recipe',
           'firefox = slapos.recipe.firefox:Recipe',
           'fontconfig = slapos.recipe.fontconfig:Recipe',
           'generate.mac = slapos.recipe.generatemac:Recipe',

slapos/recipe/erp5_bootstrap/__init__.py

@@ -33,10 +33,61 @@ import urlparse
 class Recipe(GenericBaseRecipe):
   """
   Instanciate ERP5 in Zope
+
+  Input:
+    mysql-url
+      mysql url, must contain connexion informations.
+
+    zope-url (optional)
+      Url of zope, this url will be parsed to get these values:
+      protocol://username@password:hostname:port/site_id
+
+    If specified following inputs have priority on parsed zope-url values:
+    zope-hostname (optional)
+      Zope hostname.
+      Default value: "localhost", if not contained in zope-url.
+    zope-port (optional)
+      Zope port.
+      Default value: "80", if not contained in zope-url.
+    zope-protocol (optional)
+      Protocol used.
+      Default "http" if not contained in zope-url.
+    zope-password (optional)
+      Zope user password.
+      Default value: "insecure", if not contained in zope-url.
+    zope-username (optional)
+      Zope username.
+      Default value: "zope", if not contained in zope-url.
+    site-id (optional)
+      Site id
+      ex: erp5
+      Default value: "erp5", if not contained in zope-url.
+
+    runner-path
+      The path to create the runner.
+
+    scalability (optional)
+      Boolean, default value: "False"
+      If true erp5 site will fix the site consistency and
+      will use configurator to automaticlly install a
+      small and medium buisiness.
   """
 
   def install(self):
-    parsed = urlparse.urlparse(self.options['mysql-url'])
+    
+    # Get mysql url
+    mysql_url = None
+    if self.options.get('mysql-url'):
+      mysql_url = self.options['mysql-url']
+    else:
+      # Get url of the first database of the list
+      mysql_url_list = self.options['mysql-url-list']
+      # Parse string to list
+      if not isinstance(mysql_url_list, list):
+        mysql_url_list = mysql_url_list[1:len(mysql_url_list)-1].split(',')
+      mysql_url = mysql_url_list[0]
+      
+    parsed = urlparse.urlparse(mysql_url)
     mysql_connection_string = "%(database)s@%(hostname)s:%(port)s "\
         "%(username)s %(password)s" % dict(
       database=parsed.path.split('/')[1],
@@ -46,15 +97,70 @@ class Recipe(GenericBaseRecipe):
       password=parsed.password
     )
 
-    zope_parsed = urlparse.urlparse(self.options['zope-url'])
+    # Init zope configuration
+    zope_username = None
+    zope_password = None
+    zope_hostname = None
+    zope_port = None
+    zope_protocol = None
+
+    # Get informations from zope url
+    if self.options.get('zope-url'):
+      zope_parsed = urlparse.urlparse(self.options['zope-url'])
+    # Zope hostname
+    if self.options.get('zope-hostname'):
+      zope_hostname = self.options['zope-hostname']
+    elif self.options.get('zope-url'):
+      zope_hostname = zope_parsed.hostname
+    else:
+      zope_hostname = 'localhost'
+    # Zope port
+    if self.options.get('zope-port'):
+      zope_port = self.options['zope-port']
+    elif self.options.get('zope-url'):
+      zope_port = str(zope_parsed.port)
+    else:
+      zope_port = "8080"
+    # Zope username and password
+    if self.options.get('zope-username') and self.options.get('zope-password'):
+      zope_username = self.options['zope-username']
+      zope_password = self.options['zope-password']
+    elif self.options.get('zope-url'):
+      zope_username = zope_parsed.username
+      zope_password = zope_parsed.password
+    else:
+      zope_username = 'zope'
+      zope_password = 'insecure'
+    # Zope protocol
+    if self.options.get('zope-protocol'):
+      zope_protocol = self.options['zope-protocol']
+    elif self.options.get('zope-url'):
+      zope_protocol = zope_parsed.scheme      
+    else:
+      zope_protocol = 'http'
+    # Zope site-id
+    if self.options.get('zope-site-id'):
+      zope_site_id = self.options['zope-site-id']
+    elif self.options.get('zope-url'):
+      zope_site_id = zope_parsed.path.split('/')[1],
+    else:
+      zope_site_id = 'erp5'
+
+    # Scalability
+    if self.options.get('scalability'):
+      scalability = self.options['scalability']
+    else:
+      scalability = "False"
 
     config = dict(
       python_path=sys.executable,
-      user=zope_parsed.username,
-      password=zope_parsed.password,
-      site_id=zope_parsed.path.split('/')[1],
-      host="%s:%s" % (zope_parsed.hostname, zope_parsed.port),
+      user=zope_username,
+      password=zope_password,
+      site_id=zope_site_id,
+      host="%s:%s" % (zope_hostname, str(zope_port)),
+      protocol=zope_protocol,
       sql_connection_string=mysql_connection_string,
+      scalability="True",
     )
 
     # Runners
@@ -63,4 +169,4 @@ class Recipe(GenericBaseRecipe):
       self.substituteTemplate(self.getTemplateFilename('erp5_bootstrap.in'),
                               config))
 
-    return [runner_path]
+    return [runner_path]
\ No newline at end of file

slapos/recipe/erp5_bootstrap/template/erp5_bootstrap.in

@@ -3,43 +3,242 @@
 import httplib
 import urllib
 import base64
+import time
+
+MAX_INSTALLATION_TIME = 60*30
+MAX_TESTING_TIME = 60*2
+MAX_GETTING_CONNECTION_TIME = 60*5
 
 user = "%(user)s"
 password = "%(password)s"
 host = "%(host)s"
 site_id = "%(site_id)s"
-erp5_catalog_storage = 'erp5_mysql_innodb_catalog'
 mysql_url = "%(sql_connection_string)s"
-
+protocol = "%(protocol)s"
+scalability = "%(scalability)s" == "True"
+erp5_catalog_storage = 'erp5_mysql_innodb_catalog'
 header_dict = {'Authorization': 'Basic %%s' %% \
   base64.encodestring('%%s:%%s' %% (user, password)).strip()}
-zope_connection = httplib.HTTPConnection(host)
-
-# Check if an ERP5 site is already created, as ERP5 does support having
-# 2 instances in the same zope, and this script should not destroy user data
-zope_connection.request('GET', '/isERP5SitePresent', headers=header_dict)
-result = zope_connection.getresponse()
-
-if result.status == 204: # and (result.read() == "False"):
-
-  # Use a new connection
-  zope_connection = httplib.HTTPConnection(host)
-
-  # Create the expected ERP5 instance
-  zope_connection.request(
-    'POST', '/manage_addProduct/ERP5/manage_addERP5Site',
-    urllib.urlencode({
-      'id': site_id,
-      'erp5_catalog_storage': erp5_catalog_storage,
-      'erp5_sql_connection_string': mysql_url,
-      'cmf_activity_sql_connection_string': mysql_url,
-    }),
-    headers=header_dict)
-  # Wait for the erp5 response, to prevent multiple requests
-  # been done by the same script.
+
+def getConnection():
+  print "Getting new connection"
+  start_time = time.time()
+  count = 0
+  while MAX_GETTING_CONNECTION_TIME > time.time()-start_time:
+    try:
+      count = count + 1
+      if protocol == 'https':
+        return httplib.HTTPSConnection(host)
+      elif protocol == 'http':
+        return httplib.HTTPConnection(host)
+      else:
+        raise ValueError("Protocol not implemented")
+    except:
+      print "Getting new connection failed, retry"
+      time.sleep(10)
+  raise ValueError("Cannot get new connection after %%d try (for %%s s)" %%(count, str(time.time()-start_time)))
+
+def testIfExist(page, unexcepted_content="Site Error"):
+  print "Test if %%s exists" %%(page)
+  start_time = time.time()
+  count = 0
+  while MAX_TESTING_TIME > time.time()-start_time:
+    try:
+      count = count + 1
+      zope_connection = getConnection()
+      zope_connection.request('GET', '/%%s/%%s' %%(site_id, page))
+      result = zope_connection.getresponse()
+      return not unexcepted_content in result.read()
+    except:
+      print "Test if exists failed, retry..."
+      time.sleep(10)
+  raise ValueError("Cannot testIfExist after %%d try (for %%s s)" %%(count, str(time.time()-start_time)))
+  
+  
+def waitFor0PendingActivities():
+  # TODO: tolerate 1 pending activities ? (mail server...)
+  print "waitFor0PendingActivities.."
+  start_time = time.time()
+  count = 0
+  ok = False
+  while MAX_INSTALLATION_TIME > time.time()-start_time and not ok:
+    try:
+      count = count + 1
+      zope_connection = getConnection()
+      zope_connection.request(
+        'GET', '/%%s/portal_activities/getMessageList' %%(site_id),
+        headers=header_dict
+      )
+      result = zope_connection.getresponse()
+      message_list_text = result.read()
+      message_list = [s.strip() for s in message_list_text[1:-1].split(',')]
+      
+      if len(message_list)==0:
+        print "There is no pending activities."
+        ok = True
+      #Hack to do not take into account persistent Alarm_installMailServer acitivities
+      if len(message_list)==1:
+        print "1 pending activity for 'Alarm_installMailServer'."
+        print "ok."
+        ok = True
+        
+      print "There is %%d pending activities" %%len(message_list)
+      time.sleep(5)
+        
+    except:
+      time.sleep(5)
+      print "Getting activities failed, retry."
+  
+  if not ok:
+    raise ValueError("Pending activities always here after %%d check/try (for %%s s)" %%(count, str(time.time()-start_time)))
+
+
+  
+
+def createERP5Site():
+  # Check if an ERP5 site is already created, as ERP5 does support having
+  # 2 instances in the same zope, and this script should not destroy user data
+  zope_connection = getConnection()
+  zope_connection.request('GET', '/isERP5SitePresent', headers=header_dict)
   result = zope_connection.getresponse()
 
-  # Read result make sure the site really finished to 
-  #created the ERP5 site.
-  result.read()
-  print "ERP5 site created."
+  if result.status == 204: # and (result.read() == "False"):
+    # Use a new connection
+    zope_connection = getConnection()
+    # Create the expected ERP5 instance
+    zope_connection.request(
+      'POST', '/manage_addProduct/ERP5/manage_addERP5Site',
+      urllib.urlencode({
+        'id': site_id,
+        'erp5_catalog_storage': erp5_catalog_storage,
+        'erp5_sql_connection_string': mysql_url,
+        'cmf_activity_sql_connection_string': mysql_url,
+      }),
+      headers=header_dict)
+    # Wait for the erp5 response, to prevent multiple requests
+    # been done by the same script.
+    result = zope_connection.getresponse()
+
+    # Read result make sure the site really finished to
+    # created the ERP5 site.
+    print result.read()
+    print "ERP5 site created."
+
+
+createERP5Site()
+
+# Scalability case
+if scalability:
+  
+  while True:
+    count = 0
+
+    createERP5Site()
+    
+    # Fix site consistency
+    if not testIfExist("/%%s/portal_configurator/" %%site_id):
+      count = count + 1
+      print "Going to fix site consistency..."
+      zope_connection = getConnection()
+      zope_connection.request(
+        'GET', '/%%s/ERP5Site_launchFixConfigurationConsistency' %%(site_id),
+        headers=header_dict
+      )
+      result = zope_connection.getresponse()
+      print "Check consistency..."
+      print result.read()
+      # Wait activities
+      waitFor0PendingActivities()
+      print "Site consistency checking: done."
+
+    # Install testing scalability business configuration if not exists
+    if testIfExist("/%%s/portal_configurator/" %%site_id) \
+       and not testIfExist("/%%s/sale_order_module/" %%site_id):
+      count = count + 1
+      print "Going to install testing scalability business configuration.."
+      zope_connection = getConnection()
+      zope_connection.request(
+        'GET', '/%%s/business_configuration_module/1/build' %%site_id,
+        headers=header_dict
+      )
+      result = zope_connection.getresponse()
+      print "Build scalability business configuration..."
+      print result.read()
+      # Wait activities
+      waitFor0PendingActivities()
+      print "Scalability business configuration building: done."
+
+    # Force to use random id generator
+    print "Force to use random id.."
+    try:
+      zope_connection = getConnection()
+      zope_connection.request(
+        'GET', '/%%s/use_random_id_scalability' %%site_id,
+        headers=header_dict
+      )
+      result = zope_connection.getresponse()
+      assert(result.read()=="1")
+      # Wait activities
+      waitFor0PendingActivities()
+      print "use_random_id_scalability: done."
+    except:
+      print "use_random_id_scalability: fail."
+
+    # Create scalability users
+    if testIfExist("/%%s/person_module/scalability_user/getTitle" %%site_id) \
+       and not testIfExist("/%%s/person_module/scalability_user_0/getTitle" %%site_id):
+      count = count + 1
+      print "Going to create scalability users.."
+      zope_connection = getConnection()
+      zope_connection.request(
+        'GET', '/%%s/init_scalability' %%site_id,
+        headers=header_dict
+      )
+      result = zope_connection.getresponse()
+      print "Creating scalaiblity users..."
+      assert(result.read()=="1")
+      # Wait activities
+      waitFor0PendingActivities()
+      print "Scalability users creation: done."
+
+    # Try to Updates roles and sleep
+    if count == 0:
+      print "Trying to update roles.."
+      try:
+        zope_connection = getConnection()
+        zope_connection.request(
+          'GET', '/%%s/update_roles_scalability' %%site_id,
+          headers=header_dict
+        )
+        result = zope_connection.getresponse()
+        assert(result.read()=="1")
+        # Wait activities
+        waitFor0PendingActivities()
+        print "Update roles: done."
+      except:
+        print "Update roles: fail. (but may be already done before)"
+      time.sleep(30)
+      
+    # XXX: Hack to perform load balacing
+    # TODO: Use an other way to do the load balancing (for example
+    # using zope.conf, and using family-zope group names).
+    # Apply load balacing
+    if testIfExist("/%%s/portal_activities/manageLoadBalancing" %%site_id) \
+        and testIfExist("/%%s/person_module/scalability_user_0/getTitle" %%site_id):
+      print "Load balacing.."
+      try:
+        zope_connection = getConnection()
+        zope_connection.request(
+          'GET', '/%%s/apply_load_balancing_scalability' %%site_id,
+          headers=header_dict
+        )
+        result = zope_connection.getresponse()
+        assert("Processing Nodes" in result.read())
+        # Wait activities
+        waitFor0PendingActivities()
+        print "Load balancing init: done."
+      except:
+        print "Load balancing init: fail."
+
+    # sleep
+    time.sleep(30)
\ No newline at end of file

slapos/recipe/scalability/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+import os
+
+from slapos.recipe.librecipe import GenericBaseRecipe
+
+class Recipe(GenericBaseRecipe):
+  def install(self):
+    path_list = []
+    configuration_file = self.createFile(
+      self.options['configuration-file'],
+      '',
+    )
+    path_list.append(configuration_file)
+    path_list.append(
+      self.createPythonScript(
+        self.options['wrapper'],
+        'slapos.recipe.librecipe.execute.executee',
+        [ # Executable
+          [ self.options['exec-path'],
+            '--erp5-url', self.options['erp5-url'],
+            '--test-result-path', self.options['test-result-path'],
+            '--test-suite-master-url', self.options['test-suite-master-url'],
+            '--node-title', self.options['node-title'],
+            '--test-suite', self.options['test-suite'],
+            '--revision', self.options['revision'],
+            '--log-path', self.options['log-path'],
+            '--runner-path', self.options['runner-path'],
+            '--erp5-location', self.options['erp5-location'],
+          ],
+          # Environment
+          {
+            'GIT_SSL_NO_VERIFY': '1',
+          }
+        ],
+      )
+    )
+    return path_list

software/erp5-util-testing/instance.cfg

 [buildout]
 parts =
+  scalability-runner
   test-runner
   sh-environment
 
@@ -24,6 +25,18 @@ git-executable = ${git:location}/bin/git
 <= download-source
 repository = ${erp5-util-repository:location}
 
+[scalability-runner]
+recipe = slapos.cookbook:egg_test
+run-scalability-test-suite = $${create-directory:bin}/runScalabilityTestSuite
+run-scalability-test-suite-binary = $${buildout:bin-directory}/runScalabilityTestSuite
+# The list of executables should be defined here and a combination
+# of tests should dynamically generated.
+#python-list = $${}
+test-list =
+  $${erp5-util:location}
+prepend-path = ${git:location}/bin:${libxslt:location}/bin:${python2.7:location}/bin
+environment = environment
+
 [test-runner]
 recipe = slapos.cookbook:egg_test
 run-test-suite = $${create-directory:bin}/runTestSuite

software/erp5-util-testing/software.cfg

@@ -45,7 +45,7 @@ branch = master
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
-md5sum = 6b919dd280d6972afe0f50d389ba5fe6
+md5sum = 4449ebd6037fd248dd70a0ff7a6d6805
 output =  ${buildout:directory}/template.cfg
 mode = 640
 

stack/erp5/buildout.cfg

@@ -3,6 +3,7 @@
 develop =
   ${:parts-directory}/slapos.cookbook-repository
   ${:parts-directory}/cloudooo-repository
+  ${:parts-directory}/erp5
 
 extensions =
   slapos.zcbworkarounds
@@ -14,6 +15,8 @@ find-links =
     http://dist.repoze.org
     http://www.nexedi.org/static/packages/source/
     http://www.owlfish.com/software/wsgiutils/download.html
+    https://pypi.python.org/packages/source/z/z3c.etestbrowser/
+    https://pypi.python.org/packages/source/z/zope.testbrowser/
 
 allow-hosts += pybrary.net
 
@@ -104,6 +107,8 @@ parts =
   eggs
   testrunner
   test_suite_runner
+  scalability_test_suite_runner
+  performance_tester
 
 # basic Xorg
   libXdmcp
@@ -133,6 +138,9 @@ parts =
 
 # Local development
   slapos.cookbook-repository
+  erp5.util
+  slapos.cookbook
+  erp5.util-check
   check-recipe
 # Create instance template
   template
@@ -142,12 +150,33 @@ recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:filename}
 mode = 644
 
+[erp5.util-check]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = grep parts ${buildout:develop-eggs-directory}/erp5.util.egg-link
+
+[erp5.util]
+recipe = zc.recipe.egg
+eggs = erp5.util
+scripts =
+ugly-depend-on = ${erp5:repository}
+
 # Local development
 [slapos.cookbook-repository]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/slapos.git
-branch = erp5-cluster
+#branch = erp5-cluster
 git-executable = ${git:location}/bin/git
+#repository = http://192.168.242.77:1236/slapos.git
+
+branch = erp5-cluster-scalability
+
+[slapos.cookbook]
+recipe = zc.recipe.egg
+eggs = slapos.cookbook
+scripts =
+ugly-depend-on = ${slapos.cookbook-repository:repository} ${slapos.cookbook-repository:branch}
 
 [check-recipe]
 recipe = plone.recipe.command
@@ -166,6 +195,8 @@ context =
     key bin_directory buildout:bin-directory
     key develop_eggs_directory buildout:develop-eggs-directory
     key eggs_directory buildout:eggs-directory
+    key erp5_location erp5:location
+    key erp5_dev_location erp5:location
     ${:extra-context}
 
 [template-mariadb]
@@ -213,7 +244,7 @@ md5sum = 564006953b7d7a12d40a14b6648b32f0
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 filename = template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = f71e14621d90903eba1b943401548b8a
+md5sum = 3d5a16f831a3b73f9c0e2aae7abbd811
 extra-context =
     key mariadb_link_binary template-mariadb:link-binary
     key zope_link_binary template-zope:link-binary
@@ -224,7 +255,6 @@ extra-context =
     key curl_location curl:location
     key dash_location dash:location
     key dcron_location dcron:location
-    key erp5_location erp5:location
     key file_location file:location
     key findutils_location findutils:location
     key fontconfig_location fontconfig:location
@@ -264,13 +294,14 @@ extra-context =
     key template_zeo template-zeo:target
     key template_zope template-zope:target
     key template_zope_conf template-zope-conf:target
+    key template_scalability template-scalability:target
     key wget_location wget:location
     key zlib_location zlib:location
 
 [template-erp5]
 < = download-base
 filename = instance-erp5.cfg.in
-md5sum = 5b11875e6beba48db7d45c5d462a6d2d
+md5sum = 0620af0820d202dedcf73334365a44bb
 
 [template-neo]
 < = download-base
@@ -328,6 +359,11 @@ extra-context =
     key gzip_location gzip:location
     key logrotate_location logrotate:location
 
+[template-scalability]
+< = download-base
+filename = instance-scalability.cfg.in
+md5sum = d45b0b8e2fdb7fc11a5f5e4720a66879
+
 [bt5-repository]
 # Format:
 #   <url or path> [...]
@@ -355,7 +391,8 @@ repository_id_list = erp5
 [erp5]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/erp5.git
-branch = master
+#repository = http://192.168.242.77:1235/erp5.git
+#branch = master
 git-executable = ${git:location}/bin/git
 
 [fix-products-paths]
@@ -404,16 +441,7 @@ initialization =
   os.environ['PATH'] = '${jsl:location}/bin:%s' % os.environ['PATH']
   os.environ['CGI_PATH'] = '${w3-validator:location}/httpd/cgi-bin'
 
-[test_suite_runner]
-# XXX: Workaround for fact ERP5Type is not an distribution and does not
-#      expose entry point for test runner
-recipe = zc.recipe.egg
-python = python2.7
-eggs = ${eggs:eggs}
-extra-paths = ${eggs:extra-paths}
-entry-points =
-  runTestSuite=Products.ERP5Type.tests.runTestSuite:main
-scripts = runTestSuite
+[erp5-bin-base]
 initialization =
   import os
   import sys
@@ -425,6 +453,36 @@ initialization =
   repository_id_list = list(reversed('''${erp5_repository_list:repository_id_list}'''.split()))
   sys.path[0:0] = ['/'.join(['''${buildout:parts-directory}''', x]) for x in repository_id_list]
 
+[test_suite_runner]
+< = erp5-bin-base
+# XXX: Workaround for fact ERP5Type is not an distribution and does not
+#      expose entry point for test runner
+recipe = zc.recipe.egg
+python = python2.7
+eggs = ${eggs:eggs}
+extra-paths = ${eggs:extra-paths}
+entry-points =
+  runTestSuite=Products.ERP5Type.tests.runTestSuite:main
+scripts = runTestSuite
+
+[scalability_test_suite_runner]
+< = erp5-bin-base
+recipe = zc.recipe.egg
+eggs = ${eggs:eggs}
+extra-paths = ${eggs:extra-paths}
+entry-points =
+  runScalabilityTestSuite=erp5.util.scalability.runScalabilityTestSuite:main
+scripts = runScalabilityTestSuite
+  
+[performance_tester]
+< = erp5-bin-base
+recipe = zc.recipe.egg
+eggs = ${eggs:eggs}
+extra-paths = ${eggs:extra-paths}
+entry-points =
+  performance_tester=erp5.util.benchmark.performance_tester:main
+scripts = performance_tester
+
 [eggs]
 recipe = zc.recipe.egg
 python = python2.7
@@ -465,7 +523,8 @@ eggs =
   feedparser
   argparse
   validictory
-  erp5.util
+# erp5.util doesn't have to be here because is it developed (at part/erp5)
+#  erp5.util
   huBarcode
   qrcode
   spyne
@@ -509,7 +568,12 @@ eggs =
 
 # Needed for parsing .po files from our Localizer subset
   polib
-
+# performance_tester dependencies
+  z3c.etestbrowser
+  zope.testbrowser
+# runScalabilityTestSuite dependencie
+  slapos.core
+  psutil
 # parameterizing the version of the generated python interpreter name by the
 # python section version causes dependency between this egg section and the
 # installation of python, which we don't want on an instance
@@ -536,7 +600,6 @@ entry-points =
 scripts = zodbanalyze
 
 [cloudooo-repository]
-branch = master
 revision = 5c67568c403239bd8e25993602d03c553236fcec
 
 [mysql-python]
@@ -568,6 +631,10 @@ scripts =
   zodbpack
 
 [versions]
+z3c.etestbrowser = 2.0.0
+zope.testbrowser =
+slapos.core = 0.32.3
+psutil = 
 # pin Acquisition and Products.DCWorkflow to Nexedi flavour of eggs
 Acquisition = 2.13.7nxd001
 Products.DCWorkflow = 2.2.3nxd002
@@ -658,7 +725,7 @@ coverage = 3.6
 csp-eventlet = 0.7.0
 elementtree = 1.2.7-20070827-preview
 erp5.recipe.cmmiforcei686 = 0.1.3
-erp5.util = 0.4.34
+erp5.util =
 erp5diff = 0.8.1.5
 eventlet = 0.12.1
 feedparser = 5.1.3

stack/erp5/instance-erp5.cfg.in

@@ -3,6 +3,14 @@
 {% set frontend_dict = slapparameter_dict.get('frontend', {}) %}
 {% set has_frontend = frontend_dict.get('software-url', '') != '' -%}
 {% set site_id = slapparameter_dict.get('site-id', 'erp5') -%}
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
 [request-common]
 recipe = slapos.cookbook:request.serialised
 software-url = ${slap-connection:software-release-url}
@@ -17,6 +25,7 @@ config =
   ${:extra-config}
 extra-config =
 config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
+state = ${slap-configuration:instance-state}
 
 {% macro request(name, software_type, config_key, config={}, ret={'url': True}) -%}
 {% do config.update(slapparameter_dict.get(config_key, {})) -%}

stack/erp5/instance-scalability.cfg.in

+{% if slap_software_type == software_type -%}
+{% set site_id = slapparameter_dict.get('cluster').get('site-id', 'erp5') -%}
+{% set promise_path = slapparameter_dict.get('promise-path', 'erp5') -%}
+[buildout]
+extends = 
+parts =
+  erp5-cluster
+  binary-wrap-launcher
+  binary-wrap-performance-tester
+  erp5-bootstrap
+  erp5-promise
+  promise-erp5-site
+  promise-scalability-user-data
+  scalability-launcher-instance
+  publish
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+# Request erp5-cluster instance
+[erp5-cluster]
+recipe = slapos.cookbook:request.serialised
+sla = computer_guid
+server-url = ${slap-connection:server-url}
+key-file = ${slap-connection:key-file}
+cert-file = ${slap-connection:cert-file}
+computer-id = ${slap-connection:computer-id}
+partition-id = ${slap-connection:partition-id}
+name = ERP5 Cluster
+software-url = ${slap-connection:software-release-url}
+software-type = default
+sla-computer_guid = {{ slapparameter_dict.get('launcher-computer-guid', computer_id) }}
+{# Add a zope scalability family to the erp5 cluster configuration, -#}
+{# this zope family is used by erp5-bootstrap and may serve to scalability monitoring tools. -#}
+{% set additional_scalability_zope = {'scalability':{'family':'scalability', 'computer-guid':computer_id}} -%}
+{% do slapparameter_dict.get('cluster').get('zope-partition-dict').update( additional_scalability_zope ) -%}
+{% set cluster_parameter_dict = slapparameter_dict.get('cluster', {}) -%}
+config = {{ cluster_parameter_dict.keys() | join(' ') }}
+{% for key, value in cluster_parameter_dict.items() -%}
+config-{{ key }} = {{ dumps(value) }}
+{% endfor -%}
+return = mariadb-database-list family-scalability family-user memcached-persistent-url memcached-volatile-url cloudooo-url
+state = ${slap-configuration:instance-state}
+
+# Create wrapper
+[binary-wrap-launcher]
+recipe = slapos.cookbook:wrapper
+log-path = ${basedirectory:log}
+binary-path = {{ bin_directory }}/runScalabilityTestSuite
+wrapper-path = ${rootdirectory:bin}/runScalabilityTestSuite
+output = ${binary-wrap-launcher:binary-path}
+site-id = "/{{ site_id }}"
+# Suppose that there is a user zope family in the configuration
+erp5-url = ${erp5-cluster:connection-family-user}${:site-id}
+parameters-extra = $*
+command-line = ${binary-wrap-launcher:binary-path}
+return = url
+
+# Create wrapper
+[binary-wrap-performance-tester]
+recipe = slapos.cookbook:wrapper
+binary-path = {{ bin_directory }}/performance_tester
+wrapper-path = ${rootdirectory:bin}/performance_tester
+output = ${binary-wrap-performance-tester:binary-path}
+parameters-extra = $*
+command-line = ${binary-wrap-performance-tester:binary-path}
+return = url
+
+[scalability-launcher-instance]
+recipe = slapos.cookbook:scalability
+configuration-file = ${rootdirectory:etc}/scalability_runner.conf
+wrapper = ${basedirectory:services}/scalability_runner
+exec-path = ${binary-wrap-launcher:wrapper-path}
+erp5-url = ${binary-wrap-launcher:erp5-url}
+test-result-path = {{ slapparameter_dict.get('test-result-path') }}
+revision = {{ slapparameter_dict.get('test-suite-revision') }}
+node-title = {{ slapparameter_dict.get('node-title', 'Unknown') }}
+test-suite = {{ slapparameter_dict.get('test-suite') }}
+test-suite-master-url = {{ slapparameter_dict.get('test-suite-master-url') }}
+log-path = ${binary-wrap-launcher:log-path}
+runner-path = ${binary-wrap-performance-tester:wrapper-path}
+erp5-location = {{ erp5_location }}
+
+# Create partition's directories
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = ${rootdirectory:var}/log
+services = ${rootdirectory:etc}/run
+promises = ${rootdirectory:etc}/promise/
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+var = ${buildout:directory}/var
+bin = ${buildout:directory}/bin
+
+[erp5-bootstrap]
+recipe = slapos.cookbook:erp5.bootstrap
+runner-path = ${basedirectory:services}/erp5-bootstrap
+#mysql-url =  ${erp5-cluster:mariadb-url}
+mysql-url-list =  ${erp5-cluster:connection-mariadb-database-list}
+zope-username = zope
+zope-password = insecure
+zope-site-id = {{ site_id }}
+zope-url = ${erp5-cluster:connection-family-scalability}
+scalability = True
+
+[erp5-promise]
+recipe = slapos.cookbook:erp5.promise
+promise-path = {{ promise_path }}
+kumofs-url = ${erp5-cluster:connection-memcached-persistent-url}
+memcached-url = ${erp5-cluster:connection-memcached-volatile-url}
+cloudooo-url = ${erp5-cluster:connection-cloudooo-url}
+smtp-url =
+bt5 =
+bt5-repository-url =
+
+[promise-erp5-site]
+recipe = slapos.cookbook:check_url_available
+path = ${basedirectory:promises}/erp5site
+site-id = "/{{ site_id }}"
+url = ${erp5-cluster:connection-family-scalability}${:site-id}
+dash_path = {{ dash_location }}/bin/dash
+curl_path = {{ curl_location }}/bin/curl
+
+[promise-scalability-user-data]
+recipe = slapos.cookbook:check_url_available
+path = ${basedirectory:promises}/scalabilityData
+site-id = "/{{ site_id }}"
+page = "/person_module/scalability_user_0/getTitle"
+url = ${erp5-cluster:connection-family-scalability}${:site-id}${:page}
+dash_path = {{ dash_location }}/bin/dash
+curl_path = {{ curl_location }}/bin/curl
+
+[publish]
+recipe = slapos.cookbook:publish.serialised
+erp5-frontend-url = ${scalability-launcher-instance:erp5-url}
+erp5-scalability-url = ${erp5-cluster:connection-family-scalability}
+{% endif %}

stack/erp5/instance.cfg.in

@@ -187,6 +187,29 @@ extra-context =
 # Must match the key id in [switch-softwaretype] which uses this section.
     raw software_type mariadb
 
+[dynamic-template-scalability-parameters]
+erp5 = {{ erp5_location }}
+erp5-dev = {{ erp5_dev_location }}
+local-bt5-repository = {{ local_bt5_repository }}
+curl-location = {{ curl_location }}
+dash-location = {{ dash_location }}
+bin-directory = {{ bin_directory }}
+
+
+[dynamic-template-scalability]
+< = jinja2-template-base
+template = {{ template_scalability }}
+filename = instance-scalability.cfg
+extensions = jinja2.ext.do
+extra-context =
+    key bin_directory dynamic-template-scalability-parameters:bin-directory
+    key curl_location dynamic-template-scalability-parameters:curl-location
+    key dash_location dynamic-template-scalability-parameters:dash-location
+    key erp5_location dynamic-template-scalability-parameters:erp5-dev
+    key local_bt5_repository dynamic-template-scalability-parameters:local-bt5-repository
+# Must match the key id in [switch-softwaretype] which uses this section.
+    raw software_type test 
+
 [switch-softwaretype]
 recipe = slapos.recipe.build
 script =
@@ -214,6 +237,7 @@ slapos_update_promise = ${:slapos_promise}
 override = {{ dumps(override_switch_softwaretype |default) }}
 
 default = ${dynamic-template-erp5:rendered}
+test = ${dynamic-template-scalability:rendered}
 kumofs = ${dynamic-template-kumofs:rendered}
 cloudooo = ${dynamic-template-cloudooo:rendered}
 mariadb = ${dynamic-template-mariadb:rendered}
@@ -221,3 +245,4 @@ balancer = ${dynamic-template-balancer:rendered}
 zodb-neo = ${dynamic-template-neo:rendered}
 zodb-zeo = ${dynamic-template-zeo:rendered}
 zope = ${dynamic-template-zope:rendered}
+RootSoftwareInstance = ${dynamic-template-erp5-RootSoftwareInstance:rendered}