add support for multi-repositories

- a list of repositories to synchronize can be provided
- the first repository is used for software
- transfer project revision with string like
  [repo x]=[commit count x]-[hash x],[repo y]=[commit count y][hash y]
- change related to new structure :
  recipe slapos.recipe.erp5testnode is inside slapos.cookbook egg
- allow authentification setting for several repositories
- drop support for svn

slapos/recipe/erp5testnode/__init__.py

@@ -83,36 +83,24 @@ class Recipe(BaseSlapRecipe):
               git_binary=self.options['git_binary'],
               software_root=CONFIG['software_root'],
               working_directory=CONFIG['working_directory'],
-              vcs_repository=self.parameter_dict.get('vcs_repository'),
+              vcs_repository_list=eval(self.parameter_dict.get('vcs_repository_list'),),
               node_quantity=self.parameter_dict.get('node_quantity', '1'),
-              branch=self.parameter_dict.get('branch', None),
               test_suite_master_url=self.parameter_dict.get(
                                 'test_suite_master_url', None),
               test_suite_name=self.parameter_dict.get('test_suite_name'),
               test_suite_title=self.parameter_dict.get('test_suite_title'),
+              project_title=self.parameter_dict.get('project_title'),
               bin_directory=self.bin_directory,
-              foo='bar',
               # botenvironemnt is splittable string of key=value to substitute
               # environment of running bot
               bot_environment=self.parameter_dict.get('bot_environment', ''),
               partition_reference=CONFIG['partition_reference'],
               environment=dict(PATH=os.environ['PATH']),
+              vcs_authentication_list=self.parameter_dict.get(
+                     'vcs_authentication_list')
             )
           ]))
 
-  def installLocalSvn(self):
-    svn_dict = dict(svn_binary = self.options['svn_binary'])
-    svn_dict.update(self.parameter_dict)
-    self._writeExecutable(os.path.join(self.bin_directory, 'svn'), """\
-#!/bin/sh
-%(svn_binary)s --username %(svn_username)s --password %(svn_password)s \
---non-interactive --trust-server-cert --no-auth-cache "$@" """% svn_dict)
-
-    svnversion = os.path.join(self.bin_directory, 'svnversion')
-    if os.path.lexists(svnversion):
-      os.unlink(svnversion)
-    os.symlink(self.options['svnversion_binary'], svnversion)
-
   def installLocalGit(self):
     git_dict = dict(git_binary = self.options['git_binary'])
     git_dict.update(self.parameter_dict)
@@ -122,19 +110,18 @@ class Recipe(BaseSlapRecipe):
     home_directory = os.path.join(*os.path.split(self.bin_directory)[0:-1])
     print "home_directory : %r" % home_directory
     git_dict.setdefault("git_server_name", "git.erp5.org")
-    if git_dict.get('vcs_username', None) is not None:
+    if git_dict.get('vcs_authentication_list', None) is not None:
       netrc_file = open(os.path.join(home_directory, '.netrc'), 'w')
-      netrc_file.write("""
-  machine %(git_server_name)s
-  login %(vcs_username)s
-  password %(vcs_password)s""" % git_dict)
+      for vcs_authentication_dict in git_dict['vcs_authentication_list']:
+        netrc_file.write("""
+machine %(host)s
+login %(user_name)s
+password %(password)s
+""" % vcs_authentication_dict)
       netrc_file.close()
 
   def installLocalRepository(self):
-    if self.parameter_dict.get('vcs_repository').endswith('git'):
-      self.installLocalGit()
-    else:
-      self.installLocalSvn()
+    self.installLocalGit()
 
   def installLocalZip(self):
     zip = os.path.join(self.bin_directory, 'zip')

slapos/recipe/erp5testnode/testnode.py

@@ -60,39 +60,48 @@ slapos_controler = None
 def run(args):
   config = args[0]
   slapgrid = None
-  branch = config.get('branch', None)
   supervisord_pid_file = os.path.join(config['instance_root'], 'var', 'run',
         'supervisord.pid')
   subprocess.check_call([config['git_binary'],
                 "config", "--global", "http.sslVerify", "false"])
   previous_revision = None
-  run_software = True
-  # find what will be the path of the repository
-  repository_name = config['vcs_repository'].split('/')[-1].split('.')[0]
-  repository_path = os.path.join(config['working_directory'],repository_name)
-  config['repository_path'] = repository_path
-  sys.path.append(repository_path)
 
+  run_software = True
   # Write our own software.cfg to use the local repository
   custom_profile_path = os.path.join(config['working_directory'], 'software.cfg')
   config['custom_profile_path'] = custom_profile_path
-  
-  # create a profile in order to use the repository we already have
-  custom_profile = open(custom_profile_path, 'w')
-  profile_content = """
+  vcs_repository_list = config['vcs_repository_list']
+  profile_content = None
+  assert len(vcs_repository_list), "we must have at least one repository"
+  for vcs_repository in vcs_repository_list:
+    url = vcs_repository['url']
+    buildout_section_id = vcs_repository.get('buildout_section_id', None)
+    repository_id = buildout_section_id or \
+                                  url.split('/')[-1].split('.')[0]
+    repository_path = os.path.join(config['working_directory'],repository_id)
+    vcs_repository['repository_id'] = repository_id
+    vcs_repository['repository_path'] = repository_path
+    if profile_content is None:
+      profile_content = """
 [buildout]
 extends = %(software_config_path)s
-
-[%(repository_name)s]
+""" %  {'software_config_path': os.path.join(repository_path,
+                                          config['profile_path'])}
+    if not(buildout_section_id is None):
+      profile_content += """
+[%(buildout_section_id)s]
 repository = %(repository_path)s
-""" %     {'software_config_path': os.path.join(repository_path,
-                                          config['profile_path']),
-    'repository_name': repository_name,
-    'repository_path' : repository_path}
-  if branch is not None:
-    profile_content += "\nbranch = %s" % branch
+branch = %(branch)s
+""" %  {'buildout_section_id': buildout_section_id,
+        'repository_path' : repository_path,
+        'branch' : vcs_repository.get('branch','')}
+
+  custom_profile = open(custom_profile_path, 'w')
   custom_profile.write(profile_content)
   custom_profile.close()
+  config['repository_path'] = repository_path
+  sys.path.append(repository_path)
+
   retry_software = False
   try:
     while True:
@@ -104,19 +113,24 @@ repository = %(repository_path)s
           except:
             pass
         process_group_pid_set.clear()
+        full_revision_list = []
         # Make sure we have local repository
-        if not os.path.exists(repository_path):
-          parameter_list = [config['git_binary'], 'clone',
-                            config['vcs_repository']]
-          if branch is not None:
-            parameter_list.extend(['-b',branch])
-          parameter_list.append(repository_path)
-          subprocess.check_call(parameter_list)
-          # XXX this looks like to not wait the end of the command
-        # Make sure we have local repository
-        updater = Updater(repository_path, git_binary=config['git_binary'])
-        updater.checkout()
-        revision = updater.getRevision()
+        for vcs_repository in vcs_repository_list:
+          repository_path = vcs_repository['repository_path']
+          repository_id = vcs_repository['repository_id']
+          if not os.path.exists(repository_path):
+            parameter_list = [config['git_binary'], 'clone',
+                              vcs_repository['url']]
+            if vcs_repository.get('branch') is not None:
+              parameter_list.extend(['-b',vcs_repository.get('branch')])
+            parameter_list.append(repository_path)
+            subprocess.check_call(parameter_list)
+          # Make sure we have local repository
+          updater = Updater(repository_path, git_binary=config['git_binary'])
+          updater.checkout()
+          revision = "-".join(updater.getRevision())
+          full_revision_list.append('%s=%s' % (repository_id, revision))
+        revision = ','.join(full_revision_list)
         if previous_revision == revision:
           time.sleep(120)
           if not(retry_software):
@@ -138,16 +152,19 @@ repository = %(repository_path)s
           assert master.getProtocolRevision() == 1
           test_result = safeRpcCall(master.createTestResult,
             config['test_suite_name'], revision, [],
-            False, config['test_suite_title'])
+            False, config['test_suite_title'], config['project_title'])
         print "testnode, test_result : %r" % (test_result,)
         if test_result:
           test_result_path, test_revision = test_result
           if revision != test_revision:
-            # other testnodes on other boxes are already ready to test another
-            # revision
-            updater = Updater(repository_path, git_binary=config['git_binary'],
-                              revision=test_revision)
-            updater.checkout()
+            for i, repository_revision in enumerate(test_revision.split(',')):
+              vcs_repository = vcs_repository_list[i]
+              repository_path = vcs_repository['repository_path']
+              # other testnodes on other boxes are already ready to test another
+              # revision
+              updater = Updater(repository_path, git_binary=config['git_binary'],
+                                revision=repository_revision.split('-')[1])
+              updater.checkout()
 
           # Now prepare the installation of SlapOS
           slapos_controler = SlapOSControler(config,
@@ -169,24 +186,21 @@ repository = %(repository_path)s
 
           # update repositories downloaded by buildout. Later we should get
           # from master a list of repositories
-          repository_path_list = glob(os.path.join(config['software_root'],
-                                  '*', 'parts', 'git_repository', '*'))
-          assert len(repository_path_list) >= 0
-          for repository_path in repository_path_list:
-            updater = Updater(repository_path, git_binary=config['git_binary'])
-            updater.checkout()
-            if os.path.split(repository_path)[-1] == repository_name:
-              # redo checkout with good revision, the previous one is used
-              # to pull last code
-              updater = Updater(repository_path, git_binary=config['git_binary'],
-                                revision=revision)
-              updater.checkout()
-            # calling dist/externals is only there for backward compatibility,
-            # the code will be removed soon
-            if os.path.exists(os.path.join(repository_path, 'dist/externals.py')):
-              process = subprocess.Popen(['dist/externals.py'],
-                        cwd=repository_path)
-              process.wait()
+
+          # The section below seems useless since we override buildout
+          # configuration to use local checkouts
+          #repository_path_list = glob(os.path.join(config['software_root'],
+                                  #'*', 'parts', 'git_repository', '*'))
+          #assert len(repository_path_list) >= 0
+          #for repository_path in repository_path_list:
+            #updater = Updater(repository_path, git_binary=config['git_binary'])
+            #updater.checkout()
+            #if os.path.split(repository_path)[-1] == repository_name:
+              ## redo checkout with good revision, the previous one is used
+              ## to pull last code
+              #updater = Updater(repository_path, git_binary=config['git_binary'],
+                                #revision=revision)
+              #updater.checkout()
 
           partition_path = os.path.join(config['instance_root'],
                                         config['partition_reference'])