Merge branch 'cloudooo' of https://git.erp5.org/repos/slapos into cloudooo

slapos/recipe/cloudooo/__init__.py

@@ -42,15 +42,20 @@ class Recipe(BaseSlapRecipe):
   def _install(self):
     self.path_list = []
     self.requirements, self.ws = self.egg.working_set()
-    self.killpidfromfile = zc.buildout.easy_install.scripts(
-        [('killpidfromfile', __name__ + 'slapos.recipe.erp5.killpidfromfile',
-          'killpidfromfile')], self.ws, sys.executable, self.bin_directory)[0]
-    self.path_list.append(self.killpidfromfile)
     self.cron_d = self.installCrond()
     ca_conf = self.installCertificateAuthority()
 
     key, certificate = self.requestCertificate('Cloudooo')
 
+    cloudooo_paster = os.path.join(self.bin_directory, 'cloudooo_paster')
+    ooo_paster = self.options['ooo_paster']
+    if os.path.lexists(cloudooo_paster):
+      if not os.readlink(cloudooo_paster) != ooo_paster:
+        os.unlink(cloudooo_paster)
+    if not os.path.lexists(cloudooo_paster):
+      os.symlink(ooo_paster, cloudooo_paster)
+    self.options['cloudooo_paster'] = cloudooo_paster
+
     conversion_server_conf = self.installConversionServer(
         self.getLocalIPv4Address(), 23000, 23060)
 
@@ -70,15 +75,6 @@ class Recipe(BaseSlapRecipe):
       os.symlink(runUnitTest, runCloudoooUnitTest)
     self.path_list.append(runCloudoooUnitTest)
 
-    cloudooo_paster = os.path.join(self.bin_directory, 'cloudooo_paster')
-    ooo_paster = self.options['ooo_paster']
-    if os.path.lexists(cloudooo_paster):
-      if not os.readlink(cloudooo_paster) != ooo_paster:
-        os.unlink(cloudooo_paster)
-    if not os.path.lexists(cloudooo_paster):
-      os.symlink(ooo_paster, cloudooo_paster)
-    self.path_list.append(cloudooo_paster)
-
     self.linkBinary()
     self.setConnectionDict(dict(
           site_url="https://[%s]:%s/" % (stunnel_conf['public_ip'],
@@ -190,7 +186,8 @@ class Recipe(BaseSlapRecipe):
         email_address=ca_email,
     )
     self._writeFile(openssl_configuration,
-        self.getTemplateFilename('openssl.cnf.ca.in'))
+        self.substituteTemplate(self.getTemplateFilename('openssl.cnf.ca.in'),
+          config))
     self.path_list.extend(zc.buildout.easy_install.scripts([
       ('certificate_authority',
         'slapos.recipe.erp5.certificate_authority',
@@ -241,7 +238,7 @@ class Recipe(BaseSlapRecipe):
       port=port,
       openoffice_port=openoffice_port,
       openoffice_host=ip,
-      PATH="$PATH:%s" % self.bin_directory
+      PATH=self.bin_directory
     )
     for env_line in self.options['environment'].splitlines():
       env_line = env_line.strip()
@@ -261,7 +258,7 @@ class Recipe(BaseSlapRecipe):
       'slapos.recipe.librecipe.execute',
       'execute_with_signal_translation')], self.ws,
       sys.executable, self.wrapper_directory,
-      arguments=[self.options['ooo_paster'].strip(), 'serve', config_file]))
+      arguments=[self.options['cloudooo_paster'].strip(), 'serve', config_file]))
     return {
       name + '_conf': config_file,
       name + '_port': conversion_server_dict['port'],

slapos/recipe/cloudooo/template/cloudooo.cfg.in

@@ -48,6 +48,7 @@ mimetype_registry =
   application/x-shockwave-flash * ffmpeg
   application/ogg * ffmpeg
   application/ogv * ffmpeg
+  image/png image/jpeg imagemagick
   image/png * ooo
   image/* image/* imagemagick
   text/* * ooo

slapos/recipe/cloudoootestnode/SlapOSControler.py

@@ -46,48 +46,49 @@ class SlapOSControler(object):
  'reference': config['computer_id'],
  'software_root': config['software_root']}))
 
-  def runSoftwareRelease(self, config, environment, process_group_pid_set=None):
+  def runSoftwareRelease(self, config, environment, process_group_pid_set=None,
+                         stdout=None, stderr=None):
     print "SlapOSControler.runSoftwareRelease"
-    while True:
-      cpu_count = os.sysconf("SC_NPROCESSORS_ONLN")
-      os.putenv('MAKEFLAGS', '-j%s' % cpu_count)
-      os.environ['PATH'] = environment['PATH']
-      stdout = open(os.path.join(
-                    config['instance_root'],'.runSoftwareRelease_out'),
-                    'w+')
-      stderr = open(os.path.join(
-                    config['instance_root'],'.runSoftwareRelease_err'),
-                    'w+')
-      slapgrid = subprocess.Popen([config['slapgrid_software_binary'], '-v', '-c',
-        #'--buildout-parameter',"'-U -N' -o",
-        config['slapos_config']],
-        stdout=stdout, stderr=stderr,
-        close_fds=True, preexec_fn=os.setsid)
-      process_group_pid_set.add(slapgrid.pid)
-      slapgrid.wait()
-      stdout.seek(0)
-      stderr.seek(0)
-      process_group_pid_set.remove(slapgrid.pid)
-      status_dict = {'status_code':slapgrid.returncode,
-                     'stdout':stdout.read(),
-                     'stderr':stderr.read()}
-      stdout.close()
-      stderr.close()
-      return status_dict
+    cpu_count = os.sysconf("SC_NPROCESSORS_ONLN")
+    os.putenv('MAKEFLAGS', '-j%s' % cpu_count)
+    os.environ['PATH'] = environment['PATH']
+    slapgrid = subprocess.Popen([config['slapgrid_software_binary'], '-v', '-c',
+      #'--buildout-parameter',"'-U -N' -o",
+      config['slapos_config']],
+      stdout=stdout, stderr=stderr,
+      close_fds=True, preexec_fn=os.setsid)
+    process_group_pid_set.add(slapgrid.pid)
+    slapgrid.wait()
+    stdout.seek(0)
+    stderr.seek(0)
+    process_group_pid_set.remove(slapgrid.pid)
+    status_dict = {'status_code':slapgrid.returncode,
+                    'stdout':stdout.read(),
+                    'stderr':stderr.read()}
+    stdout.close()
+    stderr.close()
+    return status_dict
 
-  def runComputerPartition(self, config, process_group_pid_set=None):
+  def runComputerPartition(self, config, environment,
+                           process_group_pid_set=None,
+                           stdout=None, stderr=None):
     print "SlapOSControler.runInstanceRelease"
     slap = slapos.slap.slap()
     slap.registerOpenOrder().request(self.software_profile,
         partition_reference='testing partition',
         partition_parameter_kw=config['instance_dict'])
     slapgrid = subprocess.Popen([config['slapgrid_partition_binary'],
-      config['slapos_config'], '-c', '-v'], close_fds=True, preexec_fn=os.setsid,
-    stdout=-1, stderr=-1)
-    ss, se = slapgrid.communicate()
-    print " \n %s \n \n %s \n" % (ss, se)
+      config['slapos_config'], '-c', '-v'],
+      stdout=stdout, stderr=stderr,
+      close_fds=True, preexec_fn=os.setsid)
     process_group_pid_set.add(slapgrid.pid)
     slapgrid.wait()
+    stdout.seek(0)
+    stderr.seek(0)
     process_group_pid_set.remove(slapgrid.pid)
-    if slapgrid.returncode != 0:
-      raise ValueError('Slapgrid instance failed')
+    status_dict = {'status_code':slapgrid.returncode,
+                    'stdout':stdout.read(),
+                    'stderr':stderr.read()}
+    stdout.close()
+    stderr.close()
+    return status_dict

slapos/recipe/cloudoootestnode/testnode.py

 from xml_marshaller import xml_marshaller
-import os, xmlrpclib, time, imp
+import os, xmlrpclib, time, imp, re
 from glob import glob
 import signal
 import slapos.slap
@@ -55,6 +55,16 @@ def safeRpcCall(function, *args):
       time.sleep(retry)
       retry += retry >> 1
 
+def getInputOutputFileList(config, command_name):
+  stdout = open(os.path.join(
+                config['instance_root'],'.%s_out' % command_name),
+                'w+')
+  stdout.write("%s\n" % command_name)
+  stderr = open(os.path.join(
+                config['instance_root'],'.%s_err' % command_name),
+                'w+')
+  return (stdout, stderr)
+
 slapos_controler = None
 
 def run(args):
@@ -84,11 +94,9 @@ def run(args):
     if profile_content is None:
       profile_content = """
 [buildout]
-extends = /home/slap/config/cloudooo.cfg
-"""
-#extends = %(software_config_path)s
-#""" %  {'software_config_path': os.path.join(repository_path,
-#                                          config['profile_path'])}
+extends = %(software_config_path)s
+""" %  {'software_config_path': os.path.join(repository_path,
+                                          config['profile_path'])}
     if not(buildout_section_id is None):
       profile_content += """\n
 [%(buildout_section_id)s]
@@ -97,6 +105,7 @@ branch = %(branch)s
 """ %  {'buildout_section_id': buildout_section_id,
         'repository_path' : repository_path,
         'branch' : vcs_repository.get('branch','cloudooo')}
+
   custom_profile = open(custom_profile_path, 'w')
   custom_profile.write(profile_content)
   custom_profile.close()
@@ -139,8 +148,9 @@ branch = %(branch)s
             continue
         retry_software = False
         previous_revision = revision
-        # Require build connection for runnig tests
+
         print config
+        # Require build connection for runnig tests
         portal_url = config['test_suite_master_url']
         test_result_path = None
         test_result = (test_result_path, revision)
@@ -168,22 +178,25 @@ branch = %(branch)s
               updater = Updater(repository_path, git_binary=config['git_binary'],
                                 revision=repository_revision.split('-')[1])
               updater.checkout()
+
           # Now prepare the installation of SlapOS
           slapos_controler = SlapOSControler(config,
             process_group_pid_set=process_group_pid_set)
-          # this should be always true later, but it is too slow for now
-          status_dict = slapos_controler.runSoftwareRelease(config,
-            config['environment'],
-            process_group_pid_set,
-            )
+          for method_name in ("runSoftwareRelease", "runComputerPartition"):
+            stdout, stderr = getInputOutputFileList(config, method_name)
+            slapos_method = getattr(slapos_controler, method_name)
+            status_dict = slapos_method(config,
+              environment=config['environment'],
+              process_group_pid_set=process_group_pid_set,
+              stdout=stdout, stderr=stderr
+              )
+            if status_dict['status_code'] != 0:
+              break
           if status_dict['status_code'] != 0:
             safeRpcCall(master.reportTaskFailure,
               test_result_path, status_dict, config['test_node_title'])
             retry_software = True
             continue
-          # create instances, it should take some seconds only
-          slapos_controler.runComputerPartition(config,
-                  process_group_pid_set=process_group_pid_set)
 
           partition_path = os.path.join(config['instance_root'],
                                         config['partition_reference'])
@@ -203,21 +216,37 @@ branch = %(branch)s
           file_object = open(run_test_suite_path, 'r')
           line = file_object.readline()
           file_object.close()
-#          cloudooo_tests = glob(
-#                    '%s/*/src/cloudooo/cloudooo/handler/*/tests/test*.py' %
-#                    config['software_root'])
-#          for test in cloudooo_tests:
-          invocation_list = []
-          if line[:2] == '#!':
-            invocation_list = line[2:].split()
-          invocation_list.extend([run_test_suite_path,
-                                  '--paster_path', cloudooo_paster,
-                                  cloudooo_conf,
-                                  'testFfmpegServer'])
-          run_test_suite = subprocess.Popen(invocation_list)
-          process_group_pid_set.add(run_test_suite.pid)
-          run_test_suite.wait()
-          process_group_pid_set.remove(run_test_suite.pid)
+
+          wait_serve = True
+          while wait_serve:
+            try:
+              conf = open(cloudooo_conf).read()
+              host, port = re.findall('host=*.*.*.*\nport\ \=.*', conf)[0].split('\n')
+              serve = xmlrpclib.Server("http://%s:%s/RPC2" % 
+                        (host.split('=')[-1].lstrip(), 
+                        port.split('=')[-1].lstrip()))
+              serve.system.listMethods()
+              if len(serve.system.listMethods()) > 0:
+                wait_serve = False
+            except socket.error, e:
+              wait_serve = True
+              time.sleep(10)
+
+          cloudooo_tests = glob(
+                    '%s/*/src/cloudooo/cloudooo/handler/*/tests/test*.py' %
+                    config['software_root'])
+          for test in cloudooo_tests:
+            invocation_list = []
+            if line[:2] == '#!':
+              invocation_list = line[2:].split()
+            invocation_list.extend([run_test_suite_path,
+                                    '--paster_path', cloudooo_paster,
+                                    cloudooo_conf,
+                                    test.split('/')[-1]])
+            run_test_suite = subprocess.Popen(invocation_list)
+            process_group_pid_set.add(run_test_suite.pid)
+            run_test_suite.wait()
+            process_group_pid_set.remove(run_test_suite.pid)
       except SubprocessError:
         time.sleep(120)
         continue