Merge branch 'slaprunner'

setup.py

@@ -130,6 +130,7 @@ setup(name=name,
           'slapmonitor = slapos.recipe.slapmonitor:Recipe',
           'slapreport = slapos.recipe.slapreport:Recipe',
           'slaprunner = slapos.recipe.slaprunner:Recipe',
+          'slaprunner.test = slapos.recipe.slaprunner:Test',
           'softwaretype = slapos.recipe.softwaretype:Recipe',
           'sphinx= slapos.recipe.sphinx:Recipe',
           'sshkeys_authority.request = slapos.recipe.sshkeys_authority:Request',

slapos/recipe/check_url_available/template/check_url.in

@@ -9,11 +9,22 @@ if [ -z $URL ]; then
   exit 3
 fi
 
-CODE=$(%(curl_path)s -k -sL $URL -w %%{http_code} -o /dev/null)
+CODE=$(%(curl_path)s -g -k -sL $URL -w %%{http_code} -o /dev/null)
+
+if [ $? -eq 3 ]; then
+  echo "URL malformed: $URL." >&2
+  exit 1
+fi
+
+if [ $? -eq 7 ]; then
+  echo "Failed to connect to host: $URL." >&2
+  exit 1
+fi
 
 if [ ! $CODE ]; then
   echo "$URL is not available (server not reachable)." >&2
   exit 1
+fi
 
 if [ $CODE -eq 000 ]; then
   echo "$URL is not available (server not reachable)." >&2

slapos/recipe/slaprunner/__init__.py

@@ -39,6 +39,7 @@ class Recipe(GenericBaseRecipe):
     self.instance_directory = options['instance-directory'].strip()
     self.partition_amount = options['partition-amount'].strip()
     self.cloud9_url = options['cloud9-url'].strip()
+    self.log_file = os.path.join(options['log_dir'].strip(), 'slaprunner.log')
     # Set slaprunner access URL
     options['access-url'] = 'http://[%s]:%s' % (self.ipv6, self.runner_port)
 
@@ -79,10 +80,12 @@ class Recipe(GenericBaseRecipe):
     path_list.append(config_file)
 
     environment = dict(
-        PATH=os.path.dirname(self.options['git-binary']) + ':' + os.environ['PATH'],
+        PATH=os.path.dirname(
+            self.options['git-binary']) + ':' + os.environ['PATH'],
         GIT_SSH=self.options['ssh_client']
     )
-    launch_args = [self.options['slaprunner'].strip(), config_file]
+    launch_args = [self.options['slaprunner'].strip(), config_file,
+                   '--logfile', self.log_file]
     if self.optionIsTrue('debug', default=False):
       launch_args.append('--debug')
 
@@ -93,3 +96,69 @@ class Recipe(GenericBaseRecipe):
     path_list.append(wrapper)
 
     return path_list
+
+class Test(GenericBaseRecipe):
+  def _options(self, options):
+    self.ipv4 = options['ipv4'].strip()
+    self.ipv6 = options['ipv6'].strip()
+    self.proxy_port = options['proxy_port'].strip()
+    self.runner_port = options['runner_port'].strip()
+    self.workdir = options['working-directory'].strip()
+    self.software_directory = options['software-directory'].strip()
+    self.instance_directory = options['instance-directory'].strip()
+    self.partition_amount = options['partition-amount'].strip()
+    self.cloud9_url = options['cloud9-url'].strip()
+    # Set slaprunner access URL
+    options['access-url'] = 'http://[%s]:%s' % (self.ipv6, self.runner_port)
+
+  def install(self):
+    path_list = []
+
+    configuration = dict(
+        software_root=self.software_directory,
+        instance_root=self.instance_directory,
+        master_url='http://%s:%s/' % (self.ipv4, self.proxy_port),
+        computer_id='slaprunner',
+        partition_amount=self.partition_amount,
+        slapgrid_sr=self.options['slapgrid_sr'],
+        slapgrid_cp=self.options['slapgrid_cp'],
+        slapproxy=self.options['slapproxy'],
+        supervisor=self.options['supervisor'],
+        supervisord_config=os.path.join(self.instance_directory, 'etc',
+          'supervisord.conf'),
+        runner_workdir=self.workdir,
+        etc_dir=self.options['etc_dir'],
+        runner_host=self.ipv6,
+        runner_port=self.runner_port,
+        ipv4_address=self.ipv4,
+        ipv6_address=self.ipv6,
+        proxy_host=self.ipv4,
+        proxy_port=self.proxy_port,
+        proxy_database=os.path.join(self.workdir, 'proxy.db'),
+        git=self.options['git-binary'],
+        ssh_client=self.options['ssh_client'],
+        public_key=self.options['public_key'],
+        private_key=self.options['private_key'],
+        cloud9_url=self.cloud9_url
+    )
+
+    config_file = self.createFile(self.options['slapos.cfg'],
+        self.substituteTemplate(self.getTemplateFilename('slapos.cfg.in'),
+        configuration))
+    path_list.append(config_file)
+
+    environment = dict(
+        PATH=os.path.dirname(
+            self.options['git-binary']) + ':' + os.environ['PATH'],
+        GIT_SSH=self.options['ssh_client'],
+        CONFIG_FILE_PATH=config_file
+    )
+    launch_args = [self.options['slaprunnertest'].strip()]
+
+    wrapper = self.createPythonScript(self.options['wrapper'],
+        'slapos.recipe.librecipe.execute.executee',
+        (launch_args, environment)
+    )
+    path_list.append(wrapper)
+
+    return path_list

slapos/recipe/slaprunner/execute.py

-import os
-
-
-def execute(args):
-  """Portable execution with process replacement"""
-  if args.get("path", None):
-    os.environ['PATH'] = args["path"]
-  os.execv(args["launch_args"][0], args["launch_args"])

software/slaprunner/instance-runner.cfg

@@ -2,6 +2,7 @@
 parts =
   cloud9
   slaprunner
+  test-runner
   sshkeys-dropbear
   dropbear-server-add-authorized-key
   sshkeys-authority
@@ -38,13 +39,18 @@ services = $${rootdirectory:etc}/run/
 run = $${rootdirectory:var}/run/
 backup = $${rootdirectory:srv}/backup/
 promises = $${rootdirectory:etc}/promise/
+test = $${rootdirectory:etc}/test/
 
 [runnerdirectory]
 recipe = slapos.cookbook:mkdirectory
 home = $${rootdirectory:srv}/runner/
+test = $${rootdirectory:srv}/test/
 project = $${:home}/project
 software-root = $${:home}/software
 instance-root = $${:home}/instance
+project-test = $${:test}/project
+software-test = $${:test}/software
+instance-test = $${:test}/instance
 
 #Create password recovery code for slaprunner
 [recovery-code]
@@ -85,9 +91,10 @@ ipv4 = $${slap-network-information:local-ipv4}
 ipv6 = $${slap-network-information:global-ipv6}
 proxy_port = 50000
 runner_port = 50000
-partition-amount = 7
+partition-amount = $${slap-parameter:instance-amount}
 cloud9-url = $${cloud9:access-url}
 wrapper = $${basedirectory:services}/slaprunner
+debug = False
 
 # Deploy dropbear (minimalist ssh server)
 [sshkeys-directory]
@@ -95,6 +102,18 @@ recipe = slapos.cookbook:mkdirectory
 requests = $${directory:sshkeys}/requests/
 keys = $${directory:sshkeys}/keys/
 
+[test-runner]
+<= slaprunner
+recipe = slapos.cookbook:slaprunner.test
+slaprunnertest = ${buildout:directory}/bin/slaprunnertest
+slapos.cfg = $${rootdirectory:etc}/slapos-test.cfg
+working-directory = $${runnerdirectory:test}
+project-directory = $${runnerdirectory:project-test}
+software-directory = $${runnerdirectory:software-test}
+instance-directory = $${runnerdirectory:instance-test}
+wrapper = $${rootdirectory:bin}/runTestSuite
+etc_dir = $${basedirectory:test}
+
 [sshkeys-authority]
 recipe = slapos.cookbook:sshkeys_authority
 request-directory = $${sshkeys-directory:requests}
@@ -182,3 +201,5 @@ port = $${dropbear-server:port}
 [slap-parameter]
 # Default value if no ssh key is specified
 authorized-key =
+# Default value of instances number in slaprunner
+instance-amount = 10

software/slaprunner/software.cfg

@@ -25,7 +25,7 @@ md5sum = 5307e4200f044ae57b504ad68444491c
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner.cfg
 output = ${buildout:directory}/template-runner.cfg
-md5sum = b57ebff565595fa92cd7d5adc6a22c7e
+md5sum = bd1c73c149b4adf2fbab80bc29c782c8
 mode = 0644
 
 [eggs]
@@ -62,36 +62,34 @@ async = 0.6.1
 buildout-versions = 1.7
 gitdb = 0.5.4
 hexagonit.recipe.cmmi = 1.6
-meld3 = 0.6.8
+meld3 = 0.6.9
 plone.recipe.command = 1.1
 pycrypto = 2.6
-slapos.cookbook = 0.64.2
 slapos.libnetworkcache = 0.13.2
-slapos.recipe.build = 0.11
+slapos.recipe.build = 0.11.4
 slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.30
 smmap = 0.8.2
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
 # slapos.core==0.30
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31-dev
 Flask = 0.9
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31-dev
 GitPython = 0.3.2.RC1
 
 # Required by:
-# slapos.cookbook==0.64.2
+# slapos.cookbook==0.67-dev
 PyXML = 0.8.4
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31-dev
 atomize = 0.1.1
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31-dev
 feedparser = 5.1.2
 
 # Required by:
@@ -99,59 +97,58 @@ feedparser = 5.1.2
 hexagonit.recipe.download = 1.5.1
 
 # Required by:
-# slapos.cookbook==0.64.2
+# slapos.cookbook==0.67-dev
 inotifyx = 0.2.0
 
 # Required by:
 # slapos.core==0.30
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31-dev
 # xml-marshaller==0.9.7
 lxml = 3.0alpha2
 
 # Required by:
-# slapos.cookbook==0.64.2
-netaddr = 0.7.9
+# slapos.cookbook==0.67-dev
+netaddr = 0.7.10
 
 # Required by:
 # slapos.core==0.30
 netifaces = 0.8
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31-dev
 paramiko = 1.7.7.2
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31-dev
 psutil = 0.6.1
 
 # Required by:
-# slapos.cookbook==0.64.2
+# slapos.cookbook==0.67-dev
 # slapos.core==0.30
 # slapos.libnetworkcache==0.13.2
-# slapos.toolbox==0.30
-# supervisor==3.0a12
+# slapos.toolbox==0.31-dev
+# supervisor==3.0b1
 # zc.buildout==1.6.0-dev-SlapOS-007
 # zc.recipe.egg==1.3.2
 # zope.interface==4.0.1
 setuptools = 0.6c12dev-r88846
 
 # Required by:
-# slapos.cookbook==0.64.2
+# slapos.cookbook==0.67-dev
 slapos.core = 0.30
 
 # Required by:
 # slapos.core==0.30
-supervisor = 3.0a12
+supervisor = 3.0b1
 
 # Required by:
-# slapos.cookbook==0.64.2
+# slapos.cookbook==0.67-dev
 xml-marshaller = 0.9.7
 
 # Required by:
-# slapos.cookbook==0.64.2
+# slapos.cookbook==0.67-dev
 zc.recipe.egg = 1.3.2
 
 # Required by:
 # slapos.core==0.30
 zope.interface = 4.0.1
-pe.egg = 1.3.2