Merge branch 'slaprunner-paas'

Conflicts:
	component/apache/buildout.cfg
	stack/monitor/buildout.cfg

software/lamp-template/software.cfg

@@ -2,6 +2,7 @@
 
 extends =
   ../../stack/lamp/buildout.cfg
+  ../../stack/slapos.cfg
 
 [application]
 recipe = slapos.recipe.build:download-unpacked
@@ -19,4 +20,5 @@ mode = 0644
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 
 [application-configuration]
-location = Student shall put here relative path to application top level directory where configuration shall be created
+#Student shall put here relative path to application top level directory where configuration shall be created
+#location = 

software/monitor/instance.cfg

+[buildout]
+
+parts =
+  publish-connection-informations
+  cron
+  certificate-authority
+  cron-entry-monitor
+  cron-entry-rss
+  deploy-index
+  deploy-settings-cgi
+  deploy-status-cgi
+  deploy-status-history-cgi
+  setup-static-files
+  certificate-authority
+  zero-parameters
+  public-symlink
+  cgi-httpd-wrapper
+  cgi-httpd-graceful-wrapper
+  monitor-promise
+  monitor-instance-log-access
+  
+extends = ${monitor-template:output}
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
\ No newline at end of file

software/monitor/software.cfg

+[buildout]
+
+extends =
+  ../../stack/monitor/buildout.cfg
+  ../../stack/slapos.cfg
+  
+parts = 
+  slapos-cookbook
+  template
+  eggs
+  
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg
+output = ${buildout:directory}/instance.cfg
+md5sum = 02442c6727c8fe8247dc91c4b085bcc6
+mode = 0644
+
+[eggs]
+recipe = z3c.recipe.scripts
+eggs = 
+  cns.recipe.symlink
+  slapos.cookbook
+  slapos.recipe.cmmi
+  hexagonit.recipe.download
+  plone.recipe.command
+  
+[versions]
+hexagonit.recipe.download = 1.7nxd002
+Jinja2 = 2.7.3
+MarkupSafe = 0.23
+PyRSS2Gen = 1.1
+Werkzeug = 0.9.6
+buildout-versions = 1.7
+cffi = 0.8.6
+cmd2 = 0.6.7
+cns.recipe.symlink = 0.2.3
+cryptography = 0.5.2
+itsdangerous = 0.24
+meld3 = 1.0.0
+mr.developer = 1.30
+plone.recipe.command = 1.1
+pyOpenSSL = 0.14
+pyparsing = 2.0.2
+six = 1.7.3
+slapos.cookbook = 0.87
+slapos.recipe.cmmi = 0.2
+slapos.recipe.template = 2.5
+stevedore = 0.15
+z3c.recipe.scripts = 1.0.1
+
+# Required by:
+# slapos.core==1.1.2
+Flask = 0.10.1
+
+# Required by:
+# cliff==1.6.1
+argparse = 1.2.1
+
+# Required by:
+# slapos.core==1.1.2
+cliff = 1.6.1
+
+# Required by:
+# slapos.cookbook==0.87
+inotifyx = 0.2.0-1
+
+# Required by:
+# slapos.cookbook==0.87
+lock-file = 2.0
+
+# Required by:
+# slapos.cookbook==0.87
+# slapos.core==1.1.2
+# xml-marshaller==0.9.7
+lxml = 3.3.5
+
+# Required by:
+# slapos.cookbook==0.87
+netaddr = 0.7.12
+
+# Required by:
+# slapos.core==1.1.2
+netifaces = 0.10.4
+
+# Required by:
+# slapos.core==1.1.2
+psutil = 2.1.1
+
+# Required by:
+# cffi==0.8.6
+pycparser = 2.10
+
+# Required by:
+# slapos.cookbook==0.87
+pytz = 2014.4
+
+# Required by:
+# slapos.core==1.1.2
+requests = 2.3.0
+
+# Required by:
+# slapos.cookbook==0.87
+slapos.core = 1.1.2
+
+# Required by:
+# slapos.core==1.1.2
+supervisor = 3.0
+
+# Required by:
+# slapos.cookbook==0.87
+xml-marshaller = 0.9.7
+
+# Required by:
+# slapos.core==1.1.2
+zope.interface = 4.1.1

software/slaprunner/README.txt

@@ -144,7 +144,7 @@ As you can see in instance-runner-*.cfg, the buildout section extends a hard-cod
 List of ports used by the webrunner:
 ------------------------------------
 8602 : slapproxy, while running tests
-8080 : shellinabox
+8949 : shellinabox
 9684 : apache (monitoring of slaprunner, git access)
 22222 : dropbear
 50000 : slapproxy
@@ -152,4 +152,4 @@ List of ports used by the webrunner:
 
 Tips:
 -----
-You can use shellinabox in fullscreen, by accessing : https://[IPV6]:8080
+You can use shellinabox in fullscreen, by accessing : https://[IPV6]:8949

software/slaprunner/common.cfg

@@ -28,7 +28,7 @@ parts =
   instance-runner-import
   instance-runner-export
   slapos-cookbook
-  slapos-cfg-template
+  template-slapos-cfg
 # XXX: we have to manually add this for resilience
   rdiff-backup
   collective.recipe.template-egg
@@ -44,7 +44,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner.cfg
 output = ${buildout:directory}/template-runner.cfg.in
-md5sum = 1c084d9aa33202ccba58be750f0698a5
+md5sum = 9878115a2969489b57e8563ef3d4d6d3
 mode = 0644
 
 [template-runner-import-script]
@@ -66,7 +66,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/runner-export.sh.jinja2
 download-only = true
-md5sum = ea478351883e6e716c044771f69a09ac
+md5sum = 246167ed081027618ef127a6fffd13ed
 filename = runner-export.sh.jinja2
 mode = 0644
 
@@ -74,13 +74,13 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner-export.cfg.in
 output = ${buildout:directory}/instance-runner-export.cfg
-md5sum = cf43eb1d6e9eea545858a86ea790a1cb
+md5sum = 6930a0d6c88896c43a8e60509ba6bd1c
 mode = 0644
 
 [template-resilient]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
-md5sum = ab564b9a9de3e44884adf072fc740d23
+md5sum = 1f7332d0b0671929a2de87ef9ce998c8
 filename = instance-resilient.cfg.jinja2
 mode = 0644
 
@@ -88,7 +88,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-resilient-test.cfg.jinja2
 download-only = true
-md5sum = 07241952e1dce8688b2da5c1a14c3e6a
+md5sum = abba49f5f397bfe6f6ea5e5dd185b1e9
 filename = instance-resilient-test.cfg.jinja2
 mode = 0644
 
@@ -104,7 +104,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/httpd_conf.in
 download-only = true
-md5sum = 47ba63d87a7fdc91568581226f0063f6
+md5sum = 63782f8eaa94af729ec6bba80e062d8f
 filename = httpd_conf.in
 mode = 0644
 
@@ -116,10 +116,10 @@ filename = launcher.in
 mode = 0644
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 
-[slapos-cfg-template]
+[template-slapos-cfg]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/${:filename}
-md5sum = 788795524769f6d946526ac282508b69
+md5sum = 847df63cf5a226ed338d7ca34b0220b5
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 filename = slapos.cfg.in
 download-only = true

software/slaprunner/development.cfg

@@ -21,7 +21,7 @@ parts =
   instance-runner-import
   instance-runner-export
   slapos-cookbook
-  slapos-cfg-template
+  template-slapos-cfg
 # XXX: we have to manually add this for resilience
   rdiff-backup
   collective.recipe.template-egg

software/slaprunner/httpd_conf.in

@@ -46,8 +46,14 @@ Alias /public {{ parameters.runner_home }}/public
 <Directory {{ parameters.runner_home }}/public>
     Order Allow,Deny
     Allow from all
+    AllowOverride All
     Satisfy Any
     Options Indexes FollowSymLinks
+    DirectoryIndex index.html
+    <Files .htaccess>
+        order allow,deny
+        deny from all
+    </Files>
 </Directory>
 
 

software/slaprunner/instance-resilient-test.cfg.jinja2

@@ -47,6 +47,7 @@ config = resiliency-backup-periodicity {{ cluster_parameter_dict.keys() | join('
 config-{{ key }} = {{ dumps(value) }}
 {% endfor -%}
 config-resiliency-backup-periodicity = */6 * * * *
+config-resilient-clone-number = 2
 # XXX hardcoded
 #config-frontend-domain = google.com
 # XXX Hack to deploy Root Instance on the same computer as the type-test Instance

software/slaprunner/instance-resilient.cfg.jinja2

@@ -5,6 +5,11 @@
 # set to false (else it could be dangerous)
 {% do slapparameter_dict.__setitem__('auto-deploy', slapparameter_dict.get('auto-deploy', 'true')) %}
 {% do slapparameter_dict.__setitem__('auto-deploy-instance', slapparameter_dict.get('auto-deploy-instance', 'false')) %}
+# let decide the number of clones we want. We can't have more than 2 clones
+{% set number_of_instances = slapparameter_dict.get('resilient-clone-number', 1)|int %}
+{% if number_of_instances > 2 %}
+  {% set number_of_instances = 2 %}
+{% endif %}
 
 {% import 'parts' as parts %}
 {% import 'replicated' as replicated %}
@@ -16,10 +21,10 @@ offline = true
 
 # += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
 parts +=
-  {{ parts.replicate("runner", "3") }}
+  {{ parts.replicate("runner", number_of_instances + 1) }}
   publish-connection-informations
 
-{{ replicated.replicate("runner", "3", "runner-export", "runner-import", slapparameter_dict=slapparameter_dict) }}
+{{ replicated.replicate("runner", number_of_instances + 1, "runner-export", "runner-import", slapparameter_dict=slapparameter_dict) }}
 
 # Bubble up the parameters
 [request-runner]
@@ -41,14 +46,13 @@ public_url = ${request-runner:connection-public_url}
 # Default parameters for distributed deployment
 # I.e state "backup1 of maria should go there, ..."
 # XXX-Cedric: Hardcoded number of backups. Should be dynamically generated.
-runner1-computer-guid =
-pbs-runner1-computer-guid =
-runner2-computer-guid =
-pbs-runner2-computer-guid =
-runner3-computer-guid =
-pbs-runner3-computer-guid =
+{% for nb in range(1,number_of_instances+1) %}
+runner{{nb}}-computer-guid =
+pbs-runner{{nb}}-computer-guid =
+{% endfor %}
 # XXX-Cedric: Hardcoded parameters. Should be dynamically generated.
 domain =
 authorized-key =
 instance-amount = 10
 debug = false
+resilient-clone-number = 1

software/slaprunner/instance-runner-export.cfg.in

@@ -26,6 +26,7 @@ parts +=
   cron-entry-backup
   cron-entry-prepare-software
   deploy-instance-parameters
+  minishell-cwd
 ## Monitoring part
 ###Parts to add for monitoring
   cron
@@ -46,6 +47,7 @@ parts +=
   monitor-instance-log-access
 ## Monitor for runner
   monitor-current-log-access
+  monitor-check-resilient-feed-file
 
 
 [exporter]

software/slaprunner/instance-runner.cfg

@@ -23,6 +23,7 @@ parts =
   cron-entry-prepare-software
   deploy-instance-parameters
   instance-software-type
+  minishell-cwd
 {% if slapparameter_dict.get('custom-frontend-backend-url') %}
   custom-frontend-promise
 {% endif %}
@@ -180,16 +181,20 @@ auto_deploy = $${slap-parameter:auto-deploy}
 auto_deploy_instance = $${slap-parameter:auto-deploy-instance}
 autorun = $${slap-parameter:autorun}
 knowledge0_file = $${buildout:directory}/$${public:filename}
+minishell_cwd_file = $${directory:etc}/.minishell-cwd
+minishell_history_file = $${directory:etc}/.minishell_history
 
 [test-runner]
 <= slaprunner
 slapos.cfg = $${directory:etc}/slapos-test.cfg
 working-directory = $${runnerdirectory:test}
 project-directory = $${runnerdirectory:project-test}
-software-directory = $${runnerdirectory:software-test}
-instance-directory = $${runnerdirectory:instance-test}
+software_root = $${runnerdirectory:software-test}
+instance_root = $${runnerdirectory:instance-test}
 proxy_port = 8602
 etc_dir = $${directory:test}
+autorun = False
+auto_deploy = True
 
 [runtestsuite]
 recipe = slapos.cookbook:wrapper
@@ -313,7 +318,7 @@ var_dir = $${directory:var}
 project_folder = $${directory:project}
 runner_home = $${runnerdirectory:home}
 git_http_backend = ${git:location}/libexec/git-core/git-http-backend
-cgi_httpd_conf = $${cgi-httpd-configuration-file:output}
+cgi_httpd_conf = $${monitor-httpd-configuration-file:rendered}
 
 [httpd-conf]
 recipe = slapos.recipe.template:jinja2
@@ -341,7 +346,8 @@ path_pid = $${directory:run}/gunicorn.pid
 
 [gunicorn-launcher]
 recipe = slapos.cookbook:wrapper
-command-line = $${gunicorn:bin_gunicorn} slapos.runner:app -p $${gunicorn:path_pid} -b unix:$${gunicorn:socket} -e RUNNER_CONFIG=$${slaprunner:slapos.cfg} --preload
+command-line = $${gunicorn:bin_gunicorn} slapos.runner:app -p $${gunicorn:path_pid} -b unix:$${gunicorn:socket} -e RUNNER_CONFIG=$${slaprunner:slapos.cfg} --error-logfile $${directory:log}/$${:error-log-file} --log-level error --preload 
+error-log-file = gunicorn-error.log
 wrapper-path = $${gunicorn:bin_launcher}
 environment = PATH=$${environ:PATH}:${git:location}/bin/
   RUNNER_CONFIG=$${slaprunner:slapos.cfg}
@@ -415,7 +421,7 @@ name = Monitor Frontend
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
 config = url domain
-config-url = https://[$${cgi-httpd-configuration-file:listening-ip}]:$${monitor-parameters:port}
+config-url = https://[$${monitor-httpd-configuration:listening-ip}]:$${monitor-parameters:port}
 config-domain = $${slap-parameter:frontend-domain}
 return = site_url domain
 
@@ -492,7 +498,7 @@ port = $${slap-parameter:monitor-port}
 
 [slapos-cfg]
 recipe = slapos.recipe.template:jinja2
-template = ${slapos-cfg-template:location}/${slapos-cfg-template:filename}
+template = ${template-slapos-cfg:location}/${template-slapos-cfg:filename}
 rendered = $${slaprunner:slapos.cfg}
 mode = 700
 context =
@@ -500,7 +506,7 @@ context =
 
 [slapos-test-cfg]
 recipe = slapos.recipe.template:jinja2
-template = ${slapos-cfg-template:location}/${slapos-cfg-template:filename}
+template = ${template-slapos-cfg:location}/${template-slapos-cfg:filename}
 rendered = $${test-runner:slapos.cfg}
 mode = 700
 context =
@@ -509,7 +515,7 @@ context =
 [shellinabox]
 recipe = slapos.cookbook:shellinabox
 ipv6 = $${slap-network-information:global-ipv6}
-port = 8080
+port = 8949
 shell = $${shell:wrapper}
 wrapper = $${directory:bin}/shellinaboxd
 shellinabox-binary = ${shellinabox:location}/bin/shellinaboxd
@@ -598,6 +604,12 @@ recovery-code = $${recovery-code:passwd}
 [zero-parameters]
 
 
+[minishell-cwd]
+recipe = plone.recipe.command
+command = if [ ! -f $${slaprunner:minishell_cwd_file} ]; then echo $${runnerdirectory:home} > $${slaprunner:minishell_cwd_file}; fi
+location = $${slaprunner:minishell_cwd_file}
+stop-on-error = true
+
 [monitor-current-log-access]
 < = monitor-directory-access
 source = $${directory:log}

software/slaprunner/node-frontend.in

-#!{{ shell_path }}
-# BEWARE: This file is operated by slapgrid
-# BEWARE: It will be overwritten automatically
-export NODE_PATH={{ node_env }}
-exec {{ node_path }} {{ conf_path }} {{ ip }} {{ port }} {{ key }} {{ certificate }} {{ backend_ip }} {{ backend_port }}
\ No newline at end of file

software/slaprunner/schema.json

@@ -79,5 +79,10 @@
                                 "type": "integer",
                                 "default": 9684
                         }
+                        "resilient-clone-number": {
+                                "description": "is the number of clones to be deployed when a resilient runner is ordered. Its value can be 1 or 2",
+                                "type": "integer",
+                                "default" : 1
+                        }
                 }
 }

software/slaprunner/software.cfg

@@ -213,7 +213,7 @@ slapos.recipe.build = 0.11.5
 slapos.recipe.cmmi = 0.2
 slapos.recipe.download = 1.0.dev-r4053
 slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.39.3
+slapos.toolbox = 0.39.4
 smmap = 0.8.2
 xml-marshaller = 0.9.7
 z3c.recipe.scripts = 1.0.1

software/slaprunner/template/runner-export.sh.jinja2

 #!{{ shell_binary }}
+LANG=en_US
+export $LANG
 umask 077
 sync_element () {
   path=$1

software/slaprunner/template/slapos.cfg.in

@@ -31,6 +31,8 @@ auto_deploy = {{ slaprunner['auto_deploy'] }}
 auto_deploy_instance = {{ slaprunner['auto_deploy_instance'] }}
 autorun = {{ slaprunner['autorun'] }}
 knowledge0_cfg = {{ slaprunner['knowledge0_file'] }}
+minishell_cwd_file = {{ slaprunner['minishell_cwd_file'] }}
+minishell_history_file = {{ slaprunner['minishell_history_file'] }}
 
 [slapproxy]
 host = {{ slaprunner['ipv4'] }}

stack/monitor/buildout.cfg

@@ -41,7 +41,7 @@ recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/monitor.cfg.in
 output = ${buildout:directory}/monitor.cfg
 filename = monitor.cfg
-md5sum = b330a6fbac771ef5db3e66933f6df8d0
+md5sum = 4d0c3b847c18a56068f04dd926487272
 mode = 0644
 
 [monitor-bin]

stack/monitor/monitor.cfg.in

@@ -193,6 +193,7 @@ recipe = plone.recipe.command
 command = if [ -d $${:source} ]; then ln -s $${:source} $${monitor-directory:private-directory}/instance-logs; fi
 update-command = if [ -d $${:source} ]; then ln -s $${:source} $${monitor-directory:private-directory}/instance-logs; fi
 source = $${monitor-directory:home}/.slapgrid/log/
+location = $${:source}
 
 [cadirectory]
 recipe = slapos.cookbook:mkdirectory