Runner: Merge development into master for release

Conflicts:
	software/slaprunner/schema.json
	software/slaprunner/software.cfg.json

software/slaprunner/TODO.txt

 - resilient sr: Cloned instances should not launch slapgrid-sr if it was not launched on export instance
 - shellinabox password should be the same in all the resilient instances
 - add test for parameter auto-deploy-instance
+- Add download facility in file browser

software/slaprunner/common.cfg

@@ -43,7 +43,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner.cfg
 output = ${buildout:directory}/template-runner.cfg.in
-md5sum = b5644037da14e79576840975415ee1a4
+md5sum = a4f3127bc82ff3a671a65a150f26c969
 mode = 0644
 
 [template-runner-import-script]
@@ -103,7 +103,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/httpd_conf.in
 download-only = true
-md5sum = 444399390eb19c7e23dd06593c067b4d
+md5sum = 94bc5d6f1afc4e075d530fed496f47c2
 filename = httpd_conf.in
 mode = 0644
 
@@ -118,7 +118,7 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
 [template-slapos-cfg]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/${:filename}
-md5sum = ba3721425ba34e265b1f8390ed7196cf
+md5sum = ca57a8bf1d4da20344fab602a69b807f
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 filename = slapos.cfg.in
 download-only = true
@@ -145,7 +145,7 @@ mode = 0644
 [template-supervisord]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/${:filename}
-md5sum = af2d78440a358ec268d7ece4fce99ec9
+md5sum = 239fe88e1522afe7686572dd34568720
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 filename = supervisord.conf.in
 download-only = true
@@ -160,6 +160,15 @@ filename = listener_slapgrid.py.in
 download-only = true
 mode = 0644
 
+[cors-domain-cgi]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/template/${:filename}
+download-only = true
+md5sum = 4c44d617d5bfd1db8695200e896480c0
+destination = ${buildout:parts-directory}/monitor-template-cors-domain-cgi
+filename = cors-domain.jinja
+mode = 0644
+
 [eggs]
 recipe = z3c.recipe.scripts
 eggs =
@@ -205,7 +214,3 @@ recipe = zc.recipe.egg
 interpreter = pythonwitheggs
 eggs +=
   supervisor
-
-[versions]
-# We still need to use the older version.
-slapos.core = 1.0.5

software/slaprunner/httpd_conf.in

@@ -27,6 +27,7 @@ LoadModule dav_module modules/mod_dav.so
 LoadModule dav_fs_module modules/mod_dav_fs.so
 LoadModule cache_module modules/mod_cache.so
 LoadModule file_cache_module modules/mod_file_cache.so
+LoadModule setenvif_module modules/mod_setenvif.so
 
 ErrorLog "{{ parameters.path_error_log }}"
 LogFormat "%h %l %u %t \"%r\" %>s %b" common
@@ -45,6 +46,11 @@ SSLHonorCipherOrder On
 SSLCipherSuite RC4-SHA:HIGH:!ADH
 SSLEngine   On
 
+Include {{ parameters.httpd_cors_file }}
+Header set Access-Control-Allow-Credentials "true"
+Header set Access-Control-Allow-Methods "PROPFIND, PROPPATCH, COPY, MOVE, DELETE, MKCOL, LOCK, UNLOCK, PUT, GETLIB, VERSION-CONTROL, CHECKIN, CHECKOUT, UNCHECKOUT, REPORT, UPDATE, CANCELUPLOAD, HEAD, OPTIONS, GET, POST"
+Header set Access-Control-Allow-Headers "Overwrite, Destination, Content-Type, Depth, User-Agent, X-File-Size, X-Requested-With, If-Modified-Since, X-File-Name, Cache-Control, Authorization"
+
 Alias /public {{ parameters.runner_home }}/public
 <Directory {{ parameters.runner_home }}/public>
     Order Allow,Deny
@@ -69,7 +75,9 @@ Alias /share {{ parameters.runner_home }}
     AuthType Basic
     AuthName "webdav"
     AuthUserFile "{{ parameters.etc_dir }}/.htpasswd"
-    Require valid-user
+    <LimitExcept OPTIONS>
+        Require valid-user
+    </LimitExcept>
 </Directory>
 
 ScriptSock {{ parameters.path_pid }}

software/slaprunner/schema.json → software/slaprunner/instance-runner-input-schema.json

@@ -91,6 +91,11 @@
             "minimum": 9683,
             "exclusiveMinimum": true,
         },
+        "cpu-usage-ratio": {
+            "description": "Ratio of the CPU use for compilation, if value is set to n, compilation will use number-of-cpu/n of cpus (need instance restart)",
+            "type": "integer",
+            "default" : 4
+        },
         "resilient-clone-number": {
             "title": "Number of Resilient Clones",
             "description": "Number of clones to be deployed when a resilient runner, if missing a single clone is provided. Its value can be 1 or 2",

software/slaprunner/instance-runner-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Values returned by Runner instanciation",
+  "properties": {
+    "backend_url": {
+      "description": "ipv6 url to directly access your runner",
+      "type": "string"
+    },
+    "access_url": {
+      "description": "Url to set up your account and then login into your webrunner",
+      "type": "string"
+    },
+    "url": {
+      "description": "Url to access your runner once you set your account",
+      "type": "string"
+    },
+    "ssh_command": {
+      "description": "SSH command used to access your runner in ssh when you provided a ssh public key"
+      "type": "string"
+    },
+    "monitor_url": {
+      "description": "Url to access the monitoring interface of your instance. Use this url to get the recovery code needed to set up your runner",
+      "type": "string"
+    },
+    "webdav_url": {
+      "description": "Url for webdav access of your runner it access the root of your runner. Same users as the one for the runner",
+      "type": "string"
+    },
+    "public_url": {
+      "description": "Url of the public folder of your runner",
+      "type": "string"
+    },
+    "git_public_url": {
+      "description": "Public url of your repositories.",
+      "type": "string"
+    },
+    "git_private_url": {
+      "description": "Private url for your repositories. You can use it to push.",
+      "type": "string"
+    }
+  },
+  "type": "object"
+}
\ No newline at end of file

software/slaprunner/instance-runner.cfg

@@ -50,6 +50,7 @@ parts =
   monitor-instance-log-access
 ## Monitor for runner
   monitor-current-log-access
+  monitor-deploy-cors-domain-cgi
 
 extends = ${monitor-template:output}
 
@@ -190,6 +191,7 @@ minishell_cwd_file = $${directory:etc}/.minishell-cwd
 minishell_history_file = $${directory:etc}/.minishell_history
 software_info_json = $${runnerdirectory:home}/software_info.json
 instance_info_json = $${runnerdirectory:home}/instance_info.json
+path = $${shell:path}
 
 [test-runner]
 <= slaprunner
@@ -207,7 +209,7 @@ auto_deploy = True
 recipe = slapos.cookbook:wrapper
 command-line = ${buildout:directory}/bin/slaprunnertest
 wrapper-path = $${directory:bin}/runTestSuite
-environment = RUNNER_CONFIG=$${slapos-test-cfg:rendered}
+environment = RUNNER_CONFIG=$${slapos-cfg:rendered}
 
 # Deploy dropbear (minimalist SSH server)
 [sshkeys-directory]
@@ -328,6 +330,7 @@ project_public_folder = $${runnerdirectory:public-project}
 runner_home = $${runnerdirectory:home}
 git_http_backend = ${git:location}/libexec/git-core/git-http-backend
 cgi_httpd_conf = $${monitor-httpd-configuration-file:rendered}
+httpd_cors_file = $${monitor-httpd-cors:location}
 
 [httpd-conf]
 recipe = slapos.recipe.template:jinja2
@@ -642,12 +645,12 @@ pidfile = $${directory:run}/supervisord.pid
 server = $${slaprunner:ipv4}:$${:port}
 port = 39986
 slapgrid-cp = slapgrid-cp
-slapgrid-cp-command = $${slaprunner:slapos} node instance --all --cfg $${slaprunner:slapos.cfg} --pidfile $${:slapgrid-cp-pid} --verbose --logfile $${:slapgrid-cp-log}
+slapgrid-cp-command = $${slaprunner:slapos} node instance --all --cfg $${:slapos-cfg} --pidfile $${:slapgrid-cp-pid} --verbose --logfile $${:slapgrid-cp-log}
 slapgrid-cp-log = $${runnerdirectory:home}/instance.log
 slapgrid-cp-pid = $${directory:run}/slapgrid-cp.pid
 slapgrid-cp-startretries = 0
 slapgrid-sr = slapgrid-sr
-slapgrid-sr-command = $${slaprunner:slapos} node software --all --cfg $${slaprunner:slapos.cfg} --pidfile $${:slapgrid-sr-pid} --verbose --logfile $${:slapgrid-sr-log}
+slapgrid-sr-command = $${slaprunner:slapos} node software --all --cfg $${:slapos-cfg} --pidfile $${:slapgrid-sr-pid} --verbose --logfile $${:slapgrid-sr-log}
 slapgrid-sr-log = $${runnerdirectory:home}/software.log
 slapgrid-sr-pid = $${directory:run}/slapgrid-sr.pid
 slapgrid-sr-startretries = 0
@@ -655,7 +658,7 @@ slapproxy = slapproxy
 slapproxy-autorestart = true
 slapproxy-autostart = true
 slapproxy-startsecs = 1
-slapproxy-command = $${slaprunner:slapos} proxy start --logfile $${:slapproxy-log} --cfg $${slaprunner:slapos.cfg}
+slapproxy-command = $${slaprunner:slapos} proxy start --logfile $${:slapproxy-log} --cfg $${:slapos-cfg}
 slapproxy-log = $${directory:log}/slapproxy.log
 socket_name = unix://$${:socket_path}
 socket_path = $${directory:tmp}/supervisord.sock
@@ -663,13 +666,17 @@ startsecs = 0
 # This file logs errors from listeners. Supervisord has its own logfile.
 # Processes should handle their logs by themselves
 stderr_logfile = $${directory:log}/supervisord-errors.log
+slapos-cfg = $${slaprunner:slapos.cfg}
 
 [supervisord-conf]
 recipe = slapos.recipe.template:jinja2
 template = ${template-supervisord:location}/${template-supervisord:filename}
 rendered = $${directory:etc}/supervisord.conf
 context =
+    import multiprocessing multiprocessing
+    import builtin __builtin__
     section supervisord supervisord
+    key slapparameter_dict slap-configuration:configuration
     key listener_slapgrid listener-slapgrid-bin:rendered
 
 [listener-slapgrid-bin]
@@ -696,3 +703,25 @@ port = $${supervisord:port}
 [monitor-current-log-access]
 < = monitor-directory-access
 source = $${directory:log}
+
+[monitor-deploy-cors-domain-cgi]
+recipe = slapos.recipe.template:jinja2
+template = ${cors-domain-cgi:location}/${cors-domain-cgi:filename}
+rendered = $${monitor-directory:knowledge0-cgi}/$${:filename}
+filename = cors-domain.cgi
+mode = 0744
+context =
+  raw config_cfg $${buildout:directory}/knowledge0.cfg
+  raw timestamp $${buildout:directory}/.timestamp
+  raw python_executable ${buildout:executable}
+  key apache_file httpd-parameters:httpd_cors_file
+  key pwd monitor-directory:knowledge0-cgi
+  key this_file :filename
+  key httpd_graceful cgi-httpd-graceful-wrapper:rendered
+
+[monitor-httpd-cors]
+recipe = plone.recipe.command
+command = if [ ! -f $${:location} ]; then touch $${:location}; fi
+location = $${directory:etc}/$${:filename}
+filename = httpd-cors.cfg
+stop-on-error = true

software/slaprunner/software.cfg

@@ -14,6 +14,7 @@ async = 0.6.1
 cns.recipe.symlink = 0.2.3
 collective.recipe.environment = 0.2.0
 collective.recipe.template = 1.11
+ecdsa = 0.11
 erp5.util = 0.4.41
 gitdb = 0.5.4
 gunicorn = 19.1.1
@@ -22,26 +23,22 @@ pycrypto = 2.6.1
 rdiff-backup = 1.0.5
 slapos.recipe.build = 0.12
 slapos.recipe.download = 1.0.dev-r4053
-slapos.toolbox = 0.40.1
+slapos.toolbox = 0.40.2
 smmap = 0.8.2
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
-# slapos.toolbox==0.39.3
+# slapos.toolbox==0.40.2
 GitPython = 0.3.2.RC1
 
 # Required by:
-# slapos.toolbox==0.39.3
+# slapos.toolbox==0.40.2
 atomize = 0.2.0
 
 # Required by:
-# paramiko==1.14.1
-ecdsa = 0.11
-
-# Required by:
-# slapos.toolbox==0.39.3
+# slapos.toolbox==0.40.2
 feedparser = 5.1.3
 
 # Required by:
-# slapos.toolbox==0.39.3
-paramiko = 1.14.1
+# slapos.toolbox==0.40.2
+paramiko = 1.15.1
\ No newline at end of file

software/slaprunner/template/cors-domain.jinja

+#!{{ python_executable }}
+
+import cgi
+import cgitb
+import ConfigParser
+import os
+import re
+import subprocess
+
+cgitb.enable(display=0, logdir="/tmp/cgi.log")
+form = cgi.FieldStorage()
+
+apache_file = "{{ apache_file }}"
+config_file = "{{ config_cfg }}"
+
+
+if not os.path.exists(config_file):
+  print "Your software does <b>not</b> embed 0-knowledge. \
+  This interface is useless in this case</body></html>"
+  exit(0)
+
+
+parser = ConfigParser.ConfigParser()
+parser.read(config_file)
+if not parser.has_section("cors"):
+  parser.add_section("cors")
+
+if not parser.has_option("cors", "cors-domain"):
+  parser.set("cors", "cors-domain", "")
+
+if "cors-domain" in form:
+  parser.set("cors", "cors-domain", form["cors-domain"].value)
+  cors_domain_list = form["cors-domain"].value.split()
+  cors_string = ""
+  for domain in cors_domain_list:
+    if cors_string:
+      cors_string += '|'
+    cors_string += re.escape(domain)
+  with open(apache_file, 'w') as file:
+    file.write('SetEnvIf Origin "^http(s)?://(.+\.)?(%s)$" origin_is=$0\n' % cors_string)
+    file.write('Header always set Access-Control-Allow-Origin %{origin_is}e env=origin_is')
+  subprocess.call('{{ httpd_graceful }}')
+
+cors_domain_parameter = parser.get("cors", "cors-domain")
+
+with open(config_file, 'w') as file:
+  parser.write(file)
+
+print "<html><head>"
+print "<link rel=\"stylesheet\" href=\"static/pure-min.css\">"
+print "<link rel=\"stylesheet\" href=\"static/style.css\">"
+print "</head><body>"
+print "<h1>Cors Domains :</h1>"
+print "<p>Enter domain names separated by space to authorized CORS on your dav storage for these domains</p>"
+print "<form action=\"/index.cgi\" method=\"post\" class=\"pure-form-aligned\">"
+print "<input type=\"hidden\" name=\"posting-script\" value=\"{{ pwd }}/{{ this_file }}\">"
+print "<div class=\"pure-control-group\">"
+print '<label for="cors-domain">CORS Domains</label>'
+print '<input type="text" name="cors-domain" value="%s">' % cors_domain_parameter
+print "</div>"
+print "<div class=\"pure-controls\"><button type=\"submit\" class=\"pure-button \
+ pure-button-primary\">Save</button></div></form>"
+print "</body></html>"

software/slaprunner/template/slapos.cfg.in

@@ -34,6 +34,7 @@ autorun = {{ slaprunner['autorun'] }}
 knowledge0_cfg = {{ slaprunner['knowledge0_file'] }}
 minishell_cwd_file = {{ slaprunner['minishell_cwd_file'] }}
 minishell_history_file = {{ slaprunner['minishell_history_file'] }}
+path = {{ slaprunner['path'] }}
 
 [slapproxy]
 host = {{ slaprunner['ipv4'] }}

software/slaprunner/template/supervisord.conf.in

@@ -26,7 +26,7 @@ autorestart = {{ supervisord['autorestart'] }}
 stdout_logfile = {{ supervisord['no_logfile'] }}
 stderr_logfile = {{ supervisord['no_logfile'] }}
 directory = {{ supervisord['directory'] }}
-environment = PATH="{{- supervisord['path'] -}}"
+environment = PATH="{{- supervisord['path'] -}}",MAKEFLAGS="-j{{- '%d' % builtin.max(1, (multiprocessing.cpu_count() / builtin.int(slapparameter_dict.get('cpu-usage-ration', 4)))) -}}"
 
 [program:{{- supervisord['slapgrid-cp'] -}}]
 command = {{ supervisord['slapgrid-cp-command'] }}