Update Release Candidate

software/kvm/buildout.hash.cfg

@@ -19,11 +19,11 @@ md5sum = 092405e2fba77c22d4dc8cefcab677d8
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = 285558df4686116a92b39250f9e00f07
+md5sum = 4e2aecca03c64d0bcff669652b581dba
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in
-md5sum = 6f4c60f4366728021a6e438ad3dc6956
+md5sum = 73b09e75d617888f6d84d363c0ada9c5
 
 [template-kvm-resilient]
 filename = instance-kvm-resilient.cfg.jinja2

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -40,7 +40,7 @@ config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
 # Request kvm instances
 {% for instance_name, kvm_parameter_dict in slapparameter_dict.get('kvm-partition-dict', {'kvm-default': {}}).items() -%}
 {% set section = 'request-' ~ instance_name -%}
-{% set use_nat = kvm_parameter_dict.get('use-nat', 'True') -%}
+{% set use_nat = kvm_parameter_dict.get('use-nat', True) -%}
 [{{ section }}]
 <= request-common
 software-type = kvm
@@ -97,7 +97,7 @@ config-auto-ballooning = {{ dumps(kvm_parameter_dict.get('auto-ballooning', True
 {% set nat_rules_list = kvm_parameter_dict.get('nat-rules', []) -%}
 {{ setconfig('nat-rules', nat_rules_list | join(' ')) }}
 config-publish-nat-url = True
-config-use-nat = {{ use_nat }}
+config-use-nat = {{ dumps(use_nat) }}
 config-use-tap = {{ dumps(kvm_parameter_dict.get('use-tap', True)) }}
 config-nat-restrict-mode = {{ dumps(kvm_parameter_dict.get('nat-restrict-mode', False)) }}
 config-enable-vhost = {{ dumps(kvm_parameter_dict.get('enable-vhost', False)) }}
@@ -142,11 +142,11 @@ sla-fw_restricted_access = {{ dumps(slapparameter_dict.get('fw-restricted-access
 
 return = 
   url
-{% if frontend_dict.get('frontend-additional-instance-guid', '') %}
+{% if frontend_dict.get('frontend-additional-instance-guid') %}
   url-additional
 {% endif %}
   backend-url
-{% if str(use_nat).lower() == 'true' -%}
+{% if use_nat -%}
 {%   for port in nat_rules_list -%}
 {%     if ':' in port -%}
 {%       set proto, port = port.split(':') -%}
@@ -170,7 +170,7 @@ return =
 {% do monitor_base_url_dict.__setitem__(instance_name, '${' ~ section ~ ':connection-monitor-base-url}') -%}
 {% do publish_dict.__setitem__(instance_name ~ '-backend-url', '${' ~ section ~ ':connection-backend-url}') -%}
 {% do publish_dict.__setitem__(instance_name ~ '-url', '${' ~ section ~ ':connection-url}') -%}
-{% if frontend_dict.get('frontend-additional-instance-guid', '') %}
+{% if frontend_dict.get('frontend-additional-instance-guid') %}
 {% do publish_dict.__setitem__(instance_name ~ '-url-additional', '${' ~ section ~ ':connection-url-additional}') -%}
 {% endif %}
 {% do kvm_instance_dict.__setitem__(instance_name, (use_nat, nat_rules_list)) -%}

software/kvm/instance-kvm.cfg.jinja2

-{% set additional_frontend = (slapparameter_dict.get('frontend-additional-instance-guid', '').strip() != '') %}
-{% set enable_http = slapparameter_dict.get('enable-http-server', 'False').lower() -%}
-{% set use_tap = slapparameter_dict.get('use-tap', 'True').lower() -%}
-{% set use_nat = slapparameter_dict.get('use-nat', 'True').lower() -%}
-{% set wipe_disk = slapparameter_dict.get('wipe-disk-ondestroy', 'False').lower() -%}
-{% set nat_restrict = slapparameter_dict.get('nat-restrict-mode', 'False').lower() -%}
+{% set additional_frontend = slapparameter_dict.get('frontend-additional-instance-guid') %}
+{% set enable_http = slapparameter_dict.get('enable-http-server', False) -%}
+{% set use_tap = slapparameter_dict.get('use-tap', True) -%}
+{% set use_nat = slapparameter_dict.get('use-nat', True) -%}
+{% set wipe_disk = slapparameter_dict.get('wipe-disk-ondestroy', False) -%}
+{% set nat_restrict = slapparameter_dict.get('nat-restrict-mode', False) -%}
 {% set name = slapparameter_dict.get('name', 'localhost') -%}
-{% set disable_ansible_promise = slapparameter_dict.get('disable-ansible-promise', 'True').lower() -%}
-{% set enable_device_hotplug = slapparameter_dict.get('enable-device-hotplug', 'false').lower() -%}
+{% set disable_ansible_promise = slapparameter_dict.get('disable-ansible-promise', True) -%}
 {% set instance_type = slapparameter_dict.get('type', 'standalone') -%}
 {% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
 {% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%}
@@ -116,7 +115,7 @@ tap-mac-address = ${create-tap-mac:mac-address}
 use-tap = ${slap-parameter:use-tap}
 use-nat = ${slap-parameter:use-nat}
 nat-rules = {{ nat_rule_list }}
-nat-restrict= {{ nat_restrict }}
+nat-restrict= {{ dumps(nat_restrict) }}
 enable-vhost = ${slap-parameter:enable-vhost}
 
 virtual-hard-drive-url = ${slap-parameter:virtual-hard-drive-url}
@@ -137,7 +136,7 @@ external-disk-number = ${slap-parameter:external-disk-number}
 external-disk-size = ${slap-parameter:external-disk-size}
 external-disk-format = ${slap-parameter:external-disk-format}
 
-{% if enable_http == 'true' -%}
+{% if enable_http -%}
 httpd-port = ${slap-parameter:httpd-port}
 {% else -%}
 httpd-port = 0
@@ -187,7 +186,7 @@ wrapper-path = ${directory:services}/6tunnel-${:ipv6-port}
 command-line = {{ sixtunnel_executable_location }} -6 -4 -d -l ${:ipv6} ${:ipv6-port} ${:ipv4} ${:ipv4-port}
 hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
-{% if use_nat == 'true' and nat_rule_list -%}
+{% if use_nat and nat_rule_list -%}
 {%   for port in nat_rule_list.split(' ') -%}
 {%     if ':' in port -%}
 {%       set proto, port = port.split(':') -%}
@@ -253,7 +252,7 @@ module = check_command_execute
 name = kvm-disk-image-corruption.py
 config-command = ${kvm-disk-image-corruption-bin:output}
 
-{% if wipe_disk == 'true' -%}
+{% if wipe_disk -%}
 {% do part_list.append('wipe-disk-wrapper') -%}
 {% set wipe_file_list = '${kvm-parameter-dict:disk-path}' -%}
 {% if storage_dict -%}
@@ -418,7 +417,7 @@ name = frontend_additional_promise.py
 config-url = ${publish-connection-information:url-additional}
 {% endif %}
 
-{% if enable_http == 'true' %}
+{% if enable_http %}
 [httpd]
 recipe = slapos.cookbook:simplehttpserver
 host = ${slap-network-information:local-ipv4}
@@ -470,7 +469,7 @@ url-additional = ${request-slave-frontend-additional:connection-secure_access}/v
 {% set disk_number = len(storage_dict) -%}
 maximum-extra-disk-amount = {{ disk_number }}
 {% set iface = 'ens3' -%}
-{% if use_nat == 'true' -%}
+{% if use_nat -%}
 {%   set iface = 'ens4' -%}
 {%   if nat_rule_list -%}
 # Publish NAT port mapping status
@@ -488,14 +487,14 @@ nat-rule-url-{{proto}}-{{port}} = [${slap-network-information:global-ipv6}]:${6t
 {%     endfor -%}
 {%   endif -%}
 {% endif -%}
-{% if use_tap == 'true' -%}
+{% if use_tap -%}
 tap-ipv4 = {{ slap_configuration.get('tap-ipv4-addr', '') }}
 tap-ipv6 = {{ slap_configuration.get('tap-ipv6-addr', '') }}
 {% endif -%}
 
 {% set kvm_http = 'http://${slap-network-information:local-ipv4}:' ~ slapparameter_dict.get('httpd-port', 8081) -%}
-{% if enable_http == 'true' %}
-{%   if use_nat == 'true' -%}
+{% if enable_http %}
+{%   if use_nat -%}
 {%     set kvm_http = 'http://10.0.2.100' -%}
 {%   endif %}
 {%   if slapparameter_dict.get('authorized-key', '') and slapparameter_dict.get('type', '') == 'cluster' -%}
@@ -503,7 +502,7 @@ key_info = Get the publick key file in your VM with the command: wget {{ kvm_htt
 {%   endif %}
 {% endif %}
 
-{% if use_tap == 'true' and slap_configuration.get('tap-ipv4-addr') -%}
+{% if use_tap and slap_configuration.get('tap-ipv4-addr') -%}
 ipv4-network-info =
   PERMANENT SOLUTION: in your VM, add the lines below in /etc/network/interfaces and then run: "ifup {{ iface }}"
   auto {{ iface }}
@@ -511,7 +510,7 @@ ipv4-network-info =
   	address {{ slap_configuration.get('tap-ipv4-addr') }}
   	netmask {{ slap_configuration.get('tap-ipv4-netmask') }}
   	gateway {{ slap_configuration.get('tap-ipv4-gateway') }}
-{% if enable_http == 'true' %}
+{% if enable_http %}
   ${helper:blank-line}
   TEMPORARY SOLUTION: run in your VM the command: "wget -O- {{ kvm_http }}/${network-config-ipv4:filename} | /bin/sh -"
   (the configuration will be gone after the next reboot)
@@ -519,14 +518,14 @@ ipv4-network-info =
 {% endif %}
 
 ipv6-network-info =
-{% if use_tap == 'true' and slap_configuration.get('tap-ipv6-addr') %}
+{% if use_tap and slap_configuration.get('tap-ipv6-addr') %}
   PERMANENT SOLUTION: in your VM, add the lines below in /etc/network/interfaces and then run: "ifup {{ iface }}"
   auto {{ iface }}
   iface {{ iface }} inet6 static
   	address {{ slap_configuration.get('tap-ipv6-gateway') }}
   	netmask {{ slap_configuration.get('tap-ipv6-network').split('/')[1] }}
   	gateway {{ slap_configuration.get('tap-ipv6-addr') }}
-{% if enable_http == 'true' %}
+{% if enable_http %}
   ${helper:blank-line}
   TEMPORARY SOLUTION: run in your VM the command: "wget -O- {{ kvm_http }}/${network-config-ipv6:filename} | /bin/sh -"
   (the configuration will be gone after the next reboot)
@@ -534,14 +533,14 @@ ipv6-network-info =
 {% endif %}
 
 
-{% if use_tap == 'true' and slap_configuration.get('tap-ipv4-addr') -%}
+{% if use_tap and slap_configuration.get('tap-ipv4-addr') -%}
 [network-config-ipv4]
 recipe = plone.recipe.command
 filename = netconfig.sh
 path = ${directory:public}/${:filename}
 ipv4-add-address = ip -4 address add {{ slap_configuration.get('tap-ipv4-addr') }}/{{ slap_configuration.get('tap-ipv4-netmask') }} dev \$IFACE noprefixroute
 ipv4-add-gateway-route = ip -4 address add {{ slap_configuration.get('tap-ipv4-gateway') }} dev \$IFACE
-{%   if nat_restrict == 'true' -%}
+{%   if nat_restrict -%}
 ipv4-add-default-route = ip route add default via {{ slap_configuration.get('tap-ipv4-gateway') }} dev \$IFACE
 {%   elif global_ipv4_prefix -%}
 ipv4-add-default-route = ip route add {{ global_ipv4_prefix }} via {{ slap_configuration.get('tap-ipv4-gateway') }} dev \$IFACE src {{ slap_configuration.get('tap-ipv4-addr') }}
@@ -564,7 +563,7 @@ command =
 update-command = ${:command}
 {% endif -%}
 
-{% if use_tap == 'true' and slap_configuration.get('tap-ipv6-addr') -%}
+{% if use_tap and slap_configuration.get('tap-ipv6-addr') -%}
 [network-config-ipv6]
 recipe = plone.recipe.command
 filename = ipv6_config.sh
@@ -602,7 +601,7 @@ mode = {{ mode }}
 {{   writefile('get-authorized-key', '${directory:public}/authorized_keys', slapparameter_dict.get('authorized-key', ''), '700') }}
 {% endif -%}
 
-{% if use_tap == 'true' and nat_restrict == 'true' -%}
+{% if use_tap and nat_restrict -%}
 # Ask to set default to tap interface in the vm
 {{ writefile('set-default-interface', '${directory:public}/delDefaultIface', iface, '600') }}
 {% do part_list.append('set-default-interface') -%}
@@ -611,7 +610,7 @@ mode = {{ mode }}
 [publish-host-config]
 recipe = plone.recipe.command
 name = {{ slapparameter_dict.get('name', 'localhost') }}
-{% if use_tap == 'true' and slap_configuration.get('tap-ipv4-addr') -%}
+{% if use_tap and slap_configuration.get('tap-ipv4-addr') -%}
 local-ipv4 = {{ slap_configuration['tap-ipv4-addr'] }}
 gateway = {{ slap_configuration.get('tap-ipv4-gateway') }}
 netmask = {{ slap_configuration.get('tap-ipv4-network') }}
@@ -679,7 +678,7 @@ context =
 
 [ansible-vm-promise]
 <= monitor-promise-base
-module = check_execute_comand
+module = check_command_execute
 name = ansible_{{ name }}.py
 config-command = ${ansible-vm-bin:rendered}
 
@@ -756,7 +755,13 @@ data-to-vm =
 keyboard-layout-language = fr
 
 {% for k, v in slapparameter_dict.items() -%}
+{% if k == 'authorized-key' and v -%}
+{% set key_list =  v.split('\n') -%}
+{{ k }} =
+  {{ key_list | join('\n  ') }}
+{% else -%}
 {{ k }} = {{ v }}
+{% endif -%}
 {% endfor -%}
 
 #############################
@@ -769,12 +774,12 @@ keyboard-layout-language = fr
 # Set Additionals parts
 {%   do part_list.append('cluster-url-path') -%}
 {% endif -%}
-{% if enable_http == 'true' %}
+{% if enable_http %}
 {%   do part_list.extend(['httpd', 'httpd-service', 'httpd-promise', 'publish-host-config']) -%}
 {% if slapparameter_dict.get('data-to-vm', '') %}
 {%   do part_list.append('vm-data-content') -%}
 {% endif -%}
-{% if disable_ansible_promise == 'false' %}
+{% if not disable_ansible_promise %}
 {%   do part_list.extend(['ansible-vm-promise', 'logrotate-vm-bootstrap']) -%}
 {% endif -%}
 {% if slapparameter_dict.get('authorized-key', '') and slapparameter_dict.get('type', '') == 'cluster' %}

software/kvm/software.cfg

@@ -52,6 +52,7 @@ eggs =
   slapos.cookbook
   erp5.util
 scripts =
+  websockify
 
 [http-proxy]
 # https://github.com/nodejitsu/node-http-proxy

software/kvm/test/test.py

@@ -54,7 +54,45 @@ else:
       self.fail('This environment is not usable for kvm testing,'
                 ' as it lacks kvm_intel kernel module')
 
+bootstrap_common_param_dict = {
+    # the bootstrap script is vm-bootstrap
+    "bootstrap-script-url": "http://shacache.org/shacache/05105cd25d1ad798b71fd46a206c9b73da2c285a078af33d0e739525a595886785725a68811578bc21f75d0a97700a66d5e75bce5b2721ca4556a0734cb13e65#c98825aa1b6c8087914d2bfcafec3058",
+    "slave-frontend": {
+        "slave-frontend-dict": {}
+        },
+    "authorized-keys": [
+        "ssh-rsa %s key_one" % ("A" * 372),
+        "ssh-rsa %s key_two" % ("B" * 372),
+        "ssh-rsa %s key_three" % ("C" * 372)
+        ],
+    "fw-restricted-access": "off",
+    "fw-authorized-sources": [],
+    "fw-reject-sources": ["10.32.0.0/13"]
+    }
 
+bootstrap_machine_param_dict = {
+    "computer-guid": "local",
+    "disable-ansible-promise": True,
+    "state": "started",
+    "auto-ballooning": True,
+    "ram-size": 4096,
+    "cpu-count": 2,
+    "disk-size": 50,
+    # Debian 10 image
+    "virtual-hard-drive-url": "http://shacache.org/shacache/9d3e6d017754fdd08e5ecf78093dec27fd792fb183df6146006adf003b6f4b98c0388d5a11566627101f7855d77f60e3dd4ba7ce66850f4a8f030573b904d5ab",
+    "virtual-hard-drive-md5sum": "b7928d7b0a2b5e2888f5ddf68f5fe422",
+    "virtual-hard-drive-gzipped": False,
+    "hard-drive-url-check-certificate": False,
+    "use-tap": True,
+    "use-nat": True,
+    "nat-restrict-mode":True,
+    "enable-vhost": True,
+    "external-disk-number": 1,
+    "external-disk-size": 100,
+    "external-disk-format": "qcow2",
+    "enable-monitor": True,
+    "keyboard-layout-language": "fr"
+    }
 @skipUnlessKvm
 class ServicesTestCase(InstanceTestCase):
   def test_hashes(self):
@@ -198,6 +236,25 @@ class TestAccessDefaultAdditional(MonitorAccessMixin, InstanceTestCase):
     )
     self.assertIn('<title>noVNC</title>', result.text)
 
+@skipUnlessKvm
+class TestAccessDefaultBootstrap(MonitorAccessMixin, InstanceTestCase):
+  __partition_reference__ = 'adb'
+  expected_partition_with_monitor_base_url_count = 1
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'_': json.dumps(dict(bootstrap_common_param_dict, **bootstrap_machine_param_dict))}
+
+  def test(self):
+    connection_parameter_dict = self.computer_partition\
+      .getConnectionParameterDict()
+
+    result = requests.get(connection_parameter_dict['url'], verify=False)
+    self.assertEqual(
+      httplib.OK,
+      result.status_code
+    )
+    self.assertIn('<title>noVNC</title>', result.text)
 
 @skipUnlessKvm
 class TestAccessKvmCluster(MonitorAccessMixin, InstanceTestCase):
@@ -270,6 +327,45 @@ class TestAccessKvmClusterAdditional(MonitorAccessMixin, InstanceTestCase):
     )
     self.assertIn('<title>noVNC</title>', result.text)
 
+@skipUnlessKvm
+class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase):
+  __partition_reference__ = 'akcb'
+  expected_partition_with_monitor_base_url_count = 3
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'kvm-cluster'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'_': json.dumps(dict(bootstrap_common_param_dict, **{
+      "kvm-partition-dict": {
+          "test-machine1": bootstrap_machine_param_dict,
+          "test-machine2": dict(bootstrap_machine_param_dict, **{
+              # Debian 9 image
+              "virtual-hard-drive-url": "http://shacache.org/shacache/ce07873dbab7fa8501d1bf5565c2737b2eed6c8b9361b4997b21daf5f5d1590972db9ac00131cc5b27d9aa353f2f94071e073f9980cc61badd6d2427f592e6e8",
+              "virtual-hard-drive-md5sum": "2b113e3cd8276b9740189622603d6f99"
+              })
+          }
+      }))}
+
+  def test(self):
+    connection_parameter_dict = self.computer_partition\
+      .getConnectionParameterDict()
+    result = requests.get(connection_parameter_dict['KVM0-url'], verify=False)
+    self.assertEqual(
+      httplib.OK,
+      result.status_code
+    )
+    self.assertIn('<title>noVNC</title>', result.text)
+
+    result = requests.get(
+      connection_parameter_dict['KVM1-url'], verify=False)
+    self.assertEqual(
+      httplib.OK,
+      result.status_code
+    )
+    self.assertIn('<title>noVNC</title>', result.text)
 
 @skipUnlessKvm
 class TestInstanceResilient(InstanceTestCase):

software/slapos-sr-testing/software.cfg

@@ -184,6 +184,7 @@ eggs =
   ${slapos.test.jupyter-setup:egg}
   ${slapos.test.nextcloud-setup:egg}
   ${slapos.test.turnserver-setup:egg}
+  ${slapos.test.theia-setup:egg}
   ${slapos.test.cloudooo-setup:egg}
   ${slapos.test.dream-setup:egg}
   ${backports.lzma:egg}

software/slaprunner/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 8d6878ff1d2e75010c50a1a2b0c13b24
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = 8d85d8d80f1ae3ff13946d14d0d8fe72
+md5sum = b03f39d483cb7f3554fb40092b5b89fa
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2
@@ -50,7 +50,7 @@ md5sum = 525e37ea8b2acf6209869999b15071a6
 
 [template-slapos-cfg]
 filename = template/slapos.cfg.in
-md5sum = 95de0677e78fc06cc8304cc6caea9169
+md5sum = fa90fc9a9010ce4bb1478ae37da9c56d
 
 [template-slapformat-definition.cfg]
 filename = template/slapformat-definition.cfg.in
@@ -66,7 +66,7 @@ md5sum = 7645048216fcf957f7773534cd0408dc
 
 [template-supervisord]
 filename = template/supervisord.conf.in
-md5sum = 37f053d75752e998fc3bb9e4bf29d776
+md5sum = 5e6c84098440c6bc163898dcafca8c9b
 
 [template-listener-slapgrid]
 filename = template/listener_slapgrid.py.in

software/slaprunner/instance-runner.cfg

@@ -791,9 +791,17 @@ slapproxy-startsecs = 1
 slapproxy-command = $${slaprunner:slapos} proxy start --logfile $${:slapproxy-log} --cfg $${:slapos-cfg}
 slapproxy-log = $${directory:log}/slapproxy.log
 slapformat = slapformat
-slapformat-command = $${slaprunner:slapos} node format --cfg $${:slapos-cfg} --verbose --logfile $${:slapformat-log} --now -i $${:slapformat-definition.cfg}
+slapformat-command = $${slaprunner:slapos} node format --cfg $${:slapos-cfg} --verbose --logfile $${:slapformat-log} --now
 slapformat-log = $${directory:log}/slapos-node-format.log
 slapformat-startretries = 0
+slapboot = slapboot
+slapboot-command = $${slaprunner:slapos} node boot --cfg $${:slapos-cfg} --verbose --logfile $${:slapboot-log}
+slapboot-log = $${directory:log}/slapos-node-boot.log
+slapboot-startretries = 0
+slapbang = slapbang
+slapbang-command = $${slaprunner:slapos} node bang --cfg $${:slapos-cfg} --verbose --logfile $${:slapbang-log}
+slapbang-log = $${directory:log}/slapos-node-bang.log
+slapbang-startretries = 0
 socket_name = unix://$${:socket_path}
 socket_path = $${directory:tmp}/supervisord.sock
 startsecs = 0

software/slaprunner/template/slapos.cfg.in

@@ -21,6 +21,7 @@ pidfile_software =  {{slaprunner['pidfile-software']}}
 pidfile_instance =  {{slaprunner['pidfile-instance']}}
 
 [slapformat]
+input_definition_file = {{ slaprunner['slapformat-definition.cfg'] }}
 partition_amount = {{ slaprunner['partition-amount'] }}
 alter_user = false
 alter_network = false

software/slaprunner/template/supervisord.conf.in

@@ -58,6 +58,34 @@ stderr_logfile = {{ supervisord['no_logfile'] }}
 directory = {{ supervisord['directory'] }}
 environment = PATH="{{- supervisord['path'] -}}"
 
+[program:{{- supervisord['slapboot'] -}}]
+command = {{ supervisord['slapboot-command'] }}
+process_name = {{ supervisord['slapboot'] }}
+numprocs = {{ supervisord['numprocs'] }}
+autostart = {{ supervisord['autostart'] }}
+exitcodes = {{ supervisord['exitcodes'] }}
+startretries = {{ supervisord['slapboot-startretries'] }}
+startsecs = {{ supervisord['startsecs'] }}
+autorestart = {{ supervisord['autorestart'] }}
+stdout_logfile = {{ supervisord['no_logfile'] }}
+stderr_logfile = {{ supervisord['no_logfile'] }}
+directory = {{ supervisord['directory'] }}
+environment = PATH="{{- supervisord['path'] -}}"
+
+[program:{{- supervisord['slapbang'] -}}]
+command = {{ supervisord['slapbang-command'] }}
+process_name = {{ supervisord['slapbang'] }}
+numprocs = {{ supervisord['numprocs'] }}
+autostart = {{ supervisord['autostart'] }}
+exitcodes = {{ supervisord['exitcodes'] }}
+startretries = {{ supervisord['slapbang-startretries'] }}
+startsecs = {{ supervisord['startsecs'] }}
+autorestart = {{ supervisord['autorestart'] }}
+stdout_logfile = {{ supervisord['no_logfile'] }}
+stderr_logfile = {{ supervisord['no_logfile'] }}
+directory = {{ supervisord['directory'] }}
+environment = PATH="{{- supervisord['path'] -}}"
+
 [program:{{- supervisord['slapproxy'] -}}]
 command = {{ supervisord['slapproxy-command'] }}
 process_name = {{ supervisord['slapproxy'] }}

software/slaprunner/test/test.py

@@ -32,6 +32,8 @@ import contextlib
 import base64
 import hashlib
 import subprocess
+import json
+import time
 
 from six.moves.urllib.parse import urlparse
 from six.moves.urllib.parse import quote
@@ -53,6 +55,214 @@ class SlaprunnerTestCase(SlapOSInstanceTestCase):
   # Slaprunner uses unix sockets, so it needs short paths.
   __partition_reference__ = 's'
 
+class SlaprunnerTestCase(SlapOSInstanceTestCase):
+  # Slaprunner uses unix sockets, so it needs short paths.
+  __partition_reference__ = 's'
+
+  def _openSoftwareRelease(self, software_release="erp5testnode/testsuite/dummy"):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    url = "%s/setCurrentProject" % parameter_dict['url']
+
+    data = {
+      "path": "workspace/slapos/software/%s" % software_release,
+    }
+    resp = self._postToSlaprunner(url, data) 
+    self.assertEqual(requests.codes.ok, resp.status_code)
+    self.assertNotEqual(json.loads(resp.text)['code'], 0,
+       'Unexpecting result in call to setCurrentProject: %s' % resp.text)
+
+  def _buildSoftwareRelease(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    url = "%s/runSoftwareProfile" % parameter_dict['url']
+    resp = self._postToSlaprunner(url, {}) 
+    self.assertEqual(requests.codes.ok, resp.status_code)
+    self.assertEqual(json.loads(resp.text)['result'], True,
+       'Unexpecting result in call to runSoftwareProfile: %s' % resp.text)
+
+  def _deployInstance(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    url = "%s/runInstanceProfile" % parameter_dict['url']
+    resp = self._postToSlaprunner(url, {}) 
+    self.assertEqual(requests.codes.ok, resp.status_code)
+    self.assertEqual(json.loads(resp.text)['result'], True,
+       'Unexpecting result in call to runSoftwareProfile: %s' % resp.text)
+
+  def _gitClone(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    url = "%s/cloneRepository" % parameter_dict['url']
+
+    data = {
+      "repo": "https://lab.nexedi.com/nexedi/slapos.git",
+      "name": "workspace/slapos",
+      "email": "slapos@slapos.org",
+      "user": "slapos"
+    }
+    resp = self._postToSlaprunner(url, data)
+    d = json.loads(resp.text)
+    if d['code'] == 0:
+      return "OK"
+
+  def _isSoftwareReleaseReady(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    url = "%s/isSRReady" % parameter_dict['url']
+    resp = self._getFromSlaprunner(url) 
+    if requests.codes.ok != resp.status_code:
+      return -1
+    return resp.text
+
+  def _waitForSoftwareBuild(self, limit=5000):
+    status = self._isSoftwareReleaseReady()
+    while limit > 0 and status != "1":
+      status = self._isSoftwareReleaseReady()
+      limit -= 1
+      if status == '0':
+        self.logger.debug("Software release is Failing to Build. Sleeping...")
+      else:
+        self.logger.debug('Software is still building. Sleeping...')
+      time.sleep(20)
+
+  def _waitForInstanceDeploy(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    url = "%s/slapgridResult" % parameter_dict['url']
+    data = {
+      "position": 0,
+      "log": ""
+      }
+    while True:
+      time.sleep(25)
+      resp = self._postToSlaprunner(url, data) 
+      if requests.codes.ok != resp.status_code:
+        continue
+      if json.loads(resp.text)["instance"]["state"] is False:
+        break
+      self.logger.info('Buildout is still running. Sleeping....')
+    self.logger.info("Instance has been deployed.")
+
+  def _getFromSlaprunner(self, url):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    return requests.get(
+        url,
+        verify=False,
+        auth=(parameter_dict['init-user'], parameter_dict['init-password']))
+
+  def _postToSlaprunner(self, url, data):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    return requests.post(
+        url,
+        verify=False,
+        data=data,
+        auth=(parameter_dict['init-user'], parameter_dict['init-password']))
+
+  def _getFileContent(self, relative_path):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    url = "%s/getFileContent" % parameter_dict['url']
+
+    data = {
+      "file": relative_path 
+    }
+    resp = self._postToSlaprunner(url, data) 
+    self.assertEqual(requests.codes.ok, resp.status_code)
+    self.assertNotEqual(json.loads(resp.text)['code'], 0,
+       'Unexpecting result in call to getFileContent: %s' % resp.text)
+
+    return json.loads(resp.text)["result"]
+
+  def _waitForCloneToBeReadyForTakeover(self, scope="runner-1", limit=500):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    takeover_url = parameter_dict["takeover-%s-url" % scope]
+
+    def getTakeoverPageContent():
+      resp = requests.get(takeover_url, verify=True)
+      self.assertEqual(requests.codes.ok, resp.status_code)
+      return resp.text
+    takeover_page_content = getTakeoverPageContent()
+    while "<b>Last valid backup:</b> No backup downloaded yet, takeover should not happen now." in takeover_page_content:
+      time.sleep(10)
+      if limit < 0:
+        raise Exception("Timeout: No valid Backup")
+      takeover_page_content = getTakeoverPageContent()
+      limit -= 1
+
+    while "<b>Importer script(s) of backup in progress:</b> True" in takeover_page_content:
+      time.sleep(10)
+      if limit < 0:
+        raise Exception("Timeout: Backup still in progress")
+      takeover_page_content = getTakeoverPageContent()
+      limit -= 1
+
+  def _doTakeover(self, scope="runner-1"):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    takeover_url = parameter_dict["takeover-%s-url" % scope]
+    takeover_password = parameter_dict["takeover-%s-password" % scope]
+    
+    resp = requests.get(
+      "%s?password=%s" % (takeover_url, takeover_password),
+      verify=True)
+    self.assertEqual(requests.codes.ok, resp.status_code)
+    self.assertNotIn("Error", resp.text,
+            "An Error occured: %s" % resp.text)
+    self.assertIn("Success", resp.text,
+            "An Success not in %s" % resp.text)
+    return resp.text
+
+class TestWebRunnerBasicUsage(SlaprunnerTestCase):
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+        'auto-deploy': 'true',
+        'software-root': os.path.join(cls.slap._instance_root, "..", "soft"),
+        'buildout-shared-folder': os.path.join(cls.slap._instance_root, "..", "shared"),
+        "slapos-reference": 'slaprunner-basic-test-resiliency'
+       }
+
+  def test_open_software_release(self):
+    self._openSoftwareRelease()
+
+  def test_git_clone(self):
+    self._gitClone()
+
+  @unittest.skip('Skip as _getFileContent dont work for now')
+  def test_basic_usage(self):
+    self._openSoftwareRelease()
+    self._buildSoftwareRelease()
+    self._waitForSoftwareBuild()
+    self._deployInstance()
+    self._waitForInstanceDeploy()
+
+    result = self._getFileContent(
+      "instance_root/slappart0/var/log/log.log")
+
+    self.assertTrue(result.startswith("Hello"),
+      result)
+
+class TestWebRunnerAutorun(SlaprunnerTestCase):
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+        # Auto deploy is required for the isSRReady works. 
+        'auto-deploy': 'true',
+        'autorun': 'true',
+        'software-root': os.path.join(cls.slap._instance_root, "..", "soft"),
+        'buildout-shared-folder': os.path.join(cls.slap._instance_root, "..", "shared"),
+        'slapos-software': 'software/erp5testnode/testsuite/dummy',
+        # XXX HACK!
+        "slapos-reference": 'slaprunner-basic-test-resiliency'
+       }
+
+  @unittest.skip('Skip as _getFileContent dont work for now')
+  def test_basic_usage(self):
+    self._openSoftwareRelease()
+    self._waitForSoftwareBuild()
+    self._waitForSoftwareBuild()
+    self._waitForInstanceDeploy()
+    self._waitForInstanceDeploy()
+
+    result = self._getFileContent(
+      "instance_root/slappart0/var/log/log.log")
+
+    self.assertTrue(result.startswith("Hello"), result)
 
 class TestWeb(SlaprunnerTestCase):
   def test_slaprunner(self):
@@ -251,8 +461,22 @@ class ServicesTestCase(SlaprunnerTestCase):
 
       self.assertIn(expected_process_name, process_names)
 
+class TestCustomFrontend(SlaprunnerTestCase):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'custom-frontend-backend-url': 'https://www.erp5.com',
+      'custom-frontend-backend-type': 'redirect',
+    }
+
+  def test(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    # slapproxy returns the backend URL when requesting a slave frontend
+    self.assertEqual(
+      parameter_dict['custom-frontend-url'],
+      'https://www.erp5.com')
 
-class TestInstanceResilient(SlaprunnerTestCase):
+class TestResilientInstance(SlaprunnerTestCase):
   instance_max_retry = 20
 
   @classmethod
@@ -278,18 +502,84 @@ class TestInstanceResilient(SlaprunnerTestCase):
         'url',
         'webdav-url']))
 
+class TestResilientCustomFrontend(TestCustomFrontend):
+  instance_max_retry = 20
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'resilient'
+
+class TestResilientWebInstance(TestWeb):
+  instance_max_retry = 20
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'resilient'
+
+  def test_public_url(self):
+    pass # Disable until we can write on runner0 rather them
+         # on root partition
+
+class TestResilientWebrunnerBasicUsage(TestWebRunnerBasicUsage):
+  instance_max_retry = 20
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'resilient'
+
+
+class TestResilientWebrunnerAutorun(TestWebRunnerAutorun):
+  instance_max_retry = 20
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'resilient'
+
+class TestResilientDummyInstance(SlaprunnerTestCase):
+  instance_max_retry = 20
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'resilient'
 
-class TestCustomFrontend(SlaprunnerTestCase):
   @classmethod
   def getInstanceParameterDict(cls):
     return {
-      'custom-frontend-backend-url': 'https://www.erp5.com',
-      'custom-frontend-backend-type': 'redirect',
+        'resiliency-backup-periodicity': '*/6 * * * *',
+        'auto-deploy-instance': 'false',
+        'software-root': os.path.join(cls.slap._instance_root, "..", "soft"),
+        'buildout-shared-folder': os.path.join(cls.slap._instance_root, "..", "shared"),
+        'auto-deploy': 'true',
+        # XXX HACK!
+        "slapos-reference": 'slaprunner-erp5-resiliency',
+        "slapos-httpd-port": '9687'
        }
 
-  def test(self):
-    parameter_dict = self.computer_partition.getConnectionParameterDict()
-    # slapproxy returns the backend URL when requesting a slave frontend
-    self.assertEqual(
-      parameter_dict['custom-frontend-url'],
-      'https://www.erp5.com')
+  @unittest.skip('Skip as _getFileContent dont work for now')
+  def test_basic_resilience(self):
+    self._openSoftwareRelease()
+    self._buildSoftwareRelease()
+    self._waitForSoftwareBuild()
+    self._deployInstance()
+    self._waitForInstanceDeploy()
+
+    result = self._getFileContent(
+      "instance_root/slappart0/var/log/log.log")
+
+    self.assertTrue(result.startswith("Hello"), result)
+
+    # We should ensure here that the resilience was indeed
+    # Propagates and test succeeded.
+    time.sleep(900)
+
+    self._waitForCloneToBeReadyForTakeover()
+    self._doTakeover()
+    self.slap.waitForInstance(20) 
+
+    previous_computer_partition = self.computer_partition
+    self.computer_partition = self.requestDefaultInstance()
+
+
+    result_after = self._getFileContent(
+      "instance_root/slappart0/var/log/log.log")
+
+    self.assertTrue(result_after.startswith("Hello"), result_after)
+
+    self.assertIn(result, result_after,
+            "%s not in %s" % (result, result_after))
+

stack/slapos.cfg

@@ -118,7 +118,7 @@ eggs =
 [versions]
 setuptools = 44.0.0
 # Use SlapOS patched zc.buildout
-zc.buildout = 2.7.1+slapos006
+zc.buildout = 2.7.1+slapos005
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
 zc.recipe.egg = 2.0.3+slapos003
 # Use own version of h.r.download to be able to open .xz and .lz archives