software/theia: Simplify and improve embedded SR

See merge request !1139

software/theia/buildout.hash.cfg

@@ -15,24 +15,28 @@
 
 [instance-theia]
 _update_hash_filename_ = instance-theia.cfg.jinja.in
-md5sum = 7ea72f2300dd2019c2d27549e7128dec
+md5sum = 7d146ca31d7e44e909386e21e0a562ec
 
 [instance]
 _update_hash_filename_ = instance.cfg.in
-md5sum = af4d92a95dc25c2f64ddbd634996e46c
+md5sum = e211c439571e2900f9f35482c9638d06
 
 [instance-import]
 _update_hash_filename_ = instance-import.cfg.jinja.in
-md5sum = 29bc1c5b76d20fd46430dff5c72d192b
+md5sum = 23c3df4a889ebfa9f0a94e873e95ad3b
 
 [instance-export]
 _update_hash_filename_ = instance-export.cfg.jinja.in
-md5sum = e2630148998c3cdf6d03b5e884d7f464
+md5sum = 84ceb4c9ee0f07fce8518ef7517ce1d4
 
 [instance-resilient]
 _update_hash_filename_ = instance-resilient.cfg.jinja
 md5sum = ad9499e7355ded4975ad313442cecb7a
 
+[slapos-standalone-script]
+_update_hash_filename_ = slapos_standalone_script.py.jinja
+md5sum = ef5b73648513caf46573f3302953790f
+
 [theia-common]
 _update_hash_filename_ = theia_common.py
 md5sum = 6a25c6a7f1beb27232a3c9acd8a76500

software/theia/instance-export.cfg.jinja.in

@@ -92,14 +92,14 @@ inline =
   then
     echo "ERROR export script last ran on " $(date -r {{ repr(exitcodefile) }})
     exit 1
-  elif [ $( cat {{ repr(exitcodefile) }}) -ne 0 ]
+  elif [ "$(cat {{ repr(exitcodefile) }})" = 0 ]
   then
+    echo "OK export script last ran on " $(date -r {{ repr(exitcodefile) }})
+    exit 0
+  else
     echo "ERROR export script failed on " $(date -r {{ repr(exitcodefile) }})
     cat {{ repr(errorfile) }}
     exit 1
-  else
-    echo "OK export script last ran on " $(date -r {{ repr(exitcodefile) }})
-    exit 0
   fi
 {%- endraw %}
 

software/theia/instance-import.cfg.jinja.in

@@ -154,14 +154,14 @@ inline =
   then
     echo "ERROR import script last ran on " $(date -r {{ repr(exitcodefile) }})
     exit 1
-  elif [ $( cat {{ repr(exitcodefile) }}) -ne 0 ]
+  elif [ "$(cat {{ repr(exitcodefile) }})" = 0 ]
   then
+    echo "OK import script last ran on " $(date -r {{ repr(exitcodefile) }})
+    exit 0
+  else
     echo "ERROR import script failed on " $(date -r {{ repr(exitcodefile) }})
     cat {{ repr(errorfile) }}
     exit 1
-  else
-    echo "OK import script last ran on " $(date -r {{ repr(exitcodefile) }})
-    exit 0
   fi
 {%- endraw %}
 

software/theia/instance-input-schema.json

@@ -15,20 +15,14 @@
       ],
       "default": "running"
     },
-    "embedded-sr": {
-      "title": "Embedded Software URL",
-      "description": "Optional URL of a software to be embedded",
-      "type": "string"
-    },
-    "embedded-sr-type": {
-      "title": "Embedded Software Type",
-      "description": "Type of the optional embedded software",
-      "type": "string"
-    },
-    "embedded-instance-parameters": {
-      "title": "Embedded Instance Parameters",
-      "description": "Parameters for the embedded instance, as a JSON dict",
-      "type": "string"
+    "initial-embedded-instance": {
+      "title": "Initial Embedded Instance Configuration",
+      "description": "One-shot optional JSON preconfiguration for an embedded instance. Only applied once when Theia is instantiated. Changing this option afterward will have no effect.",
+      "type": "string",
+      "examples": [
+        "{\"software-url\": \"~/srv/project/slapos/software/html5as-base/software.cfg\"}",
+        "{\"software-url\": \"~/srv/project/slapos/software/html5as/software.cfg\", \"software-type\": \"replicate\", \"instance-parameters\": {\"replicate-quantity\": 3}}"
+      ]
     },
     "frontend-guid": {
       "title": "Frontend Instance ID",

software/theia/instance-theia.cfg.jinja.in

-{% set parameter_dict = dict(default_parameter_dict, **parameter_dict) %}
-{% set additional_frontend = parameter_dict['additional-frontend-guid'] %}
+{%- set parameter_dict = dict(default_parameter_dict, **parameter_dict) %}
+{%- set additional_frontend = parameter_dict['additional-frontend-guid'] %}
+{%- set embedded_instance_config = parameter_dict['initial-embedded-instance'] %}
 
 [buildout]
 extends =
@@ -10,7 +11,6 @@ theia-environment-parts =
   slapos-repository
   runner-link
   settings.json
-  request-script-template
 
 theia-parts =
   frontend-reload
@@ -98,6 +98,9 @@ instance-promises =
   $${slapos-standalone-listen-promise:name}
   $${slapos-standalone-ready-promise:name}
   $${slapos-autorun-promise:name}
+  {% if embedded_instance_config %}
+  $${embedded-instance-requested-promise:name}
+  {% endif %}
 
 [theia-listen-promise]
 <= monitor-promise-base
@@ -161,6 +164,14 @@ config-service = $${slapos-autorun:service-name}
 config-expect = $${slapos-autorun:autorun}
 config-run-directory = $${directory:runner}/var/run
 
+{% if embedded_instance_config %}
+[embedded-instance-requested-promise]
+<= monitor-promise-base
+promise = check_command_execute
+name = embedded-instance-requested-promise.py
+config-command = $${embedded-instance-requested-promise-script:output}
+{% endif %}
+
 
 # Remote Caddy Frontend
 # ---------------------
@@ -402,7 +413,7 @@ base-url = $${theia-service:base-url}
 [theia-shell]
 recipe = slapos.recipe.template:jinja2
 output = $${directory:bin}/$${:_buildout_section_name_}
-inline
+inline =
   {% raw -%}
   #!{{ bash }}
   SHELL=$BASH
@@ -441,35 +452,40 @@ command =
 # Embedded Instance
 # -----------------
 
-{%- set embedded_sr = parameter_dict['embedded-sr'] %}
-{%- set embedded_sr_type = parameter_dict['embedded-sr-type'] %}
-{%- set embedded_instance_parameters = parameter_dict['embedded-instance-parameters'] %}
-{%- if embedded_sr %}
-{%-   if embedded_sr.startswith('~/') %}
-{%-     set embedded_sr = os_module.path.join(partition_root_path, embedded_sr[2:]) %}
-{%-     set embedded_sr = os_module.path.normpath(embedded_sr) %}
-{%-   endif %}
-[request-embedded-instance-script]
-recipe = slapos.recipe.template
-output = $${directory:project}/request_embedded.sh
+[embedded-instance-config]
+recipe = slapos.recipe.template:jinja2
+output = $${directory:etc}/$${:_buildout_section_name_}.json
+once = $${:output}
+config = {{ dumps(embedded_instance_config) }}
+context =
+  key config :config
 inline =
-  #!/bin/sh
-
-  slapos supply {{ embedded_sr }} slaprunner
-
-  slapos request "embedded_instance" {{ embedded_sr }}
-{%-   if embedded_sr_type %} --type {{ embedded_sr_type }} {%- endif %}
-{%-   if embedded_instance_parameters %} --parameters-file $${embedded-instance-parameters:output}
-
-[embedded-instance-parameters]
-recipe = slapos.recipe.template
-output = $${directory:project}/$${:_buildout_section_name_}.json
-inline = {{ embedded_instance_parameters | indent(2) }}
-{%-   endif %}
-{%- endif %}
+{%-     raw %}
+  {{ config or "{}"}}
+{%-     endraw %}
 
-{%- set embedded_digest = str(embedded_sr) + str(embedded_sr_type) + str(embedded_instance_parameters) %}
-{%- set embedded_digest_hash = hashlib_module.md5(embedded_digest.encode()).hexdigest() %}
+[embedded-instance-requested-promise-script]
+recipe = slapos.recipe.template:jinja2
+output = $${directory:bin}/$${:_buildout_section_name_}
+exitcode-file = $${slapos-standalone-script:embedded-request-exitcode-file}
+context =
+  key exitcodefile :exitcode-file
+{%- raw %}
+inline =
+  #!/bin/sh
+  if ! [ -f {{ repr(exitcodefile) }} ]
+  then
+    echo "ERROR embedded_instance has not been requested"
+    exit 1
+  elif [ "$(cat {{ repr(exitcodefile) }})" = 0 ]
+  then
+    echo "OK embedded_instance has been sucessfully requested"
+    exit 0
+  else
+    echo "ERROR request of embedded_instance failed"
+    exit 1
+  fi
+{%- endraw %}
 
 
 # SlapOS Standalone
@@ -485,10 +501,14 @@ ip = {{ ipv4_random }}
 ipv4 = {{ ipv4_random }}
 ipv6 = {{ ipv6_random }}
 port = $${slapos-standalone-port:port}
+base-directory = $${directory:runner}
+software-root = $${directory:runner}/software
+instance-root = $${directory:runner}/instance
 local-software-release-root = $${directory:home}
+slapos-bin = ${buildout:bin-directory}/slapos
 slapos-configuration = $${directory:runner}/etc/slapos.cfg
 computer-id = slaprunner
-abstract-socket-path = $${directory:home}/standalone-{{ embedded_digest_hash[:16] }}
+abstract-socket-path = $${directory:home}/standalone-ready
 
 [slapos-standalone-activate]
 recipe = slapos.recipe.template
@@ -500,98 +520,23 @@ inline =
   echo 'Standalone SlapOS for computer `$${slapos-standalone-config:computer-id}` activated'
 
 [slapos-standalone-script]
-recipe = slapos.recipe.template
+recipe = slapos.recipe.template:jinja2
 output = $${directory:bin}/$${:_buildout_section_name_}
-inline =
-  #!${python-for-standalone:executable}
-  import glob
-  import json
-  import os
-  import signal
-  import socket
-  import subprocess
-  import sys
-  import time
-  import traceback
-
-  import slapos.slap.standalone
-
-  # Include this hash, so that if it changes the standalone service will be restarted
-  # {{ embedded_digest_hash }}
-
-  shared_parts = """{{ '''${buildout:shared-part-list}''' | indent(2) }}"""
-  shared_part_list = [x.strip() for x in shared_parts.splitlines() if x.strip()]
-  partition_forward_configuration = (
-    slapos.slap.standalone.PartitionForwardAsPartitionConfiguration(
-      master_url="$${slap-connection:server-url}",
-      computer="$${slap-connection:computer-id}",
-      partition="$${slap-connection:partition-id}",
-      cert="$${slap-connection:cert-file}",
-      key="$${slap-connection:key-file}",
-      software_release_list=(
-        'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg',
-      ),
-    ),
-  )
-  standalone = slapos.slap.standalone.StandaloneSlapOS(
-    "$${directory:runner}",
-    "$${slapos-standalone-config:ipv4}",
-    $${slapos-standalone-config:port},
-    computer_id="$${slapos-standalone-config:computer-id}",
-    shared_part_list=shared_part_list,
-    software_root="$${directory:runner}/software",
-    instance_root="$${directory:runner}/instance",
-    partition_forward_configuration=partition_forward_configuration,
-    slapos_bin="${buildout:bin-directory}/slapos",
-    local_software_release_root="$${slapos-standalone-config:local-software-release-root}",
-  )
-
-  def signal_handler(signum, frame):
-    print("Signal {signum} received".format(signum=signum))
-    sys.exit()
-  signal.signal(signal.SIGTERM, signal_handler)
-
-  standalone.start()
-  try:
-    partition_count = 20
-    print("Standalone SlapOS: Formatting %d partitions" % partition_count)
-    standalone.format(partition_count, '$${slapos-standalone-config:ipv4}', '$${slapos-standalone-config:ipv6}')
-    print("Standalone SlapOS for computer `$${slapos-standalone-config:computer-id}` started")
-    # Run instance at least once, to start the supervisor managing instances.
-    try:
-      standalone.waitForInstance(max_retry=0)
-    except slapos.slap.standalone.SlapOSNodeCommandError as e:
-      print("Error instanciating: {}".format(e))
-{%- if embedded_sr %}
-    # Compatibility layer
-    try:
-      for cp in standalone.computer.getComputerPartitionList():
-        if cp.getInstanceParameterDict().get("instance_title") == "Embedded Instance":
-          print("Renaming 'Embedded Instance' into 'embedded_instance'")
-          cp.rename(new_name="embedded_instance")
-          break
-    except Exception:
-      print("Exception in compatibility layer, printing and moving on")
-      traceback.print_exc()
-    # Run request script
-    print("Running SlapOS script $${request-embedded-instance-script:output}")
-    slapos_env = {
-      'PATH': os.path.dirname(standalone._slapos_bin),
-      'SLAPOS_CONFIGURATION': standalone._slapos_config,
-      'SLAPOS_CLIENT_CONFIGURATION': standalone._slapos_config
-    }
-    subprocess.call(("$${request-embedded-instance-script:output}",), env=slapos_env)
-{%- endif %}
-    s = socket.socket(socket.AF_UNIX)
-    s.bind("\0$${slapos-standalone-config:abstract-socket-path}")
-    s.listen(5)
-    print("Standalone SlapOS ready")
-    while True:
-      s.accept()[0].close()
-  finally:
-    print("Stopping standalone subsystem")
-    standalone.stop()
-    print("Exiting")
+embedded-request-exitcode-file = $${directory:etc}/embedded-request-exitcode
+shared-part-list =
+  {{ """${buildout:shared-part-list}""" | indent(2) }}
+context =
+  raw python_for_standalone ${python-for-standalone:executable}
+  raw request_script_path $${directory:project}/request-embedded-instance.sh
+  raw parameters_file_path $${directory:project}/embedded-instance-parameters.json
+  key request_script_template request-script-example:inline
+  key shared_part_list :shared-part-list
+  key embedded_request_exitcode_file :embedded-request-exitcode-file
+  key embedded_instance_config embedded-instance-config:output
+  key home_path directory:home
+  section slap_connection slap-connection
+  section slapos_standalone_config slapos-standalone-config
+url = ${slapos-standalone-script:output}
 
 [slapos-standalone]
 recipe = slapos.recipe.template
@@ -725,18 +670,48 @@ recipe = slapos.cookbook:symbolic.link
 target-directory = $${directory:project}
 link-binary = $${directory:runner}
 
-[request-script-template]
-recipe = slapos.recipe.template
+[request-script-example]
+recipe = slapos.recipe.template:jinja2
 output = $${directory:project}/$${:_buildout_section_name_}.sh
+software_url = ~/srv/project/slapos/software/html5as-base/software.cfg
+request_options = html5as-1 $${:software_url}
+header_text =
+  # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+  # This template is generated automatically by buildout. #
+  # Any changes to this file may be overwritten.          #
+  # Copy and adapt it to create your own request script.  #
+  # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+context =
+  key software_url :software_url
+  key request_options :request_options
+  key header_text :header_text
 inline =
-  #!/bin/sh
-  # This template is generated automatically, copy it in order to supply and request.
-  # Any manual change to this file may be lost.
-  software_name=html5as-base #replace the software name writen in ~/srv/project/slapos/software/
-  software_release_uri=~/srv/project/slapos/software/$software_name/software.cfg
-  # slapos supply is used to add the software to the software list to be supplied to a node.
-  slapos supply $software_release_uri slaprunner
-  # slapos request the allocation of an instance to the master.
-  # slapos request also gets status and parameters of the instance if it has any
-  # (slapos request is meant to be run multiple time until you get the status).
-  slapos request $software_name'_1' $software_release_uri
+{%- raw %}
+  #!/bin/sh -e
+
+  {{ header_text }}
+
+  # slapos supply <url> <node> registers a software for installation on a node.
+  #
+  # A software is uniquely identified by its URL (the URL may be a local path).
+  # You may choose from softwares available in ~/srv/project/slapos/software.
+  #
+  # The one and only SlapOS Node embedded inside Theia is called 'slaprunner'.
+  #
+  # For more information, run:
+  #     slapos help supply
+  #
+  slapos supply {{ software_url }} slaprunner
+
+  # slapos request <name> <url> registers an instance for allocation on a node.
+  #
+  # An instance is uniquely identified by its name (and its requester).
+  #
+  # It will be allocated on a node where the software URL is already supplied.
+  # Inside Theia that node can only be 'slaprunner'.
+  #
+  # For more information, run:
+  #     slapos help request
+  #
+  slapos request {{ request_options }}
+{%  endraw %}

software/theia/instance.cfg.in

@@ -31,6 +31,7 @@ pull-backup = template-pull-backup:output
 recipe = slapos.recipe.template:jinja2
 url = ${instance-theia:output}
 output = $${buildout:directory}/instance-theia.cfg
+extensions = jinja2.ext.do
 context =
   jsonkey default_parameter_dict :default-parameters
   key parameter_dict slap-configuration:configuration
@@ -38,14 +39,12 @@ context =
   key partition_root_path buildout:directory
   key ipv6_random slap-configuration:ipv6-random
   key ipv4_random slap-configuration:ipv4-random
-  import os_module os
-  import hashlib_module hashlib
+  import os os
+  import json json
 default-parameters =
   {
     "autorun": "running",
-    "embedded-sr": null,
-    "embedded-sr-type": null,
-    "embedded-instance-parameters": null,
+    "initial-embedded-instance": null,
     "frontend-name": "Theia Frontend",
     "frontend-sr": "$${:frontend-sr}",
     "frontend-sr-type": "RootSoftwareInstance",

software/theia/slapos_standalone_script.py.jinja

+#!{{ python_for_standalone }}
+import contextlib
+import glob
+import json
+import logging
+import os
+import re
+import signal
+import socket
+import subprocess
+import sys
+import time
+
+import slapos.slap.standalone
+
+from slapos.util import unicode2str
+
+
+logging.basicConfig(format="[%(asctime)s] %(message)s", level=logging.DEBUG)
+
+
+REQUEST_SCRIPT_RAW_TEMPLATE = """{{ request_script_template }}"""
+
+{%  raw -%}
+REQUEST_SCRIPT_TEMPLATE = re.sub(
+  r"{{\s*(\w+)\s*}}",
+  r"{\1}",
+  REQUEST_SCRIPT_RAW_TEMPLATE,
+)
+{%- endraw %}
+
+REQUEST_SCRIPT_HEADER_TEXT = """
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# This script is generated once by your Theia's SlapOS Standalone service.  #
+#                                                                           #
+# It was used to pre-request an instance inside Theia's embedded SlapOS     #
+# according to the preconfiguration parameters given to your Theia.         #
+#                                                                           #
+# It will not be overwritten or recreated.                                  #
+# It will not be launched automatically.                                    #
+#                                                                           #
+# You can edit it and run it to modify your embedded instance.              #
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+""".strip()
+
+
+def signal_handler(signum, frame):
+  logging.info("Signal %d received", signum)
+  sys.exit()
+
+
+@contextlib.contextmanager
+def setupStandalone():
+  partition_forward_configuration = (
+    slapos.slap.standalone.PartitionForwardAsPartitionConfiguration(
+      master_url={{ repr(slap_connection['server-url']) }},
+      computer={{ repr(slap_connection['computer-id']) }},
+      partition={{ repr(slap_connection['partition-id']) }},
+      cert={{ repr(slap_connection['cert-file']) }},
+      key={{ repr(slap_connection['key-file']) }},
+      software_release_list=(
+        'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg',
+      ),
+    ),
+  )
+  shared_parts = {{ repr(shared_part_list) }}
+  shared_part_list = [x.strip() for x in shared_parts.splitlines() if x.strip()]
+  standalone = slapos.slap.standalone.StandaloneSlapOS(
+    {{ repr(slapos_standalone_config['base-directory']) }},
+    {{ repr(slapos_standalone_config['ipv4']) }},
+    {{ slapos_standalone_config['port'] }},
+    computer_id={{ repr(slapos_standalone_config['computer-id']) }},
+    shared_part_list=shared_part_list,
+    software_root={{ repr(slapos_standalone_config['software-root']) }},
+    instance_root={{ repr(slapos_standalone_config['instance-root']) }},
+    partition_forward_configuration=partition_forward_configuration,
+    slapos_bin={{ repr(slapos_standalone_config['slapos-bin']) }},
+    local_software_release_root={{ repr(slapos_standalone_config['local-software-release-root']) }},
+  )
+  standalone.start()
+  try:
+    partition_count = 20
+    logging.info("Standalone SlapOS: Formatting %d partitions", partition_count)
+    standalone.format(partition_count, {{ repr(slapos_standalone_config['ipv4']) }}, {{ repr(slapos_standalone_config['ipv6']) }})
+    logging.info("Standalone SlapOS for computer `%s` started", {{ repr(slapos_standalone_config['computer-id']) }})
+    # Run instance at least once, to start the supervisor managing instances.
+    try:
+      standalone.waitForInstance(max_retry=0)
+    except slapos.slap.standalone.SlapOSNodeCommandError as e:
+      logging.info("Error instanciating: {}".format(e))
+
+    yield standalone
+
+  finally:
+    logging.info("Stopping standalone subsystem")
+    standalone.stop()
+    logging.info("Exiting")
+
+
+def parseEmbeddedInstanceConfig(config_json_file):
+  with open(config_json_file) as f:
+    try:
+      config = json.load(f)
+    except json.JSONDecodeError:
+      logging.error("%s is not valid JSON", config_json_file)
+      raise
+  assert(isinstance(config, dict))
+  if config:
+    software_url = unicode2str(config['software-url'])
+    software_type = config.get('software-type')
+    instance_parameters = config.get('instance-parameters')
+    assert(isinstance(software_url, str))
+    if software_type:
+      software_type = unicode2str(software_type)
+      assert(isinstance(software_type, str))
+    if instance_parameters:
+      assert(isinstance(instance_parameters, dict))
+    return software_url, software_type, instance_parameters
+
+
+def createRequestScript(software_url, software_type, instance_parameters):
+  # Generate request script
+  request_script_path = {{ repr(request_script_path) }}
+  parameters_file_path = {{ repr(parameters_file_path) }}
+
+  request_options = "embedded_instance " + software_url
+  if software_type:
+    request_options += ' --type ' + software_type
+  if instance_parameters:
+    with open(parameters_file_path, 'w') as f:
+      json.dump(instance_parameters, f)
+    request_options += ' --parameters-file ' + parameters_file_path
+
+  with open(request_script_path, 'w') as f:
+    f.write(REQUEST_SCRIPT_TEMPLATE.format(
+      header_text = REQUEST_SCRIPT_HEADER_TEXT,
+      software_url = software_url,
+      request_options = request_options,
+    ))
+    f.write("\n")
+    os.fchmod(f.fileno(), 0o755)
+
+  return request_script_path
+
+
+def main():
+  signal.signal(signal.SIGTERM, signal_handler)
+
+  with setupStandalone() as standalone:
+
+    config_json_file = {{ repr(embedded_instance_config) }}
+    done_file = config_json_file + '.done'
+
+    if not os.path.exists(done_file):
+      with open(done_file, 'w'): pass
+      try:
+        config = parseEmbeddedInstanceConfig(config_json_file)
+      except Exception:
+        logging.info("Failed to parse embedded instance config", exc_info=True)
+        config = None
+      if config:
+        try:
+          request_script_path = createRequestScript(*config)
+          logging.info("Calling %s", request_script_path)
+          exitcode = subprocess.call((request_script_path,), env={
+            'HOME': {{ repr(home_path) }},
+            'PATH': os.path.dirname(standalone._slapos_bin),
+            'SLAPOS_CONFIGURATION': standalone._slapos_config,
+            'SLAPOS_CLIENT_CONFIGURATION': standalone._slapos_config,
+          })
+          with open({{ repr(embedded_request_exitcode_file) }}, 'w') as f:
+            f.write(str(exitcode))
+        except Exception:
+          logging.info("Failed to request embedded instance", exc_info=True)
+
+    s = socket.socket(socket.AF_UNIX)
+
+    s.bind({{ repr('\0' + slapos_standalone_config['abstract-socket-path']) }})
+    s.listen(5)
+    logging.info("Standalone SlapOS ready")
+
+    while True:
+      s.accept()[0].close()
+
+
+main()

software/theia/software.cfg

@@ -19,17 +19,9 @@ extends =
   ./buildout.hash.cfg
 
 parts =
-  theia-wrapper
   slapos-cookbook
-  python-for-resiliency
-  python-for-standalone
-  instance-theia
   instance
-  instance-import
-  instance-export
-  instance-resilient
-  theia-common
-  theia-export
+
 
 # default for slapos-standalone
 shared-part-list =
@@ -95,6 +87,10 @@ output = ${buildout:directory}/instance-export.cfg.jinja
 [instance-resilient]
 <= download-base
 
+[slapos-standalone-script]
+<= download-base
+destination = ${buildout:directory}/slapos_standalone_script.py.jinja
+
 [theia-common]
 <= download-base
 destination = ${buildout:directory}/theia_common.py
@@ -102,10 +98,12 @@ destination = ${buildout:directory}/theia_common.py
 [theia-export]
 <= download-base
 destination = ${buildout:directory}/theia_export.py
+depends = ${theia-common:recipe}
 
 [theia-import]
 <= download-base
 destination = ${buildout:directory}/theia_import.py
+depends = ${theia-common:recipe}
 
 
 # Utilities

software/theia/test/project_tests.py

@@ -62,10 +62,9 @@ class ERP5Mixin(object):
   _test_software_url = erp5_software_release_url
   _connexion_parameters_regex = re.compile(r"{.*}", re.DOTALL)
 
-  def _getERP5ConnexionParameters(self, software_type='export'):
-    slapos = self._getSlapos(software_type)
-    out = subprocess.check_output(
-      (slapos, 'request', 'test_instance', self._test_software_url),
+  def _getERP5ConnexionParameters(self, instance_type='export'):
+    out = self.captureSlapos(
+      'request', 'test_instance', self._test_software_url,
       stderr=subprocess.STDOUT,
     )
     print(out)
@@ -110,10 +109,10 @@ class ERP5Mixin(object):
       raise Exception("Found several partitions for ERP5 %s" % servicename)
     return found.pop()
 
-  def _getERP5PartitionPath(self, software_type, servicename, *paths):
+  def _getERP5PartitionPath(self, instance_type, servicename, *paths):
     partition = self._getERP5Partition(servicename)
-    return self._getPartitionPath(
-      software_type, 'srv', 'runner', 'instance', partition, *paths)
+    return self.getPartitionPath(
+      instance_type, 'srv', 'runner', 'instance', partition, *paths)
 
 
 class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
@@ -161,16 +160,15 @@ class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
 
     # Update ERP5 parameters
     print('Requesting ERP5 with parameters %s' % params)
-    slapos = self._getSlapos()
-    subprocess.check_call((slapos, 'request', 'test_instance', self._test_software_url, '--parameters', params))
+    self.checkSlapos('request', 'test_instance', self._test_software_url, '--parameters', params)
 
     # Process twice to propagate parameter changes
     for _ in range(2):
-      subprocess.check_call((slapos, 'node', 'instance'))
+      self.checkSlapos('node', 'instance')
 
     # Restart cron (actually all) services to let them take the new date into account
     # XXX this should not be required, updating ERP5 parameters should be enough
-    subprocess.call((slapos, 'node', 'restart', 'all'))
+    self.callSlapos('node', 'restart', 'all')
 
     # Wait until after the programmed backup date, and a bit more
     t = (soon - datetime.now()).total_seconds()
@@ -213,9 +211,8 @@ class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
     self.assertNotIn(self._erp5_new_title, out)
 
     # Stop all services
-    slapos = self._getSlapos()
     print("Stop all services")
-    subprocess.call((slapos, 'node', 'stop', 'all'))
+    self.callSlapos('node', 'stop', 'all')
 
     # Manually restore mariadb from backup
     mariadb_restore_script = os.path.join(mariadb_partition, 'bin', 'restore-from-backup')

software/theia/test/test.py

@@ -52,12 +52,54 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(theia_soft
 class TheiaTestCase(SlapOSInstanceTestCase):
   __partition_reference__ = 'T' # for supervisord sockets in included slapos
 
+  @classmethod
+  def getPath(cls, *components):
+    return os.path.join(cls.computer_partition_root_path, *components)
+
   @classmethod
   def _getSlapos(cls):
-    partition_root = cls.computer_partition_root_path
-    slapos = os.path.join(partition_root, 'srv', 'runner', 'bin', 'slapos')
+    try:
+      return cls._theia_slapos
+    except AttributeError:
+      cls._theia_slapos = slapos = cls.getPath('srv', 'runner', 'bin', 'slapos')
       return slapos
 
+  @classmethod
+  def callSlapos(cls, *command, **kwargs):
+    return subprocess.call((cls._getSlapos(),) + command, **kwargs)
+
+  @classmethod
+  def checkSlapos(cls, *command, **kwargs):
+    return subprocess.check_call((cls._getSlapos(),) + command, **kwargs)
+
+  @classmethod
+  def captureSlapos(cls, *command, **kwargs):
+    kwargs.setdefault('universal_newlines', kwargs.pop('text', None))
+    return subprocess.check_output((cls._getSlapos(),) + command, **kwargs)
+
+  @classmethod
+  def requestInstance(cls, parameter_dict=None, state='started'):
+    cls.slap.request(
+      software_release=cls.getSoftwareURL(),
+      software_type=cls.getInstanceSoftwareType(),
+      partition_reference=cls.default_partition_reference,
+      partition_parameter_kw=parameter_dict,
+      state=state
+    )
+
+  @classmethod
+  def restartService(cls, service):
+    with cls.slap.instance_supervisor_rpc as supervisor:
+      for process_info in supervisor.getAllProcessInfo():
+        service_name = process_info['name']
+        if service in service_name:
+          service_id = '%s:%s' % (process_info['group'], service_name)
+          supervisor.stopProcess(service_id)
+          supervisor.startProcess(service_id)
+          break
+      else:
+        raise Exception("Service %s not found" % service)
+
 
 class TestTheia(TheiaTestCase):
   def setUp(self):
@@ -104,7 +146,7 @@ class TestTheia(TheiaTestCase):
 
     # there's a public folder to serve file
     with open('{}/srv/frontend-static/public/test_file'.format(
-        self.computer_partition_root_path), 'w') as f:
+        self.getPath()), 'w') as f:
       f.write("hello")
     resp = self.get(urljoin(authenticated_url, '/public/'))
     self.assertIn('test_file', resp.text)
@@ -128,10 +170,9 @@ class TestTheia(TheiaTestCase):
     self.assertIn('ipv6', self.connection_parameters)
 
   def test_theia_slapos(self):
+    home = self.getPath()
     # Make sure we can use the shell and the integrated slapos command
-    process = pexpect.spawnu(
-        '{}/bin/theia-shell'.format(self.computer_partition_root_path),
-        env={'HOME': self.computer_partition_root_path})
+    process = pexpect.spawnu(home + '/bin/theia-shell', env={'HOME': home})
 
     # use a large enough terminal so that slapos proxy show table fit in the screen
     process.setwinsize(5000, 5000)
@@ -173,10 +214,11 @@ class TestTheia(TheiaTestCase):
     process.wait()
 
   def test_theia_shell_execute_tasks(self):
+    home = self.getPath()
     # shell needs to understand -c "command" arguments for theia tasks feature
-    test_file = '{}/test file'.format(self.computer_partition_root_path)
+    test_file = home + '/test file'
     subprocess.check_call([
-        '{}/bin/theia-shell'.format(self.computer_partition_root_path),
+        home + '/bin/theia-shell',
         '-c',
         'touch "{}"'.format(test_file)
     ])
@@ -184,26 +226,23 @@ class TestTheia(TheiaTestCase):
 
   def test_theia_request_script(self):
     script_path = os.path.join(
-      self.computer_partition_root_path,
+      self.getPath(),
       'srv',
       'project',
-      'request-script-template.sh',
+      'request-script-example.sh',
     )
     self.assertTrue(os.path.exists(script_path))
 
   def test_slapos_cli(self):
-    slapos = self._getSlapos()
-    proxy_show_output = subprocess.check_output((slapos, 'proxy', 'show'))
-    self.assertIn(b'slaprunner', proxy_show_output)
-    computer_list_output = subprocess.check_output((slapos, 'computer', 'list'))
-    self.assertIn(b'slaprunner', computer_list_output)
+    self.assertIn(b'slaprunner', self.captureSlapos('proxy', 'show'))
+    self.assertIn(b'slaprunner', self.captureSlapos('computer', 'list'))
 
 
 class TestTheiaEmbeddedSlapOSShutdown(TheiaTestCase):
   def test_stopping_instance_stops_embedded_slapos(self):
     embedded_slapos_supervisord_socket = _getSupervisordSocketPath(
         os.path.join(
-            self.computer_partition_root_path,
+            self.getPath(),
             'srv',
             'runner',
             'instance',
@@ -231,82 +270,70 @@ class TestTheiaEmbeddedSlapOSShutdown(TheiaTestCase):
     self.assertFalse(embedded_slapos_process.is_running())
 
 
-class ReRequestMixin(object):
-  def rerequest(self, parameter_dict=None, state='started'):
-    software_url = self.getSoftwareURL()
-    software_type = self.getInstanceSoftwareType()
-    name = self.default_partition_reference
-    self.slap.request(
-      software_release=software_url,
-      software_type=software_type,
-      partition_reference=name,
-      partition_parameter_kw=parameter_dict,
-      state=state)
-
-  def reinstantiate(self):
-    # Process at least twice to propagate parameter changes
-    try:
-      self.slap.waitForInstance()
-    except SlapOSNodeCommandError:
-      pass
-    self.slap.waitForInstance(self.instance_max_retry)
-
-
-class TestTheiaWithSR(TheiaTestCase, ReRequestMixin):
-  sr_url = '~/bogus/software.cfg'
-  sr_type = 'bogus_type'
-  instance_parameters = '{\n"bogus_param": "bogus_value",\n"bogus_param2": "bogus_value2"\n}'
-
-  def proxy_show(self, slapos):
-    return subprocess.check_output((slapos, 'proxy', 'show'), universal_newlines=True)
+class TestTheiaWithEmbeddedInstance(TheiaTestCase):
+  sr_url = '~/bogus/sr/url.cfg'
+  sr_type = 'bogus-type'
+  sr_config = {"bogus": "yes"}
+  regexpr = re.compile(r"([\w/\-\.]+)\s+slaprunner\s+available")
 
-  def test(self):
-    slapos = self._getSlapos()
-    home = self.computer_partition_root_path
-
-    # Check that no request script was generated
-    request_script = os.path.join(home, 'srv', 'project', 'request_embedded.sh')
-    self.assertFalse(os.path.exists(request_script))
-
-    # Manually request old-name 'Embedded Instance'
-    old_instance_name = "Embedded Instance"
-    subprocess.check_call((slapos, 'request', old_instance_name, 'bogus_url'))
-    self.assertIn(old_instance_name, self.proxy_show(slapos))
-
-    # Update Theia instance parameters
-    embedded_request_parameters = {
-      'embedded-sr': self.sr_url,
-      'embedded-sr-type': self.sr_type,
-      'embedded-instance-parameters': self.instance_parameters
+  @classmethod
+  def getInstanceParameterDict(cls, sr_url=None, sr_type=None, sr_config=None):
+    return {
+      'initial-embedded-instance': json.dumps({
+        'software-url': sr_url or cls.sr_url,
+        'software-type': sr_type or cls.sr_type,
+        'instance-parameters': sr_config or cls.sr_config,
+      }),
     }
-    self.rerequest(embedded_request_parameters)
-    self.reinstantiate()
 
-    # Check that embedded instance was requested
-    instance_name = "embedded_instance"
-    info = self.proxy_show(slapos)
-    try:
-      self.assertIn(instance_name, info)
-    except AssertionError:
-      for filename in os.listdir(home):
-        if 'standalone' in filename and '.log' in filename:
-          filepath = os.path.join(home, filename)
-          with open(filepath) as f:
-            print("Contents of filepath: " + filepath)
-            print(f.read())
-      raise
+  def expandUrl(self, url):
+    if url.startswith('~/'):
+      url = os.path.join(self.getPath(), url[2:])
+    return url
+
+  def assertSupplied(self, sr_url, info=None):
+    info = info or self.captureSlapos('proxy', 'show', text=True)
+    self.assertIn(sr_url, info)
+    self.assertIn(sr_url, self.regexpr.findall(info))
+
+  def assertNotSupplied(self, sr_url, info=None):
+    info = info or self.captureSlapos('proxy', 'show', text=True)
+    self.assertNotIn(sr_url, info)
+
+  def assertEmbedded(self, sr_url, sr_type, config):
+    proxy_info = self.captureSlapos('proxy', 'show', text=True)
+    self.assertSupplied(sr_url, info=proxy_info)
+    name = 'embedded_instance'
+    self.assertIn(name, self.captureSlapos('service', 'list', text=True))
+    info = self.captureSlapos('service', 'info', name, text=True)
+    self.assertIn(sr_url, info)
+    self.assertIn(sr_type, proxy_info)
+    self.assertIn(repr(config).replace("u'", "'"), info)
+
+  def assertNotEmbedded(self, sr_url, sr_type, config):
+    sr_url = self.expandUrl(sr_url)
+    proxy_info = self.captureSlapos('proxy', 'show', text=True)
+    self.assertNotSupplied(sr_url, info=proxy_info)
+    self.assertNotIn(sr_type, proxy_info)
 
-    # Check that old-name instance was renamed
-    self.assertNotIn(old_instance_name, info)
+  def test(self):
+    # Check that embedded instance is supplied and requested
+    initial_sr_url = self.expandUrl(self.sr_url)
+    self.assertEmbedded(initial_sr_url, self.sr_type, self.sr_config)
 
-    # Check embedded instance parameters
-    bogus_sr = os.path.join(home, self.sr_url[2:])
+    # Change parameters for embedded instance
+    sr_url = '/bogus/sr/url-2.cfg'
+    sr_type = 'bogus-type-2'
+    sr_config = {"bogus-2": "true"}
+    self.requestInstance(
+      self.getInstanceParameterDict(sr_url, sr_type, sr_config))
+    self.waitForInstance()
 
-    self.assertIsNotNone(re.search(r"%s\s+slaprunner\s+available" % (bogus_sr,), info), info)
-    self.assertIsNotNone(re.search(r"%s\s+%s\s+%s" % (bogus_sr, self.sr_type, instance_name), info), info)
+    # Check that parameters have not been taken into account
+    self.assertNotEmbedded(sr_url, sr_type, sr_config)
 
-    service_info = subprocess.check_output((slapos, 'service', 'info', instance_name), universal_newlines=True)
-    self.assertIn("{'bogus_param': 'bogus_value', 'bogus_param2': 'bogus_value2'}", service_info)
+    # Check that previous instance has not been changed
+    self.assertEmbedded(initial_sr_url, self.sr_type, self.sr_config)
 
 
 class TestTheiaFrontend(TheiaTestCase):
@@ -331,7 +358,9 @@ class TestTheiaEnv(TheiaTestCase):
   @classmethod
   def getInstanceParameterDict(cls):
     return {
-      'embedded-sr': cls.dummy_software_path,
+      'initial-embedded-instance': json.dumps({
+        'software-url': cls.dummy_software_path,
+      }),
       'autorun': 'stopped',
     }
 
@@ -339,7 +368,7 @@ class TestTheiaEnv(TheiaTestCase):
     """Make sure environment variables are the same whether we use shell or supervisor services.
     """
     # The path of the env.json file expected to be generated by building the dummy software release
-    env_json_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'software', 'env.json')
+    env_json_path = self.getPath('srv', 'runner', 'software', 'env.json')
 
     # Get the pid of the theia process from the test node's instance-supervisord
     with self.slap.instance_supervisor_rpc as supervisor:
@@ -357,7 +386,7 @@ class TestTheiaEnv(TheiaTestCase):
 
     # Start a theia shell that inherits the environment of the theia process
     # This simulates the environment of a shell launched from the browser application
-    theia_shell_process = pexpect.spawnu('{}/bin/theia-shell'.format(self.computer_partition_root_path), env=theia_env)
+    theia_shell_process = pexpect.spawnu('{}/bin/theia-shell'.format(self.getPath()), env=theia_env)
     try:
       theia_shell_process.expect_exact('Standalone SlapOS for computer `slaprunner` activated')
 
@@ -377,7 +406,7 @@ class TestTheiaEnv(TheiaTestCase):
       # Note that we have two services, slapos-node-software and slapos-node-software-all
       # The later uses --all which is what we want to use here, because the software
       # is already installed and we want to install it again, this time from supervisor
-      embedded_run_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'var', 'run')
+      embedded_run_path = self.getPath('srv', 'runner', 'var', 'run')
       embedded_supervisord_socket_path = _getSupervisordSocketPath(embedded_run_path, self.logger)
       with getSupervisorRPC(embedded_supervisord_socket_path) as embedded_supervisor:
         previous_stop_time = embedded_supervisor.getProcessInfo('slapos-node-software-all')['stop']
@@ -410,6 +439,8 @@ class ResilientTheiaMixin(object):
   @classmethod
   def setUpClass(cls):
     super(ResilientTheiaMixin, cls).setUpClass()
+    # Patch the computer root path to that of the export theia instance
+    cls.computer_partition_root_path = cls.getPartitionPath('export')
     # Add resiliency files to snapshot patterns
     cls._save_instance_file_pattern_list += (
       '*/srv/export-exitcode-file',
@@ -419,43 +450,53 @@ class ResilientTheiaMixin(object):
     )
 
   @classmethod
-  def _getPartition(cls, software_type):
+  def getPartitionId(cls, instance_type):
     software_url = cls.getSoftwareURL()
     for computer_partition in cls.slap.computer.getComputerPartitionList():
       partition_url = computer_partition.getSoftwareRelease()._software_release
       partition_type = computer_partition.getType()
-      if partition_url == software_url and partition_type == software_type:
-        return computer_partition
-    raise Exception("Theia %s partition not found" % software_type)
+      if partition_url == software_url and partition_type == instance_type:
+        return computer_partition.getId()
+    raise Exception("Theia %s partition not found" % instance_type)
+
+  @classmethod
+  def getPartitionPath(cls, instance_type='export', *paths):
+    return os.path.join(cls.slap._instance_root, cls.getPartitionId(instance_type), *paths)
+
+  @classmethod
+  def _getSlapos(cls, instance_type='export'):
+    return cls.getPartitionPath(instance_type, 'srv', 'runner', 'bin', 'slapos')
 
   @classmethod
-  def _getPartitionId(cls, software_type):
-    return cls._getPartition(software_type).getId()
+  def callSlapos(cls, *command, **kwargs):
+    instance_type = kwargs.pop('instance_type', 'export')
+    return subprocess.call((cls._getSlapos(instance_type),) + command, **kwargs)
 
   @classmethod
-  def _getPartitionPath(cls, software_type, *paths):
-    return os.path.join(cls.slap._instance_root, cls._getPartitionId(software_type), *paths)
+  def checkSlapos(cls, *command, **kwargs):
+    instance_type = kwargs.pop('instance_type', 'export')
+    return subprocess.check_call((cls._getSlapos(instance_type),) + command, **kwargs)
 
   @classmethod
-  def _getSlapos(cls, software_type='export'):
-    return cls._getPartitionPath(software_type, 'srv', 'runner', 'bin', 'slapos')
+  def captureSlapos(cls, *command, **kwargs):
+    kwargs.setdefault('universal_newlines', kwargs.pop('text', None))
+    instance_type = kwargs.pop('instance_type', 'export')
+    return subprocess.check_output((cls._getSlapos(instance_type),) + command, **kwargs)
 
   @classmethod
   def getInstanceSoftwareType(cls):
     return 'resilient'
 
+  @classmethod
+  def waitForInstance(cls):
+    # process twice to propagate to all instances
+    for _ in range(2):
+      super(ResilientTheiaMixin, cls).waitForInstance()
+
 
 class TestTheiaResilientInterface(ResilientTheiaMixin, TestTheia):
-  @classmethod
-  def setUpClass(cls):
-    super(TestTheiaResilientInterface, cls).setUpClass()
-    # Patch the computer root path to that of the export theia instance
-    cls.computer_partition_root_path = cls._getPartitionPath('export')
+  pass
 
 
-class TestTheiaResilientWithSR(ResilientTheiaMixin, TestTheiaWithSR):
-  @classmethod
-  def setUpClass(cls):
-    super(TestTheiaResilientWithSR, cls).setUpClass()
-    # Patch the computer root path to that of the export theia instance
-    cls.computer_partition_root_path = cls._getPartitionPath('export')
+class TestTheiaResilientWithEmbeddedInstance(ResilientTheiaMixin, TestTheiaWithEmbeddedInstance):
+  pass

software/theia/test/test_resiliency.py

@@ -63,11 +63,10 @@ def setUpModule():
 
 class ResilientTheiaTestCase(ResilientTheiaMixin, TheiaTestCase):
   @classmethod
-  def _processEmbeddedInstance(cls, retries=0, software_type='export'):
-    slapos = cls._getSlapos(software_type)
+  def _processEmbeddedInstance(cls, retries=0, instance_type='export'):
     for _ in range(retries):
       try:
-        output = subprocess.check_output((slapos, 'node', 'instance'), stderr=subprocess.STDOUT)
+        output = cls.captureSlapos('node', 'instance', instance_type=instance_type, stderr=subprocess.STDOUT)
       except subprocess.CalledProcessError:
         continue
       print(output)
@@ -77,19 +76,18 @@ class ResilientTheiaTestCase(ResilientTheiaMixin, TheiaTestCase):
         # Sleep a bit as an attempt to workaround monitoring boostrap not being ready
         print("Wait before running slapos node instance one last time")
         time.sleep(120)
-      subprocess.check_call((slapos, 'node', 'instance'))
+      cls.callSlapos('node', 'instance', instance_type=instance_type)
 
   @classmethod
-  def _deployEmbeddedSoftware(cls, software_url, instance_name, retries=0, software_type='export'):
-    slapos = cls._getSlapos(software_type)
-    subprocess.check_call((slapos, 'supply', software_url, 'slaprunner'))
+  def _deployEmbeddedSoftware(cls, software_url, instance_name, retries=0, instance_type='export'):
+    cls.callSlapos('supply', software_url, 'slaprunner', instance_type=instance_type)
     try:
-      subprocess.check_output((slapos, 'node', 'software'), stderr=subprocess.STDOUT)
+      cls.captureSlapos('node', 'software', instance_type=instance_type, stderr=subprocess.STDOUT)
     except subprocess.CalledProcessError as e:
       print(e.output)
       raise
-    subprocess.check_call((slapos, 'request', instance_name, software_url))
-    cls._processEmbeddedInstance(retries, software_type)
+    cls.callSlapos('request', instance_name, software_url, instance_type=instance_type)
+    cls._processEmbeddedInstance(retries, instance_type)
 
   @classmethod
   def getInstanceParameterDict(cls):
@@ -136,16 +134,16 @@ class ResilienceMixin(object):
 
 class ExportAndImportMixin(object):
   def getExportExitfile(self):
-    return self._getPartitionPath('export', 'srv', 'export-exitcode-file')
+    return self.getPartitionPath('export', 'srv', 'export-exitcode-file')
 
   def getExportErrorfile(self):
-    return self._getPartitionPath('export', 'srv', 'export-errormessage-file')
+    return self.getPartitionPath('export', 'srv', 'export-errormessage-file')
 
   def getImportExitfile(self):
-    return self._getPartitionPath('import', 'srv', 'import-exitcode-file')
+    return self.getPartitionPath('import', 'srv', 'import-exitcode-file')
 
   def getImportErrorfile(self):
-    return self._getPartitionPath('import', 'srv', 'import-errormessage-file')
+    return self.getPartitionPath('import', 'srv', 'import-errormessage-file')
 
   def makedirs(self, path):
     try:
@@ -185,7 +183,7 @@ class ExportAndImportMixin(object):
     initial_exitdate = os.path.getmtime(exitfile)
 
     # Call export script manually
-    theia_export_script = self._getPartitionPath('export', 'bin', 'theia-export-script')
+    theia_export_script = self.getPartitionPath('export', 'bin', 'theia-export-script')
     subprocess.check_call((theia_export_script,), stderr=subprocess.STDOUT)
 
     # Check that the export exitcode file was modified
@@ -198,8 +196,8 @@ class ExportAndImportMixin(object):
 
   def _doTransfer(self):
     # Copy <export>/srv/backup/theia to <import>/srv/backup/theia manually
-    export_backup_path = self._getPartitionPath('export', 'srv', 'backup', 'theia')
-    import_backup_path = self._getPartitionPath('import', 'srv', 'backup', 'theia')
+    export_backup_path = self.getPartitionPath('export', 'srv', 'backup', 'theia')
+    import_backup_path = self.getPartitionPath('import', 'srv', 'backup', 'theia')
     shutil.rmtree(import_backup_path)
     shutil.copytree(export_backup_path, import_backup_path)
 
@@ -209,7 +207,7 @@ class ExportAndImportMixin(object):
     initial_exitdate = os.path.getmtime(exitfile)
 
     # Call the import script manually
-    theia_import_script = self._getPartitionPath('import', 'bin', 'theia-import-script')
+    theia_import_script = self.getPartitionPath('import', 'bin', 'theia-import-script')
     subprocess.check_call((theia_import_script,), stderr=subprocess.STDOUT)
 
     # Check that the import exitcode file was updated
@@ -277,11 +275,11 @@ class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestC
         os.remove(path)
 
   def customSignatureScript(self, content=None):
-    custom_script = self._getPartitionPath('export', self.script_relpath)
+    custom_script = self.getPartitionPath('export', self.script_relpath)
     self.customScript(custom_script, content)
 
   def customRestoreScript(self, content=None):
-    restore_script = self._getPartitionPath('import', 'srv', 'runner-import-restore')
+    restore_script = self.getPartitionPath('import', 'srv', 'runner-import-restore')
     self.customScript(restore_script, content)
     return restore_script
 
@@ -294,7 +292,7 @@ class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestC
     self.customRestoreScript(content=None)
     self.cleanupExitfiles()
     try:
-      os.remove(self._getPartitionPath('import', self.signature_relpath))
+      os.remove(self.getPartitionPath('import', self.signature_relpath))
     except OSError:
       pass
 
@@ -309,13 +307,13 @@ class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestC
   def test_custom_hash_script(self):
     errmsg = 'Bye bye'
     self.customSignatureScript(content='>&2 echo "%s"\nexit 1' % errmsg)
-    custom_script = self._getPartitionPath('export', self.script_relpath)
+    custom_script = self.getPartitionPath('export', self.script_relpath)
     self.assertExportFailure('Compute partitions backup signatures\n ... ERROR !',
       'Custom signature script %s failed' % os.path.abspath(custom_script),
       'and stderr:\n%s' % errmsg)
 
   def test_signature_mismatch(self):
-    signature_file = self._getPartitionPath('import', self.signature_relpath)
+    signature_file = self.getPartitionPath('import', self.signature_relpath)
     self.writeFile(signature_file, 'Bogus Hash\n', mode='a')
     self.assertImportFailure('ERROR the backup signatures do not match')
 
@@ -344,7 +342,7 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
 
   def _prepareExport(self):
     # Copy ./resilience_dummy SR in export theia ~/srv/project/dummy
-    dummy_target_path = self._getPartitionPath('export', 'srv', 'project', 'dummy')
+    dummy_target_path = self.getPartitionPath('export', 'srv', 'project', 'dummy')
     shutil.copytree(os.path.dirname(dummy_software_url), dummy_target_path)
     self._test_software_url = os.path.join(dummy_target_path, 'software.cfg')
 
@@ -352,8 +350,8 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
     self._deployEmbeddedSoftware(self._test_software_url, 'dummy_instance')
 
     relpath_dummy = os.path.join('srv', 'runner', 'instance', 'slappart0')
-    self.export_dummy_root = dummy_root = self._getPartitionPath('export', relpath_dummy)
-    self.import_dummy_root = self._getPartitionPath('import', relpath_dummy)
+    self.export_dummy_root = dummy_root = self.getPartitionPath('export', relpath_dummy)
+    self.import_dummy_root = self.getPartitionPath('import', relpath_dummy)
 
     # Check that dummy instance was properly deployed
     self.initial_log = self.checkLog(os.path.join(dummy_root, 'log.log'))
@@ -373,7 +371,7 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
     self.assertTrue(os.path.exists(os.path.join(dummy_root, 'srv', '.backup_identity_script')))
 
     # Remember content of ~/etc in the import theia
-    self.etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
+    self.etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
 
   def _doSync(self):
     self._doExport()
@@ -384,14 +382,13 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
     dummy_root = self.import_dummy_root
 
     # Check that the software url is correct
-    adapted_test_url = self._getPartitionPath('import', 'srv', 'project', 'dummy', 'software.cfg')
-    proxy_content = subprocess.check_output(
-      (self._getSlapos('import'), 'proxy', 'show'), universal_newlines=True)
+    adapted_test_url = self.getPartitionPath('import', 'srv', 'project', 'dummy', 'software.cfg')
+    proxy_content = self.captureSlapos('proxy', 'show', instance_type='import', text=True)
     self.assertIn(adapted_test_url, proxy_content)
     self.assertNotIn(self._test_software_url, proxy_content)
 
     # Check that ~/etc still contains everything it did before
-    etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
+    etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
     self.assertTrue(set(self.etc_listdir).issubset(etc_listdir))
 
     # Check that ~/srv/project was exported
@@ -401,7 +398,7 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
     self.checkLog(os.path.join(dummy_root, 'log.log'), self.initial_log, newline=None)
 
     # Check that ~/srv/.backup_identity_script was detected and called
-    signature =  self._getPartitionPath(
+    signature =  self.getPartitionPath(
       'import', 'srv', 'backup', 'theia', 'slappart0.backup.signature.custom')
     self.assertTrue(os.path.exists(signature))
     with open(signature) as f:
@@ -418,7 +415,7 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
 
   def _doTakeover(self):
     # Start the dummy instance as a sort of fake takeover
-    subprocess.check_call((self._getSlapos('import'), 'node', 'instance'))
+    self.callSlapos('node', 'instance', instance_type='import')
 
   def _checkTakeover(self):
     # Check that dummy instance was properly re-deployed
@@ -493,7 +490,7 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
     # Run two synchronisations on the same instances
     # to make sure everything still works the second time
     # Check ~/etc in import theia again
-    self.etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
+    self.etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
     self._doSync()
     self._checkSync()
 
@@ -502,18 +499,18 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
     self._deployEmbeddedSoftware(self._test_software_url, 'test_instance', self.test_instance_max_retries)
 
     # Check that there is an export and import instance and get their partition IDs
-    self.export_id = self._getPartitionId('export')
-    self.import_id = self._getPartitionId('import')
+    self.export_id = self.getPartitionId('export')
+    self.import_id = self.getPartitionId('import')
 
     # Remember content of ~/etc in the import theia
-    self.etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
+    self.etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
 
   def _doSync(self):
     start = time.time()
 
     # Call exporter script instead of waiting for cron job
     # XXX Accelerate cron frequency instead ?
-    exporter_script = self._getPartitionPath('export', 'bin', 'exporter')
+    exporter_script = self.getPartitionPath('export', 'bin', 'exporter')
     transaction_id = str(int(time.time()))
     subprocess.check_call((exporter_script, '--transaction-id', transaction_id))
 
@@ -524,7 +521,7 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
 
   def _checkSync(self):
     # Check that ~/etc still contains everything it did before
-    etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
+    etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
     self.assertTrue(set(self.etc_listdir).issubset(etc_listdir))
 
   def _doTakeover(self):
@@ -541,9 +538,9 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
     # Check that there is an export, import and frozen instance and get their new partition IDs
     import_id = self.import_id
     export_id = self.export_id
-    new_export_id = self._getPartitionId('export')
-    new_import_id = self._getPartitionId('import')
-    new_frozen_id = self._getPartitionId('frozen')
+    new_export_id = self.getPartitionId('export')
+    new_import_id = self.getPartitionId('import')
+    new_frozen_id = self.getPartitionId('frozen')
 
     # Check that old export instance is now frozen
     self.assertEqual(export_id, new_frozen_id)