software/theia: Fix embedded instance request

Avoid restarting standalone when embedded instance parameters change.

software/theia/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance-theia]
 _update_hash_filename_ = instance-theia.cfg.jinja.in
-md5sum = 4df9f0d76a134a8abec9060a0c1be50b
+md5sum = 9d41697aef6c92ff1dcf633c675e6202
 
 [instance]
 _update_hash_filename_ = instance.cfg.in

software/theia/instance-theia.cfg.jinja.in

-{% set parameter_dict = dict(default_parameter_dict, **parameter_dict) %}
-{% set additional_frontend = parameter_dict['additional-frontend-guid'] %}
+{%- set parameter_dict = dict(default_parameter_dict, **parameter_dict) %}
+{%- set additional_frontend = parameter_dict['additional-frontend-guid'] %}
+{%- set embedded_computer_id = 'slaprunner' %}
+{%- set embedded_sr = parameter_dict['embedded-sr'] %}
+{%- set embedded_sr_type = parameter_dict['embedded-sr-type'] %}
+{%- set embedded_instance_parameters = parameter_dict['embedded-instance-parameters'] %}
+{%- if embedded_sr %}
+{%-   if embedded_sr.startswith('~/') %}
+{%-     set embedded_sr = os_module.path.join(partition_root_path, embedded_sr[2:]) %}
+{%-     set embedded_sr = os_module.path.normpath(embedded_sr) %}
+{%-   endif %}
+{%- endif -%}
 
 [buildout]
 extends =
@@ -51,6 +61,7 @@ dot-theia = $${:home}/.theia/
 pidfiles = $${:var}/run
 
 services = $${:etc}/service
+scripts = $${:etc}/run
 runner = $${:srv}/runner
 backup = $${:srv}/backup/theia
 project = $${:srv}/project
@@ -92,12 +103,15 @@ instance-promises =
   $${frontend-listen-promise:name}
   $${frontend-authentication-promise:name}
   $${remote-frontend-url-available-promise:name}
-  {% if additional_frontend %}
+  {%- if additional_frontend %}
   $${remote-additional-frontend-url-available-promise:name}
-  {% endif %}
+  {%- endif %}
   $${slapos-standalone-listen-promise:name}
   $${slapos-standalone-ready-promise:name}
   $${slapos-autorun-promise:name}
+  {%- if embedded_sr -%}
+  $${embedded-instance-promise:name}
+  {%- endif %}
 
 [theia-listen-promise]
 <= monitor-promise-base
@@ -161,6 +175,15 @@ config-service = $${slapos-autorun:service-name}
 config-expect = $${slapos-autorun:autorun}
 config-run-directory = $${directory:runner}/var/run
 
+{% if embedded_sr -%}
+[embedded-instance-promise]
+<= monitor-promise-base
+depends = $${request-embedded-instance:recipe}
+promise = check_command_execute
+name = embedded-instance-promise.py
+config-command = $${directory:runner}/bin/slapos service info embedded_instance
+{%- endif %}
+
 
 # Remote Caddy Frontend
 # ---------------------
@@ -447,38 +470,45 @@ command =
 # Embedded Instance
 # -----------------
 
-{%- set embedded_sr = parameter_dict['embedded-sr'] %}
-{%- set embedded_sr_type = parameter_dict['embedded-sr-type'] %}
-{%- set embedded_instance_parameters = parameter_dict['embedded-instance-parameters'] %}
-{%- if embedded_sr %}
-{%-   if embedded_sr.startswith('~/') %}
-{%-     set embedded_sr = os_module.path.join(partition_root_path, embedded_sr[2:]) %}
-{%-     set embedded_sr = os_module.path.normpath(embedded_sr) %}
-{%-   endif %}
-[request-embedded-instance-script]
-recipe = slapos.recipe.template:jinja2
-rendered = $${directory:project}/request_embedded.sh
-mode = 0700
-template =
-  inline:#!/bin/sh
+{% if embedded_sr -%}
+[request-embedded-instance]
+recipe = slapos.recipe.build
+location = $${directory:project}/request_embedded.sh
+install =
+  import os
+  import socket
+  with open(options['location'], 'w') as f:
+    f.write(options['content'])
+    os.fchmod(f.fileno(), 0o700)
+  s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+  try:
+    s.connect("\0$${slapos-standalone-config:abstract-socket-path}")
+    s.sendall(b"r")
+  except socket.error:
+    pass
+  finally:
+    s.close()
+content =
+  #!/bin/sh
+  set -e
 
   slapos supply {{ embedded_sr }} slaprunner
 
   slapos request "embedded_instance" {{ embedded_sr }}
 {%-   if embedded_sr_type %} --type {{ embedded_sr_type }} {%- endif %}
-{%-   if embedded_instance_parameters %} --parameters-file $${embedded-instance-parameters:rendered}
+{%-   if embedded_instance_parameters %} --parameters-file $${embedded-instance-parameters:location}
 
 [embedded-instance-parameters]
-recipe = slapos.recipe.template:jinja2
-rendered = $${directory:project}/$${:_buildout_section_name_}.json
-template =
-  inline:{{ embedded_instance_parameters | indent(2) }}
+recipe = slapos.recipe.build
+location = $${directory:project}/$${:_buildout_section_name_}.json
+install =
+  with open(options['location'], 'w') as f:
+    f.write(options['content'])
+content =
+  {{ embedded_instance_parameters | indent(2) }}
 {%-   endif %}
 {%- endif %}
 
-{%- set embedded_digest = str(embedded_sr) + str(embedded_sr_type) + str(embedded_instance_parameters) %}
-{%- set embedded_digest_hash = hashlib_module.md5(embedded_digest.encode()).hexdigest() %}
-
 
 # SlapOS Standalone
 # -----------------
@@ -495,8 +525,8 @@ ipv6 = {{ ipv6_random }}
 port = $${slapos-standalone-port:port}
 local-software-release-root = $${directory:home}
 slapos-configuration = $${directory:runner}/etc/slapos.cfg
-computer-id = slaprunner
-abstract-socket-path = $${directory:home}/standalone-{{ embedded_digest_hash[:16] }}
+computer-id = {{ embedded_computer_id }}
+abstract-socket-path = $${directory:home}/standalone-ready
 
 [slapos-standalone-activate]
 recipe = slapos.recipe.template:jinja2
@@ -525,9 +555,6 @@ template =
 
   import slapos.slap.standalone
 
-  # Include this hash, so that if it changes the standalone service will be restarted
-  # {{ embedded_digest_hash }}
-
   shared_parts = """{{ '''${buildout:shared-part-list}''' | indent(2) }}"""
   shared_part_list = [x.strip() for x in shared_parts.splitlines() if x.strip()]
   partition_forward_configuration = (
@@ -571,8 +598,7 @@ template =
       standalone.waitForInstance(max_retry=0)
     except slapos.slap.standalone.SlapOSNodeCommandError as e:
       print("Error instanciating: {}".format(e))
-{%- if embedded_sr %}
-    # Compatibility layer
+    # Compatibility
     try:
       for cp in standalone.computer.getComputerPartitionList():
         if cp.getInstanceParameterDict().get("instance_title") == "Embedded Instance":
@@ -580,23 +606,31 @@ template =
           cp.rename(new_name="embedded_instance")
           break
     except Exception:
-      print("Exception in compatibility layer, printing and moving on")
-      traceback.print_exc()
-    # Run request script
-    print("Running SlapOS script $${request-embedded-instance-script:rendered}")
-    slapos_env = {
-      'PATH': os.path.dirname(standalone._slapos_bin),
-      'SLAPOS_CONFIGURATION': standalone._slapos_config,
-      'SLAPOS_CLIENT_CONFIGURATION': standalone._slapos_config
-    }
-    subprocess.call(("$${request-embedded-instance-script:rendered}",), env=slapos_env)
-{%- endif %}
+      pass
+    # Open a socket to signal standalone is ready
     s = socket.socket(socket.AF_UNIX)
     s.bind("\0$${slapos-standalone-config:abstract-socket-path}")
     s.listen(5)
     print("Standalone SlapOS ready")
+    # Wait for connections and call embedded instance script when notified
+    request_script_path = "$${directory:project}/request_embedded.sh"
+    slapos_env = {
+      "PATH": "$${directory:runner}/bin",
+      "SLAPOS_CONFIGURATION": "$${slapos-standalone-config:slapos-configuration}",
+      "SLAPOS_CLIENT_CONFIGURATION": "$SLAPOS_CONFIGURATION",
+    }
+    do_request = True
     while True:
-      s.accept()[0].close()
+      if do_request:
+        try:
+          subprocess.call((request_script_path,), env=slapos_env)
+        except OSError:
+          pass
+      try:
+        conn, _ = s.accept()
+        do_request = conn.recv(1) == b"r"
+      finally:
+        conn.close()
   finally:
     print("Stopping standalone subsystem")
     standalone.stop()

software/theia/test/test.py

@@ -251,60 +251,80 @@ class ReRequestMixin(object):
 
 
 class TestTheiaWithSR(TheiaTestCase, ReRequestMixin):
-  sr_url = '~/bogus/software.cfg'
-  sr_type = 'bogus_type'
-  instance_parameters = '{\n"bogus_param": "bogus_value",\n"bogus_param2": "bogus_value2"\n}'
+  old_instance_name = 'Embedded Instance'
+  instance_name = 'embedded_instance'
 
-  def proxy_show(self, slapos):
-    return subprocess.check_output((slapos, 'proxy', 'show'), universal_newlines=True)
-
-  def test(self):
-    slapos = self._getSlapos()
-    home = self.computer_partition_root_path
-
-    # Check that no request script was generated
-    request_script = os.path.join(home, 'srv', 'project', 'request_embedded.sh')
-    self.assertFalse(os.path.exists(request_script))
-
-    # Manually request old-name 'Embedded Instance'
-    old_instance_name = "Embedded Instance"
-    subprocess.check_call((slapos, 'request', old_instance_name, 'bogus_url'))
-    self.assertIn(old_instance_name, self.proxy_show(slapos))
+  def check_output(self, *commands):
+    return subprocess.check_output(commands, universal_newlines=True)
 
+  def update(self, sr_url, sr_type, sr_parameters):
     # Update Theia instance parameters
     embedded_request_parameters = {
-      'embedded-sr': self.sr_url,
-      'embedded-sr-type': self.sr_type,
-      'embedded-instance-parameters': self.instance_parameters
+      'embedded-sr': sr_url,
+      'embedded-sr-type': sr_type,
+      'embedded-instance-parameters': json.dumps(sr_parameters, indent=1)
     }
     self.rerequest(embedded_request_parameters)
     self.reinstantiate()
 
+    slapos = self._getSlapos()
+    home = self.computer_partition_root_path
+
+    if sr_url.startswith('~/'):
+      sr_url = os.path.join(home, sr_url[2:])
+
+    # Check that request script was generated
+    request_script = os.path.join(home, 'srv', 'project', 'request_embedded.sh')
+    self.assertTrue(os.path.exists(request_script))
+
     # Check that embedded instance was requested
-    instance_name = "embedded_instance"
-    info = self.proxy_show(slapos)
-    try:
-      self.assertIn(instance_name, info)
-    except AssertionError:
-      for filename in os.listdir(home):
-        if 'standalone' in filename and '.log' in filename:
-          filepath = os.path.join(home, filename)
-          with open(filepath) as f:
-            print("Contents of filepath: " + filepath)
-            print(f.read())
-      raise
+    instance_name = self.instance_name
+    info = self.check_output(slapos, 'proxy', 'show')
+    self.assertIn(instance_name, info)
+
+    # Check embedded instance parameters
+    self.assertIsNotNone(re.search(r"%s\s+slaprunner\s+available" % (sr_url,), info), info)
+    self.assertIsNotNone(re.search(r"%s\s+%s\s+%s" % (sr_url, sr_type, instance_name), info), info)
+    service_info = self.check_output(slapos, 'service', 'info', instance_name)
+    for k, v in sr_parameters.items():
+      self.assertIn("'%s': '%s'" % (k, v), service_info)
 
     # Check that old-name instance was renamed
-    self.assertNotIn(old_instance_name, info)
+    self.assertNotIn(self.old_instance_name, self.check_output(slapos, 'proxy', 'show'))
 
-    # Check embedded instance parameters
-    bogus_sr = os.path.join(home, self.sr_url[2:])
+  def test(self):
+    slapos = self._getSlapos()
+    home = self.computer_partition_root_path
 
-    self.assertIsNotNone(re.search(r"%s\s+slaprunner\s+available" % (bogus_sr,), info), info)
-    self.assertIsNotNone(re.search(r"%s\s+%s\s+%s" % (bogus_sr, self.sr_type, instance_name), info), info)
+    # Check that no request script was generated
+    request_script = os.path.join(home, 'srv', 'project', 'request_embedded.sh')
+    self.assertFalse(os.path.exists(request_script))
 
-    service_info = subprocess.check_output((slapos, 'service', 'info', instance_name), universal_newlines=True)
-    self.assertIn("{'bogus_param': 'bogus_value', 'bogus_param2': 'bogus_value2'}", service_info)
+    # Manually request old-name 'Embedded Instance'
+    subprocess.check_call((slapos, 'request', self.old_instance_name, 'bogus_url'))
+    self.assertIn(self.old_instance_name, self.check_output(slapos, 'proxy', 'show'))
+
+    # Stop standalone so that it will be restarted to test compatibility
+    with self.slap.instance_supervisor_rpc as supervisor:
+      for process_info in supervisor.getAllProcessInfo():
+        service_name = process_info['name']
+        if 'standalone' in service_name:
+          supervisor.stopProcess('%s:%s' % (process_info['group'], service_name))
+          break
+
+    # Update Theia instance parameters and verify results
+    self.update(
+      sr_url = '~/bogus/software.cfg',
+      sr_type = 'bogus',
+      sr_parameters = {'key1': 'value1', 'key2': 'value2'}
+    )
+
+    # Update again without restarting stadalone service and verify results
+    self.update(
+      sr_url = '/bogus/software.cfg',
+      sr_type = 'bogus2',
+      sr_parameters = {'key1': 'value1', 'key3': 'value3'}
+    )
 
 
 class TestTheiaFrontend(TheiaTestCase):