Skip to content

slapos
slapos
Commit dec6333b authored by Xavier Thompson's avatar Xavier Thompson
Browse files
Options

software/theia: Simplify embedded SR options 

Remove existing "embedded instance" options:
- `embedded-sr`
- `embedded-sr-type`
- `embedded-instance-parameters`

Instead introduce `one-time-embedded-instance` option:
- Only taken into account when Theia is instantiated the first time.
- Content should be a JSON object with:
  - 'software-url': string of the software URL
  - 'software-type': (optional) string of the software type
  - 'instance-parameters': (optional) JSON object

If the option is non-empty:
- Theia will attempt once to:
  - Parse the JSON content
  - Create a supply/request script
  - Call the script
- A promise will check whether the script was successfully called.

If the promise fails forever, e.g. because of ill-formed JSON, emptying
the option will remove the promise. Editing the option after the first
time will have no effect, whether the promise succeded or not.
parent 7eb2cc60
Show whitespace changes
Inline Side-by-side
Showing with 316 additions and 164 deletions
software/theia/buildout.hash.cfg
View file @ dec6333b
......@@ -15,11 +15,11 @@
[instance-theia]
_update_hash_filename_ = instance-theia.cfg.jinja.in
md5sum = bb7c2092029713549ac325e2dd585eab
md5sum = 17395a4a388867697fbc85b6b1bb84ab
[instance]
_update_hash_filename_ = instance.cfg.in
md5sum = 313eb380fdd2d0882f329af4a1157259
md5sum = 575c414a2d44d80c6d003472ca44d028
[instance-import]
_update_hash_filename_ = instance-import.cfg.jinja.in
......@@ -33,6 +33,10 @@ md5sum = e2630148998c3cdf6d03b5e884d7f464
_update_hash_filename_ = instance-resilient.cfg.jinja
md5sum = ad9499e7355ded4975ad313442cecb7a
[slapos-standalone-script]
_update_hash_filename_ = slapos_standalone_script.py.jinja
md5sum = ef5b73648513caf46573f3302953790f
[theia-common]
_update_hash_filename_ = theia_common.py
md5sum = 6a25c6a7f1beb27232a3c9acd8a76500
......
software/theia/instance-input-schema.json
View file @ dec6333b
......@@ -15,19 +15,9 @@
],
"default": "running"
},
"embedded-sr": {
"title": "Embedded Software URL",
"description": "Optional URL of a software to be embedded",
"type": "string"
},
"embedded-sr-type": {
"title": "Embedded Software Type",
"description": "Type of the optional embedded software",
"type": "string"
},
"embedded-instance-parameters": {
"title": "Embedded Instance Parameters",
"description": "Parameters for the embedded instance, as a JSON dict",
"one-time-embedded-instance": {
"title": "One-Time-Only Embedded Software Instance Pre-Configuration",
"description": "Optional JSON configuration for an embedded instance.",
"type": "string"
},
"frontend-guid": {
......
software/theia/instance-theia.cfg.jinja.in
View file @ dec6333b
{% set parameter_dict = dict(default_parameter_dict, **parameter_dict) %}
{% set additional_frontend = parameter_dict['additional-frontend-guid'] %}
{%- set parameter_dict = dict(default_parameter_dict, **parameter_dict) %}
{%- set additional_frontend = parameter_dict['additional-frontend-guid'] %}
{%- set embedded_instance_config = parameter_dict['one-time-embedded-instance'] %}
[buildout]
extends =
......@@ -10,7 +11,6 @@ theia-environment-parts =
slapos-repository
runner-link
settings.json
request-script-template
theia-parts =
frontend-reload
......@@ -98,6 +98,9 @@ instance-promises =
$${slapos-standalone-listen-promise:name}
$${slapos-standalone-ready-promise:name}
$${slapos-autorun-promise:name}
{% if embedded_instance_config %}
$${embedded-instance-requested-promise:name}
{% endif %}
[theia-listen-promise]
<= monitor-promise-base
......@@ -161,6 +164,15 @@ config-service = $${slapos-autorun:service-name}
config-expect = $${slapos-autorun:autorun}
config-run-directory = $${directory:runner}/var/run
{% if embedded_instance_config %}
[embedded-instance-requested-promise]
<= monitor-promise-base
promise = check_command_execute
name = embedded-instance-requested-promise.py
config-command = $${embedded-instance-requested-promise-script:output}
{% endif %}
# Remote Caddy Frontend
# ---------------------
......@@ -439,36 +451,40 @@ command =
# Embedded Instance
# -----------------
{%- set embedded_sr = parameter_dict['embedded-sr'] %}
{%- set embedded_sr_type = parameter_dict['embedded-sr-type'] %}
{%- set embedded_instance_parameters = parameter_dict['embedded-instance-parameters'] %}
{%- if embedded_sr %}
{%- if embedded_sr.startswith('~/') %}
{%- set embedded_sr = os_module.path.join(partition_root_path, embedded_sr[2:]) %}
{%- set embedded_sr = os_module.path.normpath(embedded_sr) %}
{%- endif %}
[request-embedded-instance-script]
recipe = slapos.recipe.template
output = $${directory:project}/request_embedded.sh
[embedded-instance-config]
recipe = slapos.recipe.template:jinja2
output = $${directory:etc}/$${:_buildout_section_name_}.json
once = $${:output}
config = {{ dumps(parameter_dict['one-time-embedded-instance']) }}
context =
key config :config
inline =
#!/bin/sh
slapos supply {{ embedded_sr }} slaprunner
slapos request "embedded_instance" {{ embedded_sr }}
{%- if embedded_sr_type %} --type {{ embedded_sr_type }} {%- endif %}
{%- if embedded_instance_parameters %} --parameters-file $${embedded-instance-parameters:output}
{%- raw %}
{{ config or "{}"}}
{%- endraw %}
[embedded-instance-parameters]
recipe = slapos.recipe.template
output = $${directory:project}/$${:_buildout_section_name_}.json
[embedded-instance-requested-promise-script]
recipe = slapos.recipe.template:jinja2
output = $${directory:bin}/$${:_buildout_section_name_}
exitcode-file = $${slapos-standalone-script:embedded-request-exitcode-file}
context =
key exitcodefile :exitcode-file
{%- raw %}
inline =
{{ embedded_instance_parameters | indent(2) }}
{%- endif %}
{%- endif %}
{%- set embedded_digest = str(embedded_sr) + str(embedded_sr_type) + str(embedded_instance_parameters) %}
{%- set embedded_digest_hash = hashlib_module.md5(embedded_digest.encode()).hexdigest() %}
#!/bin/sh
if ! [ -f {{ repr(exitcodefile) }} ]
then
echo "ERROR embedded_instance has not been requested"
exit 1
elif [ "$(cat {{ repr(exitcodefile) }})" = 0 ]
then
echo "OK embedded_instance has been sucessfully requested"
exit 0
else
echo "ERROR request of embedded_instance failed"
exit 1
fi
{%- endraw %}
# SlapOS Standalone
......@@ -484,10 +500,14 @@ ip = {{ ipv4_random }}
ipv4 = {{ ipv4_random }}
ipv6 = {{ ipv6_random }}
port = $${slapos-standalone-port:port}
base-directory = $${directory:runner}
software-root = $${directory:runner}/software
instance-root = $${directory:runner}/instance
local-software-release-root = $${directory:home}
slapos-bin = ${buildout:bin-directory}/slapos
slapos-configuration = $${directory:runner}/etc/slapos.cfg
computer-id = slaprunner
abstract-socket-path = $${directory:home}/standalone-{{ embedded_digest_hash[:16] }}
abstract-socket-path = $${directory:home}/standalone-ready
[slapos-standalone-activate]
recipe = slapos.recipe.template
......@@ -499,98 +519,23 @@ inline =
echo 'Standalone SlapOS for computer `$${slapos-standalone-config:computer-id}` activated'
[slapos-standalone-script]
recipe = slapos.recipe.template
recipe = slapos.recipe.template:jinja2
output = $${directory:bin}/$${:_buildout_section_name_}
inline =
#!${python-for-standalone:executable}
import glob
import json
import os
import signal
import socket
import subprocess
import sys
import time
import traceback
import slapos.slap.standalone
# Include this hash, so that if it changes the standalone service will be restarted
# {{ embedded_digest_hash }}
shared_parts = """{{ '''${buildout:shared-part-list}''' | indent(2) }}"""
shared_part_list = [x.strip() for x in shared_parts.splitlines() if x.strip()]
partition_forward_configuration = (
slapos.slap.standalone.PartitionForwardAsPartitionConfiguration(
master_url="$${slap-connection:server-url}",
computer="$${slap-connection:computer-id}",
partition="$${slap-connection:partition-id}",
cert="$${slap-connection:cert-file}",
key="$${slap-connection:key-file}",
software_release_list=(
'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg',
),
),
)
standalone = slapos.slap.standalone.StandaloneSlapOS(
"$${directory:runner}",
"$${slapos-standalone-config:ipv4}",
$${slapos-standalone-config:port},
computer_id="$${slapos-standalone-config:computer-id}",
shared_part_list=shared_part_list,
software_root="$${directory:runner}/software",
instance_root="$${directory:runner}/instance",
partition_forward_configuration=partition_forward_configuration,
slapos_bin="${buildout:bin-directory}/slapos",
local_software_release_root="$${slapos-standalone-config:local-software-release-root}",
)
def signal_handler(signum, frame):
print("Signal {signum} received".format(signum=signum))
sys.exit()
signal.signal(signal.SIGTERM, signal_handler)
standalone.start()
try:
partition_count = 20
print("Standalone SlapOS: Formatting %d partitions" % partition_count)
standalone.format(partition_count, '$${slapos-standalone-config:ipv4}', '$${slapos-standalone-config:ipv6}')
print("Standalone SlapOS for computer `$${slapos-standalone-config:computer-id}` started")
# Run instance at least once, to start the supervisor managing instances.
try:
standalone.waitForInstance(max_retry=0)
except slapos.slap.standalone.SlapOSNodeCommandError as e:
print("Error instanciating: {}".format(e))
{%- if embedded_sr %}
# Compatibility layer
try:
for cp in standalone.computer.getComputerPartitionList():
if cp.getInstanceParameterDict().get("instance_title") == "Embedded Instance":
print("Renaming 'Embedded Instance' into 'embedded_instance'")
cp.rename(new_name="embedded_instance")
break
except Exception:
print("Exception in compatibility layer, printing and moving on")
traceback.print_exc()
# Run request script
print("Running SlapOS script $${request-embedded-instance-script:output}")
slapos_env = {
'PATH': os.path.dirname(standalone._slapos_bin),
'SLAPOS_CONFIGURATION': standalone._slapos_config,
'SLAPOS_CLIENT_CONFIGURATION': standalone._slapos_config
}
subprocess.call(("$${request-embedded-instance-script:output}",), env=slapos_env)
{%- endif %}
s = socket.socket(socket.AF_UNIX)
s.bind("\0$${slapos-standalone-config:abstract-socket-path}")
s.listen(5)
print("Standalone SlapOS ready")
while True:
s.accept()[0].close()
finally:
print("Stopping standalone subsystem")
standalone.stop()
print("Exiting")
embedded-request-exitcode-file = $${directory:etc}/embedded-request-exitcode
shared-part-list =
{{ """${buildout:shared-part-list}""" | indent(2) }}
context =
raw python_for_standalone ${python-for-standalone:executable}
raw request_script_path $${directory:project}/request-embedded-instance.sh
raw parameters_file_path $${directory:project}/embedded-instance-parameters.json
key request_script_template request-script-example:inline
key shared_part_list :shared-part-list
key embedded_request_exitcode_file :embedded-request-exitcode-file
key embedded_instance_config embedded-instance-config:output
key home_path directory:home
section slap_connection slap-connection
section slapos_standalone_config slapos-standalone-config
url = ${slapos-standalone-script:output}
[slapos-standalone]
recipe = slapos.recipe.template
......@@ -723,18 +668,48 @@ recipe = slapos.cookbook:symbolic.link
target-directory = $${directory:project}
link-binary = $${directory:runner}
[request-script-template]
recipe = slapos.recipe.template
[request-script-example]
recipe = slapos.recipe.template:jinja2
output = $${directory:project}/$${:_buildout_section_name_}.sh
software_url = ~/srv/project/slapos/software/html5as-base/software.cfg
request_options = html5as-1 $${:software_url}
header_text =
# # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# This template is generated automatically by buildout. #
# Any changes to this file may be overwritten. #
# Copy and adapt it to create your own request script. #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # #
context =
key software_url :software_url
key request_options :request_options
key header_text :header_text
inline =
#!/bin/sh
# This template is generated automatically, copy it in order to supply and request.
# Any manual change to this file may be lost.
software_name=html5as-base #replace the software name writen in ~/srv/project/slapos/software/
software_release_uri=~/srv/project/slapos/software/$software_name/software.cfg
# slapos supply is used to add the software to the software list to be supplied to a node.
slapos supply $software_release_uri slaprunner
# slapos request the allocation of an instance to the master.
# slapos request also gets status and parameters of the instance if it has any
# (slapos request is meant to be run multiple time until you get the status).
slapos request $software_name'_1' $software_release_uri
{%- raw %}
#!/bin/sh -e
{{ header_text }}
# slapos supply <url> <node> registers a software for installation on a node.
#
# A software is uniquely identified by its URL (the URL may be a local path).
# You may choose from softwares available in ~/srv/project/slapos/software.
#
# The one and only SlapOS Node embedded inside Theia is called 'slaprunner'.
#
# For more information, run:
# slapos help supply
#
slapos supply {{ software_url }} slaprunner
# slapos request <name> <url> registers an instance for allocation on a node.
#
# An instance is uniquely identified by its name (and its requester).
#
# It will be allocated on a node where the software URL is already supplied.
# Inside Theia that node can only be 'slaprunner'.
#
# For more information, run:
# slapos help request
#
slapos request {{ request_options }}
{% endraw %}
software/theia/instance.cfg.in
View file @ dec6333b
......@@ -31,6 +31,7 @@ pull-backup = template-pull-backup:rendered
recipe = slapos.recipe.template:jinja2
url = ${instance-theia:output}
output = $${buildout:directory}/instance-theia.cfg
extensions = jinja2.ext.do
context =
jsonkey default_parameter_dict :default-parameters
key parameter_dict slap-configuration:configuration
......@@ -38,14 +39,12 @@ context =
key partition_root_path buildout:directory
key ipv6_random slap-configuration:ipv6-random
key ipv4_random slap-configuration:ipv4-random
import os_module os
import hashlib_module hashlib
import os os
import json json
default-parameters =
{
"autorun": "running",
"embedded-sr": null,
"embedded-sr-type": null,
"embedded-instance-parameters": null,
"one-time-embedded-instance": null,
"frontend-name": "Theia Frontend",
"frontend-sr": "$${:frontend-sr}",
"frontend-sr-type": "RootSoftwareInstance",
......
software/theia/slapos_standalone_script.py.jinja 0 → 100644
View file @ dec6333b
#!{{ python_for_standalone }}
import contextlib
import glob
import json
import logging
import os
import re
import signal
import socket
import subprocess
import sys
import time
import slapos.slap.standalone
from slapos.util import unicode2str
logging.basicConfig(format="[%(asctime)s] %(message)s", level=logging.DEBUG)
REQUEST_SCRIPT_RAW_TEMPLATE = """{{ request_script_template }}"""
{% raw -%}
REQUEST_SCRIPT_TEMPLATE = re.sub(
r"{{\s*(\w+)\s*}}",
r"{\1}",
REQUEST_SCRIPT_RAW_TEMPLATE,
)
{%- endraw %}
REQUEST_SCRIPT_HEADER_TEXT = """
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# This script is generated once by your Theia's SlapOS Standalone service. #
# #
# It was used to pre-request an instance inside Theia's embedded SlapOS #
# according to the preconfiguration parameters given to your Theia. #
# #
# It will not be overwritten or recreated. #
# It will not be launched automatically. #
# #
# You can edit it and run it to modify your embedded instance. #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
""".strip()
def signal_handler(signum, frame):
logging.info("Signal %d received", signum)
sys.exit()
@contextlib.contextmanager
def setupStandalone():
partition_forward_configuration = (
slapos.slap.standalone.PartitionForwardAsPartitionConfiguration(
master_url={{ repr(slap_connection['server-url']) }},
computer={{ repr(slap_connection['computer-id']) }},
partition={{ repr(slap_connection['partition-id']) }},
cert={{ repr(slap_connection['cert-file']) }},
key={{ repr(slap_connection['key-file']) }},
software_release_list=(
'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg',
),
),
)
shared_parts = {{ repr(shared_part_list) }}
shared_part_list = [x.strip() for x in shared_parts.splitlines() if x.strip()]
standalone = slapos.slap.standalone.StandaloneSlapOS(
{{ repr(slapos_standalone_config['base-directory']) }},
{{ repr(slapos_standalone_config['ipv4']) }},
{{ slapos_standalone_config['port'] }},
computer_id={{ repr(slapos_standalone_config['computer-id']) }},
shared_part_list=shared_part_list,
software_root={{ repr(slapos_standalone_config['software-root']) }},
instance_root={{ repr(slapos_standalone_config['instance-root']) }},
partition_forward_configuration=partition_forward_configuration,
slapos_bin={{ repr(slapos_standalone_config['slapos-bin']) }},
local_software_release_root={{ repr(slapos_standalone_config['local-software-release-root']) }},
)
standalone.start()
try:
partition_count = 20
logging.info("Standalone SlapOS: Formatting %d partitions", partition_count)
standalone.format(partition_count, {{ repr(slapos_standalone_config['ipv4']) }}, {{ repr(slapos_standalone_config['ipv6']) }})
logging.info("Standalone SlapOS for computer `%s` started", {{ repr(slapos_standalone_config['computer-id']) }})
# Run instance at least once, to start the supervisor managing instances.
try:
standalone.waitForInstance(max_retry=0)
except slapos.slap.standalone.SlapOSNodeCommandError as e:
logging.info("Error instanciating: {}".format(e))
yield standalone
finally:
logging.info("Stopping standalone subsystem")
standalone.stop()
logging.info("Exiting")
def parseEmbeddedInstanceConfig(config_json_file):
with open(config_json_file) as f:
try:
config = json.load(f)
except json.JSONDecodeError:
logging.error("%s is not valid JSON", config_json_file)
raise
assert(isinstance(config, dict))
if config:
software_url = unicode2str(config['software-url'])
software_type = config.get('software-type')
instance_parameters = config.get('instance-parameters')
assert(isinstance(software_url, str))
if software_type:
software_type = unicode2str(software_type)
assert(isinstance(software_type, str))
if instance_parameters:
assert(isinstance(instance_parameters, dict))
return software_url, software_type, instance_parameters
def createRequestScript(software_url, software_type, instance_parameters):
# Generate request script
request_script_path = {{ repr(request_script_path) }}
parameters_file_path = {{ repr(parameters_file_path) }}
request_options = "embedded_instance " + software_url
if software_type:
request_options += ' --type ' + software_type
if instance_parameters:
with open(parameters_file_path, 'w') as f:
json.dump(instance_parameters, f)
request_options += ' --parameters-file ' + parameters_file_path
with open(request_script_path, 'w') as f:
f.write(REQUEST_SCRIPT_TEMPLATE.format(
header_text = REQUEST_SCRIPT_HEADER_TEXT,
software_url = software_url,
request_options = request_options,
))
f.write("\n")
os.fchmod(f.fileno(), 0o755)
return request_script_path
def main():
signal.signal(signal.SIGTERM, signal_handler)
with setupStandalone() as standalone:
config_json_file = {{ repr(embedded_instance_config) }}
done_file = config_json_file + '.done'
if not os.path.exists(done_file):
with open(done_file, 'w'): pass
try:
config = parseEmbeddedInstanceConfig(config_json_file)
except Exception:
logging.info("Failed to parse embedded instance config", exc_info=True)
config = None
if config:
try:
request_script_path = createRequestScript(*config)
logging.info("Calling %s", request_script_path)
exitcode = subprocess.call((request_script_path,), env={
'HOME': {{ repr(home_path) }},
'PATH': os.path.dirname(standalone._slapos_bin),
'SLAPOS_CONFIGURATION': standalone._slapos_config,
'SLAPOS_CLIENT_CONFIGURATION': standalone._slapos_config,
})
with open({{ repr(embedded_request_exitcode_file) }}, 'w') as f:
f.write(str(exitcode))
except Exception:
logging.info("Failed to request embedded instance", exc_info=True)
s = socket.socket(socket.AF_UNIX)
s.bind({{ repr('\0' + slapos_standalone_config['abstract-socket-path']) }})
s.listen(5)
logging.info("Standalone SlapOS ready")
while True:
s.accept()[0].close()
main()
software/theia/software.cfg
View file @ dec6333b
......@@ -19,17 +19,9 @@ extends =
./buildout.hash.cfg
parts =
theia-wrapper
slapos-cookbook
python-for-resiliency
python-for-standalone
instance-theia
instance
instance-import
instance-export
instance-resilient
theia-common
theia-export
# default for slapos-standalone
shared-part-list =
......@@ -96,6 +88,10 @@ output = ${buildout:directory}/instance-export.cfg.jinja
[instance-resilient]
<= download-base
[slapos-standalone-script]
<= download-base
destination = ${buildout:directory}/slapos_standalone_script.py.jinja
[theia-common]
<= download-base
destination = ${buildout:directory}/theia_common.py
......@@ -103,10 +99,12 @@ destination = ${buildout:directory}/theia_common.py
[theia-export]
<= download-base
destination = ${buildout:directory}/theia_export.py
depends = ${theia-common:recipe}
[theia-import]
<= download-base
destination = ${buildout:directory}/theia_import.py
depends = ${theia-common:recipe}
# Utilities
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7