Commit 88f0bcd5 authored by Xavier Thompson's avatar Xavier Thompson

software/theia: Simplify and improve embedded SR

See merge request !1139
parents c9f8bad9 1b877749
......@@ -15,24 +15,28 @@
[instance-theia]
_update_hash_filename_ = instance-theia.cfg.jinja.in
md5sum = 7ea72f2300dd2019c2d27549e7128dec
md5sum = 7d146ca31d7e44e909386e21e0a562ec
[instance]
_update_hash_filename_ = instance.cfg.in
md5sum = af4d92a95dc25c2f64ddbd634996e46c
md5sum = e211c439571e2900f9f35482c9638d06
[instance-import]
_update_hash_filename_ = instance-import.cfg.jinja.in
md5sum = 29bc1c5b76d20fd46430dff5c72d192b
md5sum = 23c3df4a889ebfa9f0a94e873e95ad3b
[instance-export]
_update_hash_filename_ = instance-export.cfg.jinja.in
md5sum = e2630148998c3cdf6d03b5e884d7f464
md5sum = 84ceb4c9ee0f07fce8518ef7517ce1d4
[instance-resilient]
_update_hash_filename_ = instance-resilient.cfg.jinja
md5sum = ad9499e7355ded4975ad313442cecb7a
[slapos-standalone-script]
_update_hash_filename_ = slapos_standalone_script.py.jinja
md5sum = ef5b73648513caf46573f3302953790f
[theia-common]
_update_hash_filename_ = theia_common.py
md5sum = 6a25c6a7f1beb27232a3c9acd8a76500
......
......@@ -92,14 +92,14 @@ inline =
then
echo "ERROR export script last ran on " $(date -r {{ repr(exitcodefile) }})
exit 1
elif [ $( cat {{ repr(exitcodefile) }}) -ne 0 ]
elif [ "$(cat {{ repr(exitcodefile) }})" = 0 ]
then
echo "OK export script last ran on " $(date -r {{ repr(exitcodefile) }})
exit 0
else
echo "ERROR export script failed on " $(date -r {{ repr(exitcodefile) }})
cat {{ repr(errorfile) }}
exit 1
else
echo "OK export script last ran on " $(date -r {{ repr(exitcodefile) }})
exit 0
fi
{%- endraw %}
......
......@@ -154,14 +154,14 @@ inline =
then
echo "ERROR import script last ran on " $(date -r {{ repr(exitcodefile) }})
exit 1
elif [ $( cat {{ repr(exitcodefile) }}) -ne 0 ]
elif [ "$(cat {{ repr(exitcodefile) }})" = 0 ]
then
echo "OK import script last ran on " $(date -r {{ repr(exitcodefile) }})
exit 0
else
echo "ERROR import script failed on " $(date -r {{ repr(exitcodefile) }})
cat {{ repr(errorfile) }}
exit 1
else
echo "OK import script last ran on " $(date -r {{ repr(exitcodefile) }})
exit 0
fi
{%- endraw %}
......
......@@ -15,20 +15,14 @@
],
"default": "running"
},
"embedded-sr": {
"title": "Embedded Software URL",
"description": "Optional URL of a software to be embedded",
"type": "string"
},
"embedded-sr-type": {
"title": "Embedded Software Type",
"description": "Type of the optional embedded software",
"type": "string"
},
"embedded-instance-parameters": {
"title": "Embedded Instance Parameters",
"description": "Parameters for the embedded instance, as a JSON dict",
"type": "string"
"initial-embedded-instance": {
"title": "Initial Embedded Instance Configuration",
"description": "One-shot optional JSON preconfiguration for an embedded instance. Only applied once when Theia is instantiated. Changing this option afterward will have no effect.",
"type": "string",
"examples": [
"{\"software-url\": \"~/srv/project/slapos/software/html5as-base/software.cfg\"}",
"{\"software-url\": \"~/srv/project/slapos/software/html5as/software.cfg\", \"software-type\": \"replicate\", \"instance-parameters\": {\"replicate-quantity\": 3}}"
]
},
"frontend-guid": {
"title": "Frontend Instance ID",
......
This diff is collapsed.
......@@ -31,6 +31,7 @@ pull-backup = template-pull-backup:output
recipe = slapos.recipe.template:jinja2
url = ${instance-theia:output}
output = $${buildout:directory}/instance-theia.cfg
extensions = jinja2.ext.do
context =
jsonkey default_parameter_dict :default-parameters
key parameter_dict slap-configuration:configuration
......@@ -38,14 +39,12 @@ context =
key partition_root_path buildout:directory
key ipv6_random slap-configuration:ipv6-random
key ipv4_random slap-configuration:ipv4-random
import os_module os
import hashlib_module hashlib
import os os
import json json
default-parameters =
{
"autorun": "running",
"embedded-sr": null,
"embedded-sr-type": null,
"embedded-instance-parameters": null,
"initial-embedded-instance": null,
"frontend-name": "Theia Frontend",
"frontend-sr": "$${:frontend-sr}",
"frontend-sr-type": "RootSoftwareInstance",
......
#!{{ python_for_standalone }}
import contextlib
import glob
import json
import logging
import os
import re
import signal
import socket
import subprocess
import sys
import time
import slapos.slap.standalone
from slapos.util import unicode2str
logging.basicConfig(format="[%(asctime)s] %(message)s", level=logging.DEBUG)
REQUEST_SCRIPT_RAW_TEMPLATE = """{{ request_script_template }}"""
{% raw -%}
REQUEST_SCRIPT_TEMPLATE = re.sub(
r"{{\s*(\w+)\s*}}",
r"{\1}",
REQUEST_SCRIPT_RAW_TEMPLATE,
)
{%- endraw %}
REQUEST_SCRIPT_HEADER_TEXT = """
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# This script is generated once by your Theia's SlapOS Standalone service. #
# #
# It was used to pre-request an instance inside Theia's embedded SlapOS #
# according to the preconfiguration parameters given to your Theia. #
# #
# It will not be overwritten or recreated. #
# It will not be launched automatically. #
# #
# You can edit it and run it to modify your embedded instance. #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
""".strip()
def signal_handler(signum, frame):
logging.info("Signal %d received", signum)
sys.exit()
@contextlib.contextmanager
def setupStandalone():
partition_forward_configuration = (
slapos.slap.standalone.PartitionForwardAsPartitionConfiguration(
master_url={{ repr(slap_connection['server-url']) }},
computer={{ repr(slap_connection['computer-id']) }},
partition={{ repr(slap_connection['partition-id']) }},
cert={{ repr(slap_connection['cert-file']) }},
key={{ repr(slap_connection['key-file']) }},
software_release_list=(
'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg',
),
),
)
shared_parts = {{ repr(shared_part_list) }}
shared_part_list = [x.strip() for x in shared_parts.splitlines() if x.strip()]
standalone = slapos.slap.standalone.StandaloneSlapOS(
{{ repr(slapos_standalone_config['base-directory']) }},
{{ repr(slapos_standalone_config['ipv4']) }},
{{ slapos_standalone_config['port'] }},
computer_id={{ repr(slapos_standalone_config['computer-id']) }},
shared_part_list=shared_part_list,
software_root={{ repr(slapos_standalone_config['software-root']) }},
instance_root={{ repr(slapos_standalone_config['instance-root']) }},
partition_forward_configuration=partition_forward_configuration,
slapos_bin={{ repr(slapos_standalone_config['slapos-bin']) }},
local_software_release_root={{ repr(slapos_standalone_config['local-software-release-root']) }},
)
standalone.start()
try:
partition_count = 20
logging.info("Standalone SlapOS: Formatting %d partitions", partition_count)
standalone.format(partition_count, {{ repr(slapos_standalone_config['ipv4']) }}, {{ repr(slapos_standalone_config['ipv6']) }})
logging.info("Standalone SlapOS for computer `%s` started", {{ repr(slapos_standalone_config['computer-id']) }})
# Run instance at least once, to start the supervisor managing instances.
try:
standalone.waitForInstance(max_retry=0)
except slapos.slap.standalone.SlapOSNodeCommandError as e:
logging.info("Error instanciating: {}".format(e))
yield standalone
finally:
logging.info("Stopping standalone subsystem")
standalone.stop()
logging.info("Exiting")
def parseEmbeddedInstanceConfig(config_json_file):
with open(config_json_file) as f:
try:
config = json.load(f)
except json.JSONDecodeError:
logging.error("%s is not valid JSON", config_json_file)
raise
assert(isinstance(config, dict))
if config:
software_url = unicode2str(config['software-url'])
software_type = config.get('software-type')
instance_parameters = config.get('instance-parameters')
assert(isinstance(software_url, str))
if software_type:
software_type = unicode2str(software_type)
assert(isinstance(software_type, str))
if instance_parameters:
assert(isinstance(instance_parameters, dict))
return software_url, software_type, instance_parameters
def createRequestScript(software_url, software_type, instance_parameters):
# Generate request script
request_script_path = {{ repr(request_script_path) }}
parameters_file_path = {{ repr(parameters_file_path) }}
request_options = "embedded_instance " + software_url
if software_type:
request_options += ' --type ' + software_type
if instance_parameters:
with open(parameters_file_path, 'w') as f:
json.dump(instance_parameters, f)
request_options += ' --parameters-file ' + parameters_file_path
with open(request_script_path, 'w') as f:
f.write(REQUEST_SCRIPT_TEMPLATE.format(
header_text = REQUEST_SCRIPT_HEADER_TEXT,
software_url = software_url,
request_options = request_options,
))
f.write("\n")
os.fchmod(f.fileno(), 0o755)
return request_script_path
def main():
signal.signal(signal.SIGTERM, signal_handler)
with setupStandalone() as standalone:
config_json_file = {{ repr(embedded_instance_config) }}
done_file = config_json_file + '.done'
if not os.path.exists(done_file):
with open(done_file, 'w'): pass
try:
config = parseEmbeddedInstanceConfig(config_json_file)
except Exception:
logging.info("Failed to parse embedded instance config", exc_info=True)
config = None
if config:
try:
request_script_path = createRequestScript(*config)
logging.info("Calling %s", request_script_path)
exitcode = subprocess.call((request_script_path,), env={
'HOME': {{ repr(home_path) }},
'PATH': os.path.dirname(standalone._slapos_bin),
'SLAPOS_CONFIGURATION': standalone._slapos_config,
'SLAPOS_CLIENT_CONFIGURATION': standalone._slapos_config,
})
with open({{ repr(embedded_request_exitcode_file) }}, 'w') as f:
f.write(str(exitcode))
except Exception:
logging.info("Failed to request embedded instance", exc_info=True)
s = socket.socket(socket.AF_UNIX)
s.bind({{ repr('\0' + slapos_standalone_config['abstract-socket-path']) }})
s.listen(5)
logging.info("Standalone SlapOS ready")
while True:
s.accept()[0].close()
main()
......@@ -19,17 +19,9 @@ extends =
./buildout.hash.cfg
parts =
theia-wrapper
slapos-cookbook
python-for-resiliency
python-for-standalone
instance-theia
instance
instance-import
instance-export
instance-resilient
theia-common
theia-export
# default for slapos-standalone
shared-part-list =
......@@ -95,6 +87,10 @@ output = ${buildout:directory}/instance-export.cfg.jinja
[instance-resilient]
<= download-base
[slapos-standalone-script]
<= download-base
destination = ${buildout:directory}/slapos_standalone_script.py.jinja
[theia-common]
<= download-base
destination = ${buildout:directory}/theia_common.py
......@@ -102,10 +98,12 @@ destination = ${buildout:directory}/theia_common.py
[theia-export]
<= download-base
destination = ${buildout:directory}/theia_export.py
depends = ${theia-common:recipe}
[theia-import]
<= download-base
destination = ${buildout:directory}/theia_import.py
depends = ${theia-common:recipe}
# Utilities
......
......@@ -62,10 +62,9 @@ class ERP5Mixin(object):
_test_software_url = erp5_software_release_url
_connexion_parameters_regex = re.compile(r"{.*}", re.DOTALL)
def _getERP5ConnexionParameters(self, software_type='export'):
slapos = self._getSlapos(software_type)
out = subprocess.check_output(
(slapos, 'request', 'test_instance', self._test_software_url),
def _getERP5ConnexionParameters(self, instance_type='export'):
out = self.captureSlapos(
'request', 'test_instance', self._test_software_url,
stderr=subprocess.STDOUT,
)
print(out)
......@@ -110,10 +109,10 @@ class ERP5Mixin(object):
raise Exception("Found several partitions for ERP5 %s" % servicename)
return found.pop()
def _getERP5PartitionPath(self, software_type, servicename, *paths):
def _getERP5PartitionPath(self, instance_type, servicename, *paths):
partition = self._getERP5Partition(servicename)
return self._getPartitionPath(
software_type, 'srv', 'runner', 'instance', partition, *paths)
return self.getPartitionPath(
instance_type, 'srv', 'runner', 'instance', partition, *paths)
class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
......@@ -161,16 +160,15 @@ class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
# Update ERP5 parameters
print('Requesting ERP5 with parameters %s' % params)
slapos = self._getSlapos()
subprocess.check_call((slapos, 'request', 'test_instance', self._test_software_url, '--parameters', params))
self.checkSlapos('request', 'test_instance', self._test_software_url, '--parameters', params)
# Process twice to propagate parameter changes
for _ in range(2):
subprocess.check_call((slapos, 'node', 'instance'))
self.checkSlapos('node', 'instance')
# Restart cron (actually all) services to let them take the new date into account
# XXX this should not be required, updating ERP5 parameters should be enough
subprocess.call((slapos, 'node', 'restart', 'all'))
self.callSlapos('node', 'restart', 'all')
# Wait until after the programmed backup date, and a bit more
t = (soon - datetime.now()).total_seconds()
......@@ -213,9 +211,8 @@ class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
self.assertNotIn(self._erp5_new_title, out)
# Stop all services
slapos = self._getSlapos()
print("Stop all services")
subprocess.call((slapos, 'node', 'stop', 'all'))
self.callSlapos('node', 'stop', 'all')
# Manually restore mariadb from backup
mariadb_restore_script = os.path.join(mariadb_partition, 'bin', 'restore-from-backup')
......
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment