Commit 430a1e2a authored by Xavier Thompson's avatar Xavier Thompson

software/theia: Generate supply/request script for embedded instance

See merge request nexedi/slapos!1105
parents 9da0377d ea38c235
Pipeline #18904 failed with stage
......@@ -15,19 +15,19 @@
[instance-theia]
_update_hash_filename_ = instance-theia.cfg.jinja.in
md5sum = 78c99ef4799063dc5b67e76b8dba2112
md5sum = 4df9f0d76a134a8abec9060a0c1be50b
[instance]
_update_hash_filename_ = instance.cfg.in
md5sum = 94703df1104405a5a73aa1bc980ea370
md5sum = f2f01a47d98a980177dc1755e618bbb7
[instance-import]
_update_hash_filename_ = instance-import.cfg.jinja.in
md5sum = 57b707cf0ed83be1959d26a88c131906
md5sum = b0a2c2b3d59fd6c8ba76c634b83a1ba2
[instance-export]
_update_hash_filename_ = instance-export.cfg.jinja.in
md5sum = 190a736471f0e0cffcb2838968e01d84
md5sum = b3f1dd83033d6a45def0bd26e70d5a9c
[instance-resilient]
_update_hash_filename_ = instance-resilient.cfg.jinja
......
......@@ -33,7 +33,7 @@ mode = 0700
exitcode-file = $${directory:srv}/export-exitcode-file
error-file = $${directory:srv}/export-errormessage-file
context =
raw python ${software-info:python-with-eggs}
raw python ${software-info:python-for-resiliency}
raw theia_export ${software-info:theia-export}
raw bash ${software-info:bash}
raw rsync ${software-info:rsync}
......
......@@ -29,6 +29,11 @@ name = Import {{ parameter_dict['additional-frontend-name'] }}
{%- endif %}
# Change standalone socket path to avoid collisions
[slapos-standalone-config]
abstract-socket-path = $${directory:home}/standalone-import-ready
# Change port ranges to avoid race conditions on port allocation
[frontend-instance-port]
minimum = 3200
......@@ -79,7 +84,7 @@ mode = 0700
exitcode-file = $${directory:srv}/import-exitcode-file
error-file = $${directory:srv}/import-errormessage-file
context =
raw python ${software-info:python-with-eggs}
raw python ${software-info:python-for-resiliency}
raw theia_import ${software-info:theia-import}
raw bash ${software-info:bash}
raw rsync ${software-info:rsync}
......
......@@ -151,7 +151,7 @@ config-port = $${slapos-standalone-instance:port}
<= monitor-promise-base
promise = check_socket_listening
name = standalone-ready-promise.py
config-abstract = $${directory:runner}/standalone_ready
config-abstract = $${slapos-standalone-config:abstract-socket-path}
[slapos-autorun-promise]
<= monitor-promise-base
......@@ -464,7 +464,7 @@ template =
slapos supply {{ embedded_sr }} slaprunner
slapos request "Embedded Instance" {{ embedded_sr }}
slapos request "embedded_instance" {{ embedded_sr }}
{%- if embedded_sr_type %} --type {{ embedded_sr_type }} {%- endif %}
{%- if embedded_instance_parameters %} --parameters-file $${embedded-instance-parameters:rendered}
......@@ -476,6 +476,9 @@ template =
{%- endif %}
{%- endif %}
{%- set embedded_digest = str(embedded_sr) + str(embedded_sr_type) + str(embedded_instance_parameters) %}
{%- set embedded_digest_hash = hashlib_module.md5(embedded_digest.encode()).hexdigest() %}
# SlapOS Standalone
# -----------------
......@@ -493,6 +496,7 @@ port = $${slapos-standalone-port:port}
local-software-release-root = $${directory:home}
slapos-configuration = $${directory:runner}/etc/slapos.cfg
computer-id = slaprunner
abstract-socket-path = $${directory:home}/standalone-{{ embedded_digest_hash[:16] }}
[slapos-standalone-activate]
recipe = slapos.recipe.template:jinja2
......@@ -504,6 +508,100 @@ template =
export SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION
echo 'Standalone SlapOS for computer `$${slapos-standalone-config:computer-id}` activated'
[slapos-standalone-script]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
template =
inline:#!${python-for-standalone:executable}
import glob
import json
import os
import signal
import socket
import subprocess
import sys
import time
import traceback
import slapos.slap.standalone
# Include this hash, so that if it changes the standalone service will be restarted
# {{ embedded_digest_hash }}
shared_parts = """{{ '''${buildout:shared-part-list}''' | indent(2) }}"""
shared_part_list = [x.strip() for x in shared_parts.splitlines() if x.strip()]
partition_forward_configuration = (
slapos.slap.standalone.PartitionForwardAsPartitionConfiguration(
master_url="$${slap-connection:server-url}",
computer="$${slap-connection:computer-id}",
partition="$${slap-connection:partition-id}",
cert="$${slap-connection:cert-file}",
key="$${slap-connection:key-file}",
software_release_list=(
'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg',
),
),
)
standalone = slapos.slap.standalone.StandaloneSlapOS(
"$${directory:runner}",
"$${slapos-standalone-config:ipv4}",
$${slapos-standalone-config:port},
computer_id="$${slapos-standalone-config:computer-id}",
shared_part_list=shared_part_list,
software_root="$${directory:runner}/software",
instance_root="$${directory:runner}/instance",
partition_forward_configuration=partition_forward_configuration,
slapos_bin="${buildout:bin-directory}/slapos",
local_software_release_root="$${slapos-standalone-config:local-software-release-root}",
)
def signal_handler(signum, frame):
print("Signal {signum} received".format(signum=signum))
sys.exit()
signal.signal(signal.SIGTERM, signal_handler)
standalone.start()
try:
partition_count = 20
print("Standalone SlapOS: Formatting %d partitions" % partition_count)
standalone.format(partition_count, '$${slapos-standalone-config:ipv4}', '$${slapos-standalone-config:ipv6}')
print("Standalone SlapOS for computer `$${slapos-standalone-config:computer-id}` started")
# Run instance at least once, to start the supervisor managing instances.
try:
standalone.waitForInstance(max_retry=0)
except slapos.slap.standalone.SlapOSNodeCommandError as e:
print("Error instanciating: {}".format(e))
{%- if embedded_sr %}
# Compatibility layer
try:
for cp in standalone.computer.getComputerPartitionList():
if cp.getInstanceParameterDict().get("instance_title") == "Embedded Instance":
print("Renaming 'Embedded Instance' into 'embedded_instance'")
cp.rename(new_name="embedded_instance")
break
except Exception:
print("Exception in compatibility layer, printing and moving on")
traceback.print_exc()
# Run request script
print("Running SlapOS script $${request-embedded-instance-script:rendered}")
slapos_env = {
'PATH': os.path.dirname(standalone._slapos_bin),
'SLAPOS_CONFIGURATION': standalone._slapos_config,
'SLAPOS_CLIENT_CONFIGURATION': standalone._slapos_config
}
subprocess.call(("$${request-embedded-instance-script:rendered}",), env=slapos_env)
{%- endif %}
s = socket.socket(socket.AF_UNIX)
s.bind("\0$${slapos-standalone-config:abstract-socket-path}")
s.listen(5)
print("Standalone SlapOS ready")
while True:
s.accept()[0].close()
finally:
print("Stopping standalone subsystem")
standalone.stop()
print("Exiting")
[slapos-standalone]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
......@@ -514,21 +612,7 @@ template =
#XXX find out where the extra nodejs in theia's PATH comes from
export PATH=${nodejs:location}/bin/:$PATH
. $${slapos-standalone-activate:rendered}
exec ${slapos-standalone:script-path} \
$${directory:runner} \
$${slapos-standalone-config:ipv4} \
$${slapos-standalone-config:ipv6} \
$${slapos-standalone-config:port} \
$${slapos-standalone-config:local-software-release-root} \
$${slapos-standalone-config:computer-id} \
{%- if embedded_sr %}
--slapos_script=$${request-embedded-instance-script:rendered} \
{%- endif %}
$${slap-connection:server-url} \
$${slap-connection:computer-id} \
$${slap-connection:partition-id} \
--key='$${slap-connection:key-file}' \
--cert='$${slap-connection:cert-file}'
exec $${slapos-standalone-script:rendered}
[slapos-standalone-instance]
recipe = slapos.cookbook:wrapper
......@@ -536,6 +620,7 @@ wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line = $${slapos-standalone:rendered}
hash-files =
$${slapos-standalone:rendered}
$${slapos-standalone-script:rendered}
hostname = $${slapos-standalone-config:ipv4}
port = $${slapos-standalone-config:port}
......
......@@ -40,6 +40,7 @@ context =
key ipv6_random slap-configuration:ipv6-random
key ipv4_random slap-configuration:ipv4-random
import os_module os
import hashlib_module hashlib
default-parameters =
{
"autorun": "running",
......
......@@ -21,7 +21,8 @@ extends =
parts =
theia-wrapper
slapos-cookbook
python-with-eggs
python-for-resiliency
python-for-standalone
instance-theia
instance
instance-import
......@@ -33,133 +34,28 @@ parts =
# default for slapos-standalone
shared-part-list =
# Versions
# --------
[gcc]
# We keep the gcc part in sync with the one from erp5 software, so that when we install
# erp5 inside theia's slapos parts can be shared.
[gcc]
max_version = 0
[nodejs]
<= nodejs-14.16.0
[yarn]
<= yarn-1.17.3
[slapos-standalone]
recipe = zc.recipe.egg
eggs =
slapos.core
scripts = ${:_buildout_section_name_}
script-path = ${buildout:bin-directory}/${:scripts}
# XXX generate a fake entry point for a non existant module, that will not
# be used because we exit in initialization step
entry-points =
${:scripts}=not_used:main
initialization =
import argparse
import glob
import json
import os
import signal
import socket
import subprocess
import sys
import time
import slapos.slap.standalone
parser = argparse.ArgumentParser()
parser.add_argument('base_directory')
parser.add_argument('ipv4')
parser.add_argument('ipv6')
parser.add_argument('server_port', type=int)
parser.add_argument('local_software_release_root')
parser.add_argument('computer_id')
parser.add_argument('--slapos_script')
forwarded_arguments = parser.add_argument_group('forwarded')
forwarded_arguments.add_argument('master_url')
forwarded_arguments.add_argument('computer')
forwarded_arguments.add_argument('partition')
# cert and key are optional
forwarded_arguments.add_argument('--cert')
forwarded_arguments.add_argument('--key')
args = parser.parse_args()
shared_part_list = [x.strip() for x in '''${buildout:shared-part-list}'''.splitlines() if x.strip()]
partition_forward_configuration = (
slapos.slap.standalone.PartitionForwardAsPartitionConfiguration(
master_url=args.master_url,
computer=args.computer,
partition=args.partition,
cert=args.cert,
key=args.key,
software_release_list=(
'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg',
),
),
)
standalone = slapos.slap.standalone.StandaloneSlapOS(
args.base_directory,
args.ipv4,
args.server_port,
computer_id=args.computer_id,
shared_part_list=shared_part_list,
software_root="%s/software" % args.base_directory,
instance_root="%s/instance" % args.base_directory,
partition_forward_configuration=partition_forward_configuration,
slapos_bin="${buildout:bin-directory}/slapos",
local_software_release_root=args.local_software_release_root,
)
def signal_handler(signum, frame):
print("Signal {signum} received".format(signum=signum))
sys.exit()
signal.signal(signal.SIGTERM, signal_handler)
standalone.start()
try:
partition_count = 20
print("Standalone SlapOS: Formatting {partition_count} partitions".format(
partition_count=partition_count))
standalone.format(partition_count, args.ipv4, args.ipv6)
print("Standalone SlapOS for computer `{}` started".format(args.computer_id))
# Run instance at least once, to start the supervisor managing instances.
try:
standalone.waitForInstance(max_retry=0)
except slapos.slap.standalone.SlapOSNodeCommandError as e:
print("Error instanciating: {}".format(e))
if args.slapos_script:
print("Running SlapOS script {}".format(args.slapos_script))
slapos_env = {
'PATH': os.path.dirname(standalone._slapos_bin),
'SLAPOS_CONFIGURATION': standalone._slapos_config,
'SLAPOS_CLIENT_CONFIGURATION': standalone._slapos_config
}
subprocess.call((args.slapos_script,), env=slapos_env)
s = socket.socket(socket.AF_UNIX)
s.bind('\0' + os.path.join(args.base_directory, 'standalone_ready'))
s.listen(5)
print("Standalone SlapOS ready")
while True:
s.accept()[0].close()
finally:
print("Stopping standalone subsystem")
standalone.stop()
print("Exiting")
[gowork]
install +=
golang.org/x/tools/gopls@v0.6.6
needs-these-eggs-scripts-in-path =
${supervisor:recipe}
${slapos-command:recipe}
[supervisor]
recipe = zc.recipe.egg
eggs =
supervisor
setuptools
# Downloads and templates
# -----------------------
[template-base]
recipe = slapos.recipe.template
......@@ -172,16 +68,6 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_}
destination = ${buildout:directory}/${:_buildout_section_name_}
output = ${:destination}
[python-language-server]
version = 0.19.0
recipe = plone.recipe.command
command =
PATH=${git:location}/bin/:$PATH bash -c "${python3:executable} -m venv --clear ${:location} && \
. ${:location}/bin/activate && \
pip install -r ${python-language-server-requirements.txt:output}"
location = ${buildout:parts-directory}/${:_buildout_section_name_}
stop-on-error = true
[python-language-server-requirements.txt]
<= download-base
......@@ -191,27 +77,6 @@ stop-on-error = true
[logo.png]
<= download-base
[gowork]
install +=
golang.org/x/tools/gopls@v0.6.6
[cli-utilities]
PATH = ${nodejs:location}/bin:${bash:location}/bin:${fish-shell:location}/bin:${tig:location}/bin:${vim:location}/bin:${tmux:location}/bin:${git:location}/bin:${curl:location}/bin:${python:location}/bin:${buildout:bin-directory}
[python-with-eggs]
recipe = zc.recipe.egg
interpreter = ${:_buildout_section_name_}
eggs =
${slapos-toolbox:eggs}
six
zc.buildout
# Only generate the interpreter script to avoid conflicts with scripts
# for eggs that are also generated by another section, like slapos.toolbox
scripts = ${:interpreter}
[instance-theia]
<= template-base
output = ${buildout:directory}/instance-theia.cfg.jinja
......@@ -243,9 +108,57 @@ destination = ${buildout:directory}/theia_export.py
<= download-base
destination = ${buildout:directory}/theia_import.py
# Utilities
# ---------
[supervisor]
recipe = zc.recipe.egg
eggs =
supervisor
setuptools
[python-language-server]
version = 0.19.0
recipe = plone.recipe.command
command =
PATH=${git:location}/bin/:$PATH bash -c "${python3:executable} -m venv --clear ${:location} && \
. ${:location}/bin/activate && \
pip install -r ${python-language-server-requirements.txt:output}"
location = ${buildout:parts-directory}/${:_buildout_section_name_}
stop-on-error = true
[cli-utilities]
PATH = ${nodejs:location}/bin:${bash:location}/bin:${fish-shell:location}/bin:${tig:location}/bin:${vim:location}/bin:${tmux:location}/bin:${git:location}/bin:${curl:location}/bin:${python:location}/bin:${buildout:bin-directory}
[python-with-eggs]
recipe = zc.recipe.egg
interpreter = ${:_buildout_section_name_}
# Only generate the interpreter script to avoid conflicts with scripts
# for eggs that are also generated by another section, like slapos.toolbox
scripts = ${:interpreter}
executable = ${buildout:bin-directory}/${:interpreter}
[python-for-resiliency]
<= python-with-eggs
eggs =
${slapos-toolbox:eggs}
six
zc.buildout
[python-for-standalone]
<= python-with-eggs
eggs =
slapos.core
needs-these-eggs-scripts-in-path =
${supervisor:recipe}
${slapos-command:recipe}
[software-info]
slapos = ${buildout:bin-directory}/slapos
python-with-eggs = ${buildout:bin-directory}/${python-with-eggs:interpreter}
python-for-resiliency = ${python-for-resiliency:executable}
python = ${python:location}/bin/python
rsync = ${rsync:location}/bin/rsync
sqlite3 = ${sqlite3:location}/bin/sqlite3
......
......@@ -40,7 +40,7 @@ import six
from six.moves.urllib.parse import urlparse, urljoin
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass, SlapOSNodeCommandError
from slapos.grid.svcbackend import getSupervisorRPC, _getSupervisordSocketPath
......@@ -229,26 +229,76 @@ class TestTheiaEmbeddedSlapOSShutdown(TheiaTestCase):
self.assertFalse(embedded_slapos_process.is_running())
class TestTheiaWithSR(TheiaTestCase):
class ReRequestMixin(object):
def rerequest(self, parameter_dict=None, state='started'):
software_url = self.getSoftwareURL()
software_type = self.getInstanceSoftwareType()
name = self.default_partition_reference
self.slap.request(
software_release=software_url,
software_type=software_type,
partition_reference=name,
partition_parameter_kw=parameter_dict,
state=state)
def reinstantiate(self):
# Process at least twice to propagate parameter changes
try:
self.slap.waitForInstance()
except SlapOSNodeCommandError:
pass
self.slap.waitForInstance(self.instance_max_retry)
class TestTheiaWithSR(TheiaTestCase, ReRequestMixin):
sr_url = '~/bogus/software.cfg'
sr_type = 'bogus_type'
instance_parameters = '{\n"bogus_param": "bogus_value",\n"bogus_param2": "bogus_value2"\n}'
@classmethod
def getInstanceParameterDict(cls):
return {
'embedded-sr': cls.sr_url,
'embedded-sr-type': cls.sr_type,
'embedded-instance-parameters': cls.instance_parameters
}
def proxy_show(self, slapos):
return subprocess.check_output((slapos, 'proxy', 'show'), universal_newlines=True)
def test(self):
slapos = self._getSlapos()
home = self.computer_partition_root_path
bogus_sr = os.path.join(home, self.sr_url[2:])
slapos = self._getSlapos()
info = subprocess.check_output((slapos, 'proxy', 'show'), universal_newlines=True)
instance_name = "Embedded Instance"
# Check that no request script was generated
request_script = os.path.join(home, 'srv', 'project', 'request_embedded.sh')
self.assertFalse(os.path.exists(request_script))
# Manually request old-name 'Embedded Instance'
old_instance_name = "Embedded Instance"
subprocess.check_call((slapos, 'request', old_instance_name, 'bogus_url'))
self.assertIn(old_instance_name, self.proxy_show(slapos))
# Update Theia instance parameters
embedded_request_parameters = {
'embedded-sr': self.sr_url,
'embedded-sr-type': self.sr_type,
'embedded-instance-parameters': self.instance_parameters
}
self.rerequest(embedded_request_parameters)
self.reinstantiate()
# Check that embedded instance was requested
instance_name = "embedded_instance"
info = self.proxy_show(slapos)
try:
self.assertIn(instance_name, info)
except AssertionError:
for filename in os.listdir(home):
if 'standalone' in filename and '.log' in filename:
filepath = os.path.join(home, filename)
with open(filepath) as f:
print("Contents of filepath: " + filepath)
print(f.read())
raise
# Check that old-name instance was renamed
self.assertNotIn(old_instance_name, info)
# Check embedded instance parameters
bogus_sr = os.path.join(home, self.sr_url[2:])
self.assertIsNotNone(re.search(r"%s\s+slaprunner\s+available" % (bogus_sr,), info), info)
self.assertIsNotNone(re.search(r"%s\s+%s\s+%s" % (bogus_sr, self.sr_type, instance_name), info), info)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment