Commit 64f10409 authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼

Update Release Candidate

parents ad5fe700 7ae4d31a
......@@ -14,7 +14,7 @@ recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/cloudooo.git
branch = master
git-executable = ${git:location}/bin/git
revision = be6c35c0156e028f31da3ccb205afc6f95728d97
revision = 0b5ff71a2ede76499e81659aed392057ae910917
[cloudooo]
recipe = zc.recipe.egg
......
......@@ -62,7 +62,7 @@ md5sum = 975177dedf677d24e14cede5d13187ce
[template-trafficserver-records-config]
_update_hash_filename_ = templates/trafficserver/records.config.jinja2
md5sum = b99403e02d1aff471a7d5ebd0afbdb6c
md5sum = 1696321871bb71083550c95d77487602
[template-trafficserver-storage-config]
_update_hash_filename_ = templates/trafficserver/storage.config.jinja2
......
......@@ -80,7 +80,7 @@ CONFIG proxy.config.http.uncacheable_requests_bypass_parent INT 1
CONFIG proxy.config.http.keep_alive_no_activity_timeout_in INT 120
CONFIG proxy.config.http.keep_alive_no_activity_timeout_out INT 120
CONFIG proxy.config.http.transaction_no_activity_timeout_in INT {{ ats_configuration['request-timeout'] }}
CONFIG proxy.config.http.transaction_no_activity_timeout_out INT 30 {{ ats_configuration['request-timeout'] }}
CONFIG proxy.config.http.transaction_no_activity_timeout_out INT {{ ats_configuration['request-timeout'] }}
CONFIG proxy.config.http.transaction_active_timeout_in INT 900
CONFIG proxy.config.http.transaction_active_timeout_out INT 0
CONFIG proxy.config.http.accept_no_activity_timeout INT 120
......
......@@ -18,4 +18,4 @@ md5sum = e986de01a57161b32425f1cd3ccac924
[template-cloudooo-instance]
filename = instance-cloudooo.cfg.in
md5sum = 9e1a66cf18d7c30c14afeb66c20afb46
md5sum = 440f2b82b119cbfa6f8c7d27652c3170
......@@ -162,6 +162,7 @@ environment =
LD_LIBRARY_PATH = {{ parameter_dict['cairo'] }}/lib:{{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['gcc'] }}/lib:{{ parameter_dict['gcc'] }}/lib64:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['glu'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['libexpat'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['mesa'] }}/lib:{{ parameter_dict['pixman'] }}/lib:{{ parameter_dict['xdamage'] }}/lib:{{ parameter_dict['xfixes'] }}/lib:{{ parameter_dict['zlib'] }}/lib
FONTCONFIG_FILE = ${fontconfig-conf:rendered}
PATH = ${binary-link:target-directory}
LANG = C.UTF-8
mimetype_entry_addition =
{% for entry in mimetype_entry_addition.splitlines() -%}
{{ " " ~ entry.strip() }}
......@@ -172,7 +173,7 @@ ooo-binary-path = {{ parameter_dict['libreoffice-bin'] }}/program
ooo-paster = {{ bin_directory }}/cloudooo_paster
ooo-uno-path = {{ parameter_dict['libreoffice-bin'] }}/basis-link/program
{% for index in range(backend_count) -%}
{% for index in range(1, backend_count + 1) -%}
{% set name = 'cloudooo-' ~ index -%}
[{{ cloudooo(name) }}]
< = cloudooo-base
......@@ -207,8 +208,8 @@ prepend-path = ${buildout:bin-directory}
run-unit-test = ${buildout:bin-directory}/runUnitTest
run-test-suite = ${buildout:bin-directory}/runTestSuite
ooo-paster = ${cloudooo-0:ooo-paster}
configuration-file = ${cloudooo-0:configuration-file}
ooo-paster = ${cloudooo-1:ooo-paster}
configuration-file = ${cloudooo-1:configuration-file}
run-unit-test-binary = {{ bin_directory }}/runCloudoooUnitTest
run-test-suite-binary = {{ bin_directory }}/runCloudoooTestSuite
......
......@@ -44,6 +44,7 @@ setup(name=name,
'slapos.core',
'slapos.cookbook',
'slapos.libnetworkcache',
'requests',
'six',
'PyPDF2',
],
......
......@@ -26,23 +26,43 @@
#
##############################################################################
import codecs
import csv
import multiprocessing
import os
import json
import six.moves.xmlrpc_client as xmlrpclib
import six.moves.urllib.parse as urllib_parse
import ssl
import base64
import io
import requests
import PyPDF2
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, CloudOooTestCase = makeModuleSetUpAndTestCaseClass(
setUpModule, _CloudOooTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
# Cloudooo needs a lot of time before being available.
CloudOooTestCase.instance_max_retry = 30
class CloudOooTestCase(_CloudOooTestCase):
# Cloudooo needs a lot of time before being available.
instance_max_retry = 30
def setUp(self):
self.url = json.loads(
self.computer_partition.getConnectionParameterDict()["_"])['cloudooo']
# XXX ignore certificate errors
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
self.server = xmlrpclib.ServerProxy(
self.url,
context=ssl_context,
allow_none=True,
)
def normalizeFontName(font_name):
......@@ -88,19 +108,6 @@ class HTMLtoPDFConversionFontTestMixin:
"""Convert the HTML source to pdf bytes.
"""
def setUp(self):
self.url = json.loads(
self.computer_partition.getConnectionParameterDict()["_"])['cloudooo']
# XXX ignore certificate errors
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
self.server = xmlrpclib.ServerProxy(
self.url,
context=ssl_context,
allow_none=True,
)
def test(self):
actual_font_mapping_mapping = {}
for font in self.expected_font_mapping:
......@@ -237,3 +244,84 @@ class TestLibreoffice(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
'html',
'pdf',
).encode())
class TestLibreOfficeTextConversion(CloudOooTestCase):
__partition_reference__ = 'txt'
def test_html_to_text(self):
self.assertEqual(
base64.decodestring(
self.server.convertFile(
base64.encodestring(
u'<html>héhé</html>'.encode('utf-8')).decode(),
'html',
'txt',
).encode()),
codecs.BOM_UTF8 + b'h\xc3\xa9h\xc3\xa9\n',
)
class TestLibreOfficeCluster(CloudOooTestCase):
__partition_reference__ = 'lc'
@classmethod
def getInstanceParameterDict(cls):
return {'backend-count': 4}
def test_multiple_conversions(self):
# make this function global so that it can be picked and used by multiprocessing
global _convert_html_to_text
def _convert_html_to_text(src_html):
return base64.decodestring(
self.server.convertFile(
base64.encodestring(src_html.encode()).decode(),
'html',
'txt',
).encode())
pool = multiprocessing.Pool(5)
# TODO py3: use with pool
converted = pool.map(_convert_html_to_text,
['<html><body>hello</body></html>'] * 100)
pool.terminate()
pool.join()
self.assertEqual(converted, [codecs.BOM_UTF8 + b'hello\n'] * 100)
# haproxy stats are exposed
res = requests.get(
urllib_parse.urljoin(self.url, '/haproxy;csv'),
verify=False,
stream=True,
)
reader = csv.DictReader(res.raw)
line_list = list(reader)
# requests have been balanced
total_hrsp_2xx = {
line['svname']: int(line['hrsp_2xx'])
for line in line_list
}
self.assertEqual(total_hrsp_2xx['FRONTEND'], 100)
self.assertEqual(total_hrsp_2xx['BACKEND'], 100)
for backend in 'cloudooo_1', 'cloudooo_2', 'cloudooo_3', 'cloudooo_4':
# ideally there should be 25% of requests on each backend, because we use
# round robin scheduling, but it can happen that some backend take longer
# to start, so we are tolerant here and just check that each backend
# process at least 15% of requests.
self.assertGreater(total_hrsp_2xx[backend], 15)
# no errors
total_eresp = {
line['svname']: int(line['eresp'] or 0)
for line in line_list
}
self.assertEqual(
total_eresp, {
'FRONTEND': 0,
'cloudooo_1': 0,
'cloudooo_2': 0,
'cloudooo_3': 0,
'cloudooo_4': 0,
'BACKEND': 0,
})
......@@ -19,7 +19,7 @@ md5sum = 0d34ff81779115bf899f7bc752877b70
[template-kvm]
filename = instance-kvm.cfg.jinja2
md5sum = 854ec0f379a05246f5f3761fdb634c36
md5sum = bf0c01ac7493693bb57ebef00bb20fa0
[template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in
......@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
[template-kvm-run]
filename = template/template-kvm-run.in
md5sum = a8afeb2f80199368e291621576a80ca1
md5sum = b8cc7c76438212e0522ebede88649393
[template-kvm-controller]
filename = template/kvm-controller-run.in
......
......@@ -359,7 +359,7 @@
},
"virtual-hard-drive-md5sum": {
"title": "Checksum of virtual hard drive",
"description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.",
"description": "MD5 checksum of virtual hard drive, required if virtual-hard-drive-url is specified.",
"type": "string"
},
"virtual-hard-drive-gzipped": {
......
......@@ -160,7 +160,7 @@
},
"virtual-hard-drive-md5sum": {
"title": "Checksum of virtual hard drive",
"description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.",
"description": "MD5 checksum of virtual hard drive, required if virtual-hard-drive-url is specified.",
"type": "string"
},
"virtual-hard-drive-gzipped": {
......
......@@ -17,6 +17,8 @@
{% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
{% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%}
{% set whitelist_domains = slapparameter_dict.get('whitelist-domains', '') -%}
{% set virtual_hard_drive_url_enabled = 'virtual-hard-drive-url' in slapparameter_dict %}
{% set virtual_hard_drive_url_gzipped = str(slapparameter_dict.get('virtual-hard-drive-gzipped', False)).lower() == 'true' %}
{% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %}
{% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %}
{% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url') -%}
......@@ -56,6 +58,11 @@ public = ${:srv}/public/
cron-entries = ${:etc}/cron.d
crontabs = ${:etc}/crontabs
cronstamps = ${:etc}/cronstamps
{%- if virtual_hard_drive_url_enabled %}
virtual-hard-drive-url-repository = ${:srv}/virtual-hard-drive-url-repository
virtual-hard-drive-url-var = ${:var}/virtual-hard-drive-url
virtual-hard-drive-url-expose = ${monitor-directory:private}/virtual-hard-drive-url
{%- endif %}
{%- if boot_image_url_list_enabled %}
boot-image-url-list-repository = ${:srv}/boot-image-url-list-repository
boot-image-url-list-var = ${:var}/boot-image-url-list
......@@ -278,6 +285,106 @@ config-filename = ${boot-image-url-list-download-wrapper:error-state-file}
## boot-image-url-list support END
{% endif %} {# if boot_image_url_list_enabled #}
{% if virtual_hard_drive_url_enabled %}
## virtual-hard-drive-url support BEGIN
[empty-file-state-base-virtual-promise]
<= monitor-promise-base
module = check_file_state
name = ${:_buildout_section_name_}.py
config-state = empty
# It's very hard to put the username and password correctly, after schema://
# and before the host, as it's not the way how one can use monitor provided
# information, so just show the information in the URL
config-url = ${monitor-base:base-url}/private/virtual-hard-drive-url/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
[virtual-hard-drive-url-source-config]
recipe = slapos.recipe.template:jinja2
template = inline:
{%- raw %}
{{ virtual_hard_drive_url }}
{% endraw -%}
{# Enforce md5sum on virtual-hard-drive-url #}
virtual-hard-drive-url = {{ slapparameter_dict['virtual-hard-drive-url'] }}#{{ slapparameter_dict['virtual-hard-drive-md5sum'] }}
context =
key virtual_hard_drive_url :virtual-hard-drive-url
rendered = ${directory:etc}/virtual-hard-drive-url.conf
[virtual-hard-drive-url-processed-config]
# compares if the current configuration has been used by
# the virtual-hard-drive-url-download, if not, exposes it as not empty file with
# information
recipe = slapos.recipe.build
install =
import os
import hashlib
if not os.path.exists(location):
os.mkdir(location)
with open('${:state-file}', 'w') as state_handler:
try:
with open('${:config-file}', 'rb') as config_handler, open('${:processed-md5sum}') as processed_handler:
config_md5sum = hashlib.md5(config_handler.read()).hexdigest()
processed_md5sum = processed_handler.read()
if config_md5sum == processed_md5sum:
state_handler.write('')
else:
state_handler.write('config %s != processed %s' % (config_md5sum, processed_md5sum))
except Exception as e:
state_handler.write(str(e))
update = ${:install}
config-file = ${virtual-hard-drive-url-source-config:rendered}
state-filename = virtual-hard-drive-url-processed-config.state
state-file = ${directory:virtual-hard-drive-url-expose}/${:state-filename}
processed-md5sum = ${directory:virtual-hard-drive-url-var}/update-image-processed.md5sum
[virtual-hard-drive-url-processed-config-promise]
# promise to check if the configuration provided by the user has been already
# processed by the virtual-hard-drive-url-download script, which runs asynchronously
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-processed-config:state-filename}
config-filename = ${virtual-hard-drive-url-processed-config:state-file}
[virtual-hard-drive-url-json-config]
# generates json configuration from user configuration
recipe = plone.recipe.command
command = {{ python_executable }} {{ image_download_config_creator }} ${virtual-hard-drive-url-source-config:rendered} ${:rendered} ${directory:virtual-hard-drive-url-repository} ${:error-state-file}
update-command = ${:command}
rendered = ${directory:virtual-hard-drive-url-var}/virtual-hard-drive-url.json
error-state-filename = virtual-hard-drive-url-json-config-error.txt
error-state-file = ${directory:virtual-hard-drive-url-expose}/${:error-state-filename}
[virtual-hard-drive-url-config-state-promise]
# promise to check if configuration has been parsed without errors
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-json-config:error-state-filename}
config-filename = ${virtual-hard-drive-url-json-config:error-state-file}
[virtual-hard-drive-url-download-wrapper]
# wrapper to execute virtual-hard-drive-url-download on each run
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:scripts}/virtual-hard-drive-url-updater
command-line = {{ python_executable }} {{ image_download_controller }} ${virtual-hard-drive-url-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${virtual-hard-drive-url-processed-config:processed-md5sum}
md5sum-state-filename = virtual-hard-drive-url-download-controller-md5sum-fail.json
md5sum-state-file = ${directory:virtual-hard-drive-url-expose}/${:md5sum-state-filename}
error-state-filename = virtual-hard-drive-url-download-controller-error.text
error-state-file = ${directory:virtual-hard-drive-url-expose}/${:error-state-filename}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[virtual-hard-drive-url-download-md5sum-promise]
# promise to report errors with problems with calculating md5sum of the
# downloaded images
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-download-wrapper:md5sum-state-filename}
config-filename = ${virtual-hard-drive-url-download-wrapper:md5sum-state-file}
[virtual-hard-drive-url-download-state-promise]
# promise to report errors during download
<= empty-file-state-base-virtual-promise
filename = ${virtual-hard-drive-url-download-wrapper:error-state-filename}
config-filename = ${virtual-hard-drive-url-download-wrapper:error-state-file}
## virtual-hard-drive-url support END
{% endif %} {# if virtual_hard_drive_url_enabled #}
[kvm-controller-parameter-dict]
python-path = {{ python_eggs_executable }}
vnc-passwd = ${gen-passwd:passwd}
......@@ -298,6 +405,11 @@ vnc-ip = ${:ipv4}
vnc-port = 5901
default-cdrom-iso = {{ debian_amd64_netinst_location }}
{% if virtual_hard_drive_url_enabled %}
virtual-hard-drive-url-json-config = ${virtual-hard-drive-url-json-config:rendered}
{% else %}
virtual-hard-drive-url-json-config =
{% endif %}
{% if boot_image_url_list_enabled %}
boot-image-url-list-json-config = ${boot-image-url-list-json-config:rendered}
{% else %}
......@@ -425,14 +537,42 @@ ipv6-port = {{ external_port }}
{% endfor -%}
{% endif -%}
{%- set depend_section_list = [] %}
{%- set hash_file_list = ['${kvm-run:rendered}'] %}
{%- macro generate_depend_section(section, key) %}
{%- do depend_section_list.append('${' + section + ':command}' ) %}
{%- do hash_file_list.append('${' + key + '}') %}
[{{ section }}]
recipe = plone.recipe.command
update-command = ${:command}
command = [ ! -f {{ '${' + key + '}' }} ] && touch {{ '${' + key + '}' }}
{%- endmacro %}
{#- Create depending sections, as state files appear late, so it's better to have empty file which will impact the hash anyway #}
{%- if boot_image_url_list_enabled %}
{{ generate_depend_section('boot-image-url-list-depend', 'boot-image-url-list-download-wrapper:md5sum-state-file') }}
{%- endif %}
{%- if boot_image_url_select_enabled %}
{{ generate_depend_section('boot-image-url-select-depend', 'boot-image-url-select-download-wrapper:md5sum-state-file') }}
{%- endif %}
{%- if virtual_hard_drive_url_enabled %}
{{ generate_depend_section('virtual-hard-drive-url-depend', 'virtual-hard-drive-url-download-wrapper:md5sum-state-file') }}
{%- endif %}
[kvm-instance]
depends =
{%- for depend_section in depend_section_list %}
{{ depend_section }}
{%- endfor %}
recipe = slapos.cookbook:wrapper
socket-path = ${kvm-controller-parameter-dict:socket-path}
wrapper-path = ${directory:services}/kvm
command-line = ${kvm-run:rendered}
kvm-controller = ${kvm-controller-wrapper:wrapper-path}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
hash-files =
{%- for hash_file in hash_file_list %}
{{ hash_file }}
{%- endfor %}
[kvm-controller-wrapper]
recipe = slapos.cookbook:wrapper
......@@ -932,8 +1072,8 @@ disk-format = qcow2
disk-device-path =
cpu-count = 1
disk-cache = writeback
disk-aio = native
disk-cache =
disk-aio =
auto-ballooning = True
machine-options =
cpu-model = host
......@@ -1108,6 +1248,13 @@ parts =
cron-service
cron-entry-logrotate
frontend-promise
{% if virtual_hard_drive_url_enabled %}
virtual-hard-drive-url-download-wrapper
virtual-hard-drive-url-config-state-promise
virtual-hard-drive-url-download-md5sum-promise
virtual-hard-drive-url-download-state-promise
virtual-hard-drive-url-processed-config-promise
{% endif %}
{% if boot_image_url_list_enabled %}
boot-image-url-list-download-wrapper
boot-image-url-list-config-state-promise
......
......@@ -6,10 +6,6 @@ import hashlib
import os
import socket
import subprocess
try:
from urllib.request import FancyURLopener
except ImportError:
from urllib import FancyURLopener
import gzip
import shutil
from random import shuffle
......@@ -17,16 +13,11 @@ import glob
import re
import json
import ssl
# XXX: give all of this through parameter, don't use this as template, but as module
qemu_img_path = '{{ parameter_dict.get("qemu-img-path") }}'
qemu_path = '{{ parameter_dict.get("qemu-path") }}'
disk_size = '{{ parameter_dict.get("disk-size") }}'
disk_type = '{{ parameter_dict.get("disk-type") }}'
disk_format = '{{ parameter_dict.get("disk-format", "qcow2")}}'
disk_format = disk_format \
if disk_format in ['qcow2', 'raw', 'vdi', 'vmdk', 'cloop', 'qed'] else 'qcow2'
qemu_img_path = {{ repr(parameter_dict["qemu-img-path"]) }}
qemu_path = {{ repr(parameter_dict["qemu-path"]) }}
disk_size = {{ repr(parameter_dict["disk-size"]) }}
disk_type = {{ repr(parameter_dict["disk-type"]) }}
socket_path = '{{ parameter_dict.get("socket-path") }}'
nbd_list = (('{{ parameter_dict.get("nbd-host") }}',
......@@ -34,11 +25,7 @@ nbd_list = (('{{ parameter_dict.get("nbd-host") }}',
('{{ parameter_dict.get("nbd2-host") }}',
{{ parameter_dict.get("nbd2-port") }}))
default_cdrom_iso = '{{ parameter_dict.get("default-cdrom-iso") }}'
disk_path = '{{ parameter_dict.get("disk-path") }}'
virtual_hard_drive_url = '{{ parameter_dict.get("virtual-hard-drive-url") }}'.strip()
virtual_hard_drive_md5sum = '{{ parameter_dict.get("virtual-hard-drive-md5sum") }}'.strip()
virtual_hard_drive_gzipped = '{{ parameter_dict.get("virtual-hard-drive-gzipped") }}'.strip().lower()
nat_rules = '{{ parameter_dict.get("nat-rules") }}'.strip()
use_tap = '{{ parameter_dict.get("use-tap") }}'.lower()
use_nat = '{{ parameter_dict.get("use-nat") }}'.lower()
......@@ -56,9 +43,7 @@ init_ram_size = {{ parameter_dict.get("init-ram-size") }}
pid_file_path = '{{ parameter_dict.get("pid-file-path") }}'
external_disk_number = {{ parameter_dict.get("external-disk-number") }}
external_disk_size = {{ parameter_dict.get("external-disk-size") }}
external_disk_format = '{{ parameter_dict.get("external-disk-format", "qcow2") }}'
external_disk_format = external_disk_format \
if external_disk_format in ['qcow2', 'raw', 'vdi', 'vmdk', 'cloop', 'qed'] else 'qcow2'
external_disk_format = {{ repr(parameter_dict["external-disk-format"]) }}
disk_storage_dict = {}
disk_storage_list = """{{ parameter_dict.get("disk-storage-list") }}""".split('\n')
map_storage_list = []
......@@ -70,11 +55,6 @@ cluster_doc_port = {{ parameter_dict.get("cluster-doc-port") }}
url_check_certificate = '{{ parameter_dict.get("hard-drive-url-check-certificate", "true") }}'.lower()
auto_ballooning = '{{ parameter_dict.get("auto-ballooning") }}' in ('true', 'True', '1')
vm_name = '{{ parameter_dict.get("name") }}'
disk_cache = '{{ parameter_dict.get("disk-cache", "writeback") }}'.strip()
disk_cache = disk_cache if disk_cache in ["none", "writeback", "unsafe",
"directsync", "writethrough"] else "writeback"
disk_aio = '{{ parameter_dict.get("disk-aio", "threads") }}'.strip()
disk_aio = disk_aio if disk_aio in ["threads", "native"] else "threads"
# If a device (ie.: /dev/sdb) is provided, use it instead
# the disk_path with disk_format
......@@ -84,16 +64,18 @@ for disk_device_path in '{{ parameter_dict.get("disk-device-path", "") }}'.split
disk_info_list.append({
'path': disk_device_path,
'format': "raw",
'io': "native",
'aio': "native",
'cache': "none"
})
if len(disk_info_list) == 0:
if not disk_info_list:
disk_info_list.append({
'path': disk_path,
'format': disk_format,
'io': disk_aio,
'cache': disk_cache,
{%- for k in 'path', 'format', 'aio', 'cache' %}
{%- set v = parameter_dict['disk-' + k] %}
{%- if v %}
{{ repr(k) }}: {{ repr(v) }},
{%- endif %}
{%- endfor %}
})
smp_count = {{ parameter_dict.get("smp-count") }}
......@@ -107,11 +89,8 @@ logfile = '{{ parameter_dict.get("log-file") }}'
boot_image_url_list_json_config = '{{ parameter_dict.get("boot-image-url-list-json-config") }}'
boot_image_url_select_json_config = '{{ parameter_dict.get("boot-image-url-select-json-config") }}'
if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false':
opener = FancyURLopener(context=ssl._create_unverified_context())
else:
opener = FancyURLopener({})
virtual_hard_drive_url_json_config = '{{ parameter_dict.get("virtual-hard-drive-url-json-config") }}'
virtual_hard_drive_gzipped = '{{ parameter_dict.get("virtual-hard-drive-gzipped") }}'.strip().lower()
def md5Checksum(file_path):
with open(file_path, 'rb') as fh:
......@@ -172,39 +151,34 @@ def getMapStorageList(disk_storage_dict, external_disk_number):
lf.write('%s' % external_disk_number)
return id_list, external_disk_number
# Download existing hard drive if needed at first boot
if len(disk_info_list) == 1 and not os.path.exists(disk_info_list[0]['path']) and virtual_hard_drive_url != '':
print('Downloading virtual hard drive...')
try:
downloaded_disk = disk_info_list[0]['path']
# Use downloaded virtual-hard-drive-url
if len(disk_info_list) == 1 and not os.path.exists(disk_info_list[0]['path']) and virtual_hard_drive_url_json_config != '':
print('Using virtual hard drive...')
with open(virtual_hard_drive_url_json_config) as fh:
image_config = json.load(fh)
if image_config['error-amount'] == 0:
image = image_config['image-list'][0]
downloaded_image = os.path.join(image_config['destination-directory'], image['destination'])
# previous version was using disk in place, but here it would result with
# redownload, so copy it
if virtual_hard_drive_gzipped == 'true':
downloaded_disk = '%s.gz' % disk_info_list[0]['path']
opener.retrieve(virtual_hard_drive_url, downloaded_disk)
except:
if os.path.exists(downloaded_disk):
os.remove(downloaded_disk)
raise
md5sum = virtual_hard_drive_md5sum.strip()
if md5sum:
print('Checking MD5 checksum...')
local_md5sum = md5Checksum(downloaded_disk)
if local_md5sum != md5sum:
os.remove(downloaded_disk)
raise Exception('MD5 mismatch. MD5 of local file is %s, Specified MD5 is %s.' % (
local_md5sum, md5sum))
print('MD5sum check passed.')
try:
with open(disk_info_list[0]['path'], 'wb') as d_fh:
with gzip.open(downloaded_image, 'rb') as s_fh:
shutil.copyfileobj(s_fh, d_fh)
except Exception:
if os.path.exists(disk_info_list[0]['path']):
os.unlink(disk_info_list[0]['path'])
raise
else:
try:
shutil.copyfile(downloaded_image, disk_info_list[0]['path'])
except Exception:
if os.path.exists(disk_info_list[0]['path']):
os.unlink(disk_info_list[0]['path'])
raise
else:
print('Warning: not checksum specified.')
if downloaded_disk.endswith('.gz'):
try:
with open(disk_info_list[0]['path'], 'w') as disk:
with gzip.open(downloaded_disk, 'rb') as disk_gz:
shutil.copyfileobj(disk_gz, disk)
except Exception:
if os.path.exists(disk_info_list[0]['path']):
os.remove(disk_info_list[0]['path'])
raise
os.remove(downloaded_disk)
raise ValueError('virtual-hard-drive-url not ready yet')
# Create disk if doesn't exist
# XXX: move to Buildout profile
......@@ -310,17 +284,12 @@ kvm_argument_list = [qemu_path,
'-nodefaults',
]
for disk_info in disk_info_list:
additional_disk_options = ''
if disk_info['io'] == 'native':
additional_disk_options += ',cache.direct=on'
if disk_info['format'] == "raw":
additional_disk_options += ',discard=on'
kvm_argument_list.extend([
kvm_argument_list += (
'-drive',
'file=%s,if=%s,cache=%s,aio=%s%s' % ( disk_info['path'], disk_type, disk_info['cache'], disk_info['io'], additional_disk_options)
])
'file=%s,if=%s,discard=on%s' % (
disk_info['path'], disk_type,
''.join(',%s=%s' % x for x in disk_info.items() if x[0] != 'path'))
)
rgx = re.compile('^[\w*\,][\=\d+\-\,\w]*$')
for numa in numa_list:
......@@ -392,6 +361,9 @@ else:
'-drive',
'file=%s,media=cdrom' % (link,)
])
else:
raise ValueError('boot-image-url-select not ready yet')
if boot_image_url_list_json_config:
# Support boot-image-url-list
with open(boot_image_url_list_json_config) as fh:
......@@ -404,6 +376,8 @@ else:
'-drive',
'file=%s,media=cdrom' % (link,)
])
else:
raise ValueError('boot-image-url-list not ready yet')
# Always add by default the default image
kvm_argument_list.extend([
'-drive', 'file=%s,media=cdrom' % default_cdrom_iso
......
This diff is collapsed.
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = 9ddae686379e8d747410c1adf82b47d6
md5sum = c115ed9d4ff0f785d79cdcacbb0bd1ad
[template-monitor]
_update_hash_filename_ = instance-monitor.cfg.jinja2
......@@ -26,11 +26,11 @@ md5sum = 2eb5596544d9c341acf653d4f7ce2680
[template-monitor-edgetest]
_update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
md5sum = a57106ee88ff3295b9ffce84105da79b
md5sum = 3c8ab4e78f66c974eb95afc595a13514
[template-monitor-edgebot]
_update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
md5sum = c1885a42aadd45bab3185a53258d4ff4
md5sum = 365a6cc6831267a73fa5ebd56ad394ee
[network-bench-cfg]
filename = network_bench.cfg.in
......@@ -42,4 +42,4 @@ md5sum = d3cfa1f6760e3fa64ccd64acf213bdfb
[template-surykatka-ini]
_update_hash_filename_ = surykatka.ini.jinja2
md5sum = 89545501f0e5bf11608978886429da3d
md5sum = 609c6cca763b73a80fa05ee56475eb20
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
"nameserver": {
"default": "",
"title": "Nameserver",
"description": "Space separated list of name servers to use.",
"type": "string"
},
"check-status-code": {
"default": "200",
"title": "Default Check HTTP Code",
"description": "Default HTTP code to check against (default: 200).",
"type": "string"
},
"check-http-header-dict": {
"default": "{}",
"title": "HTTP header dict to check",
"description": "JSON dict of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}",
"region-dict": {
"title": "Regions",
"description": "Defines regions of the cluster",
"patternProperties": {
".*": {
"properties": {
"state": {
"title": "State",
"description": "State of the node of the region. Can be used to destroy not needed regions.",
"type": "string",
"default": "started",
"enum": [
"started",
"stopped",
"destroyed"
]
},
"sla-computer_guid": {
"title": "GUID of the computer on which this region shall be deployed",
"description": "Unique identifier of the computer, like \"COMP-1234\". By default, let Master choose a computer.",
"type": "string",
"default": ""
},
"nameserver-list": {
"default": [],
"title": "Nameservers",
"description": "List of nameservers to use.",
"type": "array"
},
"check-frontend-ip-list": {
"default": [],
"title": "Default Frontend IPs to check",
"description": "List of default frontend IPs to check, if empty no constraint is used.",
"type": "array"
}
},
"type": "object"
}
},
"type": "object"
},
"check-frontend-ip": {
"default": "",
"title": "Default space separated list of Frontend IPs to check",
"description": "Default list of Frontend IPs to check, if empty no constraint is used.",
"type": "string"
},
"check-certificate-expiration-days": {
"default": "15",
"title": "Default certificate expiration days check",
"description": "Default amount of days to consider certificate as being to-be-expired (default: 15).",
"type": "string"
},
"check-maximum-elapsed-time": {
"default": "2",
"title": "Default maximum elapsed time for a site to reply (seconds)",
"description": "Default maximum elapsed time for a site to reply to be considered good (default: 2s).",
"type": "string"
"nameserver-list": {
"default": [],
"title": "Nameservers (backward compatibility)",
"description": "List of nameservers to use. Note: This is backward compatibility, use region-dict for full configuration control.",
"type": "array"
},
"failure-amount": {
"default": "1",
"title": "Default amount of failures to consider URL as in bad state",
"description": "Default amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages (default: 1).",
"type": "string"
"check-frontend-ip-list": {
"default": [],
"title": "Default Frontend IPs to check (backward compatibility)",
"description": "List of default frontend IPs to check, if empty no constraint is used. Note: This is backward compatibility, use region-dict for full configuration control.",
"type": "array"
}
}
}
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
"url": {
"title": "URL to check",
"description": "URL to check, like https://example.com",
"description": "URL to check, like https://example.com/",
"type": "string"
},
"check-status-code": {
"default": "Master default",
"title": "Default Check HTTP Code.",
"description": "HTTP code to check against (default: comes from master partition).",
"type": "string"
},
"check-http-header-dict": {
"default": "Master default",
"title": "HTTP header dict to check",
"description": "JSON dict of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}",
"type": "object"
"region-dict": {
"title": "Applicable Regions",
"description": "Puts the check on the defined regions. No definition will result with presence in all regions.",
"patternProperties": {
".*": {
"properties": {
"state": {
"title": "State",
"description": "State of the check of the region. Used only to make it correctly visible in the SlapOS Master UI if no other parameters are defined.",
"type": "string",
"default": "present",
"enum": [
"present"
]
},
"check-frontend-ip-list": {
"default": [],
"title": "Frontend IPs to check",
"description": "List of default frontend IPs to check, if empty no constraint is used. Defaults to region configuration.",
"type": "array"
}
},
"type": "object"
}
},
"type": "object",
"default": {}
},
"check-frontend-ip": {
"default": "Master default",
"title": "Space separated list of Frontend IPs to check",
"description": "List of Frontend IPs to check, if empty no constraint is used (default: comes from master partition).",
"type": "string"
"check-status-code": {
"title": "HTTP Code Check",
"description": "Expected response HTTP Code.",
"type": "number",
"default": 200,
"minimum": 100,
"maximum": 599
},
"check-certificate-expiration-days": {
"default": "Master default",
"title": "Certificate expiration days check",
"description": "Amount of days to consider certificate as being to-be-expired (default: comes from master partition).",
"type": "string"
"title": "Certificate Expiration Check (days)",
"description": "Amount of days to consider certificate as being to-be-expired.",
"type": "number",
"default": 15,
"minimum": 1
},
"check-maximum-elapsed-time": {
"default": "Master default",
"title": "Maximum elapsed time for a site to reply (seconds)",
"description": "Maximum elapsed time for a site to reply to be considered good.(default: comes from master partition).",
"type": "string"
"title": "Maximum Elapsed Check (seconds)",
"description": "Maximum elapsed time for a site to reply to be considered good.",
"type": "number",
"default": 2,
"minimum": 1
},
"check-http-header-dict": {
"title": "HTTP Header Check",
"description": "JSON object of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}. Note: Shall be expressed directly as object, without any additional qouting.",
"type": "object",
"default": {}
},
"failure-amount": {
"default": "Master default",
"title": "Amount of failures to consider URL as in bad state",
"description": "Amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages (default: comes from master partition).",
"type": "string"
"title": "Failure Amount",
"description": "Amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages.",
"type": "number",
"default": 2,
"minimum": 1
},
"check-frontend-ip-list": {
"title": "Frontend IPs to check (backward compatibility)",
"description": "List of Frontend IPs to check, if empty no constraint is used. Defaults to region configuration. Note: Use region-dict's check-frontend-ip-list to ensure specific check on each region.",
"type": "array"
}
}
}
......@@ -9,39 +9,27 @@
{%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %}
{%- if extra_slave_instance_list %}
{#- Create slaves to process with setting up defaults #}
{%- for slave in json_module.loads(extra_slave_instance_list) | sort(attribute='slave_title') %}
{%- if 'check-status-code' not in slave %}
{%- do slave.__setitem__('check-status-code', CONFIGURATION['check-status-code']) %}
{%- endif %}
{%- if 'check-http-header-dict' not in slave %}
{%- do slave.__setitem__('check-http-header-dict', CONFIGURATION['check-http-header-dict']) %}
{%- endif %}
{%- if 'check-certificate-expiration-days' not in slave %}
{%- do slave.__setitem__('check-certificate-expiration-days', CONFIGURATION['check-certificate-expiration-days']) %}
{%- endif %}
{%- if 'failure-amount' not in slave %}
{%- do slave.__setitem__('failure-amount', CONFIGURATION['failure-amount']) %}
{%- endif %}
{%- if 'check-maximum-elapsed-time' not in slave %}
{%- do slave.__setitem__('check-maximum-elapsed-time', CONFIGURATION['check-maximum-elapsed-time']) %}
{%- endif %}
{%- if 'check-frontend-ip' not in slave %}
{%- do slave.__setitem__('check-frontend-ip', CONFIGURATION['check-frontend-ip']) %}
{%- endif %}
{%- if 'url' in slave %}
{%- set class = slave['check-maximum-elapsed-time'] %}
{%- if class not in slave_instance_dict %}
{%- do slave_instance_dict.__setitem__(class, []) %}
{%- for slave in extra_slave_instance_list | sort(attribute='-slave-title') %}
{%- do slave.setdefault('check-status-code', 200) %}
{%- do slave.setdefault('check-http-header-dict', {}) %}
{%- do slave.setdefault('check-certificate-expiration-days', 15) %}
{%- do slave.setdefault('failure-amount', 2) %}
{%- do slave.setdefault('check-maximum-elapsed-time', 2) %}
{%- do slave.setdefault('check-frontend-ip-list', CONFIGURATION['check-frontend-ip-list']) %}
{%- if 'url' in slave %}
{%- set class = slave['check-maximum-elapsed-time'] %}
{%- if class not in slave_instance_dict %}
{%- do slave_instance_dict.__setitem__(class, []) %}
{%- endif %}
{%- do slave_instance_dict[class].append(slave) %}
{%- endif %}
{%- do slave_instance_dict[class].append(slave) %}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- set part_list = [] %}
{%- for class, slave_instance_list in slave_instance_dict.items() %}
{#- class is used to separate surykatka with different timeouts #}
{%- for slave in slave_instance_list | sort(attribute='slave_title') %}
{%- set part_id = 'http-query-' ~ slave['slave_reference'] ~ '-promise' %}
{%- for slave in slave_instance_list | sort(attribute='-slave-title') %}
{%- set part_id = 'http-query-' ~ hashlib_module.md5(slave['-slave-reference'].encode('utf-8')).hexdigest() ~ '-promise' %}
{%- do part_list.append(part_id) %}
{%- set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
[{{part_id}}]
......@@ -51,11 +39,11 @@ name = {{ safe_name }}.py
config-report = http_query
config-url = {{ slave['url'] }}
config-status-code = {{ slave['check-status-code'] }}
config-http-header-dict = {{ slave['check-http-header-dict'] }}
config-http-header-dict = {{ json_module.dumps(slave['check-http-header-dict']) }}
config-certificate-expiration-days = {{ slave['check-certificate-expiration-days'] }}
config-failure-amount = {{ slave['failure-amount'] }}
config-maximum-elapsed-time = {{ slave['check-maximum-elapsed-time'] }}
config-ip-list = {{ slave['check-frontend-ip'] }}
config-ip-list = {{ ' '.join(slave['check-frontend-ip-list']) }}
config-json-file = ${surykatka-config-{{ class }}:json}
{%- endfor %}
......@@ -72,7 +60,7 @@ db = ${directory:srv}/surykatka-{{ class }}.db
rendered = ${directory:etc}/surykatka-{{ class }}.ini
template = {{ template_surykatka_ini }}
slave_instance_list = {{ dumps(slave_instance_list) }}
nameserver = {{ dumps(CONFIGURATION['nameserver']) }}
nameserver_list = {{ dumps(CONFIGURATION['nameserver-list']) }}
json = ${directory:srv}/surykatka-{{ class }}.json
{#- timeout is just a bit bigger than class time #}
timeout = {{ int(class) + 2 }}
......@@ -80,7 +68,7 @@ timeout = {{ int(class) + 2 }}
context =
import json_module json
key db :db
key nameserver :nameserver
key nameserver_list :nameserver_list
key slave_instance_list :slave_instance_list
key timeout :timeout
......@@ -110,9 +98,9 @@ cron-entries = ${directory:etc}/cron.d
name = surykatka-status-{{ class }}
frequency = */2 * * * *
command = ${surykatka-status-json-{{ class }}:rendered}
{%- do part_list.append('surykatka-' + class) %}
{%- do part_list.append('surykatka-bot-promise-' + class) %}
{%- do part_list.append('cron-entry-surykatka-status-' + class) %}
{%- do part_list.append('surykatka-%i'% (class,)) %}
{%- do part_list.append('surykatka-bot-promise-%i' % (class,)) %}
{%- do part_list.append('cron-entry-surykatka-status-%i' % (class,)) %}
{%- endfor %}
[buildout]
......@@ -163,4 +151,4 @@ key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[slap-parameter]
{%- endif %}
{%- endif %} {#- if slap_software_type == software_type #}
......@@ -16,34 +16,86 @@ extensions = jinja2.ext.do
extra-context =
section slave_information slap-configuration
{% set part_list = [] -%}
# Publish information for each slave
{% set part_list = [] -%}
{%- set edgebot_software_type = 'edgebot' %}
{%- set edgebot_quantity = slapparameter_dict.pop('edgebot-quantity', '1') | int %}
{%- set edgebot_list = [] %}
{%- set edgebot_section_list = [] %}
{%- set slave_list_name = 'extra_slave_instance_list' %}
{%- set request_dict = {} %}
{%- set namebase = "edgebot" %}
{%- set authorized_slave_list = [] %}
{%- set monitor_base_url_dict = {} -%}
{%- if 'region-dict' not in slapparameter_dict %}
{#- Be nice and allow to work with default configuration #}
{%- do slapparameter_dict.__setitem__('region-dict', {
'1': {
'sla-computer_guid': slap_configuration['computer'],
'state': slap_configuration['instance-state'],
'nameserver-list': slapparameter_dict.get('nameserver-list', []),
'check-frontend-ip-list': slapparameter_dict.get('check-frontend-ip-list', []),
}
}) %}
{%- endif %}
{%- set active_region_list = [] %}
{%- for region_name in sorted(slapparameter_dict['region-dict']) %}
{%- set region_parameter_dict = slapparameter_dict['region-dict'][region_name] %}
{%- if region_parameter_dict.get('state', 'started') == 'started' %}
{%- do active_region_list.append(region_name) %}
{%- endif %}
{%- endfor %}
{%- set authorized_slave_dict = {} %}
{%- set publish_slave_dict_dict = {} %}
{%- for slave in slave_instance_list | sort(attribute='slave_title') %}
{%- do authorized_slave_list.append(slave) %}
{%- set slave_reference = slave.pop('slave_reference') %}
{%- set publish_dict = {'assigned-region-dict': {}} %}
{%- if '_' in slave %}
{%- set base_slave_dict = json_module.loads(slave.pop('_')) %} {#- XXX: Unsafe! #}
{%- do base_slave_dict.__setitem__('-slave-title', slave['slave_title']) %}
{%- do base_slave_dict.__setitem__('-slave-reference', slave_reference) %}
{%- set slave_region_dict = base_slave_dict.pop('region-dict', {}) %}
{%- if slave_region_dict == {} %}
{%- for region in active_region_list %}
{%- do slave_region_dict.__setitem__(region, {}) %}
{%- endfor %}
{%- endif %}
{%- for region in slave_region_dict %}
{%- if region in active_region_list %}
{%- set region_info = {
'nameserver-list': slapparameter_dict['region-dict'][region].get('nameserver-list') or slapparameter_dict.get('slapparameter_dict') or [],
'check-frontend-ip-list': slave_region_dict[region].get('check-frontend-ip-list') or base_slave_dict.get('check-frontend-ip-list') or slapparameter_dict['region-dict'][region].get('check-frontend-ip-list') or slapparameter_dict.get('check-frontend-ip-list') or [],
} %}
{%- do publish_dict['assigned-region-dict'].__setitem__(region, region_info) %}
{%- set slave_dict = base_slave_dict.copy() %}
{%- do slave_dict.update(region_info) %}
{%- if region not in authorized_slave_dict %}
{%- do authorized_slave_dict.__setitem__(region, [slave_dict]) %}
{%- else %}
{%- do authorized_slave_dict[region].append(slave_dict) %}
{%- endif %}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- do publish_slave_dict_dict.__setitem__(slave_reference, publish_dict) %}
{%- endfor %}
{%- set monitor_base_port = int(slap_configuration['configuration.monitor-base-port']) %}
{%- for i in range(1, edgebot_quantity + 1) %}
{%- set edgebot_name = "%s-%s" % (namebase, i) %}
{%- set request_section_title = 'request-%s' % edgebot_name %}
{%- do edgebot_list.append(edgebot_name) %}
{%- do edgebot_section_list.append(request_section_title) %}
{%- set number = {'i': 1} %}
{%- for region_name in sorted(slapparameter_dict['region-dict']) %}
{%- set region_parameter_dict = slapparameter_dict['region-dict'][region_name] %}
{%- set edgebot_name = "%s-%s" % (namebase, region_name) %}
{%- set request_section_title = 'request-%s' % (hashlib_module.md5(edgebot_name.encode('utf-8')).hexdigest(),) %}
{%- do part_list.append(request_section_title) %}
{%- do request_dict.__setitem__(request_section_title,
{
'config': {'monitor-httpd-port': monitor_base_port + i},
'name': edgebot_name,
'sla': {},
'state': 'started',
}) %}
{#- Note: monitor-httpd-port will vary on regions being added and removed,
but this is accepted, as it's only internal trick #}
{%- do request_dict.__setitem__(
request_section_title,
{
'config': {
'monitor-httpd-port': monitor_base_port + number['i'],
'check-frontend-ip-list': region_parameter_dict.get('check-frontend-ip-list', []),
'nameserver-list': region_parameter_dict.get('nameserver-list', []),
'extra_slave_instance_list': authorized_slave_dict.get(region_name, [])
},
'name': edgebot_name,
'sla': {'computer_guid': region_parameter_dict['sla-computer_guid']},
'state': region_parameter_dict.get('state', 'started'),
}) %}
{%- do number.__setitem__('i', number['i'] + 1) %}
{%- endfor %}
[replicate]
......@@ -54,54 +106,58 @@ config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd}
software-url = ${slap-connection:software-release-url}
software-type = {{edgebot_software_type}}
return = monitor-base-url
{% for section, edgebot_request in request_dict.items() %}
{%- set monitor_base_url_dict = {} -%}
{% for section, edgebot_request in request_dict.items() %}
[{{section}}]
<= replicate
name = {{ edgebot_request.get('name') }}
{%- if edgebot_request.get('state') %}
state = {{ edgebot_request.get('state') }}
{%- endif%}
{%- set slave_configuration_dict = slapparameter_dict %}
{%- do slave_configuration_dict.update(edgebot_request.get('config')) %}
{%- do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list)) %}
{%- for config_key, config_value in slave_configuration_dict.items() %}
name = {{ edgebot_request['name'] }}
state = {{ edgebot_request['state'] }}
{%- if edgebot_request['state'] != 'destroyed' %}
{%- do monitor_base_url_dict.__setitem__(section, '${' ~ section ~ ':connection-monitor-base-url}') %}
return = monitor-base-url
{%- endif %}
{%- set edgebot_configuration_dict = edgebot_request['config'] %}
{%- for config_key, config_value in edgebot_configuration_dict.items() %}
config-{{ config_key }} = {{ dumps(config_value) }}
{% endfor -%}
{%- if edgebot_request.get('sla') %}
{%- for parameter, value in edgebot_request.get('sla').items() %}
{% endfor -%}
{%- for parameter, value in edgebot_request['sla'].items() %}
sla-{{ parameter }} = {{ value }}
{%- endfor %}
{%- else %}
# As no SLA was provided, by default it is requested on the same computer
sla-computer_guid = ${slap-connection:computer-id}
{% endif %}
{%- do monitor_base_url_dict.__setitem__(section, '${' ~ section ~ ':connection-monitor-base-url}') -%}
{%- endfor %}
{%- endfor %}
{%- set directory_list = [] -%}
{%- for slave_instance in slave_instance_list -%}
{%- set publish_section_title = 'publish-%s' % slave_instance.get('slave_reference') -%}
# Publish information for each slave
{%- for slave_reference, publish_dict in publish_slave_dict_dict.items() -%}
{%- set publish_section_title = 'publish-%s' % (hashlib_module.md5(slave_reference.encode('utf-8')).hexdigest(),) -%}
{%- do part_list.append(publish_section_title) %}
[{{ publish_section_title }}]
recipe = slapos.cookbook:publish
-slave-reference = {{ slave_instance.get('slave_reference') }}
{% endfor %}
recipe = slapos.cookbook:publish.serialised
available-region-list = {{ dumps(list(active_region_list)) }}
-slave-reference = {{ slave_reference }}
{%- for key, value in publish_dict.items() %}
{{ key }} = {{ dumps(value) }}
{%- endfor %}
{% endfor %}
[monitor-conf-parameters]
monitor-title = Monitor
password = ${monitor-htpasswd:passwd}
[monitor-base-url-dict]
{% for key, value in monitor_base_url_dict.items() -%}
{% for key, value in monitor_base_url_dict.items() -%}
{{ key }} = {{ value }}
{% endfor %}
{% endfor %}
[buildout]
extends = {{ instance_base_monitor }}
parts +=
slave-test-configuration
{% for part in part_list %}
{% for part in part_list %}
{{ ' %s' % part }}
{%- endfor %}
{%- endif %}
[publish-connection-information]
recipe = slapos.cookbook:publish.serialised
active-region-list = {{ dumps(list(active_region_list)) }}
sla-computer_guid = {{ dumps(slap_configuration['computer']) }}
sla-instance_guid = {{ dumps(slap_configuration['instance-guid']) }}
{%- endif %} {#- if slap_software_type == software_type #}
......@@ -32,6 +32,7 @@ template = ${template-monitor-edgetest:target}
rendered = $${buildout:directory}/template-monitor-base-edgetest.cfg
extensions = jinja2.ext.do
context = import json_module json
import hashlib_module hashlib
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
......@@ -53,6 +54,7 @@ surykatka-binary = ${buildout:bin-directory}/${surykatka:script-name}
template-surykatka-ini = ${template-surykatka-ini:target}
context = import json_module json
import hashlib_module hashlib
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
section slap_configuration slap-configuration
......@@ -75,13 +77,8 @@ url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
# Defaults
configuration.check-status-code = 200
configuration.check-http-header-dict = {}
configuration.nameserver =
configuration.check-frontend-ip =
configuration.check-certificate-expiration-days = 15
configuration.check-maximum-elapsed-time = 2
configuration.failure-amount = 2
configuration.nameserver-list =
configuration.check-frontend-ip-list =
# use monitor-base-port to have monitor listening on each instance
# on different port and also on different port than other services
# it makes it possible to instantiate it correctly on signle IP, for
......
......@@ -15,14 +15,17 @@
"description": "Cluster of bots to perform a distributed monitoring ",
"request": "instance-edgetest-input-schema.json",
"response": "instance-default-output-schema.json",
"serialisation": "json-in-xml",
"index": 1
},
"edgetest-slave": {
"title": "Edge Test Slave",
"shared": true,
"software-type": "edgetest",
"description": "Cluster of bots to perform a distributed monitoring ",
"request": "instance-edgetest-slave-input-schema.json",
"response": "instance-default-output-schema.json",
"serialisation": "json-in-xml",
"index": 2
}
}
......
......@@ -2,7 +2,6 @@
INTERVAL = 120
TIMEOUT = {{ timeout }}
SQLITE = {{ db }}
{%- set nameserver_list = nameserver.split() %}
{%- if len(nameserver_list) > 0 %}
NAMESERVER =
{%- for nameserver_entry in sorted(nameserver_list) %}
......@@ -10,7 +9,7 @@ NAMESERVER =
{%- endfor %}
{% endif %}
URL =
{%- for slave in slave_instance_list | sort(attribute='slave_title') %}
{%- for slave in slave_instance_list | sort(attribute='-slave-title') %}
{%- if 'url' in slave %}
{{ slave['url'] }}
{%- endif -%}
......
This diff is collapsed.
......@@ -18,7 +18,7 @@ md5sum = 2bd1779425b7561682c0de5496d808ed
[root-common]
filename = root-common.cfg.in
md5sum = 8e28f599247ad604ec6e32df410412a8
md5sum = c13b4f1a5aa526a8d3f8e02bf6baf785
[instance-neo-admin]
filename = instance-neo-admin.cfg.in
......
......@@ -47,7 +47,7 @@
"type": "object"
},
"sla-dict": {
"description": "[NEO SR only] Where to request instances. Each key is a query string for criterions (e.g. \"computer_guid=foo\"), and each value is a list of partition references ('node-0', 'node-1', ...). The prefix 'node-' is mandatory and the number must start from 0. The total number of nodes here must be equal to the length of node-list.",
"description": "[NEO SR only] Where to request instances. Each key is a query string for criterions (e.g. \"computer_guid=foo\"), and each value is a list of partition references ('node-0', 'node-1', ...). The prefix 'node-' is mandatory and the number must start from 0. The total number of nodes here must be equal to the length of node-list. A node can be removed by requesting it without any admin, master and storage.",
"additionalProperties": {
"type": "array",
"items": {
......@@ -95,7 +95,7 @@
"type": "integer"
},
"storage-count": {
"description": "Number of storage nodes to deploy. One master and one admin node is deployed with each storage.",
"description": "Number of storage nodes to deploy. One master and one admin node is deployed with each storage. 0 to disable.",
"default": 1,
"type": "integer"
},
......
......@@ -89,7 +89,7 @@ software-type = {{ software_type }}
config-autostart = {{ dumps(sum(storage_count)) }}
{%- do assert(replicas < len(node_list)) %}
{%- set monitor = set() %}
{%- for i, node in enumerate(node_list) %}
{%- for node in node_list %}
{%- set port = node.get('monitor') %}
{%- if port %}
{%- do monitor.add(node.get('admin') != 0 and port) %}
......@@ -101,13 +101,14 @@ config-autostart = {{ dumps(sum(storage_count)) }}
{%- endif %}
{%- for i, node in enumerate(node_list) %}
{%- set section_id = prefix ~ i %}
{%- do section_id_list.append(section_id) %}
{%- if node.get('master') == 0 and not node.get('monitor') %}
{%- do node.setdefault('admin', 0) %}
{%- endif %}
{%- if monitor or node.get('admin') == 0 %}
{%- do node.setdefault('monitor', 0) %}
{%- endif %}
{%- for x in 'admin', 'master', 'storage-count' if node.get(x, 1) %}
{%- do section_id_list.append(section_id) %}
[{{section_id}}]
<= {{ prefix }}request-common
......@@ -136,6 +137,9 @@ config-monitor = {{ dumps(parameter_dict.get('monitor', {})) }}
config-{{ k }} = {{ dumps(v) }}
{%- endfor %}
{{ sla(section_id) }}
{%- break %}
{%- endfor %}
{%- endfor %}
{%- do assert(len(monitor) == 1, monitor) %}
......
......@@ -31,3 +31,7 @@ if returncode:
subprocess.call(
('tar', '-caf', result('db.tar.xz'), 'neo_tests'),
cwd=tempfile.gettempdir())
p = subprocess.Popen(('journalctl', '-o', 'export'), stdout=subprocess.PIPE)
with open(result('journalctl-export.xz'), 'wb') as f:
subprocess.call(('xz',), stdin=p.stdout, stdout=f)
p.wait()
......@@ -15,7 +15,7 @@
[instance]
filename = instance.cfg.in
md5sum = f3aac995aa1129ca58f9a92851ad7091
md5sum = bfd488ba023f505be25d947ec830bab3
[yarn.lock]
filename = yarn.lock
......
......@@ -7,7 +7,7 @@ theia-environment-parts =
slapos-repository
runner-link
settings.json
request-script
request-script-template
theia-parts =
frontend-reload
......@@ -230,7 +230,7 @@ recipe = plone.recipe.command
filename = logo.png
full-path = $${directory:frontend-static}/$${:filename}
command =
cp -f ${logo.png:output} $${:full-path}
cp --remove-destination ${logo.png:output} $${:full-path}
stop-on-error = true
[frontend-instance-slapos.css]
......@@ -551,17 +551,19 @@ recipe = slapos.cookbook:symbolic.link
target-directory = $${directory:project}
link-binary = $${directory:runner}
[request-script]
[request-script-template]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:project}/$${:_buildout_section_name_}.sh
mode = 0700
template =
inline:#!/bin/sh
software_name=html5as-base #replace the software name
# This template is generated automatically, copy it in order to supply and request.
# Any manual change to this file may be lost.
software_name=html5as-base #replace the software name writen in ~/srv/project/slapos/software/
software_release_uri=~/srv/project/slapos/software/$software_name/software.cfg
# slapos supply is used to add the software to the software list to be supplied to a node.
slapos supply $software_release_uri slaprunner
# slapos request the allocation of an instance to the master.
# slapos request also gets status and parameters of the instance if it has any
# slapos request the allocation of an instance to the master.
# slapos request also gets status and parameters of the instance if it has any
# (slapos request is meant to be run multiple time until you get the status).
slapos request $software_name'_1' $software_release_uri
......@@ -183,7 +183,7 @@ class TestTheia(TheiaTestCase):
self.computer_partition_root_path,
'srv',
'project',
'request-script.sh',
'request-script-template.sh',
)
self.assertTrue(os.path.exists(script_path))
......
......@@ -631,9 +631,6 @@ PasteDeploy = 1.5.2
argparse = 1.4.0
zope.dottedname = 4.1.0
# test_UserManagerInterfaces in testERP5Security fails with 1.10.0.
Products.PluggableAuthService = 1.9.0
# modified version that works fine for buildout installation
SOAPpy = 0.12.0nxd001
......@@ -664,6 +661,7 @@ Products.GenericSetup = 1.8.6
Products.LongRequestLogger = 2.1.0
# Products.MimetypesRegistry 2.1 requires AccessControl>=3.0.0Acquisition.
Products.MimetypesRegistry = 2.0.10
Products.PluggableAuthService = 1.10.0
Products.PluginRegistry = 1.4
Products.TIDStorage = 5.5.0
pyPdf = 1.13
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment