Skip to content

slapos
slapos
Commit 075b0ba2 authored by Bryton Lacquement's avatar Bryton Lacquement 🚪
Browse files
Options

kvm/slaprunner: add support for Python 3 

TODO: rdiff-backup
parents ea1eda99 04c0d5f7
Show whitespace changes
Inline Side-by-side
Showing with 333 additions and 256 deletions
slapos/recipe/simplehttpserver/__init__.py
View file @ 075b0ba2
......@@ -27,6 +27,7 @@
from slapos.recipe.librecipe import GenericBaseRecipe
import string, random
import os
from six.moves import range
class Recipe(GenericBaseRecipe):
......@@ -35,7 +36,7 @@ class Recipe(GenericBaseRecipe):
base_path = options['base-path']
if options.get('use-hash-url', 'True') in ['true', 'True']:
pool = string.letters + string.digits
hash_string = ''.join(random.choice(pool) for i in xrange(64))
hash_string = ''.join(random.choice(pool) for i in range(64))
path = os.path.join(base_path, hash_string)
if os.path.exists(base_path):
......
slapos/recipe/simplehttpserver/simplehttpserver.py
View file @ 075b0ba2
# -*- coding: utf-8 -*-
from SimpleHTTPServer import SimpleHTTPRequestHandler
from BaseHTTPServer import HTTPServer
from six.moves.SimpleHTTPServer import SimpleHTTPRequestHandler
from six.moves.BaseHTTPServer import HTTPServer
import ssl
import os
import logging
......@@ -8,6 +8,9 @@ from netaddr import valid_ipv4, valid_ipv6
import socket
import cgi, errno
from slapos.util import str2bytes
class ServerHandler(SimpleHTTPRequestHandler):
document_path = ''
......@@ -22,7 +25,7 @@ class ServerHandler(SimpleHTTPRequestHandler):
if self.restrict_root_folder and self.path and self.path == '/':
# no access to root path
self.respond(403)
self.wfile.write("Forbidden")
self.wfile.write(b"Forbidden")
return True
return False
......@@ -46,11 +49,11 @@ class ServerHandler(SimpleHTTPRequestHandler):
name = form['path'].value
content = form['content'].value
method = 'a'
if form.has_key('clear') and form['clear'].value == '1':
if 'clear' in form and form['clear'].value == '1':
method = 'w'
self.writeFile(name, content, method)
self.respond(200, type=self.headers['Content-Type'])
self.wfile.write("Content written to %s" % name)
self.wfile.write(b"Content written to %s" % str2bytes(name))
def writeFile(self, filename, content, method='a'):
file_path = os.path.join(self.document_path, filename)
......@@ -97,7 +100,7 @@ def run(args):
httpd = server((host, port), Handler)
scheme = 'http'
if args.has_key('cert-file') and args.has_key('key-file') and \
if 'cert-file' in args and 'key-file' in args and \
os.path.exists(args['cert-file']) and os.path.exists(args['key-file']):
scheme = 'https'
httpd.socket = ssl.wrap_socket (httpd.socket,
......
software/kvm/buildout.hash.cfg
View file @ 075b0ba2
......@@ -15,15 +15,15 @@
[template]
filename = instance.cfg.in
md5sum = a236b719aaac61ac342ada0ce569151a
md5sum = c9d4356b5148ed8ff8c4f3da63c137ce
[template-kvm]
filename = instance-kvm.cfg.jinja2
md5sum = fd0b26e5ae200ce2f7ee2a169731998c
md5sum = 0db4ed796808b892a2e8a0aea704b13e
[template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in
md5sum = 2e743132ba4e001f784791311df9ba6a
md5sum = 6f4c60f4366728021a6e438ad3dc6956
[template-kvm-resilient]
filename = instance-kvm-resilient.cfg.jinja2
......@@ -31,7 +31,7 @@ md5sum = 7de5756f59ef7d823cd8ed33e6d15230
[template-kvm-import]
filename = instance-kvm-import.cfg.jinja2.in
md5sum = 97a8ff8a5891678274b14481dfc5214c
md5sum = bd7e5db872b0dbe7716ec49c3907c401
[template-kvm-import-script]
filename = template/kvm-import.sh.jinja2
......@@ -39,7 +39,7 @@ md5sum = cd0008f1689dfca9b77370bc4d275b70
[template-kvm-export]
filename = instance-kvm-export.cfg.jinja2
md5sum = 00ce5e6da3c833d9d9d1825311f11a81
md5sum = f12df4256eb5bd31a01c0ddc4b3897bb
[template-kvm-export-script]
filename = template/kvm-export.sh.jinja2
......@@ -47,7 +47,7 @@ md5sum = b617d64de73de1eed518185f310bbc82
[template-nbd]
filename = instance-nbd.cfg.jinja2
md5sum = 003112c44ef8d536861a46e3dfc67a37
md5sum = 6ea26f88252bf899c966d0f5675e7176
[template-ansible-promise]
filename = template/ansible-promise.in
......@@ -55,11 +55,11 @@ md5sum = 2036bf145f472f62ef8dee5e729328fd
[template-kvm-run]
filename = template/template-kvm-run.in
md5sum = 08af4ed0e2a53e76c844e3d7325aac09
md5sum = c319ca536b6bac5425245fae1684ca49
[template-kvm-controller]
filename = template/kvm-controller-run.in
md5sum = b96cba47c97f277c857176d69e086a12
md5sum = 3827b4f7624de190cf5f5d37e3b72e86
[template-apache-conf]
filename = template/apache.conf.in
......
software/kvm/instance-kvm-cluster.cfg.jinja2.in
View file @ 075b0ba2
......@@ -19,6 +19,14 @@ config-{{ name }} = {{ dumps(value) }}
{% endif -%}
{% endmacro -%}
[slap-network-information]
global-ipv6 = {{ ipv6 }}
[slap-parameter]
{% for k, v in slapparameter_dict.items() -%}
{{ k }} = {{ v }}
{% endfor -%}
[request-common]
recipe = slapos.cookbook:request
software-url = ${slap-connection:software-release-url}
......
software/kvm/instance-kvm-export.cfg.jinja2
View file @ 075b0ba2
......@@ -20,6 +20,11 @@ parts +=
# monitor parts
monitor-base
[slap-parameter]
{% for k, v in slapparameter_dict.items() -%}
{{ k }} = {{ v }}
{% endfor -%}
# Create the exporter executable, which is a simple shell script
[exporter]
recipe = slapos.recipe.template:jinja2
......
software/kvm/instance-kvm-import.cfg.jinja2.in
View file @ 075b0ba2
......@@ -12,6 +12,11 @@ parts +=
extends =
{{ pbsready_import_template }}
[slap-parameter]
{% for k, v in slapparameter_dict.items() -%}
{{ k }} = {{ v }}
{% endfor -%}
[resilient-publish-connection-parameter]
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
monitor-url = ${monitor-publish-parameters:monitor-url}
......
software/kvm/instance-kvm.cfg.jinja2
View file @ 075b0ba2
......@@ -31,6 +31,10 @@
{% do extends_list.append(template_monitor) -%}
{% do extends_list.append(logrotate_cfg) -%}
[slap-network-information]
local-ipv4 = {{ slap_configuration['ipv4-random'] }}
global-ipv6 = {{ slap_configuration['ipv6-random'] }}
[directory]
recipe = slapos.cookbook:mkdirectory
etc = ${buildout:directory}/etc
......@@ -454,38 +458,6 @@ interface-url = {{ slapparameter_dict.get('monitor-interface-url', 'https://moni
[helper]
blank-line =
[frontend-port-execute-base]
recipe = plone.recipe.command
command =
set -e
port=$(echo '${request-slave-frontend:connection-secure_access}' | cut -d ':' -f 3 | cut -d '/' -f 1)
[ -z $port ] && port=443
echo $port > ${:output}
update-command = ${:command}
stop-on-error = True
[frontend-port-execute]
<= frontend-port-execute-base
secure_access = ${request-slave-frontend:connection-secure_access}
output = ${directory:var}/frontend_port.txt
[frontend-port]
recipe = collective.recipe.shelloutput
filename = ${frontend-port-execute:output}
commands =
port = [ -f '${:filename}' ] && cat '${:filename}' || echo "NotReady"
{% if additional_frontend %}
[frontend-additional-port-execute]
<= frontend-port-execute-base
secure_access = ${request-slave-frontend-additional:connection-secure_access}
output = ${directory:var}/frontend_additional_port.txt
[frontend-additional-port]
<= frontend-port
filename = ${frontend-additional-port-execute:output}
{% endif %}
[publish-connection-information]
<= monitor-publish
recipe = slapos.cookbook:publish
......@@ -493,7 +465,7 @@ ipv6 = ${slap-network-information:global-ipv6}
backend-url = https://[${novnc-instance:ip}]:${novnc-instance:port}/vnc.html?auto=1&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
url = ${request-slave-frontend:connection-secure_access}/vnc.html?auto=1&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
{% if additional_frontend %}
url-additional = ${request-slave-frontend-additional:connection-secure_access}/vnc.html?host=${request-slave-frontend-additional:connection-domain}&port=${frontend-additional-port:port}&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
url-additional = ${request-slave-frontend-additional:connection-secure_access}/vnc.html?auto=1&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
{% endif %}
{% set disk_number = len(storage_dict) -%}
maximum-extra-disk-amount = {{ disk_number }}
......@@ -781,6 +753,10 @@ data-to-vm =
# Change keyboard layout language (Change to en-us if you face some bad bihaviors)
keyboard-layout-language = fr
{% for k, v in slapparameter_dict.items() -%}
{{ k }} = {{ v }}
{% endfor -%}
#############################
#
# Instanciate kvm (Buildout Section)
......
software/kvm/instance-nbd.cfg.jinja2
View file @ 075b0ba2
......@@ -9,9 +9,8 @@ parts =
onetimeupload-promise
publish-connection-information
extends = {{ template_monitor }}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
{% set ipv6 = slap_configuration['ipv6-random'] -%}
[rootdirectory]
recipe = slapos.cookbook:mkdirectory
......@@ -27,7 +26,7 @@ watched-services = ${rootdirectory:etc}/service
[nbd-instance]
recipe = slapos.cookbook:nbdserver
ip = ${slap-network-information:global-ipv6}
ip = {{ ipv6 }}
port = 1024
image-path = ${onetimeupload-instance:image-path}
qemu-path = {{ qemu_nbd_executable_location }}
......@@ -55,7 +54,7 @@ bytes = 24
[onetimeupload-instance]
recipe = slapos.cookbook:onetimeupload
ip = ${slap-network-information:global-ipv6}
ip = {{ ipv6 }}
port = {{ slapparameter_dict.get('otu-port', 8080) }}
image-path = ${rootdirectory:srv}/cdrom.iso
log-path = ${rootdirectory:log}/onetimeupload.log
......@@ -80,6 +79,10 @@ upload_key = ${onetimeupload-instance:key}
status_message = ${detect-if-cdrom-present:status}
[detect-if-cdrom-present]
recipe = collective.recipe.shelloutput
commands =
status = [ -f ${onetimeupload-instance:image-path} ] && echo "image already uploaded, you can't upload it again" || echo "WARNING: no image yet, the NBD server doesn't work"
recipe = slapos.recipe.build
init =
import os
options['status'] = (
"image already uploaded, you can't upload it again"
if os.path.isfile("${onetimeupload-instance:image-path}")
else "WARNING: no image yet, the NBD server doesn't work")
software/kvm/instance.cfg.in
View file @ 075b0ba2
......@@ -8,18 +8,21 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
extends = ${template-resilient-templates:output}
[switch_softwaretype]
recipe = slapos.cookbook:softwaretype
recipe = slapos.cookbook:switch-softwaretype
default = $${:kvm}
kvm-cluster = $${dynamic-template-kvm-cluster:rendered}
kvm = $${dynamic-template-kvm:rendered}
nbd = $${dynamic-template-nbd:rendered}
kvm-cluster = dynamic-template-kvm-cluster:rendered
kvm = dynamic-template-kvm:rendered
nbd = dynamic-template-nbd:rendered
kvm-resilient = $${dynamic-template-kvm-resilient:rendered}
kvm-import = $${dynamic-template-kvm-import:rendered}
kvm-export = $${dynamic-template-kvm-export:rendered}
kvm-resilient = dynamic-template-kvm-resilient:rendered
kvm-import = dynamic-template-kvm-import:rendered
kvm-export = dynamic-template-kvm-export:rendered
frozen = ${instance-frozen:output}
pull-backup = ${template-pull-backup:output}
frozen = instance-frozen:rendered
pull-backup = template-pull-backup:rendered
# BBB
RootSoftwareInstance = $${:default}
# XXX - If this configuration is not generated by slapgrid, use empty values
[storage-configuration]
......
software/kvm/software-py3.cfg 0 → 100644
View file @ 075b0ba2
[buildout]
extends =
../../component/python3/buildout.cfg
software.cfg
python = python3
# Ignore these for now
common-parts -=
rdiff-backup
software/kvm/software.cfg
View file @ 075b0ba2
......@@ -192,8 +192,7 @@ context =
raw template_apache_conf ${template-apache-conf:location}/${template-apache-conf:filename}
[versions]
# XXX - use websockify = 0.5.1 for compatibility with kvm frontend
websockify = 0.5.1
websockify = 0.9.0
collective.recipe.environment = 0.2.0
gitdb = 0.6.4
......
software/kvm/template/kvm-controller-run.in
View file @ 075b0ba2
......@@ -57,7 +57,7 @@ def update():
'nslot': 128,
'canreboot': 1
})
except Exception, e:
except Exception as e:
write(str(e))
raise
......
software/kvm/template/template-kvm-run.in
View file @ 075b0ba2
......@@ -6,7 +6,10 @@ import hashlib
import os
import socket
import subprocess
import urllib
try:
from urllib.request import FancyURLopener
except ImportError:
from urllib import FancyURLopener
import gzip
import shutil
from random import shuffle
......@@ -96,9 +99,9 @@ enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lowe
logfile = '{{ parameter_dict.get("log-file") }}'
if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false':
opener = urllib.FancyURLopener(context=ssl._create_unverified_context())
opener = FancyURLopener(context=ssl._create_unverified_context())
else:
opener = urllib.FancyURLopener({})
opener = FancyURLopener({})
def md5Checksum(file_path):
with open(file_path, 'rb') as fh:
......@@ -112,22 +115,16 @@ def md5Checksum(file_path):
def getSocketStatus(host, port):
s = None
for res in socket.getaddrinfo(host, port,
socket.AF_UNSPEC, socket.SOCK_STREAM):
af, socktype, proto, canonname, sa = res
for af, socktype, proto, canonname, sa in socket.getaddrinfo(
host, port, socket.AF_UNSPEC, socket.SOCK_STREAM):
try:
s = socket.socket(af, socktype, proto)
except socket.error, msg:
s = None
continue
try:
s.connect(sa)
except socket.error, msg:
return s
except socket.error:
if s:
s.close()
s = None
continue
break
return s
def getMapStorageList(disk_storage_dict, external_disk_number):
map_disk_file = os.path.join(etc_directory, '.data-disk-ids')
......@@ -319,7 +316,7 @@ for numa in numa_list:
kvm_argument_list += numa_parameter
if tap_network_parameter == [] and nat_network_parameter == []:
print 'Warning : No network interface defined.'
print('Warning : No network interface defined.')
else:
kvm_argument_list += nat_network_parameter + tap_network_parameter
if language in language_list:
......@@ -361,7 +358,7 @@ for nbd_ip, nbd_port in nbd_list:
s = getSocketStatus(nbd_ip, nbd_port)
if s is None:
# NBD is not available : launch kvm without it
print 'Warning : Nbd is not available.'
print('Warning : Nbd is not available.')
else:
# NBD is available
# We close the NBD socket else qemu won't be able to use it apparently
......@@ -375,5 +372,5 @@ else:
'-drive', 'file=%s,media=cdrom' % default_cdrom_iso
])
print 'Starting KVM: \n %s' % ' '.join(kvm_argument_list)
print('Starting KVM: \n %s' % ' '.join(kvm_argument_list))
os.execv(qemu_path, kvm_argument_list)
software/kvm/test/setup.py
View file @ 075b0ba2
......@@ -47,6 +47,7 @@ setup(name=name,
'erp5.util',
'supervisor',
'psutil',
'six',
],
zip_safe=True,
test_suite='test',
......
software/kvm/test/test.py
View file @ 075b0ba2
......@@ -25,13 +25,14 @@
#
##############################################################################
import httplib
import six.moves.http_client as httplib
import json
import os
import requests
import six
import slapos.util
import sqlite3
import urlparse
from six.moves.urllib.parse import parse_qs, urlparse
import unittest
from slapos.recipe.librecipe import generateHashFromFiles
......@@ -43,7 +44,8 @@ skipUnlessKvm = unittest.skipUnless(has_kvm, 'kvm not loaded or not allowed')
if has_kvm:
setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
os.path.join(os.path.dirname(__file__), '..',
'software%s.cfg' % ("-py3" if six.PY3 else ""))))
else:
setUpModule, InstanceTestCase = None, unittest.TestCase
......@@ -118,7 +120,7 @@ class MonitorAccessMixin(object):
monitor_setup_url = connection_parameter_dict['monitor-setup-url']
monitor_url_with_auth = 'https' + monitor_setup_url.split('https')[2]
auth = urlparse.parse_qs(urlparse.urlparse(monitor_url_with_auth).path)
auth = parse_qs(urlparse(monitor_url_with_auth).path)
# check that monitor-base-url for all partitions in the tree are accessible
# with published username and password
......@@ -128,7 +130,7 @@ class MonitorAccessMixin(object):
if not connection_xml:
continue
connection_dict = slapos.util.xml2dict(
partition_information['connection_xml'].encode('utf-8'))
connection_xml if six.PY3 else connection_xml.encode('utf-8'))
monitor_base_url = connection_dict.get('monitor-base-url')
if not monitor_base_url:
continue
......@@ -162,8 +164,8 @@ class TestAccessDefault(MonitorAccessMixin, InstanceTestCase):
httplib.OK,
result.status_code
)
self.assertTrue('<title>noVNC</title>' in result.text)
self.assertFalse('url-additional' in connection_parameter_dict)
self.assertIn('<title>noVNC</title>', result.text)
self.assertNotIn('url-additional', connection_parameter_dict)
@skipUnlessKvm
......@@ -186,7 +188,7 @@ class TestAccessDefaultAdditional(MonitorAccessMixin, InstanceTestCase):
httplib.OK,
result.status_code
)
self.assertTrue('<title>noVNC</title>' in result.text)
self.assertIn('<title>noVNC</title>', result.text)
result = requests.get(
connection_parameter_dict['url-additional'], verify=False)
......@@ -194,7 +196,7 @@ class TestAccessDefaultAdditional(MonitorAccessMixin, InstanceTestCase):
httplib.OK,
result.status_code
)
self.assertTrue('<title>noVNC</title>' in result.text)
self.assertIn('<title>noVNC</title>', result.text)
@skipUnlessKvm
......@@ -219,13 +221,13 @@ class TestAccessKvmCluster(MonitorAccessMixin, InstanceTestCase):
def test(self):
connection_parameter_dict = self.computer_partition\
.getConnectionParameterDict()
result = requests.get(connection_parameter_dict['kvm0-url'], verify=False)
result = requests.get(connection_parameter_dict['KVM0-url'], verify=False)
self.assertEqual(
httplib.OK,
result.status_code
)
self.assertTrue('<title>noVNC</title>' in result.text)
self.assertFalse('kvm0-url-additional' in connection_parameter_dict)
self.assertIn('<title>noVNC</title>', result.text)
self.assertNotIn('KVM0-url-additional', connection_parameter_dict)
@skipUnlessKvm
......@@ -253,20 +255,20 @@ class TestAccessKvmClusterAdditional(MonitorAccessMixin, InstanceTestCase):
def test(self):
connection_parameter_dict = self.computer_partition\
.getConnectionParameterDict()
result = requests.get(connection_parameter_dict['kvm0-url'], verify=False)
result = requests.get(connection_parameter_dict['KVM0-url'], verify=False)
self.assertEqual(
httplib.OK,
result.status_code
)
self.assertTrue('<title>noVNC</title>' in result.text)
self.assertIn('<title>noVNC</title>', result.text)
result = requests.get(
connection_parameter_dict['kvm0-url-additional'], verify=False)
connection_parameter_dict['KVM0-url-additional'], verify=False)
self.assertEqual(
httplib.OK,
result.status_code
)
self.assertTrue('<title>noVNC</title>' in result.text)
self.assertIn('<title>noVNC</title>', result.text)
@skipUnlessKvm
......
software/slapos-sr-testing/buildout.hash.cfg
View file @ 075b0ba2
......@@ -15,4 +15,4 @@
[template]
filename = instance.cfg
md5sum = 3ad1b06673000d9f424a1e7187c6a1fa
md5sum = b21b2a9ac7f027a044a897c6eacbba56
software/slapos-sr-testing/instance.cfg
View file @ 075b0ba2
......@@ -2,44 +2,41 @@
parts =
slapos-test-runner
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[download-source]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
git-executable = {{ git_location }}/bin/git
[slapos]
<= download-source
repository = ${slapos-repository:location}
repository = {{ slapos_location }}
[directory]
recipe = slapos.cookbook:mkdirectory
bin = $${buildout:directory}/bin
working-dir = $${buildout:directory}/tmp
[test-list]
path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup},${slapos.test.theia-setup:setup},${slapos.test.grafana-setup:setup},${slapos.test.gitlab-setup:setup}
bin = ${buildout:directory}/bin
working-dir = ${buildout:directory}/tmp
[slapos-test-runner]
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:bin}/runTestSuite
wrapper-path = ${directory:bin}/runTestSuite
command-line =
${buildout:bin-directory}/runTestSuite
--python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
--source_code_path_list=$${test-list:path_list}
{{ buildout['bin-directory'] }}/runTestSuite
--python_interpreter={{ buildout['bin-directory'] }}/{{ interpreter }}
--source_code_path_list={{ ','.join(tests.splitlines()) }}
environment =
PATH=${buildout:bin-directory}:${quic_client-bin:location}:${curl:location}/bin/:/usr/bin/:/bin
SLAPOS_TEST_IPV4=$${slap-configuration:ipv4-random}
SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random}
SLAPOS_TEST_WORKING_DIR=$${directory:working-dir}
PATH={{ buildout['bin-directory'] }}:{{ quic_client_location }}:{{ curl_location }}/bin/:/usr/bin/:/bin
SLAPOS_TEST_IPV4=${slap-configuration:ipv4-random}
SLAPOS_TEST_IPV6=${slap-configuration:ipv6-random}
SLAPOS_TEST_WORKING_DIR=${directory:working-dir}
software/slapos-sr-testing/software-py3.cfg 0 → 100644
View file @ 075b0ba2
[buildout]
extends =
../../component/python3/buildout.cfg
software.cfg
python = python3
[eggs]
eggs -=
# plantuml is not Py3-compatible
${slapos.test.plantuml-setup:egg}
[template]
extra =
software/slapos-sr-testing/software.cfg
View file @ 075b0ba2
......@@ -176,10 +176,40 @@ repository = https://lab.nexedi.com/nexedi/slapos.git
branch = master
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template.cfg
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:filename}
rendered = ${buildout:directory}/template.cfg
mode = 640
context =
section buildout buildout
key git_location git:location
key slapos_location slapos-repository:location
key interpreter eggs:interpreter
key quic_client_location quic_client-bin:location
key curl_location curl:location
key tests :tests
tests =
${slapos.test.kvm-setup:setup}
${slapos.test.slaprunner-setup:setup}
${:extra}
extra =
${slapos.cookbook-setup:setup}
${slapos.test.caddy-frontend-setup:setup}
${slapos.test.erp5-setup:setup}
${slapos.test.slapos-master-setup:setup}
${slapos.test.monitor-setup:setup}
${slapos.test.plantuml-setup:setup}
${slapos.test.powerdns-setup:setup}
${slapos.test.proftpd-setup:setup}
${slapos.test.re6stnet-setup:setup}
${slapos.test.seleniumserver-setup:setup}
${slapos.test.helloworld-setup:setup}
${slapos.test.jupyter-setup:setup}
${slapos.test.nextcloud-setup:setup}
${slapos.test.turnserver-setup:setup}
${slapos.test.theia-setup:setup}
${slapos.test.grafana-setup:setup}
${slapos.test.gitlab-setup:setup}
[versions]
# slapos.core is used from the clone always
......
software/slaprunner/buildout.hash.cfg
View file @ 075b0ba2
......@@ -14,11 +14,11 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = 8b78e32b877d591400746ec7fd68ed4c
md5sum = ec70348dd71b319590a5c5837f3d2e45
[template-runner]
filename = instance-runner.cfg
md5sum = 84b81c18c77af08d7e97b0b6da4ee105
md5sum = b20894f378530e79c847ddfb61782cc5
[template-runner-import-script]
filename = template/runner-import.sh.jinja2
......@@ -34,7 +34,7 @@ md5sum = b992bb3391de9d6d422bfa8011d8ffc4
[template-resilient]
filename = instance-resilient.cfg.jinja2
md5sum = 2271c829b94542b7b2d9c589376ae538
md5sum = 105ed7f54f251c64a2d34559360a5604
[template_nginx_conf]
filename = nginx_conf.in
......@@ -62,7 +62,7 @@ md5sum = 7645048216fcf957f7773534cd0408dc
[template-supervisord]
filename = template/supervisord.conf.in
md5sum = d294d0dafd265048399de6da8c96345f
md5sum = 28f69b57c2835bddfcfbe312563eef51
[template-listener-slapgrid]
filename = template/listener_slapgrid.py.in
......
software/slaprunner/instance-resilient.cfg.jinja2
View file @ 075b0ba2
......@@ -19,7 +19,7 @@
{% set monitor_interface_url = slapparameter_dict.pop('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
{% import 'parts' as parts %}
{% import 'replicated' as replicated %}
{% import 'replicated' as replicated with context %}
[buildout]
eggs-directory = {{ eggs_directory }}
......
software/slaprunner/instance-runner.cfg
View file @ 075b0ba2
......@@ -51,9 +51,9 @@ extends =
${monitor2-template:rendered}
${template-logrotate-base:rendered}
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[slap-network-information]
local-ipv4 = $${slap-configuration:ipv4-random}
global-ipv6 = $${slap-configuration:ipv6-random}
{% if slapparameter_dict.get('custom-frontend-backend-url') -%}
[request-custom-frontend]
......@@ -287,35 +287,29 @@ extra-args=-t rsa
<=runner-sshd-ssh-keygen-base
extra-args=-t ecdsa -b 521
[runner-sshd-publickey-fingerprint-shelloutput]
recipe = collective.recipe.shelloutput
# XXX because collective.recipe.shelloutput ignore errors, we run the same
# command in a plone.recipe.command so that if fails if something goes wrong.
commands =
fingerprint = bash -o pipefail -c "${openssh-output:keygen} -lf $${runner-sshd-ssh-host-ecdsa-key:output} | cut -f 2 -d\ | sed 's/+/%2B/g' | sed 's/\//%2F/g' | sed 's/SHA256://'"
[runner-sshd-publickey-fingerprint]
# fingerprint for ssh url, see
# https://tools.ietf.org/id/draft-salowey-secsh-uri-00.html#connparam
# https://winscp.net/eng/docs/session_url#hostkey
_fingerprint = $${runner-sshd-publickey-fingerprint-shelloutput:fingerprint}
# format is host-key-alg-fingerprint, but we know that
# $${runner-sshkeys-sshd:public-key} is rsa so for host-key-alg
# we just use use rsa.
fingerprint = ssh-rsa-$${:_fingerprint}
# XXX because collective.recipe.shelloutput ignore errors and capture output
# "Error ...", we use a plone.recipe.command to check that this command did
# not fail.
# This command will always fail on first buildout run, because
# collective.recipe.shelloutput is evaluated at buildout recipes __init__ step,
# but the key file is created later at install step.
recipe = plone.recipe.command
stop-on-error = true
command = echo "$${:_fingerprint}" | ( grep ^Error || exit 0 && exit 1 )
recipe = slapos.recipe.build
init =
import os
import subprocess
from six.moves.urllib.parse import quote
keyfile = self.buildout['runner-sshd-ssh-host-ecdsa-key']['output']
if os.path.isfile(keyfile):
x = subprocess.check_output(('${openssh-output:keygen}', '-lf', keyfile))
x = x.split()[1]
assert x.startswith(b'SHA256:'), x
# format is host-key-alg-fingerprint, but we know that
# $${runner-sshd-ssh-host-ecdsa-key:output} is rsa so for host-key-alg
# we just use use rsa.
options['fingerprint'] = "ssh-rsa-" + quote(x[7:], safe='')
else:
# This command will always fail on first buildout run, because it is
# evaluated at buildout recipes __init__ step, but the key file is created
# later at install step.
options['fingerprint'] = "NotReady"
#---------------------------
#--
......@@ -665,6 +659,9 @@ monitor-cors-domains =
monitor-interface-url =
monitor-httpd-port = 8386
buildout-shared-folder = $${runnerdirectory:home}/shared
{% for k, v in slapparameter_dict.items() -%}
{{ k }} = {{ v }}
{% endfor -%}
[slapos-cfg]
recipe = slapos.recipe.template:jinja2
......@@ -745,15 +742,6 @@ recipe = plone.recipe.command
stop-on-error = true
command = SR=$${slap-parameter:slapos-software} && if [ -n "$SR" ] && [ ! -f "$${directory:etc}/.project" ]; then echo workspace/slapos/$${slap-parameter:slapos-software}/ > $${directory:etc}/.project; fi
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[minishell-cwd]
recipe = plone.recipe.command
command = if [ ! -f $${slaprunner:minishell_cwd_file} ]; then echo $${runnerdirectory:home} > $${slaprunner:minishell_cwd_file}; fi
......@@ -820,7 +808,6 @@ template = ${template-supervisord:location}/${template-supervisord:filename}
rendered = $${directory:etc}/supervisord.conf
context =
import multiprocessing multiprocessing
import builtin __builtin__
section supervisord supervisord
key slapparameter_dict slap-configuration:configuration
key listener_slapgrid listener-slapgrid-bin:rendered
......
software/slaprunner/instance.cfg
View file @ 075b0ba2
......@@ -9,15 +9,18 @@ extends =
${template-resilient-templates:output}
[switch_softwaretype]
recipe = slapos.cookbook:softwaretype
default = $${instance-base-runner:rendered}
resilient = $${instance-resilient:rendered}
runner = $${instance-base-runner:rendered}
runner-import = $${template-runner-import:rendered}
runner-export = $${template-runner-export:rendered}
recipe = slapos.cookbook:switch-softwaretype
default = $${:runner}
resilient = instance-resilient:rendered
runner = instance-base-runner:rendered
runner-import = template-runner-import:rendered
runner-export = template-runner-export:rendered
frozen = ${instance-frozen:output}
pull-backup = ${template-pull-backup:output}
frozen = instance-frozen:rendered
pull-backup = template-pull-backup:rendered
# BBB
RootSoftwareInstance = $${:default}
[instance-base-runner]
recipe = slapos.recipe.template:jinja2
......
software/slaprunner/software-py3.cfg 0 → 100644
View file @ 075b0ba2
[buildout]
extends =
../../component/python3/buildout.cfg
software.cfg
python = python3
# Ignore these for now
common-parts -=
rdiff-backup
[eggs]
eggs -=
# futures is a backport of Py3's concurrent.futures module
futures
software/slaprunner/software.cfg
View file @ 075b0ba2
......@@ -150,10 +150,10 @@ eggs =
erp5.util
lock-file
plone.recipe.command
collective.recipe.shelloutput
slapos.recipe.build
slapos.toolbox[flask_auth]
gunicorn==19.7.1
gunicorn
# for gunicorn[gthread]
futures
${slapos-cookbook:eggs}
slapos.core # listed explicitly for scripts generation
......@@ -169,7 +169,7 @@ Flask-Auth = 0.85
cns.recipe.symlink = 0.2.3
futures = 3.0.5
gitdb = 0.6.4
gunicorn = 19.7.1
gunicorn = 19.10.0
prettytable = 0.7.2
pycurl = 7.43.0
slapos.recipe.template = 4.4
......
software/slaprunner/template/supervisord.conf.in
View file @ 075b0ba2
......@@ -27,7 +27,7 @@ stdout_logfile = {{ supervisord['no_logfile'] }}
stderr_logfile = {{ supervisord['no_logfile'] }}
directory = {{ supervisord['directory'] }}
{# how many parallel build jobs to spawn when compiling software -#}
{% set njobs = builtin.max(1, (multiprocessing.cpu_count() // builtin.int(slapparameter_dict.get('cpu-usage-ratio', 4)))) -%}
{% set njobs = max(1, (multiprocessing.cpu_count() // int(slapparameter_dict.get('cpu-usage-ratio', 4)))) -%}
environment = PATH="{{- supervisord['path'] -}}",MAKEFLAGS="-j{{ njobs }}",NPY_NUM_BUILD_JOBS="{{ njobs }}",BUNDLE_JOBS="{{ njobs }}"
[program:{{- supervisord['slapgrid-cp'] -}}]
......
software/slaprunner/test/test.py
View file @ 075b0ba2
......@@ -38,13 +38,16 @@ from six.moves.urllib.parse import quote
from six.moves.urllib.parse import urljoin
from six.moves.configparser import ConfigParser
import requests
import six
from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.util import bytes2str
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
os.path.join(os.path.dirname(__file__), '..',
'software%s.cfg' % ("-py3" if six.PY3 else ""))))
class SlaprunnerTestCase(SlapOSInstanceTestCase):
......@@ -185,7 +188,7 @@ class TestSSH(SlaprunnerTestCase):
channel.settimeout(30)
received = ''
while True:
r = channel.recv(1024)
r = bytes2str(channel.recv(1024))
self.logger.debug("received >%s<", r)
if not r:
break
......@@ -197,7 +200,7 @@ class TestSSH(SlaprunnerTestCase):
# simple commands can also be executed ( this would be like `ssh bash -c 'pwd'` )
self.assertEqual(
self.computer_partition_root_path,
client.exec_command("pwd")[1].read(1000).strip())
bytes2str(client.exec_command("pwd")[1].read(1000)).strip())
class TestSlapOS(SlaprunnerTestCase):
......@@ -211,7 +214,7 @@ class TestSlapOS(SlaprunnerTestCase):
'show',
),
env={})
self.assertIn('slaprunner', proxy_show_output)
self.assertIn(b'slaprunner', proxy_show_output)
def test_shared_part_list(self):
# this slapos used shared_part_list
......
stack/resilient/buildout.hash.cfg
View file @ 075b0ba2
......@@ -14,7 +14,7 @@
# not need these here).
[pbsready]
filename = pbsready.cfg.in
md5sum = 5e0dcd4c290f0b46cb2d316dc1c9c011
md5sum = 66331047b7dbf2513c5726d5d1647320
[pbsready-import]
filename = pbsready-import.cfg.in
......@@ -22,15 +22,15 @@ md5sum = d813c43ed00eff868fb13bc75b045336
[pbsready-export]
filename = pbsready-export.cfg.in
md5sum = 2e804e06b5203c3f127c31a1704c48bd
md5sum = 2b0c71b085cfe8017f28098c160b1f49
[template-pull-backup]
filename = instance-pull-backup.cfg.in
md5sum = 0bbe16f3d805afd880a251a4f40ecaf1
md5sum = 555d528b198564f0ce1e94db1160ebf3
[template-replicated]
filename = template-replicated.cfg.in
md5sum = 290b380fe3da8736642bc10a8b1163d1
md5sum = 815fd8f7c42b9cf59b286b0fe77fa76d
[template-parts]
filename = template-parts.cfg.in
......@@ -38,7 +38,7 @@ md5sum = 071b1034ee8f5cc14f79b16fdeba2813
[template-resilient-templates]
filename = template-resilient-templates.cfg.in
md5sum = 41e82859dc6b65e94a300a006d51536e
md5sum = 097a14371efde11465ab4bd08ef3131b
[instance-frozen]
filename = instance-frozen.cfg.in
......@@ -46,7 +46,7 @@ md5sum = d21472f0e58f928fb827f2cbf22c4d4a
[resilient-web-takeover-cgi-script-download]
filename = resilient-web-takeover-cgi-script.py.in
md5sum = 60d4912fdf5e8dafaba9d9f333aa9e36
md5sum = 675ac9e1cf49ccc8f8eddb541a62d899
[template-wrapper]
filename = templates/wrapper.in
......@@ -54,9 +54,9 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
[notifier-feed-promise-template]
filename = templates/notifier-feed-promise.py.in
md5sum = d75346911dbc4cfcdb39a21e56cd5016
md5sum = fa6521daaa02fef4dd2ce06d29ef90be
[template-monitor-check-resilient-feed]
filename = templates/monitor-check-resilient-feed.in
md5sum = 19ee9055de961acf402e2dfe5b9581d2
md5sum = af9787f8440fef19924b2e765372b20f
stack/resilient/instance-pull-backup.cfg.in
View file @ 075b0ba2
......@@ -19,10 +19,6 @@ parts =
extends =
${monitor2-template:rendered}
${template-logrotate-base:rendered}
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
#----------------
#--
......@@ -90,7 +86,7 @@ rendered = $${:wrapper}
wrapper = $${basedirectory:services}/notifier
mode = 0700
command = ${buildout:bin-directory}/pubsubserver --callbacks $${directory:notifier-callbacks} --feeds $${directory:notifier-feeds} --equeue-socket $${equeue:socket} --logfile $${basedirectory:log}/notifier.log $${:host} $${:port}
host = $${slap-network-information:global-ipv6}
host = {{ ipv6 }}
port = $${notifier-port:port}
context =
key content notifier:command
......@@ -128,8 +124,8 @@ run-directory = $${basedirectory:run}
pull-push-maximum-run = 5
# XXX: this should be named "notifier-host"
notifier-url = http://[$${notifier:host}]:$${notifier:port}
slave-instance-list = $${slap-parameter:slave_instance_list}
ignore-known-hosts-file = $${slap-parameter:ignore-known-hosts-file}
slave-instance-list = {{ slapparameter_dict.get('slave_instance_list', '[]') }}
ignore-known-hosts-file = {{ slapparameter_dict.get('ignore-known-hosts-file', 'false') }}
# To get a verbose feed about PBS state
instance-root-name = $${monitor-instance-parameter:root-instance-title}
log-url = $${monitor-publish-parameters:monitor-base-url}/private/notifier/
......@@ -194,20 +190,6 @@ identity-file = $${:home}/id_rsa
command-line = ${openssh:location}/bin/ssh -T -o "UserKnownHostsFile $${pbs:known-hosts}" -i $${:identity-file}
wrapper-path = $${rootdirectory:bin}/ssh
#----------------
#--
#-- Slave instance list (empty default).
[slap-parameter]
slave_instance_list = []
ignore-known-hosts-file = false
monitor-cors-domains =
monitor-httpd-port = 8070
monitor-title = PBS Instance
monitor-password = $${monitor-htpasswd:passwd}
monitor-username = admin
#----------------
#--
#-- Resiliency promises.
......@@ -269,11 +251,11 @@ monitor-password = $${monitor-publish-parameters:monitor-password}
#-- Monitor
[monitor-instance-parameter]
monitor-httpd-port = $${slap-parameter:monitor-httpd-port}
monitor-title = $${slap-parameter:monitor-title}
cors-domains = $${slap-parameter:monitor-cors-domains}
username = $${slap-parameter:monitor-username}
password = $${slap-parameter:monitor-password}
monitor-httpd-port = {{ slapparameter_dict.get('monitor-httpd-port', 8070) }}
monitor-title = {{ slapparameter_dict.get('monitor-title', 'PBS Instance') }}
cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', '') }}
username = {{ slapparameter_dict.get('monitor-username', 'admin') }}
password = {{ slapparameter_dict.get('monitor-password', '$${monitor-htpasswd:passwd}') }}
[monitor-conf-parameters]
private-path-list +=
......
stack/resilient/pbsready-export.cfg.in
View file @ 075b0ba2
......@@ -39,7 +39,7 @@ name = exporter
title = Dumping ${slap-parameter:namebase}
executable = ${exporter:wrapper}
wrapper = ${rootdirectory:bin}/exporter
notify = ${slap-parameter:notify}
notify = {{ slapparameter_dict.get('notify', '') }}
pidfile = ${resilient-directory:pid}/${:name}.pid
max-run = 3
......@@ -85,8 +85,3 @@ recipe = slapos.cookbook:random.time
recipe = slapos.cookbook:publish-early
-init =
resiliency-backup-periodicity gen-resiliency-backup-periodicity:time
[slap-parameter]
# In cron.d format (i.e things like */15 * * * * are accepted).
resiliency-backup-periodicity =
notify =
\ No newline at end of file
stack/resilient/pbsready.cfg.in
View file @ 075b0ba2
......@@ -22,6 +22,10 @@ parts +=
extends =
${monitor2-template:rendered}
[slap-network-information]
local-ipv4 = $${slap-configuration:ipv4-random}
global-ipv6 = $${slap-configuration:ipv6-random}
#----------------
#--
#-- Creation of all needed directories.
......
stack/resilient/resilient-web-takeover-cgi-script.py.in
View file @ 075b0ba2
#!${buildout:executable}
from __future__ import print_function
equeue_database = '${equeue:database}'
equeue_lockfile = '${equeue:lockfile}'
takeover_script = '${resiliency-takeover-script:wrapper-takeover}'
......@@ -8,7 +9,11 @@ import atexit
import cgi
import cgitb
import datetime
import gdbm
try:
import dbm.gnu as gdbm
except ImportError:
import gdbm
import os
import shutil
import subprocess
......@@ -39,11 +44,12 @@ def getLatestBackupDate():
# Usually, there is only one callback (so only one key
# in the db), but if there are several:
# Take the "oldest" one (oldest value).
if not db.keys():
db_keys = db.keys()
if not db_keys:
result = False
else:
last_backup = db[db.keys()[-1]]
for callback in db.keys():
last_backup = db[db_keys[-1]]
for callback in db_keys:
timestamp = float(db[callback])
if timestamp < last_backup:
last_backup = timestamp
......@@ -79,12 +85,11 @@ if latest_backup_date == False:
else:
latest_backup_message = latest_backup_date.strftime('%Y-%m-%d %H:%M:%S')
print "Content-Type: text/html"
print
print("Content-Type: text/html\n")
form = cgi.FieldStorage()
if "password" not in form:
print """<html>
print("""<html>
<body>
<h1>This is takeover web interface.</h1>
<p>Calling takeover will stop and freeze the current main instance, and make this clone instance the new main instance, replacing the old one.</p>
......@@ -100,15 +105,15 @@ if "password" not in form:
<input type="submit" value="Take over" style="background: red;">
</form>
</body>
</html>""" % (latest_backup_message, isBackupInProgress(), getSoftwareReleaseInformationFormatted())
</html>""" % (latest_backup_message, isBackupInProgress(), getSoftwareReleaseInformationFormatted()))
sys.exit(0)
if form['password'].value != '${:password}':
print "<H1>Error</H1>"
print "Password is invalid."
print("<H1>Error</H1>")
print("Password is invalid.")
sys.exit(1)
# XXX hardcoded location
result = subprocess.check_output([takeover_script], stderr=subprocess.STDOUT)
print 'Success.'
print '<pre>%s</pre>' % result
print('Success.')
print('<pre>%s</pre>' % result)
stack/resilient/template-replicated.cfg.in
View file @ 075b0ba2
......@@ -41,7 +41,7 @@ config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}
config-name = {{namebase}}0
# Bubble up all the instance parameters to the requested export instance.
{% if slapparameter_dict is defined %}
{% for parameter_name, parameter_value in slapparameter_dict.items() %}
{% for parameter_name, parameter_value in six.iteritems(slapparameter_dict) %}
{% if parameter_value is string %}
config-{{parameter_name}} = {{ parameter_value.split('\n') | join('\n ') }}
{% else %}
......@@ -49,7 +49,7 @@ config-{{parameter_name}} = {{ parameter_value }}
{% endif %}
{% endfor %}
{% endif %}
{% for key, value in monitor_dict.iteritems() -%}
{% for key, value in six.iteritems(monitor_dict) -%}
config-{{ key }} = {{ value }}
{% endfor -%}
{% if sla_parameter_dict == {} -%}
......@@ -67,7 +67,7 @@ sla-mode = unique_by_network
{% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
{% endif -%}
{% endfor -%}
{% for key, value in sla_dict.iteritems() -%}
{% for key, value in six.iteritems(sla_dict) -%}
sla-{{ key }} = {{ value }}
{% endfor -%}
{% endif -%}
......@@ -101,7 +101,7 @@ config-number = {{id}}
config-name = {{namebase}}{{id}}
config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
{% for key, value in monitor_dict.iteritems() -%}
{% for key, value in six.iteritems(monitor_dict) -%}
config-{{ key }} = {{ value }}
{% endfor -%}
{% if sla_parameter_dict == {} -%}
......@@ -119,7 +119,7 @@ sla-mode = unique_by_network
{% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
{% endif -%}
{% endfor -%}
{% for key, value in sla_dict.iteritems() -%}
{% for key, value in six.iteritems(sla_dict) -%}
sla-{{ key }} = {{ value }}
{% endfor -%}
{% endif %}
......@@ -202,7 +202,7 @@ software-type = pull-backup
name = PBS ({{namebase}} / {{id}})
config-ignore-known-hosts-file = ${slap-parameter:ignore-known-hosts-file}
config-monitor-title = PBS ${slap-connection:computer-id}-{{namebase}}-{{id}}
{% for key, value in monitor_dict.iteritems() -%}
{% for key, value in six.iteritems(monitor_dict) -%}
config-{{ key }} = {{ value }}
{% endfor -%}
return = ssh-key notification-url feeds-url {{ monitor_return }}
......@@ -222,7 +222,7 @@ sla-mode = unique_by_network
{% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
{% endif -%}
{% endfor -%}
{% for key, value in sla_dict.iteritems() -%}
{% for key, value in six.iteritems(sla_dict) -%}
sla-{{ key }} = {{ value }}
{% endfor %}
{% endif %}
......
stack/resilient/template-resilient-templates.cfg.in
View file @ 075b0ba2
......@@ -8,3 +8,20 @@ context =
raw pbsready_template_path ${pbsready:output}
raw bash_executable_location ${bash:location}/bin/bash
raw logrotate_executable_location ${logrotate:location}/usr/sbin/logrotate
[instance-frozen]
recipe = slapos.recipe.template:jinja2
template = ${instance-frozen:output}
rendered = $${buildout:directory}/template-frozen.cfg
extensions = jinja2.ext.do
context =
key slapparameter_dict slap-configuration:configuration
[template-pull-backup]
recipe = slapos.recipe.template:jinja2
template = ${template-pull-backup:output}
rendered = $${buildout:directory}/template-pull-backup.cfg
extensions = jinja2.ext.do
context =
key slapparameter_dict slap-configuration:configuration
key ipv6 slap-configuration:ipv6-random
stack/resilient/templates/monitor-check-resilient-feed.in
View file @ 075b0ba2
#!{{ python_executable }}
from __future__ import print_function
import os
import urllib2
try:
from urllib2 import HTTPError, urlopen
except ImportError:
from urllib.error import HTTPError
from urllib.request import urlopen
import sys
input_feed_directory = '{{ input_feed_directory }}'
......@@ -12,12 +17,14 @@ feed_file_list = os.listdir(input_feed_directory)
rss_ok = True
for feed_file_name in feed_file_list:
print "Getting %s" % feed_file_name
print("Getting", feed_file_name)
url = base_url + feed_file_name
try:
feed = urllib2.urlopen(url)
feed = urlopen(url)
body = feed.read()
open(os.path.join(monitor_feed_directory, feed_file_name + '.rss'), 'w').write(body)
print "FEED is ok"
except urllib2.HTTPError as e:
with open(os.path.join(monitor_feed_directory, feed_file_name + '.rss'),
'wb') as f:
f.write(body)
print("FEED is ok")
except HTTPError as e:
sys.exit("%s is unvailable: %s" % (feed_file_name, e))
stack/resilient/templates/notifier-feed-promise.py.in
View file @ 075b0ba2
......@@ -2,7 +2,11 @@
import csv
import os
import sys
import urllib2
try:
from urllib2 import HTTPError, urlopen
except ImportError:
from urllib.error import HTTPError
from urllib.request import urlopen
csv.field_size_limit(sys.maxsize)
......@@ -15,9 +19,9 @@ for feed_file_name in feed_file_list:
url = base_url + feed_file_name
# Try feed consistency
try:
feed = urllib2.urlopen(url)
feed = urlopen(url)
body = feed.read()
except urllib2.HTTPError as e:
except HTTPError as e:
sys.exit("%s is unavailable: %s" % (feed_file_name, e))
with open(os.path.join(notifier_feed_directory, feed_file_name)) as feed_file:
reader = csv.reader(feed_file)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7