Skip to content

slapos
slapos
Commit cf4b2a73 authored by Julien Muchembled's avatar Julien Muchembled
Browse files
Options

Merge branch 'master' into erp5-cluster

parents d7562ad1 cdb2701f
Hide whitespace changes
Inline Side-by-side
Showing with 561 additions and 140 deletions
CHANGES.txt
View file @ cf4b2a73
Changes
=======
0.81 (2013-08-12)
-----------------
* KVM SR: implement resiliency test. [Cedric de Saint Martin]
0.80 (2013-08-06)
----------------
......
component/coreutils/buildout.cfg
View file @ cf4b2a73
......@@ -19,7 +19,11 @@ environment =
recipe = plone.recipe.command
stop-on-error = true
update-command = ${:command}
command = ${:test} -x ${:test} -a -x ${:cat} -a -x ${:rm}
command = ${:test} -x ${:test} -a -x ${:cat} -a -x ${:rm} -a -x ${:echo} -a -x ${:date} -a -x ${:md5sum} -a -x ${:basename}
test = ${coreutils:location}/bin/test
cat = ${coreutils:location}/bin/cat
rm = ${coreutils:location}/bin/rm
echo = ${coreutils:location}/bin/echo
date = ${coreutils:location}/bin/date
md5sum = ${coreutils:location}/bin/md5sum
basename = ${coreutils:location}/bin/basename
component/faketime/buildout.cfg 0 → 100644
View file @ cf4b2a73
# faketime - report faked system time to programs without having to change the system-wide time
# http://www.code-wizards.com/projects/libfaketime
[buildout]
parts = faketime
[faketime]
recipe = slapos.recipe.cmmi
url = http://www.code-wizards.com/projects/libfaketime/libfaketime-0.9.1.tar.gz
md5sum = ce3f996dfd5826b4ac62f1a7cc36ea27
configure-command = true
make-options =
PREFIX=${buildout:parts-directory}/${:_buildout_section_name_}
make-binary = make -e -C src
make-targets = install
post-install = sed -i -e "16c\FTPL_PATH=${buildout:parts-directory}/${:_buildout_section_name_}/lib/faketime" ${buildout:parts-directory}/${:_buildout_section_name_}/bin/faketime
component/nginx/buildout.cfg
View file @ cf4b2a73
......@@ -3,12 +3,14 @@ extends =
../pcre/buildout.cfg
../zlib/buildout.cfg
../openssl/buildout.cfg
../coreutils/buildout.cfg
parts = nginx-output
[nginx]
recipe = slapos.recipe.cmmi
#url = http://nginx.org/download/nginx-1.2.7.tar.gz
url = http://nginx.org/download/nginx-1.4.2.tar.gz
#md5sum = d252f5c689a14a668e241c744ccf5f06
url = http://nginx.org/download/nginx-1.5.3.tar.gz
md5sum = 1e735dd6a6ade2b5c20e924b67c3d355
configure-options=
--with-ipv6
--with-http_ssl_module
......@@ -17,6 +19,15 @@ configure-options=
--with-ld-opt="-L ${zlib:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib"
--with-cc-opt="-I ${pcre:location}/include -I ${openssl:location}/include -I ${zlib:location}/include"
[nginx-output]
# Shared binary location to ease migration
recipe = plone.recipe.command
stop-on-error = true
update-command = ${:command}
command = ${coreutils-output:test} -x ${:nginx} -a -f ${:mime}
nginx = ${nginx:location}/sbin/nginx
mime = ${nginx:location}/conf/mime.types
[nginx-unstable]
<= nginx
url = http://nginx.org/download/nginx-1.3.15.tar.gz
......
component/openssl/buildout.cfg
View file @ cf4b2a73
......@@ -8,9 +8,10 @@ extends =
../ca-certificates/buildout.cfg
../zlib/buildout.cfg
../patch/buildout.cfg
../coreutils/buildout.cfg
parts =
openssl
openssl-output
[openssl-nodoc.patch]
# Disable doc generation part in Makefile
......@@ -52,3 +53,11 @@ make-options =
-j1
make-targets =
all install_sw && rm -f ${buildout:parts-directory}/${:_buildout_section_name_}/etc/ssl/certs/* && for i in ${ca-certificates:location}/certs/*/*.crt; do ln -sv $i ${buildout:parts-directory}/${:_buildout_section_name_}/etc/ssl/certs/`${buildout:parts-directory}/${:_buildout_section_name_}/bin/openssl x509 -hash -noout -in $i`.0; done; true
[openssl-output]
# Shared binary location to ease migration
recipe = plone.recipe.command
stop-on-error = true
update-command = ${:command}
command = ${coreutils-output:test} -x ${:openssl}
openssl = ${openssl:location}/bin/openssl
component/qemu-kvm/buildout.cfg
View file @ cf4b2a73
......@@ -56,7 +56,7 @@ configure-options =
[debian-amd64-netinst.iso]
# Download the installer of Debian 7 (Wheezy)
recipe = slapos.recipe.download
recipe = hexagonit.recipe.download
url = http://cdimage.debian.org/debian-cd/7.1.0/amd64/iso-cd/debian-7.1.0-amd64-netinst.iso
filename = ${:_buildout_section_name_}
md5sum = 80f498a1f9daa76bc911ae13692e4495
......
setup.py
View file @ cf4b2a73
......@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob
import os
version = '0.81-dev'
version = '0.82-dev'
name = 'slapos.cookbook'
long_description = open("README.txt").read() + "\n" + \
open("CHANGES.txt").read() + "\n"
......
slapos/recipe/addresiliency/renamer.py deleted 100644 → 0
View file @ d7562ad1
# -*- coding: utf-8 -*-
import logging
import time
import slapos
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
class Renamer(object):
def __init__(self, server_url, key_file, cert_file, computer_guid,
partition_id, software_release, namebase):
self.server_url = server_url
self.key_file = key_file
self.cert_file = cert_file
self.computer_guid = computer_guid
self.partition_id = partition_id
self.software_release = software_release
self.namebase = namebase
def _failover(self):
"""\
This method does
- retrieve the broken computer partition
- change its reference to 'broken-...' and its software type to 'frozen'
- retrieve the winner computer partition (attached to this process)
- change its reference to replace the broken one.
later, slapgrid will change its software_type as well.
Then, after running slapgrid-cp a few times, the winner takes over and
a new cp is created to replace it as an importer.
"""
slap = slapos.slap.slap()
slap.initializeConnection(self.server_url, self.key_file, self.cert_file)
# partition that will take over.
cp_winner = slap.registerComputerPartition(computer_guid=self.computer_guid,
partition_id=self.partition_id)
# XXX although we can already rename cp_winner, to change its software type we need to
# get hold of the root cp as well
cp_exporter_ref = self.namebase + '0' # this is ok. the boss is always number zero.
# partition to be deactivated
cp_broken = cp_winner.request(software_release=self.software_release,
software_type='frozen',
state='stopped',
partition_reference=cp_exporter_ref)
broken_new_ref = 'broken-{}'.format(time.strftime("%d-%b_%H:%M:%S", time.gmtime()))
log.debug("Renaming {}: {}".format(cp_broken.getId(), broken_new_ref))
cp_broken.rename(new_name=broken_new_ref)
log.debug("Renaming {}: {}".format(cp_winner.getId(), cp_exporter_ref))
# update name (and later, software type) for the partition that will take over
cp_winner.rename(new_name=cp_exporter_ref)
cp_winner.bang(message='partitions have been renamed!')
def failover(self):
try:
self._failover()
log.info('Renaming done')
except slapos.slap.ServerError:
log.info('Internal server error')
slapos/recipe/addresiliency/takeover.py
View file @ cf4b2a73
# -*- coding: utf-8 -*-
import logging
import time
import slapos.recipe.addresiliency.renamer
import slapos
from slapos.slap.slap import NotFoundError
def run(args):
renamer = slapos.recipe.addresiliency.renamer.Renamer(server_url = args.pop('server_url'),
key_file = args.pop('key_file'),
cert_file = args.pop('cert_file'),
computer_guid = args.pop('computer_id'),
partition_id = args.pop('partition_id'),
software_release = args.pop('software'),
namebase = args.pop('namebase'))
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
def takeover(server_url, key_file, cert_file, computer_guid,
partition_id, software_release, namebase,
winner_instance_suffix = None):
"""
This function does
- retrieve the broken computer partition
- change its reference to 'broken-...' and its software type to 'frozen'
- retrieve the winner computer partition (attached to this process)
- change its reference to replace the broken one.
later, slapgrid will change its software_type as well.
Then, after running slapgrid-cp a few times, the winner takes over and
a new cp is created to replace it as an importer.
"""
slap = slapos.slap.slap()
slap.initializeConnection(server_url, key_file, cert_file)
current_partition = slap.registerComputerPartition(computer_guid=computer_guid,
partition_id=partition_id)
# partition that will take over.
if winner_instance_suffix:
winner_instance_name = namebase + winner_instance_suffix
# XXX: we hardcode a lot of values here, because request is a settergetter, all at once.
cp_winner = current_partition.request(software_release=software_release,
software_type='%s-import' % namebase,
partition_reference=winner_instance_name)
else:
# This script is run in the winning partition: use this one as winner
cp_winner = current_partition
# XXX although we can already rename cp_winner, to change its software type we need to
# get hold of the root cp as well
cp_exporter_ref = namebase + '0' # this is ok. the boss is always number zero.
renamer.failover()
# partition to be deactivated
cp_broken = cp_winner.request(software_release=software_release,
software_type='frozen',
state='stopped',
partition_reference=cp_exporter_ref)
broken_new_ref = 'broken-{}'.format(time.strftime("%d-%b_%H:%M:%S", time.gmtime()))
log.debug("Renaming {}: {}".format(cp_broken.getId(), broken_new_ref))
cp_broken.rename(new_name=broken_new_ref)
log.debug("Renaming {}: {}".format(cp_winner.getId(), cp_exporter_ref))
# update name (and later, software type) for the partition that will take over
while True:
time.sleep(10)
try:
cp_winner.rename(new_name=cp_exporter_ref)
break
except NotFoundError:
log.warning('Impossible to rename. Retrying in a few seconds...')
log.debug('Renamed.')
cp_winner.bang(message='partitions have been renamed!')
# Note: Root instance will reconfigure itself the winning instance (software_type
# and parameters.)
def run(args):
slapos.recipe.addresiliency.takeover.takeover(server_url = args.pop('server_url'),
key_file = args.pop('key_file'),
cert_file = args.pop('cert_file'),
computer_guid = args.pop('computer_id'),
partition_id = args.pop('partition_id'),
software_release = args.pop('software'),
namebase = args.pop('namebase'))
slapos/recipe/dcron.py
View file @ cf4b2a73
......@@ -79,11 +79,17 @@ class Part(GenericBaseRecipe):
day_of_week_dict = dict((name, dow) for dow, name in enumerate(
"sunday monday tuesday wednesday thursday friday saturday".split())
for name in (name, name[:3]))
symbolic_dict = dict(hourly = '0 * * * *',
daily = '0 0 * * *',
monthly = '0 0 1 * *',
weekly = '0 0 * * 0')
def systemd_to_cron(spec):
"""Convert from systemd.time(7) calendar spec to crontab spec"""
if spec in ("hourly", "daily", "monthly", "weekly"):
return '@' + spec
try:
return symbolic_dict[spec]
except KeyError:
pass
if not spec.strip():
raise ValueError
spec = spec.split(' ')
......
slapos/recipe/kvm/__init__.py
View file @ cf4b2a73
......@@ -48,16 +48,16 @@ class Recipe(GenericBaseRecipe):
if not self.isTrueValue(self.options.get('use-tap')):
# XXX This could be done using Jinja.
for port in self.options['nat-rules'].split():
ipv6_port = int(port) + 10000
tunnel_port = int(port) + 10000
tunnel_path = self.createExecutable(
'%s-%sto%s' % (self.options['6tunnel-wrapper-path'], port, ipv6_port),
'%s-%s' % (self.options['6tunnel-wrapper-path'], tunnel_port),
self.substituteTemplate(
self.getTemplateFilename('6to4.in'),
{
'ipv6': self.options['ipv6'],
'ipv6_port': ipv6_port,
'ipv6_port': tunnel_port,
'ipv4': self.options['ipv4'],
'ipv4_port': port,
'ipv4_port': tunnel_port,
'shell_path': self.options['shell-path'],
'6tunnel_path': self.options['6tunnel-path'],
},
......
slapos/recipe/kvm/template/kvm_run.in
View file @ cf4b2a73
......@@ -18,7 +18,7 @@ nbd_list = (('%(nbd-host)s', %(nbd-port)s), ('%(nbd2-host)s', %(nbd2-port)s))
default_disk_image = '%(default-disk-image)s'
disk_path = '%(disk-path)s'
virtual_hard_drive_url = '%(virtual-hard-drive-url)s'.strip()
virtual_hard_drive_md5_url = '%(virtual-hard-drive-md5-url)s'.strip()
virtual_hard_drive_md5sum = '%(virtual-hard-drive-md5sum)s'.strip()
nat_rules = '%(nat-rules)s'.strip()
use_tap = '%(use-tap)s'
tap_interface = '%(tap-interface)s'
......@@ -59,18 +59,27 @@ def getSocketStatus(host, port):
# Download existing hard drive if needed at first boot
if not os.path.exists(disk_path) and virtual_hard_drive_url != '':
print('Downloading virtual hard drive...')
urllib.urlretrieve(virtual_hard_drive_url, disk_path)
local_md5sum = md5Checksum(disk_path)
md5sum = urllib.urlopen(virtual_hard_drive_md5_url).read().strip()
if local_md5sum != md5sum:
os.remove(disk_path)
raise Exception('MD5 mismatch.')
md5sum = virtual_hard_drive_md5sum.strip()
if md5sum:
print('Checking MD5 checksum...')
local_md5sum = md5Checksum(disk_path)
if local_md5sum != md5sum:
os.remove(disk_path)
raise Exception('MD5 mismatch. MD5 of local file is %%s, Specified MD5 is %%s.' %% (
local_md5sum, md5sum))
print('MD5sum check passed.')
else:
print('Warning: not checksum specified.')
# Create disk if doesn't exist
# XXX: move to Buildout profile
if not os.path.exists(disk_path):
print('Creating virtual hard drive...')
subprocess.Popen([qemu_img_path, 'create' ,'-f', 'qcow2',
disk_path, '%%sG' %% disk_size])
print('Done.')
# Generate network parameters
# XXX: use_tap should be a boolean
......
software/backupserver/instance-pullrdiffbackup.cfg.in
View file @ cf4b2a73
......@@ -12,7 +12,9 @@ etc = $${buildout:directory}/etc
bin = $${buildout:directory}/bin
srv = $${buildout:directory}/srv
var = $${buildout:directory}/var
run = $${:var}/run
log = $${:var}/log
varnginx = $${:var}/nginx
# scripts = $${:etc}/run
services = $${:etc}/service
cron-entries = $${:etc}/cron.d
......@@ -20,8 +22,11 @@ cron-lines = $${:etc}/cron.lines
crontabs = $${:etc}/crontabs
cronstamps = $${:etc}/cronstamps
backup = $${:srv}/backup
status = $${:srv}/status
backupscript = $${:etc}/backup
www = $${:srv}/www
home = $${:etc}/home
ssl = $${:etc}/ssl
ssh = $${:home}/.ssh
#################################
......@@ -67,10 +72,9 @@ directory = $${directory:backup}/$${:_buildout_section_name_}
[{{ slave_reference }}-backup-private_key]
recipe = plone.recipe.command
stop-on-error = true
stop-on-error = false
command = ${dropbear-output:keygen} -t $${:type} -s 2048 -f $${:key}
key = $${directory:ssh}/$${:_buildout_section_name_}
location = $${:key}
type = rsa
[{{ slave_reference }}-backup-public_key]
......@@ -91,6 +95,7 @@ storage-path = {{ '$${' ~ slave_reference }}-backup-public_key:key}
recipe = slapos.cookbook:publish
-slave-reference = {{ slave_reference }}
authorized_key = {{ '$${' ~ slave_reference }}-backup-read-public_key:readline}
rss = https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/{{ '$${' ~ slave_reference }}-backup-script:status_name}.rss
{% do part_list.append("%s-backup-publish" % slave_reference) -%}
[{{ slave_reference }}-backup-script]
......@@ -105,6 +110,8 @@ hostname = {{ hostname }}
include = {{ include_string }}
exclude_string = {{ exclude_string }}
remote_schema = {{ remote_schema }}
status_name = {{ slave_reference }}_status.txt
status_log = $${directory:status}/$${:status_name}
[{{ slave_reference }}-backup-crontab-line]
recipe = slapos.recipe.template
......@@ -121,6 +128,25 @@ frequency = {{ frequency }}
#################################
# Generate crontab file
#################################
[update-rss-script]
recipe = slapos.recipe.template
url = ${template-update-rss-script:output}
output = $${directory:etc}/$${:_buildout_section_name_}
mode = 0700
global_rss = $${slap-connection:computer-id}-$${slap-connection:partition-id}.rss
[update-rss-crontab-line]
recipe = slapos.recipe.template
url = ${template-crontab-line:output}
output = $${directory:cron-lines}/$${:_buildout_section_name_}
mode = 0600
script = $${update-rss-script:output}
frequency = */5 * * * *
{% do crontab_line_list.append("$${update-rss-crontab-line:output}") -%}
[publish-global-rss]
recipe = slapos.cookbook:publish
rss = https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/$${update-rss-script:global_rss}
{% set crontab_line_list_string = " ".join(crontab_line_list) -%}
[activate-crontab-file]
......@@ -129,12 +155,37 @@ recipe = plone.recipe.command
stop-on-error = true
command = ${coreutils-output:cat} ${template-crontab:output} {{ crontab_line_list_string }} | ${dcron-output:crontab} -c $${directory:crontabs} -
#################################
# Nginx service
#################################
[nginx-service]
recipe = slapos.recipe.template
url = ${template-nginx-service:output}
output = $${directory:services}/nginx
mode = 0700
virtual-depends =
$${nginx-configuration:ip}
[nginx-configuration]
recipe = slapos.recipe.template
url = ${template-nginx-configuration:output}
output = $${directory:etc}/nginx.cfg
mode = 0600
access_log = $${directory:log}/nginx-access.log
error_log = $${directory:log}/nginx-error.log
ip = $${slap-network-information:global-ipv6}
port = 9443
ssl_key = $${directory:ssl}/nginx.key
ssl_csr = $${directory:ssl}/nginx.csr
ssl_crt = $${directory:ssl}/nginx.crt
# Add parts generated by template
[buildout]
parts =
dcron-service
nginx-service
activate-crontab-file
publish-global-rss
{% for part in part_list -%}
{{ ' %s' % part }}
{% endfor -%}
......
software/backupserver/software.cfg
View file @ cf4b2a73
......@@ -3,7 +3,8 @@ extends =
../../component/dash/buildout.cfg
../../component/dcron/buildout.cfg
../../component/logrotate/buildout.cfg
# ../../component/nginx/buildout.cfg
../../component/openssl/buildout.cfg
../../component/nginx/buildout.cfg
../../component/rdiff-backup/buildout.cfg
# ../../component/duplicity/buildout.cfg
# ../../component/git/buildout.cfg
......@@ -15,11 +16,13 @@ extends =
../../stack/slapos.cfg
parts =
extra-eggs
rdiff-backup
# duplicity
dcron
logrotate
# nginx
nginx
openssl
# git
# subversion
rsync
......@@ -30,6 +33,12 @@ parts =
template-crontab-line
slapos-cookbook
[extra-eggs]
recipe = zc.recipe.egg
interpreter = pythonforrssgen
eggs =
PyRSS2Gen
[networkcache]
# signature certificates of the following uploaders.
# Romain Courteaud
......@@ -163,6 +172,20 @@ signature-certificate-list =
##########################################################
# Service startup scripts and configuration files
##########################################################
[template-nginx-service]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-nginx-service.sh.in
md5sum = 5c94d952305552dcbeaeaeb27dd28f3b
output = ${buildout:directory}/template-nginx-service.sh.in
mode = 0644
[template-nginx-configuration]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-nginx.cfg.in
md5sum = c54d36f55ba71c897505ed61213e104a
output = ${buildout:directory}/template-nginx.cfg.in
mode = 0644
[template-dcron-service]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-dcron-service.sh.in
......@@ -173,7 +196,7 @@ mode = 0644
[template-backup-script]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-backup-script.sh.in
md5sum = 19c3a72913cc285bf9745ab2f869018e
md5sum = 8a076962fc4df7f154572543899328e3
output = ${buildout:directory}/template-backup-script.sh.in
mode = 0644
......@@ -191,13 +214,27 @@ md5sum = 072be0fd04896880c931d44d8eabde37
output = ${buildout:directory}/template-crontab.in
mode = 0644
[status2rss]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/status2rss.py
md5sum = 138c96e0836f2b06414b98ba2643f21c
output = ${buildout:directory}/status2rss.py
mode = 0644
[template-update-rss-script]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-update-rss.sh.in
md5sum = 529058c54e873ab26f7920c868b23c50
output = ${buildout:directory}/template-update-rss.sh.in
mode = 0644
##########################################################
# Buildout instance.cfg templates
##########################################################
[template-pullrdiffbackup]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pullrdiffbackup.cfg.in
md5sum = 213256229360f57c778308825b161321
md5sum = 9bf3a34fa41ae6fe57b183293b3ff377
output = ${buildout:directory}/template-pullrdiffbackup.cfg
mode = 0644
......@@ -209,6 +246,7 @@ output = ${buildout:directory}/template.cfg
mode = 0644
[versions]
zc.buildout = 1.6.0-dev-SlapOS-010
rdiff-backup = 1.0.5
Jinja2 = 2.7
MarkupSafe = 0.18
......@@ -223,6 +261,7 @@ slapos.recipe.build = 0.11.6
slapos.recipe.cmmi = 0.1.1
slapos.recipe.template = 2.4.2
zc.recipe.egg = 1.3.2
PyRSS2Gen = 1.1
# Required by:
# slapos.core==0.35.1
......
software/backupserver/status2rss.py 0 → 100644
View file @ cf4b2a73
import datetime
import uuid
import PyRSS2Gen
import sys
from email.utils import parsedate_tz, mktime_tz
import base64
# Based on http://thehelpfulhacker.net/2011/03/27/a-rss-feed-for-your-crontabs/
# ### Defaults
TITLE = sys.argv[1]
LINK = sys.argv[2]
DESCRIPTION = TITLE
items = []
while 1:
try:
line = sys.stdin.readline()
except KeyboardInterrupt:
break
if not line:
break
time, desc = line.split(',', 1)
rss_item = PyRSS2Gen.RSSItem(
title = desc,
description = "%s, %s" % (time, desc),
link = LINK,
pubDate = datetime.datetime.fromtimestamp(mktime_tz(parsedate_tz(time))),
guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (time, desc)))
)
items.append(rss_item)
### Build the rss feed
rss_feed = PyRSS2Gen.RSS2 (
title = TITLE,
link = LINK,
description = DESCRIPTION,
lastBuildDate = datetime.datetime.utcnow(),
items = items
)
print rss_feed.to_xml()
software/backupserver/template-backup-script.sh.in
View file @ cf4b2a73
......@@ -4,7 +4,13 @@
export HOME=$${directory:home}
set -e
# Clean status file (no history needed)
${coreutils-output:rm} -f $${:status_log}
# Inform about beginning of backup
${coreutils-output:echo} "`${coreutils-output:date} -u`, $${:hostname} backup running" >> $${:status_log}
# set -e
cd $${:datadirectory}
${rdiff-backup-output:rdiff-backup} \
$${:exclude_string} \
......@@ -13,5 +19,16 @@ ${rdiff-backup-output:rdiff-backup} \
--remote-schema '${dropbear-output:ssh} -T -y -i $${:sshkey} %s $${:remote_schema}' \
$${:connection}::/ ./
RESULT=$?
# Inform about backup status
${coreutils-output:rm} -f $${:status_log}
if [ $RESULT -eq 0 ]
then
${coreutils-output:echo} "`${coreutils-output:date} -u`, $${:hostname} backup success" >> $${:status_log}
else
${coreutils-output:echo} "`${coreutils-output:date} -u`, $${:hostname} backup failed" >> $${:status_log}
fi
# python scripts/verify_with_sudo.py ./ $${:connection}:/
software/backupserver/template-nginx-service.sh.in 0 → 100644
View file @ cf4b2a73
#!${dash-output:dash}
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
if [ ! -e $${nginx-configuration:ssl_crt} ]
then
${openssl-output:openssl} genrsa -out $${nginx-configuration:ssl_key} 2048
${openssl-output:openssl} req -new \
-subj "/C=AA/ST=Denial/L=Nowhere/O=Dis/CN=$${nginx-configuration:ip}" \
-key $${nginx-configuration:ssl_key} -out $${nginx-configuration:ssl_csr}
${openssl-output:openssl} x509 -req -days 365 \
-in $${nginx-configuration:ssl_csr} \
-signkey $${nginx-configuration:ssl_key} \
-out $${nginx-configuration:ssl_crt}
fi
exec ${nginx-output:nginx} \
-c $${nginx-configuration:output}
software/backupserver/template-nginx.cfg.in 0 → 100644
View file @ cf4b2a73
daemon off; # run in the foreground so supervisord can look after it
worker_processes 4;
pid $${directory:run}/nginx.pid;
events {
worker_connections 768;
# multi_accept on;
}
error_log $${nginx-configuration:error_log};
http {
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
server_tokens off;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
default_type application/octet-stream;
include ${nginx-output:mime};
##
# Logging Settings
##
access_log $${nginx-configuration:access_log};
error_log $${nginx-configuration:error_log};
##
# Gzip Settings
##
gzip on;
gzip_disable "msie6";
gzip_vary on;
gzip_proxied any;
gzip_comp_level 6;
gzip_buffers 16 8k;
gzip_http_version 1.1;
gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
server {
listen [$${nginx-configuration:ip}]:$${nginx-configuration:port};
ssl on;
ssl_certificate $${nginx-configuration:ssl_crt};
ssl_certificate_key $${nginx-configuration:ssl_key};
fastcgi_temp_path $${directory:varnginx} 1 2;
uwsgi_temp_path $${directory:varnginx} 1 2;
scgi_temp_path $${directory:varnginx} 1 2;
client_body_temp_path $${directory:varnginx} 1 2;
proxy_temp_path $${directory:varnginx} 1 2;
## Only allow GET and HEAD request methods
if ($request_method !~ ^(GET|HEAD)$ ) {
return 444;
}
## Serve an error 204 (No Content) for favicon.ico
location = /favicon.ico {
return 204;
}
location /
{
root $${directory:www};
# index index.html;
}
}
}
software/backupserver/template-update-rss.sh.in 0 → 100644
View file @ cf4b2a73
#!${dash-output:dash}
STATUS_DIR=$${directory:status}
RSS_DIR=$${directory:www}
CAT=${coreutils-output:cat}
PYTHON=${buildout:directory}/bin/${extra-eggs:interpreter}
STATUS2RSS=${status2rss:output}
BASENAME=${coreutils-output:basename}
for status in $STATUS_DIR/*
do
NAME=`$BASENAME $status`
$CAT $status | $PYTHON $STATUS2RSS "Backup status $NAME" "https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/$NAME.rss" > $RSS_DIR/$NAME.rss
done
$CAT $STATUS_DIR/* | $PYTHON $STATUS2RSS "Full backup status $${:global_rss}" "https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/$${:global_rss}" > $RSS_DIR/$${:global_rss}
software/kvm/common.cfg
View file @ cf4b2a73
......@@ -42,6 +42,7 @@ eggs =
websockify
slapos.cookbook
slapos.toolbox
erp5.util
[http-proxy]
# https://github.com/nodejitsu/node-http-proxy
......@@ -78,7 +79,7 @@ command =
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
md5sum = 0e84223169661462f439c164d62c2a6a
#md5sum = bdd0495ef729e7272ec9c97aca919c09
output = ${buildout:directory}/template.cfg
mode = 0644
......@@ -91,9 +92,17 @@ mode = 0644
[template-kvm-resilient]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja
url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
mode = 644
md5sum = 3ee64c654aae503b93b39e9ccd6d3643
md5sum = 6753004b582c0470bd028253ce1964ad
download-only = true
[template-kvm-resilient-test]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/instance-kvm-resilient-test.cfg.jinja2
md5sum = 027d68d9decbc6aec59365fa723975d7
mode = 0644
download-only = true
[template-kvm-import]
recipe = slapos.recipe.template
......@@ -106,7 +115,7 @@ mode = 0644
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/kvm-import.sh.in
filename = kvm-import.sh.in
md5sum = e03ed049cddd8d157228b09e1ebc071a
md5sum = a731372420dc59c0b5ba7bc5f39a14ad
download-only = true
mode = 0755
......@@ -121,7 +130,7 @@ mode = 0644
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/kvm-export.sh.in
filename = kvm-export.sh.in
md5sum = 08cd8da2221f09095b14e35e6acd2a56
md5sum = 3e878b3343c76f0d6950986fffcb6a8c
download-only = true
mode = 0755
......@@ -138,3 +147,4 @@ url = ${:_profile_base_location_}/instance-frontend.cfg.in
md5sum = cdb690495e9eb007d2b7d2f8e12f5c59
output = ${buildout:directory}/template-frontend.cfg
mode = 0644
software/kvm/development.cfg
View file @ cf4b2a73
......@@ -6,11 +6,13 @@ extends =
parts +=
slapos.cookbook-repository
slapos.toolbox-repository
erp5.util-repository
check-recipe
develop =
${:parts-directory}/slapos.cookbook-repository
${:parts-directory}/slapos.toolbox-repository
${:parts-directory}/erp5.util-repository
[slapos.cookbook-repository]
recipe = slapos.recipe.build:gitclone
......@@ -21,7 +23,13 @@ git-executable = ${git:location}/bin/git
[slapos.toolbox-repository]
recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/slapos.toolbox.git
branch = master
branch = kvmresiliency
git-executable = ${git:location}/bin/git
[erp5.util-repository]
recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/erp5.git
branch = scalability-master2
git-executable = ${git:location}/bin/git
[check-recipe]
......@@ -30,7 +38,6 @@ stop-on-error = true
update-command = ${:command}
command =
grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link &&
grep parts ${buildout:develop-eggs-directory}/erp5.util.egg-link
[versions]
websockify = 0.3.0
\ No newline at end of file
software/kvm/instance-kvm-input-schema.json
View file @ cf4b2a73
......@@ -73,6 +73,12 @@
"format": "uri",
"type": "string",
},
"virtual-hard-drive-md5sum": {
"title": "Checksum of virtual hard drive",
"description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.",
"type": "string",
},
virtual-hard-drive-md5sum
"use-tap": {
"title": "Use QEMU TAP network interface",
......
software/kvm/instance-kvm-resilient-test.cfg.jinja2 0 → 100644
View file @ cf4b2a73
[buildout]
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
parts =
deploy-resiliency-test
request-resilient-kvm
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc/
var = ${:home}/var/
srv = ${:home}/srv/
bin = ${:home}/bin/
tmp = ${:home}/tmp/
log = ${:var}/log/
services = ${:etc}/service/
scripts = ${:etc}/run/
[deploy-resiliency-test]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:scripts}/runKVMResiliencyTestSuite
testnode-parameters = --test-result-path={{ slapparameter_dict.get('test-result-path') }} --revision={{ slapparameter_dict.get('test-suite-revision') }} --node-title={{ slapparameter_dict.get('scalability-launcher-title') }} --test-suite={{ slapparameter_dict.get('test-suite') }} --test-suite-master-url={{ slapparameter_dict.get('test-suite-master-url') }} --log-path=${directory:log}
kvm-test-parameters = server_url=${slap-connection:server-url} key_file=${slap-connection:key-file} cert_file=${slap-connection:cert-file} computer_id=${slap-connection:computer-id} partition_id=${slap-connection:partition-id} software=${slap-connection:software-release-url} namebase=kvm kvm_rootinstance_name='${request-resilient-kvm:name}'
command-line = {{ bin_directory }}/runResiliencyTest ${:testnode-parameters} ${:kvm-test-parameters}
[deploy-standalone-resiliency-test]
# Used to manually run the KVM test if we don't have a running testnode.
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/runStandaloneKVMResiliencyTestSuite
command-line = {{ bin_directory }}/runStandaloneResiliencyTest --test-suite-title=kvm ${deploy-resiliency-test:kvm-test-parameters}
[request-resilient-kvm]
<= slap-connection
recipe = slapos.cookbook:request
software-url = ${slap-connection:software-release-url}
software-type = kvm-resilient
name = Resilient KVM (Root Instance)
config = virtual-hard-drive-url virtual-hard-drive-md5sum resiliency-backup-periodicity
config-virtual-hard-drive-url = ${slap-parameter:virtual-hard-drive-url}
config-virtual-hard-drive-md5sum = ${slap-parameter:virtual-hard-drive-md5sum}
config-resiliency-backup-periodicity = */5
# We don't use url parameter, but we want it to be there to make sure root instance is ready.
return = url
# XXX What to do?
#sla = instance_guid
#sla-instance_guid = ${slap-parameter:frontend-instance-guid}
[slap-parameter]
virtual-hard-drive-url = https://softinst43236.host.vifib.net/data/public/8e2138.php?dl=true
virtual-hard-drive-md5sum = de0f10c7c6538e9928879332afd9be7a
software/kvm/instance-kvm-resilient.cfg.jinja2
View file @ cf4b2a73
......@@ -10,10 +10,10 @@ offline = true
# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
parts +=
{{ parts.replicate("kvm", "2") }}
{{ parts.replicate("kvm", "3") }}
publish-connection-informations
{{ replicated.replicate("kvm", "2", "kvm-export", "kvm-import") }}
{{ replicated.replicate("kvm", "3", "kvm-export", "kvm-import") }}
# Bubble down the parameters of the requested instance to the user
[request-kvm]
......@@ -22,10 +22,10 @@ return =
# Resilient related parameters
url ssh-public-key ssh-url notification-id ip
# KVM related parameters
backend-url url
backend-url url ipv6
[publish-connection-informations]
recipe = slapos.cookbook:publish
backend-url = ${request-kvm:connection-backend-url}
url = ${request-kvm:connection-url}
ipv6 = ${request-kvm:connection-ipv6}
software/kvm/instance-kvm.cfg.in
View file @ cf4b2a73
......@@ -88,7 +88,7 @@ nat-rules = $${slap-parameter:nat-rules}
6tunnel-wrapper-path = $${directory:services}/6tunnel
virtual-hard-drive-url = $${slap-parameter:virtual-hard-drive-url}
virtual-hard-drive-md5-url = $${slap-parameter:virtual-hard-drive-md5-url}
virtual-hard-drive-md5sum = $${slap-parameter:virtual-hard-drive-md5sum}
shell-path = ${dash:location}/bin/dash
qemu-path = ${kvm:location}/bin/qemu-system-x86_64
......@@ -214,7 +214,7 @@ sla-instance_guid = $${slap-parameter:frontend-instance-guid}
recipe = slapos.cookbook:publish
backend-url = https://[$${novnc-instance:ip}]:$${novnc-instance:port}/vnc_auto.html?host=[$${novnc-instance:ip}]&port=$${novnc-instance:port}&encrypt=1&password=$${kvm-instance:vnc-passwd}
url = $${request-slave-frontend:connection-url}/vnc_auto.html?host=$${request-slave-frontend:connection-domainname}&port=$${request-slave-frontend:connection-port}&encrypt=1&path=$${request-slave-frontend:connection-resource}&password=$${kvm-instance:vnc-passwd}
ipv6 = $${slap-network-information:global-ipv6}
[frontend-promise]
recipe = slapos.cookbook:check_url_available
......@@ -243,4 +243,4 @@ nat-rules = 22 80 443
use-tap = False
virtual-hard-drive-url =
virtual-hard-drive-md5-url =
virtual-hard-drive-md5sum =
software/kvm/instance.cfg.in
View file @ cf4b2a73
......@@ -16,6 +16,9 @@ kvm-resilient = $${dynamic-template-kvm-resilient:rendered}
kvm-import = ${template-kvm-import:output}
kvm-export = ${template-kvm-export:output}
# Used for the test of resiliency. The system wants a "test" software_type.
test = $${dynamic-template-kvm-resilient-test:rendered}
frozen = ${instance-frozen:output}
pull-backup = ${template-pull-backup:output}
......@@ -29,16 +32,26 @@ cert = $${slap-connection:cert-file}
[dynamic-template-kvm-resilient]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
md5sum = 1b7a2fcc884649b4d08f238e828899c1
template = ${template-kvm-resilient:location}/instance-kvm-resilient.cfg.jinja2
rendered = $${buildout:directory}/template-kvm-resilient.cfg
context = key buildout buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
template-parts-destination = ${template-parts:destination}
template-replicated-destination = ${template-replicated:destination}
import-list = file parts :template-parts-destination
file replicated :template-replicated-destination
mode = 0644
[dynamic-template-kvm-resilient-test]
recipe = slapos.recipe.template:jinja2
template = ${template-kvm-resilient-test:location}/instance-kvm-resilient-test.cfg.jinja2
rendered = $${buildout:directory}/template-kvm-resilient-test.cfg
bin-directory = ${buildout:bin-directory}
context =
key bin_directory dynamic-template-kvm-resilient-test:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
mode = 0644
software/kvm/template/kvm-export.sh.in
View file @ cf4b2a73
......@@ -14,5 +14,5 @@ $QEMU_IMG snapshot -c $SNAPSHOT_NAME $DISK_PATH
if [ -f $BACKUP_PATH ]; then
rm $BACKUP_PATH
fi
$QEMU_IMG convert -f qcow2 -O qcow2 -s $SNAPSHOT_NAME $DISK_PATH $BACKUP_PATH
$QEMU_IMG convert -f qcow2 -O qcow2 -s $SNAPSHOT_NAME $DISK_PATH $BACKUP_PATH && \
$QEMU_IMG snapshot -d $SNAPSHOT_NAME $DISK_PATH
software/kvm/template/kvm-import.sh.in
View file @ cf4b2a73
......@@ -3,5 +3,5 @@ DISK_PATH=${:disk-path}
BACKUP_PATH=${:backup-disk-path}
# TODO: Use rdiff
rm $DISK_PATH
rm $DISK_PATH && \
cp $BACKUP_PATH $DISK_PATH
stack/resilient/buildout.cfg
View file @ cf4b2a73
......@@ -48,7 +48,7 @@ mode = 0644
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/pbsready-export.cfg.in
output = ${buildout:directory}/pbsready-export.cfg
md5sum = a0e22a5de727544c5767d6bee059a77a
md5sum = ef3861861746d3574f39f1aa3200d74e
mode = 0644
[template-pull-backup]
......@@ -84,4 +84,6 @@ output = ${buildout:directory}/instance-frozen.cfg
[versions]
# Pin Jinja2 to 2.6, as 2.7 breaks current code
Jinja2 = 2.6
# ... And newer s.r.template requires Jinja2 >= 2.7
slapos.recipe.template = 2.4.2
stack/resilient/pbsready-export.cfg.in
View file @ cf4b2a73
......@@ -39,5 +39,10 @@ notify = $${slap-parameter:notify}
<= cron
recipe = slapos.cookbook:cron.d
name = backup
frequency = 0 * * * *
frequency = $${slap-parameter:resiliency-backup-periodicity} * * * *
command = $${notifier-exporter:wrapper}
[slap-parameter]
# in minutes, modulo 60, in cron.d format (i.e */15 is accepted).
resiliency-backup-periodicity = 0
stack/slapos.cfg
View file @ cf4b2a73
......@@ -77,7 +77,7 @@ zc.buildout = 1.6.0-dev-SlapOS-012
# zc.recipe.egg 2.x is for Buildout 2
zc.recipe.egg = 1.3.2
# Use own version of h.r.download to be able to open xz-like archives
hexagonit.recipe.download = 1.6nxd002
hexagonit.recipe.download = 1.7nxd002
# Use pinned version of setuptools. Other versions work, but changing
# version makes buildout recompile everything. Developers' nightmare.
setuptools = 0.9.8
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7