Skip to content

slapos
slapos
Commit bb83c9f6 authored by Timothée Lacroix's avatar Timothée Lacroix Committed by Cédric de Saint Martin
Browse files
Options

Merge Thimothée's work on resiliency 

This is a squash merge of 'resiliency-testing' branch when
the HEAD of this branch was f725c81d7d69.

The merge was done by Antoine Catton.
parent bda7b9c1
Show whitespace changes
Inline Side-by-side
Showing with 1258 additions and 259 deletions
setup.py
View file @ bb83c9f6
......@@ -41,6 +41,7 @@ setup(name=name,
zip_safe=True,
entry_points={
'zc.buildout': [
'addresiliency = slapos.recipe.addres:Recipe',
'agent = slapos.recipe.agent:Recipe',
'apache.frontend = slapos.recipe.apache_frontend:Recipe',
'apachephp = slapos.recipe.apachephp:Recipe',
......@@ -60,6 +61,7 @@ setup(name=name,
'dropbear.add_authorized_key = slapos.recipe.dropbear:AddAuthorizedKey',
'dropbear.client = slapos.recipe.dropbear:Client',
'dropbear = slapos.recipe.dropbear:Recipe',
'dumpmdb = slapos.recipe.dumpmdb:Recipe',
'duplicity = slapos.recipe.duplicity:Recipe',
'egg_test = slapos.recipe.erp5_test:EggTestRecipe',
'equeue = slapos.recipe.equeue:Recipe',
......@@ -83,6 +85,7 @@ setup(name=name,
'gitinit = slapos.recipe.gitinit:Recipe',
'haproxy = slapos.recipe.haproxy:Recipe',
'helloworld = slapos.recipe.helloworld:Recipe',
'importmdb = slapos.recipe.importmdb:Recipe',
'java = slapos.recipe.java:Recipe',
'kumofs = slapos.recipe.kumofs:Recipe',
'kvm.frontend = slapos.recipe.kvm_frontend:Recipe',
......
slapos/recipe/addres/__init__.py 0 → 100644
View file @ bb83c9f6
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
import sys
import os
class Recipe(GenericBaseRecipe):
""" This class provides the installation of the resilience
script on the partition.
"""
def install(self):
path_list = []
param_dict = self.getComputerPartitionInstanceParameterDict()
self_id = int(param_dict['number'])
ip = param_dict['ip-list'].split(' ')
print 'Creating bully script with ips : %s\n' % ip
slap_connection = self.buildout['slap-connection']
path_conf = os.path.join(self.options['script'], 'conf.in')
path_bully = os.path.join(self.options['script'], param_dict['script'])
path_bully_new = os.path.join(self.options['script'], 'new.py')
path_run = os.path.join(self.options['run'], param_dict['wrapper'])
print 'paths: %s\n%s\n' % (path_run, path_bully)
bully_conf = dict(self_id=self_id,
ip_list=ip,
executable=sys.executable,
syspath=sys.path,
server_url=slap_connection['server-url'],
key_file=slap_connection.get('key-file'),
cert_file=slap_connection.get('cert-file'),
computer_id=slap_connection['computer-id'],
partition_id=slap_connection['partition-id'],
software=slap_connection['software-release-url'],
namebase=param_dict['namebase'],
confpath=path_conf)
try:
conf = self.createFile(path_conf,
self.substituteTemplate(
self.getTemplateFilename('conf.in.in'),
bully_conf))
path_list.append(conf)
script = self.createExecutable(path_bully,
self.substituteTemplate(
self.getTemplateFilename('bully.py.in'),
bully_conf))
path_list.append(script)
# for testing purposes only
scriptNew = self.createExecutable(path_bully_new,
self.substituteTemplate(
self.getTemplateFilename('bully_new.py.in'),
bully_conf))
path_list.append(scriptNew)
wrapper = self.createPythonScript(
path_run,
'slapos.recipe.librecipe.execute.execute',
[path_bully])
path_list.append(wrapper)
except IOError:
pass
return path_list
slapos/recipe/addres/template/bully.py.in 0 → 100644
View file @ bb83c9f6
#!%(executable)s
import select
import socket
import threading
import time
import sys
sys.path[:] = %(syspath)s
from slapos import slap as slapmodule
port = 50000
size = 1024
wait = True
def loadConnectionInfos():
connectionInfos = {}
file = open('%(confpath)s', 'r')
params = file.read().split('\n')
file.close()
ip_list = [x.strip("' ") for x in params[0].strip('[],').split(',')]
connectionInfos['self_id'] = int(params[1])
connectionInfos['server_list'] = \
[(i, ip_list[i]) for i in range(len(ip_list))]
connectionInfos['self_ip'] = ip_list[connectionInfos['self_id']]
return connectionInfos
def rename_broken_and_stop():
try:
slap = slapmodule.slap()
slap.initializeConnection('%(server_url)s',
'%(key_file)s',
'%(cert_file)s')
computer_partition = slap.registerComputerPartition('%(computer_id)s',
'%(partition_id)s')
broken = computer_partition.request('%(software)s', 'frozen', '%(namebase)s0')
broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
broken.stopped()
computer_partition.rename('%(namebase)s0')
print 'renaming done\n'
except slapos.slap.slap.ServerError:
print 'Internal server error\n'
def election():
global wait
connection = loadConnectionInfos()
message = "%%s, %%s" %% (connection['self_id'], "Election")
victory = True
for (remote_id, addr) in connection['server_list']:
if remote_id > connection['self_id']:
try:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.connect((addr, port + remote_id))
s.send(message)
reply = s.recv(size)
if reply == "%%s, %%s" %% (remote_id, "Alive"):
victory = False
except (socket.error, socket.herror, socket.gaierror, socket.timeout):
pass
finally:
s.close()
if victory:
wait = True
for (remote_id, addr) in connection['server_list']:
if remote_id < connection['self_id']:
try:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.connect((addr, port + remote_id))
s.send("%%s, %%s" %% (connection['self_id'], "Victory"))
except (socket.error, socket.herror, socket.gaierror, socket.timeout):
pass
finally:
s.close()
rename_broken_and_stop()
def failure_detect():
global wait
connection = loadConnectionInfos()
while True:
time.sleep(30)
if wait:
print 'waiting 30 minutes\n'
time.sleep(30 * 60)
wait = False
if not connection['server_list'][0]:
continue
(remote_id, addr) = connection['server_list'][0]
try:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.connect((addr, port + remote_id))
s.close()
except (socket.error, socket.herror, socket.gaierror, socket.timeout):
s.close()
election()
failure_detect_thread = threading.Thread(target=failure_detect)
failure_detect_thread.start()
connection = loadConnectionInfos()
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.bind((connection['self_ip'], port + connection['self_id']))
s.listen(5)
#election()
while True:
force_election = False
client, _ = s.accept()
client_message = client.recv(1024)
if client_message:
client_id, message = client_message.split(', ')
client_id = eval(client_id)
if message == "Victory":
wait = True
print "%%s wins" %% client_id
elif message == "Election":
print "%%s starts an election" %% client_id
if client_id < connection['self_id']:
client.send("%%s, %%s" %% (connection['self_id'], "Alive"))
force_election = True
client.close()
if force_election:
election()
slapos/recipe/addres/template/bully_new.py.in 0 → 100644
View file @ bb83c9f6
#!%(executable)s
import socket
import time
import sys
import thread
import time
import os
sys.path[:] = %(syspath)s
from slapos import slap as slapmodule
port = 50000
size = 1024
def rename_broken_and_stop():
try:
slap = slapmodule.slap()
slap.initializeConnection('%(server_url)s',
'%(key_file)s',
'%(cert_file)s')
computer_partition = slap.registerComputerPartition('%(computer_id)s',
'%(partition_id)s')
broken = computer_partition.request('%(software)s', 'frozen', '%(namebase)s0')
broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
broken.stopped()
computer_partition.rename('%(namebase)s0')
print 'renaming done\n'
except slapos.slap.slap.ServerError:
print 'Internal server error\n'
## Leader is always number 0
class ResilientInstance(object):
def __init__(self, comm):
self.comm = comm
self.id = 0
self.state = 'normal'
self.halter = 0
self.nbComp = nbComp
self.inElection = False
self.alive = True
self.lastPing = time.clock()
self.mainCanal = self.comm.canal(['ping', 'halt',
'victory'])
self.okCanal = self.comm.canal(['ok'])
self.loadConnectionInfos()
def loadConnectionInfos(self):
file = open('%(confpath)s', 'r')
params = file.read().split('\n')
file.close()
self.nbComp = len([x.strip("' ") for x in params[0].strip('[],').split(',')])
new_id = int(params[1])
if self.id != new_id:
self.halter = new_id
self.id = new_id
## Needs to be changed to use the master
def aliveManagement(self):
while self.alive:
time.sleep(30*60)
if self.id == 0:
continue
self.comm.send('ping', 0)
message, sender = self.okCanal.get()
if message:
continue
self.election()
def listen(self):
while self.alive:
self.comm.recv()
def main(self):
while self.alive:
message, sender = self.mainCanal.get()
if message == 'ping':
self.comm.send('ok', sender)
elif message == 'halt':
self.state = 'waitingConfirm'
self.halter = sender
self.comm.send('ok', sender)
elif message == 'victory':
if int(sender) == int(self.halter) and self.state == 'waitingConfirm':
print '%s thinks %s is the leader\n' % (self.id, sender)
self.comm.send('ok', sender)
self.state = 'normal'
def election(self):
self.inElection = True
self.loadConnectionInfos()
#Check if I'm the highest instance alive
for higher in range(self.id + 1, self.nbComp):
self.comm.send('ping', higher)
message, sender = self.okCanal.get()
if message:
#print '%s is alive (%s)\n' % (higher, self.id)
self.inElection = False
return False
continue
if not self.alive:
return False
#I should be the new coordinator, halt those below me
print 'Should be ME : %s \n' % self.id
self.state = 'election'
self.halter = self.id
ups = []
for lower in range(self.id):
self.comm.send('halt', lower)
message, sender = self.okCanal.get()
if message:
ups.append(lower)
#Broadcast Victory
self.state = 'reorganization'
for up in ups:
self.comm.send('victory', up)
message, sender = self.okCanal.get()
if message:
continue
print 'Something is wrong... let\'s start over\n'
return self.election()
self.state = 'normal'
self.active = True
print '%s Is THE LEADER \n' % self.id
rename_broken_and_stop()
self.inElection = False
return True
class FilteredCanal(object):
def __init__(self, accept, timeout):
self.accept = accept
self.list = []
self.lock = thread.allocate_lock()
self.timeout = timeout
def append(self, message, sender):
if message in self.accept:
self.lock.acquire()
self.list.append([message, sender])
self.lock.release()
def get(self):
start = time.clock()
while (time.clock() - start < self.timeout):
self.lock.acquire()
if self.list:
self.lock.release()
val = self.list[0]
self.list = self.list[1:]
return val
self.lock.release()
return [None, None]
class Wrapper(object):
def __init__(self, timeout=20):
self.read_pipes = [os.fdopen(x) for x in read_pipes]
self.write_pipes = write_pipes
self.canals = []
self.ips = []
self.id = 0
self.timeout = timeout
self.getConnectionInfos()
self.socket = None
def getConnectionInfos(self):
file = open('%(confpath)s', 'r')
params = file.read().split('\n')
file.close()
self.ips = [x.strip("' ") for x in params[0].strip('[],').split(',')]
self.id = int(params[1])
def start(self):
self.getConnectionInfos()
self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
self.socket.bind((self.ips[self.id], port + self.id))
s.listen(5)
def send(self, message, number):
self.getConnectionInfos()
try:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.connect((self.ips[number], port + number))
s.send(message + (' %s\n' % self.id))
except (socket.error, socket.herror, socket.gaierror, socket.timeout):
pass
finally:
s.close()
def canal(self, accept):
created = FilteredCanal(accept, self.timeout)
self.canals.append(created)
return created
def recv(self):
client, _ = s.accept()
client_message = client.recv(1024)
if client_message:
message, sender = client_message.split()
for canal in self.canals:
canal.append(message, sender)
wrapper = createWrapper(20)
computer = ResilientInstance(wrapper)
#idle waiting for connection infos
while computer.nbComp < 2 :
computer.loadConnectionInfos()
time.sleep(30)
print 'Starting\n'
computer.comm.start()
thread.start_new_thread(computer.listen, ())
thread.start_new_thread(computer.main, ())
thread.start_new_thread(computer.aliveManagement, ())
while True:
continue
slapos/recipe/apacheproxy/template/apache.in
View file @ bb83c9f6
......@@ -3,7 +3,6 @@
# Basic server configuration
PidFile "%(pid_file)s"
LockFile "%(lock_file)s"
Listen %(ip)s:%(port)s
ServerAdmin someone@email
DefaultType text/plain
......@@ -22,13 +21,15 @@ CustomLog "%(access_log)s" common
<Directory />
Options FollowSymLinks
AllowOverride None
Order deny,allow
Deny from all
Require all denied
</Directory>
ProxyPass / %(backend_url)s
# List of modules
LoadModule unixd_module modules/mod_unixd.so
LoadModule access_compat_module modules/mod_access_compat.so
LoadModule authz_core_module modules/mod_authz_core.so
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule log_config_module modules/mod_log_config.so
LoadModule setenvif_module modules/mod_setenvif.so
......
slapos/recipe/dropbear.py
View file @ bb83c9f6
......@@ -167,6 +167,7 @@ class AddAuthorizedKey(GenericBaseRecipe):
path_list.append(ssh)
authorized_keys = AuthorizedKeysFile(os.path.join(ssh, 'authorized_keys'))
authorized_keys.append(self.options['key'])
for key in self.options['key'].split(' '):
authorized_keys.append(key)
return path_list
slapos/recipe/librecipe/generic.py
View file @ bb83c9f6
......@@ -31,6 +31,8 @@ import inspect
import re
import urllib
import urlparse
from slapos import slap as slapmodule
from json import loads as unjson
import pkg_resources
import zc.buildout
......@@ -90,7 +92,7 @@ class GenericBaseRecipe(object):
def createPythonScript(self, name, absolute_function, arguments=''):
"""Create a python script using zc.buildout.easy_install.scripts
ok o
* function should look like 'module.function', or only 'function'
if it is a builtin function."""
absolute_function = tuple(absolute_function.rsplit('.', 1))
......@@ -128,6 +130,17 @@ class GenericBaseRecipe(object):
return pkg_resources.resource_filename(name,
'template/%s' % template_name)
def getComputerPartitionInstanceParameterDict(self):
slap_connection = self.buildout['slap-connection']
slap = slapmodule.slap()
slap.initializeConnection(slap_connection['server-url'],
slap_connection.get('key-file'),
slap_connection.get('cert-file'))
computer_partition = slap.registerComputerPartition(slap_connection['computer-id'],
slap_connection['partition-id'])
return computer_partition.getInstanceParameterDict()
def generatePassword(self, len_=32):
"""
The purpose of this method is to generate a password which doesn't change
......
slapos/recipe/mysql/__init__.py
View file @ bb83c9f6
......@@ -31,6 +31,7 @@ import subprocess
from slapos.recipe.librecipe import GenericBaseRecipe
from slapos.recipe.librecipe import filehash
class Recipe(GenericBaseRecipe):
def _options(self, options):
......
slapos/recipe/pbs.py
View file @ bb83c9f6
......@@ -213,6 +213,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
for slave in slaves:
path_list.extend(self.add_slave(slave, known_hosts))
else:
command = [self.options['rdiffbackup-binary']]
self.logger.info("Server mode")
......
slapos/recipe/request.py
View file @ bb83c9f6
......@@ -28,6 +28,7 @@ import logging
from slapos import slap as slapmodule
class Recipe(object):
"""
Request a partition to a slap master.
......@@ -98,8 +99,9 @@ class Recipe(object):
options['computer-id'], options['partition-id']).request
isSlave = options.get('slave', '').lower() in ['y', 'yes', 'true', '1']
print '\n Slave : %s \n' % self.isSlave
return_parameters = []
self.return_parameters = []
if 'return' in options:
return_parameters = [str(parameter).strip()
for parameter in options['return'].split()]
......@@ -119,7 +121,10 @@ class Recipe(object):
for config_parameter in options['config'].split():
partition_parameter_kw[config_parameter] = \
options['config-%s' % config_parameter]
print 'from : %s \n' % options['partition-id']
print 'requested %s (%s), parameters : %s \n\n' % (options['name'],
software_type,
partition_parameter_kw)
self.instance = instance = request(software_url, software_type,
name, partition_parameter_kw=partition_parameter_kw,
filter_kw=filter_kw, shared=isSlave)
......@@ -128,6 +133,11 @@ class Recipe(object):
try:
options['connection-%s' % param] = str(
instance.getConnectionParameter(param))
except slapmodule.NotFoundError:
try:
self.instance._synced = False
options['connection-%s' % param] = str(
self.instance.getConnectionParameter(param))
except slapmodule.NotFoundError:
options['connection-%s' % param] = ''
if self.failed is None:
......
stack/lamp/apache/instance-apache-import.cfg 0 → 100644
View file @ bb83c9f6
[buildout]
extends = ${template-apache-php:output}
${template-pbsready-export:output}
parts =
apache-proxy
logrotate
logrotate-entry-apache
cron
cron-entry-logrotate
sshkeys-authority
sshkeys-dropbear
dropbear-server
dropbear-server-pbs-authorized-key
[apache-proxy]
recipe = slapos.cookbook:apacheproxy
url = $${slap-parameter:proxy-url}
pid-file = $${basedirectory:run}/apache.pid
lock-file = $${basedirectory:run}/apache.lock
ip = $${slap-network-information:global-ipv6}
port = 8080
error-log = $${directory:httpd-log}/error.log
access-log = $${directory:httpd-log}/access.log
httpd-conf = $${rootdirectory:etc}/apache.conf
wrapper = $${basedirectory:services}/apache
promise = $${basedirectory:promises}/apache
httpd-binary = ${apache:location}/bin/httpd
[sshkeys-directory]
recipe = slapos.cookbook:mkdirectory
requests = $${directory:sshkeys}/requests/
keys = $${directory:sshkeys}/keys/
[sshkeys-authority]
recipe = slapos.cookbook:sshkeys_authority
request-directory = $${sshkeys-directory:requests}
keys-directory = $${sshkeys-directory:keys}
wrapper = $${basedirectory:services}/sshkeys_authority
keygen-binary = ${dropbear:location}/bin/dropbearkey
[sshkeys-dropbear]
<= sshkeys-authority
recipe = slapos.cookbook:sshkeys_authority.request
name = dropbear
type = rsa
executable = $${dropbear-server:wrapper}
public-key = $${dropbear-server:rsa-keyfile}.pub
private-key = $${dropbear-server:rsa-keyfile}
wrapper = $${basedirectory:services}/sshd
[dropbear-server]
recipe = slapos.cookbook:dropbear
host = $${slap-network-information:global-ipv6}
port = 2222
home = $${directory:ssh}
wrapper = $${rootdirectory:bin}/raw_sshd
shell = $${rdiff-backup-server:wrapper}
rsa-keyfile = $${directory:ssh}/server_key.rsa
dropbear-binary = ${dropbear:location}/sbin/dropbear
[dropbear-server-pbs-authorized-key]
<= dropbear-server
recipe = slapos.cookbook:dropbear.add_authorized_key
key = $${slap-parameter:authorized-key}
[rdiff-backup-server]
recipe = slapos.cookbook:pbs
client = false
path = $${directory:htdocs}
wrapper = $${rootdirectory:bin}/rdiffbackup-server
rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
[logrotate]
recipe = slapos.cookbook:logrotate
# Binaries
logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
gzip-binary = ${gzip:location}/bin/gzip
gunzip-binary = ${gzip:location}/bin/gunzip
# Directories
wrapper = $${rootdirectory:bin}/logrotate
conf = $${rootdirectory:etc}/logrotate.conf
logrotate-entries = $${directory:logrotate-entries}
backup = $${directory:logrotate-backup}
state-file = $${rootdirectory:srv}/logrotate.status
[logrotate-entry-apache]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = apache
log = $${apache-proxy:error-log} $${apache-proxy:access-log}
frequency = daily
rotate-num = 30
sharedscripts = true
notifempty = true
create = true
[cron]
recipe = slapos.cookbook:cron
dcrond-binary = ${dcron:location}/sbin/crond
cron-entries = $${directory:cron-entries}
crontabs = $${directory:crontabs}
cronstamps = $${directory:cronstamps}
catcher = $${cron-simplelogger:wrapper}
binary = $${basedirectory:services}/crond
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${rootdirectory:bin}/cron_simplelogger
log = $${basedirectory:log}/crond.log
[cron-entry-logrotate]
<= cron
recipe = slapos.cookbook:cron.d
name = logrotate
frequency = 0 0 * * *
command = $${logrotate:wrapper}
[rootdirectory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc/
var = $${buildout:directory}/var/
srv = $${buildout:directory}/srv/
bin = $${buildout:directory}/bin/
tmp = $${buildout:directory}/tmp/
[basedirectory]
recipe = slapos.cookbook:mkdirectory
log = $${rootdirectory:var}/log/
services = $${rootdirectory:etc}/run/
run = $${rootdirectory:var}/run/
backup = $${rootdirectory:srv}/backup/
promises = $${rootdirectory:etc}/promise/
[directory]
recipe = slapos.cookbook:mkdirectory
htdocs = $${rootdirectory:srv}/www/
logrotate-entries = $${rootdirectory:etc}/logrotate.d/
logrotate-backup = $${basedirectory:backup}/logrotate/
cronstamps = $${rootdirectory:etc}/cronstamps/
cron-entries = $${rootdirectory:etc}/cron.d/
crontabs = $${rootdirectory:etc}/crontabs/
ssh = $${rootdirectory:etc}/ssh/
sshkeys = $${rootdirectory:srv}/sshkeys
httpd-log = $${basedirectory:log}/apache/
stack/lamp/buildout.cfg
View file @ bb83c9f6
[buildout]
ignore-existing = true
develop =
/opt/slapdev
parts =
apache-php
mariadb
......@@ -9,13 +15,16 @@ parts =
eggs
instance-recipe-egg
template
template-apache-php
template-mariadb
template-mariadb-pbsready
template-mariadb-pbsready-import
template-mariadb-pbsready-export
#Contains the importer and exporter recipes for mariadb
template-mariadb-import
template-mariadb-export
extends =
../resilient/buildout.cfg
../../component/mariadb/buildout.cfg
../../component/apache/buildout.cfg
../../component/apache-php/buildout.cfg
......@@ -59,64 +68,60 @@ strip-top-level-dir = true
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg
md5sum = 8117f10e814a13c5376af4c01e6546d4
md5sum = 8b4660ccaccda1fa8b0e73b8ac38be11
mode = 0644
[template-apache-php]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-apache-php.cfg
url = ${:_profile_base_location_}/apache/instance-apache-php.cfg
output = ${buildout:directory}/template-apache-php.cfg
md5sum = a5dd222b3faa4e1ef2df9b3b9bb47966
mode = 0644
[template-apache-backup]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-apache-backup.cfg
url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg
output = ${buildout:directory}/template-apache-backup.cfg
md5sum = cfb77ac8785e0d125a785f69a5339014
mode = 0644
[template-backuped]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-backuped.cfg
output = ${buildout:directory}/template-backuped.cfg
md5sum = f43d1c6412ea8dc83b75573dc00daf9e
[template-resilient-lamp]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-resilient.cfg
rendered = ${buildout:directory}/template-resilient.cfg
context = key templateapache template-apache-php:output
key dropbear dropbear:location
key buildout buildout:bin-directory
import-list = file parts template-parts:destination
file replicated template-replicated:destination
md5sum = 2488daff81286a1b6be6dbc1ad3b0d7d
mode = 0644
[template-mariadb]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-mariadb.cfg
url = ${:_profile_base_location_}/mariadb/instance-mariadb.cfg
output = ${buildout:directory}/template-mariadb.cfg
md5sum = 767452bc503ff6c1b7af0ebfac590c9f
mode = 0644
[template-mariadb-pbsready]
[template-mariadb-import]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-mariadb-pbsready.cfg
output = ${buildout:directory}/template-mariadb-pbsready.cfg
md5sum = d2a580dcd7efdd528be45c5ffadfe760
url = ${:_profile_base_location_}/mariadb/instance-mariadb-import.cfg
output = ${buildout:directory}/template-mariadb-import.cfg
md5sum = fa696733db4bd5b2e3e9fb6e0b09c59b
mode = 0644
[template-mariadb-pbsready-import]
[template-mariadb-export]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-mariadb-pbsready-import.cfg
output = ${buildout:directory}/template-mariadb-pbsready-import.cfg
md5sum = 4a96ff02da3898fef7077fa8baec81ac
url = ${:_profile_base_location_}/mariadb/instance-mariadb-export.cfg
output = ${buildout:directory}/template-mariadb-export.cfg
md5sum = 4b7dec765265b27c8235419b82ca7b02
mode = 0644
[template-mariadb-pbsready-export]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-mariadb-pbsready-export.cfg
output = ${buildout:directory}/template-mariadb-pbsready-export.cfg
md5sum = 11a9e45e8bc590bc11bfdd304b07a4a5
mode = 0644
[template-pull-backup]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pull-backup.cfg
output = ${buildout:directory}/template-pull-backup.cfg
md5sum = 9aab30ba5aa23a37d4b507e7c414be00
mode = 0644
# Dummy parts in case no application configuration file is needed
[application-template]
......
stack/lamp/instance-backuped.cfg deleted 100644 → 0
View file @ bda7b9c1
[buildout]
extends =
${template-apache-php:output}
parts =
request-mariadb
request-mariadb-pseudo-replicating
request-apache-backup
request-pull-backup-server
request-pull-backup-server-mariadb
request-pull-backup-server-mariadb-backup
request-pull-backup-server-apache
request-pull-backup-server-apache-backup
url
apache-php
stunnel
certificate-authority
ca-stunnel
logrotate
logrotate-entry-apache
logrotate-entry-stunnel
cron
cron-entry-logrotate
dropbear-server
sshkeys-authority
dropbear-server-pbs-authorized-key
[sshkeys-directory]
recipe = slapos.cookbook:mkdirectory
requests = $${directory:sshkeys}/requests/
keys = $${directory:sshkeys}/keys/
[sshkeys-authority]
recipe = slapos.cookbook:sshkeys_authority
request-directory = $${sshkeys-directory:requests}
keys-directory = $${sshkeys-directory:keys}
wrapper = $${basedirectory:services}/sshkeys_authority
keygen-binary = ${dropbear:location}/bin/dropbearkey
[sshkeys-dropbear]
<= sshkeys-authority
recipe = slapos.cookbook:sshkeys_authority.request
name = dropbear
type = rsa
executable = $${dropbear-server:wrapper}
public-key = $${dropbear-server:rsa-keyfile}.pub
private-key = $${dropbear-server:rsa-keyfile}
wrapper = $${basedirectory:services}/sshd
[dropbear-server]
recipe = slapos.cookbook:dropbear
host = $${slap-network-information:global-ipv6}
port = 2222
home = $${directory:ssh}
wrapper = $${rootdirectory:bin}/raw_sshd
shell = $${rdiff-backup-server:wrapper}
rsa-keyfile = $${directory:ssh}/server_key.rsa
dropbear-binary = ${dropbear:location}/sbin/dropbear
[dropbear-server-pbs-authorized-key]
<= dropbear-server
recipe = slapos.cookbook:dropbear.add_authorized_key
key = $${request-pull-backup-server:connection-ssh-key}
[rdiff-backup-server]
recipe = slapos.cookbook:pbs
client = false
path = $${directory:htdocs}
wrapper = $${rootdirectory:bin}/rdiffbackup-server
rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
[request-pull-backup-server]
<= slap-connection
recipe = slapos.cookbook:request
name = PBS (Pull Backup Server)
software-url = $${slap-connection:software-release-url}
software-type = pull-backup
return = ssh-key notification-url feeds-url
slave = false
[request-mariadb]
software-type = mariadb-pbsready-export
config = authorized-key notify
config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
config-notify = $${request-pull-backup-server:connection-notification-url}
return = url ssh-public-key ssh-url notification-id
[request-mariadb-pseudo-replicating]
<= slap-connection
recipe = slapos.cookbook:request
name = MariaDB Backup
software-url = $${slap-connection:software-release-url}
software-type = mariadb-pbsready-import
return = url ssh-public-key ssh-url notification-url
config = authorized-key on-notification
config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
pbs-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb-push
config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${:pbs-notification-id}
[request-apache-backup]
<= slap-connection
recipe = slapos.cookbook:request
name = Apache Backup
software-url = $${slap-connection:software-release-url}
software-type = apache-backup
return = url ssh-url ssh-public-key
config = authorized-key proxy-url
config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
config-proxy-url = $${url:url}
[request-pull-backup-server-apache]
<= request-pull-backup-server
slap-connection
recipe = slapos.cookbook:request
name = PBS pulling from Apache
software-url = $${slap-connection:software-release-url}
config = url name type server-key notify notification-id frequency
config-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
config-name = $${slap-connection:computer-id}-$${slap-connection:partition-id}-apache
config-type = pull
config-server-key = $${sshkeys-dropbear:public-key-value}
config-notify = $${request-pull-backup-server:connection-notification-url}
config-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-apache-pull
config-frequency = 30 * * * *
slave = true
[request-pull-backup-server-apache-backup]
<= request-pull-backup-server
slap-connection
recipe = slapos.cookbook:request
name = PBS pushing to $${request-apache-backup:name}
software-url = $${slap-connection:software-release-url}
config = url name type server-key on-notification
config-url = $${request-apache-backup:connection-ssh-url}
config-name = $${request-pull-backup-server-apache:config-name}
config-type = push
config-server-key = $${request-apache-backup:connection-ssh-public-key}
config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${request-pull-backup-server-apache:config-notification-id}
slave = true
[request-pull-backup-server-mariadb]
<= request-pull-backup-server
slap-connection
name = PBS pulling from $${request-mariadb:name}
config = url name type server-key on-notification notify notification-id title
config-url = $${request-mariadb:connection-ssh-url}
config-name = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb
config-type = pull
config-server-key = $${request-mariadb:connection-ssh-public-key}
config-on-notification = $${request-mariadb:connection-notification-id}
config-notify = $${request-pull-backup-server:connection-notification-url}
config-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb-pull
config-title = Pulling from MariaDB
slave = true
[request-pull-backup-server-mariadb-backup]
<= request-pull-backup-server
slap-connection
name = PBS pushing on $${request-mariadb-pseudo-replicating:name}
config = url name type server-key on-notification notify notification-id title
config-url = $${request-mariadb-pseudo-replicating:connection-ssh-url}
config-name = $${request-pull-backup-server-mariadb:config-name}
config-type = push
config-server-key = $${request-mariadb-pseudo-replicating:connection-ssh-public-key}
config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${request-pull-backup-server-mariadb:config-notification-id}
config-notify = $${request-mariadb-pseudo-replicating:connection-notification-url}
config-notification-id = $${request-mariadb-pseudo-replicating:pbs-notification-id}
config-title = Pushing to MariaDB backup
slave = true
[directory]
ssh = $${rootdirectory:etc}/ssh/
sshkeys = $${rootdirectory:srv}/sshkeys
stack/lamp/instance-mariadb-pbsready-export.cfg deleted 100644 → 0
View file @ bda7b9c1
[buildout]
extends = ${template-mariadb-pbsready:output}
parts += cron-entry-mariadb-backup
[urls]
notification-id = http://[$${notifier:host}]:$${notifier:port}/get/$${notifier-mydumper:name}
[mydumper]
recipe = slapos.cookbook:mydumper
wrapper = $${rootdirectory:bin}/raw_mydumper
backup-directory = $${directory:mariadb-backup}
socket = $${mariadb:socket}
user = root
mydumper-binary = ${mydumper:location}/bin/mydumper
database = $${mariadb:database}
import = false
[notifier-mydumper]
<= notifier
recipe = slapos.cookbook:notifier.notify
name = mydumper
title = Dumping MariaDB Database
executable = $${mydumper:wrapper}
wrapper = $${rootdirectory:bin}/mydumper
notify = $${slap-parameter:notify}
[cron-entry-mariadb-backup]
<= cron
recipe = slapos.cookbook:cron.d
name = backup
frequency = 0 * * * *
command = $${notifier-mydumper:wrapper}
stack/lamp/instance-resilient.cfg 0 → 100644
View file @ bb83c9f6
{% import 'parts' as parts %}
{% import 'replicated' as replicated %}
[buildout]
extends =
# ${template-resilient:output}
{{templateapache}}
parts =
{{ parts.replicate("mariadb","3") }}
request-apache-backup-1
request-apache-backup-2
request-pull-backup-server-apache-1
request-pull-backup-server-apache-backup-1
request-pull-backup-server-apache-2
request-pull-backup-server-apache-backup-2
publish-connection-informations
apache-php
stunnel
certificate-authority
ca-stunnel
logrotate
logrotate-entry-apache
logrotate-entry-stunnel
cron
cron-entry-logrotate
dropbear-server
sshkeys-authority
dropbear-server-pbs-authorized-key
request-pull-backup-server
{{ replicated.replicate("mariadb", "3", "mariadb-export", "mariadb-import") }}
[request-pull-backup-server]
<= slap-connection
recipe = slapos.cookbook:request
name = PBS (Pull Backup Server)
software-url = ${slap-connection:software-release-url}
software-type = pull-backup
return = ssh-key notification-url feeds-url
slave = false
[sshkeys-directory]
recipe = slapos.cookbook:mkdirectory
requests = ${directory:sshkeys}/requests/
keys = ${directory:sshkeys}/keys/
[sshkeys-authority]
recipe = slapos.cookbook:sshkeys_authority
request-directory = ${sshkeys-directory:requests}
keys-directory = ${sshkeys-directory:keys}
wrapper = ${basedirectory:services}/sshkeys_authority
keygen-binary = {{dropbear}}/bin/dropbearkey
[sshkeys-dropbear]
<= sshkeys-authority
recipe = slapos.cookbook:sshkeys_authority.request
name = dropbear
type = rsa
executable = ${dropbear-server:wrapper}
public-key = ${dropbear-server:rsa-keyfile}.pub
private-key = ${dropbear-server:rsa-keyfile}
wrapper = ${basedirectory:services}/sshd
[dropbear-server]
recipe = slapos.cookbook:dropbear
host = ${slap-network-information:global-ipv6}
port = 2222
home = ${directory:ssh}
wrapper = ${rootdirectory:bin}/raw_sshd
shell = ${rdiff-backup-server:wrapper}
rsa-keyfile = ${directory:ssh}/server_key.rsa
dropbear-binary = {{dropbear}}/sbin/dropbear
[dropbear-server-pbs-authorized-key]
<= dropbear-server
recipe = slapos.cookbook:dropbear.add_authorized_key
key = ${request-pull-backup-server:connection-ssh-key}
[rdiff-backup-server]
<= apache-php
recipe = slapos.cookbook:pbs
client = false
path = ${apache-php:htdocs}
wrapper = ${rootdirectory:bin}/rdiffbackup-server
rdiffbackup-binary = {{buildout}}/rdiff-backup
[request-apache-backup-1]
<= slap-connection
recipe = slapos.cookbook:request
name = Apache Backup 1
software-url = ${slap-connection:software-release-url}
software-type = apache-backup
return = url ssh-url ssh-public-key
config = authorized-key proxy-url
config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
config-proxy-url = ${publish-connection-informations:url}
[request-apache-backup-2]
<= slap-connection
recipe = slapos.cookbook:request
name = Apache Backup 2
software-url = ${slap-connection:software-release-url}
software-type = apache-backup
return = url ssh-url ssh-public-key
config = authorized-key proxy-url
config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
config-proxy-url = ${publish-connection-informations:url}
[request-pull-backup-server-apache-1]
<= request-pull-backup-server
slap-connection
recipe = slapos.cookbook:request
name = PBS pulling from Apache 1
software-url = ${slap-connection:software-release-url}
config = url name type server-key notify notification-id frequency
config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
config-type = pull
config-server-key = ${sshkeys-dropbear:public-key-value}
config-notify = ${request-pull-backup-server:connection-notification-url}
config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
config-frequency = 30 * * * *
slave = true
[request-pull-backup-server-apache-2]
<= request-pull-backup-server
slap-connection
recipe = slapos.cookbook:request
name = PBS pulling from Apache 2
software-url = ${slap-connection:software-release-url}
config = url name type server-key notify notification-id frequency
config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
config-type = pull
config-server-key = ${sshkeys-dropbear:public-key-value}
config-notify = ${request-pull-backup-server:connection-notification-url}
config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
config-frequency = 30 * * * *
slave = true
[request-pull-backup-server-apache-backup-1]
<= request-pull-backup-server
slap-connection
recipe = slapos.cookbook:request
name = PBS pushing to ${request-apache-backup-1:name}
software-url = ${slap-connection:software-release-url}
config = url name type server-key on-notification
config-url = ${request-apache-backup-1:connection-ssh-url}
config-name = ${request-pull-backup-server-apache-1:config-name}
config-type = push
config-server-key = ${request-apache-backup-1:connection-ssh-public-key}
config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-1:config-notification-id}
slave = true
[request-pull-backup-server-apache-backup-2]
<= request-pull-backup-server
slap-connection
recipe = slapos.cookbook:request
name = PBS pushing to ${request-apache-backup-2:name}
software-url = ${slap-connection:software-release-url}
config = url name type server-key on-notification
config-url = ${request-apache-backup-2:connection-ssh-url}
config-name = ${request-pull-backup-server-apache-2:config-name}
config-type = push
config-server-key = ${request-apache-backup-2:connection-ssh-public-key}
config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-2:config-notification-id}
slave = true
[directory]
ssh = ${rootdirectory:etc}/ssh/
sshkeys = ${rootdirectory:srv}/sshkeys
stack/lamp/instance.cfg
View file @ bb83c9f6
[buildout]
extends =
${template-switchsoftware:output}
parts =
switch_softwaretype
......@@ -9,9 +13,11 @@ offline = true
[switch_softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${template-apache-php:output}
backuped = ${template-backuped:output}
resilient = ${template-resilient-lamp:rendered}
mariadb = ${template-mariadb:output}
mariadb-pbsready-import = ${template-mariadb-pbsready-import:output}
mariadb-pbsready-export = ${template-mariadb-pbsready-export:output}
mariadb-import = ${template-mariadb-import:output}
mariadb-export = ${template-mariadb-export:output}
pull-backup = ${template-pull-backup:output}
apache-backup = ${template-apache-backup:output}
frozen = ${template-frozen:output}
stack/lamp/mariadb/instance-mariadb-export.cfg 0 → 100644
View file @ bb83c9f6
[buildout]
extends = ${template-mariadb:output}
${template-pbsready-export:output}
parts += mariadb
[exporter]
recipe = slapos.cookbook:mydumper
wrapper = $${rootdirectory:bin}/$${slap-parameter:namebase}-exporter
backup-directory = $${directory:backup}
socket = $${mariadb:socket}
user = root
mydumper-binary = ${mydumper:location}/bin/mydumper
database = $${mariadb:database}
import = false
\ No newline at end of file
stack/lamp/instance-mariadb-pbsready-import.cfg → stack/lamp/mariadb/instance-mariadb-import.cfg
View file @ bb83c9f6
[buildout]
extends = ${template-mariadb:output}
${template-pbsready-import:output}
extends = ${template-mariadb-pbsready:output}
parts += mariadb
parts += mariadb-import-on-notification
[urls]
notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
[mydumper-import]
[importer]
recipe = slapos.cookbook:mydumper
wrapper = $${rootdirectory:bin}/myloader
backup-directory = $${directory:mariadb-backup}
backup-directory = $${directory:backup}
socket = $${mariadb:socket}
user = root
myloader-binary = ${mydumper:location}/bin/myloader
database = $${mariadb:database}
import = true
[mariadb-import-on-notification]
<= notifier
recipe = slapos.cookbook:notifier.callback
on-notification-id = $${slap-parameter:on-notification}
callback = $${mydumper-import:wrapper}
stack/lamp/instance-mariadb.cfg → stack/lamp/mariadb/instance-mariadb.cfg
View file @ bb83c9f6
......@@ -23,6 +23,7 @@ offline = true
[urls]
recipe = slapos.cookbook:publish
url = mysqls://$${mariadb:user}:$${mariadb:password}@[$${stunnel:remote-host}]:$${stunnel:remote-port}/$${mariadb:database}
ip = $${slap-network-information:global-ipv6}
[mariadb]
recipe = slapos.cookbook:mysql
......@@ -52,6 +53,7 @@ mysql-install-binary = ${mariadb:location}/scripts/mysql_install_db
mysql-upgrade-binary = ${mariadb:location}/bin/mysql_upgrade
mysqld-binary = ${mariadb:location}/bin/mysqld
[slapmonitor]
recipe = slapos.cookbook:slapmonitor
pid-file = $${basedirectory:run}/mariadb.pid
......@@ -195,6 +197,7 @@ recipe = slapos.cookbook:mkdirectory
log = $${rootdirectory:var}/log/
services = $${rootdirectory:etc}/run/
run = $${rootdirectory:var}/run/
script = $${rootdirectory:etc}/script/
backup = $${rootdirectory:srv}/backup/
promises = $${rootdirectory:etc}/promise/
......
stack/resilient/buildout.cfg 0 → 100644
View file @ bb83c9f6
[buildout]
parts =
#Templates needed to setup automatic backup
template-pbsready
template-pbsready-import
template-pbsready-export
template-replicated
template-parts
#Frozen is the state used to not destroy a broken instance's content
template-frozen
#
template-resilient
template-switchsoftware
[template-pbsready]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pbsready.cfg
output = ${buildout:directory}/template-pbsready.cfg
md5sum = 45e64cfb6afbcfda1f9f85e33c73bd99
mode = 0644
[template-pbsready-import]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pbsready-import.cfg
output = ${buildout:directory}/template-pbsready-import.cfg
md5sum = 5ba7477f9499a7dbde5f33ca96bd6ba4
mode = 0644
[template-pbsready-export]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pbsready-export.cfg
output = ${buildout:directory}/template-pbsready-export.cfg
md5sum = 29d36aac2008b173cb9ce5da9e88c0fa
mode = 0644
[template-pull-backup]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pull-backup.cfg
output = ${buildout:directory}/template-pull-backup.cfg
md5sum = 964ea5cb91af333254e565c1ec0f312f
mode = 0644
[template-replicated]
recipe = slapos.recipe.download
url = ${:_profile_base_location_}/template-replicated.cfg
md5sum = 0f14615814f4abe37cc580bb0e51b276
mode = 0644
destination = ${buildout:directory}/template-replicated.cfg
[template-parts]
recipe = slapos.recipe.download
url = ${:_profile_base_location_}/template-parts.cfg
md5sum = f5fc27235725f05fdbde76a78ebc363e
mode = 0644
destination = ${buildout:directory}/template-parts.cfg
[template-frozen]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-frozen.cfg
output = ${buildout:directory}/template-frozen.cfg
[template-resilient]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/resilient.cfg
output = ${buildout:directory}/resilient.cfg
md5sum = 59e74d290d623de2c1e147e48f284fba
mode = 0644
[template-switchsoftware]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/switchsoftware.cfg
output = ${buildout:directory}/switchsoftware.cfg
md5sum = c94a0ed85fce2e72254ae956dce7e40d
mode = 0644
\ No newline at end of file
stack/resilient/instance-frozen.cfg 0 → 100644
View file @ bb83c9f6
[buildout]
parts =
\ No newline at end of file
stack/resilient/instance-pbsready-export.cfg 0 → 100644
View file @ bb83c9f6
[buildout]
extends = ${template-pbsready:output}
parts += cron-entry-backup
[urls]
notification-id = http://[$${notifier:host}]:$${notifier:port}/get/$${notifier-exporter:name}
#notify launches executable, and once it's done, notifies the pull-backup-servers.
[notifier-exporter]
<= notifier
recipe = slapos.cookbook:notifier.notify
name = exporter
title = Dumping $${slap-parameter:namebase}
executable = $${exporter:wrapper}
wrapper = $${rootdirectory:bin}/exporter
notify = $${slap-parameter:notify}
#adds the exporter to cron
[cron-entry-backup]
<= cron
recipe = slapos.cookbook:cron.d
name = backup
frequency = 0 * * * *
command = $${notifier-exporter:wrapper}
stack/resilient/instance-pbsready-import.cfg 0 → 100644
View file @ bb83c9f6
[buildout]
extends = ${template-pbsready:output}
parts += import-on-notification
[urls]
notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
#Launches callback, when a notification is received
[import-on-notification]
<= notifier
recipe = slapos.cookbook:notifier.callback
on-notification-id = $${slap-parameter:on-notification}
callback = $${importer:wrapper}
stack/lamp/instance-mariadb-pbsready.cfg → stack/resilient/instance-pbsready.cfg
View file @ bb83c9f6
[buildout]
extends = ${template-mariadb:output}
parts =
resiliency
urls
mariadb
stunnel
certificate-authority
ca-stunnel
logrotate
logrotate-entry-mariadb
logrotate-entry-stunnel
logrotate-entry-cron
logrotate-entry-equeue
......@@ -21,6 +18,13 @@ parts =
dropbear-server-pbs-authorized-key
notifier
# adds the resiliency script for the bully algorithm
[resiliency]
recipe = slapos.cookbook:addresiliency
script = $${basedirectory:script}
run = $${basedirectory:services}
# sets up an rdiff-backup server (with a dropbear server for ssh)
[urls]
ssh-public-key = $${sshkeys-dropbear:public-key-value}
ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
......@@ -65,10 +69,12 @@ key = $${slap-parameter:authorized-key}
[rdiff-backup-server]
recipe = slapos.cookbook:pbs
client = false
path = $${directory:mariadb-backup}
path = $${directory:backup}
wrapper = $${rootdirectory:bin}/rdiffbackup-server
rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
## Sets up the execution queue for the notifier
[logrotate-entry-equeue]
<= logrotate
recipe = slapos.cookbook:logrotate.d
......@@ -85,6 +91,9 @@ database = $${rootdirectory:srv}/equeue.db
wrapper = $${basedirectory:services}/equeue
equeue-binary = ${buildout:bin-directory}/equeue
## notifier.notify adds the [exporter, notifier] to the execution queue
## notifier.notify.callback sets up a callback
[notifier]
recipe = slapos.cookbook:notifier
feeds = $${directory:notifier-feeds}
......@@ -99,12 +108,15 @@ notifier-binary = ${buildout:bin-directory}/pubsubnotifier
[basedirectory]
script = $${rootdirectory:etc}/script/
services = $${rootdirectory:etc}/run/
cache = $${rootdirectory:var}/cache/
notifier = $${rootdirectory:etc}/notifier/
[directory]
mariadb-backup = $${basedirectory:backup}/mariadb/
backup = $${basedirectory:backup}/$${slap-parameter:namebase}
ssh = $${rootdirectory:etc}/ssh/
sshkeys = $${rootdirectory:srv}/sshkeys
notifier-feeds = $${basedirectory:notifier}/feeds/
notifier-callbacks = $${basedirectory:notifier}/callbacks/
script = $${basedirectory:script}
stack/lamp/instance-pull-backup.cfg → stack/resilient/instance-pull-backup.cfg
View file @ bb83c9f6
......@@ -19,6 +19,8 @@ ssh-key = $${sshkeys-dropbear:public-key-value}
notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/
## sets up the equeue for the notifier
[equeue]
recipe = slapos.cookbook:equeue
socket = $${basedirectory:run}/equeue.sock
......@@ -27,6 +29,7 @@ database = $${rootdirectory:srv}/equeue.db
wrapper = $${basedirectory:services}/equeue
equeue-binary = ${buildout:bin-directory}/equeue
[notifier]
recipe = slapos.cookbook:notifier
feeds = $${directory:notifier-feeds}
......@@ -39,6 +42,8 @@ wrapper = $${basedirectory:services}/notifier
server-binary = ${buildout:bin-directory}/pubsubserver
notifier-binary = ${buildout:bin-directory}/pubsubnotifier
## Dropbear Client to provide ssh
[dropbear-client]
recipe = slapos.cookbook:dropbear.client
dbclient-binary = ${dropbear:location}/bin/dbclient
......@@ -68,6 +73,9 @@ public-key = $${dropbear-client:identity-file}.pub
private-key = $${dropbear-client:identity-file}
wrapper = $${rootdirectory:bin}/do_backup
## The pull-backup-server contains every backup (incremental).
## to prevent a corrupt dump from destroying everything.
[pbs]
<= notifier
recipe = slapos.cookbook:pbs
......@@ -82,6 +90,7 @@ wrappers-directory = $${directory:pbs-wrappers}
notifier-url = http://[$${notifier:host}]:$${notifier:port}/
slave-instance-list = $${slap-parameter:slave_instance_list}
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${rootdirectory:bin}/cron_simplelogger
......
stack/resilient/resilient.cfg 0 → 100644
View file @ bb83c9f6
## not used at the moment
[buildout]
parts =
request-pull-backup-server
[request-pull-backup-server]
<= slap-connection
recipe = slapos.cookbook:request
name = PBS (Pull Backup Server)
software-url = $${slap-connection:software-release-url}
software-type = pull-backup
return = ssh-key notification-url feeds-url
slave = false
\ No newline at end of file
stack/resilient/switchsoftware.cfg 0 → 100644
View file @ bb83c9f6
[buildout]
parts =
switch_softwaretype
[switch_softwaretype]
recipe = slapos.cookbook:softwaretype
pull-backup = ${template-pull-backup:output}
#frozen creates a syntax error, meaning it can keep its data.
#It's dirty as hell, it needs to be replaced.
frozen = ${template-frozen:output}
stack/resilient/template-parts.cfg 0 → 100644
View file @ bb83c9f6
## Parts Needed for the resiliency
{% macro replicate(namebase, nbbackup) %}
request-{{namebase}}
request-{{namebase}}-2
{% for i in range(1,nbbackup|int) %}
request-{{namebase}}-pseudo-replicating-{{i}}
request-{{namebase}}-pseudo-replicating-{{i}}-2
{% endfor %}
{% for i in range(1,nbbackup|int) %}
request-pbs-{{namebase}}-{{i}}
request-pull-backup-server-{{namebase}}-{{i}}
request-pull-backup-server-{{namebase}}-backup-{{i}}
{% endfor %}
{% endmacro %}
\ No newline at end of file
stack/resilient/template-replicated.cfg 0 → 100644
View file @ bb83c9f6
{% macro replicate(namebase, nbbackup, typeexport, typeimport, heriteLeader='', heriteBackup='') %}
## Tells the Backupable recipe that we want a backup
[resilient]
config-script = bully.py
config-wrapper = bully
config-namebase = {{namebase}}
## Every request is double to provide the 3 IPs.
[request-{{namebase}}]
<= resilient
slap-connection
{{heriteLeader}}
software-type = {{typeexport}}
name = {{namebase}}0
return = url ssh-public-key ssh-url notification-id ip
config = number script wrapper authorized-key notify ip-list namebase
config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
config-ip-list =
config-number = 0
{% for id in range(1,nbbackup|int) %}
[request-{{namebase}}-pseudo-replicating-{{id}}]
<= slap-connection
resilient
{{heriteBackup}}
recipe = slapos.cookbook:request
name = {{namebase}}{{id}}
software-url = ${slap-connection:software-release-url}
software-type = {{typeimport}}
return = url ssh-public-key ssh-url notification-url ip
config = number script wrapper authorized-key on-notification ip-list namebase
config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
config-ip-list =
config-number = {{id}}
{% endfor %}
[iplist]
config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbbackup|int) %} ${request-{{namebase}}-pseudo-replicating-{{j}}:connection-ip}{% endfor %}
[request-{{namebase}}-2]
<= resilient
slap-connection
iplist
{{heriteLeader}}
recipe = slapos.cookbook:request
name = {{namebase}}0
config = number script wrapper authorized-key notify ip-list namebase
software-url = ${slap-connection:software-release-url}
software-type = {{typeexport}}
return = url ssh-public-key ssh-url notification-id ip
config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
config-number=0
{% for id in range(1,nbbackup|int) %}
[request-{{namebase}}-pseudo-replicating-{{id}}-2]
<= slap-connection
resilient
iplist
{{heriteBackup}}
recipe = slapos.cookbook:request
name = {{namebase}}{{id}}
software-url = ${slap-connection:software-release-url}
software-type = {{typeimport}}
return = url ssh-public-key ssh-url notification-url
config = number script wrapper authorized-key on-notification ip-list namebase
config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
config-number = {{id}}
{% endfor %}
## The PBS and their push / pull slaves
## Adding a PBS provides resiliency
## Adding a backup server provides availability
## Having 3 backups pulling from the same PBS provides
##only availability, not resiliency
## WARNING : SLAVES ARE ALLOCATED AT RANDOM, THIS NEEDS TO BE FIXED.
{% for id in range(1,nbbackup|int) %}
[request-pbs-{{namebase}}-{{id}}]
<= slap-connection
recipe = slapos.cookbook:request
name = PBS ({{namebase}} / {{id}})
software-url = ${slap-connection:software-release-url}
software-type = pull-backup
return = ssh-key notification-url feeds-url
slave = false
[request-pull-backup-server-{{namebase}}-{{id}}]
<= request-pbs-{{namebase}}-{{id}}
slap-connection
name = PBS {{id}} pulling from ${request-{{namebase}}:name}
config = url name type server-key on-notification notify notification-id title
config-url = ${request-{{namebase}}:connection-ssh-url}
config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}
config-type = pull
config-server-key = ${request-{{namebase}}:connection-ssh-public-key}
config-on-notification = ${request-{{namebase}}:connection-notification-id}
config-notify = ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}
config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-pull
config-title = Pulling from {{namebase}}
slave = true
[request-pull-backup-server-{{namebase}}-backup-{{id}}]
<= request-pbs-{{namebase}}-{{id}}
slap-connection
name = PBS pushing on ${request-{{namebase}}-pseudo-replicating-{{id}}:name}
config = url name type server-key on-notification notify notification-id title
config-url = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-url}
config-name = ${request-pull-backup-server-{{namebase}}-{{id}}:config-name}
config-type = push
config-server-key = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-public-key}
config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${request-pull-backup-server-{{namebase}}-{{id}}:config-notification-id}
config-notify = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-notification-url}
config-notification-id = ${request-{{namebase}}-pseudo-replicating-{{id}}:pbs-notification-id}
config-title = Pushing to {{namebase}} backup {{id}}
slave = true
{% endfor %}
{% endmacro %}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7