Commit 690be0d0 authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki

Merge branch 'master' into erp5-cluster

Conflicts:
	stack/erp5/buildout.cfg
	stack/erp5/instance-kumofs.cfg.in
	stack/erp5/instance-mariadb.cfg.in
	stack/erp5/instance-memcached.cfg.in
	stack/erp5/instance-tidstorage.cfg.in
	stack/erp5/instance-varnish.cfg.in
parents 9193eea5 0d7d7519
...@@ -12,8 +12,8 @@ parts = ...@@ -12,8 +12,8 @@ parts =
[curl] [curl]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://curl.haxx.se/download/curl-7.39.0.tar.bz2 url = http://curl.haxx.se/download/curl-7.40.0.tar.bz2
md5sum = 1efecb5b0e43c17d968f0d228bbbbbbd md5sum = 8d30594212e65657a5c32030f0998fa9
configure-options = configure-options =
--disable-static --disable-static
--disable-ldap --disable-ldap
......
...@@ -8,8 +8,8 @@ extends = ...@@ -8,8 +8,8 @@ extends =
[file] [file]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = ftp://ftp.astron.com/pub/file/file-5.21.tar.gz url = ftp://ftp.astron.com/pub/file/file-5.22.tar.gz
md5sum = 549fe96e09041eabece9de2bb28ef923 md5sum = 8fb13e5259fe447e02c4a37bc7225add
configure-options = configure-options =
--disable-static --disable-static
environment = environment =
......
...@@ -12,8 +12,8 @@ extends = ...@@ -12,8 +12,8 @@ extends =
[groonga] [groonga]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://packages.groonga.org/source/groonga/groonga-4.0.9.tar.gz url = http://packages.groonga.org/source/groonga/groonga-4.1.0.tar.gz
md5sum = ab6c15de7764ba5028b5037cc22368e8 md5sum = 93c763da6c298595da4b6964d6b80769
# temporary patch to respect more tokens in natural language mode. # temporary patch to respect more tokens in natural language mode.
patches = patches =
${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b ${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b
......
...@@ -4,7 +4,7 @@ parts = ...@@ -4,7 +4,7 @@ parts =
[lunzip] [lunzip]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://download.savannah.gnu.org/releases/lunzip/lunzip-1.6.tar.gz url = http://download.savannah.gnu.org/releases/lzip/lunzip/lunzip-1.6.tar.gz
md5sum = 5e6ad4fe91f235be64227bc9930986fe md5sum = 5e6ad4fe91f235be64227bc9930986fe
configure-options = configure-options =
--disable-static --disable-static
...@@ -39,6 +39,7 @@ environment = ...@@ -39,6 +39,7 @@ environment =
AUTOCONF=${autoconf:location}/bin/autoconf AUTOCONF=${autoconf:location}/bin/autoconf
AUTOMAKE=${automake:location}/bin/automake AUTOMAKE=${automake:location}/bin/automake
make-options = make-options =
-j1
LIBTOOL=${libtool:location}/bin/libtool LIBTOOL=${libtool:location}/bin/libtool
[glu] [glu]
......
...@@ -28,7 +28,7 @@ from setuptools import setup, find_packages ...@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob import glob
import os import os
version = '0.92' version = '0.93'
name = 'slapos.cookbook' name = 'slapos.cookbook'
long_description = open("README.txt").read() + "\n" + \ long_description = open("README.txt").read() + "\n" + \
open("CHANGES.txt").read() + "\n" open("CHANGES.txt").read() + "\n"
......
...@@ -32,7 +32,8 @@ class Recipe(GenericBaseRecipe): ...@@ -32,7 +32,8 @@ class Recipe(GenericBaseRecipe):
parameters = [ parameters = [
'--database', self.options['database'], '--database', self.options['database'],
'-l', self.options['log'], '--logfile', self.options['log'],
'--lockfile', self.options['lockfile']
] ]
if 'loglevel' in self.options: if 'loglevel' in self.options:
......
...@@ -53,6 +53,15 @@ class Recipe(GenericBaseRecipe): ...@@ -53,6 +53,15 @@ class Recipe(GenericBaseRecipe):
else: else:
networking = 'skip-networking' networking = 'skip-networking'
log_bin = self.options.get('binlog-path', '')
if log_bin:
log_bin = 'log_bin = %s' % log_bin
expire_logs_days = self.options.get('binlog-expire-days')
if expire_logs_days > 0:
expire_logs_days = 'expire_logs_days = %s' % expire_logs_days
else:
expire_logs_days = ''
mysql_conf_file = self.createFile( mysql_conf_file = self.createFile(
self.options['conf-file'], self.options['conf-file'],
self.substituteTemplate(template_filename, { self.substituteTemplate(template_filename, {
...@@ -62,6 +71,8 @@ class Recipe(GenericBaseRecipe): ...@@ -62,6 +71,8 @@ class Recipe(GenericBaseRecipe):
'socket': self.options['socket'], 'socket': self.options['socket'],
'error_log': self.options['error-log'], 'error_log': self.options['error-log'],
'slow_query_log': self.options['slow-query-log'], 'slow_query_log': self.options['slow-query-log'],
'log_bin': log_bin,
'expire_logs_days': expire_logs_days,
}) })
) )
path_list.append(mysql_conf_file) path_list.append(mysql_conf_file)
......
...@@ -48,9 +48,8 @@ innodb_locks_unsafe_for_binlog = 1 ...@@ -48,9 +48,8 @@ innodb_locks_unsafe_for_binlog = 1
#innodb_doublewrite = 0 #innodb_doublewrite = 0
#sync_frm = 0 #sync_frm = 0
# Uncomment the following if you need binary logging, which is recommended %(log_bin)s
# on production instances (either for replication or incremental backups). %(expire_logs_days)s
#log-bin=mysql-bin
# Force utf8 usage # Force utf8 usage
collation_server = utf8_unicode_ci collation_server = utf8_unicode_ci
......
...@@ -60,9 +60,4 @@ md5sum = 22ffc8e212dcf2db8ad94cf0e5ac4772 ...@@ -60,9 +60,4 @@ md5sum = 22ffc8e212dcf2db8ad94cf0e5ac4772
[versions] [versions]
PyXML = 0.8.5 PyXML = 0.8.5
erp5.util = 0.4.42 erp5.util = 0.4.42
slapos.cookbook = 0.92
slapos.recipe.template = 2.6 slapos.recipe.template = 2.6
# Required by:
# slapos.cookbook==0.92
jsonschema = 2.4.0
...@@ -65,7 +65,7 @@ log = $${:var}/log ...@@ -65,7 +65,7 @@ log = $${:var}/log
# This recipe will try to "exec" the command-line after separating parameters. # This recipe will try to "exec" the command-line after separating parameters.
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
# Notice that there is only one $ at ${dash:location}, it is because it comes from the Software Release buildout profile. # Notice that there is only one $ at ${dash:location}, it is because it comes from the Software Release buildout profile.
command-line = ${dash:location}/bin/dash -c 'echo "Hello $${instance-parameter:configuration.name}, it is $(date). If I were a TCP server, I would be listening on [$${instance-parameter:ipv6-random}] or $${instance-parameter:ipv4-random}" > $${directory:log}/log.log; sleep 1000000;' command-line = ${dash:location}/bin/dash -c 'echo "Hello $${instance-parameter:configuration.name}, it is $(date). If I were a server application, you would reach me at $${instance-parameter:global-ipv6}" > $${directory:log}/log.log; sleep 1000000;'
# Put this shell script in the "etc/service" directory. Each executable of this # Put this shell script in the "etc/service" directory. Each executable of this
# repository will be started and monitored by supervisord. If a service # repository will be started and monitored by supervisord. If a service
# exits/crashes, it will trigger a "bang" and cause a re-run of the instance. # exits/crashes, it will trigger a "bang" and cause a re-run of the instance.
......
...@@ -26,5 +26,5 @@ recipe = slapos.recipe.template ...@@ -26,5 +26,5 @@ recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in url = ${:_profile_base_location_}/instance.cfg.in
output = ${buildout:directory}/instance.cfg output = ${buildout:directory}/instance.cfg
# MD5 checksum can be skipped for development (easier to develop), but must be filled for production # MD5 checksum can be skipped for development (easier to develop), but must be filled for production
md5sum = 532d13c3aee4f0252181da39765317f0 md5sum = 1fea8a474f3b2eb7847685659441a3f9
mode = 0644 mode = 0644
...@@ -34,10 +34,6 @@ atomize = 0.2.0 ...@@ -34,10 +34,6 @@ atomize = 0.2.0
# slapos.toolbox==0.40.4 # slapos.toolbox==0.40.4
feedparser = 5.1.3 feedparser = 5.1.3
# Required by:
# slapos.cookbook==0.87
jsonschema = 2.4.0
# Required by: # Required by:
# websockify==0.6.0 # websockify==0.6.0
numpy = 1.9.0 numpy = 1.9.0
......
...@@ -45,14 +45,14 @@ mode = 0644 ...@@ -45,14 +45,14 @@ mode = 0644
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner.cfg url = ${:_profile_base_location_}/instance-runner.cfg
output = ${buildout:directory}/template-runner.cfg.in output = ${buildout:directory}/template-runner.cfg.in
md5sum = 39458e6a1d8cddd0000d6d6a0517f017 md5sum = 3cac749e81d8b94dbbf1f1833bc031ce
mode = 0644 mode = 0644
[template-runner-import-script] [template-runner-import-script]
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/runner-import.sh.jinja2 url = ${:_profile_base_location_}/template/runner-import.sh.jinja2
download-only = true download-only = true
md5sum = c0d05a26b06ce172efaad03c52ef92ca md5sum = 4134ea7e191d0c3b552d2efbae6b5894
filename = runner-import.sh.jinja2 filename = runner-import.sh.jinja2
mode = 0644 mode = 0644
...@@ -60,14 +60,14 @@ mode = 0644 ...@@ -60,14 +60,14 @@ mode = 0644
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner-import.cfg.in url = ${:_profile_base_location_}/instance-runner-import.cfg.in
output = ${buildout:directory}/instance-runner-import.cfg output = ${buildout:directory}/instance-runner-import.cfg
md5sum = a85d054b3e2ae9243d8d188c897dc121 md5sum = e9e3692ce4af4d603666c969ec1964d2
mode = 0644 mode = 0644
[template-runner-export-script] [template-runner-export-script]
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/runner-export.sh.jinja2 url = ${:_profile_base_location_}/template/runner-export.sh.jinja2
download-only = true download-only = true
md5sum = 0f290b46c0b89ff84aee5c10477e07ca md5sum = 41c0213f5cc07ecbe5c2852ef1844ac9
filename = runner-export.sh.jinja2 filename = runner-export.sh.jinja2
mode = 0644 mode = 0644
...@@ -75,7 +75,7 @@ mode = 0644 ...@@ -75,7 +75,7 @@ mode = 0644
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner-export.cfg.in url = ${:_profile_base_location_}/instance-runner-export.cfg.in
output = ${buildout:directory}/instance-runner-export.cfg output = ${buildout:directory}/instance-runner-export.cfg
md5sum = 521bad4c571b5b2dc3eee6090802de95 md5sum = aaf86d03409018b5d3dce2dea27287cf
mode = 0644 mode = 0644
[template-resilient] [template-resilient]
...@@ -171,6 +171,15 @@ destination = ${buildout:parts-directory}/monitor-template-cors-domain-cgi ...@@ -171,6 +171,15 @@ destination = ${buildout:parts-directory}/monitor-template-cors-domain-cgi
filename = cors-domain.jinja filename = cors-domain.jinja
mode = 0644 mode = 0644
[monitor-check-webrunner-internal-instance]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/${:filename}
download-only = true
#md5sum = 4c44d617d5bfd1db8695200e896480c0
destination = ${buildout:parts-directory}/${:filename}
filename = monitor-check-webrunner-internal-instances.py
mode = 0644
[eggs] [eggs]
recipe = z3c.recipe.scripts recipe = z3c.recipe.scripts
eggs = eggs =
......
...@@ -16,13 +16,14 @@ parts += ...@@ -16,13 +16,14 @@ parts +=
publish-connection-informations publish-connection-informations
slaprunner-promise slaprunner-promise
slaprunner-frontend-promise slaprunner-frontend-promise
slaprunner-supervisord-wrapper
dropbear-promise dropbear-promise
runtestsuite runtestsuite
shellinabox-promise shellinabox-promise
symlinks symlinks
shellinabox shellinabox
slapos-cfg slapos-cfg
slapos-repo-config slapos-repo
cron-entry-backup cron-entry-backup
cron-entry-prepare-software cron-entry-prepare-software
deploy-instance-parameters deploy-instance-parameters
...@@ -50,6 +51,7 @@ parts += ...@@ -50,6 +51,7 @@ parts +=
## Monitor for runner ## Monitor for runner
monitor-current-log-access monitor-current-log-access
monitor-check-resilient-feed-file monitor-check-resilient-feed-file
monitor-check-webrunner-internal-instance
[exporter] [exporter]
......
...@@ -14,18 +14,20 @@ parts += ...@@ -14,18 +14,20 @@ parts +=
dropbear-server-add-authorized-key dropbear-server-add-authorized-key
sshkeys-authority sshkeys-authority
slaprunner-promise slaprunner-promise
slaprunner-supervisord-wrapper
dropbear-promise dropbear-promise
runtestsuite runtestsuite
shellinabox-promise shellinabox-promise
shellinabox shellinabox
symlinks symlinks
slapos-cfg slapos-cfg
slapos-repo-config slapos-repo
cron-entry-prepare-software cron-entry-prepare-software
deploy-instance-parameters deploy-instance-parameters
instance-software-type instance-software-type
bash-profile bash-profile
supervisord-wrapper supervisord-wrapper
importer-consistency-promise
# have to repeat the next one, as it's not inherited from pbsready-import # have to repeat the next one, as it's not inherited from pbsready-import
import-on-notification import-on-notification
## Monitoring part ## Monitoring part
...@@ -52,6 +54,14 @@ parts += ...@@ -52,6 +54,14 @@ parts +=
## Monitor for import runner ## Monitor for import runner
monitor-latest-restored-backup monitor-latest-restored-backup
# For the needs of importer, we run the full slaprunner
# In case both exporter and importer (aka main instance and clone instance)
# run with the same IP (usually for testing purposes),
# run slaprunner using different ports.
[slaprunner]
proxy_port = 50000
runner_port = 50005
[importer] [importer]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = ${template-runner-import-script:location}/${template-runner-import-script:filename} template = ${template-runner-import-script:location}/${template-runner-import-script:filename}
...@@ -59,12 +69,30 @@ rendered = $${directory:bin}/$${slap-parameter:namebase}-importer ...@@ -59,12 +69,30 @@ rendered = $${directory:bin}/$${slap-parameter:namebase}-importer
# backward compatibility for resilient stack # backward compatibility for resilient stack
wrapper = $${:rendered} wrapper = $${:rendered}
mode = 700 mode = 700
restore-exit-code-file=$${directory:srv}/importer-exit-code-file
context = context =
key backend_url slaprunner:access-url key backend_url slaprunner:access-url
section directory directory section directory directory
raw shell_binary ${dash:location}/bin/dash raw shell_binary ${dash:location}/bin/dash
raw rsync_binary ${rsync:location}/bin/rsync raw rsync_binary ${rsync:location}/bin/rsync
raw curl_binary ${curl:location}/bin/curl raw curl_binary ${curl:location}/bin/curl
raw restore_exit_code_file $${:restore-exit-code-file}
[importer-consistency-promise]
# Test that the importer script and "after-import" subscripts:
# 1/ Have been run in the last 24 hours
# 2/ Have succeeded
recipe = collective.recipe.template
input = inline: #!/bin/sh
EXIT_CODE_FILE=$(find "$${importer:restore-exit-code-file}" -mtime -1)
if [ -z "$EXIT_CODE_FILE" ]; then
echo "Consistency check is too old."
exit 1
fi
EXIT_CODE=$(cat $EXIT_CODE_FILE)
exit $EXIT_CODE
output = $${directory:promises}/importer-consistency-promise
mode = 755
[slap-parameter] [slap-parameter]
auto-deploy-instance = false auto-deploy-instance = false
......
...@@ -13,15 +13,17 @@ parts = ...@@ -13,15 +13,17 @@ parts =
publish-connection-informations publish-connection-informations
slaprunner-promise slaprunner-promise
slaprunner-frontend-promise slaprunner-frontend-promise
slaprunner-supervisord-wrapper
dropbear-promise dropbear-promise
runtestsuite runtestsuite
shellinabox-promise shellinabox-promise
symlinks symlinks
shellinabox shellinabox
slapos-cfg slapos-cfg
slapos-repo-config slapos-repo
cron-entry-prepare-software cron-entry-prepare-software
deploy-instance-parameters deploy-instance-parameters
instance-software
instance-software-type instance-software-type
minishell-cwd minishell-cwd
bash-profile bash-profile
...@@ -51,6 +53,7 @@ parts = ...@@ -51,6 +53,7 @@ parts =
## Monitor for runner ## Monitor for runner
monitor-current-log-access monitor-current-log-access
monitor-deploy-cors-domain-cgi monitor-deploy-cors-domain-cgi
monitor-check-webrunner-internal-instance
extends = ${monitor-template:output} extends = ${monitor-template:output}
...@@ -105,11 +108,12 @@ custom-frontend-url = https://$${request-custom-frontend:connection-domain} ...@@ -105,11 +108,12 @@ custom-frontend-url = https://$${request-custom-frontend:connection-domain}
# Create all needed directories # Create all needed directories
[directory] [directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc/ home = $${buildout:directory}
var = $${buildout:directory}/var/ etc = $${:home}/etc/
srv = $${buildout:directory}/srv/ var = $${:home}/var/
bin = $${buildout:directory}/bin/ srv = $${:home}/srv/
tmp = $${buildout:directory}/tmp/ bin = $${:home}/bin/
tmp = $${:home}/tmp/
sshkeys = $${:srv}/sshkeys sshkeys = $${:srv}/sshkeys
services = $${:etc}/service/ services = $${:etc}/service/
...@@ -120,7 +124,7 @@ run = $${:var}/run/ ...@@ -120,7 +124,7 @@ run = $${:var}/run/
backup = $${:srv}/backup/ backup = $${:srv}/backup/
promises = $${:etc}/promise/ promises = $${:etc}/promise/
test = $${:etc}/test/ test = $${:etc}/test/
nginx-data = $${directory:srv}/nginx nginx-data = $${:srv}/nginx
ca-dir = $${:srv}/ssl ca-dir = $${:srv}/ssl
project = $${:srv}/runner/project project = $${:srv}/runner/project
...@@ -189,6 +193,17 @@ software_info_json = $${runnerdirectory:home}/software_info.json ...@@ -189,6 +193,17 @@ software_info_json = $${runnerdirectory:home}/software_info.json
instance_info_json = $${runnerdirectory:home}/instance_info.json instance_info_json = $${runnerdirectory:home}/instance_info.json
path = $${shell:path} path = $${shell:path}
#---------------------------
#--
#-- supervisord managing slaprunner instance processes
[slaprunner-supervisord-wrapper]
recipe = slapos.cookbook:wrapper
# XXX hardcoded locations
command-line = $${buildout:directory}/bin/slapos node supervisord --cfg $${directory:etc}/slapos.cfg -n
wrapper-path = $${directory:services}/slaprunner-supervisord
[test-runner] [test-runner]
<= slaprunner <= slaprunner
slapos.cfg = $${directory:etc}/slapos-test.cfg slapos.cfg = $${directory:etc}/slapos-test.cfg
...@@ -262,7 +277,6 @@ scgi_temp_path = $${directory:tmp}/scgi_temp_path ...@@ -262,7 +277,6 @@ scgi_temp_path = $${directory:tmp}/scgi_temp_path
nb_workers = 2 nb_workers = 2
# Network # Network
local-ip = $${slap-network-information:local-ipv4} local-ip = $${slap-network-information:local-ipv4}
port = 30001
global-ip = $${slap-network-information:global-ipv6} global-ip = $${slap-network-information:global-ipv6}
global-port = $${slaprunner:runner_port} global-port = $${slaprunner:runner_port}
# Backend # Backend
...@@ -558,12 +572,7 @@ repository = $${slap-parameter:slapos-repository} ...@@ -558,12 +572,7 @@ repository = $${slap-parameter:slapos-repository}
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
develop = true develop = true
location = $${directory:project}/slapos location = $${directory:project}/slapos
branch = $${slap-parameter:slapos-reference}
[slapos-repo-config]
recipe = plone.recipe.command
stop-on-error = true
command = cd $${slapos-repo:location} && ${git:location}/bin/git checkout $${slap-parameter:slapos-reference} && SR=$${slap-parameter:slapos-software} && if [ -n "$SR" ] && [ ! -f "$${directory:etc}/.project" ]; then echo workspace/slapos/$${slap-parameter:slapos-software}/ > $${directory:etc}/.project; fi
update-command = true
[prepare-software] [prepare-software]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
...@@ -599,6 +608,11 @@ stop-on-error = true ...@@ -599,6 +608,11 @@ stop-on-error = true
software-type-path = $${directory:etc}/.software_type.xml software-type-path = $${directory:etc}/.software_type.xml
command = if [ ! -f $${:software-type-path} -a "$${slap-parameter:slapos-software-type}" != "" ]; then echo "$${slap-parameter:slapos-software-type}" > $${:software-type-path}; fi command = if [ ! -f $${:software-type-path} -a "$${slap-parameter:slapos-software-type}" != "" ]; then echo "$${slap-parameter:slapos-software-type}" > $${:software-type-path}; fi
[instance-software]
recipe = plone.recipe.command
stop-on-error = true
command = SR=$${slap-parameter:slapos-software} && if [ -n "$SR" ] && [ ! -f "$${directory:etc}/.project" ]; then echo workspace/slapos/$${slap-parameter:slapos-software}/ > $${directory:etc}/.project; fi
[slap-configuration] [slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id} computer = $${slap-connection:computer-id}
...@@ -628,6 +642,11 @@ context = ...@@ -628,6 +642,11 @@ context =
raw path $PATH:${nano:location}/bin:${vim:location}/bin:${screen:location}/bin:${git:location}/bin:${curl:location}/bin:${python2.7:location}/bin raw path $PATH:${nano:location}/bin:${vim:location}/bin:${screen:location}/bin:${git:location}/bin:${curl:location}/bin:${python2.7:location}/bin
key workdir runnerdirectory:home key workdir runnerdirectory:home
#---------------------------
#--
#-- supervisord managing slaprunner automation features
[supervisord] [supervisord]
autorestart = false autorestart = false
autostart = false autostart = false
...@@ -696,6 +715,7 @@ path = $${directory:promises}/supervisord ...@@ -696,6 +715,7 @@ path = $${directory:promises}/supervisord
hostname = $${slaprunner:ipv4} hostname = $${slaprunner:ipv4}
port = $${supervisord:port} port = $${supervisord:port}
[monitor-current-log-access] [monitor-current-log-access]
< = monitor-directory-access < = monitor-directory-access
source = $${directory:log} source = $${directory:log}
...@@ -715,6 +735,13 @@ context = ...@@ -715,6 +735,13 @@ context =
key this_file :filename key this_file :filename
key httpd_graceful cgi-httpd-graceful-wrapper:rendered key httpd_graceful cgi-httpd-graceful-wrapper:rendered
[monitor-check-webrunner-internal-instance]
recipe = slapos.recipe.template:jinja2
template = ${monitor-check-webrunner-internal-instance:location}/${monitor-check-webrunner-internal-instance:filename}
rendered = $${monitor-directory:monitor-custom-scripts}/$${:filename}
filename = monitor-check-webrunner-internal-instance.py
mode = 0744
[monitor-httpd-cors] [monitor-httpd-cors]
recipe = plone.recipe.command recipe = plone.recipe.command
command = if [ ! -f $${:location} ]; then touch $${:location}; fi command = if [ ! -f $${:location} ]; then touch $${:location}; fi
......
...@@ -15,31 +15,32 @@ collective.recipe.environment = 0.2.0 ...@@ -15,31 +15,32 @@ collective.recipe.environment = 0.2.0
collective.recipe.template = 1.11 collective.recipe.template = 1.11
ecdsa = 0.11 ecdsa = 0.11
erp5.util = 0.4.42 erp5.util = 0.4.42
gitdb = 0.6.0 gitdb = 0.6.4
gunicorn = 19.1.1 gunicorn = 19.1.1
plone.recipe.command = 1.1 plone.recipe.command = 1.1
pycrypto = 2.6.1 pycrypto = 2.6.1
slapos.recipe.download = 1.0.dev-r4053 slapos.recipe.download = 1.0.dev-r4053
slapos.toolbox = 0.45.3 slapos.recipe.template = 2.6
smmap = 0.8.3 slapos.toolbox = 0.46.0
smmap = 0.9.0
z3c.recipe.scripts = 1.0.1 z3c.recipe.scripts = 1.0.1
# Required by: # Required by:
# slapos.toolbox==0.45.2 # slapos.toolbox==0.46.0
GitPython = 0.3.2.1 GitPython = 0.3.5
# Required by: # Required by:
# slapos.toolbox==0.45.2 # slapos.toolbox==0.46.0
atomize = 0.2.0 atomize = 0.2.0
# Required by: # Required by:
# slapos.toolbox==0.45.2 # slapos.toolbox==0.46.0
feedparser = 5.1.3 feedparser = 5.1.3
# Required by: # Required by:
# slapos.cookbook==0.92 # slapos.toolbox==0.46.0
jsonschema = 2.4.0 lockfile = 0.10.2
# Required by: # Required by:
# slapos.toolbox==0.45.2 # slapos.toolbox==0.45.3
paramiko = 1.15.2 paramiko = 1.15.2
#!/usr/bin/python
import os
import subprocess
import sys
def runPromise(promise_path):
promise_relative_path = promise_path.replace(os.path.expanduser('~'), '~')
print 'Running promise %s...' % promise_relative_path
promise_process = subprocess.Popen(promise_path, stderr=subprocess.PIPE)
stdout, stderr = promise_process.communicate()
return_code = promise_process.returncode
if return_code == 0:
print 'Success.'
return True
else:
sys.stderr.write('Failure while running promise %s. %s\n' % (promise_relative_path, stderr))
def getPromisePathListFromPartitionPath(partition_path):
promise_directory_path = os.path.join(partition_path, 'etc/promise')
try:
promise_name_list = os.listdir(promise_directory_path)
return [os.path.join(promise_directory_path, promise_name) for promise_name in promise_name_list]
except OSError:
return []
def main():
# XXX hardcoded
partition_root_path = os.path.expanduser('~/srv/runner/instance')
success = True
for partition_name in os.listdir(partition_root_path):
partition_path = os.path.join(partition_root_path, partition_name)
for promise_path in getPromisePathListFromPartitionPath(partition_path):
result = runPromise(promise_path)
if not result:
success = False
if not success:
sys.exit(1)
if __name__ == '__main__':
main()
...@@ -2,20 +2,44 @@ ...@@ -2,20 +2,44 @@
LC_ALL=C LC_ALL=C
export LC_ALL export LC_ALL
umask 077 umask 077
srv_directory={{ directory['srv'] }}
sync_element () { sync_element () {
path=$1 path=$1
backup_path=$2 backup_path=$2
shift 2 shift 2
element_list=$* element_list=$*
# Concatenate the exclude file of each partition of webrunner
# to create a global exclude file.
for partition in $srv_directory/runner/instance/slappart*
do
exclude_file="$partition/srv/exporter.exclude"
if [ -e "$exclude_file" ]; then
partition_exclude_content_relative=$(cat "$exclude_file")
# For every line of the local exclude file, add the absolute path
for line in "$partition_exclude_content_relative"
do
if [ ! -z "$line" ]; then
exclude_content="$exclude_content\ninstance/$(basename $partition)/$line"
fi
done
fi
done
echo "$exclude_content" > $srv_directory/exporter.exclude
for element in $element_list for element in $element_list
do do
echo "Changing current directory to $path."
cd $path; cd $path;
if [ -f $element ] || [ -d $element ]; then if [ -f $element ] || [ -d $element ]; then
{{ rsync_binary }} -rlptgov --safe-links --delete $element $backup_path; echo "Running {{ rsync_binary }} -rlptgov --safe-links --exclude-from=$srv_directory/exporter.exclude --delete --delete-excluded $element $backup_path"
{{ rsync_binary }} -rlptgov --safe-links --exclude-from=$srv_directory/exporter.exclude --delete --delete-excluded $element $backup_path;
fi fi
done done
} }
sync_element {{ directory['srv'] }}/runner {{ directory['backup'] }}/runner/ instance project proxy.db sync_element $srv_directory/runner {{ directory['backup'] }}/runner/ instance project proxy.db
# We sync .* appart # We sync .* appart
date +%s -u > {{ directory['etc'] }}/.resilient-timestamp date +%s -u > {{ directory['etc'] }}/.resilient-timestamp
cp -r {{ directory['etc'] }}/.??* {{ directory['backup'] }}/etc/ cp -r {{ directory['etc'] }}/.??* {{ directory['backup'] }}/etc/
......
#!{{ shell_binary }} #!{{ shell_binary }}
set -e
LC_ALL=C LC_ALL=C
export LC_ALL export LC_ALL
umask 077 umask 077
srv_directory={{ directory['srv'] }}
restore_element () { restore_element () {
backup_path=$1 backup_path=$1
restore_path=$2 restore_path=$2
...@@ -18,14 +20,66 @@ restore_element () { ...@@ -18,14 +20,66 @@ restore_element () {
write_backup_proof () { write_backup_proof () {
cd {{ directory['backup'] }} cd {{ directory['backup'] }}
find -type f ! -name backup.signature ! -wholename "./rdiff-backup-data/*" -print0 | xargs -P4 -0 sha256sum | LC_ALL=C sort -k 66 > {{ directory['srv'] }}/proof.signature find -type f ! -name backup.signature ! -wholename "./rdiff-backup-data/*" -print0 | xargs -P4 -0 sha256sum | LC_ALL=C sort -k 66 > $srv_directory/proof.signature
diff -ruw {{ directory['backup'] }} {{ directory['srv'] }}/proof.signature > {{ directory['srv'] }}/backup.diff diff -ruw {{ directory['backup'] }} $srv_directory/proof.signature > $srv_directory/backup.diff || true # diff exits with code 1 when files are different
} }
# For now we just make the diff before # For now we just make the diff before
write_backup_proof write_backup_proof
restore_element {{ directory['backup'] }}/runner/ {{ directory['srv'] }}/runner instance project proxy.db restore_element {{ directory['backup'] }}/runner/ $srv_directory/runner instance project proxy.db
restore_element {{ directory['backup'] }}/etc/ {{ directory['etc'] }} config.json ssh restore_element {{ directory['backup'] }}/etc/ {{ directory['etc'] }} config.json ssh
cp -r {{ directory['backup'] }}/etc/.??* {{ directory['etc'] }}; cp -r {{ directory['backup'] }}/etc/.??* {{ directory['etc'] }};
{{ curl_binary }} --insecure -vg6L --max-time 5 {{ backend_url }}/isSRReady; {{ curl_binary }} --insecure -g6L --max-time 5 {{ backend_url }}/isSRReady;
# Invoke arbitrary script to perform specific restoration
# procedure.
RESTORE_EXIT_CODE=0
runner_import_restore=$srv_directory/runner-import-restore
if [ ! -e "$runner_import_restore" ]; then
touch $runner_import_restore
chmod +x $runner_import_restore
fi
echo "Running $runner_import_restore script..."
$srv_directory/runner-import-restore || RESTORE_EXIT_CODE=$?
echo "Updating slapproxy database, software release and instances..."
HOME="{{ directory['home'] }}"
SLAPOS="{{ directory['bin'] }}/slapos"
# XXX hardcoded
SQLITE3="$HOME/software_release/parts/sqlite3/bin/sqlite3"
DATABASE="$HOME/srv/runner/proxy.db"
# Change slapproxy database to point instances to new software release
# XXX hardcoded
PARTITION=$(basename $HOME)
OLD_SOFTWARE_RELEASE=$($SQLITE3 $DATABASE "select software_release from partition11 where reference='slappart0';")
SOFTWARE_RELEASE=$(echo $OLD_SOFTWARE_RELEASE | sed -e 's/slappart[0-9][0-9]/'"$PARTITION"'/')
$SQLITE3 $DATABASE "update partition11 set software_release='$SOFTWARE_RELEASE' where software_release NOT NULL;"
# Change slapproxy database to have all instances stopped
$SQLITE3 $DATABASE "update partition11 set slap_state='stopped';"
# Build newest software
$SLAPOS node software --cfg ~/etc/slapos.cfg --pidfile ~/var/run/slapos-node-software.pid --all >/dev/null 2>&1 || true
# Remove defined scripts to force buildout to recreate them to have updated paths
rm $srv_directory/runner/instance/slappart*/srv/runner-import-restore
# Run slapos node instance
# XXX hardcoded
$SLAPOS node instance --cfg ~/etc/slapos.cfg --pidfile ~/var/run/slapos-node-instance.pid >/dev/null 2>&1 || true
# Invoke defined scripts for each partition inside of slaprunner
for partition in $srv_directory/runner/instance/slappart*/
do
script=$partition/srv/runner-import-restore
if [ -e "$script" ]; then
echo "Running $script script..."
$script || RESTORE_EXIT_CODE=$?
fi
done
# Change back slapproxy database to have all instances started
$SQLITE3 $DATABASE "update partition11 set slap_state='started';"
# Write exit code to an arbitrary file that will be checked by promise/monitor
RESTORE_EXIT_CODE_FILE="{{ restore_exit_code_file }}"
echo $RESTORE_EXIT_CODE > $RESTORE_EXIT_CODE_FILE
exit $RESTORE_EXIT_CODE
...@@ -16,6 +16,7 @@ extends = ...@@ -16,6 +16,7 @@ extends =
../../component/gzip/buildout.cfg ../../component/gzip/buildout.cfg
../../component/haproxy/buildout.cfg ../../component/haproxy/buildout.cfg
../../component/hookbox/buildout.cfg ../../component/hookbox/buildout.cfg
../../component/findutils/buildout.cfg
../../component/librsvg/buildout.cfg ../../component/librsvg/buildout.cfg
../../component/imagemagick/buildout.cfg ../../component/imagemagick/buildout.cfg
../../component/inkscape/buildout.cfg ../../component/inkscape/buildout.cfg
...@@ -97,7 +98,6 @@ parts = ...@@ -97,7 +98,6 @@ parts =
userhosts userhosts
# Buildoutish # Buildoutish
patched-eggs
eggs eggs
testrunner testrunner
test_suite_runner test_suite_runner
...@@ -157,10 +157,16 @@ context = ...@@ -157,10 +157,16 @@ context =
key slapos_core_version versions:slapos.core key slapos_core_version versions:slapos.core
${:extra-context} ${:extra-context}
[mariadb-resiliency-after-import-script]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/template/instance-mariadb-resiliency-after-import-script.sh.in
md5sum = 8db483ef7c3da79a1cb5ea07ba79a0ed
mode = 755
[template-mariadb] [template-mariadb]
< = download-base < = download-base
filename = instance-mariadb.cfg.in filename = instance-mariadb.cfg.in
md5sum = 8a59e1730391fb3a859cd973cd7c3a5b md5sum = fc4a7585fe1bfea0c198b1826f904595
link-binary = link-binary =
${coreutils:location}/bin/basename ${coreutils:location}/bin/basename
${coreutils:location}/bin/cat ${coreutils:location}/bin/cat
...@@ -176,12 +182,12 @@ link-binary = ...@@ -176,12 +182,12 @@ link-binary =
[template-kumofs] [template-kumofs]
< = download-base < = download-base
filename = instance-kumofs.cfg.in filename = instance-kumofs.cfg.in
md5sum = 5ebf310655d5de27da039d71a63d209b md5sum = 00e29ccfdd2b482f7d0db35a85af2877
[template-cloudooo] [template-cloudooo]
< = download-base < = download-base
filename = instance-cloudoo.cfg.in filename = instance-cloudoo.cfg.in
md5sum = 48d5e8c3efc5bfd6fc1027b5ebe55e64 md5sum = 050fa6ff4eb397c5d4cb41a9f75afb3f
[template-zope-conf] [template-zope-conf]
< = download-base < = download-base
...@@ -260,6 +266,7 @@ extra-context = ...@@ -260,6 +266,7 @@ extra-context =
key local_bt5_repository local-bt5-repository:list key local_bt5_repository local-bt5-repository:list
key logrotate_location logrotate:location key logrotate_location logrotate:location
key mariadb_location mariadb:location key mariadb_location mariadb:location
key mariadb_resiliency_after_import_script mariadb-resiliency-after-import-script:target
key mesa_location mesa:location key mesa_location mesa:location
key openssl_location openssl:location key openssl_location openssl:location
key sixtunnel_location 6tunnel:location key sixtunnel_location 6tunnel:location
...@@ -428,22 +435,6 @@ initialization = ...@@ -428,22 +435,6 @@ initialization =
repository_id_list = list(reversed('''${erp5_repository_list:repository_id_list}'''.split())) repository_id_list = list(reversed('''${erp5_repository_list:repository_id_list}'''.split()))
sys.path[0:0] = ['/'.join(['''${buildout:parts-directory}''', x]) for x in repository_id_list] sys.path[0:0] = ['/'.join(['''${buildout:parts-directory}''', x]) for x in repository_id_list]
[patched-eggs]
recipe = minitage.recipe.egg
eggs =
Acquisition
Products.DCWorkflow
ZODB3
Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic.patch
Acquisition-patch-options = -p1
Acquisition-patch-binary = ${patch:location}/bin/patch
Products.DCWorkflow-patches = ${:_profile_base_location_}/../../component/egg-patch/Products.DCWorkflow/workflow_method.patch
Products.DCWorkflow-patch-options = -p1
Products.DCWorkflow-patch-binary = ${patch:location}/bin/patch
ZODB3-patches = ${:_profile_base_location_}/../../component/egg-patch/ZODB3-3.10.5.patch
ZODB3-patch-options = -p1
ZODB3-patch-binary = ${patch:location}/bin/patch
[eggs] [eggs]
recipe = zc.recipe.egg recipe = zc.recipe.egg
eggs = eggs =
...@@ -459,6 +450,7 @@ eggs = ...@@ -459,6 +450,7 @@ eggs =
Pympler Pympler
SOAPpy SOAPpy
chardet chardet
collective.recipe.template
coverage coverage
elementtree elementtree
erp5diff erp5diff
...@@ -466,6 +458,7 @@ eggs = ...@@ -466,6 +458,7 @@ eggs =
interval interval
ipdb ipdb
Jinja2 Jinja2
jsonschema
mechanize mechanize
paramiko paramiko
ply ply
...@@ -556,6 +549,15 @@ scripts = ...@@ -556,6 +549,15 @@ scripts =
extra-paths = extra-paths =
${erp5:location} ${erp5:location}
# patches for eggs
patch-binary = ${patch:location}/bin/patch
Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic.patch
Acquisition-patch-options = -p1
Products.DCWorkflow-patches = ${:_profile_base_location_}/../../component/egg-patch/Products.DCWorkflow/workflow_method.patch
Products.DCWorkflow-patch-options = -p1
ZODB3-patches = ${:_profile_base_location_}/../../component/egg-patch/ZODB3-3.10.5.patch
ZODB3-patch-options = -p1
[zodbanalyze] [zodbanalyze]
recipe = zc.recipe.egg recipe = zc.recipe.egg
eggs = eggs =
...@@ -592,9 +594,9 @@ scripts = ...@@ -592,9 +594,9 @@ scripts =
[versions] [versions]
# patched eggs # patched eggs
Acquisition = 2.13.8-ZMinitagePatched-AqDynamic Acquisition = 2.13.8+SlapOSPatched001
Products.DCWorkflow = 2.2.4-ZMinitagePatched-WorkflowMethod Products.DCWorkflow = 2.2.4+SlapOSPatched001
ZODB3 = 3.10.5-ZMinitagePatched-ZODB33105 ZODB3 = 3.10.5+SlapOSPatched001
# specify dev version to be sure that an old released version is not used # specify dev version to be sure that an old released version is not used
cloudooo = 1.2.5-dev cloudooo = 1.2.5-dev
...@@ -654,16 +656,16 @@ StructuredText = 2.11.1 ...@@ -654,16 +656,16 @@ StructuredText = 2.11.1
WSGIUtils = 0.7 WSGIUtils = 0.7
apache-libcloud = 0.16.0 apache-libcloud = 0.16.0
astroid = 1.3.2 astroid = 1.3.2
async = 0.6.1
chardet = 2.3.0 chardet = 2.3.0
collective.recipe.template = 1.11
csp-eventlet = 0.7.0 csp-eventlet = 0.7.0
ecdsa = 0.11 ecdsa = 0.11
elementtree = 1.2.7-20070827-preview elementtree = 1.2.6.post20050316
erp5diff = 0.8.1.5 erp5diff = 0.8.1.5
eventlet = 0.16.0 eventlet = 0.16.1
five.formlib = 1.0.4 five.formlib = 1.0.4
five.localsitemanager = 2.0.5 five.localsitemanager = 2.0.5
gitdb = 0.6.1 gitdb = 0.6.4
greenlet = 0.4.5 greenlet = 0.4.5
http-parser = 0.8.3 http-parser = 0.8.3
httplib2 = 0.9 httplib2 = 0.9
...@@ -673,25 +675,23 @@ ipdb = 0.8 ...@@ -673,25 +675,23 @@ ipdb = 0.8
ipython = 2.3.1 ipython = 2.3.1
jsonschema = 2.4.0 jsonschema = 2.4.0
logilab-common = 0.63.2 logilab-common = 0.63.2
minitage.paste = 1.4.6
minitage.recipe.egg = 1.107
numpy = 1.9.1 numpy = 1.9.1
plone.recipe.command = 1.1 plone.recipe.command = 1.1
ply = 3.4 ply = 3.4
polib = 1.0.5 polib = 1.0.6
pprofile = 1.7.2 pprofile = 1.7.2
pycountry = 1.10 pycountry = 1.10
pyflakes = 0.8.1 pyflakes = 0.8.1
pylint = 1.4.0 pylint = 1.4.0
python-ldap = 2.4.18 python-ldap = 2.4.19
python-magic = 0.4.6 python-magic = 0.4.6
python-memcached = 1.53 python-memcached = 1.53
qrcode = 5.1 qrcode = 5.1
restkit = 4.2.2 restkit = 4.2.2
rtjp-eventlet = 0.3.2 rtjp-eventlet = 0.3.2
slapos.recipe.template = 2.6 slapos.recipe.template = 2.6
slapos.toolbox = 0.45.2 slapos.toolbox = 0.45.3
smmap = 0.8.3 smmap = 0.9.0
socketpool = 0.5.3 socketpool = 0.5.3
spyne = 2.11.0 spyne = 2.11.0
suds = 0.4 suds = 0.4
...@@ -704,52 +704,27 @@ xupdate-processor = 0.4 ...@@ -704,52 +704,27 @@ xupdate-processor = 0.4
xfw = 0.10 xfw = 0.10
# Required by: # Required by:
# slapos.toolbox==0.45.2 # slapos.toolbox==0.45.3
GitPython = 0.3.3 GitPython = 0.3.5
# Required by: # Required by:
# Products.CMFCore==2.2.8 # Products.CMFCore==2.2.8
Products.ZSQLMethods = 2.13.4 Products.ZSQLMethods = 2.13.4
# Required by: # Required by:
# slapos.toolbox==0.45.2 # slapos.toolbox==0.45.3
atomize = 0.2.0 atomize = 0.2.0
# Required by: # Required by:
# cloudooo==1.2.5-dev # slapos.toolbox==0.45.3
erp5.util = 0.4.41
# Required by:
# slapos.toolbox==0.45.2
feedparser = 5.1.3 feedparser = 5.1.3
# Required by: # Required by:
# SOAPpy==0.12.0nxd001 # SOAPpy===0.12.0nxd001
fpconst = 0.7.2 fpconst = 0.7.2
# Required by: # Required by:
# minitage.recipe.egg==1.107 # slapos.toolbox==0.45.3
iniparse = 0.4
# Required by:
# minitage.core==2.0.57
minitage = 2.0.67
# Required by:
# minitage.recipe.common==1.90
# minitage.recipe.egg==1.107
minitage.core = 2.0.57
# Required by:
# minitage.recipe.egg==1.107
minitage.recipe.common = 1.90
# Required by:
# minitage.recipe.egg==1.107
ordereddict = 1.1
# Required by:
# slapos.toolbox==0.45.2
paramiko = 1.15.2 paramiko = 1.15.2
# Required by: # Required by:
......
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
parts = parts =
publish-cloudooo-connection-information publish-cloudooo-connection-information
cloudooo-instance cloudooo-instance
resiliency-exclude-file
promise promise
promise-openoffice promise-openoffice
{% if use_ipv6 %}promise-tunnel{% endif %} {% if use_ipv6 %}promise-tunnel{% endif %}
...@@ -52,6 +53,12 @@ url-list = {{ dumps(slapparameter_dict.get('font-url-list', []) | join(' ')) }} ...@@ -52,6 +53,12 @@ url-list = {{ dumps(slapparameter_dict.get('font-url-list', []) | join(' ')) }}
service-folder = ${directory:service} service-folder = ${directory:service}
onetimedownload_path = {{ bin_directory }}/onetimedownload onetimedownload_path = {{ bin_directory }}/onetimedownload
[resiliency-exclude-file]
# Generate rdiff exclude file in case of resiliency
recipe = collective.recipe.template
input = inline: **
output = ${directory:srv}/exporter.exclude
[promise] [promise]
recipe = slapos.cookbook:check_port_listening recipe = slapos.cookbook:check_port_listening
path = ${directory:promise}/cloudooo path = ${directory:promise}/cloudooo
......
...@@ -5,6 +5,7 @@ parts += ...@@ -5,6 +5,7 @@ parts +=
publish-kumofs-connection-information publish-kumofs-connection-information
kumofs-instance kumofs-instance
logrotate-entry-kumofs logrotate-entry-kumofs
resiliency-exclude-file
promise-kumofs-server promise-kumofs-server
promise-kumofs-server-listen promise-kumofs-server-listen
promise-kumofs-gateway promise-kumofs-gateway
...@@ -73,6 +74,12 @@ services = ${buildout:directory}/etc/run ...@@ -73,6 +74,12 @@ services = ${buildout:directory}/etc/run
promise = ${buildout:directory}/etc/promise promise = ${buildout:directory}/etc/promise
kumofs-data = ${buildout:directory}/srv/kumofs kumofs-data = ${buildout:directory}/srv/kumofs
[resiliency-exclude-file]
# Generate rdiff exclude file in case of resiliency
recipe = collective.recipe.template
input = inline: **
output = ${directory:srv}/exporter.exclude
# Deploy zope promises scripts # Deploy zope promises scripts
[promise-template] [promise-template]
recipe = slapos.cookbook:check_port_listening recipe = slapos.cookbook:check_port_listening
......
...@@ -211,6 +211,12 @@ var = ${buildout:directory}/var ...@@ -211,6 +211,12 @@ var = ${buildout:directory}/var
log = ${:var}/log log = ${:var}/log
run = ${:var}/run run = ${:var}/run
[resiliency-exclude-file]
# Generate rdiff exclude file in case of resiliency
recipe = collective.recipe.template
input = inline: srv/mariadb/**
output = ${directory:srv}/exporter.exclude
[promise] [promise]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
command-line = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened" "${my-cnf-parameters:ip}" "${my-cnf-parameters:port}" command-line = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened" "${my-cnf-parameters:ip}" "${my-cnf-parameters:port}"
...@@ -225,5 +231,6 @@ parts += ...@@ -225,5 +231,6 @@ parts +=
binary-link binary-link
update-mysql update-mysql
mysqld mysqld
resiliency-exclude-file
promise promise
{{ part_list | join('\n ') }} {{ part_list | join('\n ') }}
#!/bin/sh
# DO NOT RUN THIS SCRIPT ON PRODUCTION INSTANCE
# OR MYSQL DATA WILL BE ERASED.
# This script will import the dump of the mysql database to the real
# database. It is launched by the clone (importer) instance of webrunner
# in the end of the import script.
# Depending on the output, it will create a file containing
# the status of the restoration (success or failure)
mysql_executable="${mariadb-instance:mysql-binary}"
mysqldump_executable="${binary-wrap-mysqldump:wrapper-path}"
mariadb_data_directory="${directory:mariadb-data}"
mariadb_backup_directory="${directory:mariadb-backup-full}"
instance_directory="${buildout:directory}"
pid_file="${mariadb-instance:pid-file}"
binlog_path="${mariadb-instance:binlog-path}"
# Make sure mariadb is not already running
if [ -e "$pid_file" ]; then
pid=$(cat $pid_file) > /dev/null 2>&1
if kill -0 "$pid"; then
echo "Mariadb is already running with pid $pid. Aborting."
exit 1
fi
fi
echo "Deleting existing database..."
rm -r $mariadb_data_directory/*
echo "Adapting binlog database to new paths..."
new_binlog_directory="$(dirname $binlog_path)"
binlog_index_file="$new_binlog_directory/binlog.index"
old_binlog_directory="$(dirname $(head -n 1 $binlog_index_file))"
sed -e "s|$old_binlog_directory|$new_binlog_directory|g" $binlog_index_file > $binlog_index_file
echo "Starting mariadb..."
# XXX hardcoded
$instance_directory/etc/run/mariadb &
mysqld_pid=$!
$instance_directory/etc/run/mariadb_update > /dev/null 2>&1
echo "Importing data..."
# Use latest dump XXX can contain funny characters
dump=$(ls -r $mariadb_backup_directory | head -1)
zcat "$mariadb_backup_directory/$dump" | $mysql_executable -u root --socket="$instance_directory/var/run/mariadb.sock"
RESTORE_EXIT_CODE=$?
kill "$mysqld_pid"
if [ $RESTORE_EXIT_CODE -eq 0 ]; then
echo 'Backup restoration successfully completed.'
else
echo 'Backup restoration failed.'
fi
exit $RESTORE_EXIT_CODE
...@@ -119,7 +119,7 @@ mode = 0644 ...@@ -119,7 +119,7 @@ mode = 0644
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename} url = ${:_profile_base_location_}/${:filename}
download-only = true download-only = true
md5sum = 5f1b93ccdea7c3031aef396154c64938 md5sum = 6c84a826778cb059754623f39b33651b
destination = ${buildout:parts-directory}/monitor-template-rss-bin destination = ${buildout:parts-directory}/monitor-template-rss-bin
filename = status2rss.py filename = status2rss.py
mode = 0644 mode = 0644
......
...@@ -34,9 +34,12 @@ for row in rows: ...@@ -34,9 +34,12 @@ for row in rows:
event_time = datetime.datetime.fromtimestamp(line_timestamp).strftime('%Y-%m-%d %H:%M:%S') event_time = datetime.datetime.fromtimestamp(line_timestamp).strftime('%Y-%m-%d %H:%M:%S')
individual_rows = db.execute("select status, element, output from individual_status where timestamp=?", (line_timestamp,))
description = '\n'.join(['%s: %s %s' % row for row in individual_rows])
rss_item = PyRSS2Gen.RSSItem( rss_item = PyRSS2Gen.RSSItem(
title = status, title = status,
description = "%s: %s" % (event_time, status), description = "%s: %s\n%s" % (event_time, status, description),
link = LINK, link = LINK,
pubDate = event_time, pubDate = event_time,
guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (event_time, status))) guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (event_time, status)))
......
...@@ -38,7 +38,7 @@ eggs = collective.recipe.template ...@@ -38,7 +38,7 @@ eggs = collective.recipe.template
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/pbsready.cfg.in url = ${:_profile_base_location_}/pbsready.cfg.in
output = ${buildout:directory}/pbsready.cfg output = ${buildout:directory}/pbsready.cfg
md5sum = e89d8378cc610704b518a89b095d3a19 md5sum = cbc5bdb360fb5c72418dba03135df526
mode = 0644 mode = 0644
[pbsready-import] [pbsready-import]
...@@ -63,7 +63,7 @@ mode = 0644 ...@@ -63,7 +63,7 @@ mode = 0644
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pull-backup.cfg.in url = ${:_profile_base_location_}/instance-pull-backup.cfg.in
output = ${buildout:directory}/instance-pull-backup.cfg output = ${buildout:directory}/instance-pull-backup.cfg
md5sum = 3866b0d4d2872f693b7d9519a668e6bc md5sum = 0e6a95e7a9b38d402f94c11b7d10397e
mode = 0644 mode = 0644
[template-replicated] [template-replicated]
...@@ -92,7 +92,7 @@ output = ${buildout:directory}/instance-frozen.cfg ...@@ -92,7 +92,7 @@ output = ${buildout:directory}/instance-frozen.cfg
[resilient-web-takeover-cgi-script-download] [resilient-web-takeover-cgi-script-download]
recipe = slapos.recipe.download recipe = slapos.recipe.download
url = ${:_profile_base_location_}/resilient-web-takeover-cgi-script.py.in url = ${:_profile_base_location_}/resilient-web-takeover-cgi-script.py.in
md5sum = e6262c5cf9b1c4d1ea4d959fdcbe3070 md5sum = 5c90dae56885eeb490cc5d7f82d2dc5b
mode = 0644 mode = 0644
destination = ${buildout:directory}/resilient-web-takeover-cgi-script.py.in destination = ${buildout:directory}/resilient-web-takeover-cgi-script.py.in
...@@ -121,11 +121,6 @@ mode = 0644 ...@@ -121,11 +121,6 @@ mode = 0644
find-links = http://www.nexedi.org/static/packages/source/rdiff-backup-1.3.4nxd2.tar.gz find-links = http://www.nexedi.org/static/packages/source/rdiff-backup-1.3.4nxd2.tar.gz
[versions] [versions]
# Pin Jinja2 to 2.6, as 2.7 breaks current code
Jinja2 = 2.6
# ... And newer s.r.template requires Jinja2 >= 2.7
slapos.recipe.template = 2.4.2
rdiff-backup = 1.3.4nxd2 rdiff-backup = 1.3.4nxd2
slapos.cookbook = 0.92
...@@ -83,6 +83,7 @@ notifier-callbacks = $${basedirectory:notifier}/callbacks ...@@ -83,6 +83,7 @@ notifier-callbacks = $${basedirectory:notifier}/callbacks
[equeue] [equeue]
recipe = slapos.cookbook:equeue recipe = slapos.cookbook:equeue
socket = $${basedirectory:run}/equeue.sock socket = $${basedirectory:run}/equeue.sock
lockfile = $${basedirectory:run}/equeue.lock
log = $${basedirectory:log}/equeue.log log = $${basedirectory:log}/equeue.log
database = $${rootdirectory:srv}/equeue.db database = $${rootdirectory:srv}/equeue.db
wrapper = $${basedirectory:services}/equeue wrapper = $${basedirectory:services}/equeue
......
...@@ -157,6 +157,7 @@ rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup ...@@ -157,6 +157,7 @@ rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
[equeue] [equeue]
recipe = slapos.cookbook:equeue recipe = slapos.cookbook:equeue
socket = $${basedirectory:run}/equeue.sock socket = $${basedirectory:run}/equeue.sock
lockfile = $${basedirectory:run}/equeue.lock
log = $${basedirectory:log}/equeue.log log = $${basedirectory:log}/equeue.log
database = $${rootdirectory:srv}/equeue.db database = $${rootdirectory:srv}/equeue.db
wrapper = $${basedirectory:services}/equeue wrapper = $${basedirectory:services}/equeue
......
#!${buildout:executable} #!${buildout:executable}
equeue_database = '${equeue:database}'
equeue_lockfile = '${equeue:lockfile}'
import cgi import cgi
import cgitb import cgitb
import datetime
import gdbm
import os import os
import shutil
import subprocess import subprocess
import sys import sys
import tempfile
cgitb.enable() cgitb.enable()
def getLatestBackupDate():
"""
Get the date of the latest successful backup.
"""
# Create a copy of the db (locked by equeue process)
temporary_directory = tempfile.mkdtemp()
equeue_database_copy = os.path.join(temporary_directory, 'equeue.db')
shutil.copyfile(equeue_database, equeue_database_copy)
db = gdbm.open(equeue_database_copy)
# Usually, there is only one callback (so only one key
# in the db), but if there are several:
# Take the "oldest" one (oldest value).
last_backup = db[db.keys()[0]]
for callback in db.keys():
timestamp = float(db[callback])
if timestamp < last_backup:
last_backup = timestamp
return datetime.datetime.fromtimestamp(last_backup)
def isBackupInProgress():
"""
Check if backup is in progress (importer script is running)
by checking if equeue lockfile exists.
"""
# XXX: check if file is valid
return os.path.exists(equeue_lockfile)
print "Content-Type: text/html" print "Content-Type: text/html"
print print
...@@ -17,12 +52,14 @@ if "password" not in form: ...@@ -17,12 +52,14 @@ if "password" not in form:
<p>Calling takeover will stop and freeze the current main instance, and make this clone instance the new main instance, replacing the old one.</p> <p>Calling takeover will stop and freeze the current main instance, and make this clone instance the new main instance, replacing the old one.</p>
<p><b>Warning: submit the form only if you understand what you are doing.</b></p> <p><b>Warning: submit the form only if you understand what you are doing.</b></p>
<p>Note: the password asked here can be found within the parameters of your SlapOS instance page.</p> <p>Note: the password asked here can be found within the parameters of your SlapOS instance page.</p>
<p>Last valid backup: %s</p>
<p>Importer script(s) of backup in progress: %s</p>
<form action="/"> <form action="/">
Password: <input type="text" name="password"> Password: <input type="text" name="password">
<input type="submit" value="Take over" style="background: red;"> <input type="submit" value="Take over" style="background: red;">
</form> </form>
</body> </body>
</html>""" </html>""" % (getLatestBackupDate().strftime('%Y-%m-%d %H:%M:%S'), isBackupInProgress())
sys.exit(0) sys.exit(0)
if form['password'].value != '${:password}': if form['password'].value != '${:password}':
......
...@@ -20,8 +20,9 @@ extensions += ...@@ -20,8 +20,9 @@ extensions +=
# Use shacache and lxml # Use shacache and lxml
extends = extends =
../component/python-2.7/buildout.cfg ../component/git/buildout.cfg
../component/lxml-python/buildout.cfg ../component/lxml-python/buildout.cfg
../component/python-2.7/buildout.cfg
../component/python-cffi/buildout.cfg ../component/python-cffi/buildout.cfg
../component/python-cliff/buildout.cfg ../component/python-cliff/buildout.cfg
../component/python-cryptography/buildout.cfg ../component/python-cryptography/buildout.cfg
...@@ -102,9 +103,9 @@ eggs = ...@@ -102,9 +103,9 @@ eggs =
[versions] [versions]
# Use SlapOS patched zc.buildout # Use SlapOS patched zc.buildout
zc.buildout = 1.7.1-dev-SlapOS-005 zc.buildout = 1.7.1.post6
# Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2) # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
zc.recipe.egg = 1.3.2nxd001 zc.recipe.egg = 1.3.2.post2
# Use own version of h.r.download to be able to open .xz and .lz archives # Use own version of h.r.download to be able to open .xz and .lz archives
hexagonit.recipe.download = 1.7nxd003 hexagonit.recipe.download = 1.7nxd003
...@@ -115,6 +116,7 @@ buildout-versions = 1.7 ...@@ -115,6 +116,7 @@ buildout-versions = 1.7
cffi = 0.8.6 cffi = 0.8.6
cmd2 = 0.6.8 cmd2 = 0.6.8
cryptography = 0.7.1 cryptography = 0.7.1
inotifyx = 0.2.2
itsdangerous = 0.24 itsdangerous = 0.24
lxml = 3.4.1 lxml = 3.4.1
meld3 = 1.0.0 meld3 = 1.0.0
...@@ -122,19 +124,20 @@ mr.developer = 1.31 ...@@ -122,19 +124,20 @@ mr.developer = 1.31
netaddr = 0.7.13 netaddr = 0.7.13
pbr = 0.10.7 pbr = 0.10.7
prettytable = 0.7.2 prettytable = 0.7.2
psutil = 2.1.3 psutil = 2.2.0
pyOpenSSL = 0.14 pyOpenSSL = 0.14
pyparsing = 2.0.3 pyparsing = 2.0.3
pytz = 2014.10
requests = 2.5.1 requests = 2.5.1
setuptools = 7.0 setuptools = 11.3.1
simplejson = 3.6.5 simplejson = 3.6.5
six = 1.9.0 six = 1.9.0
slapos.cookbook = 0.87 slapos.cookbook = 0.93
slapos.core = 1.3.5 slapos.core = 1.3.6
slapos.libnetworkcache = 0.14.2 slapos.libnetworkcache = 0.14.2
slapos.recipe.build = 0.16 slapos.recipe.build = 0.16
slapos.recipe.cmmi = 0.2 slapos.recipe.cmmi = 0.2
stevedore = 1.1.0 stevedore = 1.2.0
xml-marshaller = 0.9.7 xml-marshaller = 0.9.7
# Required by: # Required by:
...@@ -143,7 +146,7 @@ Flask = 0.10.1 ...@@ -143,7 +146,7 @@ Flask = 0.10.1
# Required by: # Required by:
# cliff==1.9.0 # cliff==1.9.0
# stevedore==1.1.0 # stevedore==1.2.0
argparse = 1.3.0 argparse = 1.3.0
# Required by: # Required by:
...@@ -151,15 +154,15 @@ argparse = 1.3.0 ...@@ -151,15 +154,15 @@ argparse = 1.3.0
cliff = 1.9.0 cliff = 1.9.0
# Required by: # Required by:
# cryptography==0.7 # cryptography==0.7.1
enum34 = 1.0.4 enum34 = 1.0.4
# Required by: # Required by:
# slapos.cookbook==0.87 # slapos.cookbook==0.93
inotifyx = 0.2.2 jsonschema = 2.4.0
# Required by: # Required by:
# slapos.cookbook==0.87 # slapos.cookbook==0.93
lock-file = 2.0 lock-file = 2.0
# Required by: # Required by:
...@@ -171,17 +174,13 @@ netifaces = 0.10.4 ...@@ -171,17 +174,13 @@ netifaces = 0.10.4
pip = 6.0.6 pip = 6.0.6
# Required by: # Required by:
# cryptography==0.7 # cryptography==0.7.1
pyasn1 = 0.1.7 pyasn1 = 0.1.7
# Required by: # Required by:
# cffi==0.8.6 # cffi==0.8.6
pycparser = 2.10 pycparser = 2.10
# Required by:
# slapos.cookbook==0.87
pytz = 2014.10
# Required by: # Required by:
# slapos.core==1.3.5 # slapos.core==1.3.5
supervisor = 3.1.3 supervisor = 3.1.3
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment