diff --git a/component/apache/buildout.cfg b/component/apache/buildout.cfg index 598549e57986088a6b6f110e7d4f18f1d7a336a8..72f4b67b8175a2db5a410971f5e666d9d095e42a 100644 --- a/component/apache/buildout.cfg +++ b/component/apache/buildout.cfg @@ -120,8 +120,9 @@ make-targets = [apache-2.2] # inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/ recipe = slapos.recipe.cmmi -url = http://mir2.ovh.net/ftp.apache.org/dist//httpd/httpd-2.2.24.tar.gz -md5sum = 64a3392018ad60583209a16d728180d3 +version = 2.2.25 +url = http://mir2.ovh.net/ftp.apache.org/dist/httpd/httpd-${:version}.tar.bz2 +md5sum = 9ebe3070c0bb4311f21a0cd0e34f0045 patch-options = -p1 configure-options = --disable-static --enable-authn-alias @@ -182,6 +183,8 @@ environment = recipe = slapos.recipe.cmmi url = http://sourceforge.net/projects/mod-antiloris/files/mod_antiloris-0.4.tar.bz2/download md5sum = 66862bf10e9be3a023e475604a28a0b4 +depends = + ${apache-2.2:version} configure-command = ${apache-2.2:location}/bin/apxs configure-options = -c mod_antiloris.c make-binary = ${:configure-command} diff --git a/component/nodejs/buildout.cfg b/component/nodejs/buildout.cfg index 4b9c2d1cae8e7dab97f1fd72633e8a923fd90a3d..59efa3c7f942e52fbf5869a1ff2f76a275c6f243 100644 --- a/component/nodejs/buildout.cfg +++ b/component/nodejs/buildout.cfg @@ -14,8 +14,8 @@ parts = [nodejs-0.8] # Server-side Javascript. recipe = slapos.recipe.cmmi -url = http://nodejs.org/dist/v0.8.14/node-v0.8.14.tar.gz -md5sum = 284fd2c7578064c339d9cf6a3a475ac7 +url = http://nodejs.org/dist/v0.8.23/node-v0.8.23.tar.gz +md5sum = 22fe54ac365f52d3d80ecf748e7323d5 configure-options = --openssl-includes=${openssl:location}/include --openssl-libpath=${openssl:location}/lib diff --git a/setup.py b/setup.py index e587569190f85ed07e78a375b6ade2ae48c71063..369bfc7c91484bc762f3d14bd081d05aaa2a7dca 100755 --- a/setup.py +++ b/setup.py @@ -188,6 +188,8 @@ setup(name=name, 'slapreport = slapos.recipe.slapreport:Recipe', 'slaprunner = slapos.recipe.slaprunner:Recipe', 'slaprunner.test = slapos.recipe.slaprunner:Test', + 'slaprunner.export = slapos.recipe.slaprunner.backup:ExportRecipe', + 'slaprunner.import = slapos.recipe.slaprunner.backup:ImportRecipe', 'softwaretype = slapos.recipe.softwaretype:Recipe', 'sphinx= slapos.recipe.sphinx:Recipe', 'sshkeys_authority = slapos.recipe.sshkeys_authority:Recipe', diff --git a/slapos/recipe/neoppod.py b/slapos/recipe/neoppod.py index 62f440b7bccb1d62c3a88778385f47bf6ab10ab9..90926c2db96386e9074d6653f8081e1343cfe730 100644 --- a/slapos/recipe/neoppod.py +++ b/slapos/recipe/neoppod.py @@ -50,8 +50,6 @@ class NeoBaseRecipe(GenericBaseRecipe): #'-n', options['name'], '-c', options['cluster'], ] - if options['verbose']: - option_list.append('-v') option_list.extend(self._getOptionList()) return [self.createPythonScript( options['wrapper'], diff --git a/slapos/recipe/slaprunner/backup.py b/slapos/recipe/slaprunner/backup.py new file mode 100644 index 0000000000000000000000000000000000000000..e8ba31641b0c6758d3bdab8b78f4f51e554f18ec --- /dev/null +++ b/slapos/recipe/slaprunner/backup.py @@ -0,0 +1,133 @@ +############################################################################## +# +# Copyright (c) 2013 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## + +import textwrap + +from slapos.recipe.librecipe import GenericBaseRecipe + + + +class ExportRecipe(GenericBaseRecipe): + """\ + This recipe creates an exporter script for using with the resilient stack. + + Required options: + backup-directory + folder that will contain the dump file. + srv-directory + folder that contain the runner directory. + wrapper + full path of the exporter script to create. + """ + + def install(self): + wrapper = self.options['wrapper'] + self.createBackupScript(wrapper) + return [wrapper] + + + def createBackupScript(self, wrapper): + """\ + Create a script to backup the database in 'custom' format. + """ + content = textwrap.dedent("""\ + #!%(shell-binary)s + umask 077 + sync_element () { + path=$1 + backup_path=$2 + shift 2 + element_list=$* + for element in $element_list + do + cd $path; + if [ -f $element ] || [ -d $element ]; then + %(rsync-binary)s -avz --safe-links --delete $element $backup_path; + fi + done + } + sync_element %(srv-directory)s/runner %(backup-directory)s/runner/ instance project proxy.db softwareLink + sync_element %(etc-directory)s %(backup-directory)s/etc/ .rcode .project .users ssh + if [ -d %(backup-directory)s/runner/software ]; then + rm %(backup-directory)s/runner/software/* + fi + """ % self.options) + self.createExecutable(wrapper, content=content) + + + +class ImportRecipe(GenericBaseRecipe): + """\ + This recipe creates an importer script for using with the resilient stack. + + Required options: + backup-directory + folder that will contain the dump file. + srv-directory + folder that contain the runner directory. + wrapper + full path of the exporter script to create. + """ + + def install(self): + wrapper = self.options['wrapper'] + self.createRestoreScript(wrapper) + return [wrapper] + + + def createRestoreScript(self, wrapper): + """\ + Create a script to restore the database from 'custom' format. + """ + content = textwrap.dedent("""\ + #!%(shell-binary)s + umask 077 + restore_element () { + backup_path=$1 + restore_path=$2 + shift 2 + element_list=$* + for element in $element_list + do + cd $backup_path; + if [ -f $element ] || [ -d $element ]; then + %(rsync-binary)s -avz --delete $backup_path/$element $restore_path; + fi + done + } + restore_element %(backup-directory)s/runner/ %(srv-directory)s/runner instance project proxy.db softwareLink + restore_element %(backup-directory)s/etc/ %(etc-directory)s .rcode .project .users ssh + ifs=$IFS IFS=';' + read user pass remaining < %(etc-directory)s/.users + IFS=$ifs + %(curl-binary)s -vg6L -F clogin="$user" -F cpwd="$pass" --dump-header login_cookie %(backend-url)s/doLogin; + %(curl-binary)s -vg6LX POST --cookie login_cookie --max-time 5 %(backend-url)s/runSoftwareProfile; + rm -f login_cookie + """ % self.options) + self.createExecutable(wrapper, content=content) + + diff --git a/software/neoppod/README.txt b/software/neoppod/README.txt index 6260e2e0b43aa296de94233b8f9d7d92065c1c60..abb8b69cfd65a0401d8597a296708283f6cba61c 100644 --- a/software/neoppod/README.txt +++ b/software/neoppod/README.txt @@ -37,11 +37,6 @@ Defautls to 12. Number of replicates. Defaults to 0 (no resilience). -'verbose' (int, optional) -~~~~~~~~~~~~~~~~~~~~~~~~~ -Non-zero to increase verbosity. -Defaults to 0. - 'mysql-storage-count' (int, optional) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Number of MySQL-based storage nodes to deploy. One master node is deployed diff --git a/software/neoppod/cluster.cfg.in b/software/neoppod/cluster.cfg.in index a83d416eb0e9e60ebec0af08f0c28fbb36552c06..74bd0284f94e821ec5aa370d4c76e44919507b73 100644 --- a/software/neoppod/cluster.cfg.in +++ b/software/neoppod/cluster.cfg.in @@ -19,10 +19,9 @@ partition-id = ${slap-connection:partition-id} config-cluster = {{ slapparameter_dict['cluster'] }} config-partitions = {{ slapparameter_dict.get('partitions', 12) }} config-replicas = {{ slapparameter_dict.get('replicas', 0) }} -config-verbose = {{ dumps(slapparameter_dict.get('verbose', 0)) }} [request-config] -base = cluster verbose +base = cluster master = partitions replicas node = masters diff --git a/software/neoppod/instance-neo-admin.cfg.in b/software/neoppod/instance-neo-admin.cfg.in index 7a5eab2a1805c01a0e228e0e897ef1bfab27a1c2..7c896f5e7e6f1e435fecf6057eba6819c5756d7c 100644 --- a/software/neoppod/instance-neo-admin.cfg.in +++ b/software/neoppod/instance-neo-admin.cfg.in @@ -16,7 +16,6 @@ ip = ${publish:ip} port = ${publish:port} cluster = ${slap-parameter:cluster} masters = ${slap-parameter:masters} -verbose = ${slap-parameter:verbose} [publish] recipe = slapos.cookbook:publish @@ -33,6 +32,7 @@ link-binary = ${neo-admin:wrapper} [directory] recipe = slapos.cookbook:mkdirectory +promises = ${buildout:directory}/etc/promises run = ${buildout:directory}/etc/run log = ${buildout:directory}/var/log bin = ${buildout:directory}/bin diff --git a/software/neoppod/instance-neo-master.cfg.in b/software/neoppod/instance-neo-master.cfg.in index 9cedb58285efab1f83a4a28e91e3eb1c8ade1228..82bd2694427daf637ffa819d850a5b618d35631a 100644 --- a/software/neoppod/instance-neo-master.cfg.in +++ b/software/neoppod/instance-neo-master.cfg.in @@ -17,7 +17,6 @@ cluster = ${slap-parameter:cluster} partitions = ${slap-parameter:partitions} replicas = ${slap-parameter:replicas} masters = ${slap-parameter:masters} -verbose = ${slap-parameter:verbose} [publish] recipe = slapos.cookbook:publish @@ -34,6 +33,7 @@ link-binary = ${neo-master:wrapper} [directory] recipe = slapos.cookbook:mkdirectory +promises = ${buildout:directory}/etc/promises run = ${buildout:directory}/etc/run log = ${buildout:directory}/var/log bin = ${buildout:directory}/bin diff --git a/software/neoppod/instance-neo-storage-mysql.cfg.in b/software/neoppod/instance-neo-storage-mysql.cfg.in index 88af462fd31995c7f3bfc47292818b4e8777d194..3047b1a788a50bec1d61439d4714133e1bc9e9a3 100644 --- a/software/neoppod/instance-neo-storage-mysql.cfg.in +++ b/software/neoppod/instance-neo-storage-mysql.cfg.in @@ -10,6 +10,7 @@ database = neo conf-file = ${directory:etc}/mariadb.cnf # No networking (ip & port options) data-directory = ${directory:srv_mariadb} +bin-directory = ${directory:bin} pid-file = ${directory:var_run}/mariadb.pid socket = ${directory:var_run}/mariadb.sock error-log = ${directory:log}/mariadb_error.log @@ -33,7 +34,6 @@ logfile = ${directory:log}/neostorage.log ip = [${slap-network-information:global-ipv6}] cluster = ${slap-parameter:cluster} masters = ${slap-parameter:masters} -verbose = ${slap-parameter:verbose} database-adapter = MySQL database-parameters = ${mariadb-instance:user}:${mariadb-instance:password}@${mariadb-instance:database}${mariadb-instance:socket} wait-database = 60 diff --git a/software/neoppod/software.cfg b/software/neoppod/software.cfg index 9f1878c840f18b8874db493d5ec961adac801978..d9823ede2bdd344d15fa8bc29c29e738bab82511 100644 --- a/software/neoppod/software.cfg +++ b/software/neoppod/software.cfg @@ -1,6 +1,3 @@ -# Note on COOKBOOK/END COOKBOOK: the areas delimited by those tags should be -# dropped when a cookbook version containing neoppod recipe is released as an -# egg on pypi. # Note on LXML/END LXML: they delimit areas where lxml magic is needed. lxml is # a slapos.cookbook dependency, so it should be fetched automatically. But when # automatically fetched, it gets built against system headers/libs, which is @@ -10,9 +7,6 @@ [buildout] extends = ../../stack/slapos.cfg -#COOKBOOK - ../../component/git/buildout.cfg -#END COOKBOOK #LXML ../../component/lxml-python/buildout.cfg #END LXML @@ -20,19 +14,9 @@ extends = ../../component/mariadb/buildout.cfg ../../component/mysql-python/buildout.cfg -develop = -#COOKBOOK -# Note: as we develop a path we fetch, first buildout execution will fail as -# develop couldn't find the clone. Second (and later) runs will succeed. - ${:parts-directory}/slapos.cookbook-repository -#END COOKBOOK - parts = -#COOKBOOK - slapos.cookbook-repository - check-recipe -#END COOKBOOK slapos-deps-eggs + slapos-cookbook # NEO & dependencies python2.7 mariadb @@ -47,29 +31,12 @@ eggs = #LXML ${lxml-python:egg} #END LXML -#COOKBOOK - slapos.cookbook -#END COOKBOOK #LXML [lxml-python] python = python2.7 #END LXML -#COOKBOOK -[slapos.cookbook-repository] -recipe = slapos.recipe.build:gitclone -repository = http://git.erp5.org/repos/slapos.git -branch = neoppod -git-executable = ${git:location}/bin/git - -[check-recipe] -recipe = plone.recipe.command -stop-on-error = true -update-command = ${:command} -command = test -e ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link -#END COOKBOOK - [base-template] recipe = slapos.recipe.template:jinja2 template = ${:_profile_base_location_}/${:filename}.in @@ -85,12 +52,12 @@ context = [template-neo-master] < = base-template filename = instance-neo-master.cfg -md5sum = b48e9446fb8fdce24d954d7d4f4c7460 +md5sum = 8947620b15535b95163c11a0efab546e [template-neo-storage-mysql] < = base-template filename = instance-neo-storage-mysql.cfg -md5sum = e767c66e7de58232de30bdf26188efe1 +md5sum = 518c8c41ce73f0b608fbb1c3b889ca1b extra-context = key mariadb_location mariadb:location key master_cfg template-neo-master:rendered @@ -98,12 +65,12 @@ extra-context = [template-neo-admin] < = base-template filename = instance-neo-admin.cfg -md5sum = fcf1a5c937a609a934c8ccd7c1326726 +md5sum = 987f69333c563898cab21455509f8024 [template-cluster] recipe = slapos.recipe.build:download url = ${:_profile_base_location_}/cluster.cfg.in -md5sum = fe37b2ff36fee65b44945393f63e51c3 +md5sum = 7177d129e171b20044bf435ed549af43 [template-instance] < = base-template diff --git a/software/slaprunner/TODO.txt b/software/slaprunner/TODO.txt new file mode 100644 index 0000000000000000000000000000000000000000..106a496ac153cab6541aa0fe2b10fbb4b8a48568 --- /dev/null +++ b/software/slaprunner/TODO.txt @@ -0,0 +1,2 @@ +- Use https to connect to the runner. !!! Critical +- resilient sr: Cloned instances should not launch slapgrid-sr if it was not launched on export instance diff --git a/software/slaprunner/common.cfg b/software/slaprunner/common.cfg index 98f094d60715b61673d99b0f74c5b28b32b276e0..3c23fa021cf435800865eb9189267b5cebc060f3 100644 --- a/software/slaprunner/common.cfg +++ b/software/slaprunner/common.cfg @@ -1,31 +1,67 @@ [buildout] extends = + ../../component/bash/buildout.cfg ../../component/cloud9/buildout.cfg + ../../component/curl/buildout.cfg ../../component/dash/buildout.cfg ../../component/dropbear/buildout.cfg ../../component/git/buildout.cfg ../../component/lxml-python/buildout.cfg + ../../component/rsync/buildout.cfg ../../stack/flask.cfg ../../stack/shacache-client.cfg + ../../stack/resilient/buildout.cfg ../../stack/slapos.cfg parts = + slapos.cookbook-repository + rdiff-backup template - slapos-cookbook eggs + instance-runner-import + instance-runner-export + +# slapos-cookbook + [template] recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance.cfg output = ${buildout:directory}/template.cfg mode = 0644 -md5sum = 5307e4200f044ae57b504ad68444491c +#md5sum = 5307e4200f044ae57b504ad68444491c [template-runner] recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance-runner.cfg output = ${buildout:directory}/template-runner.cfg -md5sum = 91d6550c43b7a43a999724af4650ae40 +#md5sum = 91d6550c43b7a43a999724af4650ae40 +mode = 0644 + +[instance-resilient] +recipe = slapos.recipe.template:jinja2 +template = ${:_profile_base_location_}/instance-resilient.cfg.jinja2 +rendered = ${buildout:directory}/instance-resilient.cfg + +context = key buildout buildout:bin-directory + key develop_eggs_directory buildout:develop-eggs-directory + key eggs_directory buildout:eggs-directory + +import-list = file parts template-parts:destination + file replicated template-replicated:destination + +mode = 0644 + +[instance-runner-import] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/instance-runner-import.cfg.in +output = ${buildout:directory}/instance-runner-import.cfg +mode = 0644 + +[instance-runner-export] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/instance-runner-export.cfg.in +output = ${buildout:directory}/instance-runner-export.cfg mode = 0644 [eggs] diff --git a/software/slaprunner/development.cfg b/software/slaprunner/development.cfg index 21f0da315a57bcfc7fd92dacc1a80e339ca965a7..1ee36ab86d65f2d68a9c68a2e31b3bc2cd1d1a11 100644 --- a/software/slaprunner/development.cfg +++ b/software/slaprunner/development.cfg @@ -10,39 +10,43 @@ extends = common.cfg parts += slapos.cookbook-repository - slapos.core-repository - slapos.toolbox-repository - check-recipe + + +# slapos.toolbox-repository + +# slapos.core-repository +# check-recipe develop = ${:parts-directory}/slapos.cookbook-repository - ${:parts-directory}/slapos.core-repository - ${:parts-directory}/slapos.toolbox-repository +# ${:parts-directory}/slapos.toolbox-repository +# ${:parts-directory}/slapos.core-repository -[slapos.toolbox-repository] -recipe = slapos.recipe.build:gitclone -repository = http://git.erp5.org/repos/slapos.toolbox.git -branch = slaprunner -git-executable = ${git:location}/bin/git +#[slapos.toolbox-repository] +#recipe = slapos.recipe.build:gitclone +#repository = http://git.erp5.org/repos/slapos.toolbox.git +#branch = slaprunner-resiliency +#git-executable = ${git:location}/bin/git [slapos.cookbook-repository] recipe = slapos.recipe.build:gitclone repository = http://git.erp5.org/repos/slapos.git -branch = slaprunner -git-executable = ${git:location}/bin/git - -[slapos.core-repository] -recipe = slapos.recipe.build:gitclone -repository = http://git.erp5.org/repos/slapos.core.git -branch = master +branch = slaprunner-resiliency git-executable = ${git:location}/bin/git +#[slapos.core-repository] +#recipe = slapos.recipe.build:gitclone +#repository = http://git.erp5.org/repos/slapos.core.git +#branch = master +#git-executable = ${git:location}/bin/git +# [check-recipe] recipe = plone.recipe.command stop-on-error = true update-command = ${:command} command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link && - grep parts ${buildout:develop-eggs-directory}/slapos.core.egg-link && grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link + +# grep parts ${buildout:develop-eggs-directory}/slapos.core.egg-link && diff --git a/software/slaprunner/instance-resilient.cfg.jinja2 b/software/slaprunner/instance-resilient.cfg.jinja2 new file mode 100644 index 0000000000000000000000000000000000000000..f4c40fb48583574d02e5241d50b8d12dfdf1c7c1 --- /dev/null +++ b/software/slaprunner/instance-resilient.cfg.jinja2 @@ -0,0 +1,57 @@ +# vim: set ft=cfg: + +{% import 'parts' as parts %} +{% import 'replicated' as replicated %} + +[buildout] +eggs-directory = {{ eggs_directory }} +develop-eggs-directory = {{ develop_eggs_directory }} +offline = true + +# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended +parts += + {{ parts.replicate("runner", "3") }} + publish-connection-informations + +{{ replicated.replicate("runner", "3", "runner-export", "runner-import") }} + +# Bubble up the parameters +[request-runner] +return = url ssh-public-key ssh-url notification-id ip backend_url url cloud9_url ssh_command password_recovery_code +config = instance-amount debug domain number authorized-key notify ip-list namebase runner1-computer-guid pbs-runner1-computer-guid runner2-computer-guid pbs-runner2-computer-guid runner3-computer-guid pbs-runner3-computer-guid +# XXX Cedric LN Ugly hack, resilient stack and slaprunner stack sharing too much ssh sections +config-authorized-key = ${request-pbs-runner-1:connection-ssh-key} ${request-pbs-runner-2:connection-ssh-key} ${slap-parameter:authorized-key} +config-instance-amount = ${slap-parameter:instance-amount} +config-debug = ${slap-parameter:debug} +config-runner1-computer-guid = ${slap-parameter:runner1-computer-guid} +config-pbs-runner1-computer-guid = ${slap-parameter:pbs-runner1-computer-guid} +config-runner2-computer-guid = ${slap-parameter:runner2-computer-guid} +config-pbs-runner2-computer-guid = ${slap-parameter:pbs-runner2-computer-guid} +config-runner3-computer-guid = ${slap-parameter:runner3-computer-guid} +config-pbs-runner3-computer-guid = ${slap-parameter:pbs-runner3-computer-guid} +config-domain = ${slap-parameter:domain} + + +[publish-connection-informations] +recipe = slapos.cookbook:publish +backend_url = ${request-runner:connection-backend_url} +url = ${request-runner:connection-url} +cloud9_url = ${request-runner:connection-cloud9_url} +ssh_command = ${request-runner:connection-ssh_command} +password_recovery_code = ${request-runner:connection-password_recovery_code} + +[slap-parameter] +# Default parameters for distributed deployment +# I.e state "backup1 of maria should go there, ..." +# XXX-Cedric: Hardcoded number of backups. Should be dynamically generated. +runner1-computer-guid = +pbs-runner1-computer-guid = +runner2-computer-guid = +pbs-runner2-computer-guid = +runner3-computer-guid = +pbs-runner3-computer-guid = +# XXX-Cedric: Hardcoded parameters. Should be dynamically generated. +domain = +authorized-key = +instance-amount = 10 +debug = false diff --git a/software/slaprunner/instance-runner-export.cfg.in b/software/slaprunner/instance-runner-export.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..31aea46bd7c430c0de18b002be6b15817f318fca --- /dev/null +++ b/software/slaprunner/instance-runner-export.cfg.in @@ -0,0 +1,27 @@ +[buildout] +extends = ${template-runner:output} + ${pbsready-export:output} + +parts += + urls + slaprunner + cron-entry-backup + +[exporter] +recipe = slapos.cookbook:slaprunner.export +wrapper = $${directory:bin}/$${slap-parameter:namebase}-exporter +srv-directory = $${directory:srv} +etc-directory = $${directory:etc} +backup-directory = $${directory:backup} +shell-binary = ${dash:location}/bin/dash +rsync-binary = ${rsync:location}/bin/rsync + + +# Extends publish section with resilient parameters +[urls] +<= resilient-publish-connection-parameter +backend_url = $${slaprunner:access-url} +url = $${request-frontend:connection-site_url} +cloud9_url = $${cloud9:access-url} +ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port} +password_recovery_code = $${recovery-code:passwd} diff --git a/software/slaprunner/instance-runner-import.cfg.in b/software/slaprunner/instance-runner-import.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..9845d10369894332d90f8c94b2dbaaee08338760 --- /dev/null +++ b/software/slaprunner/instance-runner-import.cfg.in @@ -0,0 +1,20 @@ +[buildout] +extends = ${template-runner:output} + ${pbsready-import:output} + + +parts += + slaprunner +# have to repeat the next one, as it's not inherited from pbsready-import + import-on-notification + +[importer] +recipe = slapos.cookbook:slaprunner.import +wrapper = $${directory:bin}/$${slap-parameter:namebase}-importer +srv-directory = $${directory:srv} +etc-directory = $${directory:etc} +backup-directory = $${directory:backup} +shell-binary = ${dash:location}/bin/dash +rsync-binary = ${rsync:location}/bin/rsync +curl-binary = ${curl:location}/bin/curl +backend-url = $${slaprunner:access-url} diff --git a/software/slaprunner/instance-runner.cfg b/software/slaprunner/instance-runner.cfg index d69ebf0191223df7306f7ebc6522c8bf34f71901..34dbb51114c69b2ed06effe9042766efea75ff43 100644 --- a/software/slaprunner/instance-runner.cfg +++ b/software/slaprunner/instance-runner.cfg @@ -3,7 +3,7 @@ parts = cloud9 slaprunner test-runner - sshkeys-dropbear + sshkeys-dropbear-runner dropbear-server-add-authorized-key sshkeys-authority publish-connection-informations @@ -13,7 +13,6 @@ parts = dropbear-promise symlinks - eggs-directory = ${buildout:eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory} offline = true @@ -82,9 +81,9 @@ instance-directory = $${runnerdirectory:instance-root} etc_dir = $${directory:etc} log_dir = $${directory:log} run_dir = $${directory:run} -ssh_client = $${sshkeys-dropbear:wrapper} -public_key = $${sshkeys-dropbear:public-key} -private_key = $${sshkeys-dropbear:private-key} +ssh_client = $${sshkeys-dropbear-runner:wrapper} +public_key = $${sshkeys-dropbear-runner:public-key} +private_key = $${sshkeys-dropbear-runner:private-key} ipv4 = $${slap-network-information:local-ipv4} ipv6 = $${slap-network-information:global-ipv6} proxy_port = 50000 @@ -121,28 +120,28 @@ keys-directory = $${sshkeys-directory:keys} wrapper = $${directory:services}/sshkeys_authority keygen-binary = ${dropbear:location}/bin/dropbearkey -[dropbear-server] +[dropbear-runner-server] recipe = slapos.cookbook:dropbear host = $${slap-network-information:global-ipv6} -port = 2222 +port = 22222 home = $${directory:ssh} -wrapper = $${directory:bin}/raw_sshd -shell = /bin/bash +wrapper = $${directory:bin}/runner_sshd +shell = ${bash:location}/bin/bash rsa-keyfile = $${directory:ssh}/server_key.rsa dropbear-binary = ${dropbear:location}/sbin/dropbear -[sshkeys-dropbear] +[sshkeys-dropbear-runner] <= sshkeys-authority recipe = slapos.cookbook:sshkeys_authority.request name = dropbear type = rsa -executable = $${dropbear-server:wrapper} -public-key = $${dropbear-server:rsa-keyfile}.pub -private-key = $${dropbear-server:rsa-keyfile} -wrapper = $${directory:services}/sshd +executable = $${dropbear-runner-server:wrapper} +public-key = $${dropbear-runner-server:rsa-keyfile}.pub +private-key = $${dropbear-runner-server:rsa-keyfile} +wrapper = $${directory:services}/runner_sshd [dropbear-server-add-authorized-key] -<= dropbear-server +<= dropbear-runner-server recipe = slapos.cookbook:dropbear.add_authorized_key key = $${slap-parameter:authorized-key} @@ -166,7 +165,7 @@ recipe = slapos.cookbook:publish backend_url = $${slaprunner:access-url} url = $${request-frontend:connection-site_url} cloud9_url = $${cloud9:access-url} -ssh_command = ssh $${dropbear-server:host} -p $${dropbear-server:port} +ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port} password_recovery_code = $${recovery-code:passwd} @@ -194,8 +193,8 @@ curl_path = ${curl:location}/bin/curl [dropbear-promise] recipe = slapos.cookbook:check_port_listening path = $${directory:promises}/dropbear -hostname = $${dropbear-server:host} -port = $${dropbear-server:port} +hostname = $${dropbear-runner-server:host} +port = $${dropbear-runner-server:port} [symlinks] recipe = cns.recipe.symlink diff --git a/software/slaprunner/instance.cfg b/software/slaprunner/instance.cfg index 446a7a1983eb4ac3b50151c538856da807c06b09..2b0681dd3a031c0570aaef9f5cb6afa474d55a68 100644 --- a/software/slaprunner/instance.cfg +++ b/software/slaprunner/instance.cfg @@ -8,4 +8,11 @@ offline = true [switch_softwaretype] recipe = slapos.cookbook:softwaretype -default = ${template-runner:output} \ No newline at end of file +default = ${template-runner:output} +resilient = ${instance-resilient:rendered} +runner = ${template-runner:output} +runner-import = ${instance-runner-import:output} +runner-export = ${instance-runner-export:output} + +frozen = ${instance-frozen:output} +pull-backup = ${template-pull-backup:output} diff --git a/software/slaprunner/software.cfg b/software/slaprunner/software.cfg index 45961a88df996ab219cf1f05818f142ab06b63ad..b488b47509354d63d3d53d1f0e8232b7dfa7cd31 100644 --- a/software/slaprunner/software.cfg +++ b/software/slaprunner/software.cfg @@ -5,7 +5,7 @@ # 2/ Define list of trusted certificates for the cache. [buildout] -extends = common.cfg +extends = development.cfg [networkcache] # signature certificates of the following uploaders. @@ -56,12 +56,12 @@ netaddr = 0.7.10 plone.recipe.command = 1.1 pycrypto = 2.6 pytz = 2012j -slapos.cookbook = 0.71.1 -slapos.core = 0.34 -slapos.libnetworkcache = 0.13.3 +#slapos.cookbook = 0.71.1 +slapos.core = 0.35.1 +slapos.libnetworkcache = 0.13.4 slapos.recipe.build = 0.11.5 slapos.recipe.template = 2.4.2 -slapos.toolbox = 0.34.0 +#slapos.toolbox = 0.34.0 smmap = 0.8.2 xml-marshaller = 0.9.7 z3c.recipe.scripts = 1.0.1 diff --git a/stack/resilient/README.txt b/stack/resilient/README.txt index e6703a14eba57c152f175e30f2b12488cf16a598..063e8b61198c831812c77a51f6a0ae8dd673102f 100644 --- a/stack/resilient/README.txt +++ b/stack/resilient/README.txt @@ -127,6 +127,27 @@ wrapper. I suggest you only add options and specify your export/import recipe. + +Checking that it works +---------------------- + +To check that your software instance is resilient you can proceed this way: +Once all instances are successfully deployed, go to your export instance, connect as the instance user and run: +$ ~/bin/exporter +It is the script responsible for triggering the resiliency stack on your instance. After doing a backup of your data, it will notify the pull-backup instances of a new backup, triggering the transfer of this data to the import instances. + +Once this script is run successfully, go to your import instance, connect as its instance user and check ~/srv/backup/"your sofwtare"/, the location of the data you wanted to receive. The last part of the resiliency is up to your import script. + +DEBUGGING: +Here is a partial list of things you can check to understand what is causing the problem: + +- Check that your import script does not fail and successfully places your data in ~/srv/backup/"your software" (as the import instance user) by runnig: +$ ~/bin/"your software"-exporter +- Check the export instance script is run successfully as this instance user by running: +$ ~/bin/exporter +- Check the pull-instance system did its job by going to one of your pull-backup instance, connect as its user and check the log : ~/var/log/equeue.log + + ----------------------------------------------------------------------------------------- Finally, instance-mysoftware-import.cfg.in and diff --git a/stack/resilient/buildout.cfg b/stack/resilient/buildout.cfg index caca443c456fb1b2aa9a996cfa9e94c072f285e9..a87613d999f7e55b6a1b76c36087ec16f959b204 100644 --- a/stack/resilient/buildout.cfg +++ b/stack/resilient/buildout.cfg @@ -1,6 +1,10 @@ [buildout] +extends = + ../../component/gzip/buildout.cfg + ../../component/rdiff-backup/buildout.cfg parts = + rdiff-backup pbsready pbsready-import pbsready-export @@ -9,6 +13,9 @@ parts = instance-frozen template-resilient + # needed tools for resiliency + gzip + #---------------- #--