diff --git a/stack/lamp/buildout.cfg b/stack/lamp/buildout.cfg index e7f8727e5d39ab793ff0618af58c3e2c0b3a3da3..7edcd753281f5ae6175139a892076cd34e472ffd 100644 --- a/stack/lamp/buildout.cfg +++ b/stack/lamp/buildout.cfg @@ -90,7 +90,7 @@ mode = 0644 recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance-backuped.cfg output = ${buildout:directory}/template-backuped.cfg -md5sum = 4190baa5e3f87a02e8721aeac3b70942 +md5sum = 0fedb9ed84137b2d5518ffbaaa6d76eb mode = 0644 [template-use-backup] @@ -111,34 +111,35 @@ mode = 0644 recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance-mariadb-pbsready.cfg output = ${buildout:directory}/template-mariadb-pbsready.cfg -md5sum = cfc87bb3bb1d4aa4ffd11e4b6dfcabe7 +md5sum = d2a580dcd7efdd528be45c5ffadfe760 mode = 0644 [template-mariadb-pbsready-import] recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance-mariadb-pbsready-import.cfg output = ${buildout:directory}/template-mariadb-pbsready-import.cfg -md5sum = 483796d0e0ebdc86fb6fe6589bb8ed7e +md5sum = 4a96ff02da3898fef7077fa8baec81ac mode = 0644 [template-mariadb-pbsready-export] recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance-mariadb-pbsready-export.cfg output = ${buildout:directory}/template-mariadb-pbsready-export.cfg -md5sum = 112f259b613eefa94f8cc83a0e2aa9f3 +md5sum = 11a9e45e8bc590bc11bfdd304b07a4a5 mode = 0644 [template-pull-backup] recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance-pull-backup.cfg output = ${buildout:directory}/template-pull-backup.cfg -md5sum = 46bd46a0b46f39937ab3d604eeb20441 +md5sum = c7099cdf45605da4e9595c6e7b913327 mode = 0644 [eggs] recipe = zc.recipe.egg eggs = ${lxml-python:egg} + slapos.toolbox [versions] setuptools = 0.6c12dev-r88795 diff --git a/stack/lamp/instance-backuped.cfg b/stack/lamp/instance-backuped.cfg index b83f6fce8b844fec59c9578633b92cc8779997b7..a42783e78307c6070b1feea81c7c1443acaf173c 100644 --- a/stack/lamp/instance-backuped.cfg +++ b/stack/lamp/instance-backuped.cfg @@ -26,14 +26,15 @@ recipe = slapos.cookbook:request name = PBS (Pull Backup Server) software-url = $${slap-connection:software-release-url} software-type = pull-backup -return = ssh-key +return = ssh-key notification-url feeds-url slave = false [request-mariadb] software-type = mariadb-pbsready-export -config = authorized-key +config = authorized-key notify config-authorized-key = $${request-pull-backup-server:connection-ssh-key} -return = url ssh-public-key ssh-url +config-notify = $${request-pull-backup-server:connection-notification-url} +return = url ssh-public-key ssh-url notification-id [request-mariadb-pseudo-replicating] <= slap-connection @@ -41,30 +42,38 @@ recipe = slapos.cookbook:request name = MariaDB (backup) software-url = $${slap-connection:software-release-url} software-type = mariadb-pbsready-import -return = url ssh-public-key ssh-url -config = authorized-key +return = url ssh-public-key ssh-url notification-url +config = authorized-key on-notification config-authorized-key = $${request-pull-backup-server:connection-ssh-key} +pbs-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb-push +config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${:pbs-notification-id} [request-pull-backup-server-mariadb] <= request-pull-backup-server slap-connection name = PBS pulling from MariaDB (backuped) -config = url name type server-key frequency +config = url name type server-key on-notification notify notification-id title config-url = $${request-mariadb:connection-ssh-url} config-name = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb config-type = pull config-server-key = $${request-mariadb:connection-ssh-public-key} -config-frequency = 15 0 * * * +config-on-notification = $${request-mariadb:connection-notification-id} +config-notify = $${request-pull-backup-server:connection-notification-url} +config-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb-pull +config-title = Pulling from MariaDB slave = true [request-pull-backup-server-mariadb-backup] <= request-pull-backup-server slap-connection name = PBS pushing on MariaDB (ready to use backup) -config = url name type server-key frequency +config = url name type server-key on-notification notify notification-id title config-url = $${request-mariadb-pseudo-replicating:connection-ssh-url} config-name = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb config-type = push config-server-key = $${request-mariadb-pseudo-replicating:connection-ssh-public-key} -config-frequency = 45 0 * * * +config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${request-pull-backup-server-mariadb:config-notification-id} +config-notify = $${request-mariadb-pseudo-replicating:connection-notification-url} +config-notification-id = $${request-mariadb-pseudo-replicating:pbs-notification-id} +config-title = Pushing to MariaDB backup slave = true diff --git a/stack/lamp/instance-mariadb-pbsready-export.cfg b/stack/lamp/instance-mariadb-pbsready-export.cfg index e54edca0934acce12a7d578277d76e5e392c2b8e..95a8e674b7da445f12899db724a53fcfeb187daf 100644 --- a/stack/lamp/instance-mariadb-pbsready-export.cfg +++ b/stack/lamp/instance-mariadb-pbsready-export.cfg @@ -3,6 +3,9 @@ extends = ${template-mariadb-pbsready:output} parts += cron-entry-mariadb-backup +[urls] +notification-id = http://[$${notifier:host}]:$${notifier:port}/get/$${notifier-mydumper:name} + [mydumper] recipe = slapos.cookbook:mydumper wrapper = $${rootdirectory:bin}/raw_mydumper @@ -13,15 +16,18 @@ mydumper-binary = ${mydumper:location}/bin/mydumper database = $${mariadb:database} import = false -[mydumper-locked] -recipe = slapos.cookbook:lockfile -lock-file = $${basedirectory:run}/mysql-backup.lock -binary = $${mydumper:wrapper} +[notifier-mydumper] +<= notifier +recipe = slapos.cookbook:notifier.notify +name = mydumper +title = Dumping MariaDB Database +executable = $${mydumper:wrapper} wrapper = $${rootdirectory:bin}/mydumper +notify = $${slap-parameter:notify} [cron-entry-mariadb-backup] <= cron recipe = slapos.cookbook:cron.d name = backup frequency = 0 * * * * -command = $${mydumper-locked:wrapper} +command = $${notifier-mydumper:wrapper} diff --git a/stack/lamp/instance-mariadb-pbsready-import.cfg b/stack/lamp/instance-mariadb-pbsready-import.cfg index 31a606dc61188a0a130d7c9c94d307dff84138d4..90906f63bc21126a75afd5c24efc60ca6b4a9b85 100644 --- a/stack/lamp/instance-mariadb-pbsready-import.cfg +++ b/stack/lamp/instance-mariadb-pbsready-import.cfg @@ -2,27 +2,14 @@ extends = ${template-mariadb-pbsready:output} -parts = - urls - mariadb - stunnel - certificate-authority - ca-stunnel - logrotate - logrotate-entry-mariadb - logrotate-entry-stunnel - logrotate-entry-cron - cron - cron-entry-logrotate - sshkeys-authority - dropbear-server - sshkeys-dropbear - dropbear-server-pbs-authorized-key - cron-entry-mariadb-import +parts += mariadb-import-on-notification + +[urls] +notification-url = http://[$${notifier:host}]:$${notifier:port}/notify [mydumper-import] recipe = slapos.cookbook:mydumper -wrapper = $${rootdirectory:bin}/raw_myloader +wrapper = $${rootdirectory:bin}/myloader backup-directory = $${directory:mariadb-backup} socket = $${mariadb:socket} user = root @@ -30,16 +17,8 @@ myloader-binary = ${mydumper:location}/bin/myloader database = $${mariadb:database} import = true -[mydumper-import-locked] -recipe = slapos.cookbook:lockfile -lock-file = $${basedirectory:run}/import.lock -wrapper = $${rootdirectory:bin}/myloader -binary = $${mydumper-import:wrapper} -wait = true - -[cron-entry-mariadb-import] -<= cron -recipe = slapos.cookbook:cron.d -name = mariadb_import -frequency = 0 * * * * -command = $${mydumper-import-locked:wrapper} +[mariadb-import-on-notification] +<= notifier +recipe = slapos.cookbook:notifier.callback +on-notification-id = $${slap-parameter:on-notification} +callback = $${mydumper-import:wrapper} diff --git a/stack/lamp/instance-mariadb-pbsready.cfg b/stack/lamp/instance-mariadb-pbsready.cfg index 56d50ab06a399f6d0183158bec31f30e9cd1b562..1a2eeea74008740ffdd44c7cf5371ea6fdde3186 100644 --- a/stack/lamp/instance-mariadb-pbsready.cfg +++ b/stack/lamp/instance-mariadb-pbsready.cfg @@ -12,12 +12,14 @@ parts = logrotate-entry-mariadb logrotate-entry-stunnel logrotate-entry-cron + logrotate-entry-equeue cron cron-entry-logrotate sshkeys-authority dropbear-server sshkeys-dropbear dropbear-server-pbs-authorized-key + notifier [urls] ssh-public-key = $${sshkeys-dropbear:public-key-value} @@ -61,16 +63,48 @@ recipe = slapos.cookbook:dropbear.add_authorized_key key = $${slap-parameter:authorized-key} [rdiff-backup-server] -recipe = slapos.cookbook:rdiffbackup +recipe = slapos.cookbook:pbs client = false path = $${directory:mariadb-backup} wrapper = $${rootdirectory:bin}/rdiffbackup-server rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup +[logrotate-entry-equeue] +<= logrotate +recipe = slapos.cookbook:logrotate.d +name = equeue +log = $${equeue:log} +frequency = daily +rotate-num = 30 + +[equeue] +recipe = slapos.cookbook:equeue +socket = $${basedirectory:run}/equeue.sock +log = $${basedirectory:log}/equeue.log +database = $${rootdirectory:srv}/equeue.db +wrapper = $${basedirectory:services}/equeue +equeue-binary = ${buildout:bin-directory}/equeue + +[notifier] +recipe = slapos.cookbook:notifier +feeds = $${directory:notifier-feeds} +callbacks = $${directory:notifier-callbacks} +id-file = $${rootdirectory:etc}/notifier.id +equeue-socket = $${equeue:socket} +host = $${slap-network-information:global-ipv6} +port = 8080 +wrapper = $${basedirectory:services}/notifier +server-binary = ${buildout:bin-directory}/pubsubserver +notifier-binary = ${buildout:bin-directory}/pubsubnotifier + + [basedirectory] cache = $${rootdirectory:var}/cache/ +notifier = $${rootdirectory:etc}/notifier/ [directory] mariadb-backup = $${basedirectory:backup}/mariadb/ ssh = $${rootdirectory:etc}/ssh/ sshkeys = $${rootdirectory:srv}/sshkeys +notifier-feeds = $${basedirectory:notifier}/feeds/ +notifier-callbacks = $${basedirectory:notifier}/callbacks/ diff --git a/stack/lamp/instance-pull-backup.cfg b/stack/lamp/instance-pull-backup.cfg index f1d46d746168379ad58d73b66aaaeaf3f1b78d09..193f50bc8617380bff69712efcd0ce18fb51164e 100644 --- a/stack/lamp/instance-pull-backup.cfg +++ b/stack/lamp/instance-pull-backup.cfg @@ -2,7 +2,7 @@ parts = connection-dict - rdiff-backup + pbs cron cron-entry-logrotate logrotate @@ -16,6 +16,28 @@ offline = true [connection-dict] recipe = slapos.cookbook:publish ssh-key = $${sshkeys-dropbear:public-key-value} +notification-url = http://[$${notifier:host}]:$${notifier:port}/notify +feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/ + +[equeue] +recipe = slapos.cookbook:equeue +socket = $${basedirectory:run}/equeue.sock +log = $${basedirectory:log}/equeue.log +database = $${rootdirectory:srv}/equeue.db +wrapper = $${basedirectory:services}/equeue +equeue-binary = ${buildout:bin-directory}/equeue + +[notifier] +recipe = slapos.cookbook:notifier +feeds = $${directory:notifier-feeds} +callbacks = $${directory:notifier-callbacks} +id-file = $${rootdirectory:etc}/notifier.id +equeue-socket = $${equeue:socket} +host = $${slap-network-information:global-ipv6} +port = 8080 +wrapper = $${basedirectory:services}/notifier +server-binary = ${buildout:bin-directory}/pubsubserver +notifier-binary = ${buildout:bin-directory}/pubsubnotifier [dropbear-client] recipe = slapos.cookbook:dropbear.client @@ -39,23 +61,25 @@ wrapper = $${basedirectory:services}/sshkeys_authority [sshkeys-dropbear] <= sshkeys-authority recipe = slapos.cookbook:sshkeys_authority.request -name = rdiff-backup +name = pbs type = rsa executable = $${dropbear-client:wrapper} public-key = $${dropbear-client:identity-file}.pub private-key = $${dropbear-client:identity-file} wrapper = $${rootdirectory:bin}/do_backup -[rdiff-backup] -recipe = slapos.cookbook:rdiffbackup +[pbs] +<= notifier +recipe = slapos.cookbook:pbs client = true rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup sshclient-binary = $${dropbear-client:wrapper} known-hosts = $${directory:dot-ssh}/known_hosts promises-directory = $${basedirectory:promises} -directory = $${directory:backups} +directory = $${directory:pbs-backup} cron-entries = $${cron:cron-entries} wrappers-directory = $${directory:pbs-wrappers} +notifier-url = http://[$${notifier:host}]:$${notifier:port}/ [cron-simplelogger] recipe = slapos.cookbook:simplelogger @@ -91,6 +115,23 @@ logrotate-entries = $${directory:logrotate-entries} backup = $${directory:logrotate-backup} state-file = $${rootdirectory:srv}/logrotate.status +[logrotate-entry-equeue] +<= logrotate +recipe = slapos.cookbook:logrotate.d +name = equeue +log = $${equeue:log} +frequency = daily +rotate-num = 30 + +[logrotate-entry-equeue] +<= logrotate +recipe = slapos.cookbook:logrotate.d +name = cron +log = $${cron-simplelogger:log} +frequency = daily +rotate-num = 30 + + [rootdirectory] recipe = slapos.cookbook:mkdirectory etc = $${buildout:directory}/etc/ @@ -113,17 +154,18 @@ cronstamps = $${rootdirectory:etc}/cronstamps/ cron-entries = $${rootdirectory:etc}/cron.d/ crontabs = $${rootdirectory:etc}/crontabs/ cronoutput = $${basedirectory:log}/cron-ouput/ -rdiff-backup-backup = $${basedirectory:backup}/rdiff-backup/ +pbs-backup = $${basedirectory:backup}/pbs/ logrotate-entries = $${rootdirectory:etc}/logrotate.d/ logrotate-backup = $${basedirectory:backup}/logrotate/ -rdiff-backup = $${rootdirectory:etc}/rdiff-backup/ sshkeys = $${rootdirectory:srv}/sshkeys -backups = $${basedirectory:backup}/backups/ -pbs-wrappers = $${rootdirectory:bin}/rdiff-backup/ +pbs-wrappers = $${rootdirectory:bin}/pbs/ dot-ssh = $${basedirectory:ssh-home}/.ssh/ +notifier-feeds = $${basedirectory:notifier}/feeds/ +notifier-callbacks = $${basedirectory:notifier}/callbacks/ [basedirectory] ssh-home = $${rootdirectory:home}/ssh +notifier = $${rootdirectory:etc}/notifier/ [rootdirectory] home = $${buildout:directory}/home/