Commit b2468f4e authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki

Merge branch 'erp5-component' into erp5

parents 9d6253d6 b273bcb0
Changes
=======
0.84.2 (2013-10-04)
-------------------
* sshkeys_authority: don't allow to return None as parameter. [9e340a0]
0.84.1 (2013-10-03)
-------------------
......
[buildout]
parts =
nano
extends =
../ncurses/buildout.cfg
[nano]
recipe = slapos.recipe.cmmi
version = 2.2.6
url = http://www.nano-editor.org/dist/v2.2/nano-2.2.6.tar.gz
md5sum = 03233ae480689a008eb98feb1b599807
environment=
CFLAGS=-I${ncurses:location}/include
LDFLAGS=-L${ncurses:location}/lib/ -Wl,-rpath=${ncurses:location}/lib/
\ No newline at end of file
......@@ -10,8 +10,8 @@ parts =
recipe = slapos.recipe.cmmi
depends =
${perl:version}
version = 2.1.9
version = 2.2.5
url = http://www.percona.com/redir/downloads/percona-toolkit/${:version}/percona-toolkit-${:version}.tar.gz
md5sum = 94545d0fe6a4893dcad8a3411531107d
md5sum = 56bc17bd7ba8b9af8461ba93cfb53d25
configure-command =
${perl:location}/bin/perl Makefile.PL
......@@ -5,8 +5,8 @@ parts =
[sqlite3]
recipe = slapos.recipe.cmmi
url = http://www.sqlite.org/2013/sqlite-autoconf-3080002.tar.gz
md5sum = 6d6cc639a4da04fbbdda7b1a1a01b386
url = http://www.sqlite.org/2013/sqlite-autoconf-3080100.tar.gz
md5sum = 8b5a0a02dfcb0c7daf90856a5cfd485a
configure-options =
--disable-static
--enable-readline
......
......@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob
import os
version = '0.84.1'
version = '0.84.2'
name = 'slapos.cookbook'
long_description = open("README.txt").read() + "\n" + \
open("CHANGES.txt").read() + "\n"
......
......@@ -12,7 +12,8 @@ erp5_catalog_storage = 'erp5_mysql_innodb_catalog'
mysql_url = "%(sql_connection_string)s"
header_dict = {'Authorization': 'Basic %%s' %% \
base64.encodestring('%%s:%%s' %% (user, password)).strip()}
base64.encodestring('%%s:%%s' %% (user, password)).strip(),
'Referer':'http://%%s/manage_addProduct/ERP5/addERP5Site' %% host}
zope_connection = httplib.HTTPConnection(host)
# Check if an ERP5 site is already created, as ERP5 does support having
......
......@@ -50,6 +50,7 @@ class ERP5Updater(object):
base64string = base64.encodestring(authentication_string).strip()
self.header_dict['Authorization'] = 'Basic %s' % base64string
self.header_dict['Referer'] = 'http://%s/manage_addProduct/ERP5/addERP5Site' % host
self.host = host
self.site_id = site_id
......
......@@ -6,6 +6,8 @@
import socket
import time
# XXX: to be factored with slapos.toolbox qemu qmp wrapper.
socket_path = '%(socket-path)s'
vnc_password = '%(vnc-passwd)s'
......
......@@ -21,6 +21,7 @@ parts =
# XXX: we have to manually add this for resilience
rdiff-backup
collective.recipe.template-egg
#XXX-Cedric : Currently, one can only access to KVM using noVNC.
# Ideally one should be able to access KVM by using either NoVNC or VNC.
......@@ -79,7 +80,7 @@ command =
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
#md5sum = bdd0495ef729e7272ec9c97aca919c09
md5sum = 8617a8cc345a55688c5449528daef4d1
output = ${buildout:directory}/template.cfg
mode = 0644
......@@ -94,14 +95,14 @@ mode = 0644
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
mode = 644
#md5sum = 6753004b582c0470bd028253ce1964ad
md5sum = 45a846378215eded6c001d0dd729a1ec
download-only = true
on-update = true
[template-kvm-resilient-test]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/instance-kvm-resilient-test.cfg.jinja2
#md5sum = 027d68d9decbc6aec59365fa723975d7
md5sum = b58427f93d5fcca94bdc90661fe6080b
mode = 0644
download-only = true
on-update = true
......@@ -132,7 +133,7 @@ mode = 0644
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/kvm-export.sh.in
filename = kvm-export.sh.in
md5sum = 3e878b3343c76f0d6950986fffcb6a8c
md5sum = bf03a90f6960b37cba812ee936a13342
download-only = true
mode = 0755
......
......@@ -29,5 +29,11 @@
"description": "Periodicity of backup, in cron format.",
"type": "string"
}
"remove-backup-older-than": {
"title": "Remove backups older than...",
"description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible."
"type": "string",
"default": "3B"
}
}
}
......@@ -50,12 +50,10 @@ config-{{ key }} = {{ dumps(value) }}
config-virtual-hard-drive-url = ${slap-parameter:virtual-hard-drive-url}
config-virtual-hard-drive-md5sum = ${slap-parameter:virtual-hard-drive-md5sum}
config-resiliency-backup-periodicity = */5
# We don't use url parameter, but we want it to be there to make sure root instance is ready.
return = url
# XXX What to do?
sla = computer_guid
sla-computer_guid = ${slap-connection:computer-id}
[slap-parameter]
virtual-hard-drive-url = https://softinst43236.host.vifib.net/data/public/8e2138.php?dl=true
virtual-hard-drive-md5sum = de0f10c7c6538e9928879332afd9be7a
virtual-hard-drive-md5sum = 465e1024447997e7b86ee2e5151e031b
......@@ -12,6 +12,8 @@ offline = true
parts +=
{{ parts.replicate("kvm", "3") }}
publish-connection-informations
kvm-frontend-url-promise
kvm-backend-url-promise
{{ replicated.replicate("kvm", "3", "kvm-export", "kvm-import", slapparameter_dict=slapparameter_dict) }}
......@@ -29,3 +31,22 @@ recipe = slapos.cookbook:publish
backend-url = ${request-kvm:connection-backend-url}
url = ${request-kvm:connection-url}
ipv6 = ${request-kvm:connection-ipv6}
[kvm-frontend-url-promise]
# Check that url parameter is complete
recipe = collective.recipe.template
input = inline:#!/bin/sh
URL="${request-kvm:connection-url}"
if [[ ! "$URL" == https://* ]]; then
exit 1
fi
output = ${resilient-directory:promise}/kvm-frontend-url
mode = 700
[kvm-backend-url-promise]
# Check that backend url is reachable
recipe = slapos.cookbook:check_url_available
path = ${directory:promises}/frontend_promise
url = ${publish-connection-information:url}
dash_path = /bin/sh
curl_path = {{ curl_executable_location }}
......@@ -49,10 +49,10 @@ mode = 0644
recipe = slapos.recipe.template:jinja2
template = ${template-kvm-resilient-test:location}/instance-kvm-resilient-test.cfg.jinja2
rendered = $${buildout:directory}/template-kvm-resilient-test.cfg
bin-directory = ${buildout:bin-directory}
context =
key bin_directory dynamic-template-kvm-resilient-test:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
raw bin_directory ${buildout:bin-directory}
raw curl-executable-location ${curl:location}/bin/curl
mode = 0644
......@@ -122,34 +122,35 @@ Werkzeug = 0.9.4
apache-libcloud = 0.13.2
async = 0.6.1
buildout-versions = 1.7
collective.recipe.template = 1.10
erp5.util = 0.4.36
gitdb = 0.5.4
itsdangerous = 0.23
lxml = 3.2.3
meld3 = 0.6.10
plone.recipe.command = 1.1
psutil = 1.1.0
psutil = 1.1.1
pycrypto = 2.6
rdiff-backup = 1.0.5
slapos.cookbook = 0.84.1
slapos.cookbook = 0.84.2
slapos.recipe.cmmi = 0.2
slapos.recipe.download = 1.0.dev-r4053
slapos.toolbox = 0.37.1
slapos.toolbox = 0.37.2
smmap = 0.8.2
websockify = 0.5.1
z3c.recipe.scripts = 1.0.1
# Required by:
# slapos.core==0.35.1
# slapos.toolbox==0.37.1
# slapos.toolbox==0.37.2
Flask = 0.10.1
# Required by:
# slapos.toolbox==0.37.1
# slapos.toolbox==0.37.2
GitPython = 0.3.2.RC1
# Required by:
# slapos.toolbox==0.37.1
# slapos.toolbox==0.37.2
atomize = 0.1.1
# Required by:
......@@ -157,19 +158,19 @@ atomize = 0.1.1
ecdsa = 0.9
# Required by:
# slapos.toolbox==0.37.1
# slapos.toolbox==0.37.2
feedparser = 5.1.3
# Required by:
# slapos.cookbook==0.84.1
# slapos.cookbook==0.84.2
inotifyx = 0.2.0-1
# Required by:
# slapos.cookbook==0.84.1
# slapos.cookbook==0.84.2
lock-file = 2.0
# Required by:
# slapos.cookbook==0.84.1
# slapos.cookbook==0.84.2
netaddr = 0.7.10
# Required by:
......@@ -180,9 +181,8 @@ netifaces = 0.8-1
# websockify==0.5.1
numpy = 1.7.1
# Required by:
# slapos.toolbox==0.37.1
# slapos.toolbox==0.37.2
paramiko = 1.12.0
# Required by:
......@@ -190,12 +190,12 @@ paramiko = 1.12.0
pyflakes = 0.7.3
# Required by:
# slapos.cookbook==0.84.1
# slapos.cookbook==0.84.2
pytz = 2013.7
# Required by:
# slapos.cookbook==0.84.1
# slapos.toolbox==0.37.1
# slapos.cookbook==0.84.2
# slapos.toolbox==0.37.2
slapos.core = 0.35.1
# Required by:
......@@ -207,11 +207,10 @@ supervisor = 3.0
unittest2 = 0.5.1
# Required by:
# slapos.cookbook==0.84.1
# slapos.toolbox==0.37.1
# slapos.cookbook==0.84.2
# slapos.toolbox==0.37.2
xml-marshaller = 0.9.7
# Required by:
# slapos.core==0.35.1
zope.interface = 4.0.5
......@@ -4,13 +4,17 @@ QEMU_IMG=${kvm-instance:qemu-img-path}
SNAPSHOT_NAME=$(date +%s)
DISK_PATH=${kvm-instance:disk-path}
BACKUP_PATH=${:backup-disk-path}
QMP_CLIENT=${buildout:directory}/software_release/bin/qemu-qmp-client
if [ ! -f $DISK_PATH ]; then
echo "Nothing to backup, disk image doesn't exist yet."
exit 0;
fi
$QMP_CLIENT ${kvm-instance:socket-path} suspend && \
$QEMU_IMG snapshot -c $SNAPSHOT_NAME $DISK_PATH
$QMP_CLIENT ${kvm-instance:socket-path} resume
if [ -f $BACKUP_PATH ]; then
rm $BACKUP_PATH
fi
......
......@@ -15,7 +15,6 @@ extends =
../../stack/slapos.cfg
parts =
rdiff-backup
template
eggs
nginx
......@@ -27,6 +26,10 @@ parts =
instance-runner-export
slapos-cookbook
# XXX: we have to manually add this for resilience
rdiff-backup
collective.recipe.template-egg
####################
## Node JS proxy
####################
......
......@@ -189,7 +189,7 @@ mode = 640
[template-kumofs]
< = template-jinja2-base
filename = instance-kumofs.cfg
md5sum = 90a321be12ee977800d590bf941021ef
md5sum = 40817014a41497bceb696e512436e670
extra-context =
key dash_location dash:location
key dcron_location dcron:location
......
......@@ -34,8 +34,8 @@ gateway-wrapper = ${basedirectory:services}/kumofs_gateway
manager-wrapper = ${basedirectory:services}/kumofs_manager
server-wrapper = ${basedirectory:services}/kumofs_server
# Paths: Data
data-directory = ${directory:kumofs-data}
# Paths: Data (with 10M buckets and HDBTLARGE option)
data-path = ${directory:kumofs-data}/kumodb.tch#bnum=10485760#opts=l
# Paths: Logs
kumo-gateway-log = ${basedirectory:log}/kumo-gateway.log
......
[buildout]
extends =
../../component/dash/buildout.cfg
../../component/dropbear/buildout.cfg
../../component/gzip/buildout.cfg
../../component/rdiff-backup/buildout.cfg
../../component/rsync/buildout.cfg
parts =
rdiff-backup
collective.recipe.template-egg
pbsready
pbsready-import
pbsready-export
......@@ -16,7 +17,12 @@ parts =
# needed tools for resiliency
gzip
rdiff-backup
dash
[collective.recipe.template-egg]
recipe = zc.recipe.egg
eggs = collective.recipe.template
#----------------
#--
......@@ -30,7 +36,7 @@ parts =
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/pbsready.cfg.in
output = ${buildout:directory}/pbsready.cfg
md5sum = 570e0b54c97d510befa2ea981c1e90e0
#md5sum = 46f9d33e642467a72c599c8dc767e6c3
mode = 0644
[pbsready-import]
......@@ -39,7 +45,7 @@ mode = 0644
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/pbsready-import.cfg.in
output = ${buildout:directory}/pbsready-import.cfg
md5sum = cc9c776500ccd07cb51969beb68ffcda
md5sum = cb562bd954b9e809c8748d0f96de4116
mode = 0644
[pbsready-export]
......@@ -48,7 +54,7 @@ mode = 0644
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/pbsready-export.cfg.in
output = ${buildout:directory}/pbsready-export.cfg
md5sum = 25d05b3929fb4c6cf275866bad678d6a
md5sum = 8fb619622a08aff2321497895e04df16
mode = 0644
[template-pull-backup]
......@@ -61,14 +67,14 @@ mode = 0644
[template-replicated]
recipe = slapos.recipe.download
url = ${:_profile_base_location_}/template-replicated.cfg.in
md5sum = c762a625f65193bc8a570b4d56a0d08c
md5sum = 9e236726678d89a5359e1571a91e59e8
mode = 0644
destination = ${buildout:directory}/template-replicated.cfg.in
[template-parts]
recipe = slapos.recipe.download
url = ${:_profile_base_location_}/template-parts.cfg.in
md5sum = c942f82552fcb42fc74a5f896e0cd5f3
md5sum = a3f55a20881c3f1ec4416662146c06f7
mode = 0644
destination = ${buildout:directory}/template-parts.cfg.in
......
......@@ -14,6 +14,7 @@ parts =
sshkeys-authority
dropbear-server
sshkeys-dropbear
resilient-sshkeys-dropbear-promise
dropbear-server-pbs-authorized-key
notifier
......
......@@ -14,6 +14,7 @@ parts =
sshkeys-authority
dropbear-server
sshkeys-dropbear
resilient-sshkeys-dropbear-promise
dropbear-server-pbs-authorized-key
notifier
......
......@@ -10,6 +10,7 @@ parts =
sshkeys-authority
dropbear-server
sshkeys-dropbear
resilient-sshkeys-dropbear-promise
dropbear-server-pbs-authorized-key
notifier
......@@ -179,6 +180,27 @@ server-binary = ${buildout:bin-directory}/pubsubserver
notifier-binary = ${buildout:bin-directory}/pubsubnotifier
#----------------
#--
#-- Dropbear.
[dropbear-server]
recipe = slapos.cookbook:dropbear
host = $${slap-network-information:global-ipv6}
# Explicitely excludes to define "port" argument. It will be defined in
# pbs-ready-import.cfg.in and pbs-ready-export.cfg.in
home = $${directory:ssh}
wrapper = $${rootdirectory:bin}/raw_sshd
shell = $${rdiff-backup-server:wrapper}
rsa-keyfile = $${directory:ssh}/server_key.rsa
dropbear-binary = ${dropbear:location}/sbin/dropbear
[dropbear-server-pbs-authorized-key]
<= dropbear-server
recipe = slapos.cookbook:dropbear.add_authorized_key
key = $${slap-parameter:authorized-key}
#----------------
#--
#-- sshkeys
......@@ -205,31 +227,21 @@ public-key = $${dropbear-server:rsa-keyfile}.pub
private-key = $${dropbear-server:rsa-keyfile}
wrapper = $${basedirectory:services}/sshd
#----------------
#--
#-- Dropbear.
[dropbear-server]
recipe = slapos.cookbook:dropbear
host = $${slap-network-information:global-ipv6}
# Explicitely excludes to define "port" argument. It will be defined in
# pbs-ready-import.cfg.in and pbs-ready-export.cfg.in
home = $${directory:ssh}
wrapper = $${rootdirectory:bin}/raw_sshd
shell = $${rdiff-backup-server:wrapper}
rsa-keyfile = $${directory:ssh}/server_key.rsa
dropbear-binary = ${dropbear:location}/sbin/dropbear
[dropbear-server-pbs-authorized-key]
<= dropbear-server
recipe = slapos.cookbook:dropbear.add_authorized_key
key = $${slap-parameter:authorized-key}
[resilient-sshkeys-dropbear-promise]
# Check that public key file exists and is not empty
recipe = collective.recipe.template
input = inline:#!${dash:location}/bin/dash
PUBLIC_KEY_CONTENT="$${sshkeys-dropbear:public-key-value}"
if [ ! -n "$PUBLIC_KEY_CONTENT" ]; then
exit 1
fi
output = $${basedirectory:promises}/public-key-existence
mode = 700
#----------------
#--
#-- Conncetion informations to re-use.
#-- Connection informations to re-use.
# XXX-Cedric: when "aggregation" system is done in libslap, directly publish.
[resilient-publish-connection-parameter]
recipe = slapos.cookbook:publish
......
......@@ -4,18 +4,21 @@
request-{{namebase}}
request-{{namebase}}-2
resilient-request-{{namebase}}-public-key-promise
{% for i in range(1,nbbackup|int) %}
request-{{namebase}}-pseudo-replicating-{{i}}
request-{{namebase}}-pseudo-replicating-{{i}}-2
{% for id in range(1,nbbackup|int) %}
request-{{namebase}}-pseudo-replicating-{{id}}
request-{{namebase}}-pseudo-replicating-{{id}}-2
resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key-promise
{% endfor %}
{% for i in range(1,nbbackup|int) %}
request-pbs-{{namebase}}-{{i}}
request-pull-backup-server-{{namebase}}-{{i}}
request-pull-backup-server-{{namebase}}-backup-{{i}}
{% for id in range(1,nbbackup|int) %}
request-pbs-{{namebase}}-{{id}}
resilient-request-pbs-{{namebase}}-{{id}}-public-key-promise
request-pull-backup-server-{{namebase}}-{{id}}
request-pull-backup-server-{{namebase}}-backup-{{id}}
{% endfor %}
{% endmacro %}
......@@ -11,6 +11,13 @@
{% endif -%}
[resilient-directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc
promise = ${:etc}/promise
## Tells the Backupable recipe that we want a backup
[resilient]
recipe = slapos.cookbook:request
......@@ -28,7 +35,6 @@ software-url = ${slap-connection:software-release-url}
software-type = {{typeexport}}
name = {{namebase}}0
return = ssh-public-key ssh-url notification-id ip
config =
# Resilient related parameters
number authorized-key notify ip-list namebase
......@@ -66,6 +72,7 @@ sla-{{ key }} = {{ value }}
{% endif -%}
{% endif -%}
{% for id in range(1,nbbackup|int) %}
[request-{{namebase}}-pseudo-replicating-{{id}}]
......@@ -107,9 +114,9 @@ sla-{{ key }} = {{ value }}
{% endif %}
{% endif %}
{% endfor -%}
[iplist]
config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbbackup|int) %} ${request-{{namebase}}-pseudo-replicating-{{j}}:connection-ip}{% endfor %}
......@@ -117,11 +124,37 @@ config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbback
<= request-{{namebase}}
iplist
[resilient-request-{{namebase}}-public-key-promise]
# Check that public-key-value parameter exists and is not empty
# XXX: maybe we should consider empty values to be non-nexistent.
recipe = collective.recipe.template
# XXX: don't use system executable
input = inline:#!/bin/sh
PUBLIC_KEY_CONTENT="${request-{{namebase}}-2:connection-ssh-public-key})"
if [[ ! -n "$PUBLIC_KEY_CONTENT" -o "$PUBLIC_KEY_CONTENT" == None ]]; then
exit 1
fi
output = ${resilient-directory:promise}/resilient-request-{{namebase}}-public-key
mode = 700
{% for id in range(1,nbbackup|int) %}
[request-{{namebase}}-pseudo-replicating-{{id}}-2]
<= request-{{namebase}}-pseudo-replicating-{{id}}
iplist
[resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key-promise]
# Check that public-key-value parameter exists and is not empty
# XXX: maybe we should consider empty values to be non-nexistent.
recipe = collective.recipe.template
# XXX: don't use system executable
input = inline:#!/bin/sh
PUBLIC_KEY_CONTENT="${request-{{namebase}}-pseudo-replicating-{{id}}-2:connection-ssh-public-key})"
if [ ! -n "$PUBLIC_KEY_CONTENT" -a "$PUBLIC_KEY_CONTENT" == None ]; then
exit 1
fi
output = ${resilient-directory:promise}/resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key
mode = 700
{% endfor %}
......@@ -167,6 +200,19 @@ sla-{{ key }} = {{ value }}
{% endif %}
{% endif %}
[resilient-request-pbs-{{namebase}}-{{id}}-public-key-promise]
# Check that public-key-value parameter exists and is not empty
# XXX: maybe we should consider empty values to be non-nexistent.
recipe = collective.recipe.template
# XXX: don't use system executable
input = inline:#!/bin/sh
PUBLIC_KEY_CONTENT="${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}:connection-ssh-key})"
if [ ! -n "$PUBLIC_KEY_CONTENT" -a "$PUBLIC_KEY_CONTENT" == None ]; then
exit 1
fi
output = ${resilient-directory:promise}/resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key
mode = 700
[request-pull-backup-server-{{namebase}}-{{id}}]
<= request-pbs-common
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment