Skip to content

slapos
slapos
Commit 2f17cfdf authored by Nicolas Wavrant's avatar Nicolas Wavrant
Browse files
Options

slaprunner: use the new python version of runner-exporter

parent 6dfcbfc0
Hide whitespace changes
Inline Side-by-side
Showing with 16 additions and 166 deletions
software/slaprunner/buildout.hash.cfg
View file @ 2f17cfdf
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = 713db528880282d568278f09458d2aab
md5sum = bb2628810e873b4fd4b50937ef23645d
[template-runner]
filename = instance-runner.cfg
......@@ -28,13 +28,9 @@ md5sum = ed2e08c07a6727b2012f15da67c0705d
filename = instance-runner-import.cfg.in
md5sum = f9075dab9469c69506b24e642682938c
[template-runner-export-script]
filename = template/runner-export.sh.jinja2
md5sum = 231f9b74862f8991f54326511e76f5ec
[instance-runner-export]
filename = instance-runner-export.cfg.in
md5sum = 2abcebc7e4610566502eca45e28c2992
md5sum = 01395b98534066ec23a5ad5c96d5f1e7
[template-resilient]
filename = instance-resilient.cfg.jinja2
......
software/slaprunner/instance-runner-export.cfg.in
View file @ 2f17cfdf
......@@ -75,21 +75,19 @@ ip = ${slaprunner:ipv4}
[supervisord]
port = ${supervisord-free-port:port}
[exporter-raw]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/${slap-parameter:namebase}-exporter-raw
command-line = {{ software_release_bin }}/${slap-parameter:namebase}-exporter --srv-path ${directory:srv} --etc-path ${directory:etc} --backup-path ${directory:backup} --backup-wait-time {{ backup_wait_time }}
environment =
PATH={{ rsync_bin_folder }}:/bin:/usr/bin
[exporter]
recipe = slapos.recipe.template:jinja2
template = {{ exporter_script_path }}
rendered = ${directory:bin}/${slap-parameter:namebase}-exporter
# backward compatibility for resilient stack
wrapper = ${:rendered}
mode = 700
context =
import sys sys
import easy_install zc.buildout.easy_install
section directory directory
raw backup_wait_time {{ backup_wait_time }}
raw output_log_file ${directory:log}/resilient.log
raw shell_binary {{ bash_executable_location }}
raw rsync_binary {{ rsync_executable_location }}
recipe = slapos.cookbook:wrapper
command-line = {{ bash_executable_location }} -c "exec ${exporter-raw:wrapper-path} > >(tee -ai "${directory:log}/resilient.log") 2>&1"
wrapper-path = ${directory:bin}/${slap-parameter:namebase}-exporter
# wrapper parameter is needed by resilience stack
wrapper = ${:wrapper-path}
[monitor-httpd-free-port]
recipe = slapos.cookbook:free_port
......
software/slaprunner/instance.cfg
View file @ 2f17cfdf
......@@ -56,12 +56,12 @@ context =
key pbsready_export_template_path template-pbsready-export:rendered
key template_runner_path instance-base-runner:rendered
key slapparameter_dict slap-configuration:configuration
raw software_release_bin ${buildout:bin-directory}
raw backup_wait_time ${exporter-default-configuration:backup_wait_time}
raw exporter_script_path ${template-runner-export-script:location}/${template-runner-export-script:filename}
raw monitor_check_resilient_feed_template_path ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
raw buildout_executable_location ${buildout:executable}
raw bash_executable_location ${bash:location}/bin/bash
raw rsync_executable_location ${rsync:location}/bin/rsync
raw rsync_bin_folder ${rsync:location}/bin
[template-runner-import]
recipe = slapos.recipe.template:jinja2
......
software/slaprunner/software.cfg
View file @ 2f17cfdf
......@@ -76,10 +76,6 @@ output = ${buildout:directory}/template-runner.cfg.in
< = template-download-base
filename = runner-import.sh.jinja2
[template-runner-export-script]
< = template-download-base
filename = runner-export.sh.jinja2
[instance-runner-import]
< = download-base
recipe = slapos.recipe.build:download
......
software/slaprunner/template/runner-export.sh.jinja2 deleted 100644 → 0
View file @ 6dfcbfc0
#!{{ shell_binary }}
LC_ALL=C
export LC_ALL
umask 077
# Exit on any error, to prevent inconsistent backup
# Error on unset variable expansion
set -eu
set -o pipefail
# Redirect output to log
exec > >(tee -ai '{{ output_log_file }}')
exec 2>&1
echo -e "\n\n$0 run at : $(date)"
srv_directory='{{ directory["srv"] }}'
backup_directory='{{ directory["backup"] }}'
etc_directory='{{ directory["etc"] }}'
tmp_directory='{{ directory["tmp"] }}'
rsync () {
# Workaround for bug https://bugzilla.samba.org/show_bug.cgi?id=3653
IGNOREEXIT=24
IGNOREOUT='^(file has vanished: |rsync warning: some files vanished before they could be transferred)'
set -x
'{{ rsync_binary }}' -rlptgov --stats --safe-links --ignore-missing-args --delete --delete-excluded "$@" 2>&1 | (egrep -v "$IGNOREOUT" || true) || [ $? = "$IGNOREEXIT" ]
set +x
}
(
# XXX: code duplication with runner-import.sh.jinja2
path=$srv_directory/runner
backup_path=$backup_directory/runner/
cd "$path"
if [ -d instance ]; then
# Concatenate the exclude file of each partition of webrunner
# to create a global exclude file.
# Also, ignore all buildout-managed files.
exclude=$({{ sys.executable }} - "$path" <<EOF
if 1:
import glob, errno, os, sys
sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
from zc.buildout.configparser import parse
path = sys.argv[1]
def print_relative(path_list):
for p in path_list:
p = p.strip()
if p:
print(os.path.relpath(p, path))
print("*.sock")
print("*.socket")
print("*.pid")
print(".installed*.cfg")
for partition in glob.glob(path + "/instance/slappart*"):
try:
os.chdir(partition)
except OSError as e:
if e.errno != errno.ENOTDIR:
raise
continue
try:
with open("srv/exporter.exclude") as f:
exclude = f.readlines()
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
print_relative(exclude)
for installed in glob.glob(".installed*.cfg"):
try:
with open(installed) as f:
installed = parse(f, installed)
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
for section in installed.itervalues():
print_relative(section.get(
'__buildout_installed__', '').splitlines())
EOF
)
echo "$exclude" |rsync --exclude-from=- instance "$backup_path"
fi
test -d project && rsync project "$backup_path"
test -d public && rsync public "$backup_path"
test -f proxy.db && rsync proxy.db "$backup_path"
)
# We sync .* appart
(
cd "$etc_directory"
date +%s -u > .resilient-timestamp
rsync config.json "$backup_directory"/etc/
# Hidden files are related to the webrunner's internals
cp -r .??* "$backup_directory/etc/"
)
if [ -d "$backup_directory"/runner/software ]; then
rm "$backup_directory"/runner/software/*
fi
(
cd "$backup_directory"
find -type f ! -name backup.signature -print0 | xargs -0 sha256sum | sort -k 66 > backup.signature
)
# Check that export didn't happen during backup of instances
tmp_backup_sum=$(mktemp -p "$tmp_directory")
remove_tmp_files () {
rm "$tmp_backup_sum"
}
trap remove_tmp_files EXIT
cd {{ directory['backup'] }}
# Wait a little to increase the probability to detect an ongoing backup.
sleep 5
# Getting files from runner backup directory, as instance backup files may be
# explicitely excluded from the backup, using the srv/exporter.exclude
find -path "./runner/instance/slappart*/srv/backup/*" -type f ! -name backup.signature -print0 |
xargs -r0 sha256sum | sort -k 66 > "$tmp_backup_sum"
# If no backup found, it's over
if [ ! -s "$tmp_backup_sum" ]; then
exit 0
fi
# If the diff fails, then the notifier will restart this script
grep "instance/slappart.*/srv/backup/" "$backup_directory/backup.signature" |
diff "$tmp_backup_sum" - || {
echo "ERROR: Some backups are not consistent, exporter should be re-run."
echo "Let's sleep {{ backup_wait_time }} minutes, to let the backup end..."
sleep {{ backup_wait_time }}m
exit 1
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7