From 4ee761cd5c82b45b3838eb8b7f3cad1bfea0068b Mon Sep 17 00:00:00 2001 From: Vincent Pelletier <vincent@nexedi.com> Date: Thu, 20 Apr 2017 14:11:21 +0900 Subject: [PATCH] software/slaprunner: Rework rsync workhorse. Do not apply exclusions to unrelated paths ("project", "proxy.db" and "etc"). Simplify global exclusion file generation. Add support for absolute and partition-relative exclusion paths (previously, only partition-relative were supported). Pipe exclusion file to rsync, to avoid an intermediate disk write. Remove some directory prefix duplication by using PWD-relative paths (except in one place, for runner-import.sh.jinja2 comparison). --- software/slaprunner/buildout.hash.cfg | 6 +- software/slaprunner/common.cfg | 1 + .../slaprunner/instance-runner-export.cfg.in | 4 + software/slaprunner/instance.cfg | 4 + .../template/runner-export.sh.jinja2 | 87 ++++++++++--------- 5 files changed, 59 insertions(+), 43 deletions(-) diff --git a/software/slaprunner/buildout.hash.cfg b/software/slaprunner/buildout.hash.cfg index de765e772..1d522f2d7 100644 --- a/software/slaprunner/buildout.hash.cfg +++ b/software/slaprunner/buildout.hash.cfg @@ -15,7 +15,7 @@ # not need these here). [template] filename = instance.cfg -md5sum = 02755403c9f0b52d717160b0b2d0cfb7 +md5sum = afd426c01891d06e95f17b622ecd172f [template-runner] filename = instance-runner.cfg @@ -31,11 +31,11 @@ md5sum = 9db9957f452bda370cb2d5cc2e833e85 [template-runner-export-script] filename = template/runner-export.sh.jinja2 -md5sum = 7ba7ede9bf850551a9add58812e95d6e +md5sum = a8cb62a948300e6641af935e81ac4aef [instance-runner-export] filename = instance-runner-export.cfg.in -md5sum = 852a2ed99af566d27e5e4403334a3376 +md5sum = e9aa653417a85ed2a7a7ccfb64668ace [template-resilient] filename = instance-resilient.cfg.jinja2 diff --git a/software/slaprunner/common.cfg b/software/slaprunner/common.cfg index 80648236b..77076a4b7 100644 --- a/software/slaprunner/common.cfg +++ b/software/slaprunner/common.cfg @@ -2,6 +2,7 @@ extends = buildout.hash.cfg ../../component/bash/buildout.cfg + ../../component/coreutils/buildout.cfg ../../component/busybox/buildout.cfg ../../component/curl/buildout.cfg ../../component/dash/buildout.cfg diff --git a/software/slaprunner/instance-runner-export.cfg.in b/software/slaprunner/instance-runner-export.cfg.in index 9eaf31ee1..bb67f79ea 100644 --- a/software/slaprunner/instance-runner-export.cfg.in +++ b/software/slaprunner/instance-runner-export.cfg.in @@ -62,6 +62,9 @@ ip = ${slaprunner:ipv4} [supervisord] port = ${supervisord-free-port:port} +[exporter-configuration] +coreutils-location = {{ dumps(parameter_dict['coreutils-location']) }} + [exporter] recipe = slapos.recipe.template:jinja2 template = {{ exporter_script_path }} @@ -71,6 +74,7 @@ wrapper = ${:rendered} mode = 700 context = section directory directory + section parameter_dict exporter-configuration raw output_log_file ${directory:log}/resilient.log raw shell_binary {{ bash_executable_location }} raw rsync_binary {{ rsync_executable_location }} diff --git a/software/slaprunner/instance.cfg b/software/slaprunner/instance.cfg index a3999ead2..fdb6e138e 100644 --- a/software/slaprunner/instance.cfg +++ b/software/slaprunner/instance.cfg @@ -47,6 +47,9 @@ import-list = file parts :template-parts-destination file replicated :template-replicated-destination mode = 0644 +[template-runner-export-configuration] +coreutils-location = ${coreutils:location} + [template-runner-export] recipe = slapos.recipe.template:jinja2 template = ${instance-runner-export:target} @@ -55,6 +58,7 @@ mode = 640 context = key pbsready_export_template_path template-pbsready-export:rendered key template_runner_path instance-base-runner:rendered + section parameter_dict template-runner-export-configuration raw exporter_script_path ${template-runner-export-script:location}/${template-runner-export-script:filename} raw monitor_check_resilient_feed_template_path ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename} raw buildout_executable_location ${buildout:executable} diff --git a/software/slaprunner/template/runner-export.sh.jinja2 b/software/slaprunner/template/runner-export.sh.jinja2 index 7248cd27d..4490cab73 100644 --- a/software/slaprunner/template/runner-export.sh.jinja2 +++ b/software/slaprunner/template/runner-export.sh.jinja2 @@ -18,50 +18,57 @@ backup_directory='{{ directory["backup"] }}' etc_directory='{{ directory["etc"] }}' tmp_directory='{{ directory["tmp"] }}' -sync_element () { - path=$1 - backup_path=$2 - shift 2 - element_list=$* - - # Concatenate the exclude file of each partition of webrunner - # to create a global exclude file. - exclude_content="instance/supervisord.socket" - for partition in $srv_directory/runner/instance/slappart* - do - exclude_file="$partition/srv/exporter.exclude" - if [ -e "$exclude_file" ]; then - partition_exclude_content_relative=$(cat "$exclude_file") - # For every line of the local exclude file, add the absolute path - for line in "$partition_exclude_content_relative" - do - if [ ! -z "$line" ]; then - exclude_content="$exclude_content\ninstance/$(basename $partition)/$line" - fi - done - fi - done - echo -e "$exclude_content" > $srv_directory/exporter.exclude - - for element in $element_list - do - echo "Changing current directory to $path." - cd $path; - if [ -f $element ] || [ -d $element ]; then - command="{{ rsync_binary }} -rlptgov --stats --safe-links --exclude-from=$srv_directory/exporter.exclude --delete --delete-excluded $element $backup_path" - echo "Running : $command" - $command +rsync () { + set -x + '{{ rsync_binary }}' -rlptgov --stats --safe-links --delete --delete-excluded "$@" + set +x +} + +relativise () { + while IFS= read -r line; do + if [ ! -z "$line" ]; then + '{{ parameter_dict["coreutils-location"] }}/bin/realpath' --quiet --canonicalize-missing --no-symlinks --relative-to="$1" "$line" fi done } -sync_element $srv_directory/runner {{ directory['backup'] }}/runner/ instance project proxy.db + +( + path=$srv_directory/runner + backup_path=$backup_directory/runner/ + cd "$path" + + if [ -d instance ]; then + # Concatenate the exclude file of each partition of webrunner + # to create a global exclude file. + ( + echo "*.sock" + echo "*.socket" + echo "*.pid" + echo ".installed*.cfg" + for partition in "$path"/instance/slappart*; do + # So "relativise" can handle relative paths (which are expected to be relative to partition). + cd "$partition" + exclude_file=srv/exporter.exclude + if [ -r "$exclude_file" ]; then + relativise "$path" < "$exclude_file" + fi + done + ) | rsync --exclude-from=- instance "$backup_path" + fi + + test -d project && rsync project "$backup_path" + test -f proxy.db && rsync proxy.db "$backup_path" +) # We sync .* appart -date +%s -u > {{ directory['etc'] }}/.resilient-timestamp -sync_element {{ directory['etc'] }} {{ directory['backup'] }}/etc/ config.json -# Hidden files are related to the webrunner's internals -cp -r {{ directory['etc'] }}/.??* {{ directory['backup'] }}/etc/ -if [ -d {{ directory['backup'] }}/runner/software ]; then - rm {{ directory['backup'] }}/runner/software/* +( + cd "$etc_directory" + date +%s -u > .resilient-timestamp + rsync config.json "$backup_directory"/etc/ + # Hidden files are related to the webrunner's internals + cp -r .??* "$backup_directory/etc/" +) +if [ -d "$backup_directory"/runner/software ]; then + rm "$backup_directory"/runner/software/* fi ( -- 2.30.9