software/slaprunner: Rework rsync workhorse.

Do not apply exclusions to unrelated paths ("project", "proxy.db" and
"etc").
Simplify global exclusion file generation.
Add support for absolute and partition-relative exclusion paths
(previously, only partition-relative were supported).
Pipe exclusion file to rsync, to avoid an intermediate disk write.
Remove some directory prefix duplication by using PWD-relative paths
(except in one place, for runner-import.sh.jinja2 comparison).

software/slaprunner/buildout.hash.cfg

@@ -15,7 +15,7 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = 02755403c9f0b52d717160b0b2d0cfb7
+md5sum = afd426c01891d06e95f17b622ecd172f
 
 [template-runner]
 filename = instance-runner.cfg
@@ -31,11 +31,11 @@ md5sum = 9db9957f452bda370cb2d5cc2e833e85
 
 [template-runner-export-script]
 filename = template/runner-export.sh.jinja2
-md5sum = 7ba7ede9bf850551a9add58812e95d6e
+md5sum = a8cb62a948300e6641af935e81ac4aef
 
 [instance-runner-export]
 filename = instance-runner-export.cfg.in
-md5sum = 852a2ed99af566d27e5e4403334a3376
+md5sum = e9aa653417a85ed2a7a7ccfb64668ace
 
 [template-resilient]
 filename = instance-resilient.cfg.jinja2

software/slaprunner/common.cfg

@@ -2,6 +2,7 @@
 extends =
   buildout.hash.cfg
   ../../component/bash/buildout.cfg
+  ../../component/coreutils/buildout.cfg
   ../../component/busybox/buildout.cfg
   ../../component/curl/buildout.cfg
   ../../component/dash/buildout.cfg

software/slaprunner/instance-runner-export.cfg.in

@@ -62,6 +62,9 @@ ip = ${slaprunner:ipv4}
 [supervisord]
 port = ${supervisord-free-port:port}
 
+[exporter-configuration]
+coreutils-location = {{ dumps(parameter_dict['coreutils-location']) }}
+
 [exporter]
 recipe = slapos.recipe.template:jinja2
 template = {{ exporter_script_path }}
@@ -71,6 +74,7 @@ wrapper = ${:rendered}
 mode = 700
 context =
   section directory directory
+  section parameter_dict exporter-configuration
   raw  output_log_file ${directory:log}/resilient.log
   raw  shell_binary {{ bash_executable_location }}
   raw  rsync_binary {{ rsync_executable_location }}

software/slaprunner/instance.cfg

@@ -47,6 +47,9 @@ import-list = file parts :template-parts-destination
               file replicated :template-replicated-destination
 mode = 0644
 
+[template-runner-export-configuration]
+coreutils-location = ${coreutils:location}
+
 [template-runner-export]
 recipe = slapos.recipe.template:jinja2
 template = ${instance-runner-export:target}
@@ -55,6 +58,7 @@ mode = 640
 context =
   key pbsready_export_template_path template-pbsready-export:rendered
   key template_runner_path instance-base-runner:rendered
+  section parameter_dict template-runner-export-configuration
   raw exporter_script_path ${template-runner-export-script:location}/${template-runner-export-script:filename}
   raw monitor_check_resilient_feed_template_path ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
   raw buildout_executable_location ${buildout:executable}

software/slaprunner/template/runner-export.sh.jinja2

@@ -18,50 +18,57 @@ backup_directory='{{ directory["backup"] }}'
 etc_directory='{{ directory["etc"] }}'
 tmp_directory='{{ directory["tmp"] }}'
 
-sync_element () {
-  path=$1
-  backup_path=$2
-  shift 2
-  element_list=$*
-
-  # Concatenate the exclude file of each partition of webrunner
-  # to create a global exclude file.
-  exclude_content="instance/supervisord.socket"
-  for partition in $srv_directory/runner/instance/slappart*
-  do
-    exclude_file="$partition/srv/exporter.exclude"
-    if [ -e "$exclude_file" ]; then
-      partition_exclude_content_relative=$(cat "$exclude_file")
-      # For every line of the local exclude file, add the absolute path
-      for line in "$partition_exclude_content_relative"
-      do
-        if [ ! -z "$line" ]; then
-          exclude_content="$exclude_content\ninstance/$(basename $partition)/$line"
-       fi
-      done
-    fi
-  done
-  echo -e "$exclude_content" > $srv_directory/exporter.exclude
-
-  for element in $element_list
-  do
-    echo "Changing current directory to $path."
-    cd $path;
-    if [ -f $element ] || [ -d $element ]; then
-       command="{{ rsync_binary }} -rlptgov --stats --safe-links --exclude-from=$srv_directory/exporter.exclude --delete --delete-excluded $element  $backup_path"
-       echo "Running : $command"
-       $command
+rsync () {
+  set -x
+  '{{ rsync_binary }}' -rlptgov --stats --safe-links --delete --delete-excluded "$@"
+  set +x
+}
+
+relativise () {
+  while IFS= read -r line; do
+    if [ ! -z "$line" ]; then
+      '{{ parameter_dict["coreutils-location"] }}/bin/realpath' --quiet --canonicalize-missing --no-symlinks --relative-to="$1" "$line"
     fi
   done
 }
-sync_element $srv_directory/runner {{ directory['backup'] }}/runner/ instance project proxy.db
+
+(
+  path=$srv_directory/runner
+  backup_path=$backup_directory/runner/
+  cd "$path"
+
+  if [ -d instance ]; then
+    # Concatenate the exclude file of each partition of webrunner
+    # to create a global exclude file.
+    (
+      echo "*.sock"
+      echo "*.socket"
+      echo "*.pid"
+      echo ".installed*.cfg"
+      for partition in "$path"/instance/slappart*; do
+        # So "relativise" can handle relative paths (which are expected to be relative to partition).
+        cd "$partition"
+        exclude_file=srv/exporter.exclude
+        if [ -r "$exclude_file" ]; then
+          relativise "$path" < "$exclude_file"
+        fi
+      done
+    ) | rsync --exclude-from=- instance "$backup_path"
+  fi
+
+  test -d project  && rsync project "$backup_path"
+  test -f proxy.db && rsync proxy.db "$backup_path"
+)
 # We sync .* appart
-date +%s -u > {{ directory['etc'] }}/.resilient-timestamp
-sync_element {{ directory['etc'] }}  {{ directory['backup'] }}/etc/ config.json
-# Hidden files are related to the webrunner's internals
-cp -r {{ directory['etc'] }}/.??* {{ directory['backup'] }}/etc/
-if [ -d {{ directory['backup'] }}/runner/software ]; then
-  rm {{ directory['backup'] }}/runner/software/*
+(
+  cd "$etc_directory"
+  date +%s -u > .resilient-timestamp
+  rsync config.json "$backup_directory"/etc/
+  # Hidden files are related to the webrunner's internals
+  cp -r .??* "$backup_directory/etc/"
+)
+if [ -d "$backup_directory"/runner/software ]; then
+  rm "$backup_directory"/runner/software/*
 fi
 
 (