runner-export.sh.jinja2

#!{{ shell_binary }}
LC_ALL=C
export LC_ALL
umask 077

# Exit on any error, to prevent inconsistent backup
# Error on unset variable expansion
set -eu

# Redirect output to log
exec > >(tee -ai '{{ output_log_file }}')
exec 2>&1

echo -e "\n\n$0 run at : $(date)"

srv_directory='{{ directory["srv"] }}'
backup_directory='{{ directory["backup"] }}'
etc_directory='{{ directory["etc"] }}'
tmp_directory='{{ directory["tmp"] }}'

rsync () {
  set -x
  '{{ rsync_binary }}' -rlptgov --stats --safe-links --delete --delete-excluded "$@"
  set +x
}

relativise () {
  while IFS= read -r line; do
    if [ ! -z "$line" ]; then
      '{{ parameter_dict["coreutils-location"] }}/bin/realpath' --quiet --canonicalize-missing --no-symlinks --relative-to="$1" "$line"
    fi
  done
}

(
  path=$srv_directory/runner
  backup_path=$backup_directory/runner/
  cd "$path"

  if [ -d instance ]; then
    # Concatenate the exclude file of each partition of webrunner
    # to create a global exclude file.
    # Also, ignore all buildout-managed files.
    (
      echo "*.sock"
      echo "*.socket"
      echo "*.pid"
      echo ".installed*.cfg"
      for partition in "$path"/instance/slappart*; do
        # So "relativise" can handle relative paths (which are expected to be relative to partition).
        cd "$partition"
        exclude_file=srv/exporter.exclude
        if [ -r "$exclude_file" ]; then
          relativise "$path" < "$exclude_file"
        fi
        for installed in .installed*.cfg; do
          if [ -r "$installed" ]; then
            # Print every line from each __buildout_installed__ found.
            '{{ parameter_dict["gawk-location"] }}/bin/gawk' '
              BEGIN { do_print = 0 }
              match($0, /^__buildout_installed__\s*=\s*(\S.*)/, ary) { do_print = 1; print ary[1]; next }
              /^\S/ { do_print = 0; next }
              match($0, /^\s+(\S.*)/, ary) { if (do_print) print ary[1] }
            ' "$installed" | relativise "$path"
          fi
        done
      done
    ) | rsync --exclude-from=- instance "$backup_path"
  fi

  test -d project  && rsync project "$backup_path"
  test -f proxy.db && rsync proxy.db "$backup_path"
)
# We sync .* appart
(
  cd "$etc_directory"
  date +%s -u > .resilient-timestamp
  rsync config.json "$backup_directory"/etc/
  # Hidden files are related to the webrunner's internals
  cp -r .??* "$backup_directory/etc/"
)
if [ -d "$backup_directory"/runner/software ]; then
  rm "$backup_directory"/runner/software/*
fi

(
  cd "$backup_directory"
  find -type f ! -name backup.signature -print0 | xargs -0 sha256sum | sort -k 66 > backup.signature
)

# Check that export didn't happen during backup of instances
tmp_backup_sum=$(mktemp -p "$tmp_directory")
tmp_filtered_signature=$(mktemp -p "$tmp_directory")

remove_tmp_files () {
  rm "$tmp_backup_sum" "$tmp_filtered_signature"
}
trap remove_tmp_files EXIT

# Getting files from runner backup directory, as instance backup files may be
# explicitely excluded from the backup, using the srv/exporter.exclude
cd {{ directory['backup'] }}
backup_directory_path=$(find . -path "./runner/instance/slappart*/srv/backup/*" -type f)

# If no backup found, it's over
if [ -z "$backup_directory_path" ]; then
  exit 0
fi

sleep 5
sha256sum "$backup_directory_path" | sort -k 66 > "$tmp_backup_sum"
egrep "instance/slappart.*/srv/backup/" "$backup_directory/backup.signature" > "$tmp_filtered_signature"

# If the diff fails, then the notifier will restart this script
if diff "$tmp_backup_sum" "$tmp_filtered_signature"; then
  exit 0
fi
echo "ERROR: Some backups are not consistent, exporter should be re-run."
echo "Let's sleep 10 minutes, to let the backup end..."
sleep 10m
exit 1