runner-export.sh.jinja2 3.63 KB
Newer Older
1
#!{{ shell_binary }}
2 3
LC_ALL=C
export LC_ALL
4
umask 077
5

6
# Exit on any error, to prevent inconsistent backup
7 8
# Error on unset variable expansion
set -eu
9

10
# Redirect output to log
11
exec > >(tee -ai '{{ output_log_file }}')
12 13 14 15
exec 2>&1

echo -e "\n\n$0 run at : $(date)"

16 17 18 19
srv_directory='{{ directory["srv"] }}'
backup_directory='{{ directory["backup"] }}'
etc_directory='{{ directory["etc"] }}'
tmp_directory='{{ directory["tmp"] }}'
20

21 22 23 24 25
rsync () {
  set -x
  '{{ rsync_binary }}' -rlptgov --stats --safe-links --delete --delete-excluded "$@"
  set +x
}
26

27 28 29 30
relativise () {
  while IFS= read -r line; do
    if [ ! -z "$line" ]; then
      '{{ parameter_dict["coreutils-location"] }}/bin/realpath' --quiet --canonicalize-missing --no-symlinks --relative-to="$1" "$line"
31 32 33
    fi
  done
}
34 35 36 37 38 39 40 41 42

(
  path=$srv_directory/runner
  backup_path=$backup_directory/runner/
  cd "$path"

  if [ -d instance ]; then
    # Concatenate the exclude file of each partition of webrunner
    # to create a global exclude file.
43
    # Also, ignore all buildout-managed files.
44 45 46 47 48 49 50 51 52 53 54 55
    (
      echo "*.sock"
      echo "*.socket"
      echo "*.pid"
      echo ".installed*.cfg"
      for partition in "$path"/instance/slappart*; do
        # So "relativise" can handle relative paths (which are expected to be relative to partition).
        cd "$partition"
        exclude_file=srv/exporter.exclude
        if [ -r "$exclude_file" ]; then
          relativise "$path" < "$exclude_file"
        fi
56 57 58 59 60 61 62 63 64 65 66
        for installed in .installed*.cfg; do
          if [ -r "$installed" ]; then
            # Print every line from each __buildout_installed__ found.
            '{{ parameter_dict["gawk-location"] }}/bin/gawk' '
              BEGIN { do_print = 0 }
              match($0, /^__buildout_installed__\s*=\s*(\S.*)/, ary) { do_print = 1; print ary[1]; next }
              /^\S/ { do_print = 0; next }
              match($0, /^\s+(\S.*)/, ary) { if (do_print) print ary[1] }
            ' "$installed" | relativise "$path"
          fi
        done
67 68 69 70 71 72 73
      done
    ) | rsync --exclude-from=- instance "$backup_path"
  fi

  test -d project  && rsync project "$backup_path"
  test -f proxy.db && rsync proxy.db "$backup_path"
)
74
# We sync .* appart
75 76 77 78 79 80 81 82 83
(
  cd "$etc_directory"
  date +%s -u > .resilient-timestamp
  rsync config.json "$backup_directory"/etc/
  # Hidden files are related to the webrunner's internals
  cp -r .??* "$backup_directory/etc/"
)
if [ -d "$backup_directory"/runner/software ]; then
  rm "$backup_directory"/runner/software/*
84
fi
85

86 87
(
  cd "$backup_directory"
88
  find -type f ! -name backup.signature -print0 | xargs -0 sha256sum | sort -k 66 > backup.signature
89
)
90 91

# Check that export didn't happen during backup of instances
92 93
tmp_backup_sum=$(mktemp -p "$tmp_directory")
tmp_filtered_signature=$(mktemp -p "$tmp_directory")
94 95

remove_tmp_files () {
96
  rm "$tmp_backup_sum" "$tmp_filtered_signature"
97 98 99
}
trap remove_tmp_files EXIT

100 101 102
# Getting files from runner backup directory, as instance backup files may be
# explicitely excluded from the backup, using the srv/exporter.exclude
cd {{ directory['backup'] }}
103 104 105 106 107 108 109 110
backup_directory_path=$(find . -path "./runner/instance/slappart*/srv/backup/*" -type f)

# If no backup found, it's over
if [ -z "$backup_directory_path" ]; then
  exit 0
fi

sleep 5
111
sha256sum "$backup_directory_path" | sort -k 66 > "$tmp_backup_sum"
112
egrep "instance/slappart.*/srv/backup/" "$backup_directory/backup.signature" > "$tmp_filtered_signature"
113 114

# If the diff fails, then the notifier will restart this script
115 116
if diff "$tmp_backup_sum" "$tmp_filtered_signature"; then
  exit 0
117
fi
118 119 120 121
echo "ERROR: Some backups are not consistent, exporter should be re-run."
echo "Let's sleep 10 minutes, to let the backup end..."
sleep 10m
exit 1