Retry rsync when source files vanish during backup

changelogs/unreleased/am-modifying-files-rb-to-retry-on-rsync-error.yml

+---
+title: Retry rsync when source files vanish during backup
+merge_request: 48568
+author:
+type: added

lib/backup/files.rb

@@ -26,9 +26,15 @@ module Backup
       FileUtils.rm_f(backup_tarball)
 
       if ENV['STRATEGY'] == 'copy'
-        cmd = [%w[rsync -a], exclude_dirs(:rsync), %W[#{app_files_dir} #{Gitlab.config.backup.path}]].flatten
+        cmd = [%w[rsync -a --delete], exclude_dirs(:rsync), %W[#{app_files_dir} #{Gitlab.config.backup.path}]].flatten
         output, status = Gitlab::Popen.popen(cmd)
 
+        # Retry if rsync source files vanish
+        if status == 24
+          $stdout.puts "Warning: files vanished during rsync, retrying..."
+          output, status = Gitlab::Popen.popen(cmd)
+        end
+
         unless status == 0
           puts output
           raise Backup::Error, 'Backup failed'

spec/lib/backup/files_spec.rb

@@ -149,13 +149,27 @@ RSpec.describe Backup::Files do
       end
 
       it 'excludes tmp dirs from rsync' do
-        expect(Gitlab::Popen).to receive(:popen).with(%w(rsync -a --exclude=lost+found --exclude=/@pages.tmp /var/gitlab-pages /var/gitlab-backup)).and_return(['', 0])
+        expect(Gitlab::Popen).to receive(:popen)
+          .with(%w(rsync -a --delete --exclude=lost+found --exclude=/@pages.tmp /var/gitlab-pages /var/gitlab-backup))
+          .and_return(['', 0])
 
         subject.dump
       end
 
+      it 'retries if rsync fails due to vanishing files' do
+        expect(Gitlab::Popen).to receive(:popen)
+          .with(%w(rsync -a --delete --exclude=lost+found --exclude=/@pages.tmp /var/gitlab-pages /var/gitlab-backup))
+          .and_return(['rsync failed', 24], ['', 0])
+
+        expect do
+          subject.dump
+        end.to output(/files vanished during rsync, retrying/).to_stdout
+      end
+
       it 'raises an error and outputs an error message if rsync failed' do
-        allow(Gitlab::Popen).to receive(:popen).with(%w(rsync -a --exclude=lost+found --exclude=/@pages.tmp /var/gitlab-pages /var/gitlab-backup)).and_return(['rsync failed', 1])
+        allow(Gitlab::Popen).to receive(:popen)
+          .with(%w(rsync -a --delete --exclude=lost+found --exclude=/@pages.tmp /var/gitlab-pages /var/gitlab-backup))
+          .and_return(['rsync failed', 1])
 
         expect do
           subject.dump