Merge branch 'gitaly-backup' into 'master'

Add a new repository backup strategy using gitaly-backup [RUN ALL RSPEC] [RUN AS-IF-FOSS]

See merge request gitlab-org/gitlab!61436

config/feature_flags/development/gitaly_backup.yml

+---
+name: gitaly_backup
+introduced_by_url: https://gitlab.com/gitlab-org/gitaly/-/merge_requests/3554
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/333034
+milestone: '14.0'
+type: development
+group: group::gitaly
+default_enabled: false

lib/backup/gitaly_backup.rb

+# frozen_string_literal: true
+
+module Backup
+  # Backup and restores repositories using gitaly-backup
+  class GitalyBackup
+    def initialize(progress)
+      @progress = progress
+    end
+
+    def start(type)
+      raise Error, 'already started' if started?
+
+      command = case type
+                when :create
+                  'create'
+                when :restore
+                  'restore'
+                else
+                  raise Error, "unknown backup type: #{type}"
+                end
+
+      @read_io, @write_io = IO.pipe
+      @pid = Process.spawn(bin_path, command, '-path', backup_repos_path, in: @read_io, out: progress)
+    end
+
+    def wait
+      return unless started?
+
+      @write_io.close
+      Process.wait(@pid)
+      status = $?
+
+      @pid = nil
+
+      raise Error, "gitaly-backup exit status #{status.exitstatus}" if status.exitstatus != 0
+    end
+
+    def enqueue(container, repo_type)
+      raise Error, 'not started' unless started?
+
+      repository = repo_type.repository_for(container)
+
+      @write_io.puts({
+        storage_name: repository.storage,
+        relative_path: repository.relative_path,
+        gl_project_path: repository.gl_project_path,
+        always_create: repo_type.project?
+      }.merge(Gitlab::GitalyClient.connection_data(repository.storage)).to_json)
+    end
+
+    private
+
+    attr_reader :progress
+
+    def started?
+      @pid.present?
+    end
+
+    def backup_repos_path
+      File.absolute_path(File.join(Gitlab.config.backup.path, 'repositories'))
+    end
+
+    def bin_path
+      File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-backup'))
+    end
+  end
+end

lib/backup/repositories.rb

@@ -4,7 +4,7 @@ require 'yaml'
 
 module Backup
   class Repositories
-    def initialize(progress, strategy: GitalyRpcBackup.new(progress))
+    def initialize(progress, strategy:)
       @progress = progress
       @strategy = strategy
     end

lib/tasks/gitlab/backup.rake

@@ -109,7 +109,7 @@ namespace :gitlab do
           puts "GITLAB_BACKUP_MAX_CONCURRENCY and GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY must have a value of at least 1".color(:red)
           exit 1
         else
-          Backup::Repositories.new(progress).dump(
+          Backup::Repositories.new(progress, strategy: repository_backup_strategy).dump(
             max_concurrency: max_concurrency,
             max_storage_concurrency: max_storage_concurrency
           )
@@ -119,7 +119,7 @@ namespace :gitlab do
 
       task restore: :gitlab_environment do
         puts_time "Restoring repositories ...".color(:blue)
-        Backup::Repositories.new(progress).restore
+        Backup::Repositories.new(progress, strategy: repository_backup_strategy).restore
         puts_time "done".color(:green)
       end
     end
@@ -294,6 +294,14 @@ namespace :gitlab do
         $stdout
       end
     end
+
+    def repository_backup_strategy
+      if Feature.enabled?(:gitaly_backup)
+        Backup::GitalyBackup.new(progress)
+      else
+        Backup::GitalyRpcBackup.new(progress)
+      end
+    end
   end
   # namespace end: backup
 end

spec/lib/backup/gitaly_backup_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Backup::GitalyBackup do
+  let(:progress) do
+    Tempfile.new('progress').tap do |progress|
+      progress.unlink
+    end
+  end
+
+  after do
+    progress.close
+  end
+
+  subject { described_class.new(progress) }
+
+  context 'unknown' do
+    it 'fails to start unknown' do
+      expect { subject.start(:unknown) }.to raise_error(::Backup::Error, 'unknown backup type: unknown')
+    end
+  end
+
+  context 'create' do
+    RSpec.shared_examples 'creates a repository backup' do
+      it 'creates repository bundles', :aggregate_failures do
+        # Add data to the wiki, design repositories, and snippets, so they will be included in the dump.
+        create(:wiki_page, container: project)
+        create(:design, :with_file, issue: create(:issue, project: project))
+        project_snippet = create(:project_snippet, :repository, project: project)
+        personal_snippet = create(:personal_snippet, :repository, author: project.owner)
+
+        subject.start(:create)
+        subject.enqueue(project, Gitlab::GlRepository::PROJECT)
+        subject.enqueue(project, Gitlab::GlRepository::WIKI)
+        subject.enqueue(project, Gitlab::GlRepository::DESIGN)
+        subject.enqueue(personal_snippet, Gitlab::GlRepository::SNIPPET)
+        subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET)
+        subject.wait
+
+        expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.bundle'))
+        expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.wiki.bundle'))
+        expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.design.bundle'))
+        expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', personal_snippet.disk_path + '.bundle'))
+        expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project_snippet.disk_path + '.bundle'))
+      end
+
+      it 'raises when the exit code not zero' do
+        expect(subject).to receive(:bin_path).and_return(Gitlab::Utils.which('false'))
+
+        subject.start(:create)
+        expect { subject.wait }.to raise_error(::Backup::Error, 'gitaly-backup exit status 1')
+      end
+    end
+
+    context 'hashed storage' do
+      let_it_be(:project) { create(:project, :repository) }
+
+      it_behaves_like 'creates a repository backup'
+    end
+
+    context 'legacy storage' do
+      let_it_be(:project) { create(:project, :repository, :legacy_storage) }
+
+      it_behaves_like 'creates a repository backup'
+    end
+  end
+
+  context 'restore' do
+    let_it_be(:project) { create(:project, :repository) }
+    let_it_be(:personal_snippet) { create(:personal_snippet, author: project.owner) }
+    let_it_be(:project_snippet) { create(:project_snippet, project: project, author: project.owner) }
+
+    def copy_bundle_to_backup_path(bundle_name, destination)
+      FileUtils.mkdir_p(File.join(Gitlab.config.backup.path, 'repositories', File.dirname(destination)))
+      FileUtils.cp(Rails.root.join('spec/fixtures/lib/backup', bundle_name), File.join(Gitlab.config.backup.path, 'repositories', destination))
+    end
+
+    it 'restores from repository bundles', :aggregate_failures do
+      copy_bundle_to_backup_path('project_repo.bundle', project.disk_path + '.bundle')
+      copy_bundle_to_backup_path('wiki_repo.bundle', project.disk_path + '.wiki.bundle')
+      copy_bundle_to_backup_path('design_repo.bundle', project.disk_path + '.design.bundle')
+      copy_bundle_to_backup_path('personal_snippet_repo.bundle', personal_snippet.disk_path + '.bundle')
+      copy_bundle_to_backup_path('project_snippet_repo.bundle', project_snippet.disk_path + '.bundle')
+
+      subject.start(:restore)
+      subject.enqueue(project, Gitlab::GlRepository::PROJECT)
+      subject.enqueue(project, Gitlab::GlRepository::WIKI)
+      subject.enqueue(project, Gitlab::GlRepository::DESIGN)
+      subject.enqueue(personal_snippet, Gitlab::GlRepository::SNIPPET)
+      subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET)
+      subject.wait
+
+      collect_commit_shas = -> (repo) { repo.commits('master', limit: 10).map(&:sha) }
+
+      expect(collect_commit_shas.call(project.repository)).to eq(['393a7d860a5a4c3cc736d7eb00604e3472bb95ec'])
+      expect(collect_commit_shas.call(project.wiki.repository)).to eq(['c74b9948d0088d703ee1fafeddd9ed9add2901ea'])
+      expect(collect_commit_shas.call(project.design_repository)).to eq(['c3cd4d7bd73a51a0f22045c3a4c871c435dc959d'])
+      expect(collect_commit_shas.call(personal_snippet.repository)).to eq(['3b3c067a3bc1d1b695b51e2be30c0f8cf698a06e'])
+      expect(collect_commit_shas.call(project_snippet.repository)).to eq(['6e44ba56a4748be361a841e759c20e421a1651a1'])
+    end
+
+    it 'raises when the exit code not zero' do
+      expect(subject).to receive(:bin_path).and_return(Gitlab::Utils.which('false'))
+
+      subject.start(:restore)
+      expect { subject.wait }.to raise_error(::Backup::Error, 'gitaly-backup exit status 1')
+    end
+  end
+end

spec/tasks/gitlab/backup_rake_spec.rb

 # frozen_string_literal: true
 
-require 'spec_helper'
-require 'rake'
+require 'rake_helper'
 
 RSpec.describe 'gitlab:app namespace rake task', :delete do
   let(:enable_registry) { true }
@@ -24,14 +23,10 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
 
   before(:all) do
     Rake.application.rake_require 'active_record/railties/databases'
-    Rake.application.rake_require 'tasks/gitlab/helpers'
     Rake.application.rake_require 'tasks/gitlab/backup'
     Rake.application.rake_require 'tasks/gitlab/shell'
     Rake.application.rake_require 'tasks/gitlab/db'
     Rake.application.rake_require 'tasks/cache'
-
-    # empty task as env is already loaded
-    Rake::Task.define_task :environment
   end
 
   before do
@@ -39,6 +34,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
     FileUtils.rm(tars_glob, force: true)
     FileUtils.rm(backup_files, force: true)
     FileUtils.rm_rf(backup_directories, secure: true)
+    FileUtils.mkdir_p('tmp/tests/public/uploads')
     reenable_backup_sub_tasks
     stub_container_registry_config(enabled: enable_registry)
   end
@@ -47,12 +43,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
     FileUtils.rm(tars_glob, force: true)
     FileUtils.rm(backup_files, force: true)
     FileUtils.rm_rf(backup_directories, secure: true)
-  end
-
-  def run_rake_task(task_name)
-    FileUtils.mkdir_p('tmp/tests/public/uploads')
-    Rake::Task[task_name].reenable
-    Rake.application.invoke_task task_name
+    FileUtils.rm_rf('tmp/tests/public/uploads', secure: true)
   end
 
   def reenable_backup_sub_tasks
@@ -92,11 +83,11 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         end
 
         it 'invokes restoration on match' do
-          expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout
+          expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout_from_any_process
         end
 
         it 'prints timestamps on messages' do
-          expect { run_rake_task('gitlab:backup:restore') }.to output(/.*\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}\s[-+]\d{4}\s--\s.*/).to_stdout
+          expect { run_rake_task('gitlab:backup:restore') }.to output(/.*\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}\s[-+]\d{4}\s--\s.*/).to_stdout_from_any_process
         end
       end
     end
@@ -105,14 +96,16 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
       before do
         # We only need a backup of the repositories for this test
         stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
+
+        create(:project, :repository)
       end
 
       it 'removes stale data' do
-        expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
+        expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
 
         excluded_project = create(:project, :repository, name: 'mepmep')
 
-        expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout
+        expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout_from_any_process
 
         raw_repo = excluded_project.repository.raw
 
@@ -124,9 +117,10 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
 
     context 'when the backup is restored' do
       let!(:included_project) { create(:project, :repository) }
+      let!(:original_checksum) { included_project.repository.checksum }
 
       before do
-        expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
+        expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
 
         backup_tar = Dir.glob(File.join(Gitlab.config.backup.path, '*_gitlab_backup.tar')).last
         allow(Dir).to receive(:glob).and_return([backup_tar])
@@ -153,11 +147,12 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
       end
 
       it 'restores the data' do
-        expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout
+        expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout_from_any_process
 
         raw_repo = included_project.repository.raw
 
         expect(raw_repo.empty?).to be(false)
+        expect(included_project.repository.checksum).to eq(original_checksum)
       end
     end
   end
@@ -169,8 +164,9 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
       allow(ActiveRecord::Base.connection).to receive(:reconnect!)
     end
 
+    let!(:project) { create(:project, :repository) }
+
     describe 'backup creation and deletion using custom_hooks' do
-      let(:project) { create(:project, :repository) }
       let(:user_backup_path) { "repositories/#{project.disk_path}" }
 
       before do
@@ -184,7 +180,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
 
       context 'project uses custom_hooks and successfully creates backup' do
         it 'creates custom_hooks.tar and project bundle' do
-          expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
+          expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
 
           tar_contents, exit_status = Gitlab::Popen.popen(%W{tar -tvf #{backup_tar}})
 
@@ -195,8 +191,8 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         end
 
         it 'restores files correctly' do
-          expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
-          expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout
+          expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
+          expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout_from_any_process
 
           repo_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
             project.repository.path
@@ -210,7 +206,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
 
         it 'prints a progress message to stdout' do
           task_list.each do |task|
-            expect { run_rake_task("gitlab:backup:#{task}:create") }.to output(/Dumping /).to_stdout
+            expect { run_rake_task("gitlab:backup:#{task}:create") }.to output(/Dumping /).to_stdout_from_any_process
           end
         end
       end
@@ -219,7 +215,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
     context 'tar creation' do
       context 'archive file permissions' do
         it 'sets correct permissions on the tar file' do
-          expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
+          expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
 
           expect(File.exist?(backup_tar)).to be_truthy
           expect(File::Stat.new(backup_tar).mode.to_s(8)).to eq('100600')
@@ -231,7 +227,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
           end
 
           it 'uses the custom permissions' do
-            expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
+            expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
 
             expect(File::Stat.new(backup_tar).mode.to_s(8)).to eq('100651')
           end
@@ -239,7 +235,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
       end
 
       it 'sets correct permissions on the tar contents' do
-        expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
+        expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
 
         tar_contents, exit_status = Gitlab::Popen.popen(
           %W{tar -tvf #{backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz registry.tar.gz}
@@ -258,7 +254,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
       end
 
       it 'deletes temp directories' do
-        expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
+        expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
 
         temp_dirs = Dir.glob(
           File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,pages,lfs,registry}')
@@ -271,7 +267,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         let(:enable_registry) { false }
 
         it 'does not create registry.tar.gz' do
-          expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
+          expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
 
           tar_contents, exit_status = Gitlab::Popen.popen(
             %W{tar -tvf #{backup_tar}}
@@ -311,7 +307,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
           move_repository_to_secondary(project_b)
           move_repository_to_secondary(project_snippet_b)
 
-          expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
+          expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
 
           tar_contents, exit_status = Gitlab::Popen.popen(
             %W{tar -tvf #{backup_tar} repositories}
@@ -377,7 +373,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
             .and_call_original
         end
 
-        expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
+        expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
       end
 
       it 'passes through concurrency environment variables' do
@@ -390,7 +386,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
             .and_call_original
         end
 
-        expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
+        expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
       end
     end
   end
@@ -399,10 +395,12 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
   describe "Skipping items" do
     before do
       stub_env('SKIP', 'repositories,uploads')
+
+      create(:project, :repository)
     end
 
     it "does not contain skipped item" do
-      expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
+      expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
 
       tar_contents, _exit_status = Gitlab::Popen.popen(
         %W{tar -tvf #{backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz registry.tar.gz}
@@ -419,7 +417,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
     end
 
     it 'does not invoke repositories restore' do
-      expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
+      expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
 
       allow(Rake::Task['gitlab:shell:setup'])
         .to receive(:invoke).and_return(true)
@@ -434,17 +432,19 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
       expect(Rake::Task['gitlab:backup:lfs:restore']).to receive :invoke
       expect(Rake::Task['gitlab:backup:registry:restore']).to receive :invoke
       expect(Rake::Task['gitlab:shell:setup']).to receive :invoke
-      expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout
+      expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout_from_any_process
     end
   end
 
   describe 'skipping tar archive creation' do
     before do
       stub_env('SKIP', 'tar')
+
+      create(:project, :repository)
     end
 
     it 'created files with backup content and no tar archive' do
-      expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
+      expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
 
       dir_contents = Dir.children(Gitlab.config.backup.path)
 
@@ -463,7 +463,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
     end
 
     it 'those component files can be restored from' do
-      expect { run_rake_task("gitlab:backup:create") }.to output.to_stdout
+      expect { run_rake_task("gitlab:backup:create") }.to output.to_stdout_from_any_process
 
       allow(Rake::Task['gitlab:shell:setup'])
         .to receive(:invoke).and_return(true)
@@ -478,13 +478,13 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
       expect(Rake::Task['gitlab:backup:lfs:restore']).to receive :invoke
       expect(Rake::Task['gitlab:backup:registry:restore']).to receive :invoke
       expect(Rake::Task['gitlab:shell:setup']).to receive :invoke
-      expect { run_rake_task("gitlab:backup:restore") }.to output.to_stdout
+      expect { run_rake_task("gitlab:backup:restore") }.to output.to_stdout_from_any_process
     end
   end
 
   describe "Human Readable Backup Name" do
     it 'name has human readable time' do
-      expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
+      expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
 
       expect(backup_tar).to match(/\d+_\d{4}_\d{2}_\d{2}_\d+\.\d+\.\d+.*_gitlab_backup.tar$/)
     end