Merge branch 'port_repo_backup_to_gl_repo' into 'master'

Port repository backup/restore to use GlRepository

See merge request gitlab-org/gitlab!40682

lib/backup/repository.rb → lib/backup/repositories.rb

@@ -3,7 +3,7 @@
 require 'yaml'
 
 module Backup
-  class Repository
+  class Repositories
     attr_reader :progress
 
     def initialize(progress)
@@ -41,98 +41,23 @@ module Backup
       raise errors.pop unless errors.empty?
     end
 
-    def backup_project(project)
-      path_to_project_bundle = path_to_bundle(project)
-      Gitlab::GitalyClient::RepositoryService.new(project.repository)
-        .create_bundle(path_to_project_bundle)
-
-      backup_custom_hooks(project)
-    rescue => e
-      progress_warn(project, e, 'Failed to backup repo')
-    end
-
-    def backup_custom_hooks(project)
-      FileUtils.mkdir_p(project_backup_path(project))
-
-      custom_hooks_path = custom_hooks_tar(project)
-      Gitlab::GitalyClient::RepositoryService.new(project.repository)
-        .backup_custom_hooks(custom_hooks_path)
-    end
-
-    def restore_custom_hooks(project)
-      return unless Dir.exist?(project_backup_path(project))
-      return if Dir.glob("#{project_backup_path(project)}/custom_hooks*").none?
-
-      custom_hooks_path = custom_hooks_tar(project)
-      Gitlab::GitalyClient::RepositoryService.new(project.repository)
-        .restore_custom_hooks(custom_hooks_path)
-    end
-
     def restore
       Project.find_each(batch_size: 1000) do |project|
-        progress.print " * #{project.full_path} ... "
-
-        restore_repo_success =
-          begin
-            try_restore_repository(project)
-          rescue => err
-            progress.puts "Error: #{err}".color(:red)
-            false
-          end
-
-        if restore_repo_success
-          progress.puts "[DONE]".color(:green)
-        else
-          progress.puts "[Failed] restoring #{project.full_path} repository".color(:red)
-        end
-
-        wiki = ProjectWiki.new(project)
-        wiki.repository.remove rescue nil
-        path_to_wiki_bundle = path_to_bundle(wiki)
-
-        if File.exist?(path_to_wiki_bundle)
-          progress.print " * #{wiki.full_path} ... "
-          begin
-            wiki.repository.create_from_bundle(path_to_wiki_bundle)
-            restore_custom_hooks(wiki)
-
-            progress.puts "[DONE]".color(:green)
-          rescue => e
-            progress.puts "[Failed] restoring #{wiki.full_path} wiki".color(:red)
-            progress.puts "Error #{e}".color(:red)
-          end
-        end
+        restore_repository(project, Gitlab::GlRepository::PROJECT)
+        restore_repository(project, Gitlab::GlRepository::WIKI)
       end
 
       restore_object_pools
     end
 
-    protected
-
-    def try_restore_repository(project)
-      path_to_project_bundle = path_to_bundle(project)
-      project.repository.remove rescue nil
-
-      if File.exist?(path_to_project_bundle)
-        project.repository.create_from_bundle(path_to_project_bundle)
-        restore_custom_hooks(project)
-      else
-        project.repository.create_repository
-      end
-
-      true
-    end
-
-    def path_to_bundle(project)
-      File.join(backup_repos_path, project.disk_path + '.bundle')
-    end
-
-    def project_backup_path(project)
-      File.join(backup_repos_path, project.disk_path)
-    end
+    private
 
-    def custom_hooks_tar(project)
-      File.join(project_backup_path(project), "custom_hooks.tar")
+    def restore_repository(container, type)
+      BackupRestore.new(
+        progress,
+        type.repository_for(container),
+        backup_repos_path
+      ).restore(always_create: type.project?)
     end
 
     def backup_repos_path
@@ -145,8 +70,6 @@ module Backup
       FileUtils.mkdir(backup_repos_path, mode: 0700)
     end
 
-    private
-
     def dump_consecutive
       Project.includes(:route, :group, namespace: :owner).find_each(batch_size: 1000) do |project|
         dump_project(project)
@@ -193,43 +116,16 @@ module Backup
     end
 
     def dump_project(project)
-      progress.puts " * #{display_repo_path(project)} ... "
-
-      if project.hashed_storage?(:repository)
-        FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path)))
-      else
-        FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace
-      end
-
-      if !empty_repo?(project)
-        backup_project(project)
-        progress.puts " * #{display_repo_path(project)} ... " + "[DONE]".color(:green)
-      else
-        progress.puts " * #{display_repo_path(project)} ... " + "[SKIPPED]".color(:cyan)
-      end
-
-      wiki = ProjectWiki.new(project)
-
-      if !empty_repo?(wiki)
-        backup_project(wiki)
-        progress.puts " * #{display_repo_path(project)} ... " + "[DONE] Wiki".color(:green)
-      else
-        progress.puts " * #{display_repo_path(project)} ... " + "[SKIPPED] Wiki".color(:cyan)
-      end
-    end
-
-    def progress_warn(project, cmd, output)
-      progress.puts "[WARNING] Executing #{cmd}".color(:orange)
-      progress.puts "Ignoring error on #{display_repo_path(project)} - #{output}".color(:orange)
-    end
-
-    def empty_repo?(project_or_wiki)
-      project_or_wiki.repository.expire_emptiness_caches
-      project_or_wiki.repository.empty?
+      backup_repository(project, Gitlab::GlRepository::PROJECT)
+      backup_repository(project, Gitlab::GlRepository::WIKI)
     end
 
-    def display_repo_path(project)
-      project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path
+    def backup_repository(container, type)
+      BackupRestore.new(
+        progress,
+        type.repository_for(container),
+        backup_repos_path
+      ).backup
     end
 
     def restore_object_pools
@@ -244,6 +140,79 @@ module Backup
       end
     end
 
+    class BackupRestore
+      attr_accessor :progress, :repository, :backup_repos_path
+
+      def initialize(progress, repository, backup_repos_path)
+        @progress = progress
+        @repository = repository
+        @backup_repos_path = backup_repos_path
+      end
+
+      def backup
+        progress.puts " * #{display_repo_path} ... "
+
+        if repository.empty?
+          progress.puts " * #{display_repo_path} ... " + "[SKIPPED]".color(:cyan)
+          return
+        end
+
+        FileUtils.mkdir_p(repository_backup_path)
+
+        repository.bundle_to_disk(path_to_bundle)
+        repository.gitaly_repository_client.backup_custom_hooks(custom_hooks_tar)
+
+        progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green)
+
+      rescue => e
+        progress.puts "[Failed] backing up #{display_repo_path}".color(:red)
+        progress.puts "Error #{e}".color(:red)
+      end
+
+      def restore(always_create: false)
+        progress.puts " * #{display_repo_path} ... "
+
+        repository.remove rescue nil
+
+        if File.exist?(path_to_bundle)
+          repository.create_from_bundle(path_to_bundle)
+          restore_custom_hooks
+        elsif always_create
+          repository.create_repository
+        end
+
+        progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green)
+
+      rescue => e
+        progress.puts "[Failed] restoring #{display_repo_path}".color(:red)
+        progress.puts "Error #{e}".color(:red)
+      end
+
+      private
+
+      def display_repo_path
+        "#{repository.full_path} (#{repository.disk_path})"
+      end
+
+      def repository_backup_path
+        @repository_backup_path ||= File.join(backup_repos_path, repository.disk_path)
+      end
+
+      def path_to_bundle
+        @path_to_bundle ||= File.join(backup_repos_path, repository.disk_path + '.bundle')
+      end
+
+      def restore_custom_hooks
+        return unless File.exist?(custom_hooks_tar)
+
+        repository.gitaly_repository_client.restore_custom_hooks(custom_hooks_tar)
+      end
+
+      def custom_hooks_tar
+        File.join(repository_backup_path, "custom_hooks.tar")
+      end
+    end
+
     class InterlockSizedQueue < SizedQueue
       extend ::Gitlab::Utils::Override
 

lib/tasks/gitlab/backup.rake

@@ -107,7 +107,7 @@ namespace :gitlab do
           puts "GITLAB_BACKUP_MAX_CONCURRENCY and GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY must have a value of at least 1".color(:red)
           exit 1
         else
-          Backup::Repository.new(progress).dump(
+          Backup::Repositories.new(progress).dump(
             max_concurrency: max_concurrency,
             max_storage_concurrency: max_storage_concurrency
           )
@@ -117,7 +117,7 @@ namespace :gitlab do
 
       task restore: :gitlab_environment do
         puts_time "Restoring repositories ...".color(:blue)
-        Backup::Repository.new(progress).restore
+        Backup::Repositories.new(progress).restore
         puts_time "done".color(:green)
       end
     end

spec/lib/backup/repository_spec.rb → spec/lib/backup/repositories_spec.rb

@@ -2,9 +2,7 @@
 
 require 'spec_helper'
 
-RSpec.describe Backup::Repository do
-  let_it_be(:project) { create(:project, :wiki_repo) }
-
+RSpec.describe Backup::Repositories do
   let(:progress) { StringIO.new }
 
   subject { described_class.new(progress) }
@@ -12,7 +10,6 @@ RSpec.describe Backup::Repository do
   before do
     allow(progress).to receive(:puts)
     allow(progress).to receive(:print)
-    allow(FileUtils).to receive(:mv).and_return(true)
 
     allow_next_instance_of(described_class) do |instance|
       allow(instance).to receive(:progress).and_return(progress)
@@ -24,11 +21,19 @@ RSpec.describe Backup::Repository do
       allow(Gitlab.config.repositories.storages).to receive(:keys).and_return(storage_keys)
     end
 
-    let_it_be(:projects) { create_list(:project, 5, :wiki_repo) + [project] }
+    let_it_be(:projects) { create_list(:project, 5, :repository, :wiki_repo) }
 
     let(:storage_keys) { %w[default test_second_storage] }
 
     context 'no concurrency' do
+      it 'creates repository bundle' do
+        subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+
+        projects.each do |project|
+          expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.bundle'))
+        end
+      end
+
       it 'creates the expected number of threads' do
         expect(Thread).not_to receive(:new)
 
@@ -58,12 +63,22 @@ RSpec.describe Backup::Repository do
           subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
         end.count
 
-        create_list(:project, 2, :wiki_repo)
+        create_list(:project, 2, :repository, :wiki_repo)
 
         expect do
           subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
         end.not_to exceed_query_limit(control_count)
       end
+
+      context 'legacy storage' do
+        let_it_be(:project) { create(:project, :repository, :legacy_storage, :wiki_repo) }
+
+        it 'creates repository bundle' do
+          subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+
+          expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.bundle'))
+        end
+      end
     end
 
     [4, 10].each do |max_storage_concurrency|
@@ -120,7 +135,7 @@ RSpec.describe Backup::Repository do
             subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency)
           end.count
 
-          create_list(:project, 2, :wiki_repo)
+          create_list(:project, 2, :repository, :wiki_repo)
 
           expect do
             subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency)
@@ -131,45 +146,32 @@ RSpec.describe Backup::Repository do
   end
 
   describe '#restore' do
-    let(:timestamp) { Time.utc(2017, 3, 22) }
-    let(:temp_dirs) do
-      Gitlab.config.repositories.storages.map do |name, storage|
-        Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-          File.join(storage.legacy_disk_path, '..', 'repositories.old.' + timestamp.to_i.to_s)
-        end
-      end
-    end
-
-    around do |example|
-      Timecop.freeze(timestamp) { example.run }
-    end
-
-    after do
-      temp_dirs.each { |path| FileUtils.rm_rf(path) }
-    end
+    let_it_be(:project) { create(:project, :wiki_repo) }
 
     describe 'command failure' do
       before do
-        # Allow us to set expectations on the project directly
         expect(Project).to receive(:find_each).and_yield(project)
-        expect(project.repository).to receive(:create_repository) { raise 'Fail in tests' }
+
+        allow_next_instance_of(Repository) do |repository|
+          allow(repository).to receive(:create_repository) { raise 'Fail in tests' }
+        end
       end
 
       context 'hashed storage' do
         it 'shows the appropriate error' do
           subject.restore
 
-          expect(progress).to have_received(:puts).with("[Failed] restoring #{project.full_path} repository")
+          expect(progress).to have_received(:puts).with("[Failed] restoring #{project.full_path} (#{project.disk_path})")
         end
       end
 
       context 'legacy storage' do
-        let!(:project) { create(:project, :legacy_storage) }
+        let_it_be(:project) { create(:project, :legacy_storage) }
 
         it 'shows the appropriate error' do
           subject.restore
 
-          expect(progress).to have_received(:puts).with("[Failed] restoring #{project.full_path} repository")
+          expect(progress).to have_received(:puts).with("[Failed] restoring #{project.full_path} (#{project.disk_path})")
         end
       end
     end
@@ -188,45 +190,15 @@ RSpec.describe Backup::Repository do
     end
 
     it 'cleans existing repositories' do
-      wiki_repository_spy = spy(:wiki)
+      expect(Repository).to receive(:new).twice.and_wrap_original do |method, *original_args|
+        repository = method.call(*original_args)
 
-      allow_next_instance_of(ProjectWiki) do |project_wiki|
-        allow(project_wiki).to receive(:repository).and_return(wiki_repository_spy)
-      end
+        expect(repository).to receive(:remove)
 
-      expect_next_instance_of(Repository) do |repo|
-        expect(repo).to receive(:remove)
+        repository
       end
 
       subject.restore
-
-      expect(wiki_repository_spy).to have_received(:remove)
-    end
-  end
-
-  describe '#empty_repo?' do
-    context 'for a wiki' do
-      let(:wiki) { create(:project_wiki) }
-
-      it 'invalidates the emptiness cache' do
-        expect(wiki.repository).to receive(:expire_emptiness_caches).once
-
-        subject.send(:empty_repo?, wiki)
-      end
-
-      context 'wiki repo has content' do
-        let!(:wiki_page) { create(:wiki_page, wiki: wiki) }
-
-        it 'returns true, regardless of bad cache value' do
-          expect(subject.send(:empty_repo?, wiki)).to be(false)
-        end
-      end
-
-      context 'wiki repo does not have content' do
-        it 'returns true, regardless of bad cache value' do
-          expect(subject.send(:empty_repo?, wiki)).to be_truthy
-        end
-      end
     end
   end
 end

spec/tasks/gitlab/backup_rake_spec.rb

@@ -370,7 +370,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
       end
 
       it 'has defaults' do
-        expect_next_instance_of(::Backup::Repository) do |instance|
+        expect_next_instance_of(::Backup::Repositories) do |instance|
           expect(instance).to receive(:dump)
             .with(max_concurrency: 1, max_storage_concurrency: 1)
             .and_call_original
@@ -383,7 +383,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         stub_env('GITLAB_BACKUP_MAX_CONCURRENCY', 5)
         stub_env('GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY', 2)
 
-        expect_next_instance_of(::Backup::Repository) do |instance|
+        expect_next_instance_of(::Backup::Repositories) do |instance|
           expect(instance).to receive(:dump)
             .with(max_concurrency: 5, max_storage_concurrency: 2)
             .and_call_original