Remove feature flags from lib/backup

Moved to OPT_OUT in 7d14b725, Now, by
removing the feature gates, this is an mandatory feature.

Related issues:
- https://gitlab.com/gitlab-org/gitaly/issues/526
- https://gitlab.com/gitlab-org/gitaly/issues/1194

Closes https://gitlab.com/gitlab-org/gitaly/issues/749
Closes https://gitlab.com/gitlab-org/gitaly/issues/1212
Closes https://gitlab.com/gitlab-org/gitaly/issues/1195

lib/backup/repository.rb

 require 'yaml'
-require_relative 'helper'
 
 module Backup
   class Repository
-    include Backup::Helper
-
     attr_reader :progress
 
     def initialize(progress)
@@ -42,131 +39,36 @@ module Backup
     end
 
     def prepare_directories
-      Gitlab.config.repositories.storages.each do |name, repository_storage|
-        delete_all_repositories(name, repository_storage)
+      Gitlab.config.repositories.storages.each do |name, _repository_storage|
+        Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories
       end
     end
 
     def backup_project(project)
-      gitaly_migrate(:repository_backup, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-        if is_enabled
-          backup_project_gitaly(project)
-        else
-          backup_project_local(project)
-        end
-      end
-
-      backup_custom_hooks(project)
-    rescue => e
-      progress_warn(project, e, 'Failed to backup repo')
-    end
-
-    def backup_project_gitaly(project)
       path_to_project_bundle = path_to_bundle(project)
       Gitlab::GitalyClient::RepositoryService.new(project.repository)
         .create_bundle(path_to_project_bundle)
-    end
-
-    def backup_project_local(project)
-      path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-        path_to_repo(project)
-      end
-
-      path_to_project_bundle = path_to_bundle(project)
-
-      cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all)
-      output, status = Gitlab::Popen.popen(cmd)
-      progress_warn(project, cmd.join(' '), output) unless status.zero?
-    end
-
-    def delete_all_repositories(name, repository_storage)
-      gitaly_migrate(:delete_all_repositories, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-        if is_enabled
-          Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories
-        else
-          local_delete_all_repositories(name, repository_storage)
-        end
-      end
-    end
-
-    def local_delete_all_repositories(name, repository_storage)
-      path = repository_storage.legacy_disk_path
-      return unless File.exist?(path)
-
-      bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s)
-      FileUtils.mkdir_p(bk_repos_path, mode: 0700)
-      files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")]
-
-      begin
-        FileUtils.mv(files, bk_repos_path)
-      rescue Errno::EACCES
-        access_denied_error(path)
-      rescue Errno::EBUSY
-        resource_busy_error(path)
-      end
-    end
 
-    def local_restore_custom_hooks(project, dir)
-      path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-        path_to_repo(project)
-      end
-      cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
-      output, status = Gitlab::Popen.popen(cmd)
-      unless status.zero?
-        progress_warn(project, cmd.join(' '), output)
-      end
-    end
-
-    def gitaly_restore_custom_hooks(project, dir)
-      custom_hooks_path = path_to_tars(project, dir)
-      Gitlab::GitalyClient::RepositoryService.new(project.repository)
-        .restore_custom_hooks(custom_hooks_path)
+      backup_custom_hooks(project)
+    rescue => e
+      progress_warn(project, e, 'Failed to backup repo')
     end
 
-    def local_backup_custom_hooks(project)
-      in_path(path_to_tars(project)) do |dir|
-        path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-          path_to_repo(project)
-        end
-        break unless File.exist?(File.join(path_to_project_repo, dir))
-
-        FileUtils.mkdir_p(path_to_tars(project))
-        cmd = %W(tar -cf #{path_to_tars(project, dir)} -c #{path_to_project_repo} #{dir})
-        output, status = Gitlab::Popen.popen(cmd)
-
-        unless status.zero?
-          progress_warn(project, cmd.join(' '), output)
-        end
-      end
-    end
+    def backup_custom_hooks(project)
+      FileUtils.mkdir_p(project_backup_path(project))
 
-    def gitaly_backup_custom_hooks(project)
-      FileUtils.mkdir_p(path_to_tars(project))
-      custom_hooks_path = path_to_tars(project, 'custom_hooks')
+      custom_hooks_path = custom_hooks_tar(project)
       Gitlab::GitalyClient::RepositoryService.new(project.repository)
         .backup_custom_hooks(custom_hooks_path)
     end
 
-    def backup_custom_hooks(project)
-      gitaly_migrate(:backup_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-        if is_enabled
-          gitaly_backup_custom_hooks(project)
-        else
-          local_backup_custom_hooks(project)
-        end
-      end
-    end
-
     def restore_custom_hooks(project)
-      in_path(path_to_tars(project)) do |dir|
-        gitaly_migrate(:restore_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          if is_enabled
-            gitaly_restore_custom_hooks(project, dir)
-          else
-            local_restore_custom_hooks(project, dir)
-          end
-        end
-      end
+      return unless Dir.exist?(project_backup_path(project))
+      return if Dir.glob("#{project_backup_path(project)}/custom_hooks*").none?
+
+      custom_hooks_path = custom_hooks_tar(project)
+      Gitlab::GitalyClient::RepositoryService.new(project.repository)
+        .restore_custom_hooks(custom_hooks_path)
     end
 
     def restore
@@ -181,7 +83,8 @@ module Backup
         restore_repo_success = nil
         if File.exist?(path_to_project_bundle)
           begin
-            project.repository.create_from_bundle path_to_project_bundle
+            project.repository.create_from_bundle(path_to_project_bundle)
+            restore_custom_hooks(project)
             restore_repo_success = true
           rescue => e
             restore_repo_success = false
@@ -197,8 +100,6 @@ module Backup
           progress.puts "[Failed] restoring #{project.full_path} repository".color(:red)
         end
 
-        restore_custom_hooks(project)
-
         wiki = ProjectWiki.new(project)
         path_to_wiki_bundle = path_to_bundle(wiki)
 
@@ -219,48 +120,28 @@ module Backup
 
     protected
 
-    def path_to_repo(project)
-      project.repository.path_to_repo
-    end
-
     def path_to_bundle(project)
       File.join(backup_repos_path, project.disk_path + '.bundle')
     end
 
-    def path_to_tars(project, dir = nil)
-      path = File.join(backup_repos_path, project.disk_path)
+    def project_backup_path(project)
+      File.join(backup_repos_path, project.disk_path)
+    end
 
-      if dir
-        File.join(path, "#{dir}.tar")
-      else
-        path
-      end
+    def custom_hooks_tar(project)
+      File.join(project_backup_path(project), "custom_hooks.tar")
     end
 
     def backup_repos_path
       File.join(Gitlab.config.backup.path, 'repositories')
     end
 
-    def in_path(path)
-      return unless Dir.exist?(path)
-
-      dir_entries = Dir.entries(path)
-
-      if dir_entries.include?('custom_hooks') || dir_entries.include?('custom_hooks.tar')
-        yield('custom_hooks')
-      end
-    end
-
     def prepare
       FileUtils.rm_rf(backup_repos_path)
       FileUtils.mkdir_p(Gitlab.config.backup.path)
       FileUtils.mkdir(backup_repos_path, mode: 0700)
     end
 
-    def silent
-      { err: '/dev/null', out: '/dev/null' }
-    end
-
     private
 
     def progress_warn(project, cmd, output)
@@ -273,18 +154,8 @@ module Backup
       project_or_wiki.repository.empty?
     end
 
-    def repository_storage_paths_args
-      Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path }
-    end
-
     def display_repo_path(project)
       project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path
     end
-
-    def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block)
-      Gitlab::GitalyClient.migrate(method, status: status, &block)
-    rescue GRPC::NotFound, GRPC::BadStatus => e
-      raise Error, e
-    end
   end
 end

spec/lib/backup/repository_spec.rb

@@ -73,37 +73,27 @@ describe Backup::Repository do
     end
   end
 
-  describe '#delete_all_repositories', :seed_helper do
-    shared_examples('delete_all_repositories') do
-      before do
-        allow(FileUtils).to receive(:mkdir_p).and_call_original
-        allow(FileUtils).to receive(:mv).and_call_original
-      end
-
-      after(:all) do
-        ensure_seeds
-      end
-
-      it 'removes all repositories' do
-        # Sanity check: there should be something for us to delete
-        expect(list_repositories).to include(File.join(SEED_STORAGE_PATH, TEST_REPO_PATH))
+  describe '#prepare_directories', :seed_helper do
+    before do
+      allow(FileUtils).to receive(:mkdir_p).and_call_original
+      allow(FileUtils).to receive(:mv).and_call_original
+    end
 
-        subject.delete_all_repositories('default', Gitlab.config.repositories.storages['default'])
+    after(:all) do
+      ensure_seeds
+    end
 
-        expect(list_repositories).to be_empty
-      end
+    it' removes all repositories' do
+      # Sanity check: there should be something for us to delete
+      expect(list_repositories).to include(File.join(SEED_STORAGE_PATH, TEST_REPO_PATH))
 
-      def list_repositories
-        Dir[File.join(SEED_STORAGE_PATH, '*.git')]
-      end
-    end
+      subject.prepare_directories
 
-    context 'with gitaly' do
-      it_behaves_like 'delete_all_repositories'
+      expect(list_repositories).to be_empty
     end
 
-    context 'without gitaly', :skip_gitaly_mock do
-      it_behaves_like 'delete_all_repositories'
+    def list_repositories
+      Dir[File.join(SEED_STORAGE_PATH, '*.git')]
     end
   end
 

spec/lib/gitlab/gitaly_client/storage_service_spec.rb

+require 'spec_helper'
+
+describe Gitlab::GitalyClient::StorageService do
+  describe '#delete_all_repositories' do
+    let!(:project) { create(:project, :repository) }
+
+    it 'removes all repositories' do
+      described_class.new(project.repository_storage).delete_all_repositories
+
+      expect(project.repository.exists?).to be(false)
+    end
+  end
+end

spec/tasks/gitlab/backup_rake_spec.rb

@@ -87,6 +87,27 @@ describe 'gitlab:app namespace rake task' do
         expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout
       end
     end
+
+    context 'when the restore directory is not empty' do
+      before do
+        # We only need a backup of the repositories for this test
+        stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
+      end
+
+      it 'removes stale data' do
+        expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout
+
+        excluded_project = create(:project, :repository, name: 'mepmep')
+
+        expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout
+
+        raw_repo = excluded_project.repository.raw
+
+        # The restore will not find the repository in the backup, but will create
+        # an empty one in its place
+        expect(raw_repo.empty?).to be(true)
+      end
+    end
   end # backup_restore task
 
   describe 'backup' do