Add group wikis to Gitlab Backup

In this commit we add group wikis to the Gitlab Backup. Group
wikis are a premium feature and all the code is implemented
in the ee folders.

Since there are no free group repository features, the group
retrieval and iteration is done in the ee part.

doc/raketasks/backup_restore.md

@@ -61,6 +61,7 @@ including:
 - Container Registry images
 - GitLab Pages content
 - Snippets
+- Group wikis **(PREMIUM)**
 
 WARNING:
 GitLab does not back up any configuration files, SSL certificates, or system

doc/user/group/index.md

@@ -460,7 +460,7 @@ and above.
 There are a few limitations compared to project wikis:
 
 - Git LFS is not supported.
-- Group wikis are not included in global search, group exports, backups, and Geo replication.
+- Group wikis are not included in global search, group exports, and Geo replication.
 - Changes to group wikis don't show up in the group's activity feed.
 - Group wikis [can't be moved](../../api/project_repository_storage_moves.md#limitations) using the project
   repository moves API.

ee/changelogs/unreleased/254964-fj-add-group-wikis-to-backup.yml

+---
+title: Add group wikis to Gitlab Backup
+merge_request: 51590
+author:
+type: added

ee/lib/ee/backup/repositories.rb

+# frozen_string_literal: true
+
+module EE
+  module Backup
+    module Repositories
+      extend ::Gitlab::Utils::Override
+
+      override :restore
+      def restore
+        restore_group_repositories
+
+        super
+      end
+
+      private
+
+      override :repository_storage_klasses
+      def repository_storage_klasses
+        super << GroupWikiRepository
+      end
+
+      def restore_group_repositories
+        find_groups_in_batches do |group|
+          restore_repository(group, ::Gitlab::GlRepository::WIKI)
+        end
+      end
+
+      def group_relation
+        ::Group.includes(:route, :owners, group_wiki_repository: :shard) # rubocop: disable CodeReuse/ActiveRecord
+      end
+
+      def find_groups_in_batches(&block)
+        group_relation.find_each(batch_size: 1000) do |group| # rubocop: disable CodeReuse/ActiveRecord
+          yield(group)
+        end
+      end
+
+      override :dump_container
+      def dump_container(container)
+        if container.is_a?(Group)
+          dump_group(container)
+        else
+          super
+        end
+      end
+
+      def dump_group(group)
+        backup_repository(group, ::Gitlab::GlRepository::WIKI)
+      end
+
+      override :dump_consecutive
+      def dump_consecutive
+        dump_consecutive_groups
+
+        super
+      end
+
+      def dump_consecutive_groups
+        find_groups_in_batches do |group|
+          dump_group(group)
+        end
+      end
+
+      override :records_to_enqueue
+      def records_to_enqueue(storage)
+        super << groups_in_storage(storage)
+      end
+
+      def groups_in_storage(storage)
+        group_relation.id_in(GroupWikiRepository.for_repository_storage(storage).select(:group_id))
+      end
+    end
+  end
+end

ee/spec/lib/ee/backup/repositories_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Backup::Repositories do
+  let(:progress) { StringIO.new }
+
+  subject { described_class.new(progress) }
+
+  before do
+    allow(progress).to receive(:puts)
+    allow(progress).to receive(:print)
+
+    allow_next_instance_of(described_class) do |instance|
+      allow(instance).to receive(:progress).and_return(progress)
+    end
+  end
+
+  describe '#dump' do
+    context 'hashed storage' do
+      let_it_be(:project) { create(:project, :repository) }
+      let_it_be(:group) { create(:group, :wiki_repo) }
+
+      it 'creates repository bundles', :aggregate_failures do
+        create(:wiki_page, container: group)
+
+        subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+
+        expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.bundle'))
+        expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', group.wiki.disk_path + '.bundle'))
+      end
+    end
+
+    context 'no concurrency' do
+      let_it_be(:groups) { create_list(:group, 5, :wiki_repo) }
+
+      it 'creates the expected number of threads' do
+        expect(Thread).not_to receive(:new)
+
+        groups.each do |group|
+          expect(subject).to receive(:dump_group).with(group).and_call_original
+        end
+
+        subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+      end
+
+      describe 'command failure' do
+        it 'dump_group raises an error' do
+          allow(subject).to receive(:dump_group).and_raise(IOError)
+
+          expect { subject.dump(max_concurrency: 1, max_storage_concurrency: 1) }.to raise_error(IOError)
+        end
+
+        it 'group query raises an error' do
+          allow(Group).to receive_message_chain(:includes, :find_each).and_raise(ActiveRecord::StatementTimeout)
+
+          expect { subject.dump(max_concurrency: 1, max_storage_concurrency: 1) }.to raise_error(ActiveRecord::StatementTimeout)
+        end
+      end
+
+      it 'avoids N+1 database queries' do
+        control_count = ActiveRecord::QueryRecorder.new do
+          subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+        end.count
+
+        create_list(:group, 2, :wiki_repo)
+
+        expect do
+          subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+        end.not_to exceed_query_limit(control_count)
+      end
+    end
+  end
+
+  describe '#restore' do
+    let_it_be(:project) { create(:project) }
+    let_it_be(:group) { create(:group) }
+
+    let(:next_path_to_bundle) do
+      [
+        Rails.root.join('spec/fixtures/lib/backup/wiki_repo.bundle'),
+        Rails.root.join('spec/fixtures/lib/backup/project_repo.bundle')
+      ].to_enum
+    end
+
+    it 'restores repositories from bundles', :aggregate_failures do
+      allow_next_instance_of(described_class::BackupRestore) do |backup_restore|
+        allow(backup_restore).to receive(:path_to_bundle).and_return(next_path_to_bundle.next)
+      end
+
+      subject.restore
+
+      collect_commit_shas = -> (repo) { repo.commits('master', limit: 10).map(&:sha) }
+
+      expect(collect_commit_shas.call(project.repository)).to eq(['393a7d860a5a4c3cc736d7eb00604e3472bb95ec'])
+      expect(collect_commit_shas.call(group.wiki.repository)).to eq(['c74b9948d0088d703ee1fafeddd9ed9add2901ea'])
+    end
+  end
+end

lib/backup/repositories.rb

@@ -40,31 +40,42 @@ module Backup
     end
 
     def restore
+      restore_project_repositories
+      restore_snippets
+
+      restore_object_pools
+    end
+
+    private
+
+    def restore_project_repositories
       Project.find_each(batch_size: 1000) do |project|
         restore_repository(project, Gitlab::GlRepository::PROJECT)
         restore_repository(project, Gitlab::GlRepository::WIKI)
         restore_repository(project, Gitlab::GlRepository::DESIGN)
       end
+    end
 
+    def restore_snippets
       invalid_ids = Snippet.find_each(batch_size: 1000)
         .map { |snippet| restore_snippet_repository(snippet) }
         .compact
 
       cleanup_snippets_without_repositories(invalid_ids)
-
-      restore_object_pools
     end
 
-    private
-
     def check_valid_storages!
-      [ProjectRepository, SnippetRepository].each do |klass|
+      repository_storage_klasses.each do |klass|
         if klass.excluding_repository_storage(Gitlab.config.repositories.storages.keys).exists?
           raise Error, "repositories.storages in gitlab.yml does not include all storages used by #{klass}"
         end
       end
     end
 
+    def repository_storage_klasses
+      [ProjectRepository, SnippetRepository]
+    end
+
     def backup_repos_path
       @backup_repos_path ||= File.join(Gitlab.config.backup.path, 'repositories')
     end
@@ -103,12 +114,7 @@ module Backup
               end
 
               begin
-                case container
-                when Project
-                  dump_project(container)
-                when Snippet
-                  dump_snippet(container)
-                end
+                dump_container(container)
               rescue => e
                 errors << e
                 break
@@ -130,6 +136,15 @@ module Backup
       end
     end
 
+    def dump_container(container)
+      case container
+      when Project
+        dump_project(container)
+      when Snippet
+        dump_snippet(container)
+      end
+    end
+
     def dump_project(project)
       backup_repository(project, Gitlab::GlRepository::PROJECT)
       backup_repository(project, Gitlab::GlRepository::WIKI)
@@ -308,3 +323,5 @@ module Backup
     end
   end
 end
+
+Backup::Repositories.prepend_if_ee('EE::Backup::Repositories')