Merge branch '254964-fj-add-group-wikis-to-backup' into 'master'

Add group wikis to Gitlab Backup

See merge request gitlab-org/gitlab!51590

doc/raketasks/backup_restore.md

@@ -61,6 +61,7 @@ including:
 - Container Registry images
 - GitLab Pages content
 - Snippets
+- Group wikis **(PREMIUM)**
 
 WARNING:
 GitLab does not back up any configuration files, SSL certificates, or system

doc/user/group/index.md

@@ -460,7 +460,7 @@ and above.
 There are a few limitations compared to project wikis:
 
 - Git LFS is not supported.
-- Group wikis are not included in global search, group exports, backups, and Geo replication.
+- Group wikis are not included in global search, group exports, and Geo replication.
 - Changes to group wikis don't show up in the group's activity feed.
 - Group wikis [can't be moved](../../api/project_repository_storage_moves.md#limitations) using the project
   repository moves API.

ee/changelogs/unreleased/254964-fj-add-group-wikis-to-backup.yml

+---
+title: Add group wikis to Gitlab Backup
+merge_request: 51590
+author:
+type: added

ee/lib/ee/backup/repositories.rb

+# frozen_string_literal: true
+
+module EE
+  module Backup
+    module Repositories
+      extend ::Gitlab::Utils::Override
+
+      override :restore
+      def restore
+        restore_group_repositories
+
+        super
+      end
+
+      private
+
+      override :repository_storage_klasses
+      def repository_storage_klasses
+        super << GroupWikiRepository
+      end
+
+      def restore_group_repositories
+        find_groups_in_batches do |group|
+          restore_repository(group, ::Gitlab::GlRepository::WIKI)
+        end
+      end
+
+      def group_relation
+        ::Group.includes(:route, :owners, group_wiki_repository: :shard) # rubocop: disable CodeReuse/ActiveRecord
+      end
+
+      def find_groups_in_batches(&block)
+        group_relation.find_each(batch_size: 1000) do |group| # rubocop: disable CodeReuse/ActiveRecord
+          yield(group)
+        end
+      end
+
+      override :dump_container
+      def dump_container(container)
+        if container.is_a?(Group)
+          dump_group(container)
+        else
+          super
+        end
+      end
+
+      def dump_group(group)
+        backup_repository(group, ::Gitlab::GlRepository::WIKI)
+      end
+
+      override :dump_consecutive
+      def dump_consecutive
+        dump_consecutive_groups
+
+        super
+      end
+
+      def dump_consecutive_groups
+        find_groups_in_batches do |group|
+          dump_group(group)
+        end
+      end
+
+      override :records_to_enqueue
+      def records_to_enqueue(storage)
+        super << groups_in_storage(storage)
+      end
+
+      def groups_in_storage(storage)
+        group_relation.id_in(GroupWikiRepository.for_repository_storage(storage).select(:group_id))
+      end
+    end
+  end
+end

ee/spec/lib/ee/backup/repositories_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Backup::Repositories do
+  let(:progress) { StringIO.new }
+
+  subject { described_class.new(progress) }
+
+  before do
+    allow(progress).to receive(:puts)
+    allow(progress).to receive(:print)
+
+    allow_next_instance_of(described_class) do |instance|
+      allow(instance).to receive(:progress).and_return(progress)
+    end
+  end
+
+  describe '#dump' do
+    context 'hashed storage' do
+      let_it_be(:project) { create(:project, :repository) }
+      let_it_be(:group) { create(:group, :wiki_repo) }
+
+      it 'creates repository bundles', :aggregate_failures do
+        create(:wiki_page, container: group)
+
+        subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+
+        expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.bundle'))
+        expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', group.wiki.disk_path + '.bundle'))
+      end
+    end
+
+    context 'no concurrency' do
+      let_it_be(:groups) { create_list(:group, 5, :wiki_repo) }
+
+      it 'creates the expected number of threads' do
+        expect(Thread).not_to receive(:new)
+
+        groups.each do |group|
+          expect(subject).to receive(:dump_group).with(group).and_call_original
+        end
+
+        subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+      end
+
+      describe 'command failure' do
+        it 'dump_group raises an error' do
+          allow(subject).to receive(:dump_group).and_raise(IOError)
+
+          expect { subject.dump(max_concurrency: 1, max_storage_concurrency: 1) }.to raise_error(IOError)
+        end
+
+        it 'group query raises an error' do
+          allow(Group).to receive_message_chain(:includes, :find_each).and_raise(ActiveRecord::StatementTimeout)
+
+          expect { subject.dump(max_concurrency: 1, max_storage_concurrency: 1) }.to raise_error(ActiveRecord::StatementTimeout)
+        end
+      end
+
+      it 'avoids N+1 database queries' do
+        control_count = ActiveRecord::QueryRecorder.new do
+          subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+        end.count
+
+        create_list(:group, 2, :wiki_repo)
+
+        expect do
+          subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+        end.not_to exceed_query_limit(control_count)
+      end
+    end
+  end
+
+  describe '#restore' do
+    let_it_be(:project) { create(:project) }
+    let_it_be(:group) { create(:group) }
+
+    let(:next_path_to_bundle) do
+      [
+        Rails.root.join('spec/fixtures/lib/backup/wiki_repo.bundle'),
+        Rails.root.join('spec/fixtures/lib/backup/project_repo.bundle')
+      ].to_enum
+    end
+
+    it 'restores repositories from bundles', :aggregate_failures do
+      allow_next_instance_of(described_class::BackupRestore) do |backup_restore|
+        allow(backup_restore).to receive(:path_to_bundle).and_return(next_path_to_bundle.next)
+      end
+
+      subject.restore
+
+      collect_commit_shas = -> (repo) { repo.commits('master', limit: 10).map(&:sha) }
+
+      expect(collect_commit_shas.call(project.repository)).to eq(['393a7d860a5a4c3cc736d7eb00604e3472bb95ec'])
+      expect(collect_commit_shas.call(group.wiki.repository)).to eq(['c74b9948d0088d703ee1fafeddd9ed9add2901ea'])
+    end
+  end
+end

lib/backup/repositories.rb

@@ -40,31 +40,42 @@ module Backup
     end
 
     def restore
+      restore_project_repositories
+      restore_snippets
+
+      restore_object_pools
+    end
+
+    private
+
+    def restore_project_repositories
       Project.find_each(batch_size: 1000) do |project|
         restore_repository(project, Gitlab::GlRepository::PROJECT)
         restore_repository(project, Gitlab::GlRepository::WIKI)
         restore_repository(project, Gitlab::GlRepository::DESIGN)
       end
+    end
 
+    def restore_snippets
       invalid_ids = Snippet.find_each(batch_size: 1000)
         .map { |snippet| restore_snippet_repository(snippet) }
         .compact
 
       cleanup_snippets_without_repositories(invalid_ids)
-
-      restore_object_pools
     end
 
-    private
-
     def check_valid_storages!
-      [ProjectRepository, SnippetRepository].each do |klass|
+      repository_storage_klasses.each do |klass|
         if klass.excluding_repository_storage(Gitlab.config.repositories.storages.keys).exists?
           raise Error, "repositories.storages in gitlab.yml does not include all storages used by #{klass}"
         end
       end
     end
 
+    def repository_storage_klasses
+      [ProjectRepository, SnippetRepository]
+    end
+
     def backup_repos_path
       @backup_repos_path ||= File.join(Gitlab.config.backup.path, 'repositories')
     end
@@ -103,12 +114,7 @@ module Backup
               end
 
               begin
-                case container
-                when Project
-                  dump_project(container)
-                when Snippet
-                  dump_snippet(container)
-                end
+                dump_container(container)
               rescue => e
                 errors << e
                 break
@@ -130,6 +136,15 @@ module Backup
       end
     end
 
+    def dump_container(container)
+      case container
+      when Project
+        dump_project(container)
+      when Snippet
+        dump_snippet(container)
+      end
+    end
+
     def dump_project(project)
       backup_repository(project, Gitlab::GlRepository::PROJECT)
       backup_repository(project, Gitlab::GlRepository::WIKI)
@@ -308,3 +323,5 @@ module Backup
     end
   end
 end
+
+Backup::Repositories.prepend_if_ee('EE::Backup::Repositories')