Merge branch '207886-fj-import-export-group-wikis' into 'master'

Import and export Group Wikis

See merge request gitlab-org/gitlab!51873

app/services/groups/import_export/export_service.rb

@@ -126,3 +126,5 @@ module Groups
     end
   end
 end
+
+Groups::ImportExport::ExportService.prepend_if_ee('EE::Groups::ImportExport::ExportService')

app/services/groups/import_export/import_service.rb

@@ -123,3 +123,5 @@ module Groups
     end
   end
 end
+
+Groups::ImportExport::ImportService.prepend_if_ee('EE::Groups::ImportExport::ImportService')

config/feature_flags/development/group_wiki_import_export.yml

+---
+name: group_wiki_import_export
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/51873
+rollout_issue_url: 
+milestone: '13.9'
+type: development
+group: group::editor
+default_enabled: false

ee/app/services/ee/groups/import_export/export_service.rb

+# frozen_string_literal: true
+
+module EE
+  module Groups
+    module ImportExport
+      module ExportService
+        extend ::Gitlab::Utils::Override
+
+        override :savers
+        def savers
+          return super unless ndjson?
+          return super if ::Feature.disabled?(:group_wiki_import_export, group)
+
+          super << group_and_descendants_repo_saver
+        end
+
+        def group_and_descendants_repo_saver
+          ::Gitlab::ImportExport::Group::GroupAndDescendantsRepoSaver.new(group: group, shared: shared)
+        end
+      end
+    end
+  end
+end

ee/app/services/ee/groups/import_export/import_service.rb

+# frozen_string_literal: true
+
+module EE
+  module Groups
+    module ImportExport
+      module ImportService
+        extend ::Gitlab::Utils::Override
+
+        override :restorers
+        def restorers
+          return super unless ndjson?
+          return super if ::Feature.disabled?(:group_wiki_import_export, group)
+
+          super << group_and_descendants_repo_restorer
+        end
+
+        def group_and_descendants_repo_restorer
+          ::Gitlab::ImportExport::Group::GroupAndDescendantsRepoRestorer.new(group: group, shared: shared, tree_restorer: tree_restorer)
+        end
+      end
+    end
+  end
+end

ee/lib/ee/gitlab/import_export/wiki_repo_saver.rb

+# frozen_string_literal: true
+
+module EE
+  module Gitlab
+    module ImportExport
+      module WikiRepoSaver
+        extend ::Gitlab::Utils::Override
+
+        private
+
+        override :bundle_full_path
+        def bundle_full_path
+          return super unless exportable.is_a?(Group)
+
+          ::Gitlab::ImportExport.group_wiki_repo_bundle_path(shared, bundle_filename)
+        end
+
+        override :bundle_filename
+        def bundle_filename
+          return super unless exportable.is_a?(Group)
+
+          ::Gitlab::ImportExport.group_wiki_repo_bundle_filename(exportable.id)
+        end
+      end
+    end
+  end
+end

ee/lib/gitlab/import_export/group/group_and_descendants_repo_restorer.rb

+# frozen_string_literal: true
+
+# Given a group, this class can import the
+# wiki repositories for the main group and all its
+# descendants.
+#
+# If we want to import more group repositories in the future
+# we should extend this class.
+module Gitlab
+  module ImportExport
+    module Group
+      class GroupAndDescendantsRepoRestorer
+        attr_reader :group, :shared, :tree_restorer
+
+        def initialize(group:, shared:, tree_restorer:)
+          @group = group
+          @shared = shared
+          @tree_restorer = tree_restorer
+        end
+
+        def restore
+          # At the moment, group only have wiki repositories so, in order
+          # to avoid iterating them, we're checking the feature flag before
+          # the loop.
+          #
+          # If, at some point, we add more repositories to groups, we should
+          # move this check inside the loop, along with the other checks
+          # for the new repository type.
+          return true unless group.feature_available?(:group_wikis)
+          return true if group_mapping.empty?
+
+          group.self_and_descendants.find_each.all? do |subgroup|
+            old_id = group_mapping[subgroup]
+
+            next true unless old_id
+
+            restore_wiki(subgroup, old_id)
+          end
+        end
+
+        private
+
+        def group_mapping
+          @group_mapping ||= tree_restorer.groups_mapping.invert
+        end
+
+        def restore_wiki(group, old_id)
+          ::Gitlab::ImportExport::RepoRestorer.new(
+            path_to_bundle: ::Gitlab::ImportExport.group_wiki_repo_bundle_full_path(shared, old_id),
+            shared: shared,
+            importable: GroupWiki.new(group)).restore
+        end
+      end
+    end
+  end
+end

ee/lib/gitlab/import_export/group/group_and_descendants_repo_saver.rb

+# frozen_string_literal: true
+
+# Given a group, this class can export the
+# wiki repositories for the main group and all its
+# descendants.
+#
+# If we want to export more group repositories in the future
+# we should extend this class.
+module Gitlab
+  module ImportExport
+    module Group
+      class GroupAndDescendantsRepoSaver
+        attr_reader :group, :shared
+
+        def initialize(group:, shared:)
+          @group = group
+          @shared = shared
+        end
+
+        def save
+          group.self_and_descendants.find_each.all? do |subgroup|
+            save_wiki(subgroup)
+          end
+        end
+
+        private
+
+        def save_wiki(group)
+          ::Gitlab::ImportExport::WikiRepoSaver.new(
+            exportable: group,
+            shared: shared).save
+        end
+      end
+    end
+  end
+end

ee/lib/gitlab/import_export/group_helper.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module GroupHelper
+      def group_wiki_repo_bundle_filename(group_id)
+        "#{group_id}.wiki.bundle"
+      end
+
+      def group_wiki_repo_bundle_path(shared, filename)
+        File.join(shared.export_path, 'repositories', filename)
+      end
+
+      def group_wiki_repo_bundle_full_path(shared, group_id)
+        group_wiki_repo_bundle_path(shared, group_wiki_repo_bundle_filename(group_id))
+      end
+    end
+  end
+end

ee/spec/lib/ee/gitlab/import_export/repo_restorer_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::ImportExport::RepoRestorer do
+  include GitHelpers
+
+  describe 'restores group wiki bundles' do
+    let(:group) { create(:group) }
+    let(:shared) { Gitlab::ImportExport::Shared.new(group) }
+    let(:export_path) { "#{Dir.tmpdir}/group_tree_saver_spec" }
+    let(:bundle_path) { ::Gitlab::ImportExport.group_wiki_repo_bundle_full_path(shared, group_wiki.container.id) }
+    let(:bundler) { Gitlab::ImportExport::WikiRepoSaver.new(exportable: group_wiki.container, shared: shared) }
+    let(:gitlab_shell) { Gitlab::Shell.new }
+    let(:restorer) do
+      described_class.new(path_to_bundle: bundle_path,
+                          shared: shared,
+                          importable: GroupWiki.new(group))
+    end
+
+    before do
+      allow_next_instance_of(Gitlab::ImportExport) do |instance|
+        allow(instance).to receive(:storage_path).and_return(export_path)
+      end
+
+      bundler.save # rubocop:disable Rails/SaveBang
+    end
+
+    after do
+      FileUtils.rm_rf(export_path)
+      gitlab_shell.remove_repository(group_wiki.repository_storage, group_wiki.disk_path)
+      gitlab_shell.remove_repository(group.wiki.repository_storage, group.wiki.disk_path)
+    end
+
+    context 'when group wiki in bundle' do
+      let_it_be(:page_title) { 'index' }
+      let_it_be(:page_content) { 'test content' }
+      let_it_be(:group_wiki) do
+        create(:group_wiki).tap do |group_wiki|
+          group_wiki.create_page(page_title, page_content)
+        end
+      end
+
+      it 'restores the repo successfully', :aggregated_failures do
+        expect(group.wiki_repository_exists?).to be false
+
+        restorer.restore
+
+        pages = group.wiki.list_pages(load_content: true)
+        expect(pages.size).to eq 1
+        expect(pages.first.title).to eq page_title
+        expect(pages.first.content).to eq page_content
+      end
+    end
+
+    context 'when no group wiki in the bundle', :aggregated_failures do
+      let_it_be(:group_wiki) { create(:group_wiki) }
+
+      it 'does not creates an empty wiki' do
+        expect(restorer.restore).to be true
+        expect(group.wiki_repository_exists?).to be false
+      end
+    end
+  end
+end

ee/spec/lib/ee/gitlab/import_export/wiki_repo_saver_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::ImportExport::WikiRepoSaver do
+  let_it_be(:user) { create(:user) }
+  let_it_be(:group) do
+    create(:group, :wiki_repo).tap do |g|
+      g.add_owner(user)
+    end
+  end
+
+  let(:exportable) { group }
+  let(:shared) { Gitlab::ImportExport::Shared.new(group) }
+
+  subject { described_class.new(exportable: exportable, shared: shared) }
+
+  describe 'bundles a group wiki Git repo' do
+    let!(:group_wiki) { GroupWiki.new(group, user) }
+    let(:export_path) { "#{Dir.tmpdir}/group_tree_saver_spec" }
+
+    before do
+      allow_next_instance_of(Gitlab::ImportExport) do |instance|
+        allow(instance).to receive(:storage_path).and_return(export_path)
+      end
+
+      group_wiki.create_page('index', 'test content')
+    end
+
+    after do
+      FileUtils.rm_rf(export_path)
+    end
+
+    it 'bundles the repo successfully' do
+      expect(subject.save).to be true
+      expect(File.exist?(subject.send(:bundle_full_path))).to eq true
+    end
+
+    context 'when the repo is empty' do
+      let!(:group) { create(:group) }
+
+      it 'bundles the repo successfully' do
+        expect(subject.save).to be true
+      end
+    end
+  end
+
+  describe '#bundle_filename' do
+    context 'when exportable is a group' do
+      it 'returns the right filename for group wikis' do
+        expect(subject.send(:bundle_filename)).to eq ::Gitlab::ImportExport.group_wiki_repo_bundle_filename(exportable.id)
+      end
+    end
+
+    context 'when exportable is a project' do
+      let(:exportable) { build_stubbed(:project) }
+
+      it 'returns the right filename for project wikis' do
+        expect(subject.send(:bundle_filename)).to eq ::Gitlab::ImportExport.wiki_repo_bundle_filename
+      end
+    end
+  end
+end

ee/spec/lib/gitlab/import_export/group/group_and_descendants_repo_restorer_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::ImportExport::Group::GroupAndDescendantsRepoRestorer do
+  let_it_be_with_refind(:group) { create(:group) }
+  let_it_be(:subgroup) { create(:group, parent: group) }
+
+  let(:shared) { Gitlab::ImportExport::Shared.new(group) }
+  let(:tree_restorer) { instance_double(Gitlab::ImportExport::Group::TreeRestorer) }
+  let(:groups_mapping) { { 100 => group, 200 => subgroup } }
+
+  subject { described_class.new(group: group, shared: shared, tree_restorer: tree_restorer) }
+
+  before do
+    allow(tree_restorer).to receive(:groups_mapping).and_return(groups_mapping)
+  end
+
+  context 'when group wiki license feature is enabled' do
+    before do
+      stub_licensed_features(group_wikis: true)
+    end
+
+    it 'imports the group and subgroups wiki repo and returns true' do
+      group_bundle_path = Gitlab::ImportExport.group_wiki_repo_bundle_full_path(shared, 100)
+      expect_next_instance_of(Gitlab::ImportExport::RepoRestorer, importable: group.wiki, shared: shared, path_to_bundle: group_bundle_path) do |restorer|
+        expect(restorer).to receive(:restore).and_return(true)
+      end
+
+      subgroup_bundle_path = Gitlab::ImportExport.group_wiki_repo_bundle_full_path(shared, 200)
+      expect_next_instance_of(Gitlab::ImportExport::RepoRestorer, importable: subgroup.wiki, shared: shared, path_to_bundle: subgroup_bundle_path) do |restorer|
+        expect(restorer).to receive(:restore).and_return(true)
+      end
+
+      expect(subject.restore).to eq true
+    end
+
+    context 'if any of the wiki imports fails' do
+      it 'returns false and stops importing other groups' do
+        expect_next_instance_of(Gitlab::ImportExport::RepoRestorer, hash_including(importable: group.wiki)) do |restorer|
+          expect(restorer).to receive(:restore).and_return(false)
+        end
+
+        expect(Gitlab::ImportExport::RepoRestorer).not_to receive(:new).with(hash_including(importable: subgroup.wiki))
+
+        expect(subject.restore).to eq false
+      end
+    end
+
+    context 'when group is not inside group mappings' do
+      let(:groups_mapping) { { 100 => group } }
+
+      it 'avoids calling the restorer, continue importing, and returns true' do
+        expect(Gitlab::ImportExport::RepoRestorer).to receive(:new).and_call_original
+        expect(Gitlab::ImportExport::RepoRestorer).not_to receive(:new).with(hash_including(importable: subgroup.wiki))
+
+        expect(subject.restore).to eq true
+      end
+    end
+
+    context 'when group mapping is empty' do
+      let(:groups_mapping) { {} }
+
+      it 'does not try to import wikis and returns true' do
+        expect(Gitlab::ImportExport::RepoRestorer).not_to receive(:new)
+
+        expect(subject.restore).to eq true
+      end
+    end
+  end
+
+  context 'when group wiki license feature is not enabled' do
+    it 'does not try to import wikis and returns true' do
+      stub_licensed_features(group_wikis: false)
+
+      expect(Gitlab::ImportExport::RepoRestorer).not_to receive(:new)
+
+      expect(subject.restore).to eq true
+    end
+  end
+end

ee/spec/lib/gitlab/import_export/group/group_and_descendants_repo_saver_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::ImportExport::Group::GroupAndDescendantsRepoSaver do
+  let_it_be(:group) { create(:group) }
+  let_it_be(:subgroup) { create(:group, parent: group) }
+
+  let(:shared) { Gitlab::ImportExport::Shared.new(group) }
+
+  subject { described_class.new(group: group, shared: shared) }
+
+  it 'exports the group and subgroups wiki repo' do
+    expect_next_instance_of(Gitlab::ImportExport::WikiRepoSaver, exportable: group, shared: shared) do |saver|
+      expect(saver).to receive(:save).and_return(true)
+    end
+
+    expect_next_instance_of(Gitlab::ImportExport::WikiRepoSaver, exportable: subgroup, shared: shared) do |saver|
+      expect(saver).to receive(:save).and_return(true)
+    end
+
+    expect(subject.save).to eq true
+  end
+
+  context 'if any of the wiki exports fails' do
+    it 'returns false and stops exporting other groups' do
+      expect_next_instance_of(Gitlab::ImportExport::WikiRepoSaver, exportable: group, shared: shared) do |saver|
+        expect(saver).to receive(:save).and_return(false)
+      end
+
+      expect(Gitlab::ImportExport::WikiRepoSaver).not_to receive(:new).with(exportable: subgroup, shared: shared)
+
+      expect(subject.save).to eq false
+    end
+  end
+end

ee/spec/services/ee/groups/import_export/export_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Groups::ImportExport::ExportService do
+  let_it_be(:user) { create(:user) }
+  let_it_be(:group) do
+    create(:group).tap do |g|
+      g.add_owner(user)
+    end
+  end
+
+  let_it_be(:group_wiki) do
+    create(:group_wiki, group: group).tap do |wiki|
+      wiki.create_page('test', 'test_content')
+    end
+  end
+
+  let(:shared) { Gitlab::ImportExport::Shared.new(group) }
+  let(:archive_path) { shared.archive_path }
+
+  subject(:export_service) { described_class.new(group: group, user: user, params: { shared: shared }) }
+
+  after do
+    FileUtils.rm_rf(archive_path)
+  end
+
+  describe '#execute' do
+    it 'exports group and descendants wiki repositories' do
+      subgroup = create(:group, :wiki_repo, parent: group)
+      subgroup.wiki.create_page('test', 'test_content')
+
+      expect_next_instance_of(::Gitlab::ImportExport::Group::GroupAndDescendantsRepoSaver, group: group, shared: shared) do |exporter|
+        expect(exporter).to receive(:save).and_call_original
+      end
+
+      # Avoid cleaning the tmp files in order to check the content of the dir
+      allow(export_service).to receive(:remove_archive_tmp_dir)
+
+      allow_next_instance_of(Gitlab::ImportExport::Saver) do |saver|
+        allow(saver).to receive(:save).and_return(true)
+      end
+
+      export_service.execute
+
+      expect(File.exist?(Gitlab::ImportExport.group_wiki_repo_bundle_full_path(shared, group.id))).to eq true
+      expect(File.exist?(Gitlab::ImportExport.group_wiki_repo_bundle_full_path(shared, subgroup.id))).to eq true
+    end
+
+    context 'when feature flag :group_wiki_import_export is disabled' do
+      it 'does not export group wiki repositories' do
+        stub_feature_flags(group_wiki_import_export: false)
+
+        expect(::Gitlab::ImportExport::Group::GroupAndDescendantsRepoSaver).not_to receive(:new)
+
+        export_service.execute
+      end
+    end
+
+    context 'when ndjson is not enabled' do
+      it 'does not export group wiki repositories' do
+        allow(export_service).to receive(:ndjson?).and_return(false)
+
+        expect(::Gitlab::ImportExport::Group::GroupAndDescendantsRepoRestorer).not_to receive(:new)
+
+        export_service.execute
+      end
+    end
+  end
+end

ee/spec/services/ee/groups/import_export/import_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Groups::ImportExport::ImportService do
+  let_it_be(:import_file) { fixture_file_upload('ee/spec/fixtures/group_export_with_wikis.tar.gz') }
+  let_it_be(:user) { create(:user) }
+  let_it_be_with_reload(:group) do
+    create(:group).tap do |g|
+      g.add_owner(user)
+    end
+  end
+
+  subject(:import_service) { described_class.new(group: group, user: user) }
+
+  before do
+    ImportExportUpload.create!(group: group, import_file: import_file)
+  end
+
+  context 'when group_wikis feature is enabled' do
+    it 'imports group and descendant wiki repositories', :aggregate_failures do
+      stub_licensed_features(group_wikis: true)
+
+      expect_next_instance_of(
+        Gitlab::ImportExport::Group::GroupAndDescendantsRepoRestorer,
+        group: group,
+        shared: instance_of(Gitlab::ImportExport::Shared),
+        tree_restorer: instance_of(Gitlab::ImportExport::Group::TreeRestorer)
+      ) do |restorer|
+        expect(restorer).to receive(:restore).and_call_original
+      end
+
+      import_service.execute
+
+      expect(group.wiki.repository_exists?).to be true
+      expect(group.wiki.list_pages.first.title).to eq 'home'
+
+      group.descendants.each do |subgroup|
+        expect(subgroup.wiki.repository_exists?).to be true
+        expect(subgroup.wiki.list_pages.first.title).to eq "home_#{subgroup.path}"
+      end
+    end
+
+    context 'when feature flag :group_wiki_import_export is disabled' do
+      it 'does not export group wiki repositories' do
+        stub_feature_flags(group_wiki_import_export: false)
+
+        expect(::Gitlab::ImportExport::Group::GroupAndDescendantsRepoRestorer).not_to receive(:new)
+
+        import_service.execute
+      end
+    end
+
+    context 'when export file not in ndjson format' do
+      let(:import_file) { fixture_file_upload('spec/fixtures/legacy_group_export.tar.gz') }
+
+      it 'does not export group wiki repositories' do
+        expect(::Gitlab::ImportExport::Group::GroupAndDescendantsRepoRestorer).not_to receive(:new)
+
+        import_service.execute
+      end
+    end
+  end
+
+  context 'when group_wikis feature is not enabled' do
+    it 'does not call the group wiki restorer' do
+      expect(::Gitlab::ImportExport::RepoRestorer).not_to receive(:new)
+
+      expect { import_service.execute }.not_to raise_error(Gitlab::ImportExport::Error)
+    end
+  end
+end

lib/gitlab/import_export.rb

@@ -103,3 +103,7 @@ module Gitlab
 end
 
 Gitlab::ImportExport.prepend_if_ee('EE::Gitlab::ImportExport')
+
+# The methods in `Gitlab::ImportExport::GroupHelper` should be available as both
+# instance and class methods.
+Gitlab::ImportExport.extend_if_ee('Gitlab::ImportExport::GroupHelper')

lib/gitlab/import_export/wiki_repo_saver.rb

@@ -19,3 +19,5 @@ module Gitlab
     end
   end
 end
+
+Gitlab::ImportExport::WikiRepoSaver.prepend_if_ee('EE::Gitlab::ImportExport::WikiRepoSaver')