Merge branch 'fj-39201-import-export-project-snippets' into 'master'

Import/Export project snippets

See merge request gitlab-org/gitlab!24150

app/models/snippet.rb

@@ -302,6 +302,10 @@ class Snippet < ApplicationRecord
     field != :content || MarkupHelper.gitlab_markdown?(file_name)
   end
 
+  def hexdigest
+    Digest::SHA256.hexdigest("#{title}#{description}#{created_at}#{updated_at}")
+  end
+
   class << self
     # Searches for snippets with a matching title or file name.
     #

app/models/snippet_repository.rb

@@ -18,6 +18,12 @@ class SnippetRepository < ApplicationRecord
     end
   end
 
+  def create_file(user, path, content, **options)
+    options[:actions] = transform_file_entries([{ file_path: path, content: content }])
+
+    capture_git_error { repository.multi_action(user, **options) }
+  end
+
   def multi_files_action(user, files = [], **options)
     return if files.nil? || files.empty?
 

app/services/projects/import_export/export_service.rb

@@ -42,7 +42,7 @@ module Projects
       end
 
       def exporters
-        [version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver, lfs_saver]
+        [version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver, lfs_saver, snippets_repo_saver]
       end
 
       def version_saver
@@ -73,6 +73,10 @@ module Projects
         Gitlab::ImportExport::LfsSaver.new(project: project, shared: shared)
       end
 
+      def snippets_repo_saver
+        Gitlab::ImportExport::SnippetsRepoSaver.new(current_user: current_user, project: project, shared: shared)
+      end
+
       def cleanup
         FileUtils.rm_rf(shared.archive_path) if shared&.archive_path
       end

changelogs/unreleased/fj-39201-import-export-project-snippets.yml

+---
+title: Import/Export snippet repositories
+merge_request: 24150
+author:
+type: added

lib/gitlab/import_export.rb

@@ -42,6 +42,18 @@ module Gitlab
       "project.wiki.bundle"
     end
 
+    def snippet_repo_bundle_dir
+      'snippets'
+    end
+
+    def snippets_repo_bundle_path(absolute_path)
+      File.join(absolute_path, ::Gitlab::ImportExport.snippet_repo_bundle_dir)
+    end
+
+    def snippet_repo_bundle_filename_for(snippet)
+      "#{snippet.hexdigest}.bundle"
+    end
+
     def config_file
       Rails.root.join('lib/gitlab/import_export/project/import_export.yml')
     end

lib/gitlab/import_export/importer.rb

@@ -35,7 +35,7 @@ module Gitlab
 
       def restorers
         [repo_restorer, wiki_restorer, project_tree, avatar_restorer,
-         uploads_restorer, lfs_restorer, statistics_restorer]
+         uploads_restorer, lfs_restorer, statistics_restorer, snippets_repo_restorer]
       end
 
       def import_file
@@ -79,6 +79,12 @@ module Gitlab
         Gitlab::ImportExport::LfsRestorer.new(project: project, shared: shared)
       end
 
+      def snippets_repo_restorer
+        Gitlab::ImportExport::SnippetsRepoRestorer.new(project: project,
+                                                       shared: shared,
+                                                       user: current_user)
+      end
+
       def statistics_restorer
         Gitlab::ImportExport::StatisticsRestorer.new(project: project, shared: shared)
       end

lib/gitlab/import_export/snippet_repo_restorer.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    class SnippetRepoRestorer < RepoRestorer
+      attr_reader :snippet
+
+      def initialize(snippet:, user:, shared:, path_to_bundle:)
+        @snippet = snippet
+        @user = user
+        @repository = snippet.repository
+        @path_to_bundle = path_to_bundle.to_s
+        @shared = shared
+      end
+
+      def restore
+        if File.exist?(path_to_bundle)
+          create_repository_from_bundle
+        else
+          create_repository_from_db
+        end
+
+        true
+      rescue => e
+        shared.error(e)
+        false
+      end
+
+      private
+
+      def create_repository_from_bundle
+        repository.create_from_bundle(path_to_bundle)
+        snippet.track_snippet_repository
+      end
+
+      def create_repository_from_db
+        snippet.create_repository
+
+        commit_attrs = {
+          branch_name: 'master',
+          message: 'Initial commit'
+        }
+
+        repository.create_file(@user, snippet.file_name, snippet.content, commit_attrs)
+      end
+    end
+  end
+end

lib/gitlab/import_export/snippet_repo_saver.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    class SnippetRepoSaver < RepoSaver
+      def initialize(project:, shared:, repository:)
+        @project = project
+        @shared = shared
+        @repository = repository
+      end
+
+      private
+
+      def bundle_full_path
+        File.join(shared.export_path,
+                  ::Gitlab::ImportExport.snippet_repo_bundle_dir,
+                  ::Gitlab::ImportExport.snippet_repo_bundle_filename_for(repository.container))
+      end
+    end
+  end
+end

lib/gitlab/import_export/snippets_repo_restorer.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    class SnippetsRepoRestorer
+      def initialize(project:, shared:, user:)
+        @project = project
+        @shared = shared
+        @user = user
+      end
+
+      def restore
+        return true unless Feature.enabled?(:version_snippets, @user)
+        return true unless Dir.exist?(snippets_repo_bundle_path)
+
+        @project.snippets.find_each.all? do |snippet|
+          Gitlab::ImportExport::SnippetRepoRestorer.new(snippet: snippet,
+                                                        user: @user,
+                                                        shared: @shared,
+                                                        path_to_bundle: snippet_repo_bundle_path(snippet))
+                                                   .restore
+        end
+      end
+
+      private
+
+      def snippet_repo_bundle_path(snippet)
+        File.join(snippets_repo_bundle_path, ::Gitlab::ImportExport.snippet_repo_bundle_filename_for(snippet))
+      end
+
+      def snippets_repo_bundle_path
+        @snippets_repo_bundle_path ||= ::Gitlab::ImportExport.snippets_repo_bundle_path(@shared.export_path)
+      end
+    end
+  end
+end

lib/gitlab/import_export/snippets_repo_saver.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    class SnippetsRepoSaver
+      include Gitlab::ImportExport::CommandLineUtil
+
+      def initialize(current_user:, project:, shared:)
+        @project = project
+        @shared = shared
+        @current_user = current_user
+      end
+
+      def save
+        return true unless Feature.enabled?(:version_snippets, @current_user)
+
+        create_snippets_repo_directory
+
+        @project.snippets.find_each.all? do |snippet|
+          Gitlab::ImportExport::SnippetRepoSaver.new(project: @project,
+                                                     shared: @shared,
+                                                     repository: snippet.repository)
+                                                .save
+        end
+      end
+
+      private
+
+      def create_snippets_repo_directory
+        mkdir_p(::Gitlab::ImportExport.snippets_repo_bundle_path(@shared.export_path))
+      end
+    end
+  end
+end

spec/factories/snippets.rb

@@ -34,7 +34,7 @@ FactoryBot.define do
 
     trait :empty_repo do
       after(:create) do |snippet|
-        raise "Failed to create repository!" unless snippet.repository.create_if_not_exists
+        raise "Failed to create repository!" unless snippet.create_repository
       end
     end
   end

spec/lib/gitlab/import_export/import_export_spec.rb

@@ -21,4 +21,12 @@ describe Gitlab::ImportExport do
       expect(described_class.export_filename(exportable: project).length).to be < 70
     end
   end
+
+  describe '#snippet_repo_bundle_filename_for' do
+    let(:snippet) { build(:snippet, id: 1) }
+
+    it 'generates the snippet bundle name' do
+      expect(described_class.snippet_repo_bundle_filename_for(snippet)).to eq "#{snippet.hexdigest}.bundle"
+    end
+  end
 end

spec/lib/gitlab/import_export/importer_spec.rb

@@ -50,7 +50,8 @@ describe Gitlab::ImportExport::Importer do
         Gitlab::ImportExport::WikiRestorer,
         Gitlab::ImportExport::UploadsRestorer,
         Gitlab::ImportExport::LfsRestorer,
-        Gitlab::ImportExport::StatisticsRestorer
+        Gitlab::ImportExport::StatisticsRestorer,
+        Gitlab::ImportExport::SnippetsRepoRestorer
       ].each do |restorer|
         it "calls the #{restorer}" do
           fake_restorer = double(restorer.to_s)

spec/lib/gitlab/import_export/snippet_repo_restorer_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::ImportExport::SnippetRepoRestorer do
+  let_it_be(:user) { create(:user) }
+  let_it_be(:project) { create(:project, namespace: user.namespace) }
+  let(:snippet) { create(:project_snippet, project: project, author: user) }
+
+  let(:shared) { project.import_export_shared }
+  let(:exporter) { Gitlab::ImportExport::SnippetsRepoSaver.new(project: project, shared: shared, current_user: user) }
+  let(:restorer) do
+    described_class.new(user: user,
+                        shared: shared,
+                        snippet: snippet,
+                        path_to_bundle: snippet_bundle_path)
+  end
+
+  after do
+    FileUtils.rm_rf(shared.export_path)
+  end
+
+  shared_examples 'no bundle file present' do
+    it 'creates the repository from the database content' do
+      expect(snippet.repository_exists?).to be_falsey
+
+      aggregate_failures do
+        expect(restorer.restore).to be_truthy
+
+        expect(snippet.repository_exists?).to be_truthy
+        expect(snippet.snippet_repository).not_to be_nil
+
+        blob = snippet.repository.blob_at('HEAD', snippet.file_name)
+        expect(blob).not_to be_nil
+        expect(blob.data).to eq(snippet.content)
+      end
+    end
+  end
+
+  context 'when the snippet does not have a bundle file path' do
+    let(:snippet_bundle_path) { nil }
+
+    it_behaves_like 'no bundle file present'
+  end
+
+  context 'when the snippet bundle path is not present' do
+    let(:snippet_bundle_path) { 'foo' }
+
+    it_behaves_like 'no bundle file present'
+  end
+
+  context 'when the snippet bundle exists' do
+    let!(:snippet_with_repo) { create(:project_snippet, :repository, project: project) }
+    let(:bundle_path) { ::Gitlab::ImportExport.snippets_repo_bundle_path(shared.export_path) }
+    let(:snippet_bundle_path) { File.join(bundle_path, "#{snippet_with_repo.hexdigest}.bundle") }
+    let(:result) { exporter.save }
+
+    it 'creates the repository from the bundle' do
+      expect(exporter.save).to be_truthy
+
+      expect(snippet.repository_exists?).to be_falsey
+      expect(snippet.snippet_repository).to be_nil
+      expect(snippet.repository).to receive(:create_from_bundle).and_call_original
+
+      expect(restorer.restore).to be_truthy
+      expect(snippet.repository_exists?).to be_truthy
+      expect(snippet.snippet_repository).not_to be_nil
+    end
+  end
+end

spec/lib/gitlab/import_export/snippet_repo_saver_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::ImportExport::SnippetRepoSaver do
+  describe 'bundle a project Git repo' do
+    let_it_be(:user) { create(:user) }
+    let_it_be(:project) { create(:project, namespace: user.namespace) }
+    let_it_be(:snippet) { create(:project_snippet, :repository, project: project, author: user) }
+    let(:shared) { project.import_export_shared }
+    let(:bundler) { described_class.new(project: project, shared: shared, repository: snippet.repository) }
+    let(:bundle_path) { ::Gitlab::ImportExport.snippets_repo_bundle_path(shared.export_path) }
+
+    around do |example|
+      FileUtils.mkdir_p(bundle_path)
+      example.run
+    ensure
+      FileUtils.rm_rf(bundle_path)
+    end
+
+    context 'with project snippet' do
+      it 'bundles the repo successfully' do
+        aggregate_failures do
+          expect(bundler.save).to be_truthy
+          expect(Dir.empty?(bundle_path)).to be_falsey
+        end
+      end
+
+      context 'when snippet does not have a repository' do
+        let(:snippet) { build(:personal_snippet) }
+
+        it 'returns true' do
+          expect(bundler.save).to be_truthy
+        end
+
+        it 'does not create any file' do
+          aggregate_failures do
+            expect(snippet.repository).not_to receive(:bundle_to_disk)
+
+            bundler.save
+
+            expect(Dir.empty?(bundle_path)).to be_truthy
+          end
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/import_export/snippets_repo_restorer_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::ImportExport::SnippetsRepoRestorer do
+  include GitHelpers
+
+  describe 'bundle a snippet Git repo' do
+    let_it_be(:user) { create(:user) }
+    let_it_be(:project) { create(:project, namespace: user.namespace) }
+    let_it_be(:snippet_with_repo) { create(:project_snippet, :repository, project: project, author: user) }
+    let_it_be(:snippet_without_repo) { create(:project_snippet, project: project, author: user) }
+
+    let(:shared) { project.import_export_shared }
+    let(:exporter) { Gitlab::ImportExport::SnippetsRepoSaver.new(current_user: user, project: project, shared: shared) }
+    let(:bundle_dir) { ::Gitlab::ImportExport.snippets_repo_bundle_path(shared.export_path) }
+    let(:restorer) do
+      described_class.new(user: user,
+                          shared: shared,
+                          project: project)
+    end
+    let(:service) { instance_double(Gitlab::ImportExport::SnippetRepoRestorer) }
+
+    before do
+      exporter.save
+    end
+
+    after do
+      FileUtils.rm_rf(shared.export_path)
+    end
+
+    it 'calls SnippetRepoRestorer per each snippet with the bundle path' do
+      allow(service).to receive(:restore).and_return(true)
+
+      expect(Gitlab::ImportExport::SnippetRepoRestorer).to receive(:new).with(hash_including(snippet: snippet_with_repo, path_to_bundle: bundle_path(snippet_with_repo))).and_return(service)
+      expect(Gitlab::ImportExport::SnippetRepoRestorer).to receive(:new).with(hash_including(snippet: snippet_without_repo, path_to_bundle: bundle_path(snippet_without_repo))).and_return(service)
+
+      expect(restorer.restore).to be_truthy
+    end
+
+    context 'when one snippet cannot be saved' do
+      it 'returns false and do not process other snippets' do
+        allow(Gitlab::ImportExport::SnippetRepoRestorer).to receive(:new).with(hash_including(snippet: snippet_with_repo)).and_return(service)
+        allow(service).to receive(:restore).and_return(false)
+
+        expect(Gitlab::ImportExport::SnippetRepoRestorer).not_to receive(:new).with(hash_including(snippet: snippet_without_repo))
+        expect(restorer.restore).to be_falsey
+      end
+    end
+
+    def bundle_path(snippet)
+      File.join(bundle_dir, ::Gitlab::ImportExport.snippet_repo_bundle_filename_for(snippet))
+    end
+  end
+end

spec/lib/gitlab/import_export/snippets_repo_saver_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::ImportExport::SnippetsRepoSaver do
+  describe 'bundle a project Git repo' do
+    let_it_be(:user) { create(:user) }
+    let!(:project) { create(:project) }
+    let(:shared) { project.import_export_shared }
+    let(:bundler) { described_class.new(current_user: user, project: project, shared: shared) }
+
+    after do
+      FileUtils.rm_rf(shared.export_path)
+    end
+
+    it 'creates the snippet bundles dir if not exists' do
+      snippets_dir = ::Gitlab::ImportExport.snippets_repo_bundle_path(shared.export_path)
+      expect(Dir.exist?(snippets_dir)).to be_falsey
+
+      bundler.save
+
+      expect(Dir.exist?(snippets_dir)).to be_truthy
+    end
+
+    context 'when project does not have any snippet' do
+      it 'does not perform any action' do
+        expect(Gitlab::ImportExport::SnippetRepoSaver).not_to receive(:new)
+
+        bundler.save
+      end
+    end
+
+    context 'when project has snippets' do
+      let!(:snippet1) { create(:project_snippet, :repository, project: project, author: user) }
+      let!(:snippet2) { create(:project_snippet, project: project, author: user) }
+      let(:service) { instance_double(Gitlab::ImportExport::SnippetRepoSaver) }
+
+      it 'calls the SnippetRepoSaver for each snippet' do
+        allow(Gitlab::ImportExport::SnippetRepoSaver).to receive(:new).and_return(service)
+        expect(service).to receive(:save).and_return(true).twice
+
+        bundler.save
+      end
+
+      context 'when one snippet cannot be saved' do
+        it 'returns false and do not process other snippets' do
+          allow(Gitlab::ImportExport::SnippetRepoSaver).to receive(:new).with(hash_including(repository: snippet1.repository)).and_return(service)
+          allow(service).to receive(:save).and_return(false)
+
+          expect(Gitlab::ImportExport::SnippetRepoSaver).not_to receive(:new).with(hash_including(repository: snippet2.repository))
+          expect(bundler.save).to be_falsey
+        end
+      end
+    end
+  end
+end

spec/models/snippet_repository_spec.rb

@@ -26,6 +26,44 @@ describe SnippetRepository do
     end
   end
 
+  describe '#create_file' do
+    let(:snippet) { create(:personal_snippet, :empty_repo, author: user) }
+
+    it 'creates the file' do
+      snippet_repository.create_file(user, 'foo', 'bar', commit_opts)
+      blob = first_blob(snippet)
+
+      aggregate_failures do
+        expect(blob).not_to be_nil
+        expect(blob.path).to eq 'foo'
+        expect(blob.data).to eq 'bar'
+      end
+    end
+
+    it 'fills the file path if empty' do
+      snippet_repository.create_file(user, nil, 'bar', commit_opts)
+      blob = first_blob(snippet)
+
+      aggregate_failures do
+        expect(blob).not_to be_nil
+        expect(blob.path).to eq 'snippetfile1.txt'
+        expect(blob.data).to eq 'bar'
+      end
+    end
+
+    context 'when the file exists' do
+      let(:snippet) { create(:personal_snippet, :repository, author: user) }
+
+      it 'captures the git exception and raises a SnippetRepository::CommitError' do
+        existing_blob = first_blob(snippet)
+
+        expect do
+          snippet_repository.create_file(user, existing_blob.path, existing_blob.data, commit_opts)
+        end.to raise_error described_class::CommitError
+      end
+    end
+  end
+
   describe '#multi_files_action' do
     let(:new_file) { { file_path: 'new_file_test', content: 'bar' } }
     let(:move_file) { { previous_path: 'CHANGELOG', file_path: 'CHANGELOG_new', content: 'bar' } }

spec/services/projects/import_export/export_service_spec.rb

@@ -64,6 +64,14 @@ describe Projects::ImportExport::ExportService do
       service.execute
     end
 
+    it 'saves the snippets' do
+      expect_next_instance_of(Gitlab::ImportExport::SnippetsRepoSaver) do |instance|
+        expect(instance).to receive(:save).and_call_original
+      end
+
+      service.execute
+    end
+
     context 'when all saver services succeed' do
       before do
         allow(service).to receive(:save_services).and_return(true)