Commit 636f17bc authored by Peter Leitzen's avatar Peter Leitzen

Merge branch 'fj-39201-import-export-project-snippets' into 'master'

Import/Export project snippets

See merge request gitlab-org/gitlab!24150
parents ec95d416 aefba81a
...@@ -302,6 +302,10 @@ class Snippet < ApplicationRecord ...@@ -302,6 +302,10 @@ class Snippet < ApplicationRecord
field != :content || MarkupHelper.gitlab_markdown?(file_name) field != :content || MarkupHelper.gitlab_markdown?(file_name)
end end
def hexdigest
Digest::SHA256.hexdigest("#{title}#{description}#{created_at}#{updated_at}")
end
class << self class << self
# Searches for snippets with a matching title or file name. # Searches for snippets with a matching title or file name.
# #
......
...@@ -18,6 +18,12 @@ class SnippetRepository < ApplicationRecord ...@@ -18,6 +18,12 @@ class SnippetRepository < ApplicationRecord
end end
end end
def create_file(user, path, content, **options)
options[:actions] = transform_file_entries([{ file_path: path, content: content }])
capture_git_error { repository.multi_action(user, **options) }
end
def multi_files_action(user, files = [], **options) def multi_files_action(user, files = [], **options)
return if files.nil? || files.empty? return if files.nil? || files.empty?
......
...@@ -42,7 +42,7 @@ module Projects ...@@ -42,7 +42,7 @@ module Projects
end end
def exporters def exporters
[version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver, lfs_saver] [version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver, lfs_saver, snippets_repo_saver]
end end
def version_saver def version_saver
...@@ -73,6 +73,10 @@ module Projects ...@@ -73,6 +73,10 @@ module Projects
Gitlab::ImportExport::LfsSaver.new(project: project, shared: shared) Gitlab::ImportExport::LfsSaver.new(project: project, shared: shared)
end end
def snippets_repo_saver
Gitlab::ImportExport::SnippetsRepoSaver.new(current_user: current_user, project: project, shared: shared)
end
def cleanup def cleanup
FileUtils.rm_rf(shared.archive_path) if shared&.archive_path FileUtils.rm_rf(shared.archive_path) if shared&.archive_path
end end
......
---
title: Import/Export snippet repositories
merge_request: 24150
author:
type: added
...@@ -42,6 +42,18 @@ module Gitlab ...@@ -42,6 +42,18 @@ module Gitlab
"project.wiki.bundle" "project.wiki.bundle"
end end
def snippet_repo_bundle_dir
'snippets'
end
def snippets_repo_bundle_path(absolute_path)
File.join(absolute_path, ::Gitlab::ImportExport.snippet_repo_bundle_dir)
end
def snippet_repo_bundle_filename_for(snippet)
"#{snippet.hexdigest}.bundle"
end
def config_file def config_file
Rails.root.join('lib/gitlab/import_export/project/import_export.yml') Rails.root.join('lib/gitlab/import_export/project/import_export.yml')
end end
......
...@@ -35,7 +35,7 @@ module Gitlab ...@@ -35,7 +35,7 @@ module Gitlab
def restorers def restorers
[repo_restorer, wiki_restorer, project_tree, avatar_restorer, [repo_restorer, wiki_restorer, project_tree, avatar_restorer,
uploads_restorer, lfs_restorer, statistics_restorer] uploads_restorer, lfs_restorer, statistics_restorer, snippets_repo_restorer]
end end
def import_file def import_file
...@@ -79,6 +79,12 @@ module Gitlab ...@@ -79,6 +79,12 @@ module Gitlab
Gitlab::ImportExport::LfsRestorer.new(project: project, shared: shared) Gitlab::ImportExport::LfsRestorer.new(project: project, shared: shared)
end end
def snippets_repo_restorer
Gitlab::ImportExport::SnippetsRepoRestorer.new(project: project,
shared: shared,
user: current_user)
end
def statistics_restorer def statistics_restorer
Gitlab::ImportExport::StatisticsRestorer.new(project: project, shared: shared) Gitlab::ImportExport::StatisticsRestorer.new(project: project, shared: shared)
end end
......
# frozen_string_literal: true
module Gitlab
module ImportExport
class SnippetRepoRestorer < RepoRestorer
attr_reader :snippet
def initialize(snippet:, user:, shared:, path_to_bundle:)
@snippet = snippet
@user = user
@repository = snippet.repository
@path_to_bundle = path_to_bundle.to_s
@shared = shared
end
def restore
if File.exist?(path_to_bundle)
create_repository_from_bundle
else
create_repository_from_db
end
true
rescue => e
shared.error(e)
false
end
private
def create_repository_from_bundle
repository.create_from_bundle(path_to_bundle)
snippet.track_snippet_repository
end
def create_repository_from_db
snippet.create_repository
commit_attrs = {
branch_name: 'master',
message: 'Initial commit'
}
repository.create_file(@user, snippet.file_name, snippet.content, commit_attrs)
end
end
end
end
# frozen_string_literal: true
module Gitlab
module ImportExport
class SnippetRepoSaver < RepoSaver
def initialize(project:, shared:, repository:)
@project = project
@shared = shared
@repository = repository
end
private
def bundle_full_path
File.join(shared.export_path,
::Gitlab::ImportExport.snippet_repo_bundle_dir,
::Gitlab::ImportExport.snippet_repo_bundle_filename_for(repository.container))
end
end
end
end
# frozen_string_literal: true
module Gitlab
module ImportExport
class SnippetsRepoRestorer
def initialize(project:, shared:, user:)
@project = project
@shared = shared
@user = user
end
def restore
return true unless Feature.enabled?(:version_snippets, @user)
return true unless Dir.exist?(snippets_repo_bundle_path)
@project.snippets.find_each.all? do |snippet|
Gitlab::ImportExport::SnippetRepoRestorer.new(snippet: snippet,
user: @user,
shared: @shared,
path_to_bundle: snippet_repo_bundle_path(snippet))
.restore
end
end
private
def snippet_repo_bundle_path(snippet)
File.join(snippets_repo_bundle_path, ::Gitlab::ImportExport.snippet_repo_bundle_filename_for(snippet))
end
def snippets_repo_bundle_path
@snippets_repo_bundle_path ||= ::Gitlab::ImportExport.snippets_repo_bundle_path(@shared.export_path)
end
end
end
end
# frozen_string_literal: true
module Gitlab
module ImportExport
class SnippetsRepoSaver
include Gitlab::ImportExport::CommandLineUtil
def initialize(current_user:, project:, shared:)
@project = project
@shared = shared
@current_user = current_user
end
def save
return true unless Feature.enabled?(:version_snippets, @current_user)
create_snippets_repo_directory
@project.snippets.find_each.all? do |snippet|
Gitlab::ImportExport::SnippetRepoSaver.new(project: @project,
shared: @shared,
repository: snippet.repository)
.save
end
end
private
def create_snippets_repo_directory
mkdir_p(::Gitlab::ImportExport.snippets_repo_bundle_path(@shared.export_path))
end
end
end
end
...@@ -34,7 +34,7 @@ FactoryBot.define do ...@@ -34,7 +34,7 @@ FactoryBot.define do
trait :empty_repo do trait :empty_repo do
after(:create) do |snippet| after(:create) do |snippet|
raise "Failed to create repository!" unless snippet.repository.create_if_not_exists raise "Failed to create repository!" unless snippet.create_repository
end end
end end
end end
......
...@@ -21,4 +21,12 @@ describe Gitlab::ImportExport do ...@@ -21,4 +21,12 @@ describe Gitlab::ImportExport do
expect(described_class.export_filename(exportable: project).length).to be < 70 expect(described_class.export_filename(exportable: project).length).to be < 70
end end
end end
describe '#snippet_repo_bundle_filename_for' do
let(:snippet) { build(:snippet, id: 1) }
it 'generates the snippet bundle name' do
expect(described_class.snippet_repo_bundle_filename_for(snippet)).to eq "#{snippet.hexdigest}.bundle"
end
end
end end
...@@ -50,7 +50,8 @@ describe Gitlab::ImportExport::Importer do ...@@ -50,7 +50,8 @@ describe Gitlab::ImportExport::Importer do
Gitlab::ImportExport::WikiRestorer, Gitlab::ImportExport::WikiRestorer,
Gitlab::ImportExport::UploadsRestorer, Gitlab::ImportExport::UploadsRestorer,
Gitlab::ImportExport::LfsRestorer, Gitlab::ImportExport::LfsRestorer,
Gitlab::ImportExport::StatisticsRestorer Gitlab::ImportExport::StatisticsRestorer,
Gitlab::ImportExport::SnippetsRepoRestorer
].each do |restorer| ].each do |restorer|
it "calls the #{restorer}" do it "calls the #{restorer}" do
fake_restorer = double(restorer.to_s) fake_restorer = double(restorer.to_s)
......
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::ImportExport::SnippetRepoRestorer do
let_it_be(:user) { create(:user) }
let_it_be(:project) { create(:project, namespace: user.namespace) }
let(:snippet) { create(:project_snippet, project: project, author: user) }
let(:shared) { project.import_export_shared }
let(:exporter) { Gitlab::ImportExport::SnippetsRepoSaver.new(project: project, shared: shared, current_user: user) }
let(:restorer) do
described_class.new(user: user,
shared: shared,
snippet: snippet,
path_to_bundle: snippet_bundle_path)
end
after do
FileUtils.rm_rf(shared.export_path)
end
shared_examples 'no bundle file present' do
it 'creates the repository from the database content' do
expect(snippet.repository_exists?).to be_falsey
aggregate_failures do
expect(restorer.restore).to be_truthy
expect(snippet.repository_exists?).to be_truthy
expect(snippet.snippet_repository).not_to be_nil
blob = snippet.repository.blob_at('HEAD', snippet.file_name)
expect(blob).not_to be_nil
expect(blob.data).to eq(snippet.content)
end
end
end
context 'when the snippet does not have a bundle file path' do
let(:snippet_bundle_path) { nil }
it_behaves_like 'no bundle file present'
end
context 'when the snippet bundle path is not present' do
let(:snippet_bundle_path) { 'foo' }
it_behaves_like 'no bundle file present'
end
context 'when the snippet bundle exists' do
let!(:snippet_with_repo) { create(:project_snippet, :repository, project: project) }
let(:bundle_path) { ::Gitlab::ImportExport.snippets_repo_bundle_path(shared.export_path) }
let(:snippet_bundle_path) { File.join(bundle_path, "#{snippet_with_repo.hexdigest}.bundle") }
let(:result) { exporter.save }
it 'creates the repository from the bundle' do
expect(exporter.save).to be_truthy
expect(snippet.repository_exists?).to be_falsey
expect(snippet.snippet_repository).to be_nil
expect(snippet.repository).to receive(:create_from_bundle).and_call_original
expect(restorer.restore).to be_truthy
expect(snippet.repository_exists?).to be_truthy
expect(snippet.snippet_repository).not_to be_nil
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::ImportExport::SnippetRepoSaver do
describe 'bundle a project Git repo' do
let_it_be(:user) { create(:user) }
let_it_be(:project) { create(:project, namespace: user.namespace) }
let_it_be(:snippet) { create(:project_snippet, :repository, project: project, author: user) }
let(:shared) { project.import_export_shared }
let(:bundler) { described_class.new(project: project, shared: shared, repository: snippet.repository) }
let(:bundle_path) { ::Gitlab::ImportExport.snippets_repo_bundle_path(shared.export_path) }
around do |example|
FileUtils.mkdir_p(bundle_path)
example.run
ensure
FileUtils.rm_rf(bundle_path)
end
context 'with project snippet' do
it 'bundles the repo successfully' do
aggregate_failures do
expect(bundler.save).to be_truthy
expect(Dir.empty?(bundle_path)).to be_falsey
end
end
context 'when snippet does not have a repository' do
let(:snippet) { build(:personal_snippet) }
it 'returns true' do
expect(bundler.save).to be_truthy
end
it 'does not create any file' do
aggregate_failures do
expect(snippet.repository).not_to receive(:bundle_to_disk)
bundler.save
expect(Dir.empty?(bundle_path)).to be_truthy
end
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::ImportExport::SnippetsRepoRestorer do
include GitHelpers
describe 'bundle a snippet Git repo' do
let_it_be(:user) { create(:user) }
let_it_be(:project) { create(:project, namespace: user.namespace) }
let_it_be(:snippet_with_repo) { create(:project_snippet, :repository, project: project, author: user) }
let_it_be(:snippet_without_repo) { create(:project_snippet, project: project, author: user) }
let(:shared) { project.import_export_shared }
let(:exporter) { Gitlab::ImportExport::SnippetsRepoSaver.new(current_user: user, project: project, shared: shared) }
let(:bundle_dir) { ::Gitlab::ImportExport.snippets_repo_bundle_path(shared.export_path) }
let(:restorer) do
described_class.new(user: user,
shared: shared,
project: project)
end
let(:service) { instance_double(Gitlab::ImportExport::SnippetRepoRestorer) }
before do
exporter.save
end
after do
FileUtils.rm_rf(shared.export_path)
end
it 'calls SnippetRepoRestorer per each snippet with the bundle path' do
allow(service).to receive(:restore).and_return(true)
expect(Gitlab::ImportExport::SnippetRepoRestorer).to receive(:new).with(hash_including(snippet: snippet_with_repo, path_to_bundle: bundle_path(snippet_with_repo))).and_return(service)
expect(Gitlab::ImportExport::SnippetRepoRestorer).to receive(:new).with(hash_including(snippet: snippet_without_repo, path_to_bundle: bundle_path(snippet_without_repo))).and_return(service)
expect(restorer.restore).to be_truthy
end
context 'when one snippet cannot be saved' do
it 'returns false and do not process other snippets' do
allow(Gitlab::ImportExport::SnippetRepoRestorer).to receive(:new).with(hash_including(snippet: snippet_with_repo)).and_return(service)
allow(service).to receive(:restore).and_return(false)
expect(Gitlab::ImportExport::SnippetRepoRestorer).not_to receive(:new).with(hash_including(snippet: snippet_without_repo))
expect(restorer.restore).to be_falsey
end
end
def bundle_path(snippet)
File.join(bundle_dir, ::Gitlab::ImportExport.snippet_repo_bundle_filename_for(snippet))
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::ImportExport::SnippetsRepoSaver do
describe 'bundle a project Git repo' do
let_it_be(:user) { create(:user) }
let!(:project) { create(:project) }
let(:shared) { project.import_export_shared }
let(:bundler) { described_class.new(current_user: user, project: project, shared: shared) }
after do
FileUtils.rm_rf(shared.export_path)
end
it 'creates the snippet bundles dir if not exists' do
snippets_dir = ::Gitlab::ImportExport.snippets_repo_bundle_path(shared.export_path)
expect(Dir.exist?(snippets_dir)).to be_falsey
bundler.save
expect(Dir.exist?(snippets_dir)).to be_truthy
end
context 'when project does not have any snippet' do
it 'does not perform any action' do
expect(Gitlab::ImportExport::SnippetRepoSaver).not_to receive(:new)
bundler.save
end
end
context 'when project has snippets' do
let!(:snippet1) { create(:project_snippet, :repository, project: project, author: user) }
let!(:snippet2) { create(:project_snippet, project: project, author: user) }
let(:service) { instance_double(Gitlab::ImportExport::SnippetRepoSaver) }
it 'calls the SnippetRepoSaver for each snippet' do
allow(Gitlab::ImportExport::SnippetRepoSaver).to receive(:new).and_return(service)
expect(service).to receive(:save).and_return(true).twice
bundler.save
end
context 'when one snippet cannot be saved' do
it 'returns false and do not process other snippets' do
allow(Gitlab::ImportExport::SnippetRepoSaver).to receive(:new).with(hash_including(repository: snippet1.repository)).and_return(service)
allow(service).to receive(:save).and_return(false)
expect(Gitlab::ImportExport::SnippetRepoSaver).not_to receive(:new).with(hash_including(repository: snippet2.repository))
expect(bundler.save).to be_falsey
end
end
end
end
end
...@@ -26,6 +26,44 @@ describe SnippetRepository do ...@@ -26,6 +26,44 @@ describe SnippetRepository do
end end
end end
describe '#create_file' do
let(:snippet) { create(:personal_snippet, :empty_repo, author: user) }
it 'creates the file' do
snippet_repository.create_file(user, 'foo', 'bar', commit_opts)
blob = first_blob(snippet)
aggregate_failures do
expect(blob).not_to be_nil
expect(blob.path).to eq 'foo'
expect(blob.data).to eq 'bar'
end
end
it 'fills the file path if empty' do
snippet_repository.create_file(user, nil, 'bar', commit_opts)
blob = first_blob(snippet)
aggregate_failures do
expect(blob).not_to be_nil
expect(blob.path).to eq 'snippetfile1.txt'
expect(blob.data).to eq 'bar'
end
end
context 'when the file exists' do
let(:snippet) { create(:personal_snippet, :repository, author: user) }
it 'captures the git exception and raises a SnippetRepository::CommitError' do
existing_blob = first_blob(snippet)
expect do
snippet_repository.create_file(user, existing_blob.path, existing_blob.data, commit_opts)
end.to raise_error described_class::CommitError
end
end
end
describe '#multi_files_action' do describe '#multi_files_action' do
let(:new_file) { { file_path: 'new_file_test', content: 'bar' } } let(:new_file) { { file_path: 'new_file_test', content: 'bar' } }
let(:move_file) { { previous_path: 'CHANGELOG', file_path: 'CHANGELOG_new', content: 'bar' } } let(:move_file) { { previous_path: 'CHANGELOG', file_path: 'CHANGELOG_new', content: 'bar' } }
......
...@@ -64,6 +64,14 @@ describe Projects::ImportExport::ExportService do ...@@ -64,6 +64,14 @@ describe Projects::ImportExport::ExportService do
service.execute service.execute
end end
it 'saves the snippets' do
expect_next_instance_of(Gitlab::ImportExport::SnippetsRepoSaver) do |instance|
expect(instance).to receive(:save).and_call_original
end
service.execute
end
context 'when all saver services succeed' do context 'when all saver services succeed' do
before do before do
allow(service).to receive(:save_services).and_return(true) allow(service).to receive(:save_services).and_return(true)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment