Commit 9b7fd748 authored by Douwe Maan's avatar Douwe Maan

Merge branch 'feature/project-export' into 'master'

Export project functionality

This is a MR for the export functionality of https://gitlab.com/gitlab-org/gitlab-ce/issues/3050, which adds the ability to export single projects.

- [x] members
- DB data
  - [x] issues
  - [x] issue comments
  - [x] merge requests
  - [x] merge request diff
  - [x] merge request comments
  - [x] labels
  - [x] milestones
  - [x] snippets
  - [x] releases
  - [x] events
  - [x] commit statuses
  - [x] CI builds
- File system data
  - [x] Git repository
  - [x] wiki
  - [x] uploads
  - [ ] ~~CI build traces~~
  - [ ] ~~CI build artifacts~~
  - [ ] ~~LFS objects~~
- DB configuration
  - [x] services
  - [x] web hooks
  - [x] protected branches
  - [x] deploy keys
  - [x] CI variables
  - [x] CI triggers

See merge request !3114
parents ae4491b4 8891bef2
...@@ -100,6 +100,7 @@ v 8.9.0 (unreleased) ...@@ -100,6 +100,7 @@ v 8.9.0 (unreleased)
- Improved UX of date pickers on issue & milestone forms - Improved UX of date pickers on issue & milestone forms
- Cache on the database if a project has an active external issue tracker. - Cache on the database if a project has an active external issue tracker.
- Put project Labels and Milestones pages links under Issues and Merge Requests tabs as subnav - Put project Labels and Milestones pages links under Issues and Merge Requests tabs as subnav
- GitLab project import and export functionality
- All classes in the Banzai::ReferenceParser namespace are now instrumented - All classes in the Banzai::ReferenceParser namespace are now instrumented
- Remove deprecated issues_tracker and issues_tracker_id from project model - Remove deprecated issues_tracker and issues_tracker_id from project model
- Allow users to create confidential issues in private projects - Allow users to create confidential issues in private projects
......
...@@ -221,7 +221,7 @@ gem 'jquery-turbolinks', '~> 2.1.0' ...@@ -221,7 +221,7 @@ gem 'jquery-turbolinks', '~> 2.1.0'
gem 'addressable', '~> 2.3.8' gem 'addressable', '~> 2.3.8'
gem 'bootstrap-sass', '~> 3.3.0' gem 'bootstrap-sass', '~> 3.3.0'
gem 'font-awesome-rails', '~> 4.2' gem 'font-awesome-rails', '~> 4.6.1'
gem 'gitlab_emoji', '~> 0.3.0' gem 'gitlab_emoji', '~> 0.3.0'
gem 'gon', '~> 6.0.1' gem 'gon', '~> 6.0.1'
gem 'jquery-atwho-rails', '~> 1.3.2' gem 'jquery-atwho-rails', '~> 1.3.2'
......
...@@ -246,7 +246,7 @@ GEM ...@@ -246,7 +246,7 @@ GEM
fog-xml (0.1.2) fog-xml (0.1.2)
fog-core fog-core
nokogiri (~> 1.5, >= 1.5.11) nokogiri (~> 1.5, >= 1.5.11)
font-awesome-rails (4.5.0.1) font-awesome-rails (4.6.1.0)
railties (>= 3.2, < 5.1) railties (>= 3.2, < 5.1)
foreman (0.78.0) foreman (0.78.0)
thor (~> 0.19.1) thor (~> 0.19.1)
...@@ -866,7 +866,7 @@ DEPENDENCIES ...@@ -866,7 +866,7 @@ DEPENDENCIES
fog-google (~> 0.3) fog-google (~> 0.3)
fog-local (~> 0.3) fog-local (~> 0.3)
fog-openstack (~> 0.1) fog-openstack (~> 0.1)
font-awesome-rails (~> 4.2) font-awesome-rails (~> 4.6.1)
foreman foreman
fuubar (~> 2.0.0) fuubar (~> 2.0.0)
gemnasium-gitlab-service (~> 0.2) gemnasium-gitlab-service (~> 0.2)
......
...@@ -24,7 +24,7 @@ class ApplicationController < ActionController::Base ...@@ -24,7 +24,7 @@ class ApplicationController < ActionController::Base
protect_from_forgery with: :exception protect_from_forgery with: :exception
helper_method :abilities, :can?, :current_application_settings helper_method :abilities, :can?, :current_application_settings
helper_method :import_sources_enabled?, :github_import_enabled?, :github_import_configured?, :gitlab_import_enabled?, :gitlab_import_configured?, :bitbucket_import_enabled?, :bitbucket_import_configured?, :gitorious_import_enabled?, :google_code_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled? helper_method :import_sources_enabled?, :github_import_enabled?, :github_import_configured?, :gitlab_import_enabled?, :gitlab_import_configured?, :bitbucket_import_enabled?, :bitbucket_import_configured?, :gitorious_import_enabled?, :google_code_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled?, :gitlab_project_import_enabled?
rescue_from Encoding::CompatibilityError do |exception| rescue_from Encoding::CompatibilityError do |exception|
log_exception(exception) log_exception(exception)
...@@ -319,6 +319,10 @@ class ApplicationController < ActionController::Base ...@@ -319,6 +319,10 @@ class ApplicationController < ActionController::Base
current_application_settings.import_sources.include?('git') current_application_settings.import_sources.include?('git')
end end
def gitlab_project_import_enabled?
current_application_settings.import_sources.include?('gitlab_project')
end
def two_factor_authentication_required? def two_factor_authentication_required?
current_application_settings.require_two_factor_authentication current_application_settings.require_two_factor_authentication
end end
......
class Import::GitlabProjectsController < Import::BaseController
before_action :verify_gitlab_project_import_enabled
def new
@namespace_id = project_params[:namespace_id]
@namespace_name = Namespace.find(project_params[:namespace_id]).name
@path = project_params[:path]
end
def create
unless file_is_valid?
return redirect_back_or_default(options: { alert: "You need to upload a GitLab project export archive." })
end
@project = Gitlab::ImportExport::ProjectCreator.new(project_params[:namespace_id],
current_user,
File.expand_path(project_params[:file].path),
project_params[:path]).execute
if @project.saved?
redirect_to(
project_path(@project),
notice: "Project '#{@project.name}' is being imported."
)
else
redirect_to(
new_import_gitlab_project_path,
alert: "Project could not be imported: #{@project.errors.full_messages.join(', ')}"
)
end
end
private
def file_is_valid?
project_params[:file] && project_params[:file].respond_to?(:read)
end
def verify_gitlab_project_import_enabled
render_404 unless gitlab_project_import_enabled?
end
def project_params
params.permit(
:path, :namespace_id, :file
)
end
end
...@@ -7,7 +7,7 @@ class ProjectsController < Projects::ApplicationController ...@@ -7,7 +7,7 @@ class ProjectsController < Projects::ApplicationController
before_action :assign_ref_vars, :tree, only: [:show], if: :repo_exists? before_action :assign_ref_vars, :tree, only: [:show], if: :repo_exists?
# Authorize # Authorize
before_action :authorize_admin_project!, only: [:edit, :update, :housekeeping] before_action :authorize_admin_project!, only: [:edit, :update, :housekeeping, :download_export, :export, :remove_export, :generate_new_export]
before_action :event_filter, only: [:show, :activity] before_action :event_filter, only: [:show, :activity]
layout :determine_layout layout :determine_layout
...@@ -186,6 +186,48 @@ class ProjectsController < Projects::ApplicationController ...@@ -186,6 +186,48 @@ class ProjectsController < Projects::ApplicationController
) )
end end
def export
@project.add_export_job(current_user: current_user)
redirect_to(
edit_project_path(@project),
notice: "Project export started. A download link will be sent by email."
)
end
def download_export
export_project_path = @project.export_project_path
if export_project_path
send_file export_project_path, disposition: 'attachment'
else
redirect_to(
edit_project_path(@project),
alert: "Project export link has expired. Please generate a new export from your project settings."
)
end
end
def remove_export
if @project.remove_exports
flash[:notice] = "Project export has been deleted."
else
flash[:alert] = "Project export could not be deleted."
end
redirect_to(edit_project_path(@project))
end
def generate_new_export
if @project.remove_exports
export
else
redirect_to(
edit_project_path(@project),
alert: "Project export could not be deleted."
)
end
end
def toggle_star def toggle_star
current_user.toggle_star(@project) current_user.toggle_star(@project)
@project.reload @project.reload
......
...@@ -9,6 +9,19 @@ module Emails ...@@ -9,6 +9,19 @@ module Emails
subject: subject("Project was moved")) subject: subject("Project was moved"))
end end
def project_was_exported_email(current_user, project)
@project = project
mail(to: current_user.notification_email,
subject: subject("Project was exported"))
end
def project_was_not_exported_email(current_user, project, errors)
@project = project
@errors = errors
mail(to: current_user.notification_email,
subject: subject("Project export error"))
end
def repository_push_email(project_id, opts = {}) def repository_push_email(project_id, opts = {})
@message = @message =
Gitlab::Email::Message::RepositoryPush.new(self, project_id, opts) Gitlab::Email::Message::RepositoryPush.new(self, project_id, opts)
......
...@@ -123,7 +123,7 @@ class ApplicationSetting < ActiveRecord::Base ...@@ -123,7 +123,7 @@ class ApplicationSetting < ActiveRecord::Base
default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'],
default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'],
restricted_signup_domains: Settings.gitlab['restricted_signup_domains'], restricted_signup_domains: Settings.gitlab['restricted_signup_domains'],
import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'], import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project],
shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
max_artifacts_size: Settings.artifacts['max_size'], max_artifacts_size: Settings.artifacts['max_size'],
require_two_factor_authentication: false, require_two_factor_authentication: false,
......
...@@ -170,6 +170,10 @@ module Ci ...@@ -170,6 +170,10 @@ module Ci
builds.where.not(environment: nil).success.pluck(:environment).uniq builds.where.not(environment: nil).success.pluck(:environment).uniq
end end
def notes
Note.for_commit_id(sha)
end
private private
def build_builds_for_stages(stages, user, status, trigger_request) def build_builds_for_stages(stages, user, status, trigger_request)
......
class CommitStatus < ActiveRecord::Base class CommitStatus < ActiveRecord::Base
include Statuseable include Statuseable
include Importable
self.table_name = 'ci_builds' self.table_name = 'ci_builds'
...@@ -7,7 +8,7 @@ class CommitStatus < ActiveRecord::Base ...@@ -7,7 +8,7 @@ class CommitStatus < ActiveRecord::Base
belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id, touch: true belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id, touch: true
belongs_to :user belongs_to :user
validates :pipeline, presence: true validates :pipeline, presence: true, unless: :importing?
validates_presence_of :name validates_presence_of :name
......
module Importable
extend ActiveSupport::Concern
attr_accessor :importing
alias_method :importing?, :importing
end
class Member < ActiveRecord::Base class Member < ActiveRecord::Base
include Sortable include Sortable
include Importable
include Gitlab::Access include Gitlab::Access
attr_accessor :raw_invite_token attr_accessor :raw_invite_token
...@@ -41,11 +42,11 @@ class Member < ActiveRecord::Base ...@@ -41,11 +42,11 @@ class Member < ActiveRecord::Base
before_validation :generate_invite_token, on: :create, if: -> (member) { member.invite_email.present? } before_validation :generate_invite_token, on: :create, if: -> (member) { member.invite_email.present? }
after_create :send_invite, if: :invite? after_create :send_invite, if: :invite?, unless: :importing?
after_create :send_request, if: :request? after_create :send_request, if: :request?, unless: :importing?
after_create :create_notification_setting, unless: :pending? after_create :create_notification_setting, unless: [:pending?, :importing?]
after_create :post_create_hook, unless: :pending? after_create :post_create_hook, unless: [:pending?, :importing?]
after_update :post_update_hook, unless: :pending? after_update :post_update_hook, unless: [:pending?, :importing?]
after_destroy :post_destroy_hook, unless: :pending? after_destroy :post_destroy_hook, unless: :pending?
after_destroy :post_decline_request, if: :request? after_destroy :post_decline_request, if: :request?
......
...@@ -4,6 +4,7 @@ class MergeRequest < ActiveRecord::Base ...@@ -4,6 +4,7 @@ class MergeRequest < ActiveRecord::Base
include Referable include Referable
include Sortable include Sortable
include Taskable include Taskable
include Importable
belongs_to :target_project, foreign_key: :target_project_id, class_name: "Project" belongs_to :target_project, foreign_key: :target_project_id, class_name: "Project"
belongs_to :source_project, foreign_key: :source_project_id, class_name: "Project" belongs_to :source_project, foreign_key: :source_project_id, class_name: "Project"
...@@ -13,7 +14,7 @@ class MergeRequest < ActiveRecord::Base ...@@ -13,7 +14,7 @@ class MergeRequest < ActiveRecord::Base
serialize :merge_params, Hash serialize :merge_params, Hash
after_create :create_merge_request_diff after_create :create_merge_request_diff, unless: :importing
after_update :update_merge_request_diff after_update :update_merge_request_diff
delegate :commits, :diffs, :real_size, to: :merge_request_diff, prefix: nil delegate :commits, :diffs, :real_size, to: :merge_request_diff, prefix: nil
...@@ -95,12 +96,12 @@ class MergeRequest < ActiveRecord::Base ...@@ -95,12 +96,12 @@ class MergeRequest < ActiveRecord::Base
end end
end end
validates :source_project, presence: true, unless: :allow_broken validates :source_project, presence: true, unless: [:allow_broken, :importing?]
validates :source_branch, presence: true validates :source_branch, presence: true
validates :target_project, presence: true validates :target_project, presence: true
validates :target_branch, presence: true validates :target_branch, presence: true
validates :merge_user, presence: true, if: :merge_when_build_succeeds? validates :merge_user, presence: true, if: :merge_when_build_succeeds?
validate :validate_branches, unless: :allow_broken validate :validate_branches, unless: [:allow_broken, :importing?]
validate :validate_fork validate :validate_fork
scope :by_branch, ->(branch_name) { where("(source_branch LIKE :branch) OR (target_branch LIKE :branch)", branch: branch_name) } scope :by_branch, ->(branch_name) { where("(source_branch LIKE :branch) OR (target_branch LIKE :branch)", branch: branch_name) }
......
class MergeRequestDiff < ActiveRecord::Base class MergeRequestDiff < ActiveRecord::Base
include Sortable include Sortable
include Importable
# Prevent store of diff if commits amount more then 500 # Prevent store of diff if commits amount more then 500
COMMITS_SAFE_SIZE = 100 COMMITS_SAFE_SIZE = 100
...@@ -22,7 +23,7 @@ class MergeRequestDiff < ActiveRecord::Base ...@@ -22,7 +23,7 @@ class MergeRequestDiff < ActiveRecord::Base
serialize :st_commits serialize :st_commits
serialize :st_diffs serialize :st_diffs
after_create :reload_content after_create :reload_content, unless: :importing?
def reload_content def reload_content
reload_commits reload_commits
......
...@@ -4,6 +4,7 @@ class Note < ActiveRecord::Base ...@@ -4,6 +4,7 @@ class Note < ActiveRecord::Base
include Participable include Participable
include Mentionable include Mentionable
include Awardable include Awardable
include Importable
default_value_for :system, false default_value_for :system, false
...@@ -28,11 +29,11 @@ class Note < ActiveRecord::Base ...@@ -28,11 +29,11 @@ class Note < ActiveRecord::Base
validates :attachment, file_size: { maximum: :max_attachment_size } validates :attachment, file_size: { maximum: :max_attachment_size }
validates :noteable_type, presence: true validates :noteable_type, presence: true
validates :noteable_id, presence: true, unless: :for_commit? validates :noteable_id, presence: true, unless: [:for_commit?, :importing?]
validates :commit_id, presence: true, if: :for_commit? validates :commit_id, presence: true, if: :for_commit?
validates :author, presence: true validates :author, presence: true
validate unless: :for_commit? do |note| validate unless: [:for_commit?, :importing?] do |note|
unless note.noteable.try(:project) == note.project unless note.noteable.try(:project) == note.project
errors.add(:invalid_project, 'Note and noteable project mismatch') errors.add(:invalid_project, 'Note and noteable project mismatch')
end end
......
...@@ -367,6 +367,11 @@ class Project < ActiveRecord::Base ...@@ -367,6 +367,11 @@ class Project < ActiveRecord::Base
joins(join_body).reorder('join_note_counts.amount DESC') joins(join_body).reorder('join_note_counts.amount DESC')
end end
# Deletes gitlab project export files older than 24 hours
def remove_gitlab_exports!
Gitlab::Popen.popen(%W(find #{Gitlab::ImportExport.storage_path} -not -path #{Gitlab::ImportExport.storage_path} -mmin +1440 -delete))
end
end end
def team def team
...@@ -470,7 +475,7 @@ class Project < ActiveRecord::Base ...@@ -470,7 +475,7 @@ class Project < ActiveRecord::Base
end end
def import? def import?
external_import? || forked? external_import? || forked? || gitlab_project_import?
end end
def no_import? def no_import?
...@@ -501,6 +506,10 @@ class Project < ActiveRecord::Base ...@@ -501,6 +506,10 @@ class Project < ActiveRecord::Base
Gitlab::UrlSanitizer.new(import_url).masked_url Gitlab::UrlSanitizer.new(import_url).masked_url
end end
def gitlab_project_import?
import_type == 'gitlab_project'
end
def check_limit def check_limit
unless creator.can_create_project? or namespace.kind == 'group' unless creator.can_create_project? or namespace.kind == 'group'
projects_limit = creator.projects_limit projects_limit = creator.projects_limit
...@@ -1096,4 +1105,27 @@ class Project < ActiveRecord::Base ...@@ -1096,4 +1105,27 @@ class Project < ActiveRecord::Base
ensure ensure
@errors = original_errors @errors = original_errors
end end
def add_export_job(current_user:)
job_id = ProjectExportWorker.perform_async(current_user.id, self.id)
if job_id
Rails.logger.info "Export job started for project ID #{self.id} with job ID #{job_id}"
else
Rails.logger.error "Export job failed to start for project ID #{self.id}"
end
end
def export_path
File.join(Gitlab::ImportExport.storage_path, path_with_namespace)
end
def export_project_path
Dir.glob("#{export_path}/*export.tar.gz").max_by { |f| File.ctime(f) }
end
def remove_exports
_, status = Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete))
status.zero?
end
end end
...@@ -266,6 +266,14 @@ class NotificationService ...@@ -266,6 +266,14 @@ class NotificationService
end end
end end
def project_exported(project, current_user)
mailer.project_was_exported_email(current_user, project).deliver_later
end
def project_not_exported(project, current_user, errors)
mailer.project_was_not_exported_email(current_user, project, errors).deliver_later
end
protected protected
# Get project users with WATCH notification level # Get project users with WATCH notification level
......
...@@ -80,16 +80,18 @@ module Projects ...@@ -80,16 +80,18 @@ module Projects
def after_create_actions def after_create_actions
log_info("#{@project.owner.name} created a new project \"#{@project.name_with_namespace}\"") log_info("#{@project.owner.name} created a new project \"#{@project.name_with_namespace}\"")
@project.create_wiki if @project.wiki_enabled? unless @project.gitlab_project_import?
@project.create_wiki if @project.wiki_enabled?
@project.build_missing_services @project.build_missing_services
@project.create_labels @project.create_labels
end
event_service.create_project(@project, current_user) event_service.create_project(@project, current_user)
system_hook_service.execute_hooks_for(@project, :create) system_hook_service.execute_hooks_for(@project, :create)
unless @project.group unless @project.group || @project.gitlab_project_import?
@project.team << [current_user, :master, current_user] @project.team << [current_user, :master, current_user]
end end
end end
......
module Projects
module ImportExport
class ExportService < BaseService
def execute(_options = {})
@shared = Gitlab::ImportExport::Shared.new(relative_path: File.join(project.path_with_namespace, 'work'))
save_all
end
private
def save_all
if [version_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver].all?(&:save)
Gitlab::ImportExport::Saver.save(shared: @shared)
notify_success
else
cleanup_and_notify
end
end
def version_saver
Gitlab::ImportExport::VersionSaver.new(shared: @shared)
end
def project_tree_saver
Gitlab::ImportExport::ProjectTreeSaver.new(project: project, shared: @shared)
end
def uploads_saver
Gitlab::ImportExport::UploadsSaver.new(project: project, shared: @shared)
end
def repo_saver
Gitlab::ImportExport::RepoSaver.new(project: project, shared: @shared)
end
def wiki_repo_saver
Gitlab::ImportExport::WikiRepoSaver.new(project: project, shared: @shared)
end
def cleanup_and_notify
FileUtils.rm_rf(@shared.export_path)
notify_error
raise Gitlab::ImportExport::Error.new(@shared.errors.join(', '))
end
def notify_success
notification_service.project_exported(@project, @current_user)
end
def notify_error
notification_service.project_not_exported(@project, @current_user, @shared.errors.join(', '))
end
end
end
end
...@@ -9,26 +9,31 @@ module Projects ...@@ -9,26 +9,31 @@ module Projects
'fogbugz', 'fogbugz',
'gitlab', 'gitlab',
'github', 'github',
'google_code' 'google_code',
'gitlab_project'
] ]
def execute def execute
if unknown_url? add_repository_to_project unless project.gitlab_project_import?
# In this case, we only want to import issues, not a repository.
create_repository
else
import_repository
end
import_data import_data
success success
rescue Error => e rescue => e
error(e.message) error(e.message)
end end
private private
def add_repository_to_project
if unknown_url?
# In this case, we only want to import issues, not a repository.
create_repository
else
import_repository
end
end
def create_repository def create_repository
unless project.create_repository unless project.create_repository
raise Error, 'The repository could not be created.' raise Error, 'The repository could not be created.'
...@@ -46,7 +51,7 @@ module Projects ...@@ -46,7 +51,7 @@ module Projects
def import_data def import_data
return unless has_importer? return unless has_importer?
project.repository.before_import project.repository.before_import unless project.gitlab_project_import?
unless importer.execute unless importer.execute
raise Error, 'The remote data could not be imported.' raise Error, 'The remote data could not be imported.'
...@@ -58,6 +63,8 @@ module Projects ...@@ -58,6 +63,8 @@ module Projects
end end
def importer def importer
return Gitlab::ImportExport::Importer.new(project) if @project.gitlab_project_import?
class_name = "Gitlab::#{project.import_type.camelize}Import::Importer" class_name = "Gitlab::#{project.import_type.camelize}Import::Importer"
class_name.constantize.new(project) class_name.constantize.new(project)
end end
......
- page_title "GitLab Import"
- header_title "Projects", root_path
%h3.page-title
= icon('gitlab')
Import an exported GitLab project
%hr
= form_tag import_gitlab_project_path, class: 'form-horizontal', multipart: true do
%p
Project will be imported as
%strong
#{@namespace_name}/#{@path}
%p
To move or copy an entire GitLab project from another GitLab installation to this one, navigate to the original project's settings page, generate an export file, and upload it here.
.form-group
= hidden_field_tag :namespace_id, @namespace_id
= hidden_field_tag :path, @path
= label_tag :file, class: 'control-label' do
%span GitLab project export
.col-sm-10
= file_field_tag :file, class: ''
.form-actions
= submit_tag 'Import project', class: 'btn btn-create'
%p
Project #{@project.name} was exported successfully.
%p
The project export can be downloaded from:
= link_to download_export_namespace_project_url(@project.namespace, @project) do
= @project.name_with_namespace + " export"
%p
The download link will expire in 24 hours.
Project <%= @project.name %> was exported successfully.
The project export can be downloaded from:
<%= download_export_namespace_project_url(@project.namespace, @project) %>
The download link will expire in 24 hours.
%p
Project #{@project.name} couldn't be exported.
%p
The errors we encountered were:
%ul
- @errors.each do |error|
%li
error
Project <%= @project.name %> couldn't be exported.
The errors we encountered were:
- @errors.each do |error|
<%= error %>
\ No newline at end of file
...@@ -120,6 +120,42 @@ ...@@ -120,6 +120,42 @@
= link_to 'Housekeeping', housekeeping_namespace_project_path(@project.namespace, @project), = link_to 'Housekeeping', housekeeping_namespace_project_path(@project.namespace, @project),
method: :post, class: "btn btn-save" method: :post, class: "btn btn-save"
%hr %hr
.row.prepend-top-default
.col-lg-3
%h4.prepend-top-0
Export project
%p.append-bottom-0
%p
Export this project with all its related data in order to move your project to a new GitLab instance. Once the export is finished, you can import the file from the "New Project" page.
%p
Once the exported file is ready, you will receive a notification email with a download link.
.col-lg-9
- if @project.export_project_path
= link_to 'Download export', download_export_namespace_project_path(@project.namespace, @project),
method: :get, class: "btn btn-default"
= link_to 'Generate new export', generate_new_export_namespace_project_path(@project.namespace, @project),
method: :post, class: "btn btn-default"
- else
= link_to 'Export project', export_namespace_project_path(@project.namespace, @project),
method: :post, class: "btn btn-default"
.bs-callout.bs-callout-info
%p.append-bottom-0
%p
The following items will be exported:
%ul
%li Project and wiki repositories
%li Project uploads
%li Project configuration including web hooks and services
%li Issues with comments, merge requests with diffs and comments, labels, milestones, snippets, and other project entities
%p
The following items will NOT be exported:
%ul
%li Build traces and artifacts
%li LFS objects
%hr
- if can? current_user, :archive_project, @project - if can? current_user, :archive_project, @project
.row.prepend-top-default .row.prepend-top-default
.col-lg-3 .col-lg-3
......
...@@ -7,7 +7,7 @@ ...@@ -7,7 +7,7 @@
Forking in progress. Forking in progress.
- else - else
Import in progress. Import in progress.
- unless @project.forked? - if @project.external_import?
%p.monospace git clone --bare #{@project.safe_import_url} %p.monospace git clone --bare #{@project.safe_import_url}
%p Please wait while we import the repository for you. Refresh at will. %p Please wait while we import the repository for you. Refresh at will.
:javascript :javascript
......
...@@ -84,7 +84,12 @@ ...@@ -84,7 +84,12 @@
- if git_import_enabled? - if git_import_enabled?
= link_to "#", class: 'btn js-toggle-button import_git' do = link_to "#", class: 'btn js-toggle-button import_git' do
%i.fa.fa-git %i.fa.fa-git
%span Any repo by URL %span Repo by URL
- if gitlab_project_import_enabled?
= link_to new_import_gitlab_project_path, class: 'btn import_gitlab_project project-submit' do
%i.fa.fa-gitlab
%span GitLab export
.js-toggle-content.hide .js-toggle-content.hide
= render "shared/import_form", f: f = render "shared/import_form", f: f
...@@ -115,6 +120,33 @@ ...@@ -115,6 +120,33 @@
e.preventDefault(); e.preventDefault();
var import_modal = $(this).next(".modal").show(); var import_modal = $(this).next(".modal").show();
}); });
$('.modal-header .close').bind('click', function() { $('.modal-header .close').bind('click', function() {
$(".modal").hide(); $(".modal").hide();
}); });
$('.import_gitlab_project').bind('click', function() {
var _href = $("a.import_gitlab_project").attr("href");
$(".import_gitlab_project").attr("href", _href + '?namespace_id=' + $("#project_namespace_id").val() + '&path=' + $("#project_path").val());
});
$('.import_gitlab_project').attr('disabled',true)
$('.import_gitlab_project').attr('title', 'Project path required.');
$('.import_gitlab_project').click(function( event ) {
if($('.import_gitlab_project').attr('disabled')) {
event.preventDefault();
new Flash("Please enter a path for the project to be imported to.");
}
});
$('#project_path').keyup(function(){
if($(this).val().length !=0) {
$('.import_gitlab_project').attr('disabled', false);
$('.import_gitlab_project').attr('title','');
$(".flash-container").html("")
} else {
$('.import_gitlab_project').attr('disabled',true);
$('.import_gitlab_project').attr('title', 'Project path required.');
}
})
class GitlabRemoveProjectExportWorker
include Sidekiq::Worker
sidekiq_options queue: :default
def perform
Project.remove_gitlab_exports!
end
end
class ProjectExportWorker
include Sidekiq::Worker
sidekiq_options queue: :gitlab_shell, retry: true
def perform(current_user_id, project_id)
current_user = User.find(current_user_id)
project = Project.find(project_id)
::Projects::ImportExport::ExportService.new(project, current_user).execute
end
end
...@@ -291,6 +291,9 @@ Settings.cron_jobs['admin_email_worker']['job_class'] = 'AdminEmailWorker' ...@@ -291,6 +291,9 @@ Settings.cron_jobs['admin_email_worker']['job_class'] = 'AdminEmailWorker'
Settings.cron_jobs['repository_archive_cache_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['repository_archive_cache_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['repository_archive_cache_worker']['cron'] ||= '0 * * * *' Settings.cron_jobs['repository_archive_cache_worker']['cron'] ||= '0 * * * *'
Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'RepositoryArchiveCacheWorker' Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'RepositoryArchiveCacheWorker'
Settings.cron_jobs['gitlab_remove_project_export_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['gitlab_remove_project_export_worker']['cron'] ||= '0 * * * *'
Settings.cron_jobs['gitlab_remove_project_export_worker']['job_class'] = 'GitlabRemoveProjectExportWorker'
# #
# GitLab Shell # GitLab Shell
......
...@@ -171,6 +171,10 @@ Rails.application.routes.draw do ...@@ -171,6 +171,10 @@ Rails.application.routes.draw do
get :new_user_map, path: :user_map get :new_user_map, path: :user_map
post :create_user_map, path: :user_map post :create_user_map, path: :user_map
end end
resource :gitlab_project, only: [:create, :new] do
post :create
end
end end
# #
...@@ -462,6 +466,10 @@ Rails.application.routes.draw do ...@@ -462,6 +466,10 @@ Rails.application.routes.draw do
post :housekeeping post :housekeeping
post :toggle_star post :toggle_star
post :markdown_preview post :markdown_preview
post :export
post :remove_export
post :generate_new_export
get :download_export
get :autocomplete_sources get :autocomplete_sources
get :activity get :activity
end end
......
...@@ -14,7 +14,7 @@ Background: ...@@ -14,7 +14,7 @@ Background:
@javascript @javascript
Scenario: I should see instructions on how to import from Git URL Scenario: I should see instructions on how to import from Git URL
Given I see "New Project" page Given I see "New Project" page
When I click on "Any repo by URL" When I click on "Repo by URL"
Then I see instructions on how to import from Git URL Then I see instructions on how to import from Git URL
@javascript @javascript
......
...@@ -20,7 +20,8 @@ class Spinach::Features::NewProject < Spinach::FeatureSteps ...@@ -20,7 +20,8 @@ class Spinach::Features::NewProject < Spinach::FeatureSteps
expect(page).to have_link('GitLab.com') expect(page).to have_link('GitLab.com')
expect(page).to have_link('Gitorious.org') expect(page).to have_link('Gitorious.org')
expect(page).to have_link('Google Code') expect(page).to have_link('Google Code')
expect(page).to have_link('Any repo by URL') expect(page).to have_link('Repo by URL')
expect(page).to have_link('GitLab export')
end end
step 'I click on "Import project from GitHub"' do step 'I click on "Import project from GitHub"' do
...@@ -37,7 +38,7 @@ class Spinach::Features::NewProject < Spinach::FeatureSteps ...@@ -37,7 +38,7 @@ class Spinach::Features::NewProject < Spinach::FeatureSteps
end end
end end
step 'I click on "Any repo by URL"' do step 'I click on "Repo by URL"' do
first('.import_git').click first('.import_git').click
end end
......
...@@ -36,7 +36,7 @@ module Gitlab ...@@ -36,7 +36,7 @@ module Gitlab
default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'],
default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'],
restricted_signup_domains: Settings.gitlab['restricted_signup_domains'], restricted_signup_domains: Settings.gitlab['restricted_signup_domains'],
import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'], import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project],
shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
max_artifacts_size: Settings.artifacts['max_size'], max_artifacts_size: Settings.artifacts['max_size'],
require_two_factor_authentication: false, require_two_factor_authentication: false,
......
...@@ -11,7 +11,7 @@ module Gitlab ...@@ -11,7 +11,7 @@ module Gitlab
end end
def execute def execute
project = ::Projects::CreateService.new( ::Projects::CreateService.new(
current_user, current_user,
name: repo["name"], name: repo["name"],
path: repo["path"], path: repo["path"],
...@@ -22,8 +22,6 @@ module Gitlab ...@@ -22,8 +22,6 @@ module Gitlab
import_source: repo["path_with_namespace"], import_source: repo["path_with_namespace"],
import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@") import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@")
).execute ).execute
project
end end
end end
end end
......
module Gitlab
module ImportExport
extend self
VERSION = '0.1.0'
def export_path(relative_path:)
File.join(storage_path, relative_path)
end
def storage_path
File.join(Settings.shared['path'], 'tmp/project_exports')
end
def project_filename
"project.json"
end
def project_bundle_filename
"project.bundle"
end
def config_file
'lib/gitlab/import_export/import_export.yml'
end
def version_filename
'VERSION'
end
def version
VERSION
end
def reset_tokens?
true
end
end
end
module Gitlab
module ImportExport
class AttributesFinder
def initialize(included_attributes:, excluded_attributes:, methods:)
@included_attributes = included_attributes || {}
@excluded_attributes = excluded_attributes || {}
@methods = methods || {}
end
def find(model_object)
parsed_hash = find_attributes_only(model_object)
parsed_hash.empty? ? model_object : { model_object => parsed_hash }
end
def parse(model_object)
parsed_hash = find_attributes_only(model_object)
yield parsed_hash unless parsed_hash.empty?
end
def find_included(value)
key = key_from_hash(value)
@included_attributes[key].nil? ? {} : { only: @included_attributes[key] }
end
def find_excluded(value)
key = key_from_hash(value)
@excluded_attributes[key].nil? ? {} : { except: @excluded_attributes[key] }
end
def find_method(value)
key = key_from_hash(value)
@methods[key].nil? ? {} : { methods: @methods[key] }
end
private
def find_attributes_only(value)
find_included(value).merge(find_excluded(value)).merge(find_method(value))
end
def key_from_hash(value)
value.is_a?(Hash) ? value.keys.first : value
end
end
end
end
module Gitlab
module ImportExport
module CommandLineUtil
def tar_czf(archive:, dir:)
tar_with_options(archive: archive, dir: dir, options: 'czf')
end
def untar_zxf(archive:, dir:)
untar_with_options(archive: archive, dir: dir, options: 'zxf')
end
def git_bundle(repo_path:, bundle_path:)
execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all))
end
def git_unbundle(repo_path:, bundle_path:)
execute(%W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path}))
end
private
def tar_with_options(archive:, dir:, options:)
execute(%W(tar -#{options} #{archive} -C #{dir} .))
end
def untar_with_options(archive:, dir:, options:)
execute(%W(tar -#{options} #{archive} -C #{dir}))
end
def execute(cmd)
_output, status = Gitlab::Popen.popen(cmd)
status.zero?
end
def git_bin_path
Gitlab.config.git.bin_path
end
end
end
end
module Gitlab
module ImportExport
class Error < StandardError; end
end
end
module Gitlab
module ImportExport
class FileImporter
include Gitlab::ImportExport::CommandLineUtil
def self.import(*args)
new(*args).import
end
def initialize(archive_file:, shared:)
@archive_file = archive_file
@shared = shared
end
def import
FileUtils.mkdir_p(@shared.export_path)
decompress_archive
rescue => e
@shared.error(e)
false
end
private
def decompress_archive
untar_zxf(archive: @archive_file, dir: @shared.export_path)
end
end
end
end
# Model relationships to be included in the project import/export
project_tree:
- issues:
- notes:
:author
- :labels
- :milestones
- snippets:
- notes:
:author
- :releases
- :events
- project_members:
- :user
- merge_requests:
- notes:
:author
- :merge_request_diff
- pipelines:
- notes:
:author
- :statuses
- :variables
- :triggers
- :deploy_keys
- :services
- :hooks
- :protected_branches
# Only include the following attributes for the models specified.
included_attributes:
project:
- :description
- :issues_enabled
- :merge_requests_enabled
- :wiki_enabled
- :snippets_enabled
- :visibility_level
- :archived
user:
- :id
- :email
- :username
author:
- :name
# Do not include the following attributes for the models specified.
excluded_attributes:
snippets:
- :expired_at
methods:
statuses:
- :type
\ No newline at end of file
module Gitlab
module ImportExport
class Importer
def initialize(project)
@archive_file = project.import_source
@current_user = project.creator
@project = project
@shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace)
end
def execute
Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file,
shared: @shared)
if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore)
project_tree.restored_project
else
raise Projects::ImportService::Error.new(@shared.errors.join(', '))
end
end
private
def check_version!
Gitlab::ImportExport::VersionChecker.check!(shared: @shared)
end
def project_tree
@project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: @current_user,
shared: @shared,
project: @project)
end
def repo_restorer
Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path,
shared: @shared,
project: project_tree.restored_project)
end
def wiki_restorer
Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path,
shared: @shared,
project: ProjectWiki.new(project_tree.restored_project),
wiki: true)
end
def uploads_restorer
Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared)
end
def path_with_namespace
File.join(@project.namespace.path, @project.path)
end
def repo_path
File.join(@shared.export_path, 'project.bundle')
end
def wiki_repo_path
File.join(@shared.export_path, 'project.wiki.bundle')
end
end
end
end
module Gitlab
module ImportExport
class MembersMapper
attr_reader :missing_author_ids
def initialize(exported_members:, user:, project:)
@exported_members = exported_members
@user = user
@project = project
@missing_author_ids = []
# This needs to run first, as second call would be from #map
# which means project members already exist.
ensure_default_member!
end
def map
@map ||=
begin
@exported_members.inject(missing_keys_tracking_hash) do |hash, member|
existing_user = User.where(find_project_user_query(member)).first
old_user_id = member['user']['id']
if existing_user && add_user_as_team_member(existing_user, member)
hash[old_user_id] = existing_user.id
end
hash
end
end
end
def default_user_id
@user.id
end
private
def missing_keys_tracking_hash
Hash.new do |_, key|
@missing_author_ids << key
default_user_id
end
end
def ensure_default_member!
ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true)
end
def add_user_as_team_member(existing_user, member)
member['user'] = existing_user
ProjectMember.create(member_hash(member)).persisted?
end
def member_hash(member)
member.except('id').merge(source_id: @project.id, importing: true)
end
def find_project_user_query(member)
user_arel[:username].eq(member['user']['username']).or(user_arel[:email].eq(member['user']['email']))
end
def user_arel
@user_arel ||= User.arel_table
end
end
end
end
module Gitlab
module ImportExport
class ProjectCreator
def initialize(namespace_id, current_user, file, project_path)
@namespace_id = namespace_id
@current_user = current_user
@file = file
@project_path = project_path
end
def execute
::Projects::CreateService.new(
@current_user,
name: @project_path,
path: @project_path,
namespace_id: @namespace_id,
import_type: "gitlab_project",
import_source: @file
).execute
end
end
end
end
module Gitlab
module ImportExport
class ProjectTreeRestorer
def initialize(user:, shared:, project:)
@path = File.join(shared.export_path, 'project.json')
@user = user
@shared = shared
@project = project
end
def restore
json = IO.read(@path)
@tree_hash = ActiveSupport::JSON.decode(json)
@project_members = @tree_hash.delete('project_members')
create_relations
rescue => e
@shared.error(e)
false
end
def restored_project
@restored_project ||= restore_project
end
private
def members_mapper
@members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
user: @user,
project: restored_project)
end
# Loops through the tree of models defined in import_export.yml and
# finds them in the imported JSON so they can be instantiated and saved
# in the DB. The structure and relationships between models are guessed from
# the configuration yaml file too.
# Finally, it updates each attribute in the newly imported project.
def create_relations
saved = []
default_relation_list.each do |relation|
next unless relation.is_a?(Hash) || @tree_hash[relation.to_s].present?
create_sub_relations(relation, @tree_hash) if relation.is_a?(Hash)
relation_key = relation.is_a?(Hash) ? relation.keys.first : relation
relation_hash = create_relation(relation_key, @tree_hash[relation_key.to_s])
saved << restored_project.update_attribute(relation_key, relation_hash)
end
saved.all?
end
def default_relation_list
Gitlab::ImportExport::Reader.new(shared: @shared).tree.reject do |model|
model.is_a?(Hash) && model[:project_members]
end
end
def restore_project
return @project unless @tree_hash
project_params = @tree_hash.reject { |_key, value| value.is_a?(Array) }
@project.update(project_params)
@project
end
# Given a relation hash containing one or more models and its relationships,
# loops through each model and each object from a model type and
# and assigns its correspondent attributes hash from +tree_hash+
# Example:
# +relation_key+ issues, loops through the list of *issues* and for each individual
# issue, finds any subrelations such as notes, creates them and assign them back to the hash
def create_sub_relations(relation, tree_hash)
relation_key = relation.keys.first.to_s
tree_hash[relation_key].each do |relation_item|
relation.values.flatten.each do |sub_relation|
relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation)
relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank?
end
end
end
def assign_relation_hash(relation_item, sub_relation)
if sub_relation.is_a?(Hash)
relation_hash = relation_item[sub_relation.keys.first.to_s]
sub_relation = sub_relation.keys.first
else
relation_hash = relation_item[sub_relation.to_s]
end
[relation_hash, sub_relation]
end
def create_relation(relation, relation_hash_list)
relation_array = [relation_hash_list].flatten.map do |relation_hash|
Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym,
relation_hash: relation_hash.merge('project_id' => restored_project.id),
members_mapper: members_mapper,
user: @user)
end
relation_hash_list.is_a?(Array) ? relation_array : relation_array.first
end
end
end
end
module Gitlab
module ImportExport
class ProjectTreeSaver
attr_reader :full_path
def initialize(project:, shared:)
@project = project
@shared = shared
@full_path = File.join(@shared.export_path, ImportExport.project_filename)
end
def save
FileUtils.mkdir_p(@shared.export_path)
File.write(full_path, project_json_tree)
true
rescue => e
@shared.error(e)
false
end
private
def project_json_tree
@project.to_json(Gitlab::ImportExport::Reader.new(shared: @shared).project_tree)
end
end
end
end
module Gitlab
module ImportExport
class Reader
attr_reader :tree
def initialize(shared:)
@shared = shared
config_hash = YAML.load_file(Gitlab::ImportExport.config_file).deep_symbolize_keys
@tree = config_hash[:project_tree]
@attributes_finder = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes],
excluded_attributes: config_hash[:excluded_attributes],
methods: config_hash[:methods])
end
# Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html
# for outputting a project in JSON format, including its relations and sub relations.
def project_tree
@attributes_finder.find_included(:project).merge(include: build_hash(@tree))
rescue => e
@shared.error(e)
false
end
private
# Builds a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html
#
# +model_list+ - List of models as a relation tree to be included in the generated JSON, from the _import_export.yml_ file
def build_hash(model_list)
model_list.map do |model_objects|
if model_objects.is_a?(Hash)
build_json_config_hash(model_objects)
else
@attributes_finder.find(model_objects)
end
end
end
# Called when the model is actually a hash containing other relations (more models)
# Returns the config in the right format for calling +to_json+
# +model_object_hash+ - A model relationship such as:
# {:merge_requests=>[:merge_request_diff, :notes]}
def build_json_config_hash(model_object_hash)
@json_config_hash = {}
model_object_hash.values.flatten.each do |model_object|
current_key = model_object_hash.keys.first
@attributes_finder.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash }
handle_model_object(current_key, model_object)
process_sub_model(current_key, model_object) if model_object.is_a?(Hash)
end
@json_config_hash
end
# If the model is a hash, process the sub_models, which could also be hashes
# If there is a list, add to an existing array, otherwise use hash syntax
# +current_key+ main model that will be a key in the hash
# +model_object+ model or list of models to include in the hash
def process_sub_model(current_key, model_object)
sub_model_json = build_json_config_hash(model_object).dup
@json_config_hash.slice!(current_key)
if @json_config_hash[current_key] && @json_config_hash[current_key][:include]
@json_config_hash[current_key][:include] << sub_model_json
else
@json_config_hash[current_key] = { include: sub_model_json }
end
end
# Creates or adds to an existing hash an individual model or list
# +current_key+ main model that will be a key in the hash
# +model_object+ model or list of models to include in the hash
def handle_model_object(current_key, model_object)
if @json_config_hash[current_key]
add_model_value(current_key, model_object)
else
create_model_value(current_key, model_object)
end
end
# Constructs a new hash that will hold the configuration for that particular object
# It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+
# +current_key+ main model that will be a key in the hash
# +value+ existing model to be included in the hash
def create_model_value(current_key, value)
parsed_hash = { include: value }
@attributes_finder.parse(value) do |hash|
parsed_hash = { include: hash_or_merge(value, hash) }
end
@json_config_hash[current_key] = parsed_hash
end
# Adds new model configuration to an existing hash with key +current_key+
# It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+
# +current_key+ main model that will be a key in the hash
# +value+ existing model to be included in the hash
def add_model_value(current_key, value)
@attributes_finder.parse(value) { |hash| value = { value => hash } }
old_values = @json_config_hash[current_key][:include]
@json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten
end
# Construct a new hash or merge with an existing one a model configuration
# This is to fulfil +to_json+ requirements.
# +value+ existing model to be included in the hash
# +hash+ hash containing configuration generated mainly from +@attributes_finder+
def hash_or_merge(value, hash)
value.is_a?(Hash) ? value.merge(hash) : { value => hash }
end
end
end
end
module Gitlab
module ImportExport
class RelationFactory
OVERRIDES = { snippets: :project_snippets,
pipelines: 'Ci::Pipeline',
statuses: 'commit_status',
variables: 'Ci::Variable',
triggers: 'Ci::Trigger',
builds: 'Ci::Build',
hooks: 'ProjectHook' }.freeze
USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze
def self.create(*args)
new(*args).create
end
def initialize(relation_sym:, relation_hash:, members_mapper:, user:)
@relation_name = OVERRIDES[relation_sym] || relation_sym
@relation_hash = relation_hash.except('id', 'noteable_id')
@members_mapper = members_mapper
@user = user
end
# Creates an object from an actual model with name "relation_sym" with params from
# the relation_hash, updating references with new object IDs, mapping users using
# the "members_mapper" object, also updating notes if required.
def create
set_note_author if @relation_name == :notes
update_user_references
update_project_references
reset_ci_tokens if @relation_name == 'Ci::Trigger'
generate_imported_object
end
private
def update_user_references
USER_REFERENCES.each do |reference|
if @relation_hash[reference]
@relation_hash[reference] = @members_mapper.map[@relation_hash[reference]]
end
end
end
# Sets the author for a note. If the user importing the project
# has admin access, an actual mapping with new project members
# will be used. Otherwise, a note stating the original author name
# is left.
def set_note_author
old_author_id = @relation_hash['author_id']
# Users with admin access can map users
@relation_hash['author_id'] = admin_user? ? @members_mapper.map[old_author_id] : @members_mapper.default_user_id
author = @relation_hash.delete('author')
update_note_for_missing_author(author['name']) if missing_author?(old_author_id)
end
def missing_author?(old_author_id)
!admin_user? || @members_mapper.missing_author_ids.include?(old_author_id)
end
def missing_author_note(updated_at, author_name)
timestamp = updated_at.split('.').first
"\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*"
end
def generate_imported_object
if @relation_sym == 'commit_status' # call #trace= method after assigning the other attributes
trace = @relation_hash.delete('trace')
imported_object do |object|
object.trace = trace
object.commit_id = nil
end
else
imported_object
end
end
def update_project_references
project_id = @relation_hash.delete('project_id')
# project_id may not be part of the export, but we always need to populate it if required.
@relation_hash['project_id'] = project_id if relation_class.column_names.include?('project_id')
@relation_hash['gl_project_id'] = project_id if @relation_hash['gl_project_id']
@relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id']
@relation_hash['source_project_id'] = -1 if @relation_hash['source_project_id']
# If source and target are the same, populate them with the new project ID.
if @relation_hash['source_project_id'] && @relation_hash['target_project_id'] &&
@relation_hash['target_project_id'] == @relation_hash['source_project_id']
@relation_hash['source_project_id'] = project_id
end
end
def reset_ci_tokens
return unless Gitlab::ImportExport.reset_tokens?
# If we import/export a project to the same instance, tokens will have to be reset.
@relation_hash['token'] = nil
end
def relation_class
@relation_class ||= @relation_name.to_s.classify.constantize
end
def imported_object
imported_object = relation_class.new(@relation_hash)
yield(imported_object) if block_given?
imported_object.importing = true if imported_object.respond_to?(:importing)
imported_object
end
def update_note_for_missing_author(author_name)
@relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank?
@relation_hash['note'] += missing_author_note(@relation_hash['updated_at'], author_name)
end
def admin_user?
@user.is_admin?
end
end
end
end
module Gitlab
module ImportExport
class RepoRestorer
include Gitlab::ImportExport::CommandLineUtil
def initialize(project:, shared:, path_to_bundle:, wiki: false)
@project = project
@path_to_bundle = path_to_bundle
@shared = shared
@wiki = wiki
end
def restore
return wiki? unless File.exist?(@path_to_bundle)
FileUtils.mkdir_p(path_to_repo)
git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle)
rescue => e
@shared.error(e)
false
end
private
def repos_path
Gitlab.config.gitlab_shell.repos_path
end
def path_to_repo
@project.repository.path_to_repo
end
def wiki?
@wiki
end
end
end
end
module Gitlab
module ImportExport
class RepoSaver
include Gitlab::ImportExport::CommandLineUtil
attr_reader :full_path
def initialize(project:, shared:)
@project = project
@shared = shared
end
def save
return false if @project.empty_repo?
@full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename)
bundle_to_disk
end
private
def bundle_to_disk
FileUtils.mkdir_p(@shared.export_path)
git_bundle(repo_path: path_to_repo, bundle_path: @full_path)
rescue => e
@shared.error(e)
false
end
def path_to_repo
@project.repository.path_to_repo
end
end
end
end
module Gitlab
module ImportExport
class Saver
include Gitlab::ImportExport::CommandLineUtil
def self.save(*args)
new(*args).save
end
def initialize(shared:)
@shared = shared
end
def save
if compress_and_save
remove_export_path
Rails.logger.info("Saved project export #{archive_file}")
archive_file
else
false
end
rescue => e
@shared.error(e)
false
end
private
def compress_and_save
tar_czf(archive: archive_file, dir: @shared.export_path)
end
def remove_export_path
FileUtils.rm_rf(@shared.export_path)
end
def archive_file
@archive_file ||= File.join(@shared.export_path, '..', "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_project_export.tar.gz")
end
end
end
end
module Gitlab
module ImportExport
class Shared
attr_reader :errors, :opts
def initialize(opts)
@opts = opts
@errors = []
end
def export_path
@export_path ||= Gitlab::ImportExport.export_path(relative_path: opts[:relative_path])
end
def error(error)
error_out(error.message, caller[0].dup)
@errors << error.message
# Debug:
Rails.logger.error(error.backtrace)
end
private
def error_out(message, caller)
Rails.logger.error("Import/Export error raised on #{caller}: #{message}")
end
end
end
end
module Gitlab
module ImportExport
class UploadsRestorer < UploadsSaver
def restore
return true unless File.directory?(uploads_export_path)
copy_files(uploads_export_path, uploads_path)
rescue => e
@shared.error(e)
false
end
end
end
end
module Gitlab
module ImportExport
class UploadsSaver
def initialize(project:, shared:)
@project = project
@shared = shared
end
def save
return true unless File.directory?(uploads_path)
copy_files(uploads_path, uploads_export_path)
rescue => e
@shared.error(e)
false
end
private
def copy_files(source, destination)
FileUtils.mkdir_p(destination)
FileUtils.copy_entry(source, destination)
true
end
def uploads_export_path
File.join(@shared.export_path, 'uploads')
end
def uploads_path
File.join(Rails.root.join('public/uploads'), @project.path_with_namespace)
end
end
end
end
module Gitlab
module ImportExport
class VersionChecker
def self.check!(*args)
new(*args).check!
end
def initialize(shared:)
@shared = shared
end
def check!
version = File.open(version_file, &:readline)
verify_version!(version)
rescue => e
@shared.error(e)
false
end
private
def version_file
File.join(@shared.export_path, Gitlab::ImportExport.version_filename)
end
def verify_version!(version)
if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.version)
raise Gitlab::ImportExport::Error("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}")
else
true
end
end
end
end
end
module Gitlab
module ImportExport
class VersionSaver
def initialize(shared:)
@shared = shared
end
def save
FileUtils.mkdir_p(@shared.export_path)
File.write(version_file, Gitlab::ImportExport.version, mode: 'w')
rescue => e
@shared.error(e)
false
end
private
def version_file
File.join(@shared.export_path, Gitlab::ImportExport.version_filename)
end
end
end
end
module Gitlab
module ImportExport
class WikiRepoSaver < RepoSaver
def save
@wiki = ProjectWiki.new(@project)
return true unless wiki_repository_exists? # it's okay to have no Wiki
bundle_to_disk(File.join(@shared.export_path, project_filename))
end
def bundle_to_disk(full_path)
FileUtils.mkdir_p(@shared.export_path)
git_bundle(repo_path: path_to_repo, bundle_path: full_path)
rescue => e
@shared.error(e)
false
end
private
def project_filename
"project.wiki.bundle"
end
def path_to_repo
@wiki.repository.path_to_repo
end
def wiki_repository_exists?
File.exist?(@wiki.repository.path_to_repo) && !@wiki.repository.empty?
end
end
end
end
...@@ -20,7 +20,8 @@ module Gitlab ...@@ -20,7 +20,8 @@ module Gitlab
'Gitorious.org' => 'gitorious', 'Gitorious.org' => 'gitorious',
'Google Code' => 'google_code', 'Google Code' => 'google_code',
'FogBugz' => 'fogbugz', 'FogBugz' => 'fogbugz',
'Any repo by URL' => 'git', 'Repo by URL' => 'git',
'GitLab export' => 'gitlab_project'
} }
end end
......
require 'spec_helper'
feature 'project import', feature: true, js: true do
include Select2Helper
let(:user) { create(:admin) }
let!(:namespace) { create(:namespace, name: "asd", owner: user) }
let(:file) { File.join(Rails.root, 'spec', 'features', 'projects', 'import_export', 'test_project_export.tar.gz') }
let(:export_path) { "#{Dir::tmpdir}/import_file_spec" }
let(:project) { Project.last }
background do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
login_as(user)
end
after(:each) do
FileUtils.rm_rf(export_path, secure: true)
end
scenario 'user imports an exported project successfully' do
expect(Project.all.count).to be_zero
visit new_project_path
select2('2', from: '#project_namespace_id')
fill_in :project_path, with:'test-project-path', visible: true
click_link 'GitLab export'
expect(page).to have_content('GitLab project export')
expect(URI.parse(current_url).query).to eq('namespace_id=2&path=test-project-path')
attach_file('file', file)
click_on 'Import project' # import starts
expect(project).not_to be_nil
expect(project.issues).not_to be_empty
expect(project.merge_requests).not_to be_empty
expect(project.repo_exists?).to be true
expect(wiki_exists?).to be true
expect(project.import_status).to eq('finished')
end
def wiki_exists?
wiki = ProjectWiki.new(project)
File.exist?(wiki.repository.path_to_repo) && !wiki.repository.empty?
end
end
require 'spec_helper'
describe Gitlab::ImportExport::MembersMapper, services: true do
describe 'map members' do
let(:user) { create(:user) }
let(:project) { create(:project, :public, name: 'searchable_project') }
let(:user2) { create(:user) }
let(:exported_user_id) { 99 }
let(:exported_members) do
[{
"id" => 2,
"access_level" => 40,
"source_id" => 14,
"source_type" => "Project",
"user_id" => 19,
"notification_level" => 3,
"created_at" => "2016-03-11T10:21:44.822Z",
"updated_at" => "2016-03-11T10:21:44.822Z",
"created_by_id" => nil,
"invite_email" => nil,
"invite_token" => nil,
"invite_accepted_at" => nil,
"user" =>
{
"id" => exported_user_id,
"email" => user2.email,
"username" => user2.username
}
}]
end
let(:members_mapper) do
described_class.new(
exported_members: exported_members, user: user, project: project)
end
it 'maps a project member' do
expect(members_mapper.map[exported_user_id]).to eq(user2.id)
end
it 'defaults to importer project member if it does not exist' do
expect(members_mapper.map[-1]).to eq(user.id)
end
it 'updates missing author IDs on missing project member' do
members_mapper.map[-1]
expect(members_mapper.missing_author_ids.first).to eq(-1)
end
end
end
This diff is collapsed.
require 'spec_helper'
describe Gitlab::ImportExport::ProjectTreeRestorer, services: true do
describe 'restore project tree' do
let(:user) { create(:user) }
let(:namespace) { create(:namespace, owner: user) }
let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: "", project_path: 'path') }
let(:project) { create(:empty_project, name: 'project', path: 'project') }
let(:project_tree_restorer) { described_class.new(user: user, shared: shared, project: project) }
let(:restored_project_json) { project_tree_restorer.restore }
before do
allow(shared).to receive(:export_path).and_return('spec/lib/gitlab/import_export/')
end
context 'JSON' do
it 'restores models based on JSON' do
expect(restored_project_json).to be true
end
end
end
end
require 'spec_helper'
describe Gitlab::ImportExport::ProjectTreeSaver, services: true do
describe 'saves the project tree into a json object' do
let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.path_with_namespace) }
let(:project_tree_saver) { described_class.new(project: project, shared: shared) }
let(:export_path) { "#{Dir::tmpdir}/project_tree_saver_spec" }
let(:user) { create(:user) }
let(:project) { setup_project }
before do
project.team << [user, :master]
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
end
after do
FileUtils.rm_rf(export_path)
end
it 'saves project successfully' do
expect(project_tree_saver.save).to be true
end
context 'JSON' do
let(:saved_project_json) do
project_tree_saver.save
project_json(project_tree_saver.full_path)
end
it 'saves the correct json' do
expect(saved_project_json).to include({ "visibility_level" => 20 })
end
it 'has events' do
expect(saved_project_json['events']).not_to be_empty
end
it 'has milestones' do
expect(saved_project_json['milestones']).not_to be_empty
end
it 'has merge requests' do
expect(saved_project_json['merge_requests']).not_to be_empty
end
it 'has labels' do
expect(saved_project_json['labels']).not_to be_empty
end
it 'has snippets' do
expect(saved_project_json['snippets']).not_to be_empty
end
it 'has snippet notes' do
expect(saved_project_json['snippets'].first['notes']).not_to be_empty
end
it 'has releases' do
expect(saved_project_json['releases']).not_to be_empty
end
it 'has issues' do
expect(saved_project_json['issues']).not_to be_empty
end
it 'has issue comments' do
expect(saved_project_json['issues'].first['notes']).not_to be_empty
end
it 'has author on issue comments' do
expect(saved_project_json['issues'].first['notes'].first['author']).not_to be_empty
end
it 'has project members' do
expect(saved_project_json['project_members']).not_to be_empty
end
it 'has merge requests diffs' do
expect(saved_project_json['merge_requests'].first['merge_request_diff']).not_to be_empty
end
it 'has merge requests comments' do
expect(saved_project_json['merge_requests'].first['notes']).not_to be_empty
end
it 'has author on merge requests comments' do
expect(saved_project_json['merge_requests'].first['notes'].first['author']).not_to be_empty
end
it 'has pipeline statuses' do
expect(saved_project_json['pipelines'].first['statuses']).not_to be_empty
end
it 'has pipeline builds' do
expect(saved_project_json['pipelines'].first['statuses'].count { |hash| hash['type'] == 'Ci::Build'}).to eq(1)
end
it 'has pipeline commits' do
expect(saved_project_json['pipelines']).not_to be_empty
end
it 'has ci pipeline notes' do
expect(saved_project_json['pipelines'].first['notes']).not_to be_empty
end
end
end
def setup_project
issue = create(:issue, assignee: user)
merge_request = create(:merge_request)
label = create(:label)
snippet = create(:project_snippet)
release = create(:release)
project = create(:project,
:public,
issues: [issue],
merge_requests: [merge_request],
labels: [label],
snippets: [snippet],
releases: [release]
)
commit_status = create(:commit_status, project: project)
ci_pipeline = create(:ci_pipeline,
project: project,
sha: merge_request.last_commit.id,
ref: merge_request.source_branch,
statuses: [commit_status])
create(:ci_build, pipeline: ci_pipeline, project: project)
create(:milestone, project: project)
create(:note, noteable: issue, project: project)
create(:note, noteable: merge_request, project: project)
create(:note, noteable: snippet, project: project)
create(:note_on_commit,
author: user,
project: project,
commit_id: ci_pipeline.sha)
project
end
def project_json(filename)
JSON.parse(IO.read(filename))
end
end
require 'spec_helper'
describe Gitlab::ImportExport::Reader, lib: true do
let(:shared) { Gitlab::ImportExport::Shared.new(relative_path:'') }
let(:test_config) { 'spec/support/import_export/import_export.yml' }
let(:project_tree_hash) do
{
only: [:name, :path],
include: [:issues, :labels,
{ merge_requests: {
only: [:id],
except: [:iid],
include: [:merge_request_diff, :merge_request_test]
} },
{ commit_statuses: { include: :commit } }]
}
end
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:config_file).and_return(test_config)
end
it 'generates hash from project tree config' do
expect(described_class.new(shared: shared).project_tree).to match(project_tree_hash)
end
context 'individual scenarios' do
it 'generates the correct hash for a single project relation' do
setup_yaml(project_tree: [:issues])
expect(described_class.new(shared: shared).project_tree).to match(include: [:issues])
end
it 'generates the correct hash for a multiple project relation' do
setup_yaml(project_tree: [:issues, :snippets])
expect(described_class.new(shared: shared).project_tree).to match(include: [:issues, :snippets])
end
it 'generates the correct hash for a single sub-relation' do
setup_yaml(project_tree: [issues: [:notes]])
expect(described_class.new(shared: shared).project_tree).to match(include: [{ issues: { include: :notes } }])
end
it 'generates the correct hash for a multiple sub-relation' do
setup_yaml(project_tree: [merge_requests: [:notes, :merge_request_diff]])
expect(described_class.new(shared: shared).project_tree).to match(include: [{ merge_requests: { include: [:notes, :merge_request_diff] } }])
end
it 'generates the correct hash for a sub-relation with another sub-relation' do
setup_yaml(project_tree: [merge_requests: [notes: :author]])
expect(described_class.new(shared: shared).project_tree).to match(include: [{ merge_requests: { include: { notes: { include: :author } } } }])
end
it 'generates the correct hash for a relation with included attributes' do
setup_yaml(project_tree: [:issues], included_attributes: { issues: [:name, :description] })
expect(described_class.new(shared: shared).project_tree).to match(include: [{ issues: { only: [:name, :description] } }])
end
it 'generates the correct hash for a relation with excluded attributes' do
setup_yaml(project_tree: [:issues], excluded_attributes: { issues: [:name] })
expect(described_class.new(shared: shared).project_tree).to match(include: [{ issues: { except: [:name] } }])
end
it 'generates the correct hash for a relation with both excluded and included attributes' do
setup_yaml(project_tree: [:issues], excluded_attributes: { issues: [:name] }, included_attributes: { issues: [:description] })
expect(described_class.new(shared: shared).project_tree).to match(include: [{ issues: { except: [:name], only: [:description] } }])
end
it 'generates the correct hash for a relation with custom methods' do
setup_yaml(project_tree: [:issues], methods: { issues: [:name] })
expect(described_class.new(shared: shared).project_tree).to match(include: [{ issues: { methods: [:name] } }])
end
def setup_yaml(hash)
allow(YAML).to receive(:load_file).with(test_config).and_return(hash)
end
end
end
require 'spec_helper'
describe Gitlab::ImportExport::RepoSaver, services: true do
describe 'bundle a project Git repo' do
let(:user) { create(:user) }
let!(:project) { create(:project, :public, name: 'searchable_project') }
let(:export_path) { "#{Dir::tmpdir}/project_tree_saver_spec" }
let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.path_with_namespace) }
let(:bundler) { described_class.new(project: project, shared: shared) }
before do
project.team << [user, :master]
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
end
after do
FileUtils.rm_rf(export_path)
end
it 'bundles the repo successfully' do
expect(bundler.save).to be true
end
end
end
require 'spec_helper'
describe Gitlab::ImportExport::WikiRepoSaver, services: true do
describe 'bundle a wiki Git repo' do
let(:user) { create(:user) }
let!(:project) { create(:project, :public, name: 'searchable_project') }
let(:export_path) { "#{Dir::tmpdir}/project_tree_saver_spec" }
let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.path_with_namespace) }
let(:wiki_bundler) { described_class.new(project: project, shared: shared) }
let!(:project_wiki) { ProjectWiki.new(project, user) }
before do
project.team << [user, :master]
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
project_wiki.wiki
project_wiki.create_page("index", "test content")
end
after do
FileUtils.rm_rf(export_path)
end
it 'bundles the repo successfully' do
expect(wiki_bundler.save).to be true
end
end
end
# Class relationships to be included in the project import/export
project_tree:
- :issues
- :labels
- merge_requests:
- :merge_request_diff
- :merge_request_test
- commit_statuses:
- :commit
included_attributes:
project:
- :name
- :path
merge_requests:
- :id
excluded_attributes:
merge_requests:
- :iid
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment