Commit e707d8f1 authored by James Lopez's avatar James Lopez

Merge branch '11090-export-design-management-data' into 'master'

Export design management data with project export

Closes #11090

See merge request gitlab-org/gitlab!14702
parents 3f9ce7a1 f4def7ad
# frozen_string_literal: true
module ExportHelper
# An EE-overwriteable list of descriptions
def project_export_descriptions
[
_('Project and wiki repositories'),
_('Project uploads'),
_('Project configuration, including services'),
_('Issues with comments, merge requests with diffs and comments, labels, milestones, snippets, and other project entities'),
_('LFS objects'),
_('Issue Boards')
]
end
end
ExportHelper.prepend_if_ee('EE::ExportHelper')
...@@ -1098,6 +1098,8 @@ class Repository ...@@ -1098,6 +1098,8 @@ class Repository
raw.create_repository raw.create_repository
after_create after_create
true
end end
def blobs_metadata(paths, ref = 'HEAD') def blobs_metadata(paths, ref = 'HEAD')
......
...@@ -12,6 +12,8 @@ module Projects ...@@ -12,6 +12,8 @@ module Projects
private private
attr_accessor :shared
def execute_after_export_action(after_export_strategy) def execute_after_export_action(after_export_strategy)
return unless after_export_strategy return unless after_export_strategy
...@@ -21,50 +23,54 @@ module Projects ...@@ -21,50 +23,54 @@ module Projects
end end
def save_all! def save_all!
if save_services if save_exporters
Gitlab::ImportExport::Saver.save(project: project, shared: @shared) Gitlab::ImportExport::Saver.save(project: project, shared: shared)
notify_success notify_success
else else
cleanup_and_notify_error! cleanup_and_notify_error!
end end
end end
def save_services def save_exporters
[version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver, lfs_saver].all?(&:save) exporters.all?(&:save)
end
def exporters
[version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver, lfs_saver]
end end
def version_saver def version_saver
Gitlab::ImportExport::VersionSaver.new(shared: @shared) Gitlab::ImportExport::VersionSaver.new(shared: shared)
end end
def avatar_saver def avatar_saver
Gitlab::ImportExport::AvatarSaver.new(project: project, shared: @shared) Gitlab::ImportExport::AvatarSaver.new(project: project, shared: shared)
end end
def project_tree_saver def project_tree_saver
Gitlab::ImportExport::ProjectTreeSaver.new(project: project, current_user: @current_user, shared: @shared, params: @params) Gitlab::ImportExport::ProjectTreeSaver.new(project: project, current_user: current_user, shared: shared, params: params)
end end
def uploads_saver def uploads_saver
Gitlab::ImportExport::UploadsSaver.new(project: project, shared: @shared) Gitlab::ImportExport::UploadsSaver.new(project: project, shared: shared)
end end
def repo_saver def repo_saver
Gitlab::ImportExport::RepoSaver.new(project: project, shared: @shared) Gitlab::ImportExport::RepoSaver.new(project: project, shared: shared)
end end
def wiki_repo_saver def wiki_repo_saver
Gitlab::ImportExport::WikiRepoSaver.new(project: project, shared: @shared) Gitlab::ImportExport::WikiRepoSaver.new(project: project, shared: shared)
end end
def lfs_saver def lfs_saver
Gitlab::ImportExport::LfsSaver.new(project: project, shared: @shared) Gitlab::ImportExport::LfsSaver.new(project: project, shared: shared)
end end
def cleanup_and_notify_error def cleanup_and_notify_error
Rails.logger.error("Import/Export - Project #{project.name} with ID: #{project.id} export error - #{@shared.errors.join(', ')}") # rubocop:disable Gitlab/RailsLogger Rails.logger.error("Import/Export - Project #{project.name} with ID: #{project.id} export error - #{shared.errors.join(', ')}") # rubocop:disable Gitlab/RailsLogger
FileUtils.rm_rf(@shared.export_path) FileUtils.rm_rf(shared.export_path)
notify_error notify_error
end end
...@@ -72,7 +78,7 @@ module Projects ...@@ -72,7 +78,7 @@ module Projects
def cleanup_and_notify_error! def cleanup_and_notify_error!
cleanup_and_notify_error cleanup_and_notify_error
raise Gitlab::ImportExport::Error.new(@shared.errors.join(', ')) raise Gitlab::ImportExport::Error.new(shared.errors.to_sentence)
end end
def notify_success def notify_success
...@@ -80,8 +86,10 @@ module Projects ...@@ -80,8 +86,10 @@ module Projects
end end
def notify_error def notify_error
notification_service.project_not_exported(@project, @current_user, @shared.errors) notification_service.project_not_exported(project, current_user, shared.errors)
end end
end end
end end
end end
Projects::ImportExport::ExportService.prepend_if_ee('EE::Projects::ImportExport::ExportService')
...@@ -10,12 +10,8 @@ ...@@ -10,12 +10,8 @@
%p.append-bottom-0 %p.append-bottom-0
%p= _('The following items will be exported:') %p= _('The following items will be exported:')
%ul %ul
%li= _('Project and wiki repositories') - project_export_descriptions.each do |desc|
%li= _('Project uploads') %li= desc
%li= _('Project configuration, including services')
%li= _('Issues with comments, merge requests with diffs and comments, labels, milestones, snippets, and other project entities')
%li= _('LFS objects')
%li= _('Issue Boards')
%p= _('The following items will NOT be exported:') %p= _('The following items will NOT be exported:')
%ul %ul
%li= _('Job traces and artifacts') %li= _('Job traces and artifacts')
......
# frozen_string_literal: true
class DesignIssueIdNullable < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
change_column_null :design_management_designs, :issue_id, true
end
end
...@@ -1217,7 +1217,7 @@ ActiveRecord::Schema.define(version: 2019_09_26_041216) do ...@@ -1217,7 +1217,7 @@ ActiveRecord::Schema.define(version: 2019_09_26_041216) do
create_table "design_management_designs", force: :cascade do |t| create_table "design_management_designs", force: :cascade do |t|
t.integer "project_id", null: false t.integer "project_id", null: false
t.integer "issue_id", null: false t.integer "issue_id"
t.string "filename", null: false t.string "filename", null: false
t.index ["issue_id", "filename"], name: "index_design_management_designs_on_issue_id_and_filename", unique: true t.index ["issue_id", "filename"], name: "index_design_management_designs_on_issue_id_and_filename", unique: true
t.index ["project_id"], name: "index_design_management_designs_on_project_id" t.index ["project_id"], name: "index_design_management_designs_on_project_id"
......
...@@ -38,7 +38,6 @@ to be enabled: ...@@ -38,7 +38,6 @@ to be enabled:
- Files uploaded must have a file extension of either `png`, `jpg`, `jpeg`, `gif`, `bmp`, `tiff` or `ico`. - Files uploaded must have a file extension of either `png`, `jpg`, `jpeg`, `gif`, `bmp`, `tiff` or `ico`.
The [`svg` extension is not yet supported](https://gitlab.com/gitlab-org/gitlab/issues/12771). The [`svg` extension is not yet supported](https://gitlab.com/gitlab-org/gitlab/issues/12771).
- Design uploads are limited to 10 files at a time. - Design uploads are limited to 10 files at a time.
- [Designs cannot yet be deleted](https://gitlab.com/gitlab-org/gitlab/issues/11089).
- Design Management is - Design Management is
[not yet supported in the project export](https://gitlab.com/gitlab-org/gitlab/issues/11090). [not yet supported in the project export](https://gitlab.com/gitlab-org/gitlab/issues/11090).
- Design Management data - Design Management data
......
...@@ -65,6 +65,7 @@ The following items will be exported: ...@@ -65,6 +65,7 @@ The following items will be exported:
- Project configuration, including services - Project configuration, including services
- Issues with comments, merge requests with diffs and comments, labels, milestones, snippets, - Issues with comments, merge requests with diffs and comments, labels, milestones, snippets,
and other project entities and other project entities
- Design Management files and data **(PREMIUM)**
- LFS objects - LFS objects
- Issue boards - Issue boards
......
# frozen_string_literal: true
module EE
module ExportHelper
extend ::Gitlab::Utils::Override
override :project_export_descriptions
def project_export_descriptions
super + [_('Design Management files and data')]
end
end
end
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
module DesignManagement module DesignManagement
class Design < ApplicationRecord class Design < ApplicationRecord
include Importable
include Noteable include Noteable
include Gitlab::FileTypeDetection include Gitlab::FileTypeDetection
include Gitlab::Utils::StrongMemoize include Gitlab::Utils::StrongMemoize
...@@ -16,7 +17,8 @@ module DesignManagement ...@@ -16,7 +17,8 @@ module DesignManagement
# data # data
has_many :notes, as: :noteable, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_many :notes, as: :noteable, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent
validates :project, :issue, :filename, presence: true validates :project, :filename, presence: true
validates :issue, presence: true, unless: :importing?
validates :filename, uniqueness: { scope: :issue_id } validates :filename, uniqueness: { scope: :issue_id }
validate :validate_file_is_image validate :validate_file_is_image
......
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
module DesignManagement module DesignManagement
class Version < ApplicationRecord class Version < ApplicationRecord
include Importable
include ShaAttribute include ShaAttribute
NotSameIssue = Class.new(StandardError) NotSameIssue = Class.new(StandardError)
...@@ -34,13 +35,13 @@ module DesignManagement ...@@ -34,13 +35,13 @@ module DesignManagement
source: :design, source: :design,
inverse_of: :versions inverse_of: :versions
validates :designs, presence: true validates :designs, presence: true, unless: :importing?
validates :sha, presence: true validates :sha, presence: true
validates :sha, uniqueness: { case_sensitive: false, scope: :issue_id } validates :sha, uniqueness: { case_sensitive: false, scope: :issue_id }
# We are not validating the issue object as it incurs an extra query to fetch # We are not validating the issue object as it incurs an extra query to fetch
# the record from the DB. Instead, we rely on the foreign key constraint to # the record from the DB. Instead, we rely on the foreign key constraint to
# ensure referential integrity. # ensure referential integrity.
validates :issue_id, presence: true validates :issue_id, presence: true, unless: :importing?
sha_attribute :sha sha_attribute :sha
......
# frozen_string_literal: true
module EE::Projects::ImportExport::ExportService
extend ActiveSupport::Concern
extend ::Gitlab::Utils::Override
private
override :exporters
def exporters
super + Array.wrap(design_repo_saver)
end
def design_repo_saver
return unless Feature.enabled?(:export_designs, project, default_enabled: true)
Gitlab::ImportExport::DesignRepoSaver.new(project: project, shared: shared)
end
end
---
title: Allow Design Management files and data to be included in the project exporter/importer
merge_request: 14702
author:
type: added
# frozen_string_literal: true
module EE::Gitlab::ImportExport
extend ActiveSupport::Concern
prepended do
def design_repo_bundle_filename
'project.design.bundle'
end
end
end
# frozen_string_literal: true
module EE
module Gitlab
module ImportExport
module GroupProjectObjectBuilder
extend ::Gitlab::Utils::Override
extend ActiveSupport::Concern
private
override :where_clause_for_klass
def where_clause_for_klass
return attrs_to_arel(attributes.slice('filename')).and(table[:issue_id].eq(nil)) if design?
super
end
def design?
klass == DesignManagement::Design
end
end
end
end
end
# frozen_string_literal: true
module EE::Gitlab::ImportExport::Importer
extend ActiveSupport::Concern
extend ::Gitlab::Utils::Override
private
override :restorers
def restorers
super + Array.wrap(design_repo_restorer)
end
def design_repo_restorer
return unless Feature.enabled?(:export_designs, project, default_enabled: true)
Gitlab::ImportExport::DesignRepoRestorer.new(
path_to_bundle: design_repo_path,
shared: shared,
project: project_tree.restored_project
)
end
def design_repo_path
File.join(shared.export_path, ::Gitlab::ImportExport.design_repo_bundle_filename)
end
end
# frozen_string_literal: true
module EE
module Gitlab
module ImportExport
module ProjectTreeRestorer
extend ActiveSupport::Concern
extend ::Gitlab::Utils::Override
private
attr_accessor :project
override :remove_feature_dependent_sub_relations
def remove_feature_dependent_sub_relations(relation_item)
if relation_item.is_a?(Hash) && ::Feature.disabled?(:export_designs, project, default_enabled: true)
relation_item.except!('designs', 'design_versions')
end
end
end
end
end
end
...@@ -7,10 +7,16 @@ module EE ...@@ -7,10 +7,16 @@ module EE
extend ActiveSupport::Concern extend ActiveSupport::Concern
EE_OVERRIDES = { EE_OVERRIDES = {
design: 'DesignManagement::Design',
designs: 'DesignManagement::Design',
design_versions: 'DesignManagement::Version',
actions: 'DesignManagement::Action',
deploy_access_levels: 'ProtectedEnvironment::DeployAccessLevel', deploy_access_levels: 'ProtectedEnvironment::DeployAccessLevel',
unprotect_access_levels: 'ProtectedBranch::UnprotectAccessLevel' unprotect_access_levels: 'ProtectedBranch::UnprotectAccessLevel'
}.freeze }.freeze
EE_EXISTING_OBJECT_CHECK = %i[DesignManagement::Design].freeze
class_methods do class_methods do
extend ::Gitlab::Utils::Override extend ::Gitlab::Utils::Override
...@@ -18,6 +24,11 @@ module EE ...@@ -18,6 +24,11 @@ module EE
def overrides def overrides
super.merge(EE_OVERRIDES) super.merge(EE_OVERRIDES)
end end
override :existing_object_check
def existing_object_check
super + EE_EXISTING_OBJECT_CHECK
end
end end
end end
end end
......
# frozen_string_literal: true
module Gitlab
module ImportExport
class DesignRepoRestorer < RepoRestorer
def initialize(project:, shared:, path_to_bundle:)
super(project: project, shared: shared, path_to_bundle: path_to_bundle)
@repository = project.design_repository
end
# `restore` method is handled in super class
end
end
end
# frozen_string_literal: true
module Gitlab
module ImportExport
class DesignRepoSaver < RepoSaver
def save
@repository = project.design_repository
super
end
private
def bundle_full_path
File.join(shared.export_path, ::Gitlab::ImportExport.design_repo_bundle_filename)
end
end
end
end
...@@ -30,6 +30,12 @@ FactoryBot.modify do ...@@ -30,6 +30,12 @@ FactoryBot.modify do
end end
end end
trait :design_repo do
after(:create) do |project|
raise 'Failed to create design repository!' unless project.design_repository.create_if_not_exists
end
end
trait :import_none do trait :import_none do
import_status :none import_status :none
end end
......
...@@ -49,7 +49,6 @@ describe Security::PipelineVulnerabilitiesFinder do ...@@ -49,7 +49,6 @@ describe Security::PipelineVulnerabilitiesFinder do
let(:params) { { report_type: %w[dast] } } let(:params) { { report_type: %w[dast] } }
it 'includes only dast' do it 'includes only dast' do
# binding.pry
expect(subject.count).to eq dast_count expect(subject.count).to eq dast_count
end end
end end
......
{
"description":"",
"visibility_level":0,
"archived":false,
"merge_requests_template":null,
"merge_requests_rebase_enabled":false,
"approvals_before_merge":0,
"reset_approvals_on_push":true,
"merge_requests_ff_only_enabled":false,
"issues_template":null,
"shared_runners_enabled":true,
"build_coverage_regex":null,
"build_allow_git_fetch":true,
"build_timeout":3600,
"pending_delete":false,
"public_builds":true,
"last_repository_check_failed":null,
"container_registry_enabled":true,
"only_allow_merge_if_pipeline_succeeds":false,
"has_external_issue_tracker":false,
"request_access_enabled":false,
"has_external_wiki":false,
"ci_config_path":null,
"only_allow_merge_if_all_discussions_are_resolved":false,
"repository_size_limit":null,
"printing_merge_request_link_enabled":true,
"auto_cancel_pending_pipelines":"enabled",
"service_desk_enabled":null,
"delete_error":null,
"disable_overriding_approvers_per_merge_request":null,
"resolve_outdated_diff_discussions":false,
"jobs_cache_index":null,
"external_authorization_classification_label":null,
"pages_https_only":false,
"external_webhook_token":null,
"merge_requests_author_approval":null,
"merge_requests_require_code_owner_approval":null,
"merge_requests_disable_committers_approval":null,
"require_password_to_approve":null,
"labels":[
],
"milestones":[
],
"issues":[
{
"id":469,
"title":"issue 1",
"author_id":1,
"project_id":30,
"created_at":"2019-08-07T03:57:55.007Z",
"updated_at":"2019-08-07T03:57:55.007Z",
"description":"",
"state":"opened",
"iid":1,
"updated_by_id":null,
"weight":null,
"confidential":false,
"due_date":null,
"moved_to_id":null,
"lock_version":0,
"time_estimate":0,
"relative_position":1073742323,
"service_desk_reply_to":null,
"last_edited_at":null,
"last_edited_by_id":null,
"discussion_locked":null,
"closed_at":null,
"closed_by_id":null,
"state_id":1,
"events":[
{
"id":1775,
"project_id":30,
"author_id":1,
"target_id":469,
"created_at":"2019-08-07T03:57:55.158Z",
"updated_at":"2019-08-07T03:57:55.158Z",
"target_type":"Issue",
"action":1
}
],
"timelogs":[
],
"notes":[
],
"label_links":[
],
"resource_label_events":[
],
"issue_assignees":[
],
"designs":[
{
"id":38,
"project_id":30,
"issue_id":469,
"filename":"chirrido3.jpg",
"notes":[
]
},
{
"id":39,
"project_id":30,
"issue_id":469,
"filename":"jonathan_richman.jpg",
"notes":[
]
},
{
"id":40,
"project_id":30,
"issue_id":469,
"filename":"mariavontrap.jpeg",
"notes":[
]
}
],
"design_versions":[
{
"id":24,
"sha":"9358d1bac8ff300d3d2597adaa2572a20f7f8703",
"issue_id":469,
"actions":[
{
"design_id":38,
"version_id":24,
"event":0,
"design":{
"id":38,
"project_id":30,
"issue_id":469,
"filename":"chirrido3.jpg"
}
}
]
},
{
"id":25,
"sha":"e1a4a501bcb42f291f84e5d04c8f927821542fb6",
"issue_id":469,
"actions":[
{
"design_id":38,
"version_id":25,
"event":0,
"design":{
"id":38,
"project_id":30,
"issue_id":469,
"filename":"chirrido3.jpg"
}
},
{
"design_id":39,
"version_id":25,
"event":0,
"design":{
"id":39,
"project_id":30,
"issue_id":469,
"filename":"jonathan_richman.jpg"
}
}
]
},
{
"id":26,
"sha":"27702d08f5ee021ae938737f84e8fe7c38599e85",
"issue_id":469,
"actions":[
{
"design_id":38,
"version_id":26,
"event":0,
"design":{
"id":38,
"project_id":30,
"issue_id":469,
"filename":"chirrido3.jpg"
}
},
{
"design_id":39,
"version_id":26,
"event":0,
"design":{
"id":39,
"project_id":30,
"issue_id":469,
"filename":"jonathan_richman.jpg"
}
},
{
"design_id":40,
"version_id":26,
"event":0,
"design":{
"id":40,
"project_id":30,
"issue_id":469,
"filename":"mariavontrap.jpeg"
}
}
]
}
]
},
{
"id":470,
"title":"issue 2",
"author_id":1,
"project_id":30,
"created_at":"2019-08-07T04:15:57.607Z",
"updated_at":"2019-08-07T04:15:57.607Z",
"description":"",
"state":"opened",
"iid":2,
"updated_by_id":null,
"weight":null,
"confidential":false,
"due_date":null,
"moved_to_id":null,
"lock_version":0,
"time_estimate":0,
"relative_position":1073742823,
"service_desk_reply_to":null,
"last_edited_at":null,
"last_edited_by_id":null,
"discussion_locked":null,
"closed_at":null,
"closed_by_id":null,
"state_id":1,
"events":[
{
"id":1776,
"project_id":30,
"author_id":1,
"target_id":470,
"created_at":"2019-08-07T04:15:57.789Z",
"updated_at":"2019-08-07T04:15:57.789Z",
"target_type":"Issue",
"action":1
}
],
"timelogs":[
],
"notes":[
],
"label_links":[
],
"resource_label_events":[
],
"issue_assignees":[
],
"designs":[
{
"id":42,
"project_id":30,
"issue_id":470,
"filename":"1 (1).jpeg",
"notes":[
]
},
{
"id":43,
"project_id":30,
"issue_id":470,
"filename":"2099743.jpg",
"notes":[
]
},
{
"id":44,
"project_id":30,
"issue_id":470,
"filename":"a screenshot (1).jpg",
"notes":[
]
},
{
"id":41,
"project_id":30,
"issue_id":470,
"filename":"chirrido3.jpg",
"notes":[
]
}
],
"design_versions":[
{
"id":27,
"sha":"8587e78ab6bda3bc820a9f014c3be4a21ad4fcc8",
"issue_id":470,
"actions":[
{
"design_id":41,
"version_id":27,
"event":0,
"design":{
"id":41,
"project_id":30,
"issue_id":470,
"filename":"chirrido3.jpg"
}
}
]
},
{
"id":28,
"sha":"73f871b4c8c1d65c62c460635e023179fb53abc4",
"issue_id":470,
"actions":[
{
"design_id":42,
"version_id":28,
"event":0,
"design":{
"id":42,
"project_id":30,
"issue_id":470,
"filename":"1 (1).jpeg"
}
},
{
"design_id":43,
"version_id":28,
"event":0,
"design":{
"id":43,
"project_id":30,
"issue_id":470,
"filename":"2099743.jpg"
}
}
]
},
{
"id":29,
"sha":"c9b5f067f3e892122a4b12b0a25a8089192f3ac8",
"issue_id":470,
"actions":[
{
"design_id":42,
"version_id":29,
"event":0,
"design":{
"id":42,
"project_id":30,
"issue_id":470,
"filename":"1 (1).jpeg"
}
},
{
"design_id":44,
"version_id":29,
"event":0,
"design":{
"id":44,
"project_id":30,
"issue_id":470,
"filename":"a screenshot (1).jpg"
}
}
]
}
]
}
],
"snippets":[
],
"releases":[
],
"project_members":[
{
"id":95,
"access_level":40,
"source_id":30,
"source_type":"Project",
"user_id":1,
"notification_level":3,
"created_at":"2019-08-07T03:57:32.825Z",
"updated_at":"2019-08-07T03:57:32.825Z",
"created_by_id":1,
"invite_email":null,
"invite_token":null,
"invite_accepted_at":null,
"requested_at":null,
"expires_at":null,
"ldap":false,
"override":false,
"user":{
"id":1,
"email":"admin@example.com",
"username":"root"
}
}
],
"merge_requests":[
],
"ci_pipelines":[
],
"triggers":[
],
"pipeline_schedules":[
],
"services":[
],
"protected_branches":[
],
"protected_environments":[
],
"protected_tags":[
],
"project_feature":{
"id":30,
"project_id":30,
"merge_requests_access_level":20,
"issues_access_level":20,
"wiki_access_level":20,
"snippets_access_level":20,
"builds_access_level":20,
"created_at":"2019-08-07T03:57:32.485Z",
"updated_at":"2019-08-07T03:57:32.485Z",
"repository_access_level":20,
"pages_access_level":10
},
"custom_attributes":[
],
"prometheus_metrics":[
],
"project_badges":[
],
"ci_cd_settings":{
"group_runners_enabled":true
},
"boards":[
],
"pipelines":[
]
}
...@@ -84,9 +84,9 @@ describe Mutations::DesignManagement::Delete do ...@@ -84,9 +84,9 @@ describe Mutations::DesignManagement::Delete do
end end
end end
it 'runs no more than 27 queries' do it 'runs no more than 28 queries' do
filenames.each(&:present?) # ignore setup filenames.each(&:present?) # ignore setup
# Queries: as of 2019-08-08 # Queries: as of 2019-08-28
# ------------- # -------------
# 01. routing query # 01. routing query
# 02. find project by id # 02. find project by id
...@@ -112,10 +112,11 @@ describe Mutations::DesignManagement::Delete do ...@@ -112,10 +112,11 @@ describe Mutations::DesignManagement::Delete do
# 23. create version with sha and issue # 23. create version with sha and issue
# 24. create design-version links # 24. create design-version links
# 25. validate version.actions.present? # 25. validate version.actions.present?
# 26. validate version.sha is unique # 26. validate version.issue.present?
# 27. leave transaction 1 # 27. validate version.sha is unique
# 28. leave transaction 1
# #
expect { run_mutation }.not_to exceed_query_limit(27) expect { run_mutation }.not_to exceed_query_limit(28)
end end
end end
......
# frozen_string_literal: true
require 'spec_helper'
describe ExportHelper do
describe '#project_export_descriptions' do
it 'includes design management' do
expect(project_export_descriptions).to include('Design Management files and data')
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::ImportExport::ProjectTreeRestorer do
include ImportExport::CommonUtil
using RSpec::Parameterized::TableSyntax
set(:user) { create(:user) }
set(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') }
let(:shared) { project.import_export_shared }
let(:project_tree_restorer) { described_class.new(user: user, shared: shared, project: project) }
let(:restored_project_json) { project_tree_restorer.restore }
before do
allow(shared).to receive(:export_path).and_return('spec/fixtures/lib/gitlab/import_export/')
project_tree_restorer.instance_variable_set(:@path, 'ee/spec/fixtures/lib/gitlab/import_export/project.designs.json')
end
describe 'restoring design management data' do
context 'when the `export_designs` feature is enabled' do
before do
restored_project_json
end
it_behaves_like 'restores project correctly', issues: 2
it 'restores project associations correctly' do
expect(project.designs.size).to eq(7)
end
describe 'restores issue associations correctly' do
let(:issue) { project.issues.offset(index).first }
where(:index, :design_filenames, :version_shas) do
0 | %w[chirrido3.jpg jonathan_richman.jpg mariavontrap.jpeg] | %w[27702d08f5ee021ae938737f84e8fe7c38599e85 9358d1bac8ff300d3d2597adaa2572a20f7f8703 e1a4a501bcb42f291f84e5d04c8f927821542fb6]
1 | ['1 (1).jpeg', '2099743.jpg', 'a screenshot (1).jpg', 'chirrido3.jpg'] | %w[73f871b4c8c1d65c62c460635e023179fb53abc4 8587e78ab6bda3bc820a9f014c3be4a21ad4fcc8 c9b5f067f3e892122a4b12b0a25a8089192f3ac8]
end
with_them do
it do
expect(issue.designs.pluck(:filename)).to contain_exactly(*design_filenames)
expect(issue.design_versions.pluck(:sha)).to contain_exactly(*version_shas)
end
end
end
describe 'restores design version associations correctly' do
let(:project_designs) { project.designs.reorder(:filename, :issue_id) }
let(:design) { project_designs.offset(index).first }
where(:index, :version_shas) do
0 | %w[73f871b4c8c1d65c62c460635e023179fb53abc4 c9b5f067f3e892122a4b12b0a25a8089192f3ac8]
1 | %w[73f871b4c8c1d65c62c460635e023179fb53abc4]
2 | %w[c9b5f067f3e892122a4b12b0a25a8089192f3ac8]
3 | %w[27702d08f5ee021ae938737f84e8fe7c38599e85 9358d1bac8ff300d3d2597adaa2572a20f7f8703 e1a4a501bcb42f291f84e5d04c8f927821542fb6]
4 | %w[8587e78ab6bda3bc820a9f014c3be4a21ad4fcc8]
5 | %w[27702d08f5ee021ae938737f84e8fe7c38599e85 e1a4a501bcb42f291f84e5d04c8f927821542fb6]
6 | %w[27702d08f5ee021ae938737f84e8fe7c38599e85]
end
with_them do
it do
expect(design.versions.pluck(:sha)).to contain_exactly(*version_shas)
end
end
end
end
context 'when the `export_designs` feature is disabled' do
before do
stub_feature_flags(export_designs: false)
restored_project_json
end
it_behaves_like 'restores project correctly', issues: 2
it 'does not restore any Designs' do
expect(DesignManagement::Design).not_to exist
end
it 'does not restore any Versions' do
expect(DesignManagement::Version.exists?).to be false
end
it 'does not restore any DesignVersions' do
expect(DesignManagement::Action.exists?).to be false
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::ImportExport::ProjectTreeSaver do
describe 'saves the project tree into a json object' do
set(:user) { create(:user) }
set(:project) { create(:project) }
set(:issue) { create(:issue, project: project) }
set(:design) { create(:design, :with_file, versions_count: 2, issue: issue) }
set(:note) { create(:diff_note_on_design, noteable: design, project: project, author: user) }
set(:note2) { create(:note, noteable: issue, project: project, author: user) }
let(:shared) { project.import_export_shared }
let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec_ee" }
let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared) }
let(:saved_project_json) do
project_tree_saver.save
project_json(project_tree_saver.full_path)
end
before do
project.add_maintainer(user)
end
after do
FileUtils.rm_rf(export_path)
end
it 'saves successfully' do
expect(project_tree_saver.save).to be true
end
describe 'the designs json' do
let(:issue_json) { saved_project_json['issues'].first }
it 'saves issue.designs correctly' do
expect(issue_json['designs'].size).to eq(1)
end
it 'saves issue.design_versions correctly' do
actions = issue_json['design_versions'].map do |v|
v['actions']
end.flatten
expect(issue_json['design_versions'].size).to eq(2)
expect(actions.size).to eq(2)
actions.each do |action|
expect(action['design']).to be_present
end
end
end
end
def project_json(filename)
JSON.parse(IO.read(filename))
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::ImportExport::DesignRepoRestorer do
include GitHelpers
describe 'bundle a design Git repo' do
let(:user) { create(:user) }
let!(:project_with_design_repo) { create(:project, :design_repo) }
let!(:project) { create(:project) }
let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
let(:shared) { project.import_export_shared }
let(:bundler) { Gitlab::ImportExport::DesignRepoSaver.new(project: project_with_design_repo, shared: shared) }
let(:bundle_path) { File.join(shared.export_path, Gitlab::ImportExport.design_repo_bundle_filename) }
let(:restorer) do
described_class.new(path_to_bundle: bundle_path,
shared: shared,
project: project)
end
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
bundler.save
end
after do
FileUtils.rm_rf(export_path)
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
FileUtils.rm_rf(project_with_design_repo.design_repository.path_to_repo)
FileUtils.rm_rf(project.design_repository.path_to_repo)
end
end
it 'restores the repo successfully' do
expect(restorer.restore).to eq(true)
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::ImportExport::DesignRepoSaver do
describe 'bundle a design Git repo' do
set(:user) { create(:user) }
set(:design) { create(:design, :with_file, versions_count: 1) }
let!(:project) { create(:project, :design_repo) }
let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
let(:shared) { project.import_export_shared }
let(:design_bundler) { described_class.new(project: project, shared: shared) }
before do
project.add_maintainer(user)
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
end
after do
FileUtils.rm_rf(export_path)
end
it 'bundles the repo successfully' do
expect(design_bundler.save).to be true
end
context 'when the repo is empty' do
let!(:project) { create(:project) }
it 'bundles the repo successfully' do
expect(design_bundler.save).to be true
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::ImportExport::Importer do
describe '#execute' do
let(:project) { create(:project) }
let(:test_path) { "#{Dir.tmpdir}/importer_spec" }
let(:shared) { project.import_export_shared }
let(:import_file) { fixture_file_upload('spec/features/projects/import_export/test_project_export.tar.gz') }
subject(:importer) { described_class.new(project) }
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path)
allow_any_instance_of(Gitlab::ImportExport::FileImporter).to receive(:remove_import_file)
stub_uploads_object_storage(FileUploader)
FileUtils.mkdir_p(shared.export_path)
ImportExportUpload.create(project: project, import_file: import_file)
end
after do
FileUtils.rm_rf(test_path)
end
it 'restores the design repo' do
expect(Gitlab::ImportExport::DesignRepoRestorer).to receive(:new).and_call_original
importer.execute
end
context 'when the `export_designs` feature is disabled' do
before do
stub_feature_flags(export_designs: false)
end
it 'does not restore the design repo' do
expect(Gitlab::ImportExport::DesignRepoRestorer).not_to receive(:new)
importer.execute
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Projects::ImportExport::ExportService do
describe '#execute' do
set(:user) { create(:user) }
set(:project) { create(:project) }
let(:shared) { project.import_export_shared }
let(:service) { described_class.new(project, user) }
let!(:after_export_strategy) { Gitlab::ImportExport::AfterExportStrategies::DownloadNotificationStrategy.new }
it 'saves the design repo' do
expect(Gitlab::ImportExport::DesignRepoSaver).to receive(:new).and_call_original
service.execute
end
context 'when the `export_designs` feature is disabled' do
before do
stub_feature_flags(export_designs: false)
end
it 'does not save the design repo' do
expect(Gitlab::ImportExport::DesignRepoSaver).not_to receive(:new)
service.execute
end
end
end
end
...@@ -38,6 +38,10 @@ module Gitlab ...@@ -38,6 +38,10 @@ module Gitlab
"lfs-objects" "lfs-objects"
end end
def wiki_repo_bundle_filename
"project.wiki.bundle"
end
def config_file def config_file
Rails.root.join('lib/gitlab/import_export/import_export.yml') Rails.root.join('lib/gitlab/import_export/import_export.yml')
end end
...@@ -61,3 +65,5 @@ module Gitlab ...@@ -61,3 +65,5 @@ module Gitlab
end end
end end
end end
Gitlab::ImportExport.prepend_if_ee('EE::Gitlab::ImportExport')
...@@ -26,30 +26,60 @@ module Gitlab ...@@ -26,30 +26,60 @@ module Gitlab
end end
def find def find
find_object || @klass.create(project_attributes) find_object || klass.create(project_attributes)
end end
private private
attr_reader :klass, :attributes, :group, :project
def find_object def find_object
@klass.where(where_clause).first klass.where(where_clause).first
end end
def where_clause def where_clause
@attributes.slice('title').map do |key, value| where_clauses.reduce(:and)
scope_clause = table[:project_id].eq(@project.id) end
scope_clause = scope_clause.or(table[:group_id].eq(@group.id)) if @group
def where_clauses
[
where_clause_base,
where_clause_for_title,
where_clause_for_klass
].compact
end
# Returns Arel clause `"{table_name}"."project_id" = {project.id}`
# or, if group is present:
# `"{table_name}"."project_id" = {project.id} OR "{table_name}"."group_id" = {group.id}`
def where_clause_base
clause = table[:project_id].eq(project.id)
clause = clause.or(table[:group_id].eq(group.id)) if group
clause
end
table[key].eq(value).and(scope_clause) # Returns Arel clause `"{table_name}"."title" = '{attributes['title']}'`
end.reduce(:or) # if attributes has 'title key, otherwise `nil`.
def where_clause_for_title
attrs_to_arel(attributes.slice('title'))
end
# Returns Arel clause:
# `"{table_name}"."{attrs.keys[0]}" = '{attrs.values[0]} AND {table_name}"."{attrs.keys[1]}" = '{attrs.values[1]}"`
# from the given Hash of attributes.
def attrs_to_arel(attrs)
attrs.map do |key, value|
table[key].eq(value)
end.reduce(:and)
end end
def table def table
@table ||= @klass.arel_table @table ||= klass.arel_table
end end
def project_attributes def project_attributes
@attributes.except('group').tap do |atts| attributes.except('group').tap do |atts|
if label? if label?
atts['type'] = 'ProjectLabel' # Always create project labels atts['type'] = 'ProjectLabel' # Always create project labels
elsif milestone? elsif milestone?
...@@ -60,15 +90,17 @@ module Gitlab ...@@ -60,15 +90,17 @@ module Gitlab
claim_iid claim_iid
end end
end end
atts['importing'] = true if klass.ancestors.include?(Importable)
end end
end end
def label? def label?
@klass == Label klass == Label
end end
def milestone? def milestone?
@klass == Milestone klass == Milestone
end end
# If an existing group milestone used the IID # If an existing group milestone used the IID
...@@ -79,7 +111,7 @@ module Gitlab ...@@ -79,7 +111,7 @@ module Gitlab
def claim_iid def claim_iid
# The milestone has to be a group milestone, as it's the only case where # The milestone has to be a group milestone, as it's the only case where
# we set the IID as the maximum. The rest of them are fixed. # we set the IID as the maximum. The rest of them are fixed.
milestone = @project.milestones.find_by(iid: @attributes['iid']) milestone = project.milestones.find_by(iid: attributes['iid'])
return unless milestone return unless milestone
...@@ -87,6 +119,15 @@ module Gitlab ...@@ -87,6 +119,15 @@ module Gitlab
milestone.ensure_project_iid! milestone.ensure_project_iid!
milestone.save! milestone.save!
end end
protected
# Returns Arel clause for a particular model or `nil`.
def where_clause_for_klass
# no-op
end
end end
end end
end end
Gitlab::ImportExport::GroupProjectObjectBuilder.prepend_if_ee('EE::Gitlab::ImportExport::GroupProjectObjectBuilder')
...@@ -248,7 +248,16 @@ preloads: ...@@ -248,7 +248,16 @@ preloads:
ee: ee:
tree: tree:
project: project:
protected_branches: - issues:
- designs:
- notes:
- :author
- events:
- :push_event_payload
- design_versions:
- actions:
- :design # Duplicate export of issues.designs in order to link the record to both Issue and DesignVersion
- protected_branches:
- :unprotect_access_levels - :unprotect_access_levels
protected_environments: - protected_environments:
- :deploy_access_levels - :deploy_access_levels
...@@ -21,7 +21,7 @@ module Gitlab ...@@ -21,7 +21,7 @@ module Gitlab
if import_file && check_version! && restorers.all?(&:restore) && overwrite_project if import_file && check_version! && restorers.all?(&:restore) && overwrite_project
project_tree.restored_project project_tree.restored_project
else else
raise Projects::ImportService::Error.new(@shared.errors.join(', ')) raise Projects::ImportService::Error.new(shared.errors.to_sentence)
end end
rescue => e rescue => e
raise Projects::ImportService::Error.new(e.message) raise Projects::ImportService::Error.new(e.message)
...@@ -31,70 +31,72 @@ module Gitlab ...@@ -31,70 +31,72 @@ module Gitlab
private private
attr_accessor :archive_file, :current_user, :project, :shared
def restorers def restorers
[repo_restorer, wiki_restorer, project_tree, avatar_restorer, [repo_restorer, wiki_restorer, project_tree, avatar_restorer,
uploads_restorer, lfs_restorer, statistics_restorer] uploads_restorer, lfs_restorer, statistics_restorer]
end end
def import_file def import_file
Gitlab::ImportExport::FileImporter.import(project: @project, Gitlab::ImportExport::FileImporter.import(project: project,
archive_file: @archive_file, archive_file: archive_file,
shared: @shared) shared: shared)
end end
def check_version! def check_version!
Gitlab::ImportExport::VersionChecker.check!(shared: @shared) Gitlab::ImportExport::VersionChecker.check!(shared: shared)
end end
def project_tree def project_tree
@project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: @current_user, @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: current_user,
shared: @shared, shared: shared,
project: @project) project: project)
end end
def avatar_restorer def avatar_restorer
Gitlab::ImportExport::AvatarRestorer.new(project: project_tree.restored_project, shared: @shared) Gitlab::ImportExport::AvatarRestorer.new(project: project_tree.restored_project, shared: shared)
end end
def repo_restorer def repo_restorer
Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path,
shared: @shared, shared: shared,
project: project_tree.restored_project) project: project_tree.restored_project)
end end
def wiki_restorer def wiki_restorer
Gitlab::ImportExport::WikiRestorer.new(path_to_bundle: wiki_repo_path, Gitlab::ImportExport::WikiRestorer.new(path_to_bundle: wiki_repo_path,
shared: @shared, shared: shared,
project: ProjectWiki.new(project_tree.restored_project), project: ProjectWiki.new(project_tree.restored_project),
wiki_enabled: @project.wiki_enabled?) wiki_enabled: project.wiki_enabled?)
end end
def uploads_restorer def uploads_restorer
Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared) Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: shared)
end end
def lfs_restorer def lfs_restorer
Gitlab::ImportExport::LfsRestorer.new(project: project_tree.restored_project, shared: @shared) Gitlab::ImportExport::LfsRestorer.new(project: project_tree.restored_project, shared: shared)
end end
def statistics_restorer def statistics_restorer
Gitlab::ImportExport::StatisticsRestorer.new(project: project_tree.restored_project, shared: @shared) Gitlab::ImportExport::StatisticsRestorer.new(project: project_tree.restored_project, shared: shared)
end end
def path_with_namespace def path_with_namespace
File.join(@project.namespace.full_path, @project.path) File.join(project.namespace.full_path, project.path)
end end
def repo_path def repo_path
File.join(@shared.export_path, 'project.bundle') File.join(shared.export_path, Gitlab::ImportExport.project_bundle_filename)
end end
def wiki_repo_path def wiki_repo_path
File.join(@shared.export_path, 'project.wiki.bundle') File.join(shared.export_path, Gitlab::ImportExport.wiki_repo_bundle_filename)
end end
def remove_import_file def remove_import_file
upload = @project.import_export_upload upload = project.import_export_upload
return unless upload&.import_file&.file return unless upload&.import_file&.file
...@@ -105,10 +107,10 @@ module Gitlab ...@@ -105,10 +107,10 @@ module Gitlab
def overwrite_project def overwrite_project
project = project_tree.restored_project project = project_tree.restored_project
return unless can?(@current_user, :admin_namespace, project.namespace) return unless can?(current_user, :admin_namespace, project.namespace)
if overwrite_project? if overwrite_project?
::Projects::OverwriteProjectService.new(project, @current_user) ::Projects::OverwriteProjectService.new(project, current_user)
.execute(project_to_overwrite) .execute(project_to_overwrite)
end end
...@@ -116,7 +118,7 @@ module Gitlab ...@@ -116,7 +118,7 @@ module Gitlab
end end
def original_path def original_path
@project.import_data&.data&.fetch('original_path', nil) project.import_data&.data&.fetch('original_path', nil)
end end
def overwrite_project? def overwrite_project?
...@@ -125,9 +127,11 @@ module Gitlab ...@@ -125,9 +127,11 @@ module Gitlab
def project_to_overwrite def project_to_overwrite
strong_memoize(:project_to_overwrite) do strong_memoize(:project_to_overwrite) do
Project.find_by_full_path("#{@project.namespace.full_path}/#{original_path}") Project.find_by_full_path("#{project.namespace.full_path}/#{original_path}")
end end
end end
end end
end end
end end
Gitlab::ImportExport::Importer.prepend_if_ee('EE::Gitlab::ImportExport::Importer')
...@@ -93,6 +93,10 @@ module Gitlab ...@@ -93,6 +93,10 @@ module Gitlab
end end
end end
def remove_feature_dependent_sub_relations(_relation_item)
# no-op
end
def project_relations_without_project_members def project_relations_without_project_members
# We remove `project_members` as they are deserialized separately # We remove `project_members` as they are deserialized separately
project_relations.except(:project_members) project_relations.except(:project_members)
...@@ -171,6 +175,8 @@ module Gitlab ...@@ -171,6 +175,8 @@ module Gitlab
next next
end end
remove_feature_dependent_sub_relations(relation_item)
# The transaction at this level is less speedy than one single transaction # The transaction at this level is less speedy than one single transaction
# But we can't have it in the upper level or GC won't get rid of the AR objects # But we can't have it in the upper level or GC won't get rid of the AR objects
# after we save the batch. # after we save the batch.
...@@ -238,3 +244,5 @@ module Gitlab ...@@ -238,3 +244,5 @@ module Gitlab
end end
end end
end end
Gitlab::ImportExport::ProjectTreeRestorer.prepend_if_ee('::EE::Gitlab::ImportExport::ProjectTreeRestorer')
...@@ -34,13 +34,13 @@ module Gitlab ...@@ -34,13 +34,13 @@ module Gitlab
PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze
BUILD_MODELS = %w[Ci::Build commit_status].freeze BUILD_MODELS = %i[Ci::Build commit_status].freeze
IMPORTED_OBJECT_MAX_RETRIES = 5.freeze IMPORTED_OBJECT_MAX_RETRIES = 5.freeze
EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature].freeze EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature].freeze
TOKEN_RESET_MODELS = %w[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze TOKEN_RESET_MODELS = %i[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze
def self.create(*args) def self.create(*args)
new(*args).create new(*args).create
...@@ -56,7 +56,7 @@ module Gitlab ...@@ -56,7 +56,7 @@ module Gitlab
end end
def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:, excluded_keys: []) def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:, excluded_keys: [])
@relation_name = self.class.overrides[relation_sym] || relation_sym @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym
@relation_hash = relation_hash.except('noteable_id') @relation_hash = relation_hash.except('noteable_id')
@members_mapper = members_mapper @members_mapper = members_mapper
@user = user @user = user
...@@ -92,6 +92,10 @@ module Gitlab ...@@ -92,6 +92,10 @@ module Gitlab
OVERRIDES OVERRIDES
end end
def self.existing_object_check
EXISTING_OBJECT_CHECK
end
private private
def setup_models def setup_models
...@@ -105,7 +109,7 @@ module Gitlab ...@@ -105,7 +109,7 @@ module Gitlab
update_group_references update_group_references
remove_duplicate_assignees remove_duplicate_assignees
setup_pipeline if @relation_name == 'Ci::Pipeline' setup_pipeline if @relation_name == :'Ci::Pipeline'
reset_tokens! reset_tokens!
remove_encrypted_attributes! remove_encrypted_attributes!
...@@ -184,14 +188,14 @@ module Gitlab ...@@ -184,14 +188,14 @@ module Gitlab
end end
def update_group_references def update_group_references
return unless EXISTING_OBJECT_CHECK.include?(@relation_name) return unless self.class.existing_object_check.include?(@relation_name)
return unless @relation_hash['group_id'] return unless @relation_hash['group_id']
@relation_hash['group_id'] = @project.namespace_id @relation_hash['group_id'] = @project.namespace_id
end end
def reset_tokens! def reset_tokens!
return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name.to_s) return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name)
# If we import/export a project to the same instance, tokens will have to be reset. # If we import/export a project to the same instance, tokens will have to be reset.
# We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across. # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across.
...@@ -255,7 +259,7 @@ module Gitlab ...@@ -255,7 +259,7 @@ module Gitlab
# Only find existing records to avoid mapping tables such as milestones # Only find existing records to avoid mapping tables such as milestones
# Otherwise always create the record, skipping the extra SELECT clause. # Otherwise always create the record, skipping the extra SELECT clause.
@existing_or_new_object ||= begin @existing_or_new_object ||= begin
if EXISTING_OBJECT_CHECK.include?(@relation_name) if self.class.existing_object_check.include?(@relation_name)
attribute_hash = attribute_hash_for(['events']) attribute_hash = attribute_hash_for(['events'])
existing_object.assign_attributes(attribute_hash) if attribute_hash.any? existing_object.assign_attributes(attribute_hash) if attribute_hash.any?
...@@ -284,7 +288,7 @@ module Gitlab ...@@ -284,7 +288,7 @@ module Gitlab
end end
def legacy_trigger? def legacy_trigger?
@relation_name == 'Ci::Trigger' && @relation_hash['owner_id'].nil? @relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil?
end end
def find_or_create_object! def find_or_create_object!
...@@ -293,7 +297,7 @@ module Gitlab ...@@ -293,7 +297,7 @@ module Gitlab
# Can't use IDs as validation exists calling `group` or `project` attributes # Can't use IDs as validation exists calling `group` or `project` attributes
finder_hash = parsed_relation_hash.tap do |hash| finder_hash = parsed_relation_hash.tap do |hash|
hash['group'] = @project.group if relation_class.attribute_method?('group_id') hash['group'] = @project.group if relation_class.attribute_method?('group_id')
hash['project'] = @project hash['project'] = @project if relation_class.reflect_on_association(:project)
hash.delete('project_id') hash.delete('project_id')
end end
......
...@@ -6,19 +6,23 @@ module Gitlab ...@@ -6,19 +6,23 @@ module Gitlab
include Gitlab::ImportExport::CommandLineUtil include Gitlab::ImportExport::CommandLineUtil
def initialize(project:, shared:, path_to_bundle:) def initialize(project:, shared:, path_to_bundle:)
@project = project @repository = project.repository
@path_to_bundle = path_to_bundle @path_to_bundle = path_to_bundle
@shared = shared @shared = shared
end end
def restore def restore
return true unless File.exist?(@path_to_bundle) return true unless File.exist?(path_to_bundle)
@project.repository.create_from_bundle(@path_to_bundle) repository.create_from_bundle(path_to_bundle)
rescue => e rescue => e
@shared.error(e) shared.error(e)
false false
end end
private
attr_accessor :repository, :path_to_bundle, :shared
end end
end end
end end
...@@ -5,27 +5,35 @@ module Gitlab ...@@ -5,27 +5,35 @@ module Gitlab
class RepoSaver class RepoSaver
include Gitlab::ImportExport::CommandLineUtil include Gitlab::ImportExport::CommandLineUtil
attr_reader :full_path attr_reader :project, :repository, :shared
def initialize(project:, shared:) def initialize(project:, shared:)
@project = project @project = project
@shared = shared @shared = shared
@repository = @project.repository
end end
def save def save
return true if @project.empty_repo? # it's ok to have no repo return true unless repository_exists? # it's ok to have no repo
@full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename)
bundle_to_disk bundle_to_disk
end end
private private
def repository_exists?
repository.exists? && !repository.empty?
end
def bundle_full_path
File.join(shared.export_path, ImportExport.project_bundle_filename)
end
def bundle_to_disk def bundle_to_disk
mkdir_p(@shared.export_path) mkdir_p(shared.export_path)
@project.repository.bundle_to_disk(@full_path) repository.bundle_to_disk(bundle_full_path)
rescue => e rescue => e
@shared.error(e) shared.error(e)
false false
end end
end end
......
...@@ -4,28 +4,16 @@ module Gitlab ...@@ -4,28 +4,16 @@ module Gitlab
module ImportExport module ImportExport
class WikiRepoSaver < RepoSaver class WikiRepoSaver < RepoSaver
def save def save
@wiki = ProjectWiki.new(@project) wiki = ProjectWiki.new(project)
return true unless wiki_repository_exists? # it's okay to have no Wiki @repository = wiki.repository
bundle_to_disk(File.join(@shared.export_path, project_filename)) super
end
def bundle_to_disk(full_path)
mkdir_p(@shared.export_path)
@wiki.repository.bundle_to_disk(full_path)
rescue => e
@shared.error(e)
false
end end
private private
def project_filename def bundle_full_path
"project.wiki.bundle" File.join(shared.export_path, ImportExport.wiki_repo_bundle_filename)
end
def wiki_repository_exists?
@wiki.repository.exists? && !@wiki.repository.empty?
end end
end end
end end
......
...@@ -6,19 +6,22 @@ module Gitlab ...@@ -6,19 +6,22 @@ module Gitlab
def initialize(project:, shared:, path_to_bundle:, wiki_enabled:) def initialize(project:, shared:, path_to_bundle:, wiki_enabled:)
super(project: project, shared: shared, path_to_bundle: path_to_bundle) super(project: project, shared: shared, path_to_bundle: path_to_bundle)
@project = project
@wiki_enabled = wiki_enabled @wiki_enabled = wiki_enabled
end end
def restore def restore
@project.wiki if create_empty_wiki? project.wiki if create_empty_wiki?
super super
end end
private private
attr_accessor :project, :wiki_enabled
def create_empty_wiki? def create_empty_wiki?
!File.exist?(@path_to_bundle) && @wiki_enabled !File.exist?(path_to_bundle) && wiki_enabled
end end
end end
end end
......
...@@ -5185,6 +5185,9 @@ msgstr "" ...@@ -5185,6 +5185,9 @@ msgstr ""
msgid "Deselect all" msgid "Deselect all"
msgstr "" msgstr ""
msgid "Design Management files and data"
msgstr ""
msgid "DesignManagement|%{current_design} of %{designs_count}" msgid "DesignManagement|%{current_design} of %{designs_count}"
msgstr "" msgstr ""
......
...@@ -502,3 +502,17 @@ lists: ...@@ -502,3 +502,17 @@ lists:
milestone_releases: milestone_releases:
- milestone - milestone
- release - release
design: &design
- issue
- actions
- versions
- notes
designs: *design
actions:
- design
- version
versions: &version
- issue
- designs
- actions
design_versions: *version
...@@ -2,6 +2,8 @@ require 'spec_helper' ...@@ -2,6 +2,8 @@ require 'spec_helper'
include ImportExport::CommonUtil include ImportExport::CommonUtil
describe Gitlab::ImportExport::ProjectTreeRestorer do describe Gitlab::ImportExport::ProjectTreeRestorer do
let(:shared) { project.import_export_shared }
describe 'restore project tree' do describe 'restore project tree' do
before(:context) do before(:context) do
# Using an admin for import, so we can check assignment of existing members # Using an admin for import, so we can check assignment of existing members
...@@ -14,7 +16,7 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do ...@@ -14,7 +16,7 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
RSpec::Mocks.with_temporary_scope do RSpec::Mocks.with_temporary_scope do
@project = create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project') @project = create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project')
@shared = @project.import_export_shared @shared = @project.import_export_shared
allow(@shared).to receive(:export_path).and_return('spec/lib/gitlab/import_export/') allow(@shared).to receive(:export_path).and_return('spec/fixtures/lib/gitlab/import_export/')
allow_any_instance_of(Repository).to receive(:fetch_source_branch!).and_return(true) allow_any_instance_of(Repository).to receive(:fetch_source_branch!).and_return(true)
allow_any_instance_of(Gitlab::Git::Repository).to receive(:branch_exists?).and_return(false) allow_any_instance_of(Gitlab::Git::Repository).to receive(:branch_exists?).and_return(false)
...@@ -274,36 +276,6 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do ...@@ -274,36 +276,6 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
end end
end end
shared_examples 'restores project successfully' do
it 'correctly restores project' do
expect(shared.errors).to be_empty
expect(restored_project_json).to be_truthy
end
end
shared_examples 'restores project correctly' do |**results|
it 'has labels' do
expect(project.labels.size).to eq(results.fetch(:labels, 0))
end
it 'has label priorities' do
expect(project.labels.find_by(title: 'A project label').priorities).not_to be_empty
end
it 'has milestones' do
expect(project.milestones.size).to eq(results.fetch(:milestones, 0))
end
it 'has issues' do
expect(project.issues.size).to eq(results.fetch(:issues, 0))
end
it 'does not set params that are excluded from import_export settings' do
expect(project.import_type).to be_nil
expect(project.creator_id).not_to eq 123
end
end
shared_examples 'restores group correctly' do |**results| shared_examples 'restores group correctly' do |**results|
it 'has group label' do it 'has group label' do
expect(project.group.labels.size).to eq(results.fetch(:labels, 0)) expect(project.group.labels.size).to eq(results.fetch(:labels, 0))
...@@ -322,18 +294,17 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do ...@@ -322,18 +294,17 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
context 'Light JSON' do context 'Light JSON' do
let(:user) { create(:user) } let(:user) { create(:user) }
let(:shared) { project.import_export_shared }
let!(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') } let!(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') }
let(:project_tree_restorer) { described_class.new(user: user, shared: shared, project: project) } let(:project_tree_restorer) { described_class.new(user: user, shared: shared, project: project) }
let(:restored_project_json) { project_tree_restorer.restore } let(:restored_project_json) { project_tree_restorer.restore }
before do before do
allow(shared).to receive(:export_path).and_return('spec/lib/gitlab/import_export/') allow(shared).to receive(:export_path).and_return('spec/fixtures/lib/gitlab/import_export/')
end end
context 'with a simple project' do context 'with a simple project' do
before do before do
project_tree_restorer.instance_variable_set(:@path, "spec/lib/gitlab/import_export/project.light.json") project_tree_restorer.instance_variable_set(:@path, "spec/fixtures/lib/gitlab/import_export/project.light.json")
restored_project_json restored_project_json
end end
...@@ -341,6 +312,7 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do ...@@ -341,6 +312,7 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
it_behaves_like 'restores project correctly', it_behaves_like 'restores project correctly',
issues: 1, issues: 1,
labels: 2, labels: 2,
label_with_priorities: 'A project label',
milestones: 1, milestones: 1,
first_issue_labels: 1, first_issue_labels: 1,
services: 1 services: 1
...@@ -363,7 +335,12 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do ...@@ -363,7 +335,12 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
create(:ci_build, token: 'abcd') create(:ci_build, token: 'abcd')
end end
it_behaves_like 'restores project successfully' it_behaves_like 'restores project correctly',
issues: 1,
labels: 2,
label_with_priorities: 'A project label',
milestones: 1,
first_issue_labels: 1
end end
end end
...@@ -430,15 +407,15 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do ...@@ -430,15 +407,15 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
end end
before do before do
project_tree_restorer.instance_variable_set(:@path, "spec/lib/gitlab/import_export/project.group.json") project_tree_restorer.instance_variable_set(:@path, "spec/fixtures/lib/gitlab/import_export/project.group.json")
restored_project_json restored_project_json
end end
it_behaves_like 'restores project successfully'
it_behaves_like 'restores project correctly', it_behaves_like 'restores project correctly',
issues: 2, issues: 2,
labels: 2, labels: 2,
label_with_priorities: 'A project label',
milestones: 2, milestones: 2,
first_issue_labels: 1 first_issue_labels: 1
...@@ -459,7 +436,7 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do ...@@ -459,7 +436,7 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
end end
before do before do
project_tree_restorer.instance_variable_set(:@path, "spec/lib/gitlab/import_export/project.light.json") project_tree_restorer.instance_variable_set(:@path, "spec/fixtures/lib/gitlab/import_export/project.light.json")
end end
it 'does not import any templated services' do it 'does not import any templated services' do
...@@ -501,7 +478,7 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do ...@@ -501,7 +478,7 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
end end
it 'preserves the project milestone IID' do it 'preserves the project milestone IID' do
project_tree_restorer.instance_variable_set(:@path, "spec/lib/gitlab/import_export/project.milestone-iid.json") project_tree_restorer.instance_variable_set(:@path, "spec/fixtures/lib/gitlab/import_export/project.milestone-iid.json")
expect_any_instance_of(Gitlab::ImportExport::Shared).not_to receive(:error) expect_any_instance_of(Gitlab::ImportExport::Shared).not_to receive(:error)
...@@ -534,7 +511,6 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do ...@@ -534,7 +511,6 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
describe '#restored_project' do describe '#restored_project' do
let(:project) { create(:project) } let(:project) { create(:project) }
let(:shared) { project.import_export_shared }
let(:tree_hash) { { 'visibility_level' => visibility } } let(:tree_hash) { { 'visibility_level' => visibility } }
let(:restorer) { described_class.new(user: nil, shared: shared, project: project) } let(:restorer) { described_class.new(user: nil, shared: shared, project: project) }
......
...@@ -21,7 +21,7 @@ describe Gitlab::ImportExport::RelationRenameService do ...@@ -21,7 +21,7 @@ describe Gitlab::ImportExport::RelationRenameService do
context 'when importing' do context 'when importing' do
let(:project_tree_restorer) { Gitlab::ImportExport::ProjectTreeRestorer.new(user: user, shared: shared, project: project) } let(:project_tree_restorer) { Gitlab::ImportExport::ProjectTreeRestorer.new(user: user, shared: shared, project: project) }
let(:import_path) { 'spec/lib/gitlab/import_export' } let(:import_path) { 'spec/fixtures/lib/gitlab/import_export' }
let(:file_content) { IO.read("#{import_path}/project.json") } let(:file_content) { IO.read("#{import_path}/project.json") }
let!(:json_file) { ActiveSupport::JSON.decode(file_content) } let!(:json_file) { ActiveSupport::JSON.decode(file_content) }
......
...@@ -2,8 +2,8 @@ require 'spec_helper' ...@@ -2,8 +2,8 @@ require 'spec_helper'
describe Gitlab::ImportExport::RepoSaver do describe Gitlab::ImportExport::RepoSaver do
describe 'bundle a project Git repo' do describe 'bundle a project Git repo' do
let(:user) { create(:user) } set(:user) { create(:user) }
let!(:project) { create(:project, :public, name: 'searchable_project') } let!(:project) { create(:project, :repository) }
let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
let(:shared) { project.import_export_shared } let(:shared) { project.import_export_shared }
let(:bundler) { described_class.new(project: project, shared: shared) } let(:bundler) { described_class.new(project: project, shared: shared) }
...@@ -20,5 +20,13 @@ describe Gitlab::ImportExport::RepoSaver do ...@@ -20,5 +20,13 @@ describe Gitlab::ImportExport::RepoSaver do
it 'bundles the repo successfully' do it 'bundles the repo successfully' do
expect(bundler.save).to be true expect(bundler.save).to be true
end end
context 'when the repo is empty' do
let!(:project) { create(:project) }
it 'bundles the repo successfully' do
expect(bundler.save).to be true
end
end
end end
end end
...@@ -731,3 +731,18 @@ ExternalPullRequest: ...@@ -731,3 +731,18 @@ ExternalPullRequest:
- target_repository - target_repository
- source_sha - source_sha
- target_sha - target_sha
DesignManagement::Design:
- id
- project_id
- issue_id
- filename
DesignManagement::Action:
- design_id
- event
- version_id
DesignManagement::Version:
- id
- created_at
- sha
- issue_id
- user_id
...@@ -2,8 +2,8 @@ require 'spec_helper' ...@@ -2,8 +2,8 @@ require 'spec_helper'
describe Gitlab::ImportExport::WikiRepoSaver do describe Gitlab::ImportExport::WikiRepoSaver do
describe 'bundle a wiki Git repo' do describe 'bundle a wiki Git repo' do
let(:user) { create(:user) } set(:user) { create(:user) }
let!(:project) { create(:project, :public, :wiki_repo, name: 'searchable_project') } let!(:project) { create(:project, :wiki_repo) }
let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
let(:shared) { project.import_export_shared } let(:shared) { project.import_export_shared }
let(:wiki_bundler) { described_class.new(project: project, shared: shared) } let(:wiki_bundler) { described_class.new(project: project, shared: shared) }
...@@ -23,5 +23,13 @@ describe Gitlab::ImportExport::WikiRepoSaver do ...@@ -23,5 +23,13 @@ describe Gitlab::ImportExport::WikiRepoSaver do
it 'bundles the repo successfully' do it 'bundles the repo successfully' do
expect(wiki_bundler.save).to be true expect(wiki_bundler.save).to be true
end end
context 'when the repo is empty' do
let!(:project) { create(:project) }
it 'bundles the repo successfully' do
expect(wiki_bundler.save).to be true
end
end
end end
end end
...@@ -1075,7 +1075,7 @@ describe Repository do ...@@ -1075,7 +1075,7 @@ describe Repository do
let(:ref) { 'refs/heads/master' } let(:ref) { 'refs/heads/master' }
it 'returns nil' do it 'returns nil' do
is_expected.to eq(nil) is_expected.to be_nil
end end
end end
...@@ -2002,7 +2002,7 @@ describe Repository do ...@@ -2002,7 +2002,7 @@ describe Repository do
it 'returns nil if repo does not exist' do it 'returns nil if repo does not exist' do
allow(repository).to receive(:root_ref).and_raise(Gitlab::Git::Repository::NoRepository) allow(repository).to receive(:root_ref).and_raise(Gitlab::Git::Repository::NoRepository)
expect(repository.avatar).to eq(nil) expect(repository.avatar).to be_nil
end end
it 'returns the first avatar file found in the repository' do it 'returns the first avatar file found in the repository' do
...@@ -2604,6 +2604,10 @@ describe Repository do ...@@ -2604,6 +2604,10 @@ describe Repository do
expect { repository.create_if_not_exists }.to change { repository.exists? }.from(false).to(true) expect { repository.create_if_not_exists }.to change { repository.exists? }.from(false).to(true)
end end
it 'returns true' do
expect(repository.create_if_not_exists).to eq(true)
end
it 'calls out to the repository client to create a repo' do it 'calls out to the repository client to create a repo' do
expect(repository.raw.gitaly_repository_client).to receive(:create_repository) expect(repository.raw.gitaly_repository_client).to receive(:create_repository)
...@@ -2618,6 +2622,10 @@ describe Repository do ...@@ -2618,6 +2622,10 @@ describe Repository do
repository.create_if_not_exists repository.create_if_not_exists
end end
it 'returns nil' do
expect(repository.create_if_not_exists).to be_nil
end
end end
context 'when the repository exists but the cache is not up to date' do context 'when the repository exists but the cache is not up to date' do
...@@ -2629,6 +2637,10 @@ describe Repository do ...@@ -2629,6 +2637,10 @@ describe Repository do
expect { repository.create_if_not_exists }.not_to raise_error expect { repository.create_if_not_exists }.not_to raise_error
end end
it 'returns nil' do
expect(repository.create_if_not_exists).to be_nil
end
end end
end end
......
...@@ -35,20 +35,27 @@ describe Projects::ImportExport::ExportService do ...@@ -35,20 +35,27 @@ describe Projects::ImportExport::ExportService do
end end
it 'saves the repo' do it 'saves the repo' do
# This spec errors when run against the EE codebase as there will be a third repository
# saved (the EE-specific design repository).
#
# Instead, skip this test when run within EE. There is a spec for the EE-specific design repo
# in the corresponding EE spec.
skip if Gitlab.ee?
# once for the normal repo, once for the wiki # once for the normal repo, once for the wiki
expect(Gitlab::ImportExport::RepoSaver).to receive(:new).twice.and_call_original expect(Gitlab::ImportExport::RepoSaver).to receive(:new).twice.and_call_original
service.execute service.execute
end end
it 'saves the lfs objects' do it 'saves the wiki repo' do
expect(Gitlab::ImportExport::LfsSaver).to receive(:new).and_call_original expect(Gitlab::ImportExport::WikiRepoSaver).to receive(:new).and_call_original
service.execute service.execute
end end
it 'saves the wiki repo' do it 'saves the lfs objects' do
expect(Gitlab::ImportExport::WikiRepoSaver).to receive(:new).and_call_original expect(Gitlab::ImportExport::LfsSaver).to receive(:new).and_call_original
service.execute service.execute
end end
...@@ -98,9 +105,9 @@ describe Projects::ImportExport::ExportService do ...@@ -98,9 +105,9 @@ describe Projects::ImportExport::ExportService do
end end
end end
context 'when saver services fail' do context 'when saving services fail' do
before do before do
allow(service).to receive(:save_services).and_return(false) allow(service).to receive(:save_exporters).and_return(false)
end end
after do after do
...@@ -122,7 +129,7 @@ describe Projects::ImportExport::ExportService do ...@@ -122,7 +129,7 @@ describe Projects::ImportExport::ExportService do
expect(Rails.logger).to receive(:error) expect(Rails.logger).to receive(:error)
end end
it 'the after export strategy is not called' do it 'does not call the export strategy' do
expect(service).not_to receive(:execute_after_export_action) expect(service).not_to receive(:execute_after_export_action)
end end
end end
......
# frozen_string_literal: true
# Shared examples for ProjectTreeRestorer (shared to allow the testing
# of EE-specific features)
RSpec.shared_examples 'restores project correctly' do |**results|
it 'restores the project' do
expect(shared.errors).to be_empty
expect(restored_project_json).to be_truthy
end
it 'has labels' do
labels_size = results.fetch(:labels, 0)
expect(project.labels.size).to eq(labels_size)
end
it 'has label priorities' do
label_with_priorities = results[:label_with_priorities]
if label_with_priorities
expect(project.labels.find_by(title: label_with_priorities).priorities).not_to be_empty
end
end
it 'has milestones' do
expect(project.milestones.size).to eq(results.fetch(:milestones, 0))
end
it 'has issues' do
expect(project.issues.size).to eq(results.fetch(:issues, 0))
end
it 'does not set params that are excluded from import_export settings' do
expect(project.import_type).to be_nil
expect(project.creator_id).not_to eq 123
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment