CE-specific changes to allow exporting Design data

Moving a group of shared_examples from project_tree_restorer_spec to
its own file in spec/support/shared_examples in order for these examples
to be reused in an EE-specific test.

https://gitlab.com/gitlab-org/gitlab-ee/issues/11090

app/helpers/export_helper.rb

+# frozen_string_literal: true
+
+module ExportHelper
+  # An EE-overwriteable list of descriptions
+  def project_export_descriptions
+    [
+      _('Project and wiki repositories'),
+      _('Project uploads'),
+      _('Project configuration, including services'),
+      _('Issues with comments, merge requests with diffs and comments, labels, milestones, snippets, and other project entities'),
+      _('LFS objects'),
+      _('Issue Boards')
+    ]
+  end
+end
+
+ExportHelper.prepend_if_ee('EE::ExportHelper')

app/views/projects/_export.html.haml

@@ -10,12 +10,8 @@
     %p.append-bottom-0
       %p= _('The following items will be exported:')
       %ul
-        %li= _('Project and wiki repositories')
-        %li= _('Project uploads')
-        %li= _('Project configuration, including services')
-        %li= _('Issues with comments, merge requests with diffs and comments, labels, milestones, snippets, and other project entities')
-        %li= _('LFS objects')
-        %li= _('Issue Boards')
+        - project_export_descriptions.each do |desc|
+          %li= desc
       %p= _('The following items will NOT be exported:')
       %ul
         %li= _('Job traces and artifacts')

db/migrate/20190807023052_design_issue_id_nullable.rb

+# frozen_string_literal: true
+
+class DesignIssueIdNullable < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    change_column_null :design_management_designs, :issue_id, true
+  end
+end

db/schema.rb

@@ -1217,7 +1217,7 @@ ActiveRecord::Schema.define(version: 2019_09_26_041216) do
 
   create_table "design_management_designs", force: :cascade do |t|
     t.integer "project_id", null: false
-    t.integer "issue_id", null: false
+    t.integer "issue_id"
     t.string "filename", null: false
     t.index ["issue_id", "filename"], name: "index_design_management_designs_on_issue_id_and_filename", unique: true
     t.index ["project_id"], name: "index_design_management_designs_on_project_id"

doc/user/project/issues/design_management.md

@@ -38,7 +38,6 @@ to be enabled:
 - Files uploaded must have a file extension of either `png`, `jpg`, `jpeg`, `gif`, `bmp`, `tiff` or `ico`.
   The [`svg` extension is not yet supported](https://gitlab.com/gitlab-org/gitlab/issues/12771).
 - Design uploads are limited to 10 files at a time.
-- [Designs cannot yet be deleted](https://gitlab.com/gitlab-org/gitlab/issues/11089).
 - Design Management is
   [not yet supported in the project export](https://gitlab.com/gitlab-org/gitlab/issues/11090).
 - Design Management data

doc/user/project/settings/import_export.md

@@ -65,6 +65,7 @@ The following items will be exported:
 - Project configuration, including services
 - Issues with comments, merge requests with diffs and comments, labels, milestones, snippets,
   and other project entities
+- Design Management files and data **(PREMIUM)**
 - LFS objects
 - Issue boards
 

lib/gitlab/import_export/group_project_object_builder.rb

@@ -26,30 +26,60 @@ module Gitlab
       end
 
       def find
-        find_object || @klass.create(project_attributes)
+        find_object || klass.create(project_attributes)
       end
 
       private
 
+      attr_reader :klass, :attributes, :group, :project
+
       def find_object
-        @klass.where(where_clause).first
+        klass.where(where_clause).first
       end
 
       def where_clause
-        @attributes.slice('title').map do |key, value|
-          scope_clause = table[:project_id].eq(@project.id)
-          scope_clause = scope_clause.or(table[:group_id].eq(@group.id)) if @group
+        where_clauses.reduce(:and)
+      end
+
+      def where_clauses
+        [
+          where_clause_base,
+          where_clause_for_title,
+          where_clause_for_klass
+        ].compact
+      end
+
+      # Returns Arel clause `"{table_name}"."project_id" = {project.id}`
+      # or, if group is present:
+      # `"{table_name}"."project_id" = {project.id} OR "{table_name}"."group_id" = {group.id}`
+      def where_clause_base
+        clause = table[:project_id].eq(project.id)
+        clause = clause.or(table[:group_id].eq(group.id)) if group
+
+        clause
+      end
 
-          table[key].eq(value).and(scope_clause)
-        end.reduce(:or)
+      # Returns Arel clause `"{table_name}"."title" = '{attributes['title']}'`
+      # if attributes has 'title key, otherwise `nil`.
+      def where_clause_for_title
+        attrs_to_arel(attributes.slice('title'))
+      end
+
+      # Returns Arel clause:
+      # `"{table_name}"."{attrs.keys[0]}" = '{attrs.values[0]} AND {table_name}"."{attrs.keys[1]}" = '{attrs.values[1]}"`
+      # from the given Hash of attributes.
+      def attrs_to_arel(attrs)
+        attrs.map do |key, value|
+          table[key].eq(value)
+        end.reduce(:and)
       end
 
       def table
-        @table ||= @klass.arel_table
+        @table ||= klass.arel_table
       end
 
       def project_attributes
-        @attributes.except('group').tap do |atts|
+        attributes.except('group').tap do |atts|
           if label?
             atts['type'] = 'ProjectLabel' # Always create project labels
           elsif milestone?
@@ -60,15 +90,17 @@ module Gitlab
               claim_iid
             end
           end
+
+          atts['importing'] = true if klass.ancestors.include?(Importable)
         end
       end
 
       def label?
-        @klass == Label
+        klass == Label
       end
 
       def milestone?
-        @klass == Milestone
+        klass == Milestone
       end
 
       # If an existing group milestone used the IID
@@ -79,7 +111,7 @@ module Gitlab
       def claim_iid
         # The milestone has to be a group milestone, as it's the only case where
         # we set the IID as the maximum. The rest of them are fixed.
-        milestone = @project.milestones.find_by(iid: @attributes['iid'])
+        milestone = project.milestones.find_by(iid: attributes['iid'])
 
         return unless milestone
 
@@ -87,6 +119,13 @@ module Gitlab
         milestone.ensure_project_iid!
         milestone.save!
       end
+
+      protected
+
+      # Returns Arel clause for a particular model or `nil`.
+      def where_clause_for_klass
+        # no-op
+      end
     end
   end
 end

lib/gitlab/import_export/import_export.yml

@@ -248,7 +248,16 @@ preloads:
 ee:
   tree:
     project:
-      protected_branches:
+      - issues:
+        - designs:
+          - notes:
+            - :author
+            - events:
+              - :push_event_payload
+        - design_versions:
+          - actions:
+            - :design # Duplicate export of issues.designs in order to link the record to both Issue and DesignVersion
+      - protected_branches:
         - :unprotect_access_levels
-      protected_environments:
+      - protected_environments:
         - :deploy_access_levels

lib/gitlab/import_export/project_tree_restorer.rb

@@ -93,6 +93,10 @@ module Gitlab
         end
       end
 
+      def remove_feature_dependent_sub_relations(_relation_item)
+        # no-op
+      end
+
       def project_relations_without_project_members
         # We remove `project_members` as they are deserialized separately
         project_relations.except(:project_members)
@@ -171,6 +175,8 @@ module Gitlab
             next
           end
 
+          remove_feature_dependent_sub_relations(relation_item)
+
           # The transaction at this level is less speedy than one single transaction
           # But we can't have it in the upper level or GC won't get rid of the AR objects
           # after we save the batch.
@@ -238,3 +244,5 @@ module Gitlab
     end
   end
 end
+
+Gitlab::ImportExport::ProjectTreeRestorer.prepend_if_ee('::EE::Gitlab::ImportExport::ProjectTreeRestorer')

lib/gitlab/import_export/relation_factory.rb

@@ -34,13 +34,13 @@ module Gitlab
 
       PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze
 
-      BUILD_MODELS = %w[Ci::Build commit_status].freeze
+      BUILD_MODELS = %i[Ci::Build commit_status].freeze
 
       IMPORTED_OBJECT_MAX_RETRIES = 5.freeze
 
       EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature].freeze
 
-      TOKEN_RESET_MODELS = %w[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze
+      TOKEN_RESET_MODELS = %i[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze
 
       def self.create(*args)
         new(*args).create
@@ -56,7 +56,7 @@ module Gitlab
       end
 
       def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:, excluded_keys: [])
-        @relation_name = self.class.overrides[relation_sym] || relation_sym
+        @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym
         @relation_hash = relation_hash.except('noteable_id')
         @members_mapper = members_mapper
         @user = user
@@ -92,6 +92,10 @@ module Gitlab
         OVERRIDES
       end
 
+      def self.existing_object_check
+        EXISTING_OBJECT_CHECK
+      end
+
       private
 
       def setup_models
@@ -105,7 +109,7 @@ module Gitlab
         update_group_references
         remove_duplicate_assignees
 
-        setup_pipeline if @relation_name == 'Ci::Pipeline'
+        setup_pipeline if @relation_name == :'Ci::Pipeline'
 
         reset_tokens!
         remove_encrypted_attributes!
@@ -184,14 +188,14 @@ module Gitlab
       end
 
       def update_group_references
-        return unless EXISTING_OBJECT_CHECK.include?(@relation_name)
+        return unless self.class.existing_object_check.include?(@relation_name)
         return unless @relation_hash['group_id']
 
         @relation_hash['group_id'] = @project.namespace_id
       end
 
       def reset_tokens!
-        return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name.to_s)
+        return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name)
 
         # If we import/export a project to the same instance, tokens will have to be reset.
         # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across.
@@ -255,7 +259,7 @@ module Gitlab
         # Only find existing records to avoid mapping tables such as milestones
         # Otherwise always create the record, skipping the extra SELECT clause.
         @existing_or_new_object ||= begin
-          if EXISTING_OBJECT_CHECK.include?(@relation_name)
+          if self.class.existing_object_check.include?(@relation_name)
             attribute_hash = attribute_hash_for(['events'])
 
             existing_object.assign_attributes(attribute_hash) if attribute_hash.any?
@@ -284,7 +288,7 @@ module Gitlab
       end
 
       def legacy_trigger?
-        @relation_name == 'Ci::Trigger' && @relation_hash['owner_id'].nil?
+        @relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil?
       end
 
       def find_or_create_object!
@@ -293,7 +297,7 @@ module Gitlab
         # Can't use IDs as validation exists calling `group` or `project` attributes
         finder_hash = parsed_relation_hash.tap do |hash|
           hash['group'] = @project.group if relation_class.attribute_method?('group_id')
-          hash['project'] = @project
+          hash['project'] = @project if relation_class.reflect_on_association(:project)
           hash.delete('project_id')
         end
 

locale/gitlab.pot

@@ -5179,6 +5179,9 @@ msgstr ""
 msgid "Deselect all"
 msgstr ""
 
+msgid "Design Management files and data"
+msgstr ""
+
 msgid "DesignManagement|%{current_design} of %{designs_count}"
 msgstr ""
 

spec/lib/gitlab/import_export/all_models.yml

@@ -502,3 +502,17 @@ lists:
 milestone_releases:
 - milestone
 - release
+design: &design
+- issue
+- actions
+- versions
+- notes
+designs: *design
+actions:
+- design
+- version
+versions: &version
+- issue
+- designs
+- actions
+design_versions: *version

spec/lib/gitlab/import_export/project_tree_restorer_spec.rb

@@ -2,6 +2,8 @@ require 'spec_helper'
 include ImportExport::CommonUtil
 
 describe Gitlab::ImportExport::ProjectTreeRestorer do
+  let(:shared) { project.import_export_shared }
+
   describe 'restore project tree' do
     before(:context) do
       # Using an admin for import, so we can check assignment of existing members
@@ -274,36 +276,6 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
     end
   end
 
-  shared_examples 'restores project successfully' do
-    it 'correctly restores project' do
-      expect(shared.errors).to be_empty
-      expect(restored_project_json).to be_truthy
-    end
-  end
-
-  shared_examples 'restores project correctly' do |**results|
-    it 'has labels' do
-      expect(project.labels.size).to eq(results.fetch(:labels, 0))
-    end
-
-    it 'has label priorities' do
-      expect(project.labels.find_by(title: 'A project label').priorities).not_to be_empty
-    end
-
-    it 'has milestones' do
-      expect(project.milestones.size).to eq(results.fetch(:milestones, 0))
-    end
-
-    it 'has issues' do
-      expect(project.issues.size).to eq(results.fetch(:issues, 0))
-    end
-
-    it 'does not set params that are excluded from import_export settings' do
-      expect(project.import_type).to be_nil
-      expect(project.creator_id).not_to eq 123
-    end
-  end
-
   shared_examples 'restores group correctly' do |**results|
     it 'has group label' do
       expect(project.group.labels.size).to eq(results.fetch(:labels, 0))
@@ -322,7 +294,6 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
 
   context 'Light JSON' do
     let(:user) { create(:user) }
-    let(:shared) { project.import_export_shared }
     let!(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') }
     let(:project_tree_restorer) { described_class.new(user: user, shared: shared, project: project) }
     let(:restored_project_json) { project_tree_restorer.restore }
@@ -341,6 +312,7 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
       it_behaves_like 'restores project correctly',
                       issues: 1,
                       labels: 2,
+                      label_with_priorities: 'A project label',
                       milestones: 1,
                       first_issue_labels: 1,
                       services: 1
@@ -363,7 +335,12 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
           create(:ci_build, token: 'abcd')
         end
 
-        it_behaves_like 'restores project successfully'
+        it_behaves_like 'restores project correctly',
+                        issues: 1,
+                        labels: 2,
+                        label_with_priorities: 'A project label',
+                        milestones: 1,
+                        first_issue_labels: 1
       end
     end
 
@@ -435,10 +412,10 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
         restored_project_json
       end
 
-      it_behaves_like 'restores project successfully'
       it_behaves_like 'restores project correctly',
                       issues: 2,
                       labels: 2,
+                      label_with_priorities: 'A project label',
                       milestones: 2,
                       first_issue_labels: 1
 
@@ -534,7 +511,6 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
 
   describe '#restored_project' do
     let(:project) { create(:project) }
-    let(:shared) { project.import_export_shared }
     let(:tree_hash) { { 'visibility_level' => visibility } }
     let(:restorer) { described_class.new(user: nil, shared: shared, project: project) }
 

spec/lib/gitlab/import_export/safe_model_attributes.yml

@@ -731,3 +731,18 @@ ExternalPullRequest:
 - target_repository
 - source_sha
 - target_sha
+DesignManagement::Design:
+- id
+- project_id
+- issue_id
+- filename
+DesignManagement::Action:
+- design_id
+- event
+- version_id
+DesignManagement::Version:
+- id
+- created_at
+- sha
+- issue_id
+- user_id

spec/support/shared_examples/lib/gitlab/import_export/project_tree_restorer_shared_examples.rb

+# frozen_string_literal: true
+
+# Shared examples for ProjectTreeRestorer (shared to allow the testing
+# of EE-specific features)
+RSpec.shared_examples 'restores project correctly' do |**results|
+  it 'restores the project' do
+    expect(shared.errors).to be_empty
+    expect(restored_project_json).to be_truthy
+  end
+
+  it 'has labels' do
+    labels_size = results.fetch(:labels, 0)
+
+    expect(project.labels.size).to eq(labels_size)
+  end
+
+  it 'has label priorities' do
+    label_with_priorities = results[:label_with_priorities]
+
+    if label_with_priorities
+      expect(project.labels.find_by(title: label_with_priorities).priorities).not_to be_empty
+    end
+  end
+
+  it 'has milestones' do
+    expect(project.milestones.size).to eq(results.fetch(:milestones, 0))
+  end
+
+  it 'has issues' do
+    expect(project.issues.size).to eq(results.fetch(:issues, 0))
+  end
+
+  it 'does not set params that are excluded from import_export settings' do
+    expect(project.import_type).to be_nil
+    expect(project.creator_id).not_to eq 123
+  end
+end