Automatic merge of gitlab-org/gitlab-ce master

lib/gitlab/import_export/attributes_finder.rb

@@ -3,35 +3,19 @@
 module Gitlab
   module ImportExport
     class AttributesFinder
-      def initialize(included_attributes:, excluded_attributes:, methods:)
-        @included_attributes = included_attributes || {}
-        @excluded_attributes = excluded_attributes || {}
-        @methods = methods || {}
+      def initialize(config:)
+        @tree = config[:tree] || {}
+        @included_attributes = config[:included_attributes] || {}
+        @excluded_attributes = config[:excluded_attributes] || {}
+        @methods = config[:methods] || {}
       end
 
-      def find(model_object)
-        parsed_hash = find_attributes_only(model_object)
-        parsed_hash.empty? ? model_object : { model_object => parsed_hash }
+      def find_root(model_key)
+        find(model_key, @tree[model_key])
       end
 
-      def parse(model_object)
-        parsed_hash = find_attributes_only(model_object)
-        yield parsed_hash unless parsed_hash.empty?
-      end
-
-      def find_included(value)
-        key = key_from_hash(value)
-        @included_attributes[key].nil? ? {} : { only: @included_attributes[key] }
-      end
-
-      def find_excluded(value)
-        key = key_from_hash(value)
-        @excluded_attributes[key].nil? ? {} : { except: @excluded_attributes[key] }
-      end
-
-      def find_method(value)
-        key = key_from_hash(value)
-        @methods[key].nil? ? {} : { methods: @methods[key] }
+      def find_relations_tree(model_key)
+        @tree[model_key]
       end
 
       def find_excluded_keys(klass_name)
@@ -40,12 +24,24 @@ module Gitlab
 
       private
 
-      def find_attributes_only(value)
-        find_included(value).merge(find_excluded(value)).merge(find_method(value))
+      def find(model_key, model_tree)
+        {
+          only: @included_attributes[model_key],
+          except: @excluded_attributes[model_key],
+          methods: @methods[model_key],
+          include: resolve_model_tree(model_tree)
+        }.compact
+      end
+
+      def resolve_model_tree(model_tree)
+        return unless model_tree
+
+        model_tree
+          .map(&method(:resolve_model))
       end
 
-      def key_from_hash(value)
-        value.is_a?(Hash) ? value.first.first : value
+      def resolve_model(model_key, model_tree)
+        { model_key => find(model_key, model_tree) }
       end
     end
   end

lib/gitlab/import_export/config.rb

@@ -3,70 +3,49 @@
 module Gitlab
   module ImportExport
     class Config
+      def initialize
+        @hash = parse_yaml
+        @hash.deep_symbolize_keys!
+        @ee_hash = @hash.delete(:ee) || {}
+
+        @hash[:tree] = normalize_tree(@hash[:tree])
+        @ee_hash[:tree] = normalize_tree(@ee_hash[:tree] || {})
+      end
+
       # Returns a Hash of the YAML file, including EE specific data if EE is
       # used.
       def to_h
-        hash = parse_yaml
-        ee_hash = hash['ee']
-
-        if merge? && ee_hash
-          ee_hash.each do |key, value|
-            if key == 'project_tree'
-              merge_project_tree(value, hash[key])
+        if merge_ee?
+          deep_merge(@hash, @ee_hash)
         else
-              merge_attributes_list(value, hash[key])
-            end
+          @hash
         end
       end
 
-        # We don't want to expose this section after this point, as it is no
-        # longer needed.
-        hash.delete('ee')
+      private
 
-        hash
+      def deep_merge(hash_a, hash_b)
+        hash_a.deep_merge(hash_b) do |_, this_val, other_val|
+          this_val.to_a + other_val.to_a
         end
-
-      # Merges a project relationships tree into the target tree.
-      #
-      # @param [Array<Hash|Symbol>] source_values
-      # @param [Array<Hash|Symbol>] target_values
-      def merge_project_tree(source_values, target_values)
-        source_values.each do |value|
-          if value.is_a?(Hash)
-            # Examples:
-            #
-            # { 'project_tree' => [{ 'labels' => [...] }] }
-            # { 'notes' => [:author, { 'events' => [:push_event_payload] }] }
-            value.each do |key, val|
-              target = target_values
-                .find { |h| h.is_a?(Hash) && h[key] }
-
-              if target
-                merge_project_tree(val, target[key])
-              else
-                target_values << { key => val.dup }
       end
+
+      def normalize_tree(item)
+        case item
+        when Array
+          item.reduce({}) do |hash, subitem|
+            hash.merge!(normalize_tree(subitem))
           end
+        when Hash
+          item.transform_values(&method(:normalize_tree))
+        when Symbol
+          { item => {} }
         else
-            # Example: :priorities, :author, etc
-            target_values << value
-          end
-        end
-      end
-
-      # Merges a Hash containing a flat list of attributes, such as the entries
-      # in a `excluded_attributes` section.
-      #
-      # @param [Hash] source_values
-      # @param [Hash] target_values
-      def merge_attributes_list(source_values, target_values)
-        source_values.each do |key, values|
-          target_values[key] ||= []
-          target_values[key].concat(values)
+          raise ArgumentError, "#{item} needs to be Array, Hash, Symbol or NilClass"
         end
       end
 
-      def merge?
+      def merge_ee?
         Gitlab.ee?
       end
 

lib/gitlab/import_export/import_export.yml

@@ -3,7 +3,8 @@
 # This list _must_ only contain relationships that are available to both CE and
 # EE. EE specific relationships must be defined in the `ee` section further
 # down below.
-project_tree:
+tree:
+  project:
     - labels:
       - :priorities
     - milestones:
@@ -86,6 +87,8 @@ project_tree:
       - lists:
         - label:
           - :priorities
+  group_members:
+    - :user
 
 # Only include the following attributes for the models specified.
 included_attributes:
@@ -225,12 +228,15 @@ methods:
     - :type
   lists:
     - :list_type
+  ci_pipelines:
+    - :notes
 
 # EE specific relationships and settings to include. All of this will be merged
 # into the previous structures if EE is used.
 ee:
-  project_tree:
-    - protected_branches:
+  tree:
+    project:
+      protected_branches:
         - :unprotect_access_levels
-    - protected_environments:
+      protected_environments:
         - :deploy_access_levels

lib/gitlab/import_export/json_hash_builder.rb

-# frozen_string_literal: true
-
-module Gitlab
-  module ImportExport
-    # Generates a hash that conforms with http://apidock.com/rails/Hash/to_json
-    # and its peculiar options.
-    class JsonHashBuilder
-      def self.build(model_objects, attributes_finder)
-        new(model_objects, attributes_finder).build
-      end
-
-      def initialize(model_objects, attributes_finder)
-        @model_objects = model_objects
-        @attributes_finder = attributes_finder
-      end
-
-      def build
-        process_model_objects(@model_objects)
-      end
-
-      private
-
-      # Called when the model is actually a hash containing other relations (more models)
-      # Returns the config in the right format for calling +to_json+
-      #
-      # +model_object_hash+ - A model relationship such as:
-      #   {:merge_requests=>[:merge_request_diff, :notes]}
-      def process_model_objects(model_object_hash)
-        json_config_hash = {}
-        current_key = model_object_hash.first.first
-
-        model_object_hash.values.flatten.each do |model_object|
-          @attributes_finder.parse(current_key) { |hash| json_config_hash[current_key] ||= hash }
-          handle_model_object(current_key, model_object, json_config_hash)
-        end
-
-        json_config_hash
-      end
-
-      # Creates or adds to an existing hash an individual model or list
-      #
-      # +current_key+ main model that will be a key in the hash
-      # +model_object+ model or list of models to include in the hash
-      # +json_config_hash+ the original hash containing the root model
-      def handle_model_object(current_key, model_object, json_config_hash)
-        model_or_sub_model = model_object.is_a?(Hash) ? process_model_objects(model_object) : model_object
-
-        if json_config_hash[current_key]
-          add_model_value(current_key, model_or_sub_model, json_config_hash)
-        else
-          create_model_value(current_key, model_or_sub_model, json_config_hash)
-        end
-      end
-
-      # Constructs a new hash that will hold the configuration for that particular object
-      # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+
-      #
-      # +current_key+ main model that will be a key in the hash
-      # +value+ existing model to be included in the hash
-      # +json_config_hash+ the original hash containing the root model
-      def create_model_value(current_key, value, json_config_hash)
-        json_config_hash[current_key] = parse_hash(value) || { include: value }
-      end
-
-      # Calls attributes finder to parse the hash and add any attributes to it
-      #
-      # +value+ existing model to be included in the hash
-      # +parsed_hash+ the original hash
-      def parse_hash(value)
-        return if already_contains_methods?(value)
-
-        @attributes_finder.parse(value) do |hash|
-          { include: hash_or_merge(value, hash) }
-        end
-      end
-
-      def already_contains_methods?(value)
-        value.is_a?(Hash) && value.values.detect { |val| val[:methods]}
-      end
-
-      # Adds new model configuration to an existing hash with key +current_key+
-      # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+
-      #
-      # +current_key+ main model that will be a key in the hash
-      # +value+ existing model to be included in the hash
-      # +json_config_hash+ the original hash containing the root model
-      def add_model_value(current_key, value, json_config_hash)
-        @attributes_finder.parse(value) do |hash|
-          value = { value => hash } unless value.is_a?(Hash)
-        end
-
-        add_to_array(current_key, json_config_hash, value)
-      end
-
-      # Adds new model configuration to an existing hash with key +current_key+
-      # it creates a new array if it was previously a single value
-      #
-      # +current_key+ main model that will be a key in the hash
-      # +value+ existing model to be included in the hash
-      # +json_config_hash+ the original hash containing the root model
-      def add_to_array(current_key, json_config_hash, value)
-        old_values = json_config_hash[current_key][:include]
-
-        json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten
-      end
-
-      # Construct a new hash or merge with an existing one a model configuration
-      # This is to fulfil +to_json+ requirements.
-      #
-      # +hash+ hash containing configuration generated mainly from +@attributes_finder+
-      # +value+ existing model to be included in the hash
-      def hash_or_merge(value, hash)
-        value.is_a?(Hash) ? value.merge(hash) : { value => hash }
-      end
-    end
-  end
-end

lib/gitlab/import_export/project_tree_restorer.rb

@@ -58,11 +58,13 @@ module Gitlab
       # the configuration yaml file too.
       # Finally, it updates each attribute in the newly imported project.
       def create_relations
-        default_relation_list.each do |relation|
-          if relation.is_a?(Hash)
-            create_sub_relations(relation, @tree_hash)
-          elsif @tree_hash[relation.to_s].present?
-            save_relation_hash(@tree_hash[relation.to_s], relation)
+        project_relations_without_project_members.each do |relation_key, relation_definition|
+          relation_key_s = relation_key.to_s
+
+          if relation_definition.present?
+            create_sub_relations(relation_key_s, relation_definition, @tree_hash)
+          elsif @tree_hash[relation_key_s].present?
+            save_relation_hash(relation_key_s, @tree_hash[relation_key_s])
           end
         end
 
@@ -71,7 +73,7 @@ module Gitlab
         @saved
       end
 
-      def save_relation_hash(relation_hash_batch, relation_key)
+      def save_relation_hash(relation_key, relation_hash_batch)
         relation_hash = create_relation(relation_key, relation_hash_batch)
 
         remove_group_models(relation_hash) if relation_hash.is_a?(Array)
@@ -91,10 +93,13 @@ module Gitlab
         end
       end
 
-      def default_relation_list
-        reader.tree.reject do |model|
-          model.is_a?(Hash) && model[:project_members]
+      def project_relations_without_project_members
+        # We remove `project_members` as they are deserialized separately
+        project_relations.except(:project_members)
       end
+
+      def project_relations
+        reader.attributes_finder.find_relations_tree(:project)
       end
 
       def restore_project
@@ -150,8 +155,7 @@ module Gitlab
       # issue, finds any subrelations such as notes, creates them and assign them back to the hash
       #
       # Recursively calls this method if the sub-relation is a hash containing more sub-relations
-      def create_sub_relations(relation, tree_hash, save: true)
-        relation_key = relation.keys.first.to_s
+      def create_sub_relations(relation_key, relation_definition, tree_hash, save: true)
         return if tree_hash[relation_key].blank?
 
         tree_array = [tree_hash[relation_key]].flatten
@@ -171,13 +175,13 @@ module Gitlab
           # But we can't have it in the upper level or GC won't get rid of the AR objects
           # after we save the batch.
           Project.transaction do
-            process_sub_relation(relation, relation_item)
+            process_sub_relation(relation_key, relation_definition, relation_item)
 
             # For every subrelation that hangs from Project, save the associated records altogether
             # This effectively batches all records per subrelation item, only keeping those in memory
             # We have to keep in mind that more batch granularity << Memory, but >> Slowness
             if save
-              save_relation_hash([relation_item], relation_key)
+              save_relation_hash(relation_key, [relation_item])
               tree_hash[relation_key].delete(relation_item)
             end
           end
@@ -186,37 +190,35 @@ module Gitlab
         tree_hash.delete(relation_key) if save
       end
 
-      def process_sub_relation(relation, relation_item)
-        relation.values.flatten.each do |sub_relation|
+      def process_sub_relation(relation_key, relation_definition, relation_item)
+        relation_definition.each do |sub_relation_key, sub_relation_definition|
           # We just use author to get the user ID, do not attempt to create an instance.
-          next if sub_relation == :author
+          next if sub_relation_key == :author
 
-          create_sub_relations(sub_relation, relation_item, save: false) if sub_relation.is_a?(Hash)
+          sub_relation_key_s = sub_relation_key.to_s
 
-          relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation)
-          relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank?
-        end
+          # create dependent relations if present
+          if sub_relation_definition.present?
+            create_sub_relations(sub_relation_key_s, sub_relation_definition, relation_item, save: false)
           end
 
-      def assign_relation_hash(relation_item, sub_relation)
-        if sub_relation.is_a?(Hash)
-          relation_hash = relation_item[sub_relation.keys.first.to_s]
-          sub_relation = sub_relation.keys.first
-        else
-          relation_hash = relation_item[sub_relation.to_s]
+          # transform relation hash to actual object
+          sub_relation_hash = relation_item[sub_relation_key_s]
+          if sub_relation_hash.present?
+            relation_item[sub_relation_key_s] = create_relation(sub_relation_key, sub_relation_hash)
+          end
         end
-
-        [relation_hash, sub_relation]
       end
 
-      def create_relation(relation, relation_hash_list)
+      def create_relation(relation_key, relation_hash_list)
         relation_array = [relation_hash_list].flatten.map do |relation_hash|
-          Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym,
+          Gitlab::ImportExport::RelationFactory.create(
+            relation_sym: relation_key.to_sym,
             relation_hash: relation_hash,
             members_mapper: members_mapper,
             user: @user,
             project: @restored_project,
-                                                       excluded_keys: excluded_keys_for_relation(relation))
+            excluded_keys: excluded_keys_for_relation(relation_key))
         end.compact
 
         relation_hash_list.is_a?(Array) ? relation_array : relation_array.first

lib/gitlab/import_export/project_tree_saver.rb

@@ -18,7 +18,10 @@ module Gitlab
       def save
         mkdir_p(@shared.export_path)
 
-        File.write(full_path, project_json_tree)
+        project_tree = serialize_project_tree
+        fix_project_tree(project_tree)
+        File.write(full_path, project_tree.to_json)
+
         true
       rescue => e
         @shared.error(e)
@@ -27,27 +30,25 @@ module Gitlab
 
       private
 
-      def project_json_tree
+      def fix_project_tree(project_tree)
         if @params[:description].present?
-          project_json['description'] = @params[:description]
+          project_tree['description'] = @params[:description]
         end
 
-        project_json['project_members'] += group_members_json
-
-        RelationRenameService.add_new_associations(project_json)
+        project_tree['project_members'] += group_members_array
 
-        project_json.to_json
+        RelationRenameService.add_new_associations(project_tree)
       end
 
-      def project_json
-        @project_json ||= @project.as_json(reader.project_tree)
+      def serialize_project_tree
+        @project.as_json(reader.project_tree)
       end
 
       def reader
         @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
       end
 
-      def group_members_json
+      def group_members_array
         group_members.as_json(reader.group_members_tree).each do |group_member|
           group_member['source_type'] = 'Project' # Make group members project members of the future import
         end

lib/gitlab/import_export/reader.rb

@@ -7,42 +7,22 @@ module Gitlab
 
       def initialize(shared:)
         @shared = shared
-        config_hash = ImportExport::Config.new.to_h.deep_symbolize_keys
-        @tree = config_hash[:project_tree]
-        @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes],
-                                                                        excluded_attributes: config_hash[:excluded_attributes],
-                                                                        methods: config_hash[:methods])
+
+        @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(
+          config: ImportExport::Config.new.to_h)
       end
 
       # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html
       # for outputting a project in JSON format, including its relations and sub relations.
       def project_tree
-        attributes = @attributes_finder.find(:project)
-        project_attributes = attributes.is_a?(Hash) ? attributes[:project] : {}
-
-        project_attributes.merge(include: build_hash(@tree))
+        attributes_finder.find_root(:project)
       rescue => e
         @shared.error(e)
         false
       end
 
       def group_members_tree
-        @attributes_finder.find_included(:project_members).merge(include: @attributes_finder.find(:user))
-      end
-
-      private
-
-      # Builds a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html
-      #
-      # +model_list+ - List of models as a relation tree to be included in the generated JSON, from the _import_export.yml_ file
-      def build_hash(model_list)
-        model_list.map do |model_objects|
-          if model_objects.is_a?(Hash)
-            Gitlab::ImportExport::JsonHashBuilder.build(model_objects, @attributes_finder)
-          else
-            @attributes_finder.find(model_objects)
-          end
-        end
+        attributes_finder.find_root(:group_members)
       end
     end
   end

spec/lib/gitlab/import_export/attribute_configuration_spec.rb

@@ -12,7 +12,7 @@ describe 'Import/Export attribute configuration' do
 
   let(:config_hash) { Gitlab::ImportExport::Config.new.to_h.deep_stringify_keys }
   let(:relation_names) do
-    names = names_from_tree(config_hash['project_tree'])
+    names = names_from_tree(config_hash.dig('tree', 'project'))
 
     # Remove duplicated or add missing models
     # - project is not part of the tree, so it has to be added manually.

spec/lib/gitlab/import_export/attributes_finder_spec.rb

+# frozen_string_literal: true
+
+require 'fast_spec_helper'
+
+describe Gitlab::ImportExport::AttributesFinder do
+  describe '#find_root' do
+    subject { described_class.new(config: config).find_root(model_key) }
+
+    let(:test_config) { 'spec/support/import_export/import_export.yml' }
+    let(:config) { Gitlab::ImportExport::Config.new.to_h }
+    let(:model_key) { :project }
+
+    let(:project_tree_hash) do
+      {
+        except: [:id, :created_at],
+        include: [
+          { issues: { include: [] } },
+          { labels: { include: [] } },
+          { merge_requests: {
+            except: [:iid],
+            include: [
+              { merge_request_diff: {
+                include: []
+              } },
+              { merge_request_test: { include: [] } }
+            ],
+            only: [:id]
+            } },
+          { commit_statuses: {
+              include: [{ commit: { include: [] } }]
+            } },
+          { project_members: {
+              include: [{ user: { include: [],
+                                  only: [:email] } }]
+          } }
+        ]
+      }
+    end
+
+    before do
+      allow_any_instance_of(Gitlab::ImportExport).to receive(:config_file).and_return(test_config)
+    end
+
+    it 'generates hash from project tree config' do
+      is_expected.to match(project_tree_hash)
+    end
+
+    context 'individual scenarios' do
+      it 'generates the correct hash for a single project relation' do
+        setup_yaml(tree: { project: [:issues] })
+
+        is_expected.to match(
+          include: [{ issues: { include: [] } }]
+        )
+      end
+
+      it 'generates the correct hash for a single project feature relation' do
+        setup_yaml(tree: { project: [:project_feature] })
+
+        is_expected.to match(
+          include: [{ project_feature: { include: [] } }]
+        )
+      end
+
+      it 'generates the correct hash for a multiple project relation' do
+        setup_yaml(tree: { project: [:issues, :snippets] })
+
+        is_expected.to match(
+          include: [{ issues: { include: [] } },
+                    { snippets: { include: [] } }]
+        )
+      end
+
+      it 'generates the correct hash for a single sub-relation' do
+        setup_yaml(tree: { project: [issues: [:notes]] })
+
+        is_expected.to match(
+          include: [{ issues: { include: [{ notes: { include: [] } }] } }]
+        )
+      end
+
+      it 'generates the correct hash for a multiple sub-relation' do
+        setup_yaml(tree: { project: [merge_requests: [:notes, :merge_request_diff]] })
+
+        is_expected.to match(
+          include: [{ merge_requests:
+                      { include: [{ notes: { include: [] } },
+                                  { merge_request_diff: { include: [] } }] } }]
+        )
+      end
+
+      it 'generates the correct hash for a sub-relation with another sub-relation' do
+        setup_yaml(tree: { project: [merge_requests: [notes: [:author]]] })
+
+        is_expected.to match(
+          include: [{ merge_requests: {
+                      include: [{ notes: { include: [{ author: { include: [] } }] } }]
+                    } }]
+        )
+      end
+
+      it 'generates the correct hash for a relation with included attributes' do
+        setup_yaml(tree: { project: [:issues] },
+                  included_attributes: { issues: [:name, :description] })
+
+        is_expected.to match(
+          include: [{ issues: { include: [],
+                                only: [:name, :description] } }]
+        )
+      end
+
+      it 'generates the correct hash for a relation with excluded attributes' do
+        setup_yaml(tree: { project: [:issues] },
+                  excluded_attributes: { issues: [:name] })
+
+        is_expected.to match(
+          include: [{ issues: { except: [:name],
+                                include: [] } }]
+        )
+      end
+
+      it 'generates the correct hash for a relation with both excluded and included attributes' do
+        setup_yaml(tree: { project: [:issues] },
+                  excluded_attributes: { issues: [:name] },
+                  included_attributes: { issues: [:description] })
+
+        is_expected.to match(
+          include: [{ issues: { except: [:name],
+                                include: [],
+                                only: [:description] } }]
+        )
+      end
+
+      it 'generates the correct hash for a relation with custom methods' do
+        setup_yaml(tree: { project: [:issues] },
+                  methods: { issues: [:name] })
+
+        is_expected.to match(
+          include: [{ issues: { include: [],
+                                methods: [:name] } }]
+        )
+      end
+
+      def setup_yaml(hash)
+        allow(YAML).to receive(:load_file).with(test_config).and_return(hash)
+      end
+    end
+  end
+
+  describe '#find_relations_tree' do
+    subject { described_class.new(config: config).find_relations_tree(model_key) }
+
+    let(:tree) { { project: { issues: {} } } }
+    let(:model_key) { :project }
+
+    context 'when initialized with config including tree' do
+      let(:config) { { tree: tree } }
+
+      context 'when relation is in top-level keys of the tree' do
+        it { is_expected.to eq({ issues: {} }) }
+      end
+
+      context 'when the relation is not in top-level keys' do
+        let(:model_key) { :issues }
+
+        it { is_expected.to be_nil }
+      end
+    end
+
+    context 'when tree is not present in config' do
+      let(:config) { {} }
+
+      it { is_expected.to be_nil }
+    end
+  end
+
+  describe '#find_excluded_keys' do
+    subject { described_class.new(config: config).find_excluded_keys(klass_name) }
+
+    let(:klass_name) { 'project' }
+
+    context 'when initialized with excluded_attributes' do
+      let(:config) { { excluded_attributes: excluded_attributes } }
+      let(:excluded_attributes) { { project: [:name, :path], issues: [:milestone_id] } }
+
+      it { is_expected.to eq(%w[name path]) }
+    end
+
+    context 'when excluded_attributes are not present in config' do
+      let(:config) { {} }
+
+      it { is_expected.to eq([]) }
+    end
+  end
+end

spec/lib/gitlab/import_export/config_spec.rb

 # frozen_string_literal: true
 
-require 'spec_helper'
+require 'fast_spec_helper'
+require 'rspec-parameterized'
 
 describe Gitlab::ImportExport::Config do
   let(:yaml_file) { described_class.new }
 
   describe '#to_h' do
-    context 'when using CE' do
+    subject { yaml_file.to_h }
+
+    context 'when using default config' do
+      using RSpec::Parameterized::TableSyntax
+
+      where(:ee) do
+        [true, false]
+      end
+
+      with_them do
         before do
-        allow(yaml_file)
-          .to receive(:merge?)
-          .and_return(false)
+          allow(Gitlab).to receive(:ee?) { ee }
+        end
+
+        it 'parses default config' do
+          expect { subject }.not_to raise_error
+          expect(subject).to be_a(Hash)
+          expect(subject.keys).to contain_exactly(
+            :tree, :excluded_attributes, :included_attributes, :methods)
+        end
+      end
+    end
+
+    context 'when using custom config' do
+      let(:config) do
+        <<-EOF.strip_heredoc
+          tree:
+            project:
+              - labels:
+                - :priorities
+              - milestones:
+                - events:
+                  - :push_event_payload
+
+          included_attributes:
+            user:
+              - :id
+
+          excluded_attributes:
+            project:
+              - :name
+
+          methods:
+            labels:
+              - :type
+            events:
+              - :action
+
+          ee:
+            tree:
+              project:
+                protected_branches:
+                  - :unprotect_access_levels
+            included_attributes:
+              user:
+                - :name_ee
+            excluded_attributes:
+              project:
+                - :name_without_ee
+            methods:
+              labels:
+                - :type_ee
+              events_ee:
+                - :action_ee
+        EOF
       end
 
-      it 'just returns the parsed Hash without the EE section' do
-        expected = YAML.load_file(Gitlab::ImportExport.config_file)
-        expected.delete('ee')
+      let(:config_hash) { YAML.safe_load(config, [Symbol]) }
 
-        expect(yaml_file.to_h).to eq(expected)
+      before do
+        allow_any_instance_of(described_class).to receive(:parse_yaml) do
+          config_hash.deep_dup
         end
       end
 
-    context 'when using EE' do
+      context 'when using CE' do
         before do
-        allow(yaml_file)
-          .to receive(:merge?)
-          .and_return(true)
+          allow(Gitlab).to receive(:ee?) { false }
         end
 
-      it 'merges the EE project tree into the CE project tree' do
-        allow(yaml_file)
-          .to receive(:parse_yaml)
-          .and_return({
-            'project_tree' => [
-              {
-                'issues' => [
-                  :id,
-                  :title,
-                  { 'notes' => [:id, :note, { 'author' => [:name] }] }
-                ]
-              }
-            ],
-            'ee' => {
-              'project_tree' => [
+        it 'just returns the normalized Hash' do
+          is_expected.to eq(
             {
-                  'issues' => [
-                    :description,
-                    { 'notes' => [:date, { 'author' => [:email] }] }
-                  ]
+              tree: {
+                project: {
+                  labels: {
+                    priorities: {}
                   },
-                { 'foo' => [{ 'bar' => %i[baz] }] }
-              ]
+                  milestones: {
+                    events: {
+                      push_event_payload: {}
+                    }
+                  }
                 }
-          })
-
-        expect(yaml_file.to_h).to eq({
-          'project_tree' => [
-            {
-              'issues' => [
-                :id,
-                :title,
-                {
-                  'notes' => [
-                    :id,
-                    :note,
-                    { 'author' => [:name, :email] },
-                    :date
-                  ]
               },
-                :description
-              ]
+              included_attributes: {
+                user: [:id]
               },
-            { 'foo' => [{ 'bar' => %i[baz] }] }
-          ]
-        })
-      end
-
-      it 'merges the excluded attributes list' do
-        allow(yaml_file)
-          .to receive(:parse_yaml)
-          .and_return({
-            'project_tree' => [],
-            'excluded_attributes' => {
-              'project' => %i[id title],
-              'notes' => %i[id]
+              excluded_attributes: {
+                project: [:name]
               },
-            'ee' => {
-              'project_tree' => [],
-              'excluded_attributes' => {
-                'project' => %i[date],
-                'foo' => %i[bar baz]
+              methods: {
+                labels: [:type],
+                events: [:action]
               }
             }
-          })
-
-        expect(yaml_file.to_h).to eq({
-          'project_tree' => [],
-          'excluded_attributes' => {
-            'project' => %i[id title date],
-            'notes' => %i[id],
-            'foo' => %i[bar baz]
-          }
-        })
+          )
+        end
       end
 
-      it 'merges the included attributes list' do
-        allow(yaml_file)
-          .to receive(:parse_yaml)
-          .and_return({
-            'project_tree' => [],
-            'included_attributes' => {
-              'project' => %i[id title],
-              'notes' => %i[id]
+      context 'when using EE' do
+        before do
+          allow(Gitlab).to receive(:ee?) { true }
+        end
+
+        it 'just returns the normalized Hash' do
+          is_expected.to eq(
+            {
+              tree: {
+                project: {
+                  labels: {
+                    priorities: {}
                   },
-            'ee' => {
-              'project_tree' => [],
-              'included_attributes' => {
-                'project' => %i[date],
-                'foo' => %i[bar baz]
+                  milestones: {
+                    events: {
+                      push_event_payload: {}
                     }
+                  },
+                  protected_branches: {
+                    unprotect_access_levels: {}
                   }
-          })
-
-        expect(yaml_file.to_h).to eq({
-          'project_tree' => [],
-          'included_attributes' => {
-            'project' => %i[id title date],
-            'notes' => %i[id],
-            'foo' => %i[bar baz]
                 }
-        })
-      end
-
-      it 'merges the methods list' do
-        allow(yaml_file)
-          .to receive(:parse_yaml)
-          .and_return({
-            'project_tree' => [],
-            'methods' => {
-              'project' => %i[id title],
-              'notes' => %i[id]
               },
-            'ee' => {
-              'project_tree' => [],
-              'methods' => {
-                'project' => %i[date],
-                'foo' => %i[bar baz]
-              }
+              included_attributes: {
+                user: [:id, :name_ee]
+              },
+              excluded_attributes: {
+                project: [:name, :name_without_ee]
+              },
+              methods: {
+                labels: [:type, :type_ee],
+                events: [:action],
+                events_ee: [:action_ee]
               }
-          })
-
-        expect(yaml_file.to_h).to eq({
-          'project_tree' => [],
-          'methods' => {
-            'project' => %i[id title date],
-            'notes' => %i[id],
-            'foo' => %i[bar baz]
             }
-        })
+          )
+        end
       end
     end
   end

spec/lib/gitlab/import_export/model_configuration_spec.rb

@@ -8,7 +8,7 @@ describe 'Import/Export model configuration' do
 
   let(:config_hash) { Gitlab::ImportExport::Config.new.to_h.deep_stringify_keys }
   let(:model_names) do
-    names = names_from_tree(config_hash['project_tree'])
+    names = names_from_tree(config_hash.dig('tree', 'project'))
 
     # Remove duplicated or add missing models
     # - project is not part of the tree, so it has to be added manually.

spec/lib/gitlab/import_export/reader_spec.rb

@@ -2,96 +2,45 @@ require 'spec_helper'
 
 describe Gitlab::ImportExport::Reader do
   let(:shared) { Gitlab::ImportExport::Shared.new(nil) }
-  let(:test_config) { 'spec/support/import_export/import_export.yml' }
-  let(:project_tree_hash) do
-    {
-      except: [:id, :created_at],
-      include: [:issues, :labels,
-                { merge_requests: {
-                  only: [:id],
-                  except: [:iid],
-                  include: [:merge_request_diff, :merge_request_test]
-                } },
-                { commit_statuses: { include: :commit } },
-                { project_members: { include: { user: { only: [:email] } } } }]
-    }
-  end
 
-  before do
-    allow_any_instance_of(Gitlab::ImportExport).to receive(:config_file).and_return(test_config)
-  end
+  describe '#project_tree' do
+    subject { described_class.new(shared: shared).project_tree }
 
-  it 'generates hash from project tree config' do
-    expect(described_class.new(shared: shared).project_tree).to match(project_tree_hash)
-  end
+    it 'delegates to AttributesFinder#find_root' do
+      expect_any_instance_of(Gitlab::ImportExport::AttributesFinder)
+        .to receive(:find_root)
+        .with(:project)
 
-  context 'individual scenarios' do
-    it 'generates the correct hash for a single project relation' do
-      setup_yaml(project_tree: [:issues])
-
-      expect(described_class.new(shared: shared).project_tree).to match(include: [:issues])
+      subject
     end
 
-    it 'generates the correct hash for a single project feature relation' do
-      setup_yaml(project_tree: [:project_feature])
-
-      expect(described_class.new(shared: shared).project_tree).to match(include: [:project_feature])
-    end
-
-    it 'generates the correct hash for a multiple project relation' do
-      setup_yaml(project_tree: [:issues, :snippets])
-
-      expect(described_class.new(shared: shared).project_tree).to match(include: [:issues, :snippets])
-    end
-
-    it 'generates the correct hash for a single sub-relation' do
-      setup_yaml(project_tree: [issues: [:notes]])
-
-      expect(described_class.new(shared: shared).project_tree).to match(include: [{ issues: { include: :notes } }])
-    end
-
-    it 'generates the correct hash for a multiple sub-relation' do
-      setup_yaml(project_tree: [merge_requests: [:notes, :merge_request_diff]])
-
-      expect(described_class.new(shared: shared).project_tree).to match(include: [{ merge_requests: { include: [:notes, :merge_request_diff] } }])
+    context 'when exception raised' do
+      before do
+        expect_any_instance_of(Gitlab::ImportExport::AttributesFinder)
+          .to receive(:find_root)
+          .with(:project)
+          .and_raise(StandardError)
       end
 
-    it 'generates the correct hash for a sub-relation with another sub-relation' do
-      setup_yaml(project_tree: [merge_requests: [notes: :author]])
+      it { is_expected.to be false }
 
-      expect(described_class.new(shared: shared).project_tree).to match(include: [{ merge_requests: { include: { notes: { include: :author } } } }])
-    end
-
-    it 'generates the correct hash for a relation with included attributes' do
-      setup_yaml(project_tree: [:issues], included_attributes: { issues: [:name, :description] })
+      it 'logs the error' do
+        expect(shared).to receive(:error).with(instance_of(StandardError))
 
-      expect(described_class.new(shared: shared).project_tree).to match(include: [{ issues: { only: [:name, :description] } }])
+        subject
       end
-
-    it 'generates the correct hash for a relation with excluded attributes' do
-      setup_yaml(project_tree: [:issues], excluded_attributes: { issues: [:name] })
-
-      expect(described_class.new(shared: shared).project_tree).to match(include: [{ issues: { except: [:name] } }])
     end
-
-    it 'generates the correct hash for a relation with both excluded and included attributes' do
-      setup_yaml(project_tree: [:issues], excluded_attributes: { issues: [:name] }, included_attributes: { issues: [:description] })
-
-      expect(described_class.new(shared: shared).project_tree).to match(include: [{ issues: { except: [:name], only: [:description] } }])
   end
 
-    it 'generates the correct hash for a relation with custom methods' do
-      setup_yaml(project_tree: [:issues], methods: { issues: [:name] })
+  describe '#group_members_tree' do
+    subject { described_class.new(shared: shared).group_members_tree }
 
-      expect(described_class.new(shared: shared).project_tree).to match(include: [{ issues: { methods: [:name] } }])
-    end
-
-    it 'generates the correct hash for group members' do
-      expect(described_class.new(shared: shared).group_members_tree).to match({ include: { user: { only: [:email] } } })
-    end
+    it 'delegates to AttributesFinder#find_root' do
+      expect_any_instance_of(Gitlab::ImportExport::AttributesFinder)
+        .to receive(:find_root)
+        .with(:group_members)
 
-    def setup_yaml(hash)
-      allow(YAML).to receive(:load_file).with(test_config).and_return(hash)
+      subject
     end
   end
 end

spec/lib/gitlab/import_export/relation_rename_service_spec.rb

@@ -12,7 +12,7 @@ describe Gitlab::ImportExport::RelationRenameService do
 
   let(:user) { create(:admin) }
   let(:group) { create(:group, :nested) }
-  let!(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') }
+  let!(:project) { create(:project, :builds_disabled, :issues_disabled, group: group, name: 'project', path: 'project') }
   let(:shared) { project.import_export_shared }
 
   before do
@@ -24,7 +24,6 @@ describe Gitlab::ImportExport::RelationRenameService do
     let(:import_path) { 'spec/lib/gitlab/import_export' }
     let(:file_content) { IO.read("#{import_path}/project.json") }
     let!(:json_file) { ActiveSupport::JSON.decode(file_content) }
-    let(:tree_hash) { project_tree_restorer.instance_variable_get(:@tree_hash) }
 
     before do
       allow(shared).to receive(:export_path).and_return(import_path)
@@ -92,21 +91,25 @@ describe Gitlab::ImportExport::RelationRenameService do
   end
 
   context 'when exporting' do
-    let(:project_tree_saver) { Gitlab::ImportExport::ProjectTreeSaver.new(project: project, current_user: user, shared: shared) }
-    let(:project_tree) { project_tree_saver.send(:project_json) }
+    let(:export_content_path) { project_tree_saver.full_path }
+    let(:export_content_hash) { ActiveSupport::JSON.decode(File.read(export_content_path)) }
+    let(:injected_hash) { renames.values.product([{}]).to_h }
 
-    it 'adds old relationships to the exported file' do
-      project_tree.merge!(renames.values.map { |new_name| [new_name, []] }.to_h)
-
-      allow(project_tree_saver).to receive(:save) do |arg|
-        project_tree_saver.send(:project_json_tree)
+    let(:project_tree_saver) do
+      Gitlab::ImportExport::ProjectTreeSaver.new(
+        project: project, current_user: user, shared: shared)
     end
 
-      result = project_tree_saver.save
+    it 'adds old relationships to the exported file' do
+      # we inject relations with new names that should be rewritten
+      expect(project_tree_saver).to receive(:serialize_project_tree).and_wrap_original do |method, *args|
+        method.call(*args).merge(injected_hash)
+      end
 
-      saved_data = ActiveSupport::JSON.decode(result)
+      expect(project_tree_saver.save).to eq(true)
 
-      expect(saved_data.keys).to include(*(renames.keys + renames.values))
+      expect(export_content_hash.keys).to include(*renames.keys)
+      expect(export_content_hash.keys).to include(*renames.values)
     end
   end
 end

spec/support/import_export/import_export.yml

 # Class relationships to be included in the project import/export
-project_tree:
+tree:
+  project:
     - :issues
     - :labels
     - merge_requests:
@@ -9,6 +10,8 @@ project_tree:
       - :commit
     - project_members:
       - :user
+  group_members:
+    - :user
 
 included_attributes:
   merge_requests: