relation_factory.rb 9.29 KB
Newer Older
1
module Gitlab
2
  module ImportExport
3
    class RelationFactory
4
      OVERRIDES = { snippets: :project_snippets,
5
                    pipelines: 'Ci::Pipeline',
6 7
                    statuses: 'commit_status',
                    triggers: 'Ci::Trigger',
8
                    trigger_schedule: 'Ci::TriggerSchedule',
9
                    builds: 'Ci::Build',
10 11
                    hooks: 'ProjectHook',
                    merge_access_levels: 'ProtectedBranch::MergeAccessLevel',
James Lopez's avatar
James Lopez committed
12
                    push_access_levels: 'ProtectedBranch::PushAccessLevel',
13
                    labels: :project_labels,
14
                    priorities: :label_priorities,
15
                    label: :project_label }.freeze
16

17
      USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id merge_user_id resolved_by_id].freeze
18

19
      PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze
20

James Lopez's avatar
James Lopez committed
21 22
      BUILD_MODELS = %w[Ci::Build commit_status].freeze

23 24
      IMPORTED_OBJECT_MAX_RETRIES = 5.freeze

25
      EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels].freeze
26

James Lopez's avatar
James Lopez committed
27
      TOKEN_RESET_MODELS = %w[Ci::Trigger Ci::Build ProjectHook].freeze
28

29 30 31
      def self.create(*args)
        new(*args).create
      end
32

33
      def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:)
34
        @relation_name = OVERRIDES[relation_sym] || relation_sym
35
        @relation_hash = relation_hash.except('noteable_id').merge('project_id' => project.id)
36
        @members_mapper = members_mapper
37
        @user = user
38
        @project = project
39
        @imported_object_retries = 0
40
      end
James Lopez's avatar
James Lopez committed
41

42 43 44 45
      # Creates an object from an actual model with name "relation_sym" with params from
      # the relation_hash, updating references with new object IDs, mapping users using
      # the "members_mapper" object, also updating notes if required.
      def create
46 47
        return nil if unknown_service?

48 49 50 51 52 53 54 55 56 57 58
        setup_models

        generate_imported_object
      end

      private

      def setup_models
        if @relation_name == :notes
          set_note_author

James Lopez's avatar
James Lopez committed
59
          # attachment is deprecated and note uploads are handled by Markdown uploader
60 61 62
          @relation_hash['attachment'] = nil
        end

63 64
        update_user_references
        update_project_references
65 66

        handle_group_label if group_label?
67
        reset_tokens!
68
        remove_encrypted_attributes!
69

70
        @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data']
71
        set_st_diff_commits if @relation_name == :merge_request_diff
72 73
      end

74
      def update_user_references
75
        USER_REFERENCES.each do |reference|
76 77
          if @relation_hash[reference]
            @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]]
78 79 80 81
          end
        end
      end

82 83 84 85 86 87 88
      # Sets the author for a note. If the user importing the project
      # has admin access, an actual mapping with new project members
      # will be used. Otherwise, a note stating the original author name
      # is left.
      def set_note_author
        old_author_id = @relation_hash['author_id']
        author = @relation_hash.delete('author')
89

90
        update_note_for_missing_author(author['name']) unless has_author?(old_author_id)
James Lopez's avatar
James Lopez committed
91 92
      end

93
      def has_author?(old_author_id)
94
        admin_user? && @members_mapper.include?(old_author_id)
95 96 97
      end

      def missing_author_note(updated_at, author_name)
98 99
        timestamp = updated_at.split('.').first
        "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*"
100 101
      end

102
      def generate_imported_object
103 104
        if BUILD_MODELS.include?(@relation_name)
          @relation_hash.delete('trace') # old export files have trace
105 106
          @relation_hash.delete('token')

107 108
          imported_object do |object|
            object.commit_id = nil
James Lopez's avatar
James Lopez committed
109
          end
110 111
        elsif @relation_name == :merge_requests
          MergeRequestParser.new(@project, @relation_hash.delete('diff_head_sha'), imported_object, @relation_hash).parse!
James Lopez's avatar
James Lopez committed
112
        else
113
          imported_object
James Lopez's avatar
James Lopez committed
114 115 116
        end
      end

117 118
      def update_project_references
        project_id = @relation_hash.delete('project_id')
119

120 121
        # If source and target are the same, populate them with the new project ID.
        if @relation_hash['source_project_id']
122
          @relation_hash['source_project_id'] = same_source_and_target? ? project_id : MergeRequestParser::FORKED_PROJECT_ID
123 124
        end

125
        # project_id may not be part of the export, but we always need to populate it if required.
126
        @relation_hash['project_id'] = project_id
127
        @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id']
128
      end
129

130 131
      def same_source_and_target?
        @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id']
132 133
      end

134 135 136 137 138 139 140
      def group_label?
        @relation_hash['type'] == 'GroupLabel'
      end

      def handle_group_label
        # If there's no group, move the label to a project label
        if @relation_hash['group_id']
141
          @relation_hash['project_id'] = nil
142 143 144 145 146 147
          @relation_name = :group_label
        else
          @relation_hash['type'] = 'ProjectLabel'
        end
      end

148
      def reset_tokens!
149
        return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name.to_s)
150

151
        # If we import/export a project to the same instance, tokens will have to be reset.
152 153 154 155
        # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across.
        relation_class.attribute_names.select { |name| name.include?('token') }.each do |token|
          @relation_hash[token] = nil
        end
156
      end
157

158
      def remove_encrypted_attributes!
James Lopez's avatar
James Lopez committed
159
        return unless relation_class.respond_to?(:encrypted_attributes) && relation_class.encrypted_attributes.any?
160 161 162 163 164 165

        relation_class.encrypted_attributes.each_key do |key|
          @relation_hash[key.to_s] = nil
        end
      end

166 167
      def relation_class
        @relation_class ||= @relation_name.to_s.classify.constantize
168
      end
James Lopez's avatar
James Lopez committed
169

170
      def imported_object
171 172
        yield(existing_or_new_object) if block_given?
        existing_or_new_object.importing = true if existing_or_new_object.respond_to?(:importing)
173

174 175 176 177 178 179
        existing_or_new_object
      rescue ActiveRecord::RecordNotUnique
        # as the operation is not atomic, retry in the unlikely scenario an INSERT is
        # performed on the same object between the SELECT and the INSERT
        @imported_object_retries += 1
        retry if @imported_object_retries < IMPORTED_OBJECT_MAX_RETRIES
James Lopez's avatar
James Lopez committed
180
      end
181 182 183 184 185 186 187

      def update_note_for_missing_author(author_name)
        @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank?
        @relation_hash['note'] += missing_author_note(@relation_hash['updated_at'], author_name)
      end

      def admin_user?
188
        @user.is_admin?
189
      end
190 191

      def parsed_relation_hash
192 193
        @parsed_relation_hash ||= Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: @relation_hash,
                                                                               relation_class: relation_class)
194
      end
195

196
      def set_st_diff_commits
197
        @relation_hash['st_diffs'] = @relation_hash.delete('utf8_st_diffs')
198 199 200

        HashUtil.deep_symbolize_array!(@relation_hash['st_diffs'])
        HashUtil.deep_symbolize_array_with_date!(@relation_hash['st_commits'])
201
      end
202 203 204 205 206 207

      def existing_or_new_object
        # Only find existing records to avoid mapping tables such as milestones
        # Otherwise always create the record, skipping the extra SELECT clause.
        @existing_or_new_object ||= begin
          if EXISTING_OBJECT_CHECK.include?(@relation_name)
208
            attribute_hash = attribute_hash_for(['events'])
209

210
            existing_object.assign_attributes(attribute_hash) if attribute_hash.any?
211

212 213 214 215 216 217
            existing_object
          else
            relation_class.new(parsed_relation_hash)
          end
        end
      end
218

219 220 221 222 223 224 225
      def attribute_hash_for(attributes)
        attributes.inject({}) do |hash, value|
          hash[value] = parsed_relation_hash.delete(value) if parsed_relation_hash[value]
          hash
        end
      end

226 227 228
      def existing_object
        @existing_object ||=
          begin
229 230
            existing_object = find_or_create_object!

231 232
            # Done in two steps, as MySQL behaves differently than PostgreSQL using
            # the +find_or_create_by+ method and does not return the ID the second time.
233
            existing_object.update!(parsed_relation_hash)
234 235 236
            existing_object
          end
      end
237 238 239 240 241

      def unknown_service?
        @relation_name == :services && parsed_relation_hash['type'] &&
          !Object.const_defined?(parsed_relation_hash['type'])
      end
242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260

      def find_or_create_object!
        finder_attributes = @relation_name == :group_label ? %w[title group_id] : %w[title project_id]
        finder_hash = parsed_relation_hash.slice(*finder_attributes)

        if label?
          label = relation_class.find_or_initialize_by(finder_hash)
          parsed_relation_hash.delete('priorities') if label.persisted?

          label.save!
          label
        else
          relation_class.find_or_create_by(finder_hash)
        end
      end

      def label?
        @relation_name.to_s.include?('label')
      end
261 262 263
    end
  end
end