Merge branch 'fix/import-encrypt-atts' into 'master'

Ignore encrypted attributes in Import/Export

Closes #24458

See merge request !8739

app/views/projects/edit.html.haml

@@ -183,6 +183,8 @@
             %li Build traces and artifacts
             %li LFS objects
             %li Container registry images
+            %li CI variables
+            %li Any encrypted tokens
   %hr
   - if can? current_user, :archive_project, @project
     .row.prepend-top-default

changelogs/unreleased/fix-import-encrypt-atts.yml

+---
+title: Ignore encrypted attributes in Import/Export
+merge_request: 
+author: 

doc/user/project/settings/import_export.md

@@ -22,7 +22,8 @@ with all their related data and be moved into a new GitLab instance.
 
 | GitLab version | Import/Export version |
 | -------- | -------- |
-| 8.13.0 to current  | 0.1.5    |
+| 8.16.2 to current  | 0.1.6    |
+| 8.13.0   | 0.1.5    |
 | 8.12.0   | 0.1.4    |
 | 8.10.3   | 0.1.3    |
 | 8.10.0   | 0.1.2    |
@@ -47,6 +48,9 @@ The following items will NOT be exported:
 
 - Build traces and artifacts
 - LFS objects
+- Container registry images
+- CI variables
+- Any encrypted tokens
 
 ## Exporting a project and its data
 

lib/gitlab/import_export.rb

@@ -3,7 +3,7 @@ module Gitlab
     extend self
 
     # For every version update, the version history in import_export.md has to be kept up to date.
-    VERSION = '0.1.5'
+    VERSION = '0.1.6'
     FILENAME_LIMIT = 50
 
     def export_path(relative_path:)

lib/gitlab/import_export/import_export.yml

@@ -39,7 +39,6 @@ project_tree:
       - :author
       - :events
     - :statuses
-  - :variables
   - :triggers
   - :deploy_keys
   - :services

lib/gitlab/import_export/relation_factory.rb

@@ -4,7 +4,6 @@ module Gitlab
       OVERRIDES = { snippets: :project_snippets,
                     pipelines: 'Ci::Pipeline',
                     statuses: 'commit_status',
-                    variables: 'Ci::Variable',
                     triggers: 'Ci::Trigger',
                     builds: 'Ci::Build',
                     hooks: 'ProjectHook',
@@ -24,6 +23,8 @@ module Gitlab
 
       EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels].freeze
 
+      TOKEN_RESET_MODELS = %w[Ci::Trigger Ci::Build ProjectHook].freeze
+
       def self.create(*args)
         new(*args).create
       end
@@ -61,7 +62,9 @@ module Gitlab
         update_project_references
 
         handle_group_label if group_label?
-        reset_ci_tokens if @relation_name == 'Ci::Trigger'
+        reset_tokens!
+        remove_encrypted_attributes!
+
         @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data']
         set_st_diffs if @relation_name == :merge_request_diff
       end
@@ -140,11 +143,22 @@ module Gitlab
         end
       end
 
-      def reset_ci_tokens
-        return unless Gitlab::ImportExport.reset_tokens?
+      def reset_tokens!
+        return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name.to_s)
 
         # If we import/export a project to the same instance, tokens will have to be reset.
-        @relation_hash['token'] = nil
+        # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across.
+        relation_class.attribute_names.select { |name| name.include?('token') }.each do |token|
+          @relation_hash[token] = nil
+        end
+      end
+
+      def remove_encrypted_attributes!
+        return unless relation_class.respond_to?(:encrypted_attributes) && relation_class.encrypted_attributes.any?
+
+        relation_class.encrypted_attributes.each_key do |key|
+          @relation_hash[key.to_s] = nil
+        end
       end
 
       def relation_class

spec/features/projects/import_export/export_file_spec.rb

@@ -74,6 +74,9 @@ feature 'Import/Export - project export integration test', feature: true, js: tr
         Otherwise, please add the exception to +safe_list+ in CURRENT_SPEC using #{sensitive_word} as the key and the
         correspondent hash or model as the value.
 
+        Also, if the attribute is a generated unique token, please add it to RelationFactory::TOKEN_RESET_MODELS if it needs to be
+        reset (to prevent duplicate column problems while importing to the same instance).
+
         IMPORT_EXPORT_CONFIG: #{Gitlab::ImportExport.config_file}
         CURRENT_SPEC: #{__FILE__}
       MSG

spec/lib/gitlab/import_export/project.json

@@ -6980,12 +6980,17 @@
         }
       ]
     }
-  ],
-  "variables": [
-
   ],
   "triggers": [
-
+    {
+      "id": 123,
+      "token": "cdbfasdf44a5958c83654733449e585",
+      "project_id": null,
+      "deleted_at": null,
+      "created_at": "2017-01-16T15:25:28.637Z",
+      "updated_at": "2017-01-16T15:25:28.637Z",
+      "gl_project_id": 123
+    }
   ],
   "deploy_keys": [
 

spec/lib/gitlab/import_export/project_tree_restorer_spec.rb

@@ -197,6 +197,20 @@ describe Gitlab::ImportExport::ProjectTreeRestorer, services: true do
           expect(restored_project_json).to be true
         end
       end
+
+      context 'tokens are regenerated' do
+        before do
+          restored_project_json
+        end
+
+        it 'has a new CI trigger token' do
+          expect(Ci::Trigger.where(token: 'cdbfasdf44a5958c83654733449e585')).to be_empty
+        end
+
+        it 'has a new CI build token' do
+          expect(Ci::Build.where(token: 'abcd')).to be_empty
+        end
+      end
     end
   end
 end

spec/lib/gitlab/import_export/relation_factory_spec.rb

@@ -55,8 +55,8 @@ describe Gitlab::ImportExport::RelationFactory, lib: true do
       expect(created_object.project_id).to eq(project.id)
     end
 
-    it 'has a token' do
-      expect(created_object.token).to eq(token)
+    it 'has a nil token' do
+      expect(created_object.token).to eq(nil)
     end
 
     context 'original service exists' do
@@ -178,4 +178,15 @@ describe Gitlab::ImportExport::RelationFactory, lib: true do
       expect(created_object.author).to eq(new_user)
     end
   end
+
+  context 'encrypted attributes' do
+    let(:relation_sym) { 'Ci::Variable' }
+    let(:relation_hash) do
+      create(:ci_variable).as_json
+    end
+
+    it 'has no value for the encrypted attribute' do
+      expect(created_object.value).to be_nil
+    end
+  end
 end