Merge branch 'fix/import-export-missing-attributes' into 'master'

Include missing project attributes to Import/Export

Closes gitlab-ee#2251 and #26333

See merge request !10880

app/models/project.rb

@@ -1271,6 +1271,9 @@ class Project < ActiveRecord::Base
     else
       update_attribute(name, value)
     end
+
+  rescue ActiveRecord::RecordNotSaved => e
+    handle_update_attribute_error(e, value)
   end
 
   def pushes_since_gc
@@ -1392,4 +1395,16 @@ class Project < ActiveRecord::Base
 
     ContainerRepository.build_root_repository(self).has_tags?
   end
+
+  def handle_update_attribute_error(ex, value)
+    if ex.message.start_with?('Failed to replace')
+      if value.respond_to?(:each)
+        invalid = value.detect(&:invalid?)
+
+        raise ex, ([ex.message] + invalid.errors.full_messages).join(' ') if invalid
+      end
+    end
+
+    raise ex
+  end
 end

changelogs/unreleased/fix-import-export-missing-attributes.yml

+---
+title: Add missing project attributes to Import/Export
+merge_request:
+author:

lib/gitlab/import_export/import_export.yml

@@ -41,7 +41,6 @@ project_tree:
     - :statuses
   - triggers:
     - :trigger_schedule
-  - :deploy_keys
   - :services
   - :hooks
   - protected_branches:
@@ -53,10 +52,6 @@ project_tree:
 
 # Only include the following attributes for the models specified.
 included_attributes:
-  project:
-    - :description
-    - :visibility_level
-    - :archived
   user:
     - :id
     - :email
@@ -66,6 +61,29 @@ included_attributes:
 
 # Do not include the following attributes for the models specified.
 excluded_attributes:
+  project:
+    - :name
+    - :path
+    - :namespace_id
+    - :creator_id
+    - :import_url
+    - :import_status
+    - :avatar
+    - :import_type
+    - :import_source
+    - :import_error
+    - :mirror
+    - :runners_token
+    - :repository_storage
+    - :repository_read_only
+    - :lfs_enabled
+    - :import_jid
+    - :created_at
+    - :updated_at
+    - :import_jid
+    - :import_jid
+    - :id
+    - :star_count
   snippets:
     - :expired_at
   merge_request_diff:
@@ -94,3 +112,5 @@ methods:
     - :utf8_st_diffs
   merge_requests:
     - :diff_head_sha
+  project:
+    - :description_html
\ No newline at end of file

lib/gitlab/import_export/project_tree_restorer.rb

@@ -71,14 +71,14 @@ module Gitlab
       def restore_project
         return @project unless @tree_hash
 
-        @project.update(project_params)
+        @project.update_columns(project_params)
         @project
       end
 
       def project_params
         @tree_hash.reject do |key, value|
           # return params that are not 1 to many or 1 to 1 relations
-          value.is_a?(Array) || key == key.singularize
+          value.respond_to?(:each) && !Project.column_names.include?(key)
         end
       end
 

lib/gitlab/import_export/reader.rb

@@ -15,7 +15,10 @@ module Gitlab
       # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html
       # for outputting a project in JSON format, including its relations and sub relations.
       def project_tree
-        @attributes_finder.find_included(:project).merge(include: build_hash(@tree))
+        attributes = @attributes_finder.find(:project)
+        project_attributes = attributes.is_a?(Hash) ? attributes[:project] : {}
+
+        project_attributes.merge(include: build_hash(@tree))
       rescue => e
         @shared.error(e)
         false

spec/lib/gitlab/import_export/fork_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 describe 'forked project import', services: true do
   let(:user) { create(:user) }
   let!(:project_with_repo) { create(:project, :test_repo, name: 'test-repo-restorer', path: 'test-repo-restorer') }
-  let!(:project) { create(:empty_project) }
+  let!(:project) { create(:empty_project, name: 'test-repo-restorer-no-repo', path: 'test-repo-restorer-no-repo') }
   let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
   let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.path_with_namespace) }
   let(:forked_from_project) { create(:project) }

spec/lib/gitlab/import_export/project.json

@@ -2,6 +2,7 @@
   "description": "Nisi et repellendus ut enim quo accusamus vel magnam.",
   "visibility_level": 10,
   "archived": false,
+  "description_html": "description",
   "labels": [
     {
       "id": 2,

spec/lib/gitlab/import_export/project_tree_restorer_spec.rb

@@ -30,6 +30,10 @@ describe Gitlab::ImportExport::ProjectTreeRestorer, services: true do
         expect(project.project_feature.merge_requests_access_level).to eq(ProjectFeature::ENABLED)
       end
 
+      it 'has the project html description' do
+        expect(Project.find_by_path('project').description_html).to eq('description')
+      end
+
       it 'has the same label associated to two issues' do
         expect(ProjectLabel.find_by_title('test2').issues.count).to eq(2)
       end

spec/lib/gitlab/import_export/project_tree_saver_spec.rb

@@ -189,6 +189,16 @@ describe Gitlab::ImportExport::ProjectTreeSaver, services: true do
           end
         end
       end
+
+      context 'project attributes' do
+        it 'contains the html description' do
+          expect(saved_project_json).to include("description_html" => 'description')
+        end
+
+        it 'does not contain the runners token' do
+          expect(saved_project_json).not_to include("runners_token" => 'token')
+        end
+      end
     end
   end
 
@@ -209,6 +219,7 @@ describe Gitlab::ImportExport::ProjectTreeSaver, services: true do
                      releases: [release],
                      group: group
                     )
+    project.update(description_html: 'description')
     project_label = create(:label, project: project)
     group_label = create(:group_label, group: group)
     create(:label_link, label: project_label, target: issue)

spec/lib/gitlab/import_export/reader_spec.rb

@@ -5,7 +5,7 @@ describe Gitlab::ImportExport::Reader, lib: true  do
   let(:test_config) { 'spec/support/import_export/import_export.yml' }
   let(:project_tree_hash) do
     {
-      only: [:name, :path],
+      except: [:id, :created_at],
       include: [:issues, :labels,
                 { merge_requests: {
                   only: [:id],

spec/lib/gitlab/import_export/safe_model_attributes.yml

@@ -329,6 +329,28 @@ Project:
 - snippets_enabled
 - visibility_level
 - archived
+- created_at
+- updated_at
+- last_activity_at
+- star_count
+- ci_id
+- shared_runners_enabled
+- build_coverage_regex
+- build_allow_git_fetchs
+- build_timeout
+- pending_delete
+- public_builds
+- last_repository_check_failed
+- last_repository_check_at
+- container_registry_enabled
+- only_allow_merge_if_pipeline_succeeds
+- has_external_issue_tracker
+- request_access_enabled
+- has_external_wiki
+- only_allow_merge_if_all_discussions_are_resolved
+- auto_cancel_pending_pipelines
+- printing_merge_request_link_enabled
+- build_allow_git_fetch
 Author:
 - name
 ProjectFeature:

spec/models/project_spec.rb

@@ -1906,4 +1906,23 @@ describe Project, models: true do
       expect(project.pipeline_status).to be_loaded
     end
   end
+
+  describe '#append_or_update_attribute' do
+    let(:project) { create(:project) }
+
+    it 'shows full error updating an invalid MR' do
+      error_message = 'Failed to replace merge_requests because one or more of the new records could not be saved.'\
+                      ' Validate fork Source project is not a fork of the target project'
+
+      expect { project.append_or_update_attribute(:merge_requests, [create(:merge_request)]) }.
+        to raise_error(ActiveRecord::RecordNotSaved, error_message)
+    end
+
+    it 'updates the project succesfully' do
+      merge_request = create(:merge_request, target_project: project, source_project: project)
+
+      expect { project.append_or_update_attribute(:merge_requests, [merge_request]) }.
+        not_to raise_error
+    end
+  end
 end

spec/support/import_export/import_export.yml

@@ -11,9 +11,6 @@ project_tree:
     - :user
 
 included_attributes:
-  project:
-    - :name
-    - :path
   merge_requests:
     - :id
   user:
@@ -21,4 +18,7 @@ included_attributes:
 
 excluded_attributes:
   merge_requests:
-    - :iid
+    - :iid
\ No newline at end of file
+  project:
+    - :id
+    - :created_at
\ No newline at end of file