Merge branch 'migrate-all-merge-request-user-mentions' into 'master'

Migrate all merge request user mentions to db table

See merge request gitlab-org/gitlab!34378

changelogs/unreleased/migrate-all-merge-request-user-mentions.yml

+---
+title: Store user mentions from merge request title or description in the DB
+merge_request: 34378
+author:
+type: changed

db/post_migrate/20200601120434_migrate_all_merge_request_user_mentions_to_db.rb

+# frozen_string_literal: true
+
+class MigrateAllMergeRequestUserMentionsToDb < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  DELAY = 2.minutes.to_i
+  BATCH_SIZE = 100_000
+  MIGRATION = 'UserMentions::CreateResourceUserMention'
+
+  JOIN = "LEFT JOIN merge_request_user_mentions on merge_requests.id = merge_request_user_mentions.merge_request_id"
+  QUERY_CONDITIONS = "(description LIKE '%@%' OR title LIKE '%@%') AND merge_request_user_mentions.merge_request_id IS NULL"
+
+  disable_ddl_transaction!
+
+  class MergeRequest < ActiveRecord::Base
+    include EachBatch
+  end
+
+  def up
+    delay = DELAY
+
+    MergeRequest.each_batch(of: BATCH_SIZE) do |batch, _|
+      range = batch.pluck('MIN(merge_requests.id)', 'MAX(merge_requests.id)').first
+      records_count = MergeRequest.joins(JOIN).where(QUERY_CONDITIONS).where(id: range.first..range.last).count
+
+      if records_count > 0
+        migrate_in(delay, MIGRATION, ['MergeRequest', JOIN, QUERY_CONDITIONS, false, *range])
+        delay += [DELAY, (records_count / 500 + 1).minutes.to_i].max
+      end
+    end
+  end
+
+  def down
+    # no-op
+  end
+end

db/structure.sql

@@ -23937,6 +23937,7 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200528123703
 20200528125905
 20200528171933
+20200601120434
 20200601210148
 20200602013900
 20200602013901

ee/spec/lib/gitlab/background_migration/user_mentions/create_resource_user_mention_spec.rb

@@ -68,6 +68,14 @@ RSpec.describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMent
 
         it_behaves_like 'resource mentions migration', MigrateEpicMentionsToDb, Epic
 
+        context 'when FF disabled' do
+          before do
+            stub_feature_flags(migrate_user_mentions: false)
+          end
+
+          it_behaves_like 'resource migration not run', MigrateEpicMentionsToDb, Epic
+        end
+
         context 'mentions in epic notes' do
           let!(:note1) { notes.create!(noteable_id: epic.id, noteable_type: 'Epic', author_id: author.id, note: description_mentions) }
           let!(:note2) { notes.create!(noteable_id: epic.id, noteable_type: 'Epic', author_id: author.id, note: 'sample note') }
@@ -78,6 +86,14 @@ RSpec.describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMent
           let!(:note5) { notes.create!(noteable_id: non_existing_record_id, noteable_type: 'Epic', author_id: author.id, note: description_mentions, project_id: project.id) }
 
           it_behaves_like 'resource notes mentions migration', MigrateEpicNotesMentionsToDb, Epic
+
+          context 'when FF disabled' do
+            before do
+              stub_feature_flags(migrate_user_mentions: false)
+            end
+
+            it_behaves_like 'resource notes migration not run', MigrateEpicNotesMentionsToDb, Epic
+          end
         end
       end
     end
@@ -102,6 +118,14 @@ RSpec.describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMent
       let(:resource) { design }
 
       it_behaves_like 'resource notes mentions migration', MigrateDesignNotesMentionsToDb, DesignManagement::Design
+
+      context 'when FF disabled' do
+        before do
+          stub_feature_flags(migrate_user_mentions: false)
+        end
+
+        it_behaves_like 'resource notes migration not run', MigrateDesignNotesMentionsToDb, DesignManagement::Design
+      end
     end
   end
 

lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb

@@ -12,26 +12,22 @@ module Gitlab
         ISOLATION_MODULE = 'Gitlab::BackgroundMigration::UserMentions::Models'
 
         def perform(resource_model, join, conditions, with_notes, start_id, end_id)
+          return unless Feature.enabled?(:migrate_user_mentions, default_enabled: true)
+
           resource_model = "#{ISOLATION_MODULE}::#{resource_model}".constantize if resource_model.is_a?(String)
           model = with_notes ? Gitlab::BackgroundMigration::UserMentions::Models::Note : resource_model
           resource_user_mention_model = resource_model.user_mention_model
 
           records = model.joins(join).where(conditions).where(id: start_id..end_id)
 
-          records.in_groups_of(BULK_INSERT_SIZE, false).each do |records|
+          records.each_batch(of: BULK_INSERT_SIZE) do |records|
             mentions = []
             records.each do |record|
               mention_record = record.build_mention_values(resource_user_mention_model.resource_foreign_key)
               mentions << mention_record unless mention_record.blank?
             end
 
-            Gitlab::Database.bulk_insert( # rubocop:disable Gitlab/BulkInsert
-              resource_user_mention_model.table_name,
-              mentions,
-              return_ids: true,
-              disable_quote: resource_model.no_quote_columns,
-              on_conflict: :do_nothing
-            )
+            resource_user_mention_model.insert_all(mentions) unless mentions.empty?
           end
         end
       end

lib/gitlab/background_migration/user_mentions/models/commit.rb

@@ -6,6 +6,7 @@ module Gitlab
     module UserMentions
       module Models
         class Commit
+          include EachBatch
           include Concerns::IsolatedMentionable
           include Concerns::MentionableMigrationMethods
 

lib/gitlab/background_migration/user_mentions/models/design_management/design.rb

@@ -7,6 +7,7 @@ module Gitlab
       module Models
         module DesignManagement
           class Design < ActiveRecord::Base
+            include EachBatch
             include Concerns::MentionableMigrationMethods
 
             def self.user_mention_model

lib/gitlab/background_migration/user_mentions/models/epic.rb

@@ -6,6 +6,7 @@ module Gitlab
     module UserMentions
       module Models
         class Epic < ActiveRecord::Base
+          include EachBatch
           include Concerns::IsolatedMentionable
           include Concerns::MentionableMigrationMethods
           include CacheMarkdownField

lib/gitlab/background_migration/user_mentions/models/merge_request.rb

@@ -6,6 +6,7 @@ module Gitlab
     module UserMentions
       module Models
         class MergeRequest < ActiveRecord::Base
+          include EachBatch
           include Concerns::IsolatedMentionable
           include CacheMarkdownField
           include Concerns::MentionableMigrationMethods

lib/gitlab/background_migration/user_mentions/models/note.rb

@@ -6,6 +6,7 @@ module Gitlab
     module UserMentions
       module Models
         class Note < ActiveRecord::Base
+          include EachBatch
           include Concerns::IsolatedMentionable
           include CacheMarkdownField
 

spec/lib/gitlab/background_migration/user_mentions/create_resource_user_mention_spec.rb

@@ -75,6 +75,14 @@ RSpec.describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMent
       let(:resource) { merge_request }
 
       it_behaves_like 'resource mentions migration', MigrateMergeRequestMentionsToDb, MergeRequest
+
+      context 'when FF disabled' do
+        before do
+          stub_feature_flags(migrate_user_mentions: false)
+        end
+
+        it_behaves_like 'resource migration not run', MigrateMergeRequestMentionsToDb, MergeRequest
+      end
     end
 
     context 'migrate commit mentions' do
@@ -96,6 +104,14 @@ RSpec.describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMent
       let(:resource) { commit }
 
       it_behaves_like 'resource notes mentions migration', MigrateCommitNotesMentionsToDb, Commit
+
+      context 'when FF disabled' do
+        before do
+          stub_feature_flags(migrate_user_mentions: false)
+        end
+
+        it_behaves_like 'resource notes migration not run', MigrateCommitNotesMentionsToDb, Commit
+      end
     end
   end
 

spec/migrations/migrate_all_merge_request_user_mentions_to_db_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20200601120434_migrate_all_merge_request_user_mentions_to_db')
+
+RSpec.describe MigrateAllMergeRequestUserMentionsToDb, :migration do
+  let(:users) { table(:users) }
+  let(:projects) { table(:projects) }
+  let(:namespaces) { table(:namespaces) }
+  let(:merge_requests) { table(:merge_requests) }
+  let(:merge_request_user_mentions) { table(:merge_request_user_mentions) }
+
+  let(:user) { users.create!(name: 'root', email: 'root@example.com', username: 'root', projects_limit: 0) }
+  let(:group) { namespaces.create!(name: 'group1', path: 'group1', owner_id: user.id, type: 'Group') }
+  let(:project) { projects.create!(name: 'gitlab1', path: 'gitlab1', namespace_id: group.id, visibility_level: 0) }
+
+  let(:opened_state) { 1 }
+  let(:closed_state) { 2 }
+  let(:merged_state) { 3 }
+
+  # migrateable resources
+  let(:common_args) { { source_branch: 'master', source_project_id: project.id, target_project_id: project.id, author_id: user.id, description: 'mr description with @root mention' } }
+  let!(:resource1) { merge_requests.create!(common_args.merge(title: "title 1", state_id: opened_state, target_branch: 'feature1')) }
+  let!(:resource2) { merge_requests.create!(common_args.merge(title: "title 2", state_id: closed_state, target_branch: 'feature2')) }
+  let!(:resource3) { merge_requests.create!(common_args.merge(title: "title 3", state_id: merged_state, target_branch: 'feature3')) }
+
+  # non-migrateable resources
+  # this merge request is already migrated, as it has a record in the merge_request_user_mentions table
+  let!(:resource4) { merge_requests.create!(common_args.merge(title: "title 3", state_id: opened_state, target_branch: 'feature4')) }
+  let!(:user_mention) { merge_request_user_mentions.create!(merge_request_id: resource4.id, mentioned_users_ids: [1]) }
+
+  let!(:resource5) { merge_requests.create!(common_args.merge(title: "title 3", description: 'description with no mention', state_id: opened_state, target_branch: 'feature5')) }
+
+  it_behaves_like 'schedules resource mentions migration', MergeRequest, false
+end

spec/support/shared_examples/lib/gitlab/background_migration/mentions_migration_shared_examples.rb

@@ -82,3 +82,25 @@ RSpec.shared_examples 'schedules resource mentions migration' do |resource_class
     end
   end
 end
+
+RSpec.shared_examples 'resource migration not run' do |migration_class, resource_class|
+  it 'does not migrate mentions' do
+    join = migration_class::JOIN
+    conditions = migration_class::QUERY_CONDITIONS
+
+    expect do
+      subject.perform(resource_class.name, join, conditions, false, resource_class.minimum(:id), resource_class.maximum(:id))
+    end.to change { user_mentions.count }.by(0)
+  end
+end
+
+RSpec.shared_examples 'resource notes migration not run' do |migration_class, resource_class|
+  it 'does not migrate mentions' do
+    join = migration_class::JOIN
+    conditions = migration_class::QUERY_CONDITIONS
+
+    expect do
+      subject.perform(resource_class.name, join, conditions, true, Note.minimum(:id), Note.maximum(:id))
+    end.to change { user_mentions.count }.by(0)
+  end
+end