Merge branch '21801-migrate-epic-mentions-to-db-table' into 'master'

Migrate epic mentions to epic_user_mentions

Closes #21801

See merge request gitlab-org/gitlab!22333

changelogs/unreleased/21801-migrate-epic-and-epic-notes-mentions-to-epic-user-mentions-table.yml

+---
+title: Migrate epic, epic notes mentions to respective DB table
+merge_request: 22333
+author:
+type: changed

db/post_migrate/20191115115043_migrate_epic_mentions_to_db.rb

+# frozen_string_literal: true
+
+class MigrateEpicMentionsToDb < ActiveRecord::Migration[5.2]
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  DELAY = 2.minutes.to_i
+  BATCH_SIZE = 10000
+  MIGRATION = 'UserMentions::CreateResourceUserMention'
+
+  JOIN = "LEFT JOIN epic_user_mentions on epics.id = epic_user_mentions.epic_id"
+  QUERY_CONDITIONS = "(description like '%@%' OR title like '%@%') AND epic_user_mentions.epic_id is null"
+
+  class Epic < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'epics'
+  end
+
+  def up
+    return unless Gitlab.ee?
+
+    Epic
+      .joins(JOIN)
+      .where(QUERY_CONDITIONS)
+      .each_batch(of: BATCH_SIZE) do |batch, index|
+      range = batch.pluck(Arel.sql('MIN(epics.id)'), Arel.sql('MAX(epics.id)')).first
+      BackgroundMigrationWorker.perform_in(index * DELAY, MIGRATION, ['Epic', JOIN, QUERY_CONDITIONS, false, *range])
+    end
+  end
+
+  def down
+    # no-op
+  end
+end

db/post_migrate/20191115115522_migrate_epic_notes_mentions_to_db.rb

+# frozen_string_literal: true
+
+class MigrateEpicNotesMentionsToDb < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  DELAY = 2.minutes.to_i
+  BATCH_SIZE = 10000
+  MIGRATION = 'UserMentions::CreateResourceUserMention'
+
+  INDEX_NAME = 'epic_mentions_temp_index'
+  INDEX_CONDITION = "note LIKE '%@%'::text AND notes.noteable_type = 'Epic'"
+  QUERY_CONDITIONS = "#{INDEX_CONDITION} AND epic_user_mentions.epic_id IS NULL"
+  JOIN = 'LEFT JOIN epic_user_mentions ON notes.id = epic_user_mentions.note_id'
+
+  class Note < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'notes'
+  end
+
+  def up
+    return unless Gitlab.ee?
+
+    # create temporary index for notes with mentions, may take well over 1h
+    add_concurrent_index(:notes, :id, where: INDEX_CONDITION, name: INDEX_NAME)
+
+    Note
+      .joins(JOIN)
+      .where(QUERY_CONDITIONS)
+      .each_batch(of: BATCH_SIZE) do |batch, index|
+      range = batch.pluck(Arel.sql('MIN(notes.id)'), Arel.sql('MAX(notes.id)')).first
+      BackgroundMigrationWorker.perform_in(index * DELAY, MIGRATION, ['Epic', JOIN, QUERY_CONDITIONS, true, *range])
+    end
+  end
+
+  def down
+    # no-op
+    # temporary index is to be dropped in a different migration in an upcoming release:
+    # https://gitlab.com/gitlab-org/gitlab/issues/196842
+  end
+end

db/schema.rb

@@ -2778,6 +2778,7 @@ ActiveRecord::Schema.define(version: 2020_01_27_090233) do
     t.index ["commit_id"], name: "index_notes_on_commit_id"
     t.index ["created_at"], name: "index_notes_on_created_at"
     t.index ["discussion_id"], name: "index_notes_on_discussion_id"
+    t.index ["id"], name: "epic_mentions_temp_index", where: "((note ~~ '%@%'::text) AND ((noteable_type)::text = 'Epic'::text))"
     t.index ["line_code"], name: "index_notes_on_line_code"
     t.index ["note"], name: "index_notes_on_note_trigram", opclass: :gin_trgm_ops, using: :gin
     t.index ["noteable_id", "noteable_type"], name: "index_notes_on_noteable_id_and_noteable_type"

ee/spec/lib/gitlab/background_migration/user_mentions/create_resource_user_mention_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require './db/post_migrate/20191115115043_migrate_epic_mentions_to_db'
+require './db/post_migrate/20191115115522_migrate_epic_notes_mentions_to_db'
+
+describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMention do
+  include MigrationsHelpers
+
+  let(:users) { table(:users) }
+  let(:namespaces) { table(:namespaces) }
+  let(:epics) { table(:epics) }
+  let(:notes) { table(:notes) }
+  let(:epic_user_mentions) { table(:epic_user_mentions) }
+
+  let(:author) { users.create!(email: 'author@example.com', notification_email: 'author@example.com', name: 'author', username: 'author', projects_limit: 10, state: 'active') }
+  let(:member) { users.create!(email: 'member@example.com', notification_email: 'member@example.com', name: 'member', username: 'member', projects_limit: 10, state: 'active') }
+  let(:admin) { users.create!(email: 'administrator@example.com', notification_email: 'administrator@example.com', name: 'administrator', username: 'administrator', admin: 1, projects_limit: 10, state: 'active') }
+  let(:john_doe) { users.create!(email: 'john_doe@example.com', notification_email: 'john_doe@example.com', name: 'john_doe', username: 'john_doe', projects_limit: 10, state: 'active') }
+  let(:skipped) { users.create!(email: 'skipped@example.com', notification_email: 'skipped@example.com', name: 'skipped', username: 'skipped', projects_limit: 10, state: 'active') }
+
+  let(:mentioned_users) { [author, member, admin, john_doe, skipped] }
+  let(:user_mentions) { mentioned_users.map { |u| "@#{u.username}" }.join(' ') }
+
+  let(:group) { namespaces.create!(name: 'test1', path: 'test1', runners_token: 'my-token1', project_creation_level: 1, visibility_level: 20, type: 'Group') }
+  let(:inaccessible_group) { namespaces.create!(name: 'test2', path: 'test2', runners_token: 'my-token2', project_creation_level: 1, visibility_level: 0, type: 'Group') }
+
+  let(:mentioned_groups) { [group, inaccessible_group] }
+  let(:group_mentions) { [group, inaccessible_group].map { |gr| "@#{gr.path}" }.join(' ') }
+  let(:description_mentions) { "description with mentions #{user_mentions} and #{group_mentions}" }
+
+  before do
+    # build personal namespaces and routes for users
+    mentioned_users.each { |u| u.becomes(User).save! }
+
+    # build namespaces and routes for groups
+    mentioned_groups.each do |gr|
+      gr.name += '-org'
+      gr.path += '-org'
+      gr.becomes(Namespace).save!
+    end
+  end
+
+  context 'mentions in epic description' do
+    let(:epic) do
+      epics.create!(iid: 1, group_id: group.id, author_id: author.id, title: "epic title @#{author.username}",
+                    title_html: "epic title  @#{author.username}", description: description_mentions)
+    end
+
+    it 'migrates mentions' do
+      join = MigrateEpicMentionsToDb::JOIN
+      conditions = MigrateEpicMentionsToDb::QUERY_CONDITIONS
+
+      expect do
+        subject.perform('Epic', join, conditions, false, epic.id, epic.id)
+      end.to change { epic_user_mentions.count }.by(1)
+
+      epic_user_mention = epic_user_mentions.last
+      expect(epic_user_mention.mentioned_users_ids.sort).to eq(mentioned_users.pluck(:id).sort)
+      expect(epic_user_mention.mentioned_groups_ids.sort).to eq([group.id])
+      expect(epic_user_mention.mentioned_groups_ids.sort).not_to include(inaccessible_group.id)
+    end
+
+    context 'mentions in epic notes' do
+      let(:epic_note) { notes.create!(noteable_id: epic.id, noteable_type: 'Epic', author_id: author.id, note: description_mentions) }
+      let(:epic_note2) { notes.create!(noteable_id: epic.id, noteable_type: 'Epic', author_id: author.id, note: 'sample note') }
+      let(:system_epic_note) { notes.create!(noteable_id: epic.id, noteable_type: 'Epic', author_id: author.id, note: description_mentions, system: true) }
+
+      before do
+        epic_note.becomes(Note).save!
+        epic_note2.becomes(Note).save!
+        system_epic_note.becomes(Note).save!
+      end
+
+      it 'migrates mentions from note' do
+        join = MigrateEpicNotesMentionsToDb::JOIN
+        conditions = MigrateEpicNotesMentionsToDb::QUERY_CONDITIONS
+
+        expect do
+          subject.perform('Epic', join, conditions, true, Note.minimum(:id), Note.maximum(:id))
+        end.to change { epic_user_mentions.where(note_id: epic_note.id).count }.by(1)
+
+        # check that the epic_user_mention for regular note is created
+        epic_user_mention = epic_user_mentions.first
+        expect(epic_user_mention.becomes(EpicUserMention).note.system).to be false
+        expect(epic_user_mention.mentioned_users_ids.sort).to eq(users.pluck(:id).sort)
+        expect(epic_user_mention.mentioned_groups_ids.sort).to eq([group.id])
+        expect(epic_user_mention.mentioned_groups_ids.sort).not_to include(inaccessible_group.id)
+
+        # check that the epic_user_mention for system note is created
+        epic_user_mention = epic_user_mentions.second
+        expect(epic_user_mention.becomes(EpicUserMention).note.system).to be true
+        expect(epic_user_mention.mentioned_users_ids.sort).to eq(users.pluck(:id).sort)
+        expect(epic_user_mention.mentioned_groups_ids.sort).to eq([group.id])
+        expect(epic_user_mention.mentioned_groups_ids.sort).not_to include(inaccessible_group.id)
+      end
+    end
+  end
+end

ee/spec/migrations/migrate_epic_mentions_to_db_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20191115115043_migrate_epic_mentions_to_db')
+
+describe MigrateEpicMentionsToDb, :migration, :sidekiq do
+  let(:users) { table(:users) }
+  let(:namespaces) { table(:namespaces) }
+  let(:epics) { table(:epics) }
+
+  before do
+    stub_const("#{described_class.name}::BATCH_SIZE", 1)
+
+    users.create!(id: 1, name: 'root', email: 'root@example.com', username: 'root', projects_limit: 0)
+    namespaces.create!(id: 1, name: 'group1', path: 'group1', owner_id: 1)
+    epics.create!(id: 1, iid: 1, title: "title1", title_html: 'title1', description: 'epic description with @root mention', group_id: 1, author_id: 1)
+    epics.create!(id: 2, iid: 2, title: "title2", title_html: "title2", description: 'epic description with @group mention', group_id: 1, author_id: 1)
+    epics.create!(id: 3, iid: 3, title: "title3", title_html: "title3", description: 'epic description with @project mention', group_id: 1, author_id: 1)
+  end
+
+  it 'schedules epic mentions migrations' do
+    Sidekiq::Testing.fake! do
+      Timecop.freeze do
+        migrate!
+
+        migration = described_class::MIGRATION
+        join = described_class::JOIN
+        conditions = described_class::QUERY_CONDITIONS
+
+        expect(migration).to be_scheduled_delayed_migration(2.minutes, 'Epic', join, conditions, false, 1, 1)
+        expect(migration).to be_scheduled_delayed_migration(4.minutes, 'Epic', join, conditions, false, 2, 2)
+        expect(migration).to be_scheduled_delayed_migration(6.minutes, 'Epic', join, conditions, false, 3, 3)
+        expect(BackgroundMigrationWorker.jobs.size).to eq 3
+      end
+    end
+  end
+end

ee/spec/migrations/migrate_epic_notes_mentions_to_db_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20191115115522_migrate_epic_notes_mentions_to_db')
+
+describe MigrateEpicNotesMentionsToDb, :migration, :sidekiq do
+  let(:users) { table(:users) }
+  let(:namespaces) { table(:namespaces) }
+  let(:epics) { table(:epics) }
+  let(:notes) { table(:notes) }
+
+  before do
+    stub_const("#{described_class.name}::BATCH_SIZE", 1)
+
+    users.create!(id: 1, name: 'root', email: 'root@example.com', username: 'root', projects_limit: 0)
+    namespaces.create!(id: 1, name: 'group1', path: 'group1', owner_id: 1)
+    epics.create!(id: 1, iid: 1, title: "title", title_html: 'title', description: 'epic description', group_id: 1, author_id: 1)
+
+    notes.create!(note: 'note1 for @root to check', noteable_id: 1, noteable_type: 'Epic')
+    notes.create!(note: 'note2 for @root to check', noteable_id: 1, noteable_type: 'Epic', system: true)
+    notes.create!(note: 'note3 for @root to check', noteable_id: 1, noteable_type: 'Epic')
+  end
+
+  it 'schedules epic mentions migrations' do
+    Sidekiq::Testing.fake! do
+      Timecop.freeze do
+        migrate!
+
+        migration = described_class::MIGRATION
+        join = described_class::JOIN
+        conditions = described_class::QUERY_CONDITIONS
+
+        expect(migration).to be_scheduled_delayed_migration(2.minutes, 'Epic', join, conditions, true, 1, 1)
+        expect(migration).to be_scheduled_delayed_migration(4.minutes, 'Epic', join, conditions, true, 2, 2)
+        expect(migration).to be_scheduled_delayed_migration(6.minutes, 'Epic', join, conditions, true, 3, 3)
+        expect(BackgroundMigrationWorker.jobs.size).to eq 3
+      end
+    end
+  end
+end

lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb

+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    module UserMentions
+      class CreateResourceUserMention
+        # Resources that have mentions to be migrated:
+        # issue, merge_request, epic, commit, snippet, design
+
+        BULK_INSERT_SIZE = 5000
+        ISOLATION_MODULE = 'Gitlab::BackgroundMigration::UserMentions::Models'
+
+        def perform(resource_model, join, conditions, with_notes, start_id, end_id)
+          resource_model = "#{ISOLATION_MODULE}::#{resource_model}".constantize if resource_model.is_a?(String)
+          model = with_notes ? "#{ISOLATION_MODULE}::Note".constantize : resource_model
+          resource_user_mention_model = resource_model.user_mention_model
+
+          records = model.joins(join).where(conditions).where(id: start_id..end_id)
+
+          records.in_groups_of(BULK_INSERT_SIZE, false).each do |records|
+            mentions = []
+            records.each do |record|
+              mentions << record.build_mention_values
+            end
+
+            no_quote_columns = [:note_id]
+            no_quote_columns << resource_user_mention_model.resource_foreign_key
+
+            Gitlab::Database.bulk_insert(
+              resource_user_mention_model.table_name,
+              mentions,
+              return_ids: true,
+              disable_quote: no_quote_columns,
+              on_conflict: :do_nothing
+            )
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/background_migration/user_mentions/models/epic.rb

+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    module UserMentions
+      module Models
+        class Epic < ActiveRecord::Base
+          include IsolatedMentionable
+          include CacheMarkdownField
+
+          attr_mentionable :title, pipeline: :single_line
+          attr_mentionable :description
+          cache_markdown_field :title, pipeline: :single_line
+          cache_markdown_field :description, issuable_state_filter_enabled: true
+
+          self.table_name = 'epics'
+
+          belongs_to :author, class_name: "User"
+          belongs_to :project
+          belongs_to :group
+
+          def self.user_mention_model
+            Gitlab::BackgroundMigration::UserMentions::Models::EpicUserMention
+          end
+
+          def user_mention_model
+            self.class.user_mention_model
+          end
+
+          def project
+            nil
+          end
+
+          def mentionable_params
+            { group: group, label_url_method: :group_epics_url }
+          end
+
+          def user_mention_resource_id
+            id
+          end
+
+          def user_mention_note_id
+            'NULL'
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb

+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    module UserMentions
+      module Models
+        class EpicUserMention < ActiveRecord::Base
+          self.table_name = 'epic_user_mentions'
+
+          def self.resource_foreign_key
+            :epic_id
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/background_migration/user_mentions/models/isolated_mentionable.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module UserMentions
+      module Models
+        # == IsolatedMentionable concern
+        #
+        # Shortcutted for isolation version of Mentionable to be used in mentions migrations
+        #
+        module IsolatedMentionable
+          extend ::ActiveSupport::Concern
+
+          class_methods do
+            # Indicate which attributes of the Mentionable to search for GFM references.
+            def attr_mentionable(attr, options = {})
+              attr = attr.to_s
+              mentionable_attrs << [attr, options]
+            end
+          end
+
+          included do
+            # Accessor for attributes marked mentionable.
+            cattr_accessor :mentionable_attrs, instance_accessor: false do
+              []
+            end
+
+            if self < Participable
+              participant -> (user, ext) { all_references(user, extractor: ext) }
+            end
+          end
+
+          def all_references(current_user = nil, extractor: nil)
+            # Use custom extractor if it's passed in the function parameters.
+            if extractor
+              extractors[current_user] = extractor
+            else
+              extractor = extractors[current_user] ||= ::Gitlab::ReferenceExtractor.new(project, current_user)
+
+              extractor.reset_memoized_values
+            end
+
+            self.class.mentionable_attrs.each do |attr, options|
+              text    = __send__(attr) # rubocop:disable GitlabSecurity/PublicSend
+              options = options.merge(
+                cache_key: [self, attr],
+                author: author,
+                skip_project_check: skip_project_check?
+              ).merge(mentionable_params)
+
+              cached_html = self.try(:updated_cached_html_for, attr.to_sym)
+              options[:rendered] = cached_html if cached_html
+
+              extractor.analyze(text, options)
+            end
+
+            extractor
+          end
+
+          def extractors
+            @extractors ||= {}
+          end
+
+          def skip_project_check?
+            false
+          end
+
+          def build_mention_values
+            refs = all_references(author)
+
+            {
+              "#{self.user_mention_model.resource_foreign_key}": user_mention_resource_id,
+              note_id: user_mention_note_id,
+              mentioned_users_ids: array_to_sql(refs.mentioned_users.pluck(:id)),
+              mentioned_projects_ids: array_to_sql(refs.mentioned_projects.pluck(:id)),
+              mentioned_groups_ids: array_to_sql(refs.mentioned_groups.pluck(:id))
+            }
+          end
+
+          def array_to_sql(ids_array)
+            return unless ids_array.present?
+
+            '{' + ids_array.join(", ") + '}'
+          end
+
+          private
+
+          def mentionable_params
+            {}
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/background_migration/user_mentions/models/note.rb

+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    module UserMentions
+      module Models
+        class Note < ActiveRecord::Base
+          include IsolatedMentionable
+          include CacheMarkdownField
+
+          self.table_name = 'notes'
+          self.inheritance_column = :_type_disabled
+
+          attr_mentionable :note, pipeline: :note
+          cache_markdown_field :note, pipeline: :note, issuable_state_filter_enabled: true
+
+          belongs_to :author, class_name: "User"
+          belongs_to :noteable, polymorphic: true
+          belongs_to :project
+
+          def user_mention_model
+            "#{CreateResourceUserMention::ISOLATION_MODULE}::#{noteable.class}".constantize.user_mention_model
+          end
+
+          def for_personal_snippet?
+            noteable.class.name == 'PersonalSnippet'
+          end
+
+          def for_project_noteable?
+            !for_personal_snippet?
+          end
+
+          def skip_project_check?
+            !for_project_noteable?
+          end
+
+          def for_epic?
+            noteable.class.name == 'Epic'
+          end
+
+          def user_mention_resource_id
+            noteable_id || commit_id
+          end
+
+          def user_mention_note_id
+            id
+          end
+
+          private
+
+          def mentionable_params
+            return super unless for_epic?
+
+            super.merge(banzai_context_params)
+          end
+
+          def banzai_context_params
+            { group: noteable.group, label_url_method: :group_epics_url }
+          end
+        end
+      end
+    end
+  end
+end

spec/support/shared_examples/models/mentionable_shared_examples.rb

@@ -229,16 +229,17 @@ RSpec.shared_examples 'mentions in description' do |mentionable_type|
 
     context 'when mentionable description contains mentions' do
       let(:user) { create(:user) }
+      let(:user2) { create(:user) }
       let(:group) { create(:group) }
 
-      let(:mentionable_desc) { "#{user.to_reference} some description #{group.to_reference(full: true)} and @all" }
+      let(:mentionable_desc) { "#{user.to_reference} #{user2.to_reference} #{user.to_reference} some description #{group.to_reference(full: true)} and #{user2.to_reference} @all" }
       let(:mentionable) { create(mentionable_type, description: mentionable_desc) }
 
       it 'stores mentions' do
         add_member(user)
 
         expect(mentionable.user_mentions.count).to eq 1
-        expect(mentionable.referenced_users).to match_array([user])
+        expect(mentionable.referenced_users).to match_array([user, user2])
         expect(mentionable.referenced_projects(user)).to match_array([mentionable.project].compact) # epic.project is nil, and we want empty []
         expect(mentionable.referenced_groups(user)).to match_array([group])
       end
@@ -249,8 +250,9 @@ end
 RSpec.shared_examples 'mentions in notes' do |mentionable_type|
   context 'when mentionable notes contain mentions' do
     let(:user) { create(:user) }
+    let(:user2) { create(:user) }
     let(:group) { create(:group) }
-    let(:note_desc) { "#{user.to_reference} and #{group.to_reference(full: true)} and @all" }
+    let(:note_desc) { "#{user.to_reference} #{user2.to_reference} #{user.to_reference} and #{group.to_reference(full: true)} and #{user2.to_reference} @all" }
     let!(:mentionable) { note.noteable }
 
     before do
@@ -261,7 +263,7 @@ RSpec.shared_examples 'mentions in notes' do |mentionable_type|
 
     it 'returns all mentionable mentions' do
       expect(mentionable.user_mentions.count).to eq 1
-      expect(mentionable.referenced_users).to eq [user]
+      expect(mentionable.referenced_users).to eq [user, user2]
       expect(mentionable.referenced_projects(user)).to eq [mentionable.project].compact # epic.project is nil, and we want empty []
       expect(mentionable.referenced_groups(user)).to eq [group]
     end