Add CustomEmoji Banzai Filter

This MR adds custom emoji banzai filter and is
used in GFM pipeline. It adds rspec for the filter.

Fix perl backrefs cop

Add changelog for the change

Add query recorder spec, multiple emoji

Apply suggestions to filter

Change method body for some filter functions

Use common regex from custom emoji model

Move regex to constant in emoji model

Revert regex changes

Add two same custom emoji test

Reuse part of regex in banzai filter

Use filter in appropriate places

Fix pipeline spec failures

Check fallbackEmoji before setting fallback

RUn yarn prettier

Apply suggestions and add new regex

Escape hypen for regex to fix rubocop warning

Use custom emoji filter only in GFM pipeline

Use single quotes for regex

Use regexp object and move hyphen to end

Use regexp object

Use regexp object

Use freeze for regexp object

app/assets/javascripts/behaviors/gl_emoji.js

@@ -17,6 +17,10 @@ class GlEmoji extends HTMLElement {
 
         if (emojiInfo) {
           if (name !== emojiInfo.name) {
+            if (emojiInfo.fallback && this.innerHTML) {
+              return; // When fallback emoji is used, but there is a <img> provided, use the <img> instead
+            }
+
             ({ name } = emojiInfo);
             this.dataset.name = emojiInfo.name;
           }

app/assets/javascripts/emoji/index.js

@@ -181,6 +181,11 @@ export function searchEmoji(query, opts) {
   } = opts || {};
 
   const fallbackEmoji = emojiMap.grey_question;
+
+  if (fallbackEmoji) {
+    fallbackEmoji.fallback = true;
+  }
+
   if (!query) {
     if (fallback) {
       return raw ? [{ emoji: fallbackEmoji }] : [fallbackEmoji];

app/models/custom_emoji.rb

 # frozen_string_literal: true
 
 class CustomEmoji < ApplicationRecord
+  NAME_REGEXP = /[a-z0-9_-]+/.freeze
+
   belongs_to :namespace, inverse_of: :custom_emoji
 
   belongs_to :group, -> { where(type: 'Group') }, foreign_key: 'namespace_id'
@@ -17,7 +19,12 @@ class CustomEmoji < ApplicationRecord
     uniqueness: { scope: [:namespace_id, :name] },
     presence: true,
     length: { maximum: 36 },
-    format: { with: /\A[a-z0-9][a-z0-9\-_]*[a-z0-9]\z/ }
+
+    format: { with: /\A#{NAME_REGEXP}\z/ }
+
+  scope :by_name, -> (names) { where(name: names) }
+
+  alias_attribute :url, :file # this might need a change in https://gitlab.com/gitlab-org/gitlab/-/issues/230467
 
   private
 

changelogs/unreleased/custom-emoji-banzai-filter.yml

+---
+title: Add Banzai filter for CustomEmoji
+merge_request: 47122
+author: Rajendra Kadam
+type: added

lib/banzai/filter/custom_emoji_filter.rb

+# frozen_string_literal: true
+
+module Banzai
+  module Filter
+    class CustomEmojiFilter < HTML::Pipeline::Filter
+      IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set
+
+      def call
+        return doc unless context[:project]
+        return doc unless Feature.enabled?(:custom_emoji, context[:project])
+
+        doc.search(".//text()").each do |node|
+          content = node.to_html
+
+          next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS)
+          next unless content.include?(':')
+          next unless namespace && namespace.custom_emoji.any?
+
+          html = custom_emoji_name_element_filter(content)
+
+          node.replace(html) unless html == content
+        end
+
+        doc
+      end
+
+      def custom_emoji_pattern
+        @emoji_pattern ||=
+          /(?<=[^[:alnum:]:]|\n|^)
+          :(#{CustomEmoji::NAME_REGEXP}):
+          (?=[^[:alnum:]:]|$)/x
+      end
+
+      def custom_emoji_name_element_filter(text)
+        text.gsub(custom_emoji_pattern) do |match|
+          name = Regexp.last_match[1]
+          custom_emoji = all_custom_emoji[name]
+
+          if custom_emoji
+            Gitlab::Emoji.custom_emoji_tag(custom_emoji.name, custom_emoji.url)
+          else
+            match
+          end
+        end
+      end
+
+      private
+
+      def namespace
+        context[:project].namespace.root_ancestor
+      end
+
+      def custom_emoji_candidates
+        doc.to_html.scan(/:(#{CustomEmoji::NAME_REGEXP}):/).flatten
+      end
+
+      def all_custom_emoji
+        @all_custom_emoji ||= namespace.custom_emoji.by_name(custom_emoji_candidates).index_by(&:name)
+      end
+    end
+  end
+end

lib/banzai/pipeline/gfm_pipeline.rb

@@ -34,6 +34,7 @@ module Banzai
           Filter::FootnoteFilter,
           *reference_filters,
           Filter::EmojiFilter,
+          Filter::CustomEmojiFilter,
           Filter::TaskListFilter,
           Filter::InlineDiffFilter,
           Filter::SetDirectionFilter

lib/gitlab/emoji.rb

@@ -63,6 +63,16 @@ module Gitlab
       ActionController::Base.helpers.content_tag('gl-emoji', emoji_info['moji'], options)
     end
 
+    def custom_emoji_tag(name, image_source)
+      data = {
+        name: name
+      }
+
+      ActionController::Base.helpers.content_tag('gl-emoji', title: name, data: data) do
+        emoji_image_tag(name, image_source).html_safe
+      end
+    end
+
     private
 
     def emoji_unicode_versions_by_name

spec/lib/banzai/filter/custom_emoji_filter_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Banzai::Filter::CustomEmojiFilter do
+  include FilterSpecHelper
+
+  let_it_be(:group) { create(:group) }
+  let_it_be(:project) { create(:project, group: group) }
+  let_it_be(:custom_emoji) { create(:custom_emoji, name: 'tanuki', group: group) }
+  let_it_be(:custom_emoji2) { create(:custom_emoji, name: 'happy_tanuki', group: group, file: 'https://foo.bar/happy.png') }
+
+  it 'replaces supported name custom emoji' do
+    doc = filter('<p>:tanuki:</p>', project: project)
+
+    expect(doc.css('gl-emoji').first.attributes['title'].value).to eq('tanuki')
+    expect(doc.css('gl-emoji img').size).to eq 1
+  end
+
+  it 'ignores non existent custom emoji' do
+    exp = act = '<p>:foo:</p>'
+    doc = filter(act)
+
+    expect(doc.to_html).to match Regexp.escape(exp)
+  end
+
+  it 'correctly uses the custom emoji URL' do
+    doc = filter('<p>:tanuki:</p>')
+
+    expect(doc.css('img').first.attributes['src'].value).to eq(custom_emoji.file)
+  end
+
+  it 'matches with adjacent text' do
+    doc = filter('tanuki (:tanuki:)')
+
+    expect(doc.css('img').size).to eq 1
+  end
+
+  it 'matches multiple same custom emoji' do
+    doc = filter(':tanuki: :tanuki:')
+
+    expect(doc.css('img').size).to eq 2
+  end
+
+  it 'matches multiple custom emoji' do
+    doc = filter(':tanuki: (:happy_tanuki:)')
+
+    expect(doc.css('img').size).to eq 2
+  end
+
+  it 'does not match enclosed colons' do
+    doc = filter('tanuki:tanuki:')
+
+    expect(doc.css('img').size).to be 0
+  end
+
+  it 'keeps whitespace intact' do
+    doc = filter('This deserves a :tanuki:, big time.')
+
+    expect(doc.to_html).to match(/^This deserves a <gl-emoji.+>, big time\.\z/)
+  end
+
+  it 'does not match emoji in a string' do
+    doc = filter("'2a00:tanuki:100::1'")
+
+    expect(doc.css('gl-emoji').size).to eq 0
+  end
+
+  it 'does not do N+1 query' do
+    create(:custom_emoji, name: 'party-parrot', group: group)
+
+    control_count = ActiveRecord::QueryRecorder.new(skip_cached: false) do
+      filter('<p>:tanuki:</p>')
+    end
+
+    expect do
+      filter('<p>:tanuki: :party-parrot:</p>')
+    end.not_to exceed_all_query_limit(control_count.count)
+  end
+end

spec/lib/banzai/pipeline/broadcast_message_pipeline_spec.rb

@@ -3,11 +3,14 @@
 require 'spec_helper'
 
 RSpec.describe Banzai::Pipeline::BroadcastMessagePipeline do
+  let_it_be(:group) { create(:group) }
+  let_it_be(:project) { create(:project, group: group) }
+
   before do
     stub_commonmark_sourcepos_disabled
   end
 
-  subject { described_class.to_html(exp, project: spy) }
+  subject { described_class.to_html(exp, project: project) }
 
   context "allows `a` elements" do
     let(:exp) { "<a>Link</a>" }