Merge remote-tracking branch 'origin/master' into better-asana-refs

* origin/master:
  Use lazy reference extractor to get issue's MRs
  Banzai::XFilter -> Banzai::Filter::XFilter
  Move Markdown/reference logic from Gitlab::Markdown to Banzai

app/helpers/gitlab_markdown_helper.rb

@@ -20,7 +20,7 @@ module GitlabMarkdownHelper
                    end
 
     user = current_user if defined?(current_user)
-    gfm_body = Gitlab::Markdown.render(escaped_body, project: @project, current_user: user, pipeline: :single_line)
+    gfm_body = Banzai.render(escaped_body, project: @project, current_user: user, pipeline: :single_line)
 
     fragment = Nokogiri::HTML::DocumentFragment.parse(gfm_body)
     if fragment.children.size == 1 && fragment.children[0].name == 'a'
@@ -50,7 +50,7 @@ module GitlabMarkdownHelper
 
     context[:project] ||= @project
 
-    html = Gitlab::Markdown.render(text, context)
+    html = Banzai.render(text, context)
 
     context.merge!(
       current_user:   (current_user if defined?(current_user)),
@@ -61,7 +61,7 @@ module GitlabMarkdownHelper
       ref:            @ref
     )
 
-    Gitlab::Markdown.post_process(html, context)
+    Banzai.post_process(html, context)
   end
 
   def asciidoc(text)

app/helpers/issues_helper.rb

@@ -121,6 +121,6 @@ module IssuesHelper
     end
   end
 
-  # Required for Gitlab::Markdown::IssueReferenceFilter
+  # Required for Banzai::Filter::IssueReferenceFilter
   module_function :url_for_issue
 end

app/helpers/labels_helper.rb

@@ -107,6 +107,6 @@ module LabelsHelper
     options_from_collection_for_select(grouped_labels, 'name', 'title', params[:label_name])
   end
 
-  # Required for Gitlab::Markdown::LabelReferenceFilter
+  # Required for Banzai::Filter::LabelReferenceFilter
   module_function :render_colored_label, :text_color_for_bg, :escape_once
 end

app/models/concerns/mentionable.rb

@@ -23,7 +23,7 @@ module Mentionable
 
   included do
     if self < Participable
-      participant ->(current_user) { mentioned_users(current_user, load_lazy_references: false) }
+      participant ->(current_user) { mentioned_users(current_user) }
     end
   end
 
@@ -43,8 +43,8 @@ module Mentionable
     self
   end
 
-  def all_references(current_user = self.author, text = nil, load_lazy_references: true)
-    ext = Gitlab::ReferenceExtractor.new(self.project, current_user, load_lazy_references: load_lazy_references)
+  def all_references(current_user = self.author, text = nil)
+    ext = Gitlab::ReferenceExtractor.new(self.project, current_user)
 
     if text
       ext.analyze(text)
@@ -59,13 +59,13 @@ module Mentionable
     ext
   end
 
-  def mentioned_users(current_user = nil, load_lazy_references: true)
-    all_references(current_user, load_lazy_references: load_lazy_references).users
+  def mentioned_users(current_user = nil)
+    all_references(current_user).users
   end
 
   # Extract GFM references to other Mentionables from this Mentionable. Always excludes its #local_reference.
-  def referenced_mentionables(current_user = self.author, text = nil, load_lazy_references: true)
-    refs = all_references(current_user, text, load_lazy_references: load_lazy_references)
+  def referenced_mentionables(current_user = self.author, text = nil)
+    refs = all_references(current_user, text)
     refs = (refs.issues + refs.merge_requests + refs.commits)
 
     # We're using this method instead of Array diffing because that requires

app/models/concerns/participable.rb

@@ -38,7 +38,9 @@ module Participable
   # Be aware that this method makes a lot of sql queries.
   # Save result into variable if you are going to reuse it inside same request
   def participants(current_user = self.author, load_lazy_references: true)
-    participants = self.class.participant_attrs.flat_map do |attr|
+    participants =
+      Gitlab::ReferenceExtractor.lazily do
+        self.class.participant_attrs.flat_map do |attr|
           value =
             if attr.respond_to?(:call)
               instance_exec(current_user, &attr)
@@ -48,10 +50,9 @@ module Participable
 
           participants_for(value, current_user)
         end.compact.uniq
+      end
 
-    if load_lazy_references
-      participants = Gitlab::Markdown::ReferenceFilter::LazyReference.load(participants).uniq
-
+    unless Gitlab::ReferenceExtractor.lazy?
       participants.select! do |user|
         user.can?(:read_project, project)
       end
@@ -64,12 +65,12 @@ module Participable
 
   def participants_for(value, current_user = nil)
     case value
-    when User, Gitlab::Markdown::ReferenceFilter::LazyReference
+    when User, Banzai::LazyReference
       [value]
     when Enumerable, ActiveRecord::Relation
       value.flat_map { |v| participants_for(v, current_user) }
     when Participable
-      value.participants(current_user, load_lazy_references: false)
+      value.participants(current_user)
     end
   end
 end

app/models/issue.rb

@@ -84,11 +84,11 @@ class Issue < ActiveRecord::Base
   end
 
   def referenced_merge_requests
-    references = [self, *notes].flat_map do |note|
+    Gitlab::ReferenceExtractor.lazily do
+      [self, *notes].flat_map do |note|
         note.all_references(load_lazy_references: false).merge_requests
-    end.uniq
-
-    Gitlab::Markdown::ReferenceFilter::LazyReference.load(references).uniq.sort_by(&:iid)
+      end
+    end.sort_by(&:iid)
   end
 
   # Reset issue events cache

app/models/note.rb

@@ -373,11 +373,11 @@ class Note < ActiveRecord::Base
   end
 
   def contains_emoji_only?
-    note =~ /\A#{Gitlab::Markdown::EmojiFilter.emoji_pattern}\s?\Z/
+    note =~ /\A#{Banzai::Filter::EmojiFilter.emoji_pattern}\s?\Z/
   end
 
   def award_emoji_name
-    original_name = note.match(Gitlab::Markdown::EmojiFilter.emoji_pattern)[1]
+    original_name = note.match(Banzai::Filter::EmojiFilter.emoji_pattern)[1]
     AwardEmoji.normilize_emoji_name(original_name)
   end
 end

lib/banzai.rb

+module Banzai
+  def self.render(text, context = {})
+    Renderer.render(text, context)
+  end
+
+  def self.render_result(text, context = {})
+    Renderer.render_result(text, context)
+  end
+
+  def self.post_process(html, context)
+    Renderer.post_process(html, context)
+  end
+end

lib/banzai/cross_project_reference.rb

+require 'banzai'
+
+module Banzai
+  # Common methods for ReferenceFilters that support an optional cross-project
+  # reference.
+  module CrossProjectReference
+    # Given a cross-project reference string, get the Project record
+    #
+    # Defaults to value of `context[:project]` if:
+    # * No reference is given OR
+    # * Reference given doesn't exist
+    #
+    # ref - String reference.
+    #
+    # Returns a Project, or nil if the reference can't be found
+    def project_from_ref(ref)
+      return context[:project] unless ref
+
+      Project.find_with_namespace(ref)
+    end
+  end
+end

lib/banzai/filter.rb

+require 'active_support/core_ext/string/output_safety'
+require 'banzai'
+
+module Banzai
+  module Filter
+    def self.[](name)
+      const_get("#{name.to_s.camelize}Filter")
+    end
+  end
+end

lib/gitlab/markdown/abstract_reference_filter.rb → lib/banzai/filter/abstract_reference_filter.rb

-require 'gitlab/markdown'
+require 'banzai'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Filter
     # Issues, Merge Requests, Snippets, Commits and Commit Ranges share
     # similar functionality in reference filtering.
     class AbstractReferenceFilter < ReferenceFilter

lib/gitlab/markdown/filter/autolink_filter.rb → lib/banzai/filter/autolink_filter.rb

-require 'gitlab/markdown'
+require 'banzai'
 require 'html/pipeline/filter'
 require 'uri'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Filter
     # HTML Filter for auto-linking URLs in HTML.
     #
     # Based on HTML::Pipeline::AutolinkFilter

lib/gitlab/markdown/filter/commit_range_reference_filter.rb → lib/banzai/filter/commit_range_reference_filter.rb

-require 'gitlab/markdown'
+require 'banzai'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Filter
     # HTML filter that replaces commit range references with links.
     #
     # This filter supports cross-project references.

lib/gitlab/markdown/filter/commit_reference_filter.rb → lib/banzai/filter/commit_reference_filter.rb

-require 'gitlab/markdown'
+require 'banzai'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Filter
     # HTML filter that replaces commit references with links.
     #
     # This filter supports cross-project references.

lib/gitlab/markdown/filter/emoji_filter.rb → lib/banzai/filter/emoji_filter.rb

 require 'action_controller'
-require 'gitlab/markdown'
+require 'banzai'
 require 'gitlab_emoji'
 require 'html/pipeline/filter'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Filter
     # HTML filter that replaces :emoji: with images.
     #
     # Based on HTML::Pipeline::EmojiFilter

lib/gitlab/markdown/filter/external_issue_reference_filter.rb → lib/banzai/filter/external_issue_reference_filter.rb

-require 'gitlab/markdown'
+require 'banzai'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Filter
     # HTML filter that replaces external issue tracker references with links.
     # References are ignored if the project doesn't use an external issue
     # tracker.

lib/gitlab/markdown/filter/external_link_filter.rb → lib/banzai/filter/external_link_filter.rb

-require 'gitlab/markdown'
+require 'banzai'
 require 'html/pipeline/filter'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Filter
     # HTML Filter to add a `rel="nofollow"` attribute to external links
     #
     class ExternalLinkFilter < HTML::Pipeline::Filter

lib/gitlab/markdown/filter/issue_reference_filter.rb → lib/banzai/filter/issue_reference_filter.rb

-require 'gitlab/markdown'
+require 'banzai'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Filter
     # HTML filter that replaces issue references with links. References to
     # issues that do not exist are ignored.
     #

lib/gitlab/markdown/filter/label_reference_filter.rb → lib/banzai/filter/label_reference_filter.rb

-require 'gitlab/markdown'
+require 'banzai'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Filter
     # HTML filter that replaces label references with links.
     class LabelReferenceFilter < ReferenceFilter
       # Public: Find label references in text

lib/gitlab/markdown/filter/markdown_filter.rb → lib/banzai/filter/markdown_filter.rb

-module Gitlab
-  module Markdown
+require 'banzai'
+require 'html/pipeline/filter'
+
+module Banzai
+  module Filter
     class MarkdownFilter < HTML::Pipeline::TextFilter
       def initialize(text, context = nil, result = nil)
         super text, context, result

lib/gitlab/markdown/filter/merge_request_reference_filter.rb → lib/banzai/filter/merge_request_reference_filter.rb

-require 'gitlab/markdown'
+require 'banzai'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Filter
     # HTML filter that replaces merge request references with links. References
     # to merge requests that do not exist are ignored.
     #

lib/gitlab/markdown/filter/redactor_filter.rb → lib/banzai/filter/redactor_filter.rb

-require 'gitlab/markdown'
+require 'banzai'
 require 'html/pipeline/filter'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Filter
     # HTML filter that removes references to records that the current user does
     # not have permission to view.
     #
@@ -27,7 +27,7 @@ module Gitlab
       def user_can_reference?(node)
         if node.has_attribute?('data-reference-filter')
           reference_type = node.attr('data-reference-filter')
-          reference_filter = Gitlab::Markdown.const_get(reference_type)
+          reference_filter = Banzai::Filter.const_get(reference_type)
 
           reference_filter.user_can_reference?(current_user, node, context)
         else

lib/gitlab/markdown/reference_filter.rb → lib/banzai/filter/reference_filter.rb

 require 'active_support/core_ext/string/output_safety'
-require 'gitlab/markdown'
+require 'banzai'
 require 'html/pipeline/filter'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Filter
     # Base class for GitLab Flavored Markdown reference filters.
     #
     # References within <pre>, <code>, <a>, and <style> elements are ignored.
@@ -12,27 +12,6 @@ module Gitlab
     #   :project (required) - Current project, ignored if reference is cross-project.
     #   :only_path          - Generate path-only links.
     class ReferenceFilter < HTML::Pipeline::Filter
-      LazyReference = Struct.new(:klass, :ids) do
-        def self.load(refs)
-          lazy_references, values = refs.partition { |ref| ref.is_a?(self) }
-
-          lazy_values = lazy_references.group_by(&:klass).flat_map do |klass, refs|
-            ids = refs.flat_map(&:ids)
-            klass.where(id: ids)
-          end
-
-          values + lazy_values
-        end
-
-        def load
-          self.klass.where(id: self.ids)
-        end
-      end
-
-      def self.[](name)
-        Markdown.const_get("#{name.to_s.camelize}ReferenceFilter")
-      end
-
       def self.user_can_reference?(user, node, context)
         if node.has_attribute?('data-project')
           project_id = node.attr('data-project').to_i

lib/gitlab/markdown/filter/reference_gatherer_filter.rb → lib/banzai/filter/reference_gatherer_filter.rb

-require 'gitlab/markdown'
+require 'banzai'
 require 'html/pipeline/filter'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Filter
     # HTML filter that gathers all referenced records that the current user has
     # permission to view.
     #
@@ -20,7 +20,7 @@ module Gitlab
           gather_references(node)
         end
 
-        load_lazy_references unless context[:load_lazy_references] == false
+        load_lazy_references unless ReferenceExtractor.lazy?
 
         doc
       end
@@ -31,7 +31,7 @@ module Gitlab
         return unless node.has_attribute?('data-reference-filter')
 
         reference_type = node.attr('data-reference-filter')
-        reference_filter = Gitlab::Markdown.const_get(reference_type)
+        reference_filter = Banzai::Filter.const_get(reference_type)
 
         return if context[:reference_filter] && reference_filter != context[:reference_filter]
 
@@ -47,11 +47,10 @@ module Gitlab
         end
       end
 
-      # Will load all references of one type using one query.
       def load_lazy_references
         refs = result[:references]
         refs.each do |type, values|
-          refs[type] = ReferenceFilter::LazyReference.load(values)
+          refs[type] = ReferenceExtractor.lazily(values)
         end
       end
 

lib/gitlab/markdown/filter/relative_link_filter.rb → lib/banzai/filter/relative_link_filter.rb

-require 'gitlab/markdown'
+require 'banzai'
 require 'html/pipeline/filter'
 require 'uri'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Filter
     # HTML filter that "fixes" relative links to files in a repository.
     #
     # Context options:

lib/gitlab/markdown/filter/sanitization_filter.rb → lib/banzai/filter/sanitization_filter.rb

-require 'gitlab/markdown'
+require 'banzai'
 require 'html/pipeline/filter'
 require 'html/pipeline/sanitization_filter'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Filter
     # Sanitize HTML
     #
     # Extends HTML::Pipeline::SanitizationFilter with a custom whitelist.

lib/gitlab/markdown/filter/snippet_reference_filter.rb → lib/banzai/filter/snippet_reference_filter.rb

-require 'gitlab/markdown'
+require 'banzai'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Filter
     # HTML filter that replaces snippet references with links. References to
     # snippets that do not exist are ignored.
     #

lib/gitlab/markdown/filter/syntax_highlight_filter.rb → lib/banzai/filter/syntax_highlight_filter.rb

-require 'gitlab/markdown'
+require 'banzai'
 require 'html/pipeline/filter'
 require 'rouge/plugins/redcarpet'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Filter
     # HTML Filter to highlight fenced code blocks
     #
     class SyntaxHighlightFilter < HTML::Pipeline::Filter

lib/gitlab/markdown/filter/table_of_contents_filter.rb → lib/banzai/filter/table_of_contents_filter.rb

-require 'gitlab/markdown'
+require 'banzai'
 require 'html/pipeline/filter'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Filter
     # HTML filter that adds an anchor child element to all Headers in a
     # document, so that they can be linked to.
     #

lib/gitlab/markdown/filter/task_list_filter.rb → lib/banzai/filter/task_list_filter.rb

-require 'gitlab/markdown'
+require 'banzai'
 require 'task_list/filter'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Filter
     # Work around a bug in the default TaskList::Filter that adds a `task-list`
     # class to every list element, regardless of whether or not it contains a
     # task list.

lib/gitlab/markdown/filter/upload_link_filter.rb → lib/banzai/filter/upload_link_filter.rb

-require 'gitlab/markdown'
+require 'banzai'
 require 'html/pipeline/filter'
 require 'uri'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Filter
     # HTML filter that "fixes" relative upload links to files.
     # Context options:
     #   :project (required) - Current project

lib/gitlab/markdown/filter/user_reference_filter.rb → lib/banzai/filter/user_reference_filter.rb

-require 'gitlab/markdown'
+require 'banzai'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Filter
     # HTML filter that replaces user or group references with links.
     #
     # A special `@all` reference is also supported.

lib/banzai/lazy_reference.rb

+require 'banzai'
+
+module Banzai
+  class LazyReference
+    def self.load(refs)
+      lazy_references, values = refs.partition { |ref| ref.is_a?(self) }
+
+      lazy_values = lazy_references.group_by(&:klass).flat_map do |klass, refs|
+        ids = refs.flat_map(&:ids)
+        klass.where(id: ids)
+      end
+
+      values + lazy_values
+    end
+
+    attr_reader :klass, :ids
+
+    def initialize(klass, ids)
+      @klass = klass
+      @ids = Array.wrap(ids).map(&:to_i)
+    end
+
+    def load
+      self.klass.where(id: self.ids)
+    end
+  end
+end

lib/banzai/pipeline.rb

+require 'banzai'
+
+module Banzai
+  module Pipeline
+    def self.[](name)
+      name ||= :full
+      const_get("#{name.to_s.camelize}Pipeline")
+    end
+  end
+end

lib/gitlab/markdown/pipeline/asciidoc_pipeline.rb → lib/banzai/pipeline/asciidoc_pipeline.rb

-require 'gitlab/markdown'
+require 'banzai'
 
-module Gitlab
-  module Markdown
-    class AsciidocPipeline < Pipeline
+module Banzai
+  module Pipeline
+    class AsciidocPipeline < BasePipeline
       def self.filters
         [
-          Gitlab::Markdown::RelativeLinkFilter
+          Filter::RelativeLinkFilter
         ]
       end
     end

lib/gitlab/markdown/pipeline/atom_pipeline.rb → lib/banzai/pipeline/atom_pipeline.rb

-require 'gitlab/markdown'
+require 'banzai'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Pipeline
     class AtomPipeline < FullPipeline
       def self.transform_context(context)
         super(context).merge(

lib/banzai/pipeline/base_pipeline.rb

+require 'banzai'
+require 'html/pipeline'
+
+module Banzai
+  module Pipeline
+    class BasePipeline
+      def self.filters
+        []
+      end
+
+      def self.transform_context(context)
+        context
+      end
+
+      def self.html_pipeline
+        @html_pipeline ||= HTML::Pipeline.new(filters)
+      end
+
+      class << self
+        %i(call to_document to_html).each do |meth|
+          define_method(meth) do |text, context|
+            context = transform_context(context)
+
+            html_pipeline.send(meth, text, context)
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/markdown/combined_pipeline.rb → lib/banzai/pipeline/combined_pipeline.rb

-require 'gitlab/markdown'
+require 'banzai'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Pipeline
     module CombinedPipeline
       def self.new(*pipelines)
-        Class.new(Pipeline) do
+        Class.new(BasePipeline) do
           const_set :PIPELINES, pipelines
 
           def self.pipelines

lib/gitlab/markdown/pipeline/description_pipeline.rb → lib/banzai/pipeline/description_pipeline.rb

-require 'gitlab/markdown'
+require 'banzai'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Pipeline
     class DescriptionPipeline < FullPipeline
       def self.transform_context(context)
         super(context).merge(

lib/gitlab/markdown/pipeline/email_pipeline.rb → lib/banzai/pipeline/email_pipeline.rb

-require 'gitlab/markdown'
+require 'banzai'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Pipeline
     class EmailPipeline < FullPipeline
       def self.transform_context(context)
         super(context).merge(

lib/gitlab/markdown/pipeline/full_pipeline.rb → lib/banzai/pipeline/full_pipeline.rb

-require 'gitlab/markdown'
+require 'banzai'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Pipeline
     class FullPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GfmPipeline)
 
     end

lib/banzai/pipeline/gfm_pipeline.rb

+require 'banzai'
+
+module Banzai
+  module Pipeline
+    class GfmPipeline < BasePipeline
+      def self.filters
+        @filters ||= [
+          Filter::SyntaxHighlightFilter,
+          Filter::SanitizationFilter,
+
+          Filter::UploadLinkFilter,
+          Filter::EmojiFilter,
+          Filter::TableOfContentsFilter,
+          Filter::AutolinkFilter,
+          Filter::ExternalLinkFilter,
+
+          Filter::UserReferenceFilter,
+          Filter::IssueReferenceFilter,
+          Filter::ExternalIssueReferenceFilter,
+          Filter::MergeRequestReferenceFilter,
+          Filter::SnippetReferenceFilter,
+          Filter::CommitRangeReferenceFilter,
+          Filter::CommitReferenceFilter,
+          Filter::LabelReferenceFilter,
+
+          Filter::TaskListFilter
+        ]
+      end
+
+      def self.transform_context(context)
+        context.merge(
+          only_path: true,
+
+          # EmojiFilter
+          asset_host: Gitlab::Application.config.asset_host,
+          asset_root: Gitlab.config.gitlab.base_url
+        )
+      end
+    end
+  end
+end

lib/gitlab/markdown/pipeline/note_pipeline.rb → lib/banzai/pipeline/note_pipeline.rb

-require 'gitlab/markdown'
+require 'banzai'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Pipeline
     class NotePipeline < FullPipeline
       def self.transform_context(context)
         super(context).merge(

lib/gitlab/markdown/pipeline/plain_markdown_pipeline.rb → lib/banzai/pipeline/plain_markdown_pipeline.rb

-require 'gitlab/markdown'
+require 'banzai'
 
-module Gitlab
-  module Markdown
-    class PlainMarkdownPipeline < Pipeline
+module Banzai
+  module Pipeline
+    class PlainMarkdownPipeline < BasePipeline
       def self.filters
         [
-          Gitlab::Markdown::MarkdownFilter
+          Filter::MarkdownFilter
         ]
       end
     end

lib/gitlab/markdown/pipeline/post_process_pipeline.rb → lib/banzai/pipeline/post_process_pipeline.rb

-require 'gitlab/markdown'
+require 'banzai'
 
-module Gitlab
-  module Markdown
-    class PostProcessPipeline < Pipeline
+module Banzai
+  module Pipeline
+    class PostProcessPipeline < BasePipeline
       def self.filters
         [
-          Gitlab::Markdown::RelativeLinkFilter, 
-          Gitlab::Markdown::RedactorFilter
+          Filter::RelativeLinkFilter,
+          Filter::RedactorFilter
         ]
       end
 

lib/gitlab/markdown/pipeline/reference_extraction_pipeline.rb → lib/banzai/pipeline/reference_extraction_pipeline.rb

-require 'gitlab/markdown'
+require 'banzai'
 
-module Gitlab
-  module Markdown
-    class ReferenceExtractionPipeline < Pipeline
+module Banzai
+  module Pipeline
+    class ReferenceExtractionPipeline < BasePipeline
       def self.filters
         [
-          Gitlab::Markdown::ReferenceGathererFilter
+          Filter::ReferenceGathererFilter
         ]
       end
     end

lib/gitlab/markdown/pipeline/single_line_pipeline.rb → lib/banzai/pipeline/single_line_pipeline.rb

-require 'gitlab/markdown'
+require 'banzai'
 
-module Gitlab
-  module Markdown
+module Banzai
+  module Pipeline
     class SingleLinePipeline < GfmPipeline
 
     end

lib/banzai/reference_extractor.rb

+require 'banzai'
+
+module Banzai
+  # Extract possible GFM references from an arbitrary String for further processing.
+  class ReferenceExtractor
+    class << self
+      LAZY_KEY = :banzai_reference_extractor_lazy
+
+      def lazy?
+        Thread.current[LAZY_KEY]
+      end
+
+      def lazily(values = nil, &block)
+        return (values || block.call).uniq if lazy?
+
+        begin
+          Thread.current[LAZY_KEY] = true
+
+          values ||= block.call
+
+          Banzai::LazyReference.load(values.uniq).uniq
+        ensure
+          Thread.current[LAZY_KEY] = false
+        end
+      end
+    end
+
+    def initialize
+      @texts = []
+    end
+
+    def analyze(text, context = {})
+      @texts << Renderer.render(text, context)
+    end
+
+    def references(type, context = {})
+      filter = Banzai::Filter["#{type}_reference"]
+
+      context.merge!(
+        pipeline: :reference_extraction,
+
+        # ReferenceGathererFilter
+        reference_filter: filter
+      )
+
+      self.class.lazily do
+        @texts.flat_map do |html|
+          text_context = context.dup
+          result = Renderer.render_result(html, text_context)
+          result[:references][type]
+        end.uniq
+      end
+    end
+  end
+end

lib/gitlab/markdown.rb → lib/banzai/renderer.rb

-require 'html/pipeline'
-
-module Gitlab
-  # Custom parser for GitLab-flavored Markdown
-  #
-  # See the files in `lib/gitlab/markdown/` for specific processing information.
-  module Markdown
+module Banzai
+  module Renderer
     # Convert a Markdown String into an HTML-safe String of HTML
     #
     # Note that while the returned HTML will have been sanitized of dangerous
@@ -75,41 +70,7 @@ module Gitlab
 
     def self.full_cache_key(cache_key, pipeline_name)
       return unless cache_key
-      ["markdown", *cache_key, pipeline_name || :full]
+      ["banzai", *cache_key, pipeline_name || :full]
     end
-
-    # Provide autoload paths for filters to prevent a circular dependency error
-    autoload :AutolinkFilter,               'gitlab/markdown/filter/autolink_filter'
-    autoload :CommitRangeReferenceFilter,   'gitlab/markdown/filter/commit_range_reference_filter'
-    autoload :CommitReferenceFilter,        'gitlab/markdown/filter/commit_reference_filter'
-    autoload :EmojiFilter,                  'gitlab/markdown/filter/emoji_filter'
-    autoload :ExternalIssueReferenceFilter, 'gitlab/markdown/filter/external_issue_reference_filter'
-    autoload :ExternalLinkFilter,           'gitlab/markdown/filter/external_link_filter'
-    autoload :IssueReferenceFilter,         'gitlab/markdown/filter/issue_reference_filter'
-    autoload :LabelReferenceFilter,         'gitlab/markdown/filter/label_reference_filter'
-    autoload :MarkdownFilter,               'gitlab/markdown/filter/markdown_filter'
-    autoload :MergeRequestReferenceFilter,  'gitlab/markdown/filter/merge_request_reference_filter'
-    autoload :RedactorFilter,               'gitlab/markdown/filter/redactor_filter'
-    autoload :ReferenceGathererFilter,      'gitlab/markdown/filter/reference_gatherer_filter'
-    autoload :RelativeLinkFilter,           'gitlab/markdown/filter/relative_link_filter'
-    autoload :SanitizationFilter,           'gitlab/markdown/filter/sanitization_filter'
-    autoload :SnippetReferenceFilter,       'gitlab/markdown/filter/snippet_reference_filter'
-    autoload :SyntaxHighlightFilter,        'gitlab/markdown/filter/syntax_highlight_filter'
-    autoload :TableOfContentsFilter,        'gitlab/markdown/filter/table_of_contents_filter'
-    autoload :TaskListFilter,               'gitlab/markdown/filter/task_list_filter'
-    autoload :UserReferenceFilter,          'gitlab/markdown/filter/user_reference_filter'
-    autoload :UploadLinkFilter,             'gitlab/markdown/filter/upload_link_filter'
-
-    autoload :AsciidocPipeline,             'gitlab/markdown/pipeline/asciidoc_pipeline'
-    autoload :AtomPipeline,                 'gitlab/markdown/pipeline/atom_pipeline'
-    autoload :DescriptionPipeline,          'gitlab/markdown/pipeline/description_pipeline'
-    autoload :EmailPipeline,                'gitlab/markdown/pipeline/email_pipeline'
-    autoload :FullPipeline,                 'gitlab/markdown/pipeline/full_pipeline'
-    autoload :GfmPipeline,                  'gitlab/markdown/pipeline/gfm_pipeline'
-    autoload :NotePipeline,                 'gitlab/markdown/pipeline/note_pipeline'
-    autoload :PlainMarkdownPipeline,        'gitlab/markdown/pipeline/plain_markdown_pipeline'
-    autoload :PostProcessPipeline,          'gitlab/markdown/pipeline/post_process_pipeline'
-    autoload :ReferenceExtractionPipeline,  'gitlab/markdown/pipeline/reference_extraction_pipeline'
-    autoload :SingleLinePipeline,           'gitlab/markdown/pipeline/single_line_pipeline'
   end
 end

lib/gitlab/asciidoc.rb

@@ -32,7 +32,7 @@ module Gitlab
       html = ::Asciidoctor.convert(input, asciidoc_opts)
 
       if context[:project]
-        html = Gitlab::Markdown.render(html, context.merge(pipeline: :asciidoc))
+        html = Banzai.render(html, context.merge(pipeline: :asciidoc))
       end
 
       html.html_safe

lib/gitlab/markdown/cross_project_reference.rb

-require 'gitlab/markdown'
-
-module Gitlab
-  module Markdown
-    # Common methods for ReferenceFilters that support an optional cross-project
-    # reference.
-    module CrossProjectReference
-      # Given a cross-project reference string, get the Project record
-      #
-      # Defaults to value of `context[:project]` if:
-      # * No reference is given OR
-      # * Reference given doesn't exist
-      #
-      # ref - String reference.
-      #
-      # Returns a Project, or nil if the reference can't be found
-      def project_from_ref(ref)
-        return context[:project] unless ref
-
-        Project.find_with_namespace(ref)
-      end
-    end
-  end
-end

lib/gitlab/markdown/pipeline.rb

-require 'gitlab/markdown'
+require 'banzai'
 
 module Gitlab
   module Markdown
     class Pipeline
       def self.[](name)
         name ||= :full
-        Markdown.const_get("#{name.to_s.camelize}Pipeline")
+        const_get("#{name.to_s.camelize}Pipeline")
       end
 
       def self.filters

lib/gitlab/markdown/pipeline/gfm_pipeline.rb

-require 'gitlab/markdown'
-
-module Gitlab
-  module Markdown
-    class GfmPipeline < Pipeline
-      def self.filters
-        @filters ||= [
-          Gitlab::Markdown::SyntaxHighlightFilter,
-          Gitlab::Markdown::SanitizationFilter,
-
-          Gitlab::Markdown::UploadLinkFilter,
-          Gitlab::Markdown::EmojiFilter,
-          Gitlab::Markdown::TableOfContentsFilter,
-          Gitlab::Markdown::AutolinkFilter,
-          Gitlab::Markdown::ExternalLinkFilter,
-
-          Gitlab::Markdown::UserReferenceFilter,
-          Gitlab::Markdown::IssueReferenceFilter,
-          Gitlab::Markdown::ExternalIssueReferenceFilter,
-          Gitlab::Markdown::MergeRequestReferenceFilter,
-          Gitlab::Markdown::SnippetReferenceFilter,
-          Gitlab::Markdown::CommitRangeReferenceFilter,
-          Gitlab::Markdown::CommitReferenceFilter,
-          Gitlab::Markdown::LabelReferenceFilter,
-
-          Gitlab::Markdown::TaskListFilter
-        ]
-      end
-
-      def self.transform_context(context)
-        context.merge(
-          only_path: true,
-
-          # EmojiFilter
-          asset_host: Gitlab::Application.config.asset_host,
-          asset_root: Gitlab.config.gitlab.base_url
-        )
-      end
-    end
-  end
-end

lib/gitlab/reference_extractor.rb

-require 'gitlab/markdown'
+require 'banzai'
 
 module Gitlab
   # Extract possible GFM references from an arbitrary String for further processing.
-  class ReferenceExtractor
-    attr_accessor :project, :current_user, :load_lazy_references
+  class ReferenceExtractor < Banzai::ReferenceExtractor
+    attr_accessor :project, :current_user
 
-    def initialize(project, current_user = nil, load_lazy_references: true)
+    def initialize(project, current_user = nil)
       @project = project
       @current_user = current_user
-      @load_lazy_references = load_lazy_references
 
-      @texts = []
       @references = {}
+
+      super()
     end
 
-    def analyze(text, options = {})
-      @texts << Gitlab::Markdown.render(text, options.merge(project: project))
+    def analyze(text, context = {})
+      super(text, context.merge(project: project))
     end
 
     %i(user label issue merge_request snippet commit commit_range).each do |type|
       define_method("#{type}s") do
-        @references[type] ||= pipeline_result(type)
-      end
+        @references[type] ||= references(type, project: project, current_user: current_user)
       end
-
-    private
-
-    # Instantiate and call HTML::Pipeline with a single reference filter type,
-    # returning the result
-    #
-    # filter_type - Symbol reference type (e.g., :commit, :issue, etc.)
-    #
-    # Returns the results Array for the requested filter type
-    def pipeline_result(filter_type)
-      filter = Gitlab::Markdown::ReferenceFilter[filter_type]
-
-      context = {
-        pipeline: :reference_extraction,
-
-        project:      project,
-        current_user: current_user,
-
-        # ReferenceGathererFilter
-        load_lazy_references: false,
-        reference_filter:     filter
-      }
-
-      values = @texts.flat_map do |html|
-        text_context = context.dup
-        result = Gitlab::Markdown.render_result(html, text_context)
-        result[:references][filter_type]
-      end.uniq
-
-      if @load_lazy_references
-        values = Gitlab::Markdown::ReferenceFilter::LazyReference.load(values).uniq
-      end
-
-      values
     end
   end
 end

spec/benchmarks/lib/gitlab/markdown/reference_filter_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Markdown::ReferenceFilter, benchmark: true do
+describe Banzai::Filter::ReferenceFilter, benchmark: true do
   let(:input) do
     html = <<-EOF
 <p>Hello @alice and @bob, how are you doing today?</p>

spec/lib/gitlab/markdown/cross_project_reference_spec.rb → spec/lib/banzai/cross_project_reference_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Markdown::CrossProjectReference, lib: true do
+describe Banzai::CrossProjectReference, lib: true do
   include described_class
 
   describe '#project_from_ref' do

spec/lib/gitlab/markdown/filter/autolink_filter_spec.rb → spec/lib/banzai/filter/autolink_filter_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Markdown::AutolinkFilter, lib: true do
+describe Banzai::Filter::AutolinkFilter, lib: true do
   include FilterSpecHelper
 
   let(:link) { 'http://about.gitlab.com/' }

spec/lib/gitlab/markdown/filter/commit_range_reference_filter_spec.rb → spec/lib/banzai/filter/commit_range_reference_filter_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Markdown::CommitRangeReferenceFilter, lib: true do
+describe Banzai::Filter::CommitRangeReferenceFilter, lib: true do
   include FilterSpecHelper
 
   let(:project) { create(:project, :public) }

spec/lib/gitlab/markdown/filter/commit_reference_filter_spec.rb → spec/lib/banzai/filter/commit_reference_filter_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Markdown::CommitReferenceFilter, lib: true do
+describe Banzai::Filter::CommitReferenceFilter, lib: true do
   include FilterSpecHelper
 
   let(:project) { create(:project, :public) }

spec/lib/gitlab/markdown/filter/emoji_filter_spec.rb → spec/lib/banzai/filter/emoji_filter_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Markdown::EmojiFilter, lib: true do
+describe Banzai::Filter::EmojiFilter, lib: true do
   include FilterSpecHelper
 
   before do

spec/lib/gitlab/markdown/filter/external_issue_reference_filter_spec.rb → spec/lib/banzai/filter/external_issue_reference_filter_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Markdown::ExternalIssueReferenceFilter, lib: true do
+describe Banzai::Filter::ExternalIssueReferenceFilter, lib: true do
   include FilterSpecHelper
 
   def helper

spec/lib/gitlab/markdown/filter/external_link_filter_spec.rb → spec/lib/banzai/filter/external_link_filter_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Markdown::ExternalLinkFilter, lib: true do
+describe Banzai::Filter::ExternalLinkFilter, lib: true do
   include FilterSpecHelper
 
   it 'ignores elements without an href attribute' do

spec/lib/gitlab/markdown/filter/issue_reference_filter_spec.rb → spec/lib/banzai/filter/issue_reference_filter_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Markdown::IssueReferenceFilter, lib: true do
+describe Banzai::Filter::IssueReferenceFilter, lib: true do
   include FilterSpecHelper
 
   def helper

spec/lib/gitlab/markdown/filter/label_reference_filter_spec.rb → spec/lib/banzai/filter/label_reference_filter_spec.rb

 require 'spec_helper'
 require 'html/pipeline'
 
-describe Gitlab::Markdown::LabelReferenceFilter, lib: true do
+describe Banzai::Filter::LabelReferenceFilter, lib: true do
   include FilterSpecHelper
 
   let(:project)   { create(:empty_project, :public) }

spec/lib/gitlab/markdown/filter/merge_request_reference_filter_spec.rb → spec/lib/banzai/filter/merge_request_reference_filter_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Markdown::MergeRequestReferenceFilter, lib: true do
+describe Banzai::Filter::MergeRequestReferenceFilter, lib: true do
   include FilterSpecHelper
 
   let(:project) { create(:project, :public) }

spec/lib/gitlab/markdown/filter/redactor_filter_spec.rb → spec/lib/banzai/filter/redactor_filter_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Markdown::RedactorFilter, lib: true do
+describe Banzai::Filter::RedactorFilter, lib: true do
   include ActionView::Helpers::UrlHelper
   include FilterSpecHelper
 

spec/lib/gitlab/markdown/filter/reference_gatherer_filter_spec.rb → spec/lib/banzai/filter/reference_gatherer_filter_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Markdown::ReferenceGathererFilter, lib: true do
+describe Banzai::Filter::ReferenceGathererFilter, lib: true do
   include ActionView::Helpers::UrlHelper
   include FilterSpecHelper
 

spec/lib/gitlab/markdown/filter/relative_link_filter_spec.rb → spec/lib/banzai/filter/relative_link_filter_spec.rb

@@ -2,7 +2,7 @@
 
 require 'spec_helper'
 
-describe Gitlab::Markdown::RelativeLinkFilter, lib: true do
+describe Banzai::Filter::RelativeLinkFilter, lib: true do
   def filter(doc, contexts = {})
     contexts.reverse_merge!({
       commit:         project.commit,

spec/lib/gitlab/markdown/filter/sanitization_filter_spec.rb → spec/lib/banzai/filter/sanitization_filter_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Markdown::SanitizationFilter, lib: true do
+describe Banzai::Filter::SanitizationFilter, lib: true do
   include FilterSpecHelper
 
   describe 'default whitelist' do

spec/lib/gitlab/markdown/filter/snippet_reference_filter_spec.rb → spec/lib/banzai/filter/snippet_reference_filter_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Markdown::SnippetReferenceFilter, lib: true do
+describe Banzai::Filter::SnippetReferenceFilter, lib: true do
   include FilterSpecHelper
 
   let(:project)   { create(:empty_project, :public) }

spec/lib/gitlab/markdown/filter/syntax_highlight_filter_spec.rb → spec/lib/banzai/filter/syntax_highlight_filter_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Markdown::SyntaxHighlightFilter, lib: true do
+describe Banzai::Filter::SyntaxHighlightFilter, lib: true do
   include FilterSpecHelper
 
   it 'highlights valid code blocks' do

spec/lib/gitlab/markdown/filter/table_of_contents_filter_spec.rb → spec/lib/banzai/filter/table_of_contents_filter_spec.rb

@@ -2,7 +2,7 @@
 
 require 'spec_helper'
 
-describe Gitlab::Markdown::TableOfContentsFilter, lib: true do
+describe Banzai::Filter::TableOfContentsFilter, lib: true do
   include FilterSpecHelper
 
   def header(level, text)

spec/lib/gitlab/markdown/filter/task_list_filter_spec.rb → spec/lib/banzai/filter/task_list_filter_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Markdown::TaskListFilter, lib: true do
+describe Banzai::Filter::TaskListFilter, lib: true do
   include FilterSpecHelper
 
   it 'does not apply `task-list` class to non-task lists' do

spec/lib/gitlab/markdown/filter/upload_link_filter_spec.rb → spec/lib/banzai/filter/upload_link_filter_spec.rb

@@ -2,7 +2,7 @@
 
 require 'spec_helper'
 
-describe Gitlab::Markdown::UploadLinkFilter, lib: true do
+describe Banzai::Filter::UploadLinkFilter, lib: true do
   def filter(doc, contexts = {})
     contexts.reverse_merge!({
       project: project

spec/lib/gitlab/markdown/filter/user_reference_filter_spec.rb → spec/lib/banzai/filter/user_reference_filter_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Markdown::UserReferenceFilter, lib: true do
+describe Banzai::Filter::UserReferenceFilter, lib: true do
   include FilterSpecHelper
 
   let(:project)   { create(:empty_project, :public) }

spec/lib/gitlab/asciidoc_spec.rb

@@ -50,7 +50,7 @@ module Gitlab
         filtered_html = '<b>ASCII</b>'
 
         allow(Asciidoctor).to receive(:convert).and_return(html)
-        expect(Gitlab::Markdown).to receive(:render)
+        expect(Banzai).to receive(:render)
           .with(html, context.merge(pipeline: :asciidoc))
           .and_return(filtered_html)
 

spec/support/filter_spec_helper.rb

-# Helper methods for Gitlab::Markdown filter specs
+# Helper methods for Banzai filter specs
 #
 # Must be included into specs manually
 module FilterSpecHelper
@@ -10,49 +10,49 @@ module FilterSpecHelper
   # if none is provided.
   #
   # html     - HTML String to pass to the filter's `call` method.
-  # contexts - Hash context for the filter. (default: {project: project})
+  # context - Hash context for the filter. (default: {project: project})
   #
   # Returns a Nokogiri::XML::DocumentFragment
-  def filter(html, contexts = {})
+  def filter(html, context = {})
     if defined?(project)
-      contexts.reverse_merge!(project: project)
+      context.reverse_merge!(project: project)
     end
 
-    described_class.call(html, contexts)
+    described_class.call(html, context)
   end
 
   # Run text through HTML::Pipeline with the current filter and return the
   # result Hash
   #
   # body     - String text to run through the pipeline
-  # contexts - Hash context for the filter. (default: {project: project})
+  # context - Hash context for the filter. (default: {project: project})
   #
   # Returns the Hash
-  def pipeline_result(body, contexts = {})
-    contexts.reverse_merge!(project: project) if defined?(project)
+  def pipeline_result(body, context = {})
+    context.reverse_merge!(project: project) if defined?(project)
 
-    pipeline = HTML::Pipeline.new([described_class], contexts)
+    pipeline = HTML::Pipeline.new([described_class], context)
     pipeline.call(body)
   end
 
-  def reference_pipeline(contexts = {})
-    contexts.reverse_merge!(project: project) if defined?(project)
+  def reference_pipeline(context = {})
+    context.reverse_merge!(project: project) if defined?(project)
 
     filters = [
-      Gitlab::Markdown::AutolinkFilter,
+      Banzai::Filter::AutolinkFilter,
       described_class,
-      Gitlab::Markdown::ReferenceGathererFilter
+      Banzai::Filter::ReferenceGathererFilter
     ]
 
-    HTML::Pipeline.new(filters, contexts)
+    HTML::Pipeline.new(filters, context)
   end
 
-  def reference_pipeline_result(body, contexts = {})
-    reference_pipeline(contexts).call(body)
+  def reference_pipeline_result(body, context = {})
+    reference_pipeline(context).call(body)
   end
 
-  def reference_filter(html, contexts = {})
-    reference_pipeline(contexts).to_document(html)
+  def reference_filter(html, context = {})
+    reference_pipeline(context).to_document(html)
   end
 
   # Modify a String reference to make it invalid