Merge branch 'replace_redcarpet_with_cmark' into 'master'

Add CommonMark markdown engine

See merge request gitlab-org/gitlab-ce!14835

Gemfile

@@ -126,6 +126,7 @@ gem 'html-pipeline', '~> 1.11.0'
 gem 'deckar01-task_list', '2.0.0'
 gem 'gitlab-markup', '~> 1.6.2'
 gem 'redcarpet', '~> 3.4'
+gem 'commonmarker', '~> 0.17'
 gem 'RedCloth', '~> 4.3.2'
 gem 'rdoc', '~> 4.2'
 gem 'org-ruby', '~> 0.9.12'

Gemfile.lock

@@ -131,6 +131,8 @@ GEM
     coercible (1.0.0)
       descendants_tracker (~> 0.0.1)
     colorize (0.7.7)
+    commonmarker (0.17.8)
+      ruby-enum (~> 0.5)
     concord (0.1.5)
       adamantium (~> 0.2.0)
       equalizer (~> 0.0.9)
@@ -797,6 +799,8 @@ GEM
       rubocop (>= 0.51)
     rubocop-rspec (1.22.1)
       rubocop (>= 0.52.1)
+    ruby-enum (0.7.2)
+      i18n
     ruby-fogbugz (0.2.1)
       crack (~> 0.4)
     ruby-prof (0.16.2)
@@ -1019,6 +1023,7 @@ DEPENDENCIES
   charlock_holmes (~> 0.7.5)
   chronic (~> 0.10.2)
   chronic_duration (~> 0.10.6)
+  commonmarker (~> 0.17)
   concurrent-ruby (~> 1.0.5)
   connection_pool (~> 2.0)
   creole (~> 0.5.0)

changelogs/unreleased/replace_redcarpet_with_cmark.yml

+---
+title: Add CommonMark markdown engine (experimental)
+merge_request: 14835
+author: blackst0ne
+type: added

config/initializers/8_metrics.rb

@@ -94,6 +94,7 @@ def instrument_classes(instrumentation)
 
   instrumentation.instrument_instance_methods(RepositoryCheck::SingleRepositoryWorker)
 
+  instrumentation.instrument_instance_methods(Rouge::Plugins::CommonMark)
   instrumentation.instrument_instance_methods(Rouge::Plugins::Redcarpet)
   instrumentation.instrument_instance_methods(Rouge::Formatters::HTMLGitlab)
 

lib/banzai/filter/markdown_engines/common_mark.rb

+# `CommonMark` markdown engine for GitLab's Banzai markdown filter.
+# This module is used in Banzai::Filter::MarkdownFilter.
+# Used gem is `commonmarker` which is a ruby wrapper for libcmark (CommonMark parser)
+# including GitHub's GFM extensions.
+# Homepage: https://github.com/gjtorikian/commonmarker
+
+module Banzai
+  module Filter
+    module MarkdownEngines
+      class CommonMark
+        EXTENSIONS = [
+          :autolink,      # provides support for automatically converting URLs to anchor tags.
+          :strikethrough, # provides support for strikethroughs.
+          :table,         # provides support for tables.
+          :tagfilter      # strips out several "unsafe" HTML tags from being used: https://github.github.com/gfm/#disallowed-raw-html-extension-
+        ].freeze
+
+        PARSE_OPTIONS = [
+          :FOOTNOTES,                  # parse footnotes.
+          :STRIKETHROUGH_DOUBLE_TILDE, # parse strikethroughs by double tildes (as redcarpet does).
+          :VALIDATE_UTF8	             # replace illegal sequences with the replacement character U+FFFD.
+        ].freeze
+
+        # The `:GITHUB_PRE_LANG` option is not used intentionally because
+        # it renders a fence block with language as `<pre lang="LANG"><code>some code\n</code></pre>`
+        # while GitLab's syntax is `<pre><code lang="LANG">some code\n</code></pre>`.
+        # If in the future the syntax is about to be made GitHub-compatible, please, add `:GITHUB_PRE_LANG` render option below
+        # and remove `code_block` method from `lib/banzai/renderer/common_mark/html.rb`.
+        RENDER_OPTIONS = [
+          :DEFAULT # default rendering system. Nothing special.
+        ].freeze
+
+        def initialize
+          @renderer = Banzai::Renderer::CommonMark::HTML.new(options: RENDER_OPTIONS)
+        end
+
+        def render(text)
+          doc = CommonMarker.render_doc(text, PARSE_OPTIONS, EXTENSIONS)
+
+          @renderer.render(doc)
+        end
+      end
+    end
+  end
+end

lib/banzai/filter/markdown_engines/redcarpet.rb

+# `Redcarpet` markdown engine for GitLab's Banzai markdown filter.
+# This module is used in Banzai::Filter::MarkdownFilter.
+# Used gem is `redcarpet` which is a ruby library for markdown processing.
+# Homepage: https://github.com/vmg/redcarpet
+
+module Banzai
+  module Filter
+    module MarkdownEngines
+      class Redcarpet
+        OPTIONS = {
+          fenced_code_blocks:  true,
+          footnotes:           true,
+          lax_spacing:         true,
+          no_intra_emphasis:   true,
+          space_after_headers: true,
+          strikethrough:       true,
+          superscript:         true,
+          tables:              true
+        }.freeze
+
+        def initialize
+          html_renderer = Banzai::Renderer::Redcarpet::HTML.new
+          @renderer = ::Redcarpet::Markdown.new(html_renderer, OPTIONS)
+        end
+
+        def render(text)
+          @renderer.render(text)
+        end
+      end
+    end
+  end
+end

lib/banzai/filter/markdown_filter.rb

 module Banzai
   module Filter
     class MarkdownFilter < HTML::Pipeline::TextFilter
-      # https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use
-      REDCARPET_OPTIONS = {
-        fenced_code_blocks:  true,
-        footnotes:           true,
-        lax_spacing:         true,
-        no_intra_emphasis:   true,
-        space_after_headers: true,
-        strikethrough:       true,
-        superscript:         true,
-        tables:              true
-      }.freeze
-
       def initialize(text, context = nil, result = nil)
-        super text, context, result
-        @text = @text.delete "\r"
+        super(text, context, result)
+
+        @renderer = renderer(context[:markdown_engine]).new
+        @text = @text.delete("\r")
       end
 
       def call
-        html = self.class.renderer.render(@text)
-        html.rstrip!
-        html
+        @renderer.render(@text).rstrip
       end
 
-      def self.renderer
-        Thread.current[:banzai_markdown_renderer] ||= begin
-          renderer = Banzai::Renderer::HTML.new
-          Redcarpet::Markdown.new(renderer, REDCARPET_OPTIONS)
+      private
+
+      DEFAULT_ENGINE = :redcarpet
+
+      def engine(engine_from_context)
+        engine_from_context ||= DEFAULT_ENGINE
+
+        engine_from_context.to_s.classify
       end
+
+      def renderer(engine_from_context)
+        "Banzai::Filter::MarkdownEngines::#{engine(engine_from_context)}".constantize
+      rescue NameError
+        raise NameError, "`#{engine_from_context}` is unknown markdown engine"
       end
     end
   end

lib/banzai/filter/syntax_highlight_filter.rb

+require 'rouge/plugins/common_mark'
 require 'rouge/plugins/redcarpet'
 
 module Banzai

lib/banzai/renderer/common_mark/html.rb

+module Banzai
+  module Renderer
+    module CommonMark
+      class HTML < CommonMarker::HtmlRenderer
+        def code_block(node)
+          block do
+            code      = node.string_content
+            lang      = node.fence_info
+            lang_attr = lang.present? ? %Q{ lang="#{lang}"} : ''
+            result    =
+              "<pre>" \
+                "<code#{lang_attr}>#{html_escape(code)}</code>" \
+              "</pre>"
+
+            out(result)
+          end
+        end
+      end
+    end
+  end
+end

lib/banzai/renderer/html.rb

-module Banzai
-  module Renderer
-    class HTML < Redcarpet::Render::HTML
-      def block_code(code, lang)
-        lang_attr = lang ? %Q{ lang="#{lang}"} : ''
-
-        "\n<pre>" \
-          "<code#{lang_attr}>#{html_escape(code)}</code>" \
-        "</pre>"
-      end
-    end
-  end
-end

lib/banzai/renderer/redcarpet/html.rb

+module Banzai
+  module Renderer
+    module Redcarpet
+      class HTML < ::Redcarpet::Render::HTML
+        def block_code(code, lang)
+          lang_attr = lang ? %Q{ lang="#{lang}"} : ''
+
+          "\n<pre>" \
+            "<code#{lang_attr}>#{html_escape(code)}</code>" \
+          "</pre>"
+        end
+      end
+    end
+  end
+end

lib/rouge/plugins/common_mark.rb

+# A rouge plugin for CommonMark markdown engine.
+# Used to highlight code generated by CommonMark.
+
+module Rouge
+  module Plugins
+    module CommonMark
+      def code_block(code, language)
+        lexer = Lexer.find_fancy(language, code) || Lexers::PlainText
+
+        formatter = rouge_formatter(lexer)
+        formatter.format(lexer.lex(code))
+      end
+
+      # override this method for custom formatting behavior
+      def rouge_formatter(lexer)
+        Formatters::HTMLLegacy.new(css_class: "highlight #{lexer.tag}")
+      end
+    end
+  end
+end