Merge branch 'katex-math' into 'master'

Render math in Asciidoc and Markdown with KaTeX using code blocks

Closes #13690 and #13180

See merge request !8003

app/assets/javascripts/blob_edit/edit_blob.js

@@ -60,7 +60,7 @@
           content: this.editor.getValue()
         }, function(response) {
           currentPane.empty().append(response);
-          return currentPane.syntaxHighlight();
+          return currentPane.renderGFM();
         });
       } else {
         this.$toggleButton.show();

app/assets/javascripts/notes.js

@@ -309,7 +309,7 @@
       }
       row = form.closest("tr");
       note_html = $(note.html);
-      note_html.syntaxHighlight();
+      note_html.renderGFM();
       // is this the first note of discussion?
       discussionContainer = $(".notes[data-discussion-id='" + note.discussion_id + "']");
       if ((note.original_discussion_id != null) && discussionContainer.length === 0) {
@@ -326,7 +326,7 @@
         discussionContainer.append(note_html);
         // Init discussion on 'Discussion' page if it is merge request page
         if ($('body').attr('data-page').indexOf('projects:merge_request') === 0) {
-          $('ul.main-notes-list').append(note.discussion_html).syntaxHighlight();
+          $('ul.main-notes-list').append(note.discussion_html).renderGFM();
         }
       } else {
         // append new note to all matching discussions
@@ -467,7 +467,7 @@
       // Convert returned HTML to a jQuery object so we can modify it further
       $html = $(note.html);
       gl.utils.localTimeAgo($('.js-timeago', $html));
-      $html.syntaxHighlight();
+      $html.renderGFM();
       $html.find('.js-task-list-container').taskList('enable');
       // Find the note's `li` element by ID and replace it with the updated HTML
       $note_li = $('.note-row-' + note.id);

app/assets/javascripts/preview_markdown.js

@@ -28,7 +28,7 @@
         return this.renderMarkdown(mdText, (function(_this) {
           return function(response) {
             preview.html(response.body);
-            preview.syntaxHighlight();
+            preview.renderGFM();
             return _this.renderReferencedUsers(response.references.users, form);
           };
         })(this));

app/assets/javascripts/render_gfm.js

+/* eslint-disable func-names, space-before-function-paren, consistent-return, no-var, no-undef, no-else-return, prefer-arrow-callback, padded-blocks, max-len */
+// Render Gitlab flavoured Markdown
+//
+// Delegates to syntax highlight and render math
+//
+(function() {
+  $.fn.renderGFM = function() {
+    this.find('.js-syntax-highlight').syntaxHighlight();
+    this.find('.js-render-math').renderMath();
+  };
+
+  $(document).on('ready page:load', function() {
+    return $('body').renderGFM();
+  });
+
+}).call(this);

app/assets/javascripts/render_math.js

+/* eslint-disable func-names, space-before-function-paren, consistent-return, no-var, no-undef, no-else-return, prefer-arrow-callback, padded-blocks, max-len */
+// Renders math using KaTeX in any element with the
+// `js-render-math` class
+//
+// ### Example Markup
+//
+//   <code class="js-render-math"></div>
+//
+(function() {
+  // Only load once
+  var katexLoaded = false;
+
+  // Loop over all math elements and render math
+  var renderWithKaTeX = function (elements) {
+    elements.each(function () {
+      var mathNode = $('<span></span>');
+      var $this = $(this);
+
+      var display = $this.attr('data-math-style') === 'display';
+      try {
+        katex.render($this.text(), mathNode.get(0), { displayMode: display });
+        mathNode.insertAfter($this);
+        $this.remove();
+      } catch (err) {
+        // What can we do??
+        console.log(err.message);
+      }
+    });
+  };
+
+  $.fn.renderMath = function() {
+    var $this = this;
+    if ($this.length === 0) return;
+
+    if (katexLoaded) renderWithKaTeX($this);
+    else {
+      // Request CSS file so it is in the cache
+      $.get(gon.katex_css_url, function() {
+        var css = $('<link>',
+          { rel: 'stylesheet',
+            type: 'text/css',
+            href: gon.katex_css_url,
+          });
+        css.appendTo('head');
+
+        // Load KaTeX js
+        $.getScript(gon.katex_js_url, function() {
+          katexLoaded = true;
+          renderWithKaTeX($this); // Run KaTeX
+        });
+      });
+    }
+  };
+
+}).call(this);

app/assets/javascripts/syntax_highlight.js

@@ -10,8 +10,10 @@
 //   <div class="js-syntax-highlight"></div>
 //
 (function() {
+
   $.fn.syntaxHighlight = function() {
     var $children;
+
     if ($(this).hasClass('js-syntax-highlight')) {
       // Given the element itself, apply highlighting
       return $(this).addClass(gon.user_color_scheme);
@@ -24,8 +26,4 @@
     }
   };
 
-  $(document).on('ready page:load', function() {
-    return $('.js-syntax-highlight').syntaxHighlight();
-  });
-
 }).call(this);

changelogs/unreleased/8003-katex-math.yml

+---
+title: Added support for math rendering, using KaTeX, in Markdown and asciidoc
+merge_request: 8003
+author: Munken

config/application.rb

@@ -85,6 +85,8 @@ module Gitlab
     config.assets.precompile << "print.css"
     config.assets.precompile << "notify.css"
     config.assets.precompile << "mailers/*.css"
+    config.assets.precompile << "katex.css"
+    config.assets.precompile << "katex.js"
     config.assets.precompile << "graphs/graphs_bundle.js"
     config.assets.precompile << "users/users_bundle.js"
     config.assets.precompile << "network/network_bundle.js"

config/initializers/math_lexer.rb

+# Touch the lexers so it is registered with Rouge
+Rouge::Lexers::Math

doc/user/markdown.md

@@ -319,6 +319,40 @@ Here's a sample video:
 
 ![Sample Video](img/markdown_video.mp4)
 
+### Math
+
+> If this is not rendered correctly, see
+https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/user/markdown.md#math
+
+It is possible to have math written with the LaTeX syntax rendered using [KaTeX][katex].
+
+Math written inside ```$``$``` will be rendered inline with the text.
+
+Math written inside triple back quotes, with the language declared as `math`, will be rendered on a separate line.
+
+Example:
+
+    This math is inline $`a^2+b^2=c^2`$.
+
+    This is on a separate line
+    ```math
+    a^2+b^2=c^2
+    ```
+
+Becomes:
+
+This math is inline $`a^2+b^2=c^2`$.
+
+This is on a separate line
+```math
+a^2+b^2=c^2
+```
+
+_Be advised that KaTeX only supports a [subset][katex-subset] of LaTeX._
+
+>**Note:**
+This also works for the asciidoctor `:stem: latexmath`. For details see the [asciidoctor user manual][asciidoctor-manual].
+
 ## Standard Markdown
 
 ### Headers
@@ -764,3 +798,6 @@ A link starting with a `/` is relative to the wiki root.
 [markdown.md]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/user/markdown.md
 [rouge]: http://rouge.jneen.net/ "Rouge website"
 [redcarpet]: https://github.com/vmg/redcarpet "Redcarpet website"
+[katex]: https://github.com/Khan/KaTeX "KaTeX website"
+[katex-subset]: https://github.com/Khan/KaTeX/wiki/Function-Support-in-KaTeX "Macros supported by KaTeX"
+[asciidoctor-manual]: http://asciidoctor.org/docs/user-manual/#activating-stem-support "Asciidoctor user manual"
\ No newline at end of file

lib/banzai/filter/math_filter.rb

+require 'uri'
+
+module Banzai
+  module Filter
+    # HTML filter that adds class="code math" and removes the dollar sign in $`2+2`$.
+    #
+    class MathFilter < HTML::Pipeline::Filter
+      # This picks out <code>...</code>.
+      INLINE_MATH = 'descendant-or-self::code'.freeze
+
+      # Pick out a code block which is declared math
+      DISPLAY_MATH = "descendant-or-self::pre[contains(@class, 'math') and contains(@class, 'code')]".freeze
+
+      # Attribute indicating inline or display math.
+      STYLE_ATTRIBUTE = 'data-math-style'.freeze
+
+      # Class used for tagging elements that should be rendered
+      TAG_CLASS = 'js-render-math'.freeze
+
+      INLINE_CLASSES = "code math #{TAG_CLASS}".freeze
+
+      DOLLAR_SIGN = '$'.freeze
+
+      def call
+        doc.xpath(INLINE_MATH).each do |code|
+          closing = code.next
+          opening = code.previous
+
+          # We need a sibling before and after.
+          # They should end and start with $ respectively.
+          if closing && opening &&
+              closing.content.first == DOLLAR_SIGN &&
+              opening.content.last == DOLLAR_SIGN
+
+            code[:class] = INLINE_CLASSES
+            code[STYLE_ATTRIBUTE] = 'inline'
+            closing.content = closing.content[1..-1]
+            opening.content = opening.content[0..-2]
+          end
+        end
+
+        doc.xpath(DISPLAY_MATH).each do |el|
+          el[STYLE_ATTRIBUTE] = 'display'
+          el[:class] += " #{TAG_CLASS}"
+        end
+
+        doc
+      end
+    end
+  end
+end

lib/banzai/pipeline/gfm_pipeline.rb

@@ -6,6 +6,7 @@ module Banzai
           Filter::SyntaxHighlightFilter,
           Filter::SanitizationFilter,
 
+          Filter::MathFilter,
           Filter::UploadLinkFilter,
           Filter::VideoLinkFilter,
           Filter::ImageLinkFilter,

lib/gitlab/asciidoc.rb

 require 'asciidoctor'
+require 'asciidoctor/converter/html5'
 
 module Gitlab
   # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters
@@ -23,7 +24,7 @@ module Gitlab
     def self.render(input, context, asciidoc_opts = {})
       asciidoc_opts.reverse_merge!(
         safe: :secure,
-        backend: :html5,
+        backend: :gitlab_html5,
         attributes: []
       )
       asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS)
@@ -36,3 +37,31 @@ module Gitlab
     end
   end
 end
+
+module Gitlab
+  module Asciidoc
+    class Html5Converter < Asciidoctor::Converter::Html5Converter
+      extend Asciidoctor::Converter::Config
+
+      register_for 'gitlab_html5'
+
+      def stem(node)
+        return super unless node.style.to_sym == :latexmath
+
+        %(<pre#{id_attribute(node)} class="code math js-render-math #{node.role}" data-math-style="display"><code>#{node.content}</code></pre>)
+      end
+
+      def inline_quoted(node)
+        return super unless node.type.to_sym == :latexmath
+
+        %(<code#{id_attribute(node)} class="code math js-render-math #{node.role}" data-math-style="inline">#{node.text}</code>)
+      end
+
+      private
+
+      def id_attribute(node)
+        node.id ? %( id="#{node.id}") : nil
+      end
+    end
+  end
+end

lib/gitlab/gon_helper.rb

@@ -8,6 +8,8 @@ module Gitlab
       gon.shortcuts_path         = help_page_path('shortcuts')
       gon.user_color_scheme      = Gitlab::ColorSchemes.for_user(current_user).css_class
       gon.award_menu_url         = emojis_path
+      gon.katex_css_url          = ActionController::Base.helpers.asset_path('katex.css')
+      gon.katex_js_url           = ActionController::Base.helpers.asset_path('katex.js')
 
       if current_user
         gon.current_user_id = current_user.id

lib/rouge/lexers/math.rb

+module Rouge
+  module Lexers
+    class Math < Lexer
+      title "A passthrough lexer used for LaTeX input"
+      desc "A boring lexer that doesn't highlight anything"
+
+      tag 'math'
+      mimetypes 'text/plain'
+
+      default_options token: 'Text'
+
+      def token
+        @token ||= Token[option :token]
+      end
+
+      def stream_tokens(string, &b)
+        yield self.token, string
+      end
+    end
+  end
+end

spec/lib/banzai/filter/math_filter_spec.rb

+require 'spec_helper'
+
+describe Banzai::Filter::MathFilter, lib: true do
+  include FilterSpecHelper
+
+  it 'leaves regular inline code unchanged' do
+    input = "<code>2+2</code>"
+    doc = filter(input)
+
+    expect(doc.to_s).to eq input
+  end
+
+  it 'removes surrounding dollar signs and adds class code, math and js-render-math' do
+    doc = filter("$<code>2+2</code>$")
+
+    expect(doc.to_s).to eq '<code class="code math js-render-math" data-math-style="inline">2+2</code>'
+  end
+
+  it 'only removes surrounding dollar signs' do
+    doc = filter("test $<code>2+2</code>$ test")
+    before = doc.xpath('descendant-or-self::text()[1]').first
+    after = doc.xpath('descendant-or-self::text()[3]').first
+
+    expect(before.to_s).to eq 'test '
+    expect(after.to_s).to eq ' test'
+  end
+
+  it 'only removes surrounding single dollar sign' do
+    doc = filter("test $$<code>2+2</code>$$ test")
+    before = doc.xpath('descendant-or-self::text()[1]').first
+    after = doc.xpath('descendant-or-self::text()[3]').first
+
+    expect(before.to_s).to eq 'test $'
+    expect(after.to_s).to eq '$ test'
+  end
+
+  it 'adds data-math-style inline attribute to inline math' do
+    doc = filter('$<code>2+2</code>$')
+    code = doc.xpath('descendant-or-self::code').first
+
+    expect(code['data-math-style']).to eq 'inline'
+  end
+
+  it 'adds class code and math to inline math' do
+    doc = filter('$<code>2+2</code>$')
+    code = doc.xpath('descendant-or-self::code').first
+
+    expect(code[:class]).to include("code")
+    expect(code[:class]).to include("math")
+  end
+
+  it 'adds js-render-math class to inline math' do
+    doc = filter('$<code>2+2</code>$')
+    code = doc.xpath('descendant-or-self::code').first
+
+    expect(code[:class]).to include("js-render-math")
+  end
+
+  # Cases with faulty syntax. Should be a no-op
+
+  it 'ignores cases with missing dolar sign at the end' do
+    input = "test $<code>2+2</code> test"
+    doc = filter(input)
+
+    expect(doc.to_s).to eq input
+  end
+
+  it 'ignores cases with missing dolar sign at the beginning' do
+    input = "test <code>2+2</code>$ test"
+    doc = filter(input)
+
+    expect(doc.to_s).to eq input
+  end
+
+  it 'ignores dollar signs if it is not adjacent' do
+    input = '<p>We check strictly $<code>2+2</code> and  <code>2+2</code>$ </p>'
+    doc = filter(input)
+
+    expect(doc.to_s).to eq input
+  end
+
+  # Display math
+
+  it 'adds data-math-style display attribute to display math' do
+    doc = filter('<pre class="code highlight js-syntax-highlight math" v-pre="true"><code>2+2</code></pre>')
+    pre = doc.xpath('descendant-or-self::pre').first
+
+    expect(pre['data-math-style']).to eq 'display'
+  end
+
+  it 'adds js-render-math class to display math' do
+    doc = filter('<pre class="code highlight js-syntax-highlight math" v-pre="true"><code>2+2</code></pre>')
+    pre = doc.xpath('descendant-or-self::pre').first
+
+    expect(pre[:class]).to include("js-render-math")
+  end
+
+  it 'ignores code blocks that are not math' do
+    input = '<pre class="code highlight js-syntax-highlight plaintext" v-pre="true"><code>2+2</code></pre>'
+    doc = filter(input)
+
+    expect(doc.to_s).to eq input
+  end
+
+  it 'requires the pre to contain both code and math' do
+    input = '<pre class="highlight js-syntax-highlight plaintext math" v-pre="true"><code>2+2</code></pre>'
+    doc = filter(input)
+
+    expect(doc.to_s).to eq input
+  end
+
+  it 'dollar signs around to display math' do
+    doc = filter('$<pre class="code highlight js-syntax-highlight math" v-pre="true"><code>2+2</code></pre>$')
+    before = doc.xpath('descendant-or-self::text()[1]').first
+    after = doc.xpath('descendant-or-self::text()[3]').first
+
+    expect(before.to_s).to eq '$'
+    expect(after.to_s).to eq '$'
+  end
+end

spec/lib/gitlab/asciidoc_spec.rb

@@ -11,7 +11,7 @@ module Gitlab
       it "converts the input using Asciidoctor and default options" do
         expected_asciidoc_opts = {
             safe: :secure,
-            backend: :html5,
+            backend: :gitlab_html5,
             attributes: described_class::DEFAULT_ADOC_ATTRS
         }
 
@@ -27,7 +27,7 @@ module Gitlab
         it "merges the options with default ones" do
           expected_asciidoc_opts = {
               safe: :safe,
-              backend: :html5,
+              backend: :gitlab_html5,
               attributes: described_class::DEFAULT_ADOC_ATTRS + ['foo']
           }