Merge branch '52007-frontmatter-toml-json' into 'master'

Changed frontmatter filtering to support YAML, JSON, TOML, and arbitrary languages

Closes #52007

See merge request gitlab-org/gitlab-ce!23331

changelogs/unreleased/52007-frontmatter-toml-json.yml

+---
+title: Changed frontmatter filtering to support YAML, JSON, TOML, and arbitrary languages
+merge_request: 23331
+author: Travis Miller
+type: changed

lib/banzai/filter/front_matter_filter.rb

+# frozen_string_literal: true
+
+module Banzai
+  module Filter
+    class FrontMatterFilter < HTML::Pipeline::Filter
+      DELIM_LANG = {
+        '---' => 'yaml',
+        '+++' => 'toml',
+        ';;;' => 'json'
+      }.freeze
+
+      DELIM = Regexp.union(DELIM_LANG.keys)
+
+      PATTERN = %r{
+        \A(?:[^\r\n]*coding:[^\r\n]*)?         # optional encoding line
+        \s*
+        ^(?<delim>#{DELIM})[ \t]*(?<lang>\S*)  # opening front matter marker (optional language specifier)
+        \s*
+        ^(?<front_matter>.*?)                  # front matter (not greedy)
+        \s*
+        ^\k<delim>                             # closing front matter marker
+        \s*
+      }mx
+
+      def call
+        html.sub(PATTERN) do |_match|
+          lang = $~[:lang].presence || DELIM_LANG[$~[:delim]]
+
+          ["```#{lang}", $~[:front_matter], "```", "\n"].join("\n")
+        end
+      end
+    end
+  end
+end

lib/banzai/filter/yaml_front_matter_filter.rb

-# frozen_string_literal: true
-
-module Banzai
-  module Filter
-    class YamlFrontMatterFilter < HTML::Pipeline::Filter
-      DELIM = '---'.freeze
-
-      # Hat-tip to Middleman: https://git.io/v2e0z
-      PATTERN = %r{
-        \A(?:[^\r\n]*coding:[^\r\n]*\r?\n)?
-        (?<start>#{DELIM})[ ]*\r?\n
-        (?<frontmatter>.*?)[ ]*\r?\n?
-        ^(?<stop>#{DELIM})[ ]*\r?\n?
-        \r?\n?
-        (?<content>.*)
-      }mx.freeze
-
-      def call
-        match = PATTERN.match(html)
-
-        return html unless match
-
-        "```yaml\n#{match['frontmatter']}\n```\n\n#{match['content']}"
-      end
-    end
-  end
-end

lib/banzai/pipeline/pre_process_pipeline.rb

@@ -5,7 +5,7 @@ module Banzai
     class PreProcessPipeline < BasePipeline
       def self.filters
         FilterArray[
-          Filter::YamlFrontMatterFilter,
+          Filter::FrontMatterFilter,
           Filter::BlockquoteFenceFilter,
         ]
       end

spec/lib/banzai/filter/front_matter_filter_spec.rb

+require 'rails_helper'
+
+describe Banzai::Filter::FrontMatterFilter do
+  include FilterSpecHelper
+
+  it 'allows for `encoding:` before the front matter' do
+    content = <<~MD
+      # encoding: UTF-8
+      ---
+      foo: foo
+      bar: bar
+      ---
+
+      # Header
+
+      Content
+    MD
+
+    output = filter(content)
+
+    expect(output).not_to match 'encoding'
+  end
+
+  it 'converts YAML front matter to a fenced code block' do
+    content = <<~MD
+      ---
+      foo: :foo_symbol
+      bar: :bar_symbol
+      ---
+
+      # Header
+
+      Content
+    MD
+
+    output = filter(content)
+
+    aggregate_failures do
+      expect(output).not_to include '---'
+      expect(output).to include "```yaml\nfoo: :foo_symbol\n"
+    end
+  end
+
+  it 'converts TOML frontmatter to a fenced code block' do
+    content = <<~MD
+      +++
+      foo = :foo_symbol
+      bar = :bar_symbol
+      +++
+
+      # Header
+
+      Content
+    MD
+
+    output = filter(content)
+
+    aggregate_failures do
+      expect(output).not_to include '+++'
+      expect(output).to include "```toml\nfoo = :foo_symbol\n"
+    end
+  end
+
+  it 'converts JSON front matter to a fenced code block' do
+    content = <<~MD
+      ;;;
+      {
+        "foo": ":foo_symbol",
+        "bar": ":bar_symbol"
+      }
+      ;;;
+
+      # Header
+
+      Content
+    MD
+
+    output = filter(content)
+
+    aggregate_failures do
+      expect(output).not_to include ';;;'
+      expect(output).to include "```json\n{\n  \"foo\": \":foo_symbol\",\n"
+    end
+  end
+
+  it 'converts arbitrary front matter to a fenced code block' do
+    content = <<~MD
+      ---arbitrary
+      foo = :foo_symbol
+      bar = :bar_symbol
+      ---
+
+      # Header
+
+      Content
+    MD
+
+    output = filter(content)
+
+    aggregate_failures do
+      expect(output).not_to include '---arbitrary'
+      expect(output).to include "```arbitrary\nfoo = :foo_symbol\n"
+    end
+  end
+
+  context 'on content without front matter' do
+    it 'returns the content unmodified' do
+      content = <<~MD
+        # This is some Markdown
+
+        It has no YAML front matter to parse.
+      MD
+
+      expect(filter(content)).to eq content
+    end
+  end
+
+  context 'on front matter without content' do
+    it 'converts YAML front matter to a fenced code block' do
+      content = <<~MD
+        ---
+        foo: :foo_symbol
+        bar: :bar_symbol
+        ---
+      MD
+
+      output = filter(content)
+
+      aggregate_failures do
+        expect(output).to eq <<~MD
+          ```yaml
+          foo: :foo_symbol
+          bar: :bar_symbol
+          ```
+
+        MD
+      end
+    end
+  end
+end

spec/lib/banzai/filter/yaml_front_matter_filter_spec.rb

-require 'rails_helper'
-
-describe Banzai::Filter::YamlFrontMatterFilter do
-  include FilterSpecHelper
-
-  it 'allows for `encoding:` before the frontmatter' do
-    content = <<-MD.strip_heredoc
-      # encoding: UTF-8
-      ---
-      foo: foo
-      ---
-
-      # Header
-
-      Content
-    MD
-
-    output = filter(content)
-
-    expect(output).not_to match 'encoding'
-  end
-
-  it 'converts YAML frontmatter to a fenced code block' do
-    content = <<-MD.strip_heredoc
-      ---
-      bar: :bar_symbol
-      ---
-
-      # Header
-
-      Content
-    MD
-
-    output = filter(content)
-
-    aggregate_failures do
-      expect(output).not_to include '---'
-      expect(output).to include "```yaml\nbar: :bar_symbol\n```"
-    end
-  end
-
-  context 'on content without frontmatter' do
-    it 'returns the content unmodified' do
-      content = <<-MD.strip_heredoc
-        # This is some Markdown
-
-        It has no YAML frontmatter to parse.
-      MD
-
-      expect(filter(content)).to eq content
-    end
-  end
-end