Merge branch '44361-remove-gitlab-grit' into 'master'

Remove gitlab-grit as a transitive dependency of GitLab

Closes #44361

See merge request gitlab-org/gitlab-ce!22373

Gemfile

@@ -79,10 +79,6 @@ gem 'gpgme'
 gem 'gitlab_omniauth-ldap', '~> 2.0.4', require: 'omniauth-ldap'
 gem 'net-ldap'
 
-# Git Wiki
-# Only used to compute wiki page slugs
-gem 'gitlab-gollum-lib', '~> 4.2', require: false
-
 # API
 gem 'grape', '~> 1.1'
 gem 'grape-entity', '~> 0.7.1'
@@ -210,7 +206,7 @@ gem 'hipchat', '~> 1.5.0'
 gem 'jira-ruby', '~> 1.4'
 
 # Flowdock integration
-gem 'gitlab-flowdock-git-hook', '~> 1.0.1'
+gem 'flowdock', '~> 0.7'
 
 # Slack integration
 gem 'slack-notifier', '~> 1.5.1'

Gemfile.lock

@@ -278,23 +278,6 @@ GEM
       google-protobuf (~> 3.1)
       grpc (~> 1.10)
     github-markup (1.7.0)
-    gitlab-flowdock-git-hook (1.0.1)
-      flowdock (~> 0.7)
-      gitlab-grit (>= 2.4.1)
-      multi_json
-    gitlab-gollum-lib (4.2.7.5)
-      gemojione (~> 3.2)
-      github-markup (~> 1.6)
-      gollum-grit_adapter (~> 1.0)
-      nokogiri (>= 1.6.1, < 2.0)
-      rouge (~> 3.1)
-      sanitize (~> 4.6.4)
-      stringex (~> 2.6)
-    gitlab-grit (2.8.2)
-      charlock_holmes (~> 0.6)
-      diff-lcs (~> 1.1)
-      mime-types (>= 1.16)
-      posix-spawn (~> 0.3)
     gitlab-markup (1.6.4)
     gitlab-sidekiq-fetcher (0.3.0)
       sidekiq (~> 5)
@@ -309,8 +292,6 @@ GEM
       rubyntlm (~> 0.5)
     globalid (0.4.1)
       activesupport (>= 4.2.0)
-    gollum-grit_adapter (1.0.1)
-      gitlab-grit (~> 2.7, >= 2.7.1)
     gon (6.2.0)
       actionpack (>= 3.0)
       multi_json
@@ -594,7 +575,6 @@ GEM
     pg (0.18.4)
     po_to_json (1.0.1)
       json (>= 1.6.0)
-    posix-spawn (0.3.13)
     powerpack (0.1.1)
     premailer (1.10.4)
       addressable
@@ -865,7 +845,6 @@ GEM
     state_machines-activerecord (0.5.1)
       activerecord (>= 4.1, < 6.0)
       state_machines-activemodel (>= 0.5.0)
-    stringex (2.8.4)
     sys-filesystem (1.1.6)
       ffi
     sysexits (1.2.0)
@@ -1003,6 +982,7 @@ DEPENDENCIES
   flipper (~> 0.13.0)
   flipper-active_record (~> 0.13.0)
   flipper-active_support_cache_store (~> 0.13.0)
+  flowdock (~> 0.7)
   fog-aliyun (~> 0.2.0)
   fog-aws (~> 2.0.1)
   fog-core (~> 1.44)
@@ -1019,8 +999,6 @@ DEPENDENCIES
   gettext_i18n_rails_js (~> 1.3)
   gitaly-proto (~> 0.118.1)
   github-markup (~> 1.7.0)
-  gitlab-flowdock-git-hook (~> 1.0.1)
-  gitlab-gollum-lib (~> 4.2)
   gitlab-markup (~> 1.6.4)
   gitlab-sidekiq-fetcher
   gitlab-styles (~> 2.4)

Gemfile.rails5.lock

@@ -281,23 +281,6 @@ GEM
       google-protobuf (~> 3.1)
       grpc (~> 1.10)
     github-markup (1.7.0)
-    gitlab-flowdock-git-hook (1.0.1)
-      flowdock (~> 0.7)
-      gitlab-grit (>= 2.4.1)
-      multi_json
-    gitlab-gollum-lib (4.2.7.5)
-      gemojione (~> 3.2)
-      github-markup (~> 1.6)
-      gollum-grit_adapter (~> 1.0)
-      nokogiri (>= 1.6.1, < 2.0)
-      rouge (~> 3.1)
-      sanitize (~> 4.6.4)
-      stringex (~> 2.6)
-    gitlab-grit (2.8.2)
-      charlock_holmes (~> 0.6)
-      diff-lcs (~> 1.1)
-      mime-types (>= 1.16)
-      posix-spawn (~> 0.3)
     gitlab-markup (1.6.4)
     gitlab-sidekiq-fetcher (0.3.0)
       sidekiq (~> 5)
@@ -312,8 +295,6 @@ GEM
       rubyntlm (~> 0.5)
     globalid (0.4.1)
       activesupport (>= 4.2.0)
-    gollum-grit_adapter (1.0.1)
-      gitlab-grit (~> 2.7, >= 2.7.1)
     gon (6.2.0)
       actionpack (>= 3.0)
       multi_json
@@ -598,7 +579,6 @@ GEM
     pg (0.18.4)
     po_to_json (1.0.1)
       json (>= 1.6.0)
-    posix-spawn (0.3.13)
     powerpack (0.1.1)
     premailer (1.10.4)
       addressable
@@ -873,7 +853,6 @@ GEM
     state_machines-activerecord (0.5.1)
       activerecord (>= 4.1, < 6.0)
       state_machines-activemodel (>= 0.5.0)
-    stringex (2.8.4)
     sys-filesystem (1.1.6)
       ffi
     sysexits (1.2.0)
@@ -1012,6 +991,7 @@ DEPENDENCIES
   flipper (~> 0.13.0)
   flipper-active_record (~> 0.13.0)
   flipper-active_support_cache_store (~> 0.13.0)
+  flowdock (~> 0.7)
   fog-aliyun (~> 0.2.0)
   fog-aws (~> 2.0.1)
   fog-core (~> 1.44)
@@ -1028,8 +1008,6 @@ DEPENDENCIES
   gettext_i18n_rails_js (~> 1.3)
   gitaly-proto (~> 0.118.1)
   github-markup (~> 1.7.0)
-  gitlab-flowdock-git-hook (~> 1.0.1)
-  gitlab-gollum-lib (~> 4.2)
   gitlab-markup (~> 1.6.4)
   gitlab-sidekiq-fetcher
   gitlab-styles (~> 2.4)

app/models/project_services/flowdock_service.rb

 # frozen_string_literal: true
 
-require "flowdock-git-hook"
-
-# Flow dock depends on Grit to compute the number of commits between two given
-# commits. To make this depend on Gitaly, a monkey patch is applied
-module Flowdock
-  class Git
-    # pass down a Repository all the way down
-    def repo
-      @options[:repo]
-    end
-
-    def config
-      {}
-    end
-
-    def messages
-      Git::Builder.new(repo: repo,
-                       ref: @ref,
-                       before: @from,
-                       after: @to,
-                       commit_url: @commit_url,
-                       branch_url: @branch_url,
-                       diff_url: @diff_url,
-                       repo_url: @repo_url,
-                       repo_name: @repo_name,
-                       permanent_refs: @permanent_refs,
-                       tags: tags
-                      ).to_hashes
-    end
-
-    class Builder
-      def commits
-        @repo.commits_between(@before, @after).map do |commit|
-          {
-            url: @opts[:commit_url] ? @opts[:commit_url] % [commit.sha] : nil,
-            id: commit.sha,
-            message: commit.message,
-            author: {
-              name: commit.author_name,
-              email: commit.author_email
-            }
-          }
-        end
-      end
-    end
-  end
-end
-
 class FlowdockService < Service
   prop_accessor :token
   validates :token, presence: true, if: :activated?

lib/flowdock/git.rb

+# frozen_string_literal: true
+require 'flowdock'
+require 'flowdock/git/builder'
+
+module Flowdock
+  class Git
+    TokenError = Class.new(StandardError)
+
+    DEFAULT_PERMANENT_REFS = [
+      Regexp.new('refs/heads/master')
+    ].freeze
+
+    class << self
+      def post(ref, from, to, options = {})
+        Git.new(ref, from, to, options).post
+      end
+    end
+
+    def initialize(ref, from, to, options = {})
+      raise TokenError.new("Flowdock API token not found") unless options[:token]
+
+      @ref = ref
+      @from = from
+      @to = to
+      @options = options
+      @token = options[:token]
+      @commit_url = options[:commit_url]
+      @diff_url = options[:diff_url]
+      @repo_url = options[:repo_url]
+      @repo_name = options[:repo_name]
+      @permanent_refs = options.fetch(:permanent_refs, DEFAULT_PERMANENT_REFS)
+    end
+
+    # Send git push notification to Flowdock
+    def post
+      messages.each do |message|
+        Flowdock::Client.new(flow_token: @token).post_to_thread(message)
+      end
+    end
+
+    def repo
+      @options[:repo]
+    end
+
+    private
+
+    def messages
+      Git::Builder.new(repo: repo,
+                       ref: @ref,
+                       before: @from,
+                       after: @to,
+                       commit_url: @commit_url,
+                       branch_url: @branch_url,
+                       diff_url: @diff_url,
+                       repo_url: @repo_url,
+                       repo_name: @repo_name,
+                       permanent_refs: @permanent_refs,
+                       tags: tags
+                      ).to_hashes
+    end
+
+    # Flowdock tags attached to the push notification
+    def tags
+      Array(@options[:tags]).map { |tag| CGI.escape(tag) }
+    end
+  end
+end

lib/flowdock/git/builder.rb

+# frozen_string_literal: true
+module Flowdock
+  class Git
+    class Commit
+      def initialize(external_thread_id, thread, tags, commit)
+        @commit = commit
+        @external_thread_id = external_thread_id
+        @thread = thread
+        @tags = tags
+      end
+
+      def to_hash
+        hash = {
+          external_thread_id: @external_thread_id,
+          event: "activity",
+          author: {
+            name: @commit[:author][:name],
+            email: @commit[:author][:email]
+          },
+          title: title,
+          thread: @thread,
+          body: body
+        }
+        hash[:tags] = @tags if @tags
+        encode(hash)
+      end
+
+      private
+
+      def encode(hash)
+        return hash unless "".respond_to?(:encode)
+
+        encode_as_utf8(hash)
+      end
+
+      # This only works on Ruby 1.9
+      def encode_as_utf8(obj)
+        if obj.is_a? Hash
+          obj.each_pair do |key, val|
+            encode_as_utf8(val)
+          end
+        elsif obj.is_a?(Array)
+          obj.each do |val|
+            encode_as_utf8(val)
+          end
+        elsif obj.is_a?(String) && obj.encoding != Encoding::UTF_8
+          unless obj.force_encoding("UTF-8").valid_encoding?
+            obj.force_encoding("ISO-8859-1").encode!(Encoding::UTF_8, invalid: :replace, undef: :replace)
+          end
+        end
+      end
+
+      def body
+        content = @commit[:message][first_line.size..-1]
+        content.strip! if content
+        "<pre>#{content}</pre>" unless content.empty?
+      end
+
+      def first_line
+        @first_line ||= (@commit[:message].split("\n")[0] || @commit[:message])
+      end
+
+      def title
+        commit_id = @commit[:id][0, 7]
+        if @commit[:url]
+          "<a href=\"#{@commit[:url]}\">#{commit_id}</a> #{message_title}"
+        else
+          "#{commit_id} #{message_title}"
+        end
+      end
+
+      def message_title
+        CGI.escape_html(first_line.strip)
+      end
+    end
+
+    # Class used to build Git payload
+    class Builder
+      include ::Gitlab::Utils::StrongMemoize
+
+      def initialize(opts)
+        @repo = opts[:repo]
+        @ref = opts[:ref]
+        @before = opts[:before]
+        @after = opts[:after]
+        @opts = opts
+      end
+
+      def commits
+        @repo.commits_between(@before, @after).map do |commit|
+          {
+            url: @opts[:commit_url] ? @opts[:commit_url] % [commit.sha] : nil,
+            id: commit.sha,
+            message: commit.message,
+            author: {
+              name: commit.author_name,
+              email: commit.author_email
+            }
+          }
+        end
+      end
+
+      def ref_name
+        @ref.to_s.sub(%r{\Arefs/(heads|tags)/}, '')
+      end
+
+      def to_hashes
+        commits.map do |commit|
+          Commit.new(external_thread_id, thread, @opts[:tags], commit).to_hash
+        end
+      end
+
+      private
+
+      def thread
+        @thread ||= {
+          title: thread_title,
+          external_url: @opts[:repo_url]
+        }
+      end
+
+      def permanent?
+        strong_memoize(:permanent) do
+          @opts[:permanent_refs].any? { |regex| regex.match(@ref) }
+        end
+      end
+
+      def thread_title
+        action = "updated" if permanent?
+        type = @ref =~ %r(^refs/heads/) ? "branch" : "tag"
+
+        [@opts[:repo_name], type, ref_name, action].compact.join(" ")
+      end
+
+      def external_thread_id
+        @external_thread_id ||=
+          if permanent?
+            SecureRandom.hex
+          else
+            @ref
+          end
+      end
+    end
+  end
+end

lib/gitlab/git/wiki.rb

-# We only need Gollum::Page so let's not load all of gollum-lib.
-require 'gollum-lib/pagination'
-require 'gollum-lib/wiki'
-require 'gollum-lib/page'
-
 module Gitlab
   module Git
     class Wiki
@@ -16,7 +11,43 @@ module Gitlab
           { user_id: user_id, username: username, name: name, email: email, message: message }
         end
       end
-      PageBlob = Struct.new(:name)
+
+      # GollumSlug inlines just enough knowledge from Gollum::Page to generate a
+      # slug, which is used when previewing pages that haven't been persisted
+      class GollumSlug
+        class << self
+          def cname(name, char_white_sub = '-', char_other_sub = '-')
+            if name.respond_to?(:gsub)
+              name.gsub(/\s/, char_white_sub).gsub(/[<>+]/, char_other_sub)
+            else
+              ''
+            end
+          end
+
+          def format_to_ext(format)
+            format == :markdown ? "md" : format.to_s
+          end
+
+          def canonicalize_filename(filename)
+            ::File.basename(filename, ::File.extname(filename)).tr('-', ' ')
+          end
+
+          def generate(title, format)
+            ext = format_to_ext(format.to_sym)
+            name = cname(title) + '.' + ext
+            canonical_name = canonicalize_filename(name)
+
+            path =
+              if name.include?('/')
+                name.sub(%r{/[^/]+$}, '/')
+              else
+                ''
+              end
+
+            path + cname(canonical_name, '-', '-')
+          end
+        end
+      end
 
       attr_reader :repository
 
@@ -90,15 +121,7 @@ module Gitlab
       end
 
       def preview_slug(title, format)
-        # Adapted from gollum gem (Gollum::Wiki#preview_page) to avoid
-        # using Rugged through a Gollum::Wiki instance
-        page_class = Gollum::Page
-        page = page_class.new(nil)
-        ext = page_class.format_to_ext(format.to_sym)
-        name = page_class.cname(title) + '.' + ext
-        blob = PageBlob.new(name)
-        page.populate(blob)
-        page.url_path
+        GollumSlug.generate(title, format)
       end
 
       def page_formatted_data(title:, dir: nil, version: nil)

spec/lib/gitlab/git/wiki_spec.rb

 require 'spec_helper'
 
 describe Gitlab::Git::Wiki do
+  using RSpec::Parameterized::TableSyntax
+
   let(:project) { create(:project) }
   let(:user) { project.owner }
   let(:project_wiki) { ProjectWiki.new(project, user) }
-  subject { project_wiki.wiki }
+
+  subject(:wiki) { project_wiki.wiki }
 
   describe '#pages' do
     before do
@@ -64,8 +67,44 @@ describe Gitlab::Git::Wiki do
     end
   end
 
-  def create_page(name, content)
-    subject.write_page(name, :markdown, content, commit_details(name))
+  describe '#preview_slug' do
+    where(:title, :format, :expected_slug) do
+      'The Best Thing'       | :markdown  | 'The-Best-Thing'
+      'The Best Thing'       | :md        | 'The-Best-Thing'
+      'The Best Thing'       | :txt       | 'The-Best-Thing'
+      'A Subject/Title Here' | :txt       | 'A-Subject/Title-Here'
+      'A subject'            | :txt       | 'A-subject'
+      'A 1/B 2/C 3'          | :txt       | 'A-1/B-2/C-3'
+      'subject/title'        | :txt       | 'subject/title'
+      'subject/title.md'     | :txt       | 'subject/title.md'
+      'foo<bar>+baz'         | :txt       | 'foo-bar--baz'
+      'foo%2Fbar'            | :txt       | 'foo%2Fbar'
+      ''                     | :markdown  | '.md'
+      ''                     | :md        | '.md'
+      ''                     | :txt       | '.txt'
+    end
+
+    with_them do
+      subject { wiki.preview_slug(title, format) }
+
+      let(:gitaly_slug) { wiki.pages.first }
+
+      it { is_expected.to eq(expected_slug) }
+
+      it 'matches the slug generated by gitaly' do
+        skip('Gitaly cannot generate a slug for an empty title') unless title.present?
+
+        create_page(title, 'content', format: format)
+
+        gitaly_slug = wiki.pages.first.url_path
+
+        is_expected.to eq(gitaly_slug)
+      end
+    end
+  end
+
+  def create_page(name, content, format: :markdown)
+    wiki.write_page(name, format, content, commit_details(name))
   end
 
   def commit_details(name)
@@ -73,7 +112,7 @@ describe Gitlab::Git::Wiki do
   end
 
   def destroy_page(title, dir = '')
-    page = subject.page(title: title, dir: dir)
+    page = wiki.page(title: title, dir: dir)
     project_wiki.delete_page(page, "test commit")
   end
 end