Merge branch 'ee-34600-performance-wiki-pages' into 'master'

CE Port: Performance issues when loading large number of wiki pages

See merge request gitlab-org/gitlab-ee!3420

app/controllers/projects/wikis_controller.rb

@@ -74,7 +74,11 @@ class Projects::WikisController < Projects::ApplicationController
   def history
     @page = @project_wiki.find_page(params[:id])
 
-    unless @page
+    if @page
+      @page_versions = Kaminari.paginate_array(@page.versions(page: params[:page]),
+                                               total_count: @page.count_versions)
+                               .page(params[:page])
+    else
       redirect_to(
         project_wiki_path(@project, :home),
         notice: "Page not found"
@@ -102,7 +106,7 @@ class Projects::WikisController < Projects::ApplicationController
 
     # Call #wiki to make sure the Wiki Repo is initialized
     @project_wiki.wiki
-    @sidebar_wiki_entries = WikiPage.group_by_directory(@project_wiki.pages.first(15))
+    @sidebar_wiki_entries = WikiPage.group_by_directory(@project_wiki.pages(limit: 15))
   rescue ProjectWiki::CouldNotCreateWikiError
     flash[:notice] = "Could not create Wiki Repository at this time. Please try again later."
     redirect_to project_path(@project)

app/models/project_wiki.rb

@@ -85,8 +85,8 @@ class ProjectWiki
 
   # Returns an Array of Gitlab WikiPage instances or an
   # empty Array if this Wiki has no pages.
-  def pages
-    wiki.pages.map { |page| WikiPage.new(self, page, true) }
+  def pages(limit: nil)
+    wiki.pages(limit: limit).map { |page| WikiPage.new(self, page, true) }
   end
 
   # Finds a page within the repository based on a tile

app/models/wiki_page.rb

@@ -127,19 +127,24 @@ class WikiPage
     @version ||= @page.version
   end
 
-  # Returns an array of Gitlab Commit instances.
-  def versions
+  def versions(options = {})
     return [] unless persisted?
 
-    wiki.wiki.page_versions(@page.path)
+    wiki.wiki.page_versions(@page.path, options)
   end
 
-  def commit
-    versions.first
+  def count_versions
+    return [] unless persisted?
+
+    wiki.wiki.count_page_versions(@page.path)
+  end
+
+  def last_version
+    @last_version ||= versions(limit: 1).first
   end
 
   def last_commit_sha
-    commit&.sha
+    last_version&.sha
   end
 
   # Returns the Date that this latest version was
@@ -151,7 +156,7 @@ class WikiPage
   # Returns boolean True or False if this instance
   # is an old version of the page.
   def historical?
-    @page.historical? && versions.first.sha != version.sha
+    @page.historical? && last_version.sha != version.sha
   end
 
   # Returns boolean True or False if this instance

app/views/projects/wikis/_pages_wiki_page.html.haml

@@ -2,4 +2,4 @@
   = link_to wiki_page.title, project_wiki_path(@project, wiki_page)
   %small (#{wiki_page.format})
   .pull-right
-    %small= (s_("Last edited %{date}") % { date: time_ago_with_tooltip(wiki_page.commit.authored_date) }).html_safe
+    %small= (s_("Last edited %{date}") % { date: time_ago_with_tooltip(wiki_page.last_version.authored_date) }).html_safe

app/views/projects/wikis/history.html.haml

@@ -21,7 +21,7 @@
         %th= _("Last updated")
         %th= _("Format")
     %tbody
-      - @page.versions.each_with_index do |version, index|
+      - @page_versions.each_with_index do |version, index|
         - commit = version
         %tr
           %td
@@ -37,5 +37,6 @@
           %td
             %strong
               = version.format
+= paginate @page_versions, theme: 'gitlab'
 
 = render 'sidebar'

app/views/projects/wikis/show.html.haml

@@ -11,8 +11,8 @@
   .nav-text
     %h2.wiki-page-title= @page.title.capitalize
     %span.wiki-last-edit-by
-      = (_("Last edited by %{name}") % { name: "<strong>#{@page.commit.author_name}</strong>" }).html_safe
-      #{time_ago_with_tooltip(@page.commit.authored_date)}
+      = (_("Last edited by %{name}") % { name: "<strong>#{@page.last_version.author_name}</strong>" }).html_safe
+      #{time_ago_with_tooltip(@page.last_version.authored_date)}
 
   .nav-controls
     = render 'main_links'

config/initializers/gollum.rb

@@ -10,4 +10,30 @@ module Gollum
       index.send(name, *args)
     end
   end
+
+  class Wiki
+    def pages(treeish = nil, limit: nil)
+      tree_list((treeish || @ref), limit: limit)
+    end
+
+    def tree_list(ref, limit: nil)
+      if (sha = @access.ref_to_sha(ref))
+        commit = @access.commit(sha)
+        tree_map_for(sha).inject([]) do |list, entry|
+          next list unless @page_class.valid_page_name?(entry.name)
+          list << entry.page(self, commit)
+          break list if limit && list.size >= limit
+          list
+        end
+      else
+        []
+      end
+    end
+  end
+end
+
+Rails.application.configure do
+  config.after_initialize do
+    Gollum::Page.per_page = Kaminari.config.default_per_page
+  end
 end

lib/gitlab/git/wiki.rb

@@ -58,12 +58,12 @@ module Gitlab
         end
       end
 
-      def pages
-        @repository.gitaly_migrate(:wiki_get_all_pages) do |is_enabled|
+      def pages(limit: nil)
+        @repository.gitaly_migrate(:wiki_get_all_pages, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled|
           if is_enabled
             gitaly_get_all_pages
           else
-            gollum_get_all_pages
+            gollum_get_all_pages(limit: limit)
           end
         end
       end
@@ -88,14 +88,23 @@ module Gitlab
         end
       end
 
-      def page_versions(page_path)
+      # options:
+      #  :page     - The Integer page number.
+      #  :per_page - The number of items per page.
+      #  :limit    - Total number of items to return.
+      def page_versions(page_path, options = {})
         current_page = gollum_page_by_path(page_path)
-        current_page.versions.map do |gollum_git_commit|
-          gollum_page = gollum_wiki.page(current_page.title, gollum_git_commit.id)
-          new_version(gollum_page, gollum_git_commit.id)
+
+        commits_from_page(current_page, options).map do |gitlab_git_commit|
+          gollum_page = gollum_wiki.page(current_page.title, gitlab_git_commit.id)
+          Gitlab::Git::WikiPageVersion.new(gitlab_git_commit, gollum_page&.format)
         end
       end
 
+      def count_page_versions(page_path, options = {})
+        @repository.count_commits(ref: 'HEAD', path: page_path)
+      end
+
       def preview_slug(title, format)
         # Adapted from gollum gem (Gollum::Wiki#preview_page) to avoid
         # using Rugged through a Gollum::Wiki instance
@@ -110,6 +119,22 @@ module Gitlab
 
       private
 
+      # options:
+      #  :page     - The Integer page number.
+      #  :per_page - The number of items per page.
+      #  :limit    - Total number of items to return.
+      def commits_from_page(gollum_page, options = {})
+        unless options[:limit]
+          options[:offset] = ([1, options.delete(:page).to_i].max - 1) * Gollum::Page.per_page
+          options[:limit] = (options.delete(:per_page) || Gollum::Page.per_page).to_i
+        end
+
+        @repository.log(ref: gollum_page.last_version.id,
+                        path: gollum_page.path,
+                        limit: options[:limit],
+                        offset: options[:offset])
+      end
+
       def gollum_wiki
         @gollum_wiki ||= Gollum::Wiki.new(@repository.path)
       end
@@ -126,8 +151,17 @@ module Gitlab
       end
 
       def new_version(gollum_page, commit_id)
-        commit = Gitlab::Git::Commit.find(@repository, commit_id)
-        Gitlab::Git::WikiPageVersion.new(commit, gollum_page&.format)
+        Gitlab::Git::WikiPageVersion.new(version(commit_id), gollum_page&.format)
+      end
+
+      def version(commit_id)
+        commit_find_proc = -> { Gitlab::Git::Commit.find(@repository, commit_id) }
+
+        if RequestStore.active?
+          RequestStore.fetch([:wiki_version_commit, commit_id]) { commit_find_proc.call }
+        else
+          commit_find_proc.call
+        end
       end
 
       def assert_type!(object, klass)
@@ -185,8 +219,8 @@ module Gitlab
         Gitlab::Git::WikiFile.new(gollum_file)
       end
 
-      def gollum_get_all_pages
-        gollum_wiki.pages.map { |gollum_page| new_page(gollum_page) }
+      def gollum_get_all_pages(limit: nil)
+        gollum_wiki.pages(limit: limit).map { |gollum_page| new_page(gollum_page) }
       end
 
       def gitaly_write_page(name, format, content, commit_details)

spec/models/wiki_page_spec.rb

@@ -373,7 +373,7 @@ describe WikiPage do
     end
 
     it 'returns commit sha' do
-      expect(@page.last_commit_sha).to eq @page.commit.sha
+      expect(@page.last_commit_sha).to eq @page.last_version.sha
     end
 
     it 'is changed after page updated' do