Merge branch 'ah-keyset-paginator' into 'master'

Keyset paginator for AR::Relation

See merge request gitlab-org/gitlab!60412

app/helpers/keyset_helper.rb

+# frozen_string_literal: true
+
+module KeysetHelper
+  def keyset_paginate(paginator, without_first_and_last_pages: false)
+    page_params = params.to_unsafe_h
+
+    render('kaminari/gitlab/keyset_paginator', {
+      paginator: paginator,
+      without_first_and_last_pages: without_first_and_last_pages,
+      page_params: page_params
+    })
+  end
+end

app/views/kaminari/gitlab/_keyset_paginator.html.haml

+- previous_path = url_for(page_params.merge(cursor: paginator.cursor_for_previous_page))
+- next_path = url_for(page_params.merge(cursor: paginator.cursor_for_next_page))
+
+.gl-pagination.gl-mt-3
+  %ul.pagination.justify-content-center
+
+    - if paginator.has_previous_page?
+      - unless without_first_and_last_pages
+        %li.page-item
+          - first_page_path = url_for(page_params.merge(cursor: paginator.cursor_for_first_page))
+          = link_to first_page_path, rel: 'first', class: 'page-link' do
+            = sprite_icon('angle-double-left', size: 8)
+            = s_('Pagination|First')
+
+      %li.page-item.prev
+        = link_to previous_path, rel: 'prev', class: 'page-link' do
+          = sprite_icon('angle-left', size: 8)
+          = s_('Pagination|Prev')
+
+    - if paginator.has_next_page?
+      %li.page-item.next
+        = link_to next_path, rel: 'next', class: 'page-link' do
+          = s_('Pagination|Next')
+          = sprite_icon('angle-right', size: 8)
+      - unless without_first_and_last_pages
+        %li.page-item
+          - last_page_path = url_for(page_params.merge(cursor: paginator.cursor_for_last_page))
+          = link_to last_page_path, rel: 'last', class: 'page-link' do
+            = s_('Pagination|Last')
+            = sprite_icon('angle-double-right', size: 8)

config/initializers/active_record_keyset_pagination.rb

+# frozen_string_literal: true
+
+module PaginatorExtension
+  # This method loads the records for the requested page and returns a keyset paginator object.
+  def keyset_paginate(cursor: nil, per_page: 20)
+    Gitlab::Pagination::Keyset::Paginator.new(scope: self.dup, cursor: cursor, per_page: per_page)
+  end
+end
+
+ActiveSupport.on_load(:active_record) do
+  ActiveRecord::Relation.include(PaginatorExtension)
+end

lib/gitlab/pagination/keyset/paginator.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Pagination
+    module Keyset
+      class Paginator
+        include Enumerable
+
+        module Base64CursorConverter
+          def self.dump(cursor_attributes)
+            Base64.urlsafe_encode64(Gitlab::Json.dump(cursor_attributes))
+          end
+
+          def self.parse(cursor)
+            Gitlab::Json.parse(Base64.urlsafe_decode64(cursor)).with_indifferent_access
+          end
+        end
+
+        FORWARD_DIRECTION = 'n'
+        BACKWARD_DIRECTION = 'p'
+
+        UnsupportedScopeOrder = Class.new(StandardError)
+
+        # scope                  - ActiveRecord::Relation object with order by clause
+        # cursor                 - Encoded cursor attributes as String. Empty value will requests the first page.
+        # per_page               - Number of items per page.
+        # cursor_converter       - Object that serializes and de-serializes the cursor attributes. Implements dump and parse methods.
+        # direction_key          - Symbol that will be the hash key of the direction within the cursor. (default: _kd => keyset direction)
+        def initialize(scope:, cursor: nil, per_page: 20, cursor_converter: Base64CursorConverter, direction_key: :_kd)
+          @keyset_scope = build_scope(scope)
+          @order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(@keyset_scope)
+          @per_page = per_page
+          @cursor_converter = cursor_converter
+          @direction_key = direction_key
+          @has_another_page = false
+          @at_last_page = false
+          @at_first_page = false
+          @cursor_attributes = decode_cursor_attributes(cursor)
+
+          set_pagination_helper_flags!
+        end
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        def records
+          @records ||= begin
+            items = if paginate_backward?
+                      reversed_order
+                        .apply_cursor_conditions(keyset_scope, cursor_attributes)
+                        .reorder(reversed_order)
+                        .limit(per_page_plus_one)
+                        .to_a
+                    else
+                      order
+                        .apply_cursor_conditions(keyset_scope, cursor_attributes)
+                        .limit(per_page_plus_one)
+                        .to_a
+                    end
+
+            @has_another_page = items.size == per_page_plus_one
+            items.pop if @has_another_page
+            items.reverse! if paginate_backward?
+            items
+          end
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+
+        # This and has_previous_page? methods are direction aware. In case we paginate backwards,
+        # has_next_page? will mean that we have a previous page.
+        def has_next_page?
+          records
+
+          if at_last_page?
+            false
+          elsif paginate_forward?
+            @has_another_page
+          elsif paginate_backward?
+            true
+          end
+        end
+
+        def has_previous_page?
+          records
+
+          if at_first_page?
+            false
+          elsif paginate_backward?
+            @has_another_page
+          elsif paginate_forward?
+            true
+          end
+        end
+
+        def cursor_for_next_page
+          if has_next_page?
+            data = order.cursor_attributes_for_node(records.last)
+            data[direction_key] = FORWARD_DIRECTION
+            cursor_converter.dump(data)
+          else
+            nil
+          end
+        end
+
+        def cursor_for_previous_page
+          if has_previous_page?
+            data = order.cursor_attributes_for_node(records.first)
+            data[direction_key] = BACKWARD_DIRECTION
+            cursor_converter.dump(data)
+          end
+        end
+
+        def cursor_for_first_page
+          cursor_converter.dump({ direction_key => FORWARD_DIRECTION })
+        end
+
+        def cursor_for_last_page
+          cursor_converter.dump({ direction_key => BACKWARD_DIRECTION })
+        end
+
+        delegate :each, :empty?, :any?, to: :records
+
+        private
+
+        attr_reader :keyset_scope, :order, :per_page, :cursor_converter, :direction_key, :cursor_attributes
+
+        delegate :reversed_order, to: :order
+
+        def at_last_page?
+          @at_last_page
+        end
+
+        def at_first_page?
+          @at_first_page
+        end
+
+        def per_page_plus_one
+          per_page + 1
+        end
+
+        def decode_cursor_attributes(cursor)
+          cursor.blank? ? {} : cursor_converter.parse(cursor)
+        end
+
+        def set_pagination_helper_flags!
+          @direction = cursor_attributes.delete(direction_key.to_s)
+
+          if cursor_attributes.blank? && @direction.blank?
+            @at_first_page = true
+            @direction = FORWARD_DIRECTION
+          elsif cursor_attributes.blank?
+            if paginate_forward?
+              @at_first_page = true
+            else
+              @at_last_page = true
+            end
+          end
+        end
+
+        def paginate_backward?
+          @direction == BACKWARD_DIRECTION
+        end
+
+        def paginate_forward?
+          @direction == FORWARD_DIRECTION
+        end
+
+        def build_scope(scope)
+          keyset_aware_scope, success = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(scope)
+
+          raise(UnsupportedScopeOrder, 'The order on the scope does not support keyset pagination') unless success
+
+          keyset_aware_scope
+        end
+      end
+    end
+  end
+end

lib/gitlab/pagination/keyset/simple_order_builder.rb

@@ -26,6 +26,8 @@ module Gitlab
         def build
           order = if order_values.empty?
                     primary_key_descending_order
+                  elsif Gitlab::Pagination::Keyset::Order.keyset_aware?(scope)
+                    Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope)
                   elsif ordered_by_primary_key?
                     primary_key_order
                   elsif ordered_by_other_column?

locale/gitlab.pot

@@ -23703,6 +23703,9 @@ msgstr ""
 msgid "Pages Domain"
 msgstr ""
 
+msgid "Pagination|First"
+msgstr ""
+
 msgid "Pagination|Go to first page"
 msgstr ""
 
@@ -23715,6 +23718,9 @@ msgstr ""
 msgid "Pagination|Go to previous page"
 msgstr ""
 
+msgid "Pagination|Last"
+msgstr ""
+
 msgid "Pagination|Last »"
 msgstr ""
 

spec/helpers/keyset_helper_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+RSpec.describe KeysetHelper, type: :controller do
+  controller(Admin::UsersController) do
+    def index
+      @users = User
+        .where(admin: false)
+        .order(id: :desc)
+        .keyset_paginate(cursor: params[:cursor], per_page: 2)
+
+      render inline: "<%= keyset_paginate @users %>", layout: false # rubocop: disable Rails/RenderInline
+    end
+  end
+
+  render_views
+
+  let(:admin) { create(:admin) }
+
+  before do
+    sign_in(admin)
+  end
+
+  context 'with admin mode', :enable_admin_mode do
+    context 'when no users are present' do
+      it 'does not render pagination links' do
+        get :index
+
+        expect(response.body).not_to include(s_('Pagination|First'))
+        expect(response.body).not_to include(s_('Pagination|Prev'))
+        expect(response.body).not_to include(s_('Pagination|Next'))
+        expect(response.body).not_to include(s_('Pagination|Last'))
+      end
+    end
+
+    context 'when one user is present' do
+      before do
+        create(:user)
+      end
+
+      it 'does not render pagination links' do
+        get :index
+
+        expect(response.body).not_to include(s_('Pagination|First'))
+        expect(response.body).not_to include(s_('Pagination|Prev'))
+        expect(response.body).not_to include(s_('Pagination|Next'))
+        expect(response.body).not_to include(s_('Pagination|Last'))
+      end
+    end
+
+    context 'when more users are present' do
+      let_it_be(:users) { create_list(:user, 5) }
+
+      let(:paginator) { User.where(admin: false).order(id: :desc).keyset_paginate(per_page: 2) }
+
+      context 'when on the first page' do
+        it 'renders the next and last links' do
+          get :index
+
+          expect(response.body).not_to include(s_('Pagination|First'))
+          expect(response.body).not_to include(s_('Pagination|Prev'))
+          expect(response.body).to include(s_('Pagination|Next'))
+          expect(response.body).to include(s_('Pagination|Last'))
+        end
+      end
+
+      context 'when at the last page' do
+        it 'renders the prev and first links' do
+          cursor = paginator.cursor_for_last_page
+
+          get :index, params: { cursor: cursor }
+
+          expect(response.body).to include(s_('Pagination|First'))
+          expect(response.body).to include(s_('Pagination|Prev'))
+          expect(response.body).not_to include(s_('Pagination|Next'))
+          expect(response.body).not_to include(s_('Pagination|Last'))
+        end
+      end
+
+      context 'when at the second page' do
+        it 'renders all links' do
+          cursor = paginator.cursor_for_next_page
+
+          get :index, params: { cursor: cursor }
+
+          expect(response.body).to include(s_('Pagination|First'))
+          expect(response.body).to include(s_('Pagination|Prev'))
+          expect(response.body).to include(s_('Pagination|Next'))
+          expect(response.body).to include(s_('Pagination|Last'))
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/pagination/keyset/paginator_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Pagination::Keyset::Paginator do
+  let_it_be(:project_1) { create(:project, created_at: 10.weeks.ago) }
+  let_it_be(:project_2) { create(:project, created_at: 2.weeks.ago) }
+  let_it_be(:project_3) { create(:project, created_at: 3.weeks.ago) }
+  let_it_be(:project_4) { create(:project, created_at: 5.weeks.ago) }
+  let_it_be(:project_5) { create(:project, created_at: 2.weeks.ago) }
+
+  describe 'pagination' do
+    let(:per_page) { 10 }
+    let(:cursor) { nil }
+    let(:scope) { Project.order(created_at: :asc, id: :asc) }
+    let(:expected_order) { [project_1, project_4, project_3, project_2, project_5] }
+
+    subject(:paginator) { scope.keyset_paginate(cursor: cursor, per_page: per_page) }
+
+    context 'when per_page is greater than the record count' do
+      it { expect(paginator.records).to eq(expected_order) }
+      it { is_expected.not_to have_next_page }
+      it { is_expected.not_to have_previous_page }
+
+      it 'has no next and previous cursor values' do
+        expect(paginator.cursor_for_next_page).to be_nil
+        expect(paginator.cursor_for_previous_page).to be_nil
+      end
+    end
+
+    context 'when 0 records are returned' do
+      let(:scope) { Project.where(id: non_existing_record_id).order(created_at: :asc, id: :asc) }
+
+      it { expect(paginator.records).to be_empty }
+      it { is_expected.not_to have_next_page }
+      it { is_expected.not_to have_previous_page }
+    end
+
+    context 'when page size is smaller than the record count' do
+      let(:per_page) { 2 }
+
+      it { expect(paginator.records).to eq(expected_order.first(2)) }
+      it { is_expected.to have_next_page }
+      it { is_expected.not_to have_previous_page }
+
+      it 'has next page cursor' do
+        expect(paginator.cursor_for_next_page).not_to be_nil
+      end
+
+      it 'does not have previous page cursor' do
+        expect(paginator.cursor_for_previous_page).to be_nil
+      end
+
+      context 'when on the second page' do
+        let(:cursor) { scope.keyset_paginate(per_page: per_page).cursor_for_next_page }
+
+        it { expect(paginator.records).to eq(expected_order[2...4]) }
+        it { is_expected.to have_next_page }
+        it { is_expected.to have_previous_page }
+
+        context 'and then going back to the first page' do
+          let(:previous_page_cursor) { scope.keyset_paginate(cursor: cursor, per_page: per_page).cursor_for_previous_page }
+
+          subject(:paginator) { scope.keyset_paginate(cursor: previous_page_cursor, per_page: per_page) }
+
+          it { expect(paginator.records).to eq(expected_order.first(2)) }
+          it { is_expected.to have_next_page }
+          it { is_expected.not_to have_previous_page }
+        end
+      end
+
+      context 'when jumping to the last page' do
+        let(:cursor) { scope.keyset_paginate(per_page: per_page).cursor_for_last_page }
+
+        it { expect(paginator.records).to eq(expected_order.last(2)) }
+        it { is_expected.not_to have_next_page }
+        it { is_expected.to have_previous_page }
+
+        context 'when paginating backwards' do
+          let(:previous_page_cursor) { scope.keyset_paginate(cursor: cursor, per_page: per_page).cursor_for_previous_page }
+
+          subject(:paginator) { scope.keyset_paginate(cursor: previous_page_cursor, per_page: per_page) }
+
+          it { expect(paginator.records).to eq(expected_order[-4...-2]) }
+          it { is_expected.to have_next_page }
+          it { is_expected.to have_previous_page }
+        end
+
+        context 'when jumping to the first page' do
+          let(:first_page_cursor) { scope.keyset_paginate(cursor: cursor, per_page: per_page).cursor_for_first_page }
+
+          subject(:paginator) { scope.keyset_paginate(cursor: first_page_cursor, per_page: per_page) }
+
+          it { expect(paginator.records).to eq(expected_order.first(2)) }
+          it { is_expected.to have_next_page }
+          it { is_expected.not_to have_previous_page }
+        end
+      end
+    end
+
+    describe 'default keyset direction parameter' do
+      let(:cursor_converter_class) { Gitlab::Pagination::Keyset::Paginator::Base64CursorConverter }
+      let(:per_page) { 2 }
+
+      it 'exposes the direction parameter in the cursor' do
+        cursor = paginator.cursor_for_next_page
+
+        expect(cursor_converter_class.parse(cursor)[:_kd]).to eq(described_class::FORWARD_DIRECTION)
+      end
+    end
+  end
+
+  context 'when unsupported order is given' do
+    it 'raises error' do
+      scope = Project.order(path: :asc, name: :asc, id: :desc) # Cannot build 3 column order automatically
+
+      expect { scope.keyset_paginate }.to raise_error(/does not support keyset pagination/)
+    end
+  end
+end