Add indexes for authenticated project api calls

This is to support queries like the following:

```sql
SELECT
    "projects".*
FROM
    "projects"
WHERE (EXISTS (
        SELECT
            1
        FROM
            "project_authorizations"
        WHERE
            "project_authorizations"."user_id" = 1562869
            AND (project_authorizations.project_id = projects.id))
        OR projects.visibility_level IN (10, 20))
AND "projects"."pending_delete" = FALSE
ORDER BY
    "projects"."updated_at" DESC,
    "projects"."id" DESC
LIMIT 20 OFFSET 0;
```

With this change, we add indexes for all order_by options. The regularly
sorted index (`asc, asc`) has the common name whereas the one with
`desc, desc` is optimized for API calls. It may be subject to removal
later (see https://gitlab.com/gitlab-org/gitlab/issues/195881).

Relates to https://gitlab.com/gitlab-org/gitlab/issues/36390.

A similar MR for unauthenticated calls is
https://gitlab.com/gitlab-org/gitlab/merge_requests/22784.

changelogs/unreleased/ab-projects-api-indexes-authenticated-calls.yml

+---
+title: Add indexes for authenticated Project API calls
+merge_request: 22886
+author:
+type: performance

db/migrate/20200113133352_add_indexes_for_projects_api_authenticated.rb

+# frozen_string_literal: true
+
+class AddIndexesForProjectsApiAuthenticated < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  COLUMNS = %i(updated_at name)
+
+  def up
+    add_concurrent_index :projects, %i(created_at id), order: { id: :desc }, name: 'index_projects_api_created_at_id_desc'
+
+    add_concurrent_index :projects, %i(last_activity_at id), name: 'index_projects_on_last_activity_at_and_id'
+    remove_concurrent_index :projects, :last_activity_at
+    add_concurrent_index :projects, %i(last_activity_at id), order: { id: :desc }, name: 'index_projects_api_last_activity_at_id_desc'
+
+    add_concurrent_index :projects, %i(path id), name: 'index_projects_on_path_and_id'
+    remove_concurrent_index_by_name :projects, 'index_projects_on_path'
+    add_concurrent_index :projects, %i(path id), order: { id: :desc }, name: 'index_projects_api_path_id_desc'
+
+    COLUMNS.each do |column|
+      add_concurrent_index :projects, [column, :id], name: "index_projects_on_#{column}_and_id"
+      add_concurrent_index :projects, [column, :id], order: { id: :desc }, name: "index_projects_api_#{column}_id_desc"
+    end
+  end
+
+  def down
+    remove_concurrent_index_by_name :projects, 'index_projects_api_created_at_id_desc'
+
+    remove_concurrent_index_by_name :projects, 'index_projects_on_last_activity_at_and_id'
+    add_concurrent_index :projects, :last_activity_at, name: 'index_projects_on_last_activity_at'
+    remove_concurrent_index_by_name :projects, 'index_projects_api_last_activity_at_id_desc'
+
+    remove_concurrent_index_by_name :projects, 'index_projects_on_path_and_id'
+    add_concurrent_index :projects, :path, name: 'index_projects_on_path'
+    remove_concurrent_index_by_name :projects, 'index_projects_api_path_id_desc'
+
+    COLUMNS.each do |column|
+      remove_concurrent_index_by_name :projects, "index_projects_on_#{column}_and_id"
+      remove_concurrent_index_by_name :projects, "index_projects_api_#{column}_id_desc"
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_01_10_144316) do
+ActiveRecord::Schema.define(version: 2020_01_13_133352) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"
@@ -3353,6 +3353,7 @@ ActiveRecord::Schema.define(version: 2020_01_10_144316) do
     t.boolean "autoclose_referenced_issues"
     t.string "suggestion_commit_message", limit: 255
     t.index "lower((name)::text)", name: "index_projects_on_lower_name"
+    t.index ["created_at", "id"], name: "index_projects_api_created_at_id_desc", order: { id: :desc }
     t.index ["created_at", "id"], name: "index_projects_api_vis20_created_at", where: "(visibility_level = 20)"
     t.index ["created_at", "id"], name: "index_projects_api_vis20_created_at_id_desc", order: { id: :desc }, where: "(visibility_level = 20)"
     t.index ["created_at", "id"], name: "index_projects_on_created_at_and_id"
@@ -3362,9 +3363,10 @@ ActiveRecord::Schema.define(version: 2020_01_10_144316) do
     t.index ["id"], name: "index_on_id_partial_with_legacy_storage", where: "((storage_version < 2) OR (storage_version IS NULL))"
     t.index ["id"], name: "index_projects_on_id_partial_for_visibility", unique: true, where: "(visibility_level = ANY (ARRAY[10, 20]))"
     t.index ["id"], name: "index_projects_on_mirror_and_mirror_trigger_builds_both_true", where: "((mirror IS TRUE) AND (mirror_trigger_builds IS TRUE))"
+    t.index ["last_activity_at", "id"], name: "index_projects_api_last_activity_at_id_desc", order: { id: :desc }
     t.index ["last_activity_at", "id"], name: "index_projects_api_vis20_last_activity_at", where: "(visibility_level = 20)"
     t.index ["last_activity_at", "id"], name: "index_projects_api_vis20_last_activity_at_id_desc", order: { id: :desc }, where: "(visibility_level = 20)"
-    t.index ["last_activity_at"], name: "index_projects_on_last_activity_at"
+    t.index ["last_activity_at", "id"], name: "index_projects_on_last_activity_at_and_id"
     t.index ["last_repository_check_at"], name: "index_projects_on_last_repository_check_at", where: "(last_repository_check_at IS NOT NULL)"
     t.index ["last_repository_check_failed"], name: "index_projects_on_last_repository_check_failed"
     t.index ["last_repository_updated_at"], name: "index_projects_on_last_repository_updated_at"
@@ -3372,13 +3374,16 @@ ActiveRecord::Schema.define(version: 2020_01_10_144316) do
     t.index ["marked_for_deletion_by_user_id"], name: "index_projects_on_marked_for_deletion_by_user_id", where: "(marked_for_deletion_by_user_id IS NOT NULL)"
     t.index ["mirror_last_successful_update_at"], name: "index_projects_on_mirror_last_successful_update_at"
     t.index ["mirror_user_id"], name: "index_projects_on_mirror_user_id"
+    t.index ["name", "id"], name: "index_projects_api_name_id_desc", order: { id: :desc }
     t.index ["name", "id"], name: "index_projects_api_vis20_name", where: "(visibility_level = 20)"
     t.index ["name", "id"], name: "index_projects_api_vis20_name_id_desc", order: { id: :desc }, where: "(visibility_level = 20)"
+    t.index ["name", "id"], name: "index_projects_on_name_and_id"
     t.index ["name"], name: "index_projects_on_name_trigram", opclass: :gin_trgm_ops, using: :gin
     t.index ["namespace_id"], name: "index_projects_on_namespace_id"
+    t.index ["path", "id"], name: "index_projects_api_path_id_desc", order: { id: :desc }
     t.index ["path", "id"], name: "index_projects_api_vis20_path", where: "(visibility_level = 20)"
     t.index ["path", "id"], name: "index_projects_api_vis20_path_id_desc", order: { id: :desc }, where: "(visibility_level = 20)"
-    t.index ["path"], name: "index_projects_on_path"
+    t.index ["path", "id"], name: "index_projects_on_path_and_id"
     t.index ["path"], name: "index_projects_on_path_trigram", opclass: :gin_trgm_ops, using: :gin
     t.index ["pending_delete"], name: "index_projects_on_pending_delete"
     t.index ["pool_repository_id"], name: "index_projects_on_pool_repository_id", where: "(pool_repository_id IS NOT NULL)"
@@ -3387,8 +3392,10 @@ ActiveRecord::Schema.define(version: 2020_01_10_144316) do
     t.index ["runners_token"], name: "index_projects_on_runners_token"
     t.index ["runners_token_encrypted"], name: "index_projects_on_runners_token_encrypted"
     t.index ["star_count"], name: "index_projects_on_star_count"
+    t.index ["updated_at", "id"], name: "index_projects_api_updated_at_id_desc", order: { id: :desc }
     t.index ["updated_at", "id"], name: "index_projects_api_vis20_updated_at", where: "(visibility_level = 20)"
     t.index ["updated_at", "id"], name: "index_projects_api_vis20_updated_at_id_desc", order: { id: :desc }, where: "(visibility_level = 20)"
+    t.index ["updated_at", "id"], name: "index_projects_on_updated_at_and_id"
   end
 
   create_table "prometheus_alert_events", force: :cascade do |t|

spec/lib/gitlab/database/migration_helpers_spec.rb

@@ -1475,7 +1475,11 @@ describe Gitlab::Database::MigrationHelpers do
 
   describe '#index_exists_by_name?' do
     it 'returns true if an index exists' do
-      expect(model.index_exists_by_name?(:projects, 'index_projects_on_path'))
+      ActiveRecord::Base.connection.execute(
+        'CREATE INDEX test_index_for_index_exists ON projects (path);'
+      )
+
+      expect(model.index_exists_by_name?(:projects, 'test_index_for_index_exists'))
         .to be_truthy
     end