Commit d3b91e2d authored by Stan Hu's avatar Stan Hu

Merge branch '31031-gitaly-n-1-queries-in-api-version-groups-id' into 'master'

Resolve "Gitaly N+1 queries in /api/:version/groups/:id"

Closes #31031

See merge request gitlab-org/gitlab!20023
parents 3d6e073f 49522f94
...@@ -11,6 +11,7 @@ ...@@ -11,6 +11,7 @@
# options: # options:
# only_owned: boolean # only_owned: boolean
# only_shared: boolean # only_shared: boolean
# limit: integer
# params: # params:
# sort: string # sort: string
# visibility_level: int # visibility_level: int
...@@ -20,6 +21,8 @@ ...@@ -20,6 +21,8 @@
# non_archived: boolean # non_archived: boolean
# #
class GroupProjectsFinder < ProjectsFinder class GroupProjectsFinder < ProjectsFinder
DEFAULT_PROJECTS_LIMIT = 100
attr_reader :group, :options attr_reader :group, :options
def initialize(group:, params: {}, options: {}, current_user: nil, project_ids_relation: nil) def initialize(group:, params: {}, options: {}, current_user: nil, project_ids_relation: nil)
...@@ -32,8 +35,19 @@ class GroupProjectsFinder < ProjectsFinder ...@@ -32,8 +35,19 @@ class GroupProjectsFinder < ProjectsFinder
@options = options @options = options
end end
def execute
collection = super
limit(collection)
end
private private
def limit(collection)
limit = options[:limit]
limit.present? ? collection.with_limit(limit) : collection
end
def init_collection def init_collection
projects = if current_user projects = if current_user
collection_with_user collection_with_user
......
...@@ -443,6 +443,7 @@ class Project < ApplicationRecord ...@@ -443,6 +443,7 @@ class Project < ApplicationRecord
scope :with_merge_requests_available_for_user, ->(current_user) { with_feature_available_for_user(:merge_requests, current_user) } scope :with_merge_requests_available_for_user, ->(current_user) { with_feature_available_for_user(:merge_requests, current_user) }
scope :with_merge_requests_enabled, -> { with_feature_enabled(:merge_requests) } scope :with_merge_requests_enabled, -> { with_feature_enabled(:merge_requests) }
scope :with_remote_mirrors, -> { joins(:remote_mirrors).where(remote_mirrors: { enabled: true }).distinct } scope :with_remote_mirrors, -> { joins(:remote_mirrors).where(remote_mirrors: { enabled: true }).distinct }
scope :with_limit, -> (maximum) { limit(maximum) }
scope :with_group_runners_enabled, -> do scope :with_group_runners_enabled, -> do
joins(:ci_cd_settings) joins(:ci_cd_settings)
......
---
title: Limit number of projects displayed in GET /groups/:id API
merge_request: 20023
author:
type: deprecated
...@@ -5,6 +5,8 @@ ...@@ -5,6 +5,8 @@
Get a list of visible groups for the authenticated user. When accessed without Get a list of visible groups for the authenticated user. When accessed without
authentication, only public groups are returned. authentication, only public groups are returned.
By default, this request returns 20 results at a time because the API results [are paginated](README.md#pagination).
Parameters: Parameters:
| Attribute | Type | Required | Description | | Attribute | Type | Required | Description |
...@@ -106,6 +108,8 @@ GET /groups?custom_attributes[key]=value&custom_attributes[other_key]=other_valu ...@@ -106,6 +108,8 @@ GET /groups?custom_attributes[key]=value&custom_attributes[other_key]=other_valu
Get a list of visible direct subgroups in this group. Get a list of visible direct subgroups in this group.
When accessed without authentication, only public groups are returned. When accessed without authentication, only public groups are returned.
By default, this request returns 20 results at a time because the API results [are paginated](README.md#pagination).
Parameters: Parameters:
| Attribute | Type | Required | Description | | Attribute | Type | Required | Description |
...@@ -154,8 +158,9 @@ GET /groups/:id/subgroups ...@@ -154,8 +158,9 @@ GET /groups/:id/subgroups
## List a group's projects ## List a group's projects
Get a list of projects in this group. When accessed without authentication, only Get a list of projects in this group. When accessed without authentication, only public projects are returned.
public projects are returned.
By default, this request returns 20 results at a time because the API results [are paginated](README.md#pagination).
``` ```
GET /groups/:id/projects GET /groups/:id/projects
...@@ -247,6 +252,13 @@ Parameters: ...@@ -247,6 +252,13 @@ Parameters:
curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/4 curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/groups/4
``` ```
This endpoint returns:
- All projects and shared projects in GitLab 12.5 and earlier.
- A maximum of 100 projects and shared projects [in GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/issues/31031)
and later. To get the details of all projects within a group, use the
[list a group's projects endpoint](#list-a-groups-projects) instead.
Example response: Example response:
```json ```json
...@@ -439,6 +451,18 @@ Example response: ...@@ -439,6 +451,18 @@ Example response:
} }
``` ```
### Disabling the results limit
The 100 results limit can be disabled if it breaks integrations developed using GitLab
12.4 and earlier.
To disable the limit while migrating to using the [list a group's projects](#list-a-groups-projects) endpoint, ask a GitLab administrator
with Rails console access to run the following command:
```ruby
Feature.disable(:limit_projects_in_groups_api)
```
## New group ## New group
Creates a new project group. Available only for users who can create groups. Creates a new project group. Available only for users who can create groups.
...@@ -518,6 +542,13 @@ curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab ...@@ -518,6 +542,13 @@ curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab
``` ```
This endpoint returns:
- All projects and shared projects in GitLab 12.5 and earlier.
- A maximum of 100 projects and shared projects [in GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/issues/31031)
and later. To get the details of all projects within a group, use the
[list a group's projects endpoint](#list-a-groups-projects) instead.
Example response: Example response:
```json ```json
...@@ -577,6 +608,19 @@ Example response: ...@@ -577,6 +608,19 @@ Example response:
} }
``` ```
### Disabling the results limit
The 100 results limit can be disabled if it breaks integrations developed using GitLab
12.4 and earlier.
To disable the limit while migrating to using the
[list a group's projects](#list-a-groups-projects) endpoint, ask a GitLab administrator
with Rails console access to run the following command:
```ruby
Feature.disable(:limit_projects_in_groups_api)
```
## Remove group ## Remove group
Removes group with all projects inside. Only available to group owners and administrators. Removes group with all projects inside. Only available to group owners and administrators.
......
...@@ -415,7 +415,7 @@ module API ...@@ -415,7 +415,7 @@ module API
projects = GroupProjectsFinder.new( projects = GroupProjectsFinder.new(
group: group, group: group,
current_user: options[:current_user], current_user: options[:current_user],
options: { only_owned: true } options: { only_owned: true, limit: projects_limit }
).execute ).execute
Entities::Project.preload_and_batch_count!(projects) Entities::Project.preload_and_batch_count!(projects)
...@@ -425,11 +425,19 @@ module API ...@@ -425,11 +425,19 @@ module API
projects = GroupProjectsFinder.new( projects = GroupProjectsFinder.new(
group: group, group: group,
current_user: options[:current_user], current_user: options[:current_user],
options: { only_shared: true } options: { only_shared: true, limit: projects_limit }
).execute ).execute
Entities::Project.preload_and_batch_count!(projects) Entities::Project.preload_and_batch_count!(projects)
end end
def projects_limit
if ::Feature.enabled?(:limit_projects_in_groups_api, default_enabled: true)
GroupProjectsFinder::DEFAULT_PROJECTS_LIMIT
else
nil
end
end
end end
class DiffRefs < Grape::Entity class DiffRefs < Grape::Entity
......
...@@ -168,4 +168,20 @@ describe GroupProjectsFinder do ...@@ -168,4 +168,20 @@ describe GroupProjectsFinder do
end end
end end
end end
describe 'limiting' do
context 'without limiting' do
it 'returns all projects' do
expect(subject.count).to eq(3)
end
end
context 'with limiting' do
let(:options) { { limit: 1 } }
it 'returns only the number of projects specified by the limit' do
expect(subject.count).to eq(1)
end
end
end
end end
...@@ -1349,6 +1349,14 @@ describe Project do ...@@ -1349,6 +1349,14 @@ describe Project do
end end
end end
describe '.with_limit' do
it 'limits the number of projects returned' do
create_list(:project, 3)
expect(described_class.with_limit(1).count).to eq(1)
end
end
describe '.visible_to_user' do describe '.visible_to_user' do
let!(:project) { create(:project, :private) } let!(:project) { create(:project, :private) }
let!(:user) { create(:user) } let!(:user) { create(:user) }
......
...@@ -488,6 +488,51 @@ describe API::Groups do ...@@ -488,6 +488,51 @@ describe API::Groups do
expect(response).to have_gitlab_http_status(404) expect(response).to have_gitlab_http_status(404)
end end
end end
context 'limiting the number of projects and shared_projects in the response' do
let(:limit) { 1 }
before do
stub_const("GroupProjectsFinder::DEFAULT_PROJECTS_LIMIT", limit)
# creates 3 public projects
create_list(:project, 3, :public, namespace: group1)
# creates 3 shared projects
public_group = create(:group, :public)
projects_to_be_shared = create_list(:project, 3, :public, namespace: public_group)
projects_to_be_shared.each do |project|
create(:project_group_link, project: project, group: group1)
end
end
context 'when limiting feature is enabled' do
before do
stub_feature_flags(limit_projects_in_groups_api: true)
end
it 'limits projects and shared_projects' do
get api("/groups/#{group1.id}")
expect(json_response['projects'].count).to eq(limit)
expect(json_response['shared_projects'].count).to eq(limit)
end
end
context 'when limiting feature is not enabled' do
before do
stub_feature_flags(limit_projects_in_groups_api: false)
end
it 'does not limit projects and shared_projects' do
get api("/groups/#{group1.id}")
expect(json_response['projects'].count).to eq(3)
expect(json_response['shared_projects'].count).to eq(3)
end
end
end
end end
describe 'PUT /groups/:id' do describe 'PUT /groups/:id' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment