Commit e272ca51 authored by George Koltsov's avatar George Koltsov

Convert Import/Export rate limits to application settings

  - Change hard coded Project/Group Import/Export rate
    limits to configurable application limits in order
    to give admins an ability to tighten or loosen
    these rate up
parent ca12428c
...@@ -321,7 +321,13 @@ module ApplicationSettingsHelper ...@@ -321,7 +321,13 @@ module ApplicationSettingsHelper
:email_restrictions_enabled, :email_restrictions_enabled,
:email_restrictions, :email_restrictions,
:issues_create_limit, :issues_create_limit,
:raw_blob_request_limit :raw_blob_request_limit,
:project_import_limit,
:project_export_limit,
:project_download_export_limit,
:group_import_limit,
:group_export_limit,
:group_download_export_limit
] ]
end end
......
...@@ -158,7 +158,13 @@ module ApplicationSettingImplementation ...@@ -158,7 +158,13 @@ module ApplicationSettingImplementation
snowplow_iglu_registry_url: nil, snowplow_iglu_registry_url: nil,
custom_http_clone_url_root: nil, custom_http_clone_url_root: nil,
productivity_analytics_start_date: Time.current, productivity_analytics_start_date: Time.current,
snippet_size_limit: 50.megabytes snippet_size_limit: 50.megabytes,
project_import_limit: 6,
project_export_limit: 6,
project_download_export_limit: 1,
group_import_limit: 6,
group_export_limit: 6,
group_download_export_limit: 1
} }
end end
......
= form_for @application_setting, url: network_admin_application_settings_path(anchor: 'js-import-export-limits-settings'), html: { class: 'fieldset-form' } do |f|
= form_errors(@application_setting)
%fieldset
.form-group
= f.label :project_import_limit, _('Max Project Import requests per minute per user'), class: 'label-bold'
= f.number_field :project_import_limit, class: 'form-control'
%fieldset
.form-group
= f.label :project_export_limit, _('Max Project Export requests per minute per user'), class: 'label-bold'
= f.number_field :project_export_limit, class: 'form-control'
%fieldset
.form-group
= f.label :project_download_export_limit, _('Max Project Export Download requests per minute per user'), class: 'label-bold'
= f.number_field :project_download_export_limit, class: 'form-control'
%fieldset
.form-group
= f.label :group_import_limit, _('Max Group Import requests per minute per user'), class: 'label-bold'
= f.number_field :group_import_limit, class: 'form-control'
%fieldset
.form-group
= f.label :group_export_limit, _('Max Group Export requests per minute per user'), class: 'label-bold'
= f.number_field :group_export_limit, class: 'form-control'
%fieldset
.form-group
= f.label :group_download_export_limit, _('Max Group Export Download requests per minute per user'), class: 'label-bold'
= f.number_field :group_download_export_limit, class: 'form-control'
= f.submit 'Save changes', class: "btn btn-success", data: { qa_selector: 'save_changes_button' }
...@@ -57,4 +57,15 @@ ...@@ -57,4 +57,15 @@
.settings-content .settings-content
= render 'issue_limits' = render 'issue_limits'
%section.settings.as-import-export-limits.no-animate#js-import-export-limits-settings{ class: ('expanded' if expanded_by_default?) }
.settings-header
%h4
= _('Import/Export Rate Limits')
%button.btn.btn-default.js-settings-toggle{ type: 'button' }
= expanded_by_default? ? _('Collapse') : _('Expand')
%p
= _('Configure limits for Project/Group Import/Export.')
.settings-content
= render 'import_export_limits'
= render_if_exists 'admin/application_settings/ee_network_settings' = render_if_exists 'admin/application_settings/ee_network_settings'
---
title: Convert Import/Export rate limits to configurable application settings
merge_request: 35728
author:
type: added
# frozen_string_literal: true
class AddImportExportLimitsToApplicationSettings < ActiveRecord::Migration[6.0]
DOWNTIME = false
def change
add_column :application_settings, :project_import_limit, :integer, default: 6, null: false
add_column :application_settings, :project_export_limit, :integer, default: 6, null: false
add_column :application_settings, :project_download_export_limit, :integer, default: 1, null: false
add_column :application_settings, :group_import_limit, :integer, default: 6, null: false
add_column :application_settings, :group_export_limit, :integer, default: 6, null: false
add_column :application_settings, :group_download_export_limit, :integer, default: 1, null: false
end
end
...@@ -9145,6 +9145,12 @@ CREATE TABLE public.application_settings ( ...@@ -9145,6 +9145,12 @@ CREATE TABLE public.application_settings (
compliance_frameworks smallint[] DEFAULT '{}'::smallint[] NOT NULL, compliance_frameworks smallint[] DEFAULT '{}'::smallint[] NOT NULL,
notify_on_unknown_sign_in boolean DEFAULT true NOT NULL, notify_on_unknown_sign_in boolean DEFAULT true NOT NULL,
default_branch_name text, default_branch_name text,
project_import_limit integer DEFAULT 6 NOT NULL,
project_export_limit integer DEFAULT 6 NOT NULL,
project_download_export_limit integer DEFAULT 1 NOT NULL,
group_import_limit integer DEFAULT 6 NOT NULL,
group_export_limit integer DEFAULT 6 NOT NULL,
group_download_export_limit integer DEFAULT 1 NOT NULL,
CONSTRAINT check_51700b31b5 CHECK ((char_length(default_branch_name) <= 255)), CONSTRAINT check_51700b31b5 CHECK ((char_length(default_branch_name) <= 255)),
CONSTRAINT check_d03919528d CHECK ((char_length(container_registry_vendor) <= 255)), CONSTRAINT check_d03919528d CHECK ((char_length(container_registry_vendor) <= 255)),
CONSTRAINT check_d820146492 CHECK ((char_length(spam_check_endpoint_url) <= 255)), CONSTRAINT check_d820146492 CHECK ((char_length(spam_check_endpoint_url) <= 255)),
...@@ -23573,6 +23579,7 @@ COPY "schema_migrations" (version) FROM STDIN; ...@@ -23573,6 +23579,7 @@ COPY "schema_migrations" (version) FROM STDIN;
20200626060151 20200626060151
20200626130220 20200626130220
20200630110826 20200630110826
20200701093859
20200702123805 20200702123805
20200703154822 20200703154822
20200704143633 20200704143633
......
...@@ -26,6 +26,7 @@ similarly mitigated by a rate limit. ...@@ -26,6 +26,7 @@ similarly mitigated by a rate limit.
- [User and IP rate limits](../user/admin_area/settings/user_and_ip_rate_limits.md). - [User and IP rate limits](../user/admin_area/settings/user_and_ip_rate_limits.md).
- [Raw endpoints rate limits](../user/admin_area/settings/rate_limits_on_raw_endpoints.md). - [Raw endpoints rate limits](../user/admin_area/settings/rate_limits_on_raw_endpoints.md).
- [Protected paths](../user/admin_area/settings/protected_paths.md). - [Protected paths](../user/admin_area/settings/protected_paths.md).
- [Import/Export rate limits](../user/admin_area/settings/import_export_rate_limits.md).
## Rack Attack initializer ## Rack Attack initializer
......
---
type: reference
stage: Manage
group: Import
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers
---
# Project/Group Import/Export rate limits
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35728) in GitLab 13.2.
The following table includes configurable rate limits. The following table includes limits on a
per minute per user basis:
| Limit | Default (per minute per user) |
|--------------------------|-------------------------------|
| Project Import | 6 |
| Project Export | 6 |
| Project Export Download | 1 |
| Group Import | 6 |
| Group Export | 6 |
| Group Export Download | 1 |
All rate limits are:
- Configurable at **(admin)** **Admin Area > Settings > Network > Import/Export Rate Limits**
- Applied per minute per user
- Not applied per IP address
- Active by default. To disable, set the option to `0`
- Logged to `auth.log` file if exceed rate limit
![Import/Export rate limits](img/import_export_rate_limits_v13_2.png)
...@@ -11,7 +11,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w ...@@ -11,7 +11,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w
This setting allows you to rate limit the requests to the issue creation endpoint. This setting allows you to rate limit the requests to the issue creation endpoint.
It defaults to 300 requests per minute. It defaults to 300 requests per minute.
You can change it in **Admin Area > Settings > Network > Performance Optimization**. You can change it in **Admin Area > Settings > Network > Issues Rate Limits**.
For example, requests using the For example, requests using the
[Projects::IssuesController#create](https://gitlab.com/gitlab-org/gitlab/raw/master/app/controllers/projects/issues_controller.rb) [Projects::IssuesController#create](https://gitlab.com/gitlab-org/gitlab/raw/master/app/controllers/projects/issues_controller.rb)
......
...@@ -19,17 +19,17 @@ module Gitlab ...@@ -19,17 +19,17 @@ module Gitlab
# and only do that when it's needed. # and only do that when it's needed.
def rate_limits def rate_limits
{ {
issues_create: { threshold: -> { Gitlab::CurrentSettings.current_application_settings.issues_create_limit }, interval: 1.minute }, issues_create: { threshold: -> { application_settings.issues_create_limit }, interval: 1.minute },
project_export: { threshold: 30, interval: 5.minutes }, project_export: { threshold: -> { application_settings.project_export_limit }, interval: 1.minute },
project_download_export: { threshold: 10, interval: 10.minutes }, project_download_export: { threshold: -> { application_settings.project_download_export_limit }, interval: 1.minute },
project_repositories_archive: { threshold: 5, interval: 1.minute }, project_repositories_archive: { threshold: 5, interval: 1.minute },
project_generate_new_export: { threshold: 30, interval: 5.minutes }, project_generate_new_export: { threshold: -> { application_settings.project_export_limit }, interval: 1.minute },
project_import: { threshold: 30, interval: 5.minutes }, project_import: { threshold: -> { application_settings.project_import_limit }, interval: 1.minute },
play_pipeline_schedule: { threshold: 1, interval: 1.minute }, play_pipeline_schedule: { threshold: 1, interval: 1.minute },
show_raw_controller: { threshold: -> { Gitlab::CurrentSettings.current_application_settings.raw_blob_request_limit }, interval: 1.minute }, show_raw_controller: { threshold: -> { application_settings.raw_blob_request_limit }, interval: 1.minute },
group_export: { threshold: 30, interval: 5.minutes }, group_export: { threshold: -> { application_settings.group_export_limit }, interval: 1.minute },
group_download_export: { threshold: 10, interval: 10.minutes }, group_download_export: { threshold: -> { application_settings.group_download_export_limit }, interval: 1.minute },
group_import: { threshold: 30, interval: 5.minutes } group_import: { threshold: -> { application_settings.group_import_limit }, interval: 1.minute }
}.freeze }.freeze
end end
...@@ -130,6 +130,10 @@ module Gitlab ...@@ -130,6 +130,10 @@ module Gitlab
"application_rate_limiter:#{serialized}" "application_rate_limiter:#{serialized}"
end end
def application_settings
Gitlab::CurrentSettings.current_application_settings
end
end end
end end
end end
...@@ -6077,6 +6077,9 @@ msgstr "" ...@@ -6077,6 +6077,9 @@ msgstr ""
msgid "Configure limit for issues created per minute by web and API requests." msgid "Configure limit for issues created per minute by web and API requests."
msgstr "" msgstr ""
msgid "Configure limits for Project/Group Import/Export."
msgstr ""
msgid "Configure limits for web and API requests." msgid "Configure limits for web and API requests."
msgstr "" msgstr ""
...@@ -12218,6 +12221,9 @@ msgstr "" ...@@ -12218,6 +12221,9 @@ msgstr ""
msgid "Import timed out. Import took longer than %{import_jobs_expiration} seconds" msgid "Import timed out. Import took longer than %{import_jobs_expiration} seconds"
msgstr "" msgstr ""
msgid "Import/Export Rate Limits"
msgstr ""
msgid "Import/Export illustration" msgid "Import/Export illustration"
msgstr "" msgstr ""
...@@ -13975,6 +13981,24 @@ msgstr "" ...@@ -13975,6 +13981,24 @@ msgstr ""
msgid "Maven Metadata" msgid "Maven Metadata"
msgstr "" msgstr ""
msgid "Max Group Export Download requests per minute per user"
msgstr ""
msgid "Max Group Export requests per minute per user"
msgstr ""
msgid "Max Group Import requests per minute per user"
msgstr ""
msgid "Max Project Export Download requests per minute per user"
msgstr ""
msgid "Max Project Export requests per minute per user"
msgstr ""
msgid "Max Project Import requests per minute per user"
msgstr ""
msgid "Max access level" msgid "Max access level"
msgstr "" msgstr ""
......
...@@ -941,7 +941,7 @@ RSpec.describe GroupsController do ...@@ -941,7 +941,7 @@ RSpec.describe GroupsController do
allow(Gitlab::ApplicationRateLimiter) allow(Gitlab::ApplicationRateLimiter)
.to receive(:increment) .to receive(:increment)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits[:group_export][:threshold] + 1) .and_return(Gitlab::ApplicationRateLimiter.rate_limits[:group_export][:threshold].call + 1)
end end
it 'throttles the endpoint' do it 'throttles the endpoint' do
...@@ -1015,7 +1015,7 @@ RSpec.describe GroupsController do ...@@ -1015,7 +1015,7 @@ RSpec.describe GroupsController do
allow(Gitlab::ApplicationRateLimiter) allow(Gitlab::ApplicationRateLimiter)
.to receive(:increment) .to receive(:increment)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits[:group_download_export][:threshold] + 1) .and_return(Gitlab::ApplicationRateLimiter.rate_limits[:group_download_export][:threshold].call + 1)
end end
it 'throttles the endpoint' do it 'throttles the endpoint' do
......
...@@ -1197,7 +1197,7 @@ RSpec.describe ProjectsController do ...@@ -1197,7 +1197,7 @@ RSpec.describe ProjectsController do
before do before do
allow(Gitlab::ApplicationRateLimiter) allow(Gitlab::ApplicationRateLimiter)
.to receive(:increment) .to receive(:increment)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits["project_#{action}".to_sym][:threshold] + 1) .and_return(Gitlab::ApplicationRateLimiter.rate_limits["project_#{action}".to_sym][:threshold].call + 1)
end end
it 'prevents requesting project export' do it 'prevents requesting project export' do
...@@ -1264,7 +1264,7 @@ RSpec.describe ProjectsController do ...@@ -1264,7 +1264,7 @@ RSpec.describe ProjectsController do
before do before do
allow(Gitlab::ApplicationRateLimiter) allow(Gitlab::ApplicationRateLimiter)
.to receive(:increment) .to receive(:increment)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits[:project_download_export][:threshold] + 1) .and_return(Gitlab::ApplicationRateLimiter.rate_limits[:project_download_export][:threshold].call + 1)
end end
it 'prevents requesting project export' do it 'prevents requesting project export' do
......
...@@ -33,6 +33,10 @@ RSpec.describe API::GroupExport do ...@@ -33,6 +33,10 @@ RSpec.describe API::GroupExport do
context 'group_import_export feature flag enabled' do context 'group_import_export feature flag enabled' do
before do before do
stub_feature_flags(group_import_export: true) stub_feature_flags(group_import_export: true)
allow(Gitlab::ApplicationRateLimiter)
.to receive(:increment)
.and_return(0)
end end
context 'when export file exists' do context 'when export file exists' do
...@@ -87,7 +91,7 @@ RSpec.describe API::GroupExport do ...@@ -87,7 +91,7 @@ RSpec.describe API::GroupExport do
before do before do
allow(Gitlab::ApplicationRateLimiter) allow(Gitlab::ApplicationRateLimiter)
.to receive(:increment) .to receive(:increment)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits[:group_download_export][:threshold] + 1) .and_return(Gitlab::ApplicationRateLimiter.rate_limits[:group_download_export][:threshold].call + 1)
end end
it 'throttles the endpoint' do it 'throttles the endpoint' do
...@@ -162,7 +166,7 @@ RSpec.describe API::GroupExport do ...@@ -162,7 +166,7 @@ RSpec.describe API::GroupExport do
allow(Gitlab::ApplicationRateLimiter) allow(Gitlab::ApplicationRateLimiter)
.to receive(:increment) .to receive(:increment)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits[:group_export][:threshold] + 1) .and_return(Gitlab::ApplicationRateLimiter.rate_limits[:group_export][:threshold].call + 1)
end end
it 'throttles the endpoint' do it 'throttles the endpoint' do
......
...@@ -237,7 +237,7 @@ RSpec.describe API::ProjectExport, :clean_gitlab_redis_cache do ...@@ -237,7 +237,7 @@ RSpec.describe API::ProjectExport, :clean_gitlab_redis_cache do
before do before do
allow(Gitlab::ApplicationRateLimiter) allow(Gitlab::ApplicationRateLimiter)
.to receive(:increment) .to receive(:increment)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits[:project_download_export][:threshold] + 1) .and_return(Gitlab::ApplicationRateLimiter.rate_limits[:project_download_export][:threshold].call + 1)
end end
it 'prevents requesting project export' do it 'prevents requesting project export' do
...@@ -362,7 +362,7 @@ RSpec.describe API::ProjectExport, :clean_gitlab_redis_cache do ...@@ -362,7 +362,7 @@ RSpec.describe API::ProjectExport, :clean_gitlab_redis_cache do
before do before do
allow(Gitlab::ApplicationRateLimiter) allow(Gitlab::ApplicationRateLimiter)
.to receive(:increment) .to receive(:increment)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits[:project_export][:threshold] + 1) .and_return(Gitlab::ApplicationRateLimiter.rate_limits[:project_export][:threshold].call + 1)
end end
it 'prevents requesting project export' do it 'prevents requesting project export' do
......
...@@ -16,6 +16,8 @@ RSpec.describe Import::GitlabGroupsController do ...@@ -16,6 +16,8 @@ RSpec.describe Import::GitlabGroupsController do
expect(import_export).to receive(:storage_path).and_return(import_path) expect(import_export).to receive(:storage_path).and_return(import_path)
end end
allow(Gitlab::ApplicationRateLimiter).to receive(:throttled?).and_return(false)
stub_uploads_object_storage(ImportExportUploader) stub_uploads_object_storage(ImportExportUploader)
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment