Commit c75c0662 authored by James Lopez's avatar James Lopez

Merge branch 'georgekoltsov/import-export-rate-limits-as-application-settings' into 'master'

Convert Import/Export rate limits to application settings

See merge request gitlab-org/gitlab!35728
parents a2cf569e e272ca51
......@@ -321,7 +321,13 @@ module ApplicationSettingsHelper
:email_restrictions_enabled,
:email_restrictions,
:issues_create_limit,
:raw_blob_request_limit
:raw_blob_request_limit,
:project_import_limit,
:project_export_limit,
:project_download_export_limit,
:group_import_limit,
:group_export_limit,
:group_download_export_limit
]
end
......
......@@ -158,7 +158,13 @@ module ApplicationSettingImplementation
snowplow_iglu_registry_url: nil,
custom_http_clone_url_root: nil,
productivity_analytics_start_date: Time.current,
snippet_size_limit: 50.megabytes
snippet_size_limit: 50.megabytes,
project_import_limit: 6,
project_export_limit: 6,
project_download_export_limit: 1,
group_import_limit: 6,
group_export_limit: 6,
group_download_export_limit: 1
}
end
......
= form_for @application_setting, url: network_admin_application_settings_path(anchor: 'js-import-export-limits-settings'), html: { class: 'fieldset-form' } do |f|
= form_errors(@application_setting)
%fieldset
.form-group
= f.label :project_import_limit, _('Max Project Import requests per minute per user'), class: 'label-bold'
= f.number_field :project_import_limit, class: 'form-control'
%fieldset
.form-group
= f.label :project_export_limit, _('Max Project Export requests per minute per user'), class: 'label-bold'
= f.number_field :project_export_limit, class: 'form-control'
%fieldset
.form-group
= f.label :project_download_export_limit, _('Max Project Export Download requests per minute per user'), class: 'label-bold'
= f.number_field :project_download_export_limit, class: 'form-control'
%fieldset
.form-group
= f.label :group_import_limit, _('Max Group Import requests per minute per user'), class: 'label-bold'
= f.number_field :group_import_limit, class: 'form-control'
%fieldset
.form-group
= f.label :group_export_limit, _('Max Group Export requests per minute per user'), class: 'label-bold'
= f.number_field :group_export_limit, class: 'form-control'
%fieldset
.form-group
= f.label :group_download_export_limit, _('Max Group Export Download requests per minute per user'), class: 'label-bold'
= f.number_field :group_download_export_limit, class: 'form-control'
= f.submit 'Save changes', class: "btn btn-success", data: { qa_selector: 'save_changes_button' }
......@@ -57,4 +57,15 @@
.settings-content
= render 'issue_limits'
%section.settings.as-import-export-limits.no-animate#js-import-export-limits-settings{ class: ('expanded' if expanded_by_default?) }
.settings-header
%h4
= _('Import/Export Rate Limits')
%button.btn.btn-default.js-settings-toggle{ type: 'button' }
= expanded_by_default? ? _('Collapse') : _('Expand')
%p
= _('Configure limits for Project/Group Import/Export.')
.settings-content
= render 'import_export_limits'
= render_if_exists 'admin/application_settings/ee_network_settings'
---
title: Convert Import/Export rate limits to configurable application settings
merge_request: 35728
author:
type: added
# frozen_string_literal: true
class AddImportExportLimitsToApplicationSettings < ActiveRecord::Migration[6.0]
DOWNTIME = false
def change
add_column :application_settings, :project_import_limit, :integer, default: 6, null: false
add_column :application_settings, :project_export_limit, :integer, default: 6, null: false
add_column :application_settings, :project_download_export_limit, :integer, default: 1, null: false
add_column :application_settings, :group_import_limit, :integer, default: 6, null: false
add_column :application_settings, :group_export_limit, :integer, default: 6, null: false
add_column :application_settings, :group_download_export_limit, :integer, default: 1, null: false
end
end
......@@ -9145,6 +9145,12 @@ CREATE TABLE public.application_settings (
compliance_frameworks smallint[] DEFAULT '{}'::smallint[] NOT NULL,
notify_on_unknown_sign_in boolean DEFAULT true NOT NULL,
default_branch_name text,
project_import_limit integer DEFAULT 6 NOT NULL,
project_export_limit integer DEFAULT 6 NOT NULL,
project_download_export_limit integer DEFAULT 1 NOT NULL,
group_import_limit integer DEFAULT 6 NOT NULL,
group_export_limit integer DEFAULT 6 NOT NULL,
group_download_export_limit integer DEFAULT 1 NOT NULL,
CONSTRAINT check_51700b31b5 CHECK ((char_length(default_branch_name) <= 255)),
CONSTRAINT check_d03919528d CHECK ((char_length(container_registry_vendor) <= 255)),
CONSTRAINT check_d820146492 CHECK ((char_length(spam_check_endpoint_url) <= 255)),
......@@ -23577,6 +23583,7 @@ COPY "schema_migrations" (version) FROM STDIN;
20200626060151
20200626130220
20200630110826
20200701093859
20200702123805
20200703154822
20200704143633
......
......@@ -26,6 +26,7 @@ similarly mitigated by a rate limit.
- [User and IP rate limits](../user/admin_area/settings/user_and_ip_rate_limits.md).
- [Raw endpoints rate limits](../user/admin_area/settings/rate_limits_on_raw_endpoints.md).
- [Protected paths](../user/admin_area/settings/protected_paths.md).
- [Import/Export rate limits](../user/admin_area/settings/import_export_rate_limits.md).
## Rack Attack initializer
......
---
type: reference
stage: Manage
group: Import
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers
---
# Project/Group Import/Export rate limits
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35728) in GitLab 13.2.
The following table includes configurable rate limits. The following table includes limits on a
per minute per user basis:
| Limit | Default (per minute per user) |
|--------------------------|-------------------------------|
| Project Import | 6 |
| Project Export | 6 |
| Project Export Download | 1 |
| Group Import | 6 |
| Group Export | 6 |
| Group Export Download | 1 |
All rate limits are:
- Configurable at **(admin)** **Admin Area > Settings > Network > Import/Export Rate Limits**
- Applied per minute per user
- Not applied per IP address
- Active by default. To disable, set the option to `0`
- Logged to `auth.log` file if exceed rate limit
![Import/Export rate limits](img/import_export_rate_limits_v13_2.png)
......@@ -11,7 +11,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w
This setting allows you to rate limit the requests to the issue creation endpoint.
It defaults to 300 requests per minute.
You can change it in **Admin Area > Settings > Network > Performance Optimization**.
You can change it in **Admin Area > Settings > Network > Issues Rate Limits**.
For example, requests using the
[Projects::IssuesController#create](https://gitlab.com/gitlab-org/gitlab/raw/master/app/controllers/projects/issues_controller.rb)
......
......@@ -19,17 +19,17 @@ module Gitlab
# and only do that when it's needed.
def rate_limits
{
issues_create: { threshold: -> { Gitlab::CurrentSettings.current_application_settings.issues_create_limit }, interval: 1.minute },
project_export: { threshold: 30, interval: 5.minutes },
project_download_export: { threshold: 10, interval: 10.minutes },
issues_create: { threshold: -> { application_settings.issues_create_limit }, interval: 1.minute },
project_export: { threshold: -> { application_settings.project_export_limit }, interval: 1.minute },
project_download_export: { threshold: -> { application_settings.project_download_export_limit }, interval: 1.minute },
project_repositories_archive: { threshold: 5, interval: 1.minute },
project_generate_new_export: { threshold: 30, interval: 5.minutes },
project_import: { threshold: 30, interval: 5.minutes },
play_pipeline_schedule: { threshold: 1, interval: 1.minute },
show_raw_controller: { threshold: -> { Gitlab::CurrentSettings.current_application_settings.raw_blob_request_limit }, interval: 1.minute },
group_export: { threshold: 30, interval: 5.minutes },
group_download_export: { threshold: 10, interval: 10.minutes },
group_import: { threshold: 30, interval: 5.minutes }
project_generate_new_export: { threshold: -> { application_settings.project_export_limit }, interval: 1.minute },
project_import: { threshold: -> { application_settings.project_import_limit }, interval: 1.minute },
play_pipeline_schedule: { threshold: 1, interval: 1.minute },
show_raw_controller: { threshold: -> { application_settings.raw_blob_request_limit }, interval: 1.minute },
group_export: { threshold: -> { application_settings.group_export_limit }, interval: 1.minute },
group_download_export: { threshold: -> { application_settings.group_download_export_limit }, interval: 1.minute },
group_import: { threshold: -> { application_settings.group_import_limit }, interval: 1.minute }
}.freeze
end
......@@ -130,6 +130,10 @@ module Gitlab
"application_rate_limiter:#{serialized}"
end
def application_settings
Gitlab::CurrentSettings.current_application_settings
end
end
end
end
......@@ -6082,6 +6082,9 @@ msgstr ""
msgid "Configure limit for issues created per minute by web and API requests."
msgstr ""
msgid "Configure limits for Project/Group Import/Export."
msgstr ""
msgid "Configure limits for web and API requests."
msgstr ""
......@@ -12226,6 +12229,9 @@ msgstr ""
msgid "Import timed out. Import took longer than %{import_jobs_expiration} seconds"
msgstr ""
msgid "Import/Export Rate Limits"
msgstr ""
msgid "Import/Export illustration"
msgstr ""
......@@ -13989,6 +13995,24 @@ msgstr ""
msgid "Maven Metadata"
msgstr ""
msgid "Max Group Export Download requests per minute per user"
msgstr ""
msgid "Max Group Export requests per minute per user"
msgstr ""
msgid "Max Group Import requests per minute per user"
msgstr ""
msgid "Max Project Export Download requests per minute per user"
msgstr ""
msgid "Max Project Export requests per minute per user"
msgstr ""
msgid "Max Project Import requests per minute per user"
msgstr ""
msgid "Max access level"
msgstr ""
......
......@@ -941,7 +941,7 @@ RSpec.describe GroupsController do
allow(Gitlab::ApplicationRateLimiter)
.to receive(:increment)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits[:group_export][:threshold] + 1)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits[:group_export][:threshold].call + 1)
end
it 'throttles the endpoint' do
......@@ -1015,7 +1015,7 @@ RSpec.describe GroupsController do
allow(Gitlab::ApplicationRateLimiter)
.to receive(:increment)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits[:group_download_export][:threshold] + 1)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits[:group_download_export][:threshold].call + 1)
end
it 'throttles the endpoint' do
......
......@@ -1197,7 +1197,7 @@ RSpec.describe ProjectsController do
before do
allow(Gitlab::ApplicationRateLimiter)
.to receive(:increment)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits["project_#{action}".to_sym][:threshold] + 1)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits["project_#{action}".to_sym][:threshold].call + 1)
end
it 'prevents requesting project export' do
......@@ -1264,7 +1264,7 @@ RSpec.describe ProjectsController do
before do
allow(Gitlab::ApplicationRateLimiter)
.to receive(:increment)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits[:project_download_export][:threshold] + 1)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits[:project_download_export][:threshold].call + 1)
end
it 'prevents requesting project export' do
......
......@@ -33,6 +33,10 @@ RSpec.describe API::GroupExport do
context 'group_import_export feature flag enabled' do
before do
stub_feature_flags(group_import_export: true)
allow(Gitlab::ApplicationRateLimiter)
.to receive(:increment)
.and_return(0)
end
context 'when export file exists' do
......@@ -87,7 +91,7 @@ RSpec.describe API::GroupExport do
before do
allow(Gitlab::ApplicationRateLimiter)
.to receive(:increment)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits[:group_download_export][:threshold] + 1)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits[:group_download_export][:threshold].call + 1)
end
it 'throttles the endpoint' do
......@@ -162,7 +166,7 @@ RSpec.describe API::GroupExport do
allow(Gitlab::ApplicationRateLimiter)
.to receive(:increment)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits[:group_export][:threshold] + 1)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits[:group_export][:threshold].call + 1)
end
it 'throttles the endpoint' do
......
......@@ -237,7 +237,7 @@ RSpec.describe API::ProjectExport, :clean_gitlab_redis_cache do
before do
allow(Gitlab::ApplicationRateLimiter)
.to receive(:increment)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits[:project_download_export][:threshold] + 1)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits[:project_download_export][:threshold].call + 1)
end
it 'prevents requesting project export' do
......@@ -362,7 +362,7 @@ RSpec.describe API::ProjectExport, :clean_gitlab_redis_cache do
before do
allow(Gitlab::ApplicationRateLimiter)
.to receive(:increment)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits[:project_export][:threshold] + 1)
.and_return(Gitlab::ApplicationRateLimiter.rate_limits[:project_export][:threshold].call + 1)
end
it 'prevents requesting project export' do
......
......@@ -16,6 +16,8 @@ RSpec.describe Import::GitlabGroupsController do
expect(import_export).to receive(:storage_path).and_return(import_path)
end
allow(Gitlab::ApplicationRateLimiter).to receive(:throttled?).and_return(false)
stub_uploads_object_storage(ImportExportUploader)
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment