Merge branch 'fix/max_import_size' into 'master'

Add Max import file size option

Closes #216938

See merge request gitlab-org/gitlab!33215

app/controllers/concerns/workhorse_import_export_upload.rb

@@ -14,7 +14,7 @@ module WorkhorseImportExportUpload
 
     authorized = ImportExportUploader.workhorse_authorize(
       has_length: false,
-      maximum_size: ImportExportUpload::MAXIMUM_IMPORT_FILE_SIZE
+      maximum_size: Gitlab::CurrentSettings.max_import_size.megabytes
     )
 
     render json: authorized

app/helpers/application_settings_helper.rb

@@ -228,6 +228,7 @@ module ApplicationSettingsHelper
       :import_sources,
       :max_artifacts_size,
       :max_attachment_size,
+      :max_import_size,
       :max_pages_size,
       :metrics_method_call_threshold,
       :minimum_password_length,

app/models/application_setting.rb

@@ -143,6 +143,10 @@ class ApplicationSetting < ApplicationRecord
             presence: true,
             numericality: { only_integer: true, greater_than: 0 }
 
+  validates :max_import_size,
+            presence: true,
+            numericality: { only_integer: true, greater_than_or_equal_to: 0 }
+
   validates :max_pages_size,
             presence: true,
             numericality: { only_integer: true, greater_than_or_equal_to: 0,

app/models/application_setting_implementation.rb

@@ -86,6 +86,7 @@ module ApplicationSettingImplementation
         local_markdown_version: 0,
         max_artifacts_size: Settings.artifacts['max_size'],
         max_attachment_size: Settings.gitlab['max_attachment_size'],
+        max_import_size: 50,
         mirror_available: true,
         outbound_local_requests_whitelist: [],
         password_authentication_enabled_for_git: true,

app/models/import_export_upload.rb

@@ -4,8 +4,6 @@ class ImportExportUpload < ApplicationRecord
   include WithUploads
   include ObjectStorage::BackgroundMove
 
-  MAXIMUM_IMPORT_FILE_SIZE = 50.megabytes.freeze
-
   belongs_to :project
   belongs_to :group
 

app/views/admin/application_settings/_account_and_limit.html.haml

@@ -28,6 +28,10 @@
     .form-group
       = f.label :receive_max_input_size, _('Maximum push size (MB)'), class: 'label-light'
       = f.number_field :receive_max_input_size, class: 'form-control qa-receive-max-input-size-field', title: _('Maximum size limit for a single commit.'), data: { toggle: 'tooltip', container: 'body' }
+    .form-group
+      = f.label :max_import_size, _('Maximum import size (MB)'), class: 'label-light'
+      = f.number_field :max_import_size, class: 'form-control qa-receive-max-import-size-field', title: _('Maximum size of import files.'), data: { toggle: 'tooltip', container: 'body' }
+      %span.form-text.text-muted= _('0 for unlimited, only effective with remote storage enabled.')
     .form-group
       = f.label :session_expire_delay, _('Session duration (minutes)'), class: 'label-light'
       = f.number_field :session_expire_delay, class: 'form-control', title: _('Maximum duration of a session.'), data: { toggle: 'tooltip', container: 'body' }

changelogs/unreleased/fix-max_import_size.yml

+---
+title: Add max import file size option
+merge_request: 33215
+author: Roger Meier
+type: added

db/migrate/20200527211000_add_max_import_size.rb

+# frozen_string_literal: true
+
+class AddMaxImportSize < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    add_column(:application_settings, :max_import_size, :integer, default: 50, null: false)
+  end
+
+  def down
+    remove_column(:application_settings, :max_import_size)
+  end
+end

db/structure.sql

@@ -458,6 +458,7 @@ CREATE TABLE public.application_settings (
     spam_check_endpoint_enabled boolean DEFAULT false NOT NULL,
     elasticsearch_pause_indexing boolean DEFAULT false NOT NULL,
     repository_storages_weighted jsonb DEFAULT '{}'::jsonb NOT NULL,
+    max_import_size integer DEFAULT 50 NOT NULL,
     CONSTRAINT check_d03919528d CHECK ((char_length(container_registry_vendor) <= 255)),
     CONSTRAINT check_d820146492 CHECK ((char_length(spam_check_endpoint_url) <= 255)),
     CONSTRAINT check_e5aba18f02 CHECK ((char_length(container_registry_version) <= 255))
@@ -13798,6 +13799,7 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200527151413
 20200527152116
 20200527152657
+20200527211000
 20200528054112
 20200528123703
 20200528125905

doc/api/group_import_export.md

@@ -86,6 +86,10 @@ by `@`. For example:
 curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "name=imported-group" --form "path=imported-group" --form "file=@/path/to/file" "https://gitlab.example.com/api/v4/groups/import"
 ```
 
+NOTE: **Note:**
+The maximum import file size can be set by the Administrator, default is 50MB.
+As an administrator, you can modify the maximum import file size. To do so, use the `max_import_size` option in the [Application settings API](settings.md#change-application-settings) or the [Admin UI](../user/admin_area/settings/account_and_limit_settings.md).
+
 ## Important notes
 
 Note the following:

doc/api/project_import_export.md

@@ -181,6 +181,10 @@ requests.post(url, headers=headers, data=data, files=files)
 }
 ```
 
+NOTE: **Note:**
+The maximum import file size can be set by the Administrator, default is 50MB.
+As an administrator, you can modify the maximum import file size. To do so, use the `max_import_size` option in the [Application settings API](settings.md#change-application-settings) or the [Admin UI](../user/admin_area/settings/account_and_limit_settings.md).
+
 ## Import status
 
 Get the status of an import.

doc/api/settings.md

@@ -30,6 +30,7 @@ Example response:
    "password_authentication_enabled_for_web" : true,
    "after_sign_out_path" : null,
    "max_attachment_size" : 10,
+   "max_import_size": 50,
    "user_oauth_applications" : true,
    "updated_at" : "2016-01-04T15:44:55.176Z",
    "session_expire_delay" : 10080,
@@ -118,6 +119,7 @@ Example response:
   "default_branch_protection": 2,
   "restricted_visibility_levels": [],
   "max_attachment_size": 10,
+  "max_import_size": 50,
   "session_expire_delay": 10080,
   "default_ci_config_path" : null,
   "default_project_visibility": "internal",
@@ -280,6 +282,7 @@ are listed in the descriptions of the relevant settings.
 | `local_markdown_version`                 | integer          | no                                   | Increase this value when any cached Markdown should be invalidated. |
 | `max_artifacts_size`                     | integer          | no                                   | Maximum artifacts size in MB |
 | `max_attachment_size`                    | integer          | no                                   | Limit attachment size in MB |
+| `max_import_size`                        | integer          | no                                   | Maximum import size in MB. 0 for unlimited. Default = 50 |
 | `max_pages_size`                         | integer          | no                                   | Maximum size of pages repositories in MB |
 | `max_personal_access_token_lifetime`     | integer          | no                                   | **(ULTIMATE ONLY)** Maximum allowable lifetime for personal access tokens in days |
 | `metrics_method_call_threshold`          | integer          | no                                   | A method call is only tracked when it takes longer than the given amount of milliseconds. |

doc/user/admin_area/settings/account_and_limit_settings.md

@@ -15,6 +15,17 @@ If you choose a size larger than what is currently configured for the web server
 you will likely get errors. See the [troubleshooting section](#troubleshooting) for more
 details.
 
+## Max import size
+
+You can change the maximum file size for imports in GitLab.
+Navigate to **Admin Area (wrench icon) > Settings > General**, then expand **Account and Limit**.
+From here, you can increase or decrease by changing the value in `Maximum import size (MB)`.
+
+NOTE: **Note:**
+If you choose a size larger than what is currently configured for the web server,
+you will likely get errors. See the [troubleshooting section](#troubleshooting) for more
+details.
+
 ## Maximum namespace storage size
 
 This sets a maximum size limit on each namespace. The following are included in the namespace size:

doc/user/group/settings/import_export.md

@@ -75,6 +75,10 @@ For more details on the specific data persisted in a group export, see the
 1. Alternatively, you can come back to the project settings and download the
    file from there by clicking **Download export**, or generate a new file by clicking **Regenerate export**.
 
+NOTE: **Note:**
+The maximum import file size can be set by the Administrator, default is 50MB.
+As an administrator, you can modify the maximum import file size. To do so, use the `max_import_size` option in the [Application settings API](../../../api/settings.md#change-application-settings) or the [Admin UI](../../admin_area/settings/account_and_limit_settings.md).
+
 ### Between CE and EE
 
 You can export groups from the [Community Edition to the Enterprise Edition](https://about.gitlab.com/install/ce-or-ee/) and vice versa.

doc/user/project/settings/import_export.md

@@ -158,6 +158,10 @@ If use of the `Internal` visibility level
 [is restricted](../../../public_access/public_access.md#restricting-the-use-of-public-or-internal-projects),
 all imported projects are given the visibility of `Private`.
 
+NOTE: **Note:**
+The maximum import file size can be set by the Administrator, default is 50MB.
+As an administrator, you can modify the maximum import file size. To do so, use the `max_import_size` option in the [Application settings API](../../../api/settings.md#change-application-settings) or the [Admin UI](../../admin_area/settings/account_and_limit_settings.md).
+
 ## Rate limits
 
 To help avoid abuse, users are rate limited to:

lib/api/group_import.rb

@@ -40,7 +40,7 @@ module API
 
         ImportExportUploader.workhorse_authorize(
           has_length: false,
-          maximum_size: ImportExportUpload::MAXIMUM_IMPORT_FILE_SIZE
+          maximum_size: Gitlab::CurrentSettings.max_import_size.megabytes
         )
       end
 

lib/api/project_import.rb

@@ -30,7 +30,10 @@ module API
         status 200
         content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
 
-        ImportExportUploader.workhorse_authorize(has_length: false, maximum_size: MAXIMUM_FILE_SIZE)
+        ImportExportUploader.workhorse_authorize(
+          has_length: false,
+          maximum_size: Gitlab::CurrentSettings.max_import_size.megabytes
+        )
       end
 
       params do

lib/api/settings.rb

@@ -83,6 +83,7 @@ module API
                                 desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com'
       optional :max_artifacts_size, type: Integer, desc: "Set the maximum file size for each job's artifacts"
       optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB'
+      optional :max_import_size, type: Integer, desc: 'Maximum import size in MB'
       optional :max_pages_size, type: Integer, desc: 'Maximum size of pages in MB'
       optional :metrics_method_call_threshold, type: Integer, desc: 'A method call is only tracked when it takes longer to complete than the given amount of milliseconds.'
       optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5

locale/gitlab.pot

@@ -747,6 +747,9 @@ msgstr ""
 msgid "0 for unlimited"
 msgstr ""
 
+msgid "0 for unlimited, only effective with remote storage enabled."
+msgstr ""
+
 msgid "1 %{type} addition"
 msgid_plural "%{count} %{type} additions"
 msgstr[0] ""
@@ -13544,6 +13547,9 @@ msgstr ""
 msgid "Maximum field length"
 msgstr ""
 
+msgid "Maximum import size (MB)"
+msgstr ""
+
 msgid "Maximum job timeout"
 msgstr ""
 
@@ -13583,6 +13589,9 @@ msgstr ""
 msgid "Maximum size of Elasticsearch bulk indexing requests."
 msgstr ""
 
+msgid "Maximum size of import files."
+msgstr ""
+
 msgid "Maximum size of individual attachments in comments."
 msgstr ""
 

spec/features/admin/admin_settings_spec.rb

@@ -105,6 +105,16 @@ describe 'Admin updates settings', :clean_gitlab_redis_shared_state, :do_not_moc
         expect(page).to have_content "Application settings saved successfully"
       end
 
+      it 'Change Maximum import size' do
+        page.within('.as-account-limit') do
+          fill_in 'Maximum import size (MB)', with: 15
+          click_button 'Save changes'
+        end
+
+        expect(current_settings.max_import_size).to eq 15
+        expect(page).to have_content "Application settings saved successfully"
+      end
+
       it 'Change New users set to external', :js do
         user_internal_regex = find('#application_setting_user_default_internal_regex', visible: :all)
 

spec/models/application_setting_spec.rb

@@ -283,6 +283,14 @@ describe ApplicationSetting do
         .is_greater_than(0)
     end
 
+    it { is_expected.to validate_presence_of(:max_import_size) }
+
+    it do
+      is_expected.to validate_numericality_of(:max_import_size)
+        .only_integer
+        .is_greater_than_or_equal_to(0)
+    end
+
     it do
       is_expected.to validate_numericality_of(:local_markdown_version)
         .only_integer