Expose cleanup policies limits in the admin area

Document them properly so that interested self managed
users can try them

app/helpers/application_settings_helper.rb

@@ -337,7 +337,9 @@ module ApplicationSettingsHelper
       :group_download_export_limit,
       :wiki_page_max_content_bytes,
       :container_registry_delete_tags_service_timeout,
-      :rate_limiting_response_text
+      :rate_limiting_response_text,
+      :container_registry_expiration_policies_worker_capacity,
+      :container_registry_cleanup_tags_service_max_list_size
     ]
   end
 

app/helpers/container_registry_helper.rb

 # frozen_string_literal: true
 
 module ContainerRegistryHelper
-  def limit_delete_tags_service?
+  def container_registry_expiration_policies_throttling?
     Feature.enabled?(:container_registry_expiration_policies_throttling) &&
       ContainerRegistry::Client.supports_tag_delete?
   end

app/views/admin/application_settings/_registry.html.haml

@@ -14,11 +14,21 @@
         .form-text.text-muted
           = _("Existing projects will be able to use expiration policies. Avoid enabling this if an external Container Registry is being used, as there is a performance risk if many images exist on one project.")
           = link_to sprite_icon('question-o'), help_page_path('user/packages/container_registry/index', anchor: 'use-with-external-container-registries')
-    - if limit_delete_tags_service?
+    - if container_registry_expiration_policies_throttling?
       .form-group
         = f.label :container_registry_delete_tags_service_timeout, _('Cleanup policy maximum processing time (seconds)'), class: 'label-bold'
         = f.number_field :container_registry_delete_tags_service_timeout, min: 0, class: 'form-control'
         .form-text.text-muted
           = _("Tags are deleted until the timeout is reached. Any remaining tags are included the next time the policy runs. To remove the time limit, set it to 0.")
+      .form-group
+        = f.label :container_registry_expiration_policies_worker_capacity, _('Cleanup policy maximum workers running concurrently'), class: 'label-bold'
+        = f.number_field :container_registry_expiration_policies_worker_capacity, min: 0, class: 'form-control'
+        .form-text.text-muted
+          = _("Cleanup policies are executed by background workers. This setting defines the maximum number of workers that can run concurrently. Set it to 0 to remove all workers and not execute the cleanup policies.")
+      .form-group
+        = f.label :container_registry_cleanup_tags_service_max_list_size, _('Cleanup policy maximum number of tags to be deleted'), class: 'label-bold'
+        = f.number_field :container_registry_cleanup_tags_service_max_list_size, min: 0, class: 'form-control'
+        .form-text.text-muted
+          = _("The maximum number of tags that a single worker accepts for cleanup. If the number of tags goes above this limit, the list of tags to delete is truncated to this number. To remove this limit, set it to 0.")
 
   = f.submit 'Save changes', class: "gl-button btn btn-success"

doc/user/packages/container_registry/index.md

@@ -571,6 +571,52 @@ Here are examples of regex patterns you may want to use:
   (?:v.+|master|release)
   ```
 
+### Set cleanup limits to conserve resources 
+
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/288812) in GitLab 13.9.
+> - It's [deployed behind a feature flag](../../feature_flags.md), disabled by default.
+> - It's disabled on GitLab.com.
+> - It's not recommended for production use.
+> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-cleanup-policy-limits).
+
+Cleanup policies are executed as a background process. This process is complex, and depending on the number of tags to delete,
+the process can take time to finish.
+
+To prevent server resource starvation, the following application settings are available:
+
+- `container_registry_expiration_policies_worker_capacity`. The maximum number of cleanup workers running concurrently. This must be greater than `1`.
+   We recommend starting with a low number and increasing it after monitoring the resources used by the background workers.
+- `container_registry_delete_tags_service_timeout`. The maximum time, in seconds, that the cleanup process can take to delete a batch of tags.
+- `container_registry_cleanup_tags_service_max_list_size`. The maximum number of tags that can be deleted in a single execution. Additional tags must be deleted in another execution.
+   We recommend starting with a low number, like `100`, and increasing it after monitoring that container images are properly deleted.
+   
+For self-managed instances, those settings can be updated in the [Rails console](../../../administration/operations/rails_console.md#starting-a-rails-console-session):
+
+  ```ruby
+  ApplicationSetting.last.update(container_registry_expiration_policies_worker_capacity: 3)
+  ```
+
+Alternatively, once the limits are [enabled](#enable-or-disable-cleanup-policy-limits), they are available in the [admin area](../../admin_area/index.md) **Settings > CI/CD > Container Registry**.
+
+#### Enable or disable cleanup policy limits
+
+The cleanup policies limits are under development and not ready for production use. They are
+deployed behind a feature flag that is **disabled by default**.
+[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md)
+can enable it.
+
+To enable it:
+
+```ruby
+Feature.enable(:container_registry_expiration_policies_throttling)
+```
+
+To disable it:
+
+```ruby
+Feature.disable(:container_registry_expiration_policies_throttling)
+```
+
 ### Use the cleanup policy API
 
 You can set, update, and disable the cleanup policies using the GitLab API.

locale/gitlab.pot

@@ -5821,9 +5821,18 @@ msgstr ""
 msgid "Clean up image tags"
 msgstr ""
 
+msgid "Cleanup policies are executed by background workers. This setting defines the maximum number of workers that can run concurrently. Set it to 0 to remove all workers and not execute the cleanup policies."
+msgstr ""
+
+msgid "Cleanup policy maximum number of tags to be deleted"
+msgstr ""
+
 msgid "Cleanup policy maximum processing time (seconds)"
 msgstr ""
 
+msgid "Cleanup policy maximum workers running concurrently"
+msgstr ""
+
 msgid "Clear"
 msgstr ""
 
@@ -28349,6 +28358,9 @@ msgstr ""
 msgid "The maximum file size is %{size}."
 msgstr ""
 
+msgid "The maximum number of tags that a single worker accepts for cleanup. If the number of tags goes above this limit, the list of tags to delete is truncated to this number. To remove this limit, set it to 0."
+msgstr ""
+
 msgid "The merge conflicts for this merge request cannot be resolved through GitLab. Please try to resolve them locally."
 msgstr ""
 

spec/features/admin/admin_settings_spec.rb

@@ -315,50 +315,59 @@ RSpec.describe 'Admin updates settings' do
       end
 
       context 'Container Registry' do
-        context 'delete tags service execution timeout' do
-          let(:feature_flag_enabled) { true }
-          let(:client_support) { true }
-
-          before do
-            stub_container_registry_config(enabled: true)
-            stub_feature_flags(container_registry_expiration_policies_throttling: feature_flag_enabled)
-            allow(ContainerRegistry::Client).to receive(:supports_tag_delete?).and_return(client_support)
-          end
+        let(:feature_flag_enabled) { true }
+        let(:client_support) { true }
+        let(:settings_titles) do
+          {
+            container_registry_delete_tags_service_timeout: 'Container Registry delete tags service execution timeout',
+            container_registry_expiration_policies_worker_capacity: 'Cleanup policy maximum workers running concurrently',
+            container_registry_cleanup_tags_service_max_list_size: 'Cleanup policy maximum number of tags to be deleted'
+          }
+        end
+
+        before do
+          stub_container_registry_config(enabled: true)
+          stub_feature_flags(container_registry_expiration_policies_throttling: feature_flag_enabled)
+          allow(ContainerRegistry::Client).to receive(:supports_tag_delete?).and_return(client_support)
+        end
 
-          RSpec.shared_examples 'not having service timeout settings' do
-            it 'lacks the timeout settings' do
-              visit ci_cd_admin_application_settings_path
+        shared_examples 'not having container registry setting' do |registry_setting|
+          it "lacks the container setting #{registry_setting}" do
+            visit ci_cd_admin_application_settings_path
 
-              expect(page).not_to have_content "Container Registry delete tags service execution timeout"
-            end
+            expect(page).not_to have_content(settings_titles[registry_setting])
           end
+        end
 
-          context 'with feature flag enabled' do
-            context 'with client supporting tag delete' do
-              it 'changes the timeout' do
-                visit ci_cd_admin_application_settings_path
+        %i[container_registry_delete_tags_service_timeout container_registry_expiration_policies_worker_capacity container_registry_cleanup_tags_service_max_list_size].each do |setting|
+          context "for container registry setting #{setting}" do
+            context 'with feature flag enabled' do
+              context 'with client supporting tag delete' do
+                it 'changes the setting' do
+                  visit ci_cd_admin_application_settings_path
 
-                page.within('.as-registry') do
-                  fill_in 'application_setting_container_registry_delete_tags_service_timeout', with: 400
-                  click_button 'Save changes'
-                end
+                  page.within('.as-registry') do
+                    fill_in "application_setting_#{setting}", with: 400
+                    click_button 'Save changes'
+                  end
 
-                expect(current_settings.container_registry_delete_tags_service_timeout).to eq(400)
-                expect(page).to have_content "Application settings saved successfully"
+                  expect(current_settings.public_send(setting)).to eq(400)
+                  expect(page).to have_content "Application settings saved successfully"
+                end
               end
-            end
 
-            context 'with client not supporting tag delete' do
-              let(:client_support) { false }
+              context 'with client not supporting tag delete' do
+                let(:client_support) { false }
 
-              it_behaves_like 'not having service timeout settings'
+                it_behaves_like 'not having container registry setting', setting
+              end
             end
-          end
 
-          context 'with feature flag disabled' do
-            let(:feature_flag_enabled) { false }
+            context 'with feature flag disabled' do
+              let(:feature_flag_enabled) { false }
 
-            it_behaves_like 'not having service timeout settings'
+              it_behaves_like 'not having container registry setting', setting
+            end
           end
         end
       end

spec/helpers/container_registry_helper_spec.rb

@@ -5,8 +5,8 @@ require 'spec_helper'
 RSpec.describe ContainerRegistryHelper do
   using RSpec::Parameterized::TableSyntax
 
-  describe '#limit_delete_tags_service?' do
-    subject { helper.limit_delete_tags_service? }
+  describe '#container_registry_expiration_policies_throttling?' do
+    subject { helper.container_registry_expiration_policies_throttling? }
 
     where(:feature_flag_enabled, :client_support, :expected_result) do
       true  | true  | true