Limit Sidekiq push_bulk to a maximum of 1000 jobs in one go by default

This MR introduces limiting Sidekiq
jobs enqueued via `push_bulk` to a maximum
of 1000 jobs in one go.

Changelog: added

app/workers/concerns/application_worker.rb

@@ -189,12 +189,8 @@ module ApplicationWorker
         schedule_at = bulk_schedule_at
       end
 
-      if Feature.enabled?(:sidekiq_push_bulk_in_batches)
-        in_safe_limit_batches(args_list, schedule_at) do |args_batch, schedule_at_for_batch|
-          Sidekiq::Client.push_bulk('class' => self, 'args' => args_batch, 'at' => schedule_at_for_batch)
-        end
-      else
-        Sidekiq::Client.push_bulk('class' => self, 'args' => args_list, 'at' => schedule_at)
+      in_safe_limit_batches(args_list, schedule_at) do |args_batch, schedule_at_for_batch|
+        Sidekiq::Client.push_bulk('class' => self, 'args' => args_batch, 'at' => schedule_at_for_batch)
       end
     end
 
@@ -207,12 +203,8 @@ module ApplicationWorker
     private
 
     def do_push_bulk(args_list)
-      if Feature.enabled?(:sidekiq_push_bulk_in_batches)
-        in_safe_limit_batches(args_list) do |args_batch, _|
-          Sidekiq::Client.push_bulk('class' => self, 'args' => args_batch)
-        end
-      else
-        Sidekiq::Client.push_bulk('class' => self, 'args' => args_list)
+      in_safe_limit_batches(args_list) do |args_batch, _|
+        Sidekiq::Client.push_bulk('class' => self, 'args' => args_batch)
       end
     end
 

config/feature_flags/development/sidekiq_push_bulk_in_batches.yml

----
-name: sidekiq_push_bulk_in_batches
-introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/72263
-rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/343740
-milestone: '14.5'
-type: development
-group: group::access
-default_enabled: false

spec/workers/concerns/application_worker_spec.rb

@@ -287,12 +287,6 @@ RSpec.describe ApplicationWorker do
   end
 
   context 'different kinds of push_bulk' do
-    shared_context 'disable the `sidekiq_push_bulk_in_batches` feature flag' do
-      before do
-        stub_feature_flags(sidekiq_push_bulk_in_batches: false)
-      end
-    end
-
     shared_context 'set safe limit beyond the number of jobs to be enqueued' do
       before do
         stub_const("#{described_class}::SAFE_PUSH_BULK_LIMIT", args.count + 1)
@@ -408,27 +402,6 @@ RSpec.describe ApplicationWorker do
           it_behaves_like 'returns job_id of all enqueued jobs'
           it_behaves_like 'does not schedule the jobs for any specific time'
         end
-
-        context 'when the feature flag `sidekiq_push_bulk_in_batches` is disabled' do
-          include_context 'disable the `sidekiq_push_bulk_in_batches` feature flag'
-
-          context 'when the number of jobs to be enqueued does not exceed the safe limit' do
-            include_context 'set safe limit beyond the number of jobs to be enqueued'
-
-            it_behaves_like 'enqueues jobs in one go'
-            it_behaves_like 'logs bulk insertions'
-            it_behaves_like 'returns job_id of all enqueued jobs'
-            it_behaves_like 'does not schedule the jobs for any specific time'
-          end
-
-          context 'when the number of jobs to be enqueued exceeds safe limit' do
-            include_context 'set safe limit below the number of jobs to be enqueued'
-
-            it_behaves_like 'enqueues jobs in one go'
-            it_behaves_like 'returns job_id of all enqueued jobs'
-            it_behaves_like 'does not schedule the jobs for any specific time'
-          end
-        end
       end
     end
 
@@ -476,26 +449,6 @@ RSpec.describe ApplicationWorker do
             it_behaves_like 'returns job_id of all enqueued jobs'
             it_behaves_like 'schedules all the jobs at a specific time'
           end
-
-          context 'when the feature flag `sidekiq_push_bulk_in_batches` is disabled' do
-            include_context 'disable the `sidekiq_push_bulk_in_batches` feature flag'
-
-            context 'when the number of jobs to be enqueued does not exceed the safe limit' do
-              include_context 'set safe limit beyond the number of jobs to be enqueued'
-
-              it_behaves_like 'enqueues jobs in one go'
-              it_behaves_like 'returns job_id of all enqueued jobs'
-              it_behaves_like 'schedules all the jobs at a specific time'
-            end
-
-            context 'when the number of jobs to be enqueued exceeds safe limit' do
-              include_context 'set safe limit below the number of jobs to be enqueued'
-
-              it_behaves_like 'enqueues jobs in one go'
-              it_behaves_like 'returns job_id of all enqueued jobs'
-              it_behaves_like 'schedules all the jobs at a specific time'
-            end
-          end
         end
       end
 
@@ -575,26 +528,6 @@ RSpec.describe ApplicationWorker do
             it_behaves_like 'returns job_id of all enqueued jobs'
             it_behaves_like 'schedules all the jobs at a specific time, per batch'
           end
-
-          context 'when the feature flag `sidekiq_push_bulk_in_batches` is disabled' do
-            include_context 'disable the `sidekiq_push_bulk_in_batches` feature flag'
-
-            context 'when the number of jobs to be enqueued does not exceed the safe limit' do
-              include_context 'set safe limit beyond the number of jobs to be enqueued'
-
-              it_behaves_like 'enqueues jobs in one go'
-              it_behaves_like 'returns job_id of all enqueued jobs'
-              it_behaves_like 'schedules all the jobs at a specific time, per batch'
-            end
-
-            context 'when the number of jobs to be enqueued exceeds safe limit' do
-              include_context 'set safe limit below the number of jobs to be enqueued'
-
-              it_behaves_like 'enqueues jobs in one go'
-              it_behaves_like 'returns job_id of all enqueued jobs'
-              it_behaves_like 'schedules all the jobs at a specific time, per batch'
-            end
-          end
         end
       end
     end