Merge branch 'idempotent-worker-with-deduplication-ff' into 'master'

Add feature flag option to worker deduplication

See merge request gitlab-org/gitlab!63226

app/workers/concerns/worker_attributes.rb

@@ -186,6 +186,12 @@ module WorkerAttributes
       class_attributes[:deduplication_options] || {}
     end
 
+    def deduplication_enabled?
+      return true unless get_deduplication_options[:feature_flag]
+
+      Feature.enabled?(get_deduplication_options[:feature_flag], default_enabled: :yaml)
+    end
+
     def big_payload!
       set_class_attribute(:big_payload, true)
     end

lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb

@@ -19,6 +19,7 @@ module Gitlab
       class DuplicateJob
         DUPLICATE_KEY_TTL = 6.hours
         DEFAULT_STRATEGY = :until_executing
+        STRATEGY_NONE = :none
 
         attr_reader :existing_jid
 
@@ -102,6 +103,7 @@ module Gitlab
         def strategy
           return DEFAULT_STRATEGY unless worker_klass
           return DEFAULT_STRATEGY unless worker_klass.respond_to?(:idempotent?)
+          return STRATEGY_NONE unless worker_klass.deduplication_enabled?
 
           worker_klass.get_deduplicate_strategy
         end

spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb

@@ -18,14 +18,43 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob, :clean_gi
   end
 
   describe '#schedule' do
-    it 'calls schedule on the strategy' do
-      expect do |block|
-        expect_next_instance_of(Gitlab::SidekiqMiddleware::DuplicateJobs::Strategies::UntilExecuting) do |strategy|
-          expect(strategy).to receive(:schedule).with(job, &block)
+    shared_examples 'scheduling with deduplication class' do |strategy_class|
+      it 'calls schedule on the strategy' do
+        expect do |block|
+          expect_next_instance_of("Gitlab::SidekiqMiddleware::DuplicateJobs::Strategies::#{strategy_class}".constantize) do |strategy|
+            expect(strategy).to receive(:schedule).with(job, &block)
+          end
+
+          duplicate_job.schedule(&block)
+        end.to yield_control
+      end
+    end
+
+    it_behaves_like 'scheduling with deduplication class', 'UntilExecuting'
+
+    context 'when the deduplication depends on a FF' do
+      before do
+        skip_feature_flags_yaml_validation
+        skip_default_enabled_yaml_check
+
+        allow(AuthorizedProjectsWorker).to receive(:get_deduplication_options).and_return(feature_flag: :my_feature_flag)
+      end
+
+      context 'when the feature flag is enabled' do
+        before do
+          stub_feature_flags(my_feature_flag: true)
         end
 
-        duplicate_job.schedule(&block)
-      end.to yield_control
+        it_behaves_like 'scheduling with deduplication class', 'UntilExecuting'
+      end
+
+      context 'when the feature flag is disabled' do
+        before do
+          stub_feature_flags(my_feature_flag: false)
+        end
+
+        it_behaves_like 'scheduling with deduplication class', 'None'
+      end
     end
   end
 

spec/workers/concerns/worker_attributes_spec.rb

@@ -62,6 +62,12 @@ RSpec.describe WorkerAttributes do
   end
 
   describe '.idempotent!' do
+    it 'sets `idempotent` attribute of the worker class to true' do
+      worker.idempotent!
+
+      expect(worker.send(:class_attributes)[:idempotent]).to eq(true)
+    end
+
     context 'when data consistency is not :always' do
       it 'raise exception' do
         worker.data_consistency(:sticky)
@@ -71,4 +77,66 @@ RSpec.describe WorkerAttributes do
       end
     end
   end
+
+  describe '.idempotent?' do
+    subject(:idempotent?) { worker.idempotent? }
+
+    context 'when the worker is idempotent' do
+      before do
+        worker.idempotent!
+      end
+
+      it { is_expected.to be_truthy }
+    end
+
+    context 'when the worker is not idempotent' do
+      it { is_expected.to be_falsey }
+    end
+  end
+
+  describe '.deduplicate' do
+    it 'sets deduplication_strategy and deduplication_options' do
+      worker.deduplicate(:until_executing, including_scheduled: true)
+
+      expect(worker.send(:class_attributes)[:deduplication_strategy]).to eq(:until_executing)
+      expect(worker.send(:class_attributes)[:deduplication_options]).to eq(including_scheduled: true)
+    end
+  end
+
+  describe '#deduplication_enabled?' do
+    subject(:deduplication_enabled?) { worker.deduplication_enabled? }
+
+    context 'when no feature flag is set' do
+      before do
+        worker.deduplicate(:until_executing)
+      end
+
+      it { is_expected.to eq(true) }
+    end
+
+    context 'when feature flag is set' do
+      before do
+        skip_feature_flags_yaml_validation
+        skip_default_enabled_yaml_check
+
+        worker.deduplicate(:until_executing, feature_flag: :my_feature_flag)
+      end
+
+      context 'when the FF is enabled' do
+        before do
+          stub_feature_flags(my_feature_flag: true)
+        end
+
+        it { is_expected.to eq(true) }
+      end
+
+      context 'when the FF is disabled' do
+        before do
+          stub_feature_flags(my_feature_flag: false)
+        end
+
+        it { is_expected.to eq(false) }
+      end
+    end
+  end
 end