Merge branch '341377-enable-idempotency-for-expire-job-cache-worker' into 'master'

Enable deduplication for ExpireJobCacheWorker

See merge request gitlab-org/gitlab!71199

app/workers/all_queues.yml

@@ -1427,7 +1427,7 @@
   :urgency: :high
   :resource_boundary: :unknown
   :weight: 3
-  :idempotent:
+  :idempotent: true
   :tags: []
 - :name: pipeline_cache:expire_pipeline_cache
   :worker_name: ExpirePipelineCacheWorker

app/workers/expire_job_cache_worker.rb

@@ -10,11 +10,9 @@ class ExpireJobCacheWorker # rubocop:disable Scalability/IdempotentWorker
 
   queue_namespace :pipeline_cache
   urgency :high
-  # This worker should be idempotent, but we're switching to data_consistency
-  # :sticky and there is an ongoing incompatibility, so it needs to be disabled for
-  # now. The following line can be uncommented and this comment removed once
-  # https://gitlab.com/gitlab-org/gitlab/-/issues/325291 is resolved.
-  # idempotent!
+
+  deduplicate :until_executing, including_scheduled: true
+  idempotent!
 
   # rubocop: disable CodeReuse/ActiveRecord
   def perform(job_id)

spec/workers/expire_job_cache_worker_spec.rb

@@ -13,6 +13,8 @@ RSpec.describe ExpireJobCacheWorker do
 
       let(:job_args) { job.id }
 
+      it_behaves_like 'an idempotent worker'
+
       it_behaves_like 'worker with data consistency',
         described_class,
         data_consistency: :delayed