Add workers to handle annotations prune cron task

In order to prevent database from overloading with stale annotatinos
data we need to periodically clean it.

app/workers/all_queues.yml

@@ -163,6 +163,14 @@
   :weight: 1
   :idempotent: 
   :tags: []
+- :name: cronjob:metrics_dashboard_schedule_annotations_prune
+  :feature_category: :metrics
+  :has_external_dependencies: 
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: cronjob:namespaces_prune_aggregation_schedules
   :feature_category: :source_code_management
   :has_external_dependencies: 
@@ -1347,6 +1355,14 @@
   :weight: 1
   :idempotent: true
   :tags: []
+- :name: metrics_dashboard_prune_old_annotations
+  :feature_category: :metrics
+  :has_external_dependencies: 
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: migrate_external_diffs
   :feature_category: :source_code_management
   :has_external_dependencies: 

app/workers/metrics/dashboard/prune_old_annotations_worker.rb

+# frozen_string_literal: true
+
+module Metrics
+  module Dashboard
+    class PruneOldAnnotationsWorker
+      include ApplicationWorker
+
+      DELETE_LIMIT = 10_000
+      DEFAULT_CUT_OFF_PERIOD = 2.weeks
+
+      feature_category :metrics
+
+      idempotent! # in the scope of 24 hours
+
+      def perform
+        stale_annotations = ::Metrics::Dashboard::Annotation.ending_before(DEFAULT_CUT_OFF_PERIOD.ago.beginning_of_day)
+        stale_annotations.delete_with_limit(DELETE_LIMIT)
+
+        self.class.perform_async if stale_annotations.exists?
+      end
+    end
+  end
+end

app/workers/metrics/dashboard/schedule_annotations_prune_worker.rb

+# frozen_string_literal: true
+
+module Metrics
+  module Dashboard
+    class ScheduleAnnotationsPruneWorker
+      include ApplicationWorker
+      # rubocop:disable Scalability/CronWorkerContext
+      # This worker does not perform work scoped to a context
+      include CronjobQueue
+      # rubocop:enable Scalability/CronWorkerContext
+
+      feature_category :metrics
+
+      idempotent! # PruneOldAnnotationsWorker worker is idempotent in the scope of 24 hours
+
+      def perform
+        # Process is split into two jobs to avoid long running jobs, which are more prone to be disrupted
+        # mid work, which may cause some data not be delete, especially because cronjobs has retry option disabled
+        PruneOldAnnotationsWorker.perform_async
+      end
+    end
+  end
+end

changelogs/unreleased/mwaw-211433-metrics-dashboard-annotations-reaper-job-worker.yml

+---
+title: Remove metrics dashboard annotations attached to time periods older than two weeks.
+merge_request: 32838
+author:
+type: added

config/initializers/1_settings.rb

@@ -481,6 +481,9 @@ Settings.cron_jobs['issue_due_scheduler_worker']['job_class'] = 'IssueDueSchedul
 Settings.cron_jobs['prune_web_hook_logs_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['prune_web_hook_logs_worker']['cron'] ||= '0 */1 * * *'
 Settings.cron_jobs['prune_web_hook_logs_worker']['job_class'] = 'PruneWebHookLogsWorker'
+Settings.cron_jobs['metrics_dashboard_schedule_annotations_prune_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['metrics_dashboard_schedule_annotations_prune_worker']['cron'] ||= '0 1 * * *'
+Settings.cron_jobs['metrics_dashboard_schedule_annotations_prune_worker']['job_class'] = 'Metrics::Dashboard::ScheduleAnnotationsPruneWorker'
 Settings.cron_jobs['schedule_migrate_external_diffs_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['schedule_migrate_external_diffs_worker']['cron'] ||= '15 * * * *'
 Settings.cron_jobs['schedule_migrate_external_diffs_worker']['job_class'] = 'ScheduleMigrateExternalDiffsWorker'

config/sidekiq_queues.yml

@@ -150,6 +150,8 @@
   - 5
 - - merge_request_mergeability_check
   - 1
+- - metrics_dashboard_prune_old_annotations
+  - 1
 - - migrate_external_diffs
   - 1
 - - namespaceless_project_destroy

doc/user/project/integrations/prometheus.md

@@ -827,6 +827,14 @@ You can create annotations by making requests to the
 
 ![Annotations UI](img/metrics_dashboard_annotations_ui_v13.0.png)
 
+#### Retention policy
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/211433) in GitLab 13.01.
+
+To avoid excessive storage space consumption by stale annotations, records attached
+to time periods older than two weeks are removed daily. This recurring background
+job runs at 1:00 a.m. local server time.
+
 ### Expand panel
 
 > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3100) in GitLab 13.0.

spec/workers/metrics/dashboard/prune_old_annotations_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Metrics::Dashboard::PruneOldAnnotationsWorker do
+  let_it_be(:now) { DateTime.parse('2020-06-02T00:12:00Z') }
+  let_it_be(:two_weeks_old_annotation) { create(:metrics_dashboard_annotation, starting_at: now.advance(weeks: -2)) }
+  let_it_be(:one_day_old_annotation) { create(:metrics_dashboard_annotation, starting_at: now.advance(days: -1)) }
+  let_it_be(:month_old_annotation) { create(:metrics_dashboard_annotation, starting_at: now.advance(months: -1)) }
+
+  describe '#perform' do
+    it 'removes all annotations older than cut off', :aggregate_failures do
+      Timecop.freeze(now) do
+        described_class.new.perform
+
+        expect(Metrics::Dashboard::Annotation.all).to match_array([one_day_old_annotation, two_weeks_old_annotation])
+
+        # is idempotent in the scope of 24h
+        expect { described_class.new.perform }.not_to change { Metrics::Dashboard::Annotation.all.to_a }
+        Timecop.travel(24.hours.from_now) do
+          described_class.new.perform
+          expect(Metrics::Dashboard::Annotation.all).to match_array([one_day_old_annotation])
+        end
+      end
+    end
+
+    context 'batch to be deleted is bigger than upper limit' do
+      it 'schedules second job to clear remaining records' do
+        Timecop.freeze(now) do
+          create(:metrics_dashboard_annotation, starting_at: 1.month.ago)
+          stub_const("#{described_class}::DELETE_LIMIT", 1)
+
+          expect(described_class).to receive(:perform_async)
+
+          described_class.new.perform
+        end
+      end
+    end
+  end
+end

spec/workers/metrics/dashboard/schedule_annotations_prune_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Metrics::Dashboard::ScheduleAnnotationsPruneWorker do
+  describe '#perform' do
+    it 'schedules annotations prune job with default cut off date' do
+      expect(Metrics::Dashboard::PruneOldAnnotationsWorker).to receive(:perform_async)
+
+      described_class.new.perform
+    end
+  end
+end