Merge branch '217918-revert-revert-metrics-paused-indexing-Sidekiq-queue-size' into 'master'

Revert revert of Add Paused Indexing Sidekiq queue_size to Metrics

See merge request gitlab-org/gitlab!36473

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -173,6 +173,7 @@ configuration option in `gitlab.yml`. These metrics are served from the
 | `geo_repositories_retrying_verification_count` | Gauge   | 11.2  | Number of repositories verification failures that Geo is actively trying to correct on secondary  | `url` |
 | `geo_wikis_retrying_verification_count`        | Gauge   | 11.2  | Number of wikis verification failures that Geo is actively trying to correct on secondary | `url` |
 | `global_search_bulk_cron_queue_size`           | Gauge   | 12.10 | Number of database records waiting to be synchronized to Elasticsearch | |
+| `global_search_awaiting_indexing_queue_size`   | Gauge   | 13.2  | Number of database updates waiting to be synchronized to Elasticsearch while indexing is paused | |
 | `package_files_count`                          | Gauge   | 13.0  | Number of package files on primary | `url` |
 | `package_files_checksummed_count`              | Gauge   | 13.0  | Number of package files checksummed on primary | `url` |
 | `package_files_checksum_failed_count`          | Gauge   | 13.0  | Number of package files failed to calculate the checksum on primary

ee/app/services/elastic/indexing_control_service.rb

@@ -28,6 +28,12 @@ module Elastic
       def resume_processing!(klass)
         new(klass).resume_processing!
       end
+
+      def queue_size
+        Elastic::IndexingControl::WORKERS.sum do |worker_class| # rubocop:disable CodeReuse/ActiveRecord
+          new(worker_class).queue_size
+        end
+      end
     end
 
     def add_to_waiting_queue!(args, context)

ee/changelogs/unreleased/217918-follow-up-from-implement-a-switch-to-pause-all-of-the-write-req.yml

+---
+title: Add Prometheus Metrics for Index Pause Sidekiq Queue.
+merge_request: 36473
+author:
+type: changed

ee/lib/elastic/metrics_update_service.rb

@@ -11,6 +11,9 @@ module Elastic
 
       initial_gauge = Gitlab::Metrics.gauge(:global_search_bulk_cron_initial_queue_size, 'Number of initial database updates waiting to be synchronized to Elasticsearch', {}, :max)
       initial_gauge.set({}, Elastic::ProcessInitialBookkeepingService.queue_size)
+
+      awaiting_indexing_gauge = Gitlab::Metrics.gauge(:global_search_awaiting_indexing_queue_size, 'Number of database updates waiting to be synchronized to Elasticsearch while indexing is paused.', {}, :max)
+      awaiting_indexing_gauge.set({}, Elastic::IndexingControlService.queue_size)
     end
 
     private

ee/spec/services/elastic/indexing_control_service_spec.rb

@@ -54,6 +54,20 @@ RSpec.describe Elastic::IndexingControlService, :clean_gitlab_redis_shared_state
     end
   end
 
+  describe '.queue_size' do
+    it 'reports the queue size' do
+      stub_const("Elastic::IndexingControl::WORKERS", [worker_class])
+
+      expect(described_class.queue_size).to eq(0)
+
+      subject.add_to_waiting_queue!(worker_args, worker_context)
+
+      expect(described_class.queue_size).to eq(1)
+
+      expect { subject.resume_processing! }.to change(described_class, :queue_size).by(-1)
+    end
+  end
+
   describe '#add_to_waiting_queue!' do
     it 'adds a job to the set' do
       expect { subject.add_to_waiting_queue!(worker_args, worker_context) }.to change { subject.queue_size }.from(0).to(1)

ee/spec/services/elastic/metrics_update_service_spec.rb

@@ -14,6 +14,7 @@ RSpec.describe Elastic::MetricsUpdateService, :prometheus do
     it 'sets gauges' do
       expect(Elastic::ProcessBookkeepingService).to receive(:queue_size).and_return(4)
       expect(Elastic::ProcessInitialBookkeepingService).to receive(:queue_size).and_return(6)
+      expect(Elastic::IndexingControlService).to receive(:queue_size).and_return(2)
 
       incremental_gauge_double = instance_double(Prometheus::Client::Gauge)
       expect(Gitlab::Metrics).to receive(:gauge)
@@ -25,8 +26,14 @@ RSpec.describe Elastic::MetricsUpdateService, :prometheus do
         .with(:global_search_bulk_cron_initial_queue_size, anything, {}, :max)
         .and_return(initial_gauge_double)
 
+      awaiting_indexing_gauge = instance_double(Prometheus::Client::Gauge)
+      expect(Gitlab::Metrics).to receive(:gauge)
+        .with(:global_search_awaiting_indexing_queue_size, anything, {}, :max)
+        .and_return(awaiting_indexing_gauge)
+
       expect(incremental_gauge_double).to receive(:set).with({}, 4)
       expect(initial_gauge_double).to receive(:set).with({}, 6)
+      expect(awaiting_indexing_gauge).to receive(:set).with({}, 2)
 
       subject.execute
     end