Add metrics for paused indexing Sidekiq queue size

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -173,6 +173,7 @@ configuration option in `gitlab.yml`. These metrics are served from the
 | `geo_repositories_retrying_verification_count` | Gauge   | 11.2  | Number of repositories verification failures that Geo is actively trying to correct on secondary  | `url` |
 | `geo_wikis_retrying_verification_count`        | Gauge   | 11.2  | Number of wikis verification failures that Geo is actively trying to correct on secondary | `url` |
 | `global_search_bulk_cron_queue_size`           | Gauge   | 12.10 | Number of database records waiting to be synchronized to Elasticsearch | |
+| `global_search_awaiting_indexing_queue_size`   | Gauge   | 13.2  | Number of database updates waiting to be synchronized to Elasticsearch while indexing is paused | |
 | `package_files_count`                          | Gauge   | 13.0  | Number of package files on primary | `url` |
 | `package_files_checksummed_count`              | Gauge   | 13.0  | Number of package files checksummed on primary | `url` |
 | `package_files_checksum_failed_count`          | Gauge   | 13.0  | Number of package files failed to calculate the checksum on primary

ee/changelogs/unreleased/217918-follow-up-from-implement-a-switch-to-pause-all-of-the-write-req.yml

+---
+title: Add Prometheus Metrics for Index Pause Sidekiq Queue.
+merge_request: 35871
+author:
+type: changed

ee/lib/elastic/metrics_update_service.rb

@@ -11,6 +11,9 @@ module Elastic
 
       initial_gauge = Gitlab::Metrics.gauge(:global_search_bulk_cron_initial_queue_size, 'Number of initial database updates waiting to be synchronized to Elasticsearch', {}, :max)
       initial_gauge.set({}, Elastic::ProcessInitialBookkeepingService.queue_size)
+
+      awaiting_indexing_gauge = Gitlab::Metrics.gauge(:global_search_awaiting_indexing_queue_size, 'Number of database updates waiting to be synchronized to Elasticsearch while indexing is paused.', {}, :max)
+      awaiting_indexing_gauge.set({}, Elastic::IndexingControlService.queue_size)
     end
 
     private

ee/spec/services/elastic/metrics_update_service_spec.rb

@@ -14,6 +14,7 @@ RSpec.describe Elastic::MetricsUpdateService, :prometheus do
     it 'sets gauges' do
       expect(Elastic::ProcessBookkeepingService).to receive(:queue_size).and_return(4)
       expect(Elastic::ProcessInitialBookkeepingService).to receive(:queue_size).and_return(6)
+      expect(Elastic::IndexingControlService).to receive(:queue_size).and_return(2)
 
       incremental_gauge_double = instance_double(Prometheus::Client::Gauge)
       expect(Gitlab::Metrics).to receive(:gauge)
@@ -25,8 +26,14 @@ RSpec.describe Elastic::MetricsUpdateService, :prometheus do
         .with(:global_search_bulk_cron_initial_queue_size, anything, {}, :max)
         .and_return(initial_gauge_double)
 
+      awaiting_indexing_gauge = instance_double(Prometheus::Client::Gauge)
+      expect(Gitlab::Metrics).to receive(:gauge)
+        .with(:global_search_awaiting_indexing_queue_size, anything, {}, :max)
+        .and_return(awaiting_indexing_gauge)
+
       expect(incremental_gauge_double).to receive(:set).with({}, 4)
       expect(initial_gauge_double).to receive(:set).with({}, 6)
+      expect(awaiting_indexing_gauge).to receive(:set).with({}, 2)
 
       subject.execute
     end