Merge branch 'am-add-new-output-support-to-service-ping-report' into 'master'

Add queries and non_queries to ServicePingReport

See merge request gitlab-org/gitlab!80078

app/controllers/admin/application_settings_controller.rb

@@ -62,11 +62,11 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
   def usage_data
     respond_to do |format|
       format.html do
-        usage_data_json = Gitlab::Json.pretty_generate(Gitlab::Usage::ServicePingReport.for(mode: :values, cached: true))
+        usage_data_json = Gitlab::Json.pretty_generate(Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values, cached: true))
 
         render html: Gitlab::Highlight.highlight('payload.json', usage_data_json, language: 'json')
       end
-      format.json { render json: Gitlab::Usage::ServicePingReport.for(mode: :values, cached: true).to_json }
+      format.json { render json: Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values, cached: true).to_json }
     end
   end
 

app/controllers/admin/instance_review_controller.rb

@@ -16,7 +16,7 @@ class Admin::InstanceReviewController < Admin::ApplicationController
     }
 
     if Gitlab::CurrentSettings.usage_ping_enabled?
-      data = Gitlab::Usage::ServicePingReport.for(mode: :values, cached: true)
+      data = Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values, cached: true)
       counts = data[:counts]
 
       result[:instance_review].merge!(

app/services/service_ping/build_payload_service.rb

@@ -19,7 +19,7 @@ module ServicePing
     end
 
     def raw_payload
-      @raw_payload ||= ::Gitlab::Usage::ServicePingReport.for(mode: :values)
+      @raw_payload ||= ::Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values)
     end
   end
 end

app/services/service_ping/submit_service.rb

@@ -33,7 +33,7 @@ module ServicePing
         }
         submit_payload({ error: error_payload }, url: error_url)
 
-        usage_data = Gitlab::Usage::ServicePingReport.for(mode: :values)
+        usage_data = Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values)
         response = submit_usage_data_payload(usage_data)
       end
 

doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md

@@ -1324,7 +1324,7 @@ has more information about Service Ping.
 ### Generate or get the cached Service Ping
 
 ```ruby
-Gitlab::Usage::ServicePingReport.for(mode: :values, cached: true)
+Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values, cached: true)
 ```
 
 ### Generate a fresh new Service Ping
@@ -1332,7 +1332,7 @@ Gitlab::Usage::ServicePingReport.for(mode: :values, cached: true)
 This also refreshes the cached Service Ping displayed in the Admin Area
 
 ```ruby
-Gitlab::Usage::ServicePingReport.for(mode: :values)
+Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values)
 ```
 
 ### Generate and print

doc/development/service_ping/index.md

@@ -198,8 +198,8 @@ sequenceDiagram
 ## How Service Ping works
 
 1. The Service Ping [cron job](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/gitlab_service_ping_worker.rb#L24) is set in Sidekiq to run weekly.
-1. When the cron job runs, it calls [`Gitlab::Usage::ServicePingReport.for(mode: :values)`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/service_ping/submit_service.rb).
-1. `Gitlab::Usage::ServicePingReport.for(mode: :values)` [cascades down](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb) to ~400+ other counter method calls.
+1. When the cron job runs, it calls [`Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values)`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/service_ping/submit_service.rb).
+1. `Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values)` [cascades down](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb) to ~400+ other counter method calls.
 1. The response of all methods calls are [merged together](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb#L68) into a single JSON payload.
 1. The JSON payload is then [posted to the Versions application](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/service_ping/submit_service.rb#L20)
    If a firewall exception is needed, the required URL depends on several things. If

ee/spec/config/metrics/every_metric_definition_spec.rb

@@ -5,7 +5,7 @@ require 'spec_helper'
 RSpec.describe 'Every metric definition' do
   include UsageDataHelpers
 
-  let(:usage_ping) { Gitlab::Usage::ServicePingReport.for(mode: :values) }
+  let(:usage_ping) { Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values) }
   let(:ignored_usage_ping_key_patterns) do
     %w(
       license_add_ons

lib/api/usage_data_non_sql_metrics.rb

@@ -18,7 +18,7 @@ module API
       get 'non_sql_metrics' do
         Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/325534')
 
-        data = Gitlab::UsageDataNonSqlMetrics.data
+        data = Gitlab::Usage::ServicePingReport.for(output: :non_sql_metrics_values)
 
         present data
       end

lib/api/usage_data_queries.rb

@@ -18,7 +18,7 @@ module API
       get 'queries' do
         Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/325534')
 
-        queries = Gitlab::UsageDataQueries.data
+        queries = Gitlab::Usage::ServicePingReport.for(output: :metrics_queries)
 
         present queries
       end

lib/gitlab/usage/service_ping_report.rb

@@ -4,20 +4,32 @@ module Gitlab
   module Usage
     class ServicePingReport
       class << self
-        def for(mode:, cached: false)
-          case mode.to_sym
-          when :values
-            usage_data(cached)
+        def for(output:, cached: false)
+          case output.to_sym
+          when :all_metrics_values
+            all_metrics_values(cached)
+          when :metrics_queries
+            metrics_queries
+          when :non_sql_metrics_values
+            non_sql_metrics_values
           end
         end
 
         private
 
-        def usage_data(cached)
+        def all_metrics_values(cached)
           Rails.cache.fetch('usage_data', force: !cached, expires_in: 2.weeks) do
             Gitlab::UsageData.data
           end
         end
+
+        def metrics_queries
+          Gitlab::UsageDataQueries.data
+        end
+
+        def non_sql_metrics_values
+          Gitlab::UsageDataNonSqlMetrics.data
+        end
       end
     end
   end

lib/tasks/gitlab/usage_data.rake

@@ -4,17 +4,17 @@ namespace :gitlab do
   namespace :usage_data do
     desc 'GitLab | UsageData | Generate raw SQLs for usage ping in YAML'
     task dump_sql_in_yaml: :environment do
-      puts Gitlab::UsageDataQueries.data.to_yaml
+      puts Gitlab::Usage::ServicePingReport.for(output: :metrics_queries).to_yaml
     end
 
     desc 'GitLab | UsageData | Generate raw SQLs for usage ping in JSON'
     task dump_sql_in_json: :environment do
-      puts Gitlab::Json.pretty_generate(Gitlab::UsageDataQueries.data)
+      puts Gitlab::Json.pretty_generate(Gitlab::Usage::ServicePingReport.for(output: :metrics_queries))
     end
 
     desc 'GitLab | UsageData | Generate usage ping in JSON'
     task generate: :environment do
-      puts Gitlab::Json.pretty_generate(Gitlab::Usage::ServicePingReport.for(mode: :values))
+      puts Gitlab::Json.pretty_generate(Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values))
     end
 
     desc 'GitLab | UsageData | Generate usage ping and send it to Versions Application'

spec/controllers/admin/instance_review_controller_spec.rb

@@ -23,7 +23,7 @@ RSpec.describe Admin::InstanceReviewController do
         stub_application_setting(usage_ping_enabled: true)
         stub_usage_data_connections
         stub_database_flavor_check
-        ::Gitlab::Usage::ServicePingReport.for(mode: :values)
+        ::Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values)
         subject
       end
 

spec/lib/gitlab/usage/service_ping_report_spec.rb

@@ -5,11 +5,27 @@ require 'spec_helper'
 RSpec.describe Gitlab::Usage::ServicePingReport, :use_clean_rails_memory_store_caching do
   let(:usage_data) { { uuid: "1111" } }
 
-  context 'for mode: :values' do
+  context 'for output: :all_metrics_values' do
     it 'generates the service ping' do
       expect(Gitlab::UsageData).to receive(:data)
 
-      described_class.for(mode: :values)
+      described_class.for(output: :all_metrics_values)
+    end
+  end
+
+  context 'for output: :metrics_queries' do
+    it 'generates the service ping' do
+      expect(Gitlab::UsageDataQueries).to receive(:data)
+
+      described_class.for(output: :metrics_queries)
+    end
+  end
+
+  context 'for output: :non_sql_metrics_values' do
+    it 'generates the service ping' do
+      expect(Gitlab::UsageDataNonSqlMetrics).to receive(:data)
+
+      described_class.for(output: :non_sql_metrics_values)
     end
   end
 
@@ -20,8 +36,8 @@ RSpec.describe Gitlab::Usage::ServicePingReport, :use_clean_rails_memory_store_c
       it 'caches the values' do
         allow(Gitlab::UsageData).to receive(:data).and_return(usage_data, new_usage_data)
 
-        expect(described_class.for(mode: :values)).to eq(usage_data)
-        expect(described_class.for(mode: :values, cached: true)).to eq(usage_data)
+        expect(described_class.for(output: :all_metrics_values)).to eq(usage_data)
+        expect(described_class.for(output: :all_metrics_values, cached: true)).to eq(usage_data)
 
         expect(Rails.cache.fetch('usage_data')).to eq(usage_data)
       end
@@ -29,9 +45,9 @@ RSpec.describe Gitlab::Usage::ServicePingReport, :use_clean_rails_memory_store_c
       it 'writes to cache and returns fresh data' do
         allow(Gitlab::UsageData).to receive(:data).and_return(usage_data, new_usage_data)
 
-        expect(described_class.for(mode: :values)).to eq(usage_data)
-        expect(described_class.for(mode: :values)).to eq(new_usage_data)
-        expect(described_class.for(mode: :values, cached: true)).to eq(new_usage_data)
+        expect(described_class.for(output: :all_metrics_values)).to eq(usage_data)
+        expect(described_class.for(output: :all_metrics_values)).to eq(new_usage_data)
+        expect(described_class.for(output: :all_metrics_values, cached: true)).to eq(new_usage_data)
 
         expect(Rails.cache.fetch('usage_data')).to eq(new_usage_data)
       end
@@ -43,8 +59,8 @@ RSpec.describe Gitlab::Usage::ServicePingReport, :use_clean_rails_memory_store_c
       it 'returns fresh data' do
         allow(Gitlab::UsageData).to receive(:data).and_return(usage_data, new_usage_data)
 
-        expect(described_class.for(mode: :values)).to eq(usage_data)
-        expect(described_class.for(mode: :values)).to eq(new_usage_data)
+        expect(described_class.for(output: :all_metrics_values)).to eq(usage_data)
+        expect(described_class.for(output: :all_metrics_values)).to eq(new_usage_data)
 
         expect(Rails.cache.fetch('usage_data')).to eq(new_usage_data)
       end

spec/services/service_ping/submit_service_ping_service_spec.rb

@@ -118,7 +118,7 @@ RSpec.describe ServicePing::SubmitService do
     it 'generates service ping' do
       stub_response(body: with_dev_ops_score_params)
 
-      expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(mode: :values).and_call_original
+      expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(output: :all_metrics_values).and_call_original
 
       subject.execute
     end
@@ -151,7 +151,7 @@ RSpec.describe ServicePing::SubmitService do
     it 'forces a refresh of usage data statistics before submitting' do
       stub_response(body: with_dev_ops_score_params)
 
-      expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(mode: :values).and_call_original
+      expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(output: :all_metrics_values).and_call_original
 
       subject.execute
     end
@@ -167,7 +167,7 @@ RSpec.describe ServicePing::SubmitService do
         recorded_at = Time.current
         usage_data = { uuid: 'uuid', recorded_at: recorded_at }
 
-        expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(mode: :values).and_return(usage_data)
+        expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(output: :all_metrics_values).and_return(usage_data)
 
         subject.execute
 
@@ -190,7 +190,7 @@ RSpec.describe ServicePing::SubmitService do
         recorded_at = Time.current
         usage_data = { uuid: 'uuid', recorded_at: recorded_at }
 
-        expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(mode: :values).and_return(usage_data)
+        expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(output: :all_metrics_values).and_return(usage_data)
 
         subject.execute
 
@@ -235,7 +235,7 @@ RSpec.describe ServicePing::SubmitService do
         recorded_at = Time.current
         usage_data = { uuid: 'uuid', recorded_at: recorded_at }
 
-        expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(mode: :values).and_return(usage_data)
+        expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(output: :all_metrics_values).and_return(usage_data)
 
         subject.execute
 
@@ -260,7 +260,7 @@ RSpec.describe ServicePing::SubmitService do
 
     context 'and usage data is empty string' do
       before do
-        allow(Gitlab::Usage::ServicePingReport).to receive(:for).with(mode: :values).and_return({})
+        allow(Gitlab::Usage::ServicePingReport).to receive(:for).with(output: :all_metrics_values).and_return({})
       end
 
       it_behaves_like 'does not send a blank usage ping payload'
@@ -269,7 +269,7 @@ RSpec.describe ServicePing::SubmitService do
     context 'and usage data is nil' do
       before do
         allow(ServicePing::BuildPayloadService).to receive(:execute).and_return(nil)
-        allow(Gitlab::Usage::ServicePingReport).to receive(:for).with(mode: :values).and_return(nil)
+        allow(Gitlab::Usage::ServicePingReport).to receive(:for).with(output: :all_metrics_values).and_return(nil)
       end
 
       it_behaves_like 'does not send a blank usage ping payload'
@@ -285,7 +285,7 @@ RSpec.describe ServicePing::SubmitService do
       it 'calls Gitlab::Usage::ServicePingReport .for method' do
         usage_data = build_usage_data
 
-        expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(mode: :values).and_return(usage_data)
+        expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(output: :all_metrics_values).and_return(usage_data)
 
         subject.execute
       end
@@ -329,7 +329,7 @@ RSpec.describe ServicePing::SubmitService do
       it 'calls Gitlab::Usage::ServicePingReport .for method' do
         usage_data = build_usage_data
 
-        expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(mode: :values).and_return(usage_data)
+        expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(output: :all_metrics_values).and_return(usage_data)
 
         # SubmissionError is raised as a result of 404 in response from HTTP Request
         expect { subject.execute }.to raise_error(described_class::SubmissionError)