Merge branch '298781/collect_data_on_git_fetch_requests_from_prometheus' into 'master'

Collect git fetch data

See merge request gitlab-org/gitlab!53888

doc/development/usage_ping/dictionary.md

@@ -1028,6 +1028,18 @@ Status: `data_available`
 
 Tiers: `premium`, `ultimate`
 
+### `counts.geo_node_usage.git_fetch_event_count_weekly`
+
+Number of Git fetch events from Prometheus on the Geo secondary
+
+[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/config/metrics/counts_7d/20210309194425_git_fetch_event_count_weekly.yml)
+
+Group: `group::geo`
+
+Status: `implemented`
+
+Tiers: `premium`, `ultimate`
+
 ### `counts.geo_nodes`
 
 Total number of sites in a Geo deployment

ee/app/models/geo/secondary_usage_data.rb

@@ -3,12 +3,16 @@
 # This class is used to store usage data on a secondary for transmission
 # to the primary during a status update.
 class Geo::SecondaryUsageData < Geo::TrackingBase
+  include Gitlab::Utils::UsageData
+
+  GIT_FETCH_EVENT_COUNT_WEEKLY_QUERY = 'round(sum(increase(grpc_server_handled_total{grpc_method=~"SSHUploadPack|PostUploadPack"}[7d])))'.freeze
+
   # Eventually, we'll find a way to auto-load this
   # from the metric yaml files that include something
   # like `run_on_secondary: true`, but for now we'll
   # just enumerate them.
   PAYLOAD_COUNT_FIELDS = %w(
-    git_fetch_event_count
+    git_fetch_event_count_weekly
   ).freeze
 
   store_accessor :payload, *PAYLOAD_COUNT_FIELDS
@@ -25,6 +29,14 @@ class Geo::SecondaryUsageData < Geo::TrackingBase
   end
 
   def self.update_metrics!
-    # This should go through and collect metrics
+    usage_data = new
+    usage_data.collect_prometheus_metrics
+    usage_data.save!
+  end
+
+  def collect_prometheus_metrics
+    self.git_fetch_event_count_weekly = with_prometheus_client(fallback: nil, verify: false) do |client|
+      client.query(GIT_FETCH_EVENT_COUNT_WEEKLY_QUERY).dig(0, "value", 1)&.to_i
+    end
   end
 end

ee/config/metrics/counts_7d/20210309194425_git_fetch_event_count_weekly.yml

+---
+key_path: counts.geo_node_usage.git_fetch_event_count_weekly
+description: Number of Git fetch events from Prometheus on the Geo secondary
+product_section: enablement
+product_stage: enablement
+product_group: group::geo
+product_category: disaster_recovery
+value_type: number
+status: implemented
+milestone: "13.10"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53888
+time_frame: 7d
+data_source: prometheus
+distribution:
+  - ee
+tier:
+ - premium
+ - ultimate

ee/spec/models/geo/secondary_usage_data_spec.rb

@@ -5,6 +5,8 @@ require 'spec_helper'
 RSpec.describe Geo::SecondaryUsageData, :geo, type: :model do
   subject { create(:geo_secondary_usage_data) }
 
+  let(:prometheus_client) { Gitlab::PrometheusClient.new('http://localhost:9090') }
+
   it 'is valid' do
     expect(subject).to be_valid
   end
@@ -40,4 +42,52 @@ RSpec.describe Geo::SecondaryUsageData, :geo, type: :model do
       it_behaves_like 'a payload count field', field
     end
   end
+
+  describe '#update_metrics!' do
+    let(:new_data) { double(Geo::SecondaryUsageData) }
+
+    before do
+      allow_next_instance_of(described_class) do |instance|
+        allow(instance).to receive(:with_prometheus_client).and_yield(prometheus_client)
+      end
+    end
+
+    context 'metric git_fetch_event_count_weekly' do
+      it 'gets metrics from prometheus' do
+        expected_result = 48
+        allow(prometheus_client).to receive(:query).with(Geo::SecondaryUsageData::GIT_FETCH_EVENT_COUNT_WEEKLY_QUERY).and_return([{ "value" => [1614029769.82, expected_result.to_s] }])
+
+        expect do
+          described_class.update_metrics!
+        end.to change { described_class.count }.by(1)
+
+        expect(described_class.last).to be_valid
+        expect(described_class.last.git_fetch_event_count_weekly).to eq(expected_result)
+      end
+
+      it 'returns nil if metric is unavailable' do
+        expect(prometheus_client).to receive(:query).with(Geo::SecondaryUsageData::GIT_FETCH_EVENT_COUNT_WEEKLY_QUERY).and_return([])
+
+        expect do
+          described_class.update_metrics!
+        end.to change { described_class.count }.by(1)
+
+        expect(described_class.last).to be_valid
+        expect(described_class.last.git_fetch_event_count_weekly).to be_nil
+      end
+
+      it 'returns nil if it cannot reach prometheus' do
+        expect_next_instance_of(described_class) do |instance|
+          expect(instance).to receive(:with_prometheus_client).and_return(nil)
+        end
+
+        expect do
+          described_class.update_metrics!
+        end.to change { described_class.count }.by(1)
+
+        expect(described_class.last).to be_valid
+        expect(described_class.last.git_fetch_event_count_weekly).to be_nil
+      end
+    end
+  end
 end