Commit 70568e0a authored by Valery Sizov's avatar Valery Sizov

Add git push counter to geo node status

On geo secondaries, we'd like to collect and transmit
usage ping data about git operations.
That includs pushes and pulls.

Changelog: changed
EE: true
parent ffe7f77f
...@@ -16260,6 +16260,18 @@ Status: `data_available` ...@@ -16260,6 +16260,18 @@ Status: `data_available`
Tiers: `premium`, `ultimate` Tiers: `premium`, `ultimate`
### `usage_activity_by_stage.enablement.counts.geo_node_usage.git_push_event_count_weekly`
Number of Git push events from Prometheus on the Geo secondary
[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/config/metrics/counts_7d/20210604110603_git_push_event_count_weekly.yml)
Group: `group::geo`
Status: `data_available`
Tiers: `premium`, `ultimate`
### `usage_activity_by_stage.enablement.geo_secondary_web_oauth_users` ### `usage_activity_by_stage.enablement.geo_secondary_web_oauth_users`
Missing description Missing description
......
...@@ -6,6 +6,7 @@ class Geo::SecondaryUsageData < Geo::TrackingBase ...@@ -6,6 +6,7 @@ class Geo::SecondaryUsageData < Geo::TrackingBase
include Gitlab::Utils::UsageData include Gitlab::Utils::UsageData
GIT_FETCH_EVENT_COUNT_WEEKLY_QUERY = 'round(sum(increase(grpc_server_handled_total{grpc_method=~"SSHUploadPack|PostUploadPack"}[7d])))' GIT_FETCH_EVENT_COUNT_WEEKLY_QUERY = 'round(sum(increase(grpc_server_handled_total{grpc_method=~"SSHUploadPack|PostUploadPack"}[7d])))'
GIT_PUSH_EVENT_COUNT_WEEKLY_QUERY = 'round(sum(increase(grpc_server_handled_total{grpc_method=~"SSHReceivePack|PostReceivePack"}[7d])))'
# Eventually, we'll find a way to auto-load this # Eventually, we'll find a way to auto-load this
# from the metric yaml files that include something # from the metric yaml files that include something
...@@ -13,6 +14,7 @@ class Geo::SecondaryUsageData < Geo::TrackingBase ...@@ -13,6 +14,7 @@ class Geo::SecondaryUsageData < Geo::TrackingBase
# just enumerate them. # just enumerate them.
PAYLOAD_COUNT_FIELDS = %w( PAYLOAD_COUNT_FIELDS = %w(
git_fetch_event_count_weekly git_fetch_event_count_weekly
git_push_event_count_weekly
).freeze ).freeze
store_accessor :payload, *PAYLOAD_COUNT_FIELDS store_accessor :payload, *PAYLOAD_COUNT_FIELDS
...@@ -35,8 +37,9 @@ class Geo::SecondaryUsageData < Geo::TrackingBase ...@@ -35,8 +37,9 @@ class Geo::SecondaryUsageData < Geo::TrackingBase
end end
def collect_prometheus_metrics def collect_prometheus_metrics
self.git_fetch_event_count_weekly = with_prometheus_client(fallback: nil, verify: false) do |client| with_prometheus_client(fallback: nil, verify: false) do |client|
client.query(GIT_FETCH_EVENT_COUNT_WEEKLY_QUERY).dig(0, "value", 1)&.to_i self.git_fetch_event_count_weekly = client.query(GIT_FETCH_EVENT_COUNT_WEEKLY_QUERY).dig(0, "value", 1)&.to_i
self.git_push_event_count_weekly = client.query(GIT_PUSH_EVENT_COUNT_WEEKLY_QUERY).dig(0, "value", 1)&.to_i
end end
end end
end end
---
key_path: usage_activity_by_stage.enablement.counts.geo_node_usage.git_push_event_count_weekly
description: Number of git push events from Prometheus on the Geo secondary
product_section: enablement
product_stage: enablement
product_group: group::geo
product_category: geo_replication
value_type: number
status: data_available
milestone: "14.0"
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63427/
time_frame: 7d
data_source: prometheus
distribution:
- ee
tier:
- premium
- ultimate
...@@ -146,6 +146,7 @@ ...@@ -146,6 +146,7 @@
"replication_slots_used_in_percentage", "replication_slots_used_in_percentage",
"replication_slots_max_retained_wal_bytes", "replication_slots_max_retained_wal_bytes",
"git_fetch_event_count_weekly", "git_fetch_event_count_weekly",
"git_push_event_count_weekly",
"last_event_id", "last_event_id",
"last_event_timestamp", "last_event_timestamp",
"cursor_last_event_id", "cursor_last_event_id",
...@@ -311,6 +312,7 @@ ...@@ -311,6 +312,7 @@
"replication_slots_used_in_percentage": { "type": "string" }, "replication_slots_used_in_percentage": { "type": "string" },
"replication_slots_max_retained_wal_bytes": { "type": ["integer", "null"] }, "replication_slots_max_retained_wal_bytes": { "type": ["integer", "null"] },
"git_fetch_event_count_weekly": { "type": ["integer", "null"] }, "git_fetch_event_count_weekly": { "type": ["integer", "null"] },
"git_push_event_count_weekly": { "type": ["integer", "null"] },
"last_event_id": { "type": ["integer", "null"] }, "last_event_id": { "type": ["integer", "null"] },
"last_event_timestamp": { "type": ["integer", "null"] }, "last_event_timestamp": { "type": ["integer", "null"] },
"cursor_last_event_id": { "type": ["integer", "null"] }, "cursor_last_event_id": { "type": ["integer", "null"] },
......
...@@ -50,6 +50,8 @@ RSpec.describe Geo::SecondaryUsageData, :geo, type: :model do ...@@ -50,6 +50,8 @@ RSpec.describe Geo::SecondaryUsageData, :geo, type: :model do
allow_next_instance_of(described_class) do |instance| allow_next_instance_of(described_class) do |instance|
allow(instance).to receive(:with_prometheus_client).and_yield(prometheus_client) allow(instance).to receive(:with_prometheus_client).and_yield(prometheus_client)
end end
allow(prometheus_client).to receive(:query).and_return([])
end end
context 'metric git_fetch_event_count_weekly' do context 'metric git_fetch_event_count_weekly' do
...@@ -66,7 +68,7 @@ RSpec.describe Geo::SecondaryUsageData, :geo, type: :model do ...@@ -66,7 +68,7 @@ RSpec.describe Geo::SecondaryUsageData, :geo, type: :model do
end end
it 'returns nil if metric is unavailable' do it 'returns nil if metric is unavailable' do
expect(prometheus_client).to receive(:query).with(Geo::SecondaryUsageData::GIT_FETCH_EVENT_COUNT_WEEKLY_QUERY).and_return([]) allow(prometheus_client).to receive(:query).with(Geo::SecondaryUsageData::GIT_FETCH_EVENT_COUNT_WEEKLY_QUERY).and_return([])
expect do expect do
described_class.update_metrics! described_class.update_metrics!
...@@ -89,5 +91,43 @@ RSpec.describe Geo::SecondaryUsageData, :geo, type: :model do ...@@ -89,5 +91,43 @@ RSpec.describe Geo::SecondaryUsageData, :geo, type: :model do
expect(described_class.last.git_fetch_event_count_weekly).to be_nil expect(described_class.last.git_fetch_event_count_weekly).to be_nil
end end
end end
context 'metric git_push_event_count_weekly' do
it 'gets metrics from prometheus' do
expected_result = 48
allow(prometheus_client).to receive(:query).with(Geo::SecondaryUsageData::GIT_PUSH_EVENT_COUNT_WEEKLY_QUERY).and_return([{ "value" => [1614029769.82, expected_result.to_s] }])
expect do
described_class.update_metrics!
end.to change { described_class.count }.by(1)
expect(described_class.last).to be_valid
expect(described_class.last.git_push_event_count_weekly).to eq(expected_result)
end
it 'returns nil if metric is unavailable' do
allow(prometheus_client).to receive(:query).with(Geo::SecondaryUsageData::GIT_PUSH_EVENT_COUNT_WEEKLY_QUERY).and_return([])
expect do
described_class.update_metrics!
end.to change { described_class.count }.by(1)
expect(described_class.last).to be_valid
expect(described_class.last.git_push_event_count_weekly).to be_nil
end
it 'returns nil if it cannot reach prometheus' do
expect_next_instance_of(described_class) do |instance|
expect(instance).to receive(:with_prometheus_client).and_return(nil)
end
expect do
described_class.update_metrics!
end.to change { described_class.count }.by(1)
expect(described_class.last).to be_valid
expect(described_class.last.git_push_event_count_weekly).to be_nil
end
end
end end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment