1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
module Geo
class MetricsUpdateService
METRIC_PREFIX = 'geo_'.freeze
def execute
return unless Gitlab::Geo.enabled?
if Gitlab::Geo.primary?
fetch_secondary_geo_nodes_metrics
else
fetch_current_geo_node_metrics
end
end
private
def fetch_secondary_geo_nodes_metrics
Gitlab::Geo.secondary_nodes.find_each { |node| fetch_geo_node_metrics(node) }
end
def fetch_current_geo_node_metrics
fetch_geo_node_metrics(Gitlab::Geo.current_node)
end
def fetch_geo_node_metrics(node)
return unless node.enabled?
return unless Gitlab::Geo.primary? || Gitlab::Metrics.prometheus_metrics_enabled?
status = node_status(node)
unless status.success
increment_failed_status_counter(node)
return
end
update_db_metrics(node, status) if Gitlab::Geo.primary?
update_prometheus_metrics(node, status) if Gitlab::Metrics.prometheus_metrics_enabled?
end
def update_db_metrics(node, status)
db_status = node.find_or_build_status
db_status.update_attributes(status.attributes.compact.merge(last_successful_status_check_at: Time.now.utc))
end
def update_prometheus_metrics(node, status)
GeoNodeStatus::PROMETHEUS_METRICS.each do |column, docstring|
value = status[column]
next unless value.is_a?(Integer)
gauge = Gitlab::Metrics.gauge(gauge_metric_name(column), docstring, {}, :max)
gauge.set(metric_labels(node), value)
end
end
def node_status(node)
NodeStatusFetchService.new.call(node)
end
def increment_failed_status_counter(node)
failed_status_counter(node).increment
end
def failed_status_counter(node)
Gitlab::Metrics.counter(
:geo_status_failed_total,
'Total number of times status for Geo node failed to retrieve',
metric_labels(node))
end
def gauge_metric_name(name)
# Prometheus naming conventions in
# https://prometheus.io/docs/instrumenting/writing_exporters/#naming says
# that _count and _total should be reserved for counters
base_name = name.to_s.gsub(/(_count|_total)$/, '')
(METRIC_PREFIX + base_name).to_sym
end
def metric_labels(node)
{ url: node.url }
end
end
end