1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
|
# frozen_string_literal: true
module Gitlab
module Database
module HealthStatus
module Indicators
class PrometheusAlertIndicator
include Gitlab::Utils::StrongMemoize
ALERT_CONDITIONS = {
above: :above,
below: :below
}.freeze
def initialize(context)
@gitlab_schema = context.gitlab_schema.to_sym
end
def evaluate
return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled?
connection_error_message = fetch_connection_error_message
return unknown_signal(connection_error_message) if connection_error_message.present?
sli = fetch_sli(sli_query)
return unknown_signal("#{indicator_name} can not be calculated") unless sli.present?
if alert_condition == ALERT_CONDITIONS[:above] ? sli.to_f > slo.to_f : sli.to_f < slo.to_f
Signals::Normal.new(self.class, reason: "#{indicator_name} SLI condition met")
else
Signals::Stop.new(self.class, reason: "#{indicator_name} SLI condition not met")
end
end
private
attr_reader :gitlab_schema
def indicator_name
self.class.name.demodulize
end
# By default SLIs are expected to be above SLOs, but there can be cases
# where we want it to be below SLO (eg: WAL rate). For such indicators
# the sub-class should override this default alert_condition.
def alert_condition
ALERT_CONDITIONS[:above]
end
def enabled?
raise NotImplementedError, "prometheus alert based indicators must implement #{__method__}"
end
def slo_key
raise NotImplementedError, "prometheus alert based indicators must implement #{__method__}"
end
def sli_key
raise NotImplementedError, "prometheus alert based indicators must implement #{__method__}"
end
def fetch_connection_error_message
return 'Prometheus Settings not configured' unless prometheus_alert_db_indicators_settings.present?
return 'Prometheus client is not ready' unless client.ready?
return "#{indicator_name} SLI query is not configured" unless sli_query
return "#{indicator_name} SLO is not configured" unless slo
end
def prometheus_alert_db_indicators_settings
@prometheus_alert_db_indicators_settings ||= Gitlab::CurrentSettings
.prometheus_alert_db_indicators_settings&.with_indifferent_access
end
def client
@client ||= Gitlab::PrometheusClient.new(
prometheus_alert_db_indicators_settings[:prometheus_api_url],
allow_local_requests: true,
verify: true
)
end
def sli_query
{
gitlab_main: prometheus_alert_db_indicators_settings[sli_query_key][:main],
gitlab_ci: prometheus_alert_db_indicators_settings[sli_query_key][:ci]
}.fetch(gitlab_schema)
end
strong_memoize_attr :sli_query
def slo
{
gitlab_main: prometheus_alert_db_indicators_settings[slo_key][:main],
gitlab_ci: prometheus_alert_db_indicators_settings[slo_key][:ci]
}.fetch(gitlab_schema)
end
strong_memoize_attr :slo
def fetch_sli(query)
response = client.query(query)
metric = response&.first || {}
value = metric.fetch('value', [])
Array.wrap(value).second
end
def unknown_signal(reason)
Signals::Unknown.new(self.class, reason: reason)
end
end
end
end
end
end
|
