1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
|
# frozen_string_literal: true
module WebHooks
module AutoDisabling
extend ActiveSupport::Concern
include ::Gitlab::Loggable
ENABLED_HOOK_TYPES = %w[ProjectHook].freeze
MAX_FAILURES = 100
FAILURE_THRESHOLD = 3
EXCEEDED_FAILURE_THRESHOLD = FAILURE_THRESHOLD + 1
INITIAL_BACKOFF = 1.minute.freeze
MAX_BACKOFF = 1.day.freeze
BACKOFF_GROWTH_FACTOR = 2.0
class_methods do
def auto_disabling_enabled?
enabled_hook_types.include?(name) &&
Gitlab::SafeRequestStore.fetch(:auto_disabling_web_hooks) do
Feature.enabled?(:auto_disabling_web_hooks, type: :ops)
end
end
private
def enabled_hook_types
ENABLED_HOOK_TYPES
end
end
included do
delegate :auto_disabling_enabled?, to: :class, private: true
# A hook is disabled if:
#
# - we are no longer in the grace-perod (recent_failures > ?)
# - and either:
# - disabled_until is nil (i.e. this was set by WebHook#fail!)
# - or disabled_until is in the future (i.e. this was set by WebHook#backoff!)
# - OR silent mode is enabled.
scope :disabled, -> do
return all if Gitlab::SilentMode.enabled?
return none unless auto_disabling_enabled?
where(
'recent_failures > ? AND (disabled_until IS NULL OR disabled_until >= ?)',
FAILURE_THRESHOLD,
Time.current
)
end
# A hook is executable if:
#
# - we are still in the grace-period (recent_failures <= ?)
# - OR we have exceeded the grace period and neither of the following is true:
# - disabled_until is nil (i.e. this was set by WebHook#fail!)
# - disabled_until is in the future (i.e. this was set by WebHook#backoff!)
# - AND silent mode is not enabled.
scope :executable, -> do
return none if Gitlab::SilentMode.enabled?
return all unless auto_disabling_enabled?
where(
'recent_failures <= ? OR (recent_failures > ? AND (disabled_until IS NOT NULL) AND (disabled_until < ?))',
FAILURE_THRESHOLD,
FAILURE_THRESHOLD,
Time.current
)
end
end
def executable?
return true unless auto_disabling_enabled?
!temporarily_disabled? && !permanently_disabled?
end
def temporarily_disabled?
return false unless auto_disabling_enabled?
disabled_until.present? && disabled_until >= Time.current && recent_failures > FAILURE_THRESHOLD
end
def permanently_disabled?
return false unless auto_disabling_enabled?
recent_failures > FAILURE_THRESHOLD && disabled_until.blank?
end
def enable!
return unless auto_disabling_enabled?
return if recent_failures == 0 && disabled_until.nil? && backoff_count == 0
attrs = { recent_failures: 0, disabled_until: nil, backoff_count: 0 }
assign_attributes(attrs)
logger.info(hook_id: id, action: 'enable', **attrs)
save(validate: false)
end
# Don't actually back-off until FAILURE_THRESHOLD failures have been seen
# we mark the grace-period using the recent_failures counter
def backoff!
return unless auto_disabling_enabled?
return if permanently_disabled? || (backoff_count >= MAX_FAILURES && temporarily_disabled?)
attrs = { recent_failures: next_failure_count }
if recent_failures >= FAILURE_THRESHOLD
attrs[:backoff_count] = next_backoff_count
attrs[:disabled_until] = next_backoff.from_now
end
assign_attributes(attrs)
return unless changed?
logger.info(hook_id: id, action: 'backoff', **attrs)
save(validate: false)
end
def failed!
return unless auto_disabling_enabled?
return unless recent_failures < MAX_FAILURES
attrs = { disabled_until: nil, backoff_count: 0, recent_failures: next_failure_count }
assign_attributes(**attrs)
logger.info(hook_id: id, action: 'disable', **attrs)
save(validate: false)
end
def next_backoff
return MAX_BACKOFF if backoff_count >= 8 # optimization to prevent expensive exponentiation and possible overflows
(INITIAL_BACKOFF * (BACKOFF_GROWTH_FACTOR**backoff_count))
.clamp(INITIAL_BACKOFF, MAX_BACKOFF)
.seconds
end
def alert_status
return :executable unless auto_disabling_enabled?
if temporarily_disabled?
:temporarily_disabled
elsif permanently_disabled?
:disabled
else
:executable
end
end
private
def logger
@logger ||= Gitlab::WebHooks::Logger.build
end
def next_failure_count
recent_failures.succ.clamp(1, MAX_FAILURES)
end
def next_backoff_count
backoff_count.succ.clamp(1, MAX_FAILURES)
end
end
end
WebHooks::AutoDisabling.prepend_mod
WebHooks::AutoDisabling::ClassMethods.prepend_mod
|
