1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
|
# frozen_string_literal: true
module Gitlab
module Memory
class Watchdog
class Configurator
DEFAULT_PUMA_WORKER_RSS_LIMIT_MB = 1200
DEFAULT_SLEEP_INTERVAL_S = 60
DEFAULT_SIDEKIQ_SLEEP_INTERVAL_S = 3
MIN_SIDEKIQ_SLEEP_INTERVAL_S = 2
DEFAULT_MAX_STRIKES = 5
DEFAULT_MAX_HEAP_FRAG = 0.5
DEFAULT_MAX_MEM_GROWTH = 3.0
# grace_time / sleep_interval = max_strikes allowed for Sidekiq process to violate defined limits.
DEFAULT_SIDEKIQ_GRACE_TIME_S = 300
class << self
def configure_for_puma
->(config) do
config.handler = Gitlab::Memory::Watchdog::PumaHandler.new
config.sleep_time_seconds = ENV.fetch('GITLAB_MEMWD_SLEEP_TIME_SEC', DEFAULT_SLEEP_INTERVAL_S).to_i
config.monitors(&configure_monitors_for_puma)
end
end
def configure_for_sidekiq
->(config) do
config.handler = Gitlab::Memory::Watchdog::TermProcessHandler.new
config.sleep_time_seconds = sidekiq_sleep_time
config.monitors(&configure_monitors_for_sidekiq)
config.event_reporter = SidekiqEventReporter.new
end
end
private
def configure_monitors_for_puma
->(stack) do
max_strikes = ENV.fetch('GITLAB_MEMWD_MAX_STRIKES', DEFAULT_MAX_STRIKES).to_i
if Gitlab::Utils.to_boolean(ENV['DISABLE_PUMA_WORKER_KILLER'])
max_heap_frag = ENV.fetch('GITLAB_MEMWD_MAX_HEAP_FRAG', DEFAULT_MAX_HEAP_FRAG).to_f
max_mem_growth = ENV.fetch('GITLAB_MEMWD_MAX_MEM_GROWTH', DEFAULT_MAX_MEM_GROWTH).to_f
# stack.push MonitorClass, args*, max_strikes:, kwargs**, &block
stack.push Gitlab::Memory::Watchdog::Monitor::HeapFragmentation,
max_heap_fragmentation: max_heap_frag,
max_strikes: max_strikes
stack.push Gitlab::Memory::Watchdog::Monitor::UniqueMemoryGrowth,
max_mem_growth: max_mem_growth,
max_strikes: max_strikes
else
memory_limit = ENV.fetch('PUMA_WORKER_MAX_MEMORY', DEFAULT_PUMA_WORKER_RSS_LIMIT_MB).to_i
stack.push Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit,
memory_limit_bytes: memory_limit.megabytes,
max_strikes: max_strikes
end
end
end
def sidekiq_sleep_time
[
ENV.fetch('SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', DEFAULT_SIDEKIQ_SLEEP_INTERVAL_S).to_i,
MIN_SIDEKIQ_SLEEP_INTERVAL_S
].max
end
def configure_monitors_for_sidekiq
->(stack) do
if ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'].to_i.nonzero?
soft_limit_bytes = ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'].to_i.kilobytes
grace_time = ENV.fetch('SIDEKIQ_MEMORY_KILLER_GRACE_TIME', DEFAULT_SIDEKIQ_GRACE_TIME_S).to_i
max_strikes = grace_time / sidekiq_sleep_time
stack.push Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit,
memory_limit_bytes: soft_limit_bytes,
max_strikes: max_strikes.to_i,
monitor_name: :rss_memory_soft_limit
end
if ENV['SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS'].to_i.nonzero?
hard_limit_bytes = ENV['SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS'].to_i.kilobytes
stack.push Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit,
memory_limit_bytes: hard_limit_bytes,
max_strikes: 0,
monitor_name: :rss_memory_hard_limit
end
end
end
end
end
end
end
end
|
