Add latest changes from gitlab-org/gitlab@master

5 files changed, 67 insertions, 46 deletions

diff --git a/lib/gitlab/memory/watchdog/configuration.rb b/lib/gitlab/memory/watchdog/configuration.rb
index 4bad9475531..885772d6119 100644
--- a/lib/gitlab/memory/watchdog/configuration.rb
+++ b/lib/gitlab/memory/watchdog/configuration.rb
@@ -10,7 +10,6 @@ module Gitlab
           end
 
           def push(monitor_class, *args, **kwargs, &block)
-            remove(monitor_class)
             @monitors.push(build_monitor_state(monitor_class, *args, **kwargs, &block))
           end
 
@@ -22,14 +21,11 @@ module Gitlab
 
           private
 
-          def remove(monitor_class)
-            @monitors.delete_if { |monitor| monitor.monitor_class == monitor_class }
-          end
-
-          def build_monitor_state(monitor_class, *args, max_strikes:, **kwargs, &block)
+          def build_monitor_state(monitor_class, *args, max_strikes:, monitor_name: nil, **kwargs, &block)
             monitor = build_monitor(monitor_class, *args, **kwargs, &block)
+            monitor_name ||= monitor_class.name.demodulize.underscore
 
-            Gitlab::Memory::Watchdog::MonitorState.new(monitor, max_strikes: max_strikes)
+            Gitlab::Memory::Watchdog::MonitorState.new(monitor, max_strikes: max_strikes, monitor_name: monitor_name)
           end
 
           def build_monitor(monitor_class, *args, **kwargs, &block)
diff --git a/lib/gitlab/memory/watchdog/configurator.rb b/lib/gitlab/memory/watchdog/configurator.rb
index b6917873e5a..610d8ca9e97 100644
--- a/lib/gitlab/memory/watchdog/configurator.rb
+++ b/lib/gitlab/memory/watchdog/configurator.rb
@@ -4,25 +4,33 @@ module Gitlab
   module Memory
     class Watchdog
       class Configurator
+        DEFAULT_PUMA_WORKER_RSS_LIMIT_MB = 1200
+        DEFAULT_SLEEP_INTERVAL_S = 60
+        DEFAULT_SIDEKIQ_SLEEP_INTERVAL_S = 3
+        MIN_SIDEKIQ_SLEEP_INTERVAL_S = 2
+        DEFAULT_MAX_STRIKES = 5
+        DEFAULT_MAX_HEAP_FRAG = 0.5
+        DEFAULT_MAX_MEM_GROWTH = 3.0
+        # grace_time / sleep_interval = max_strikes allowed for Sidekiq process to violate defined limits.
+        DEFAULT_SIDEKIQ_GRACE_TIME_S = 300
+
         class << self
           def configure_for_puma
-            lambda do |config|
+            ->(config) do
               config.logger = Gitlab::AppLogger
               config.handler = Gitlab::Memory::Watchdog::PumaHandler.new
               config.write_heap_dumps = write_heap_dumps?
-              config.sleep_time_seconds = ENV.fetch('GITLAB_MEMWD_SLEEP_TIME_SEC', 60).to_i
+              config.sleep_time_seconds = ENV.fetch('GITLAB_MEMWD_SLEEP_TIME_SEC', DEFAULT_SLEEP_INTERVAL_S).to_i
               config.monitors(&configure_monitors_for_puma)
             end
           end
 
           def configure_for_sidekiq
-            lambda do |config|
+            ->(config) do
               config.logger = Sidekiq.logger
               config.handler = Gitlab::Memory::Watchdog::TermProcessHandler.new
               config.write_heap_dumps = write_heap_dumps?
-              config.sleep_time_seconds = [
-                ENV.fetch('SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', 3).to_i, 2
-              ].max
+              config.sleep_time_seconds = sidekiq_sleep_time
               config.monitors(&configure_monitors_for_sidekiq)
             end
           end
@@ -34,12 +42,12 @@ module Gitlab
           end
 
           def configure_monitors_for_puma
-            lambda do |stack|
-              max_strikes = ENV.fetch('GITLAB_MEMWD_MAX_STRIKES', 5).to_i
+            ->(stack) do
+              max_strikes = ENV.fetch('GITLAB_MEMWD_MAX_STRIKES', DEFAULT_MAX_STRIKES).to_i
 
               if Gitlab::Utils.to_boolean(ENV['DISABLE_PUMA_WORKER_KILLER'])
-                max_heap_frag = ENV.fetch('GITLAB_MEMWD_MAX_HEAP_FRAG', 0.5).to_f
-                max_mem_growth = ENV.fetch('GITLAB_MEMWD_MAX_MEM_GROWTH', 3.0).to_f
+                max_heap_frag = ENV.fetch('GITLAB_MEMWD_MAX_HEAP_FRAG', DEFAULT_MAX_HEAP_FRAG).to_f
+                max_mem_growth = ENV.fetch('GITLAB_MEMWD_MAX_MEM_GROWTH', DEFAULT_MAX_MEM_GROWTH).to_f
 
                 # stack.push MonitorClass, args*, max_strikes:, kwargs**, &block
                 stack.push Gitlab::Memory::Watchdog::Monitor::HeapFragmentation,
@@ -50,17 +58,44 @@ module Gitlab
                            max_mem_growth: max_mem_growth,
                            max_strikes: max_strikes
               else
-                memory_limit = ENV.fetch('PUMA_WORKER_MAX_MEMORY', 1200).to_i
+                memory_limit = ENV.fetch('PUMA_WORKER_MAX_MEMORY', DEFAULT_PUMA_WORKER_RSS_LIMIT_MB).to_i
 
                 stack.push Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit,
-                           memory_limit: memory_limit.megabytes,
+                           memory_limit_bytes: memory_limit.megabytes,
                            max_strikes: max_strikes
               end
             end
           end
 
+          def sidekiq_sleep_time
+            [
+              ENV.fetch('SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', DEFAULT_SIDEKIQ_SLEEP_INTERVAL_S).to_i,
+              MIN_SIDEKIQ_SLEEP_INTERVAL_S
+            ].max
+          end
+
           def configure_monitors_for_sidekiq
-            # NOP - At the moment we don't run watchdog for Sidekiq
+            ->(stack) do
+              if ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'].to_i.nonzero?
+                soft_limit_bytes = ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'].to_i.kilobytes
+                grace_time = ENV.fetch('SIDEKIQ_MEMORY_KILLER_GRACE_TIME', DEFAULT_SIDEKIQ_GRACE_TIME_S).to_i
+                max_strikes = grace_time / sidekiq_sleep_time
+
+                stack.push Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit,
+                           memory_limit_bytes: soft_limit_bytes,
+                           max_strikes: max_strikes.to_i,
+                           monitor_name: :rss_memory_soft_limit
+              end
+
+              if ENV['SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS'].to_i.nonzero?
+                hard_limit_bytes = ENV['SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS'].to_i.kilobytes
+
+                stack.push Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit,
+                           memory_limit_bytes: hard_limit_bytes,
+                           max_strikes: 0,
+                           monitor_name: :rss_memory_hard_limit
+              end
+            end
           end
         end
       end
diff --git a/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb b/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb
index 8f230980eac..ce99b68464e 100644
--- a/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb
+++ b/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb
@@ -4,10 +4,7 @@ module Gitlab
   module Memory
     class Watchdog
       module Monitor
-        # A monitor that observes Ruby heap fragmentation and calls
-        # memory_violation_callback when the Ruby heap has been fragmented for an extended
-        # period of time.
-        #
+        # A monitor that observes Ruby heap fragmentation.
         # See Gitlab::Metrics::Memory for how heap fragmentation is defined.
         class HeapFragmentation
           attr_reader :max_heap_fragmentation
diff --git a/lib/gitlab/memory/watchdog/monitor/rss_memory_limit.rb b/lib/gitlab/memory/watchdog/monitor/rss_memory_limit.rb
index 3e7de024630..bcd122f0090 100644
--- a/lib/gitlab/memory/watchdog/monitor/rss_memory_limit.rb
+++ b/lib/gitlab/memory/watchdog/monitor/rss_memory_limit.rb
@@ -5,27 +5,27 @@ module Gitlab
     class Watchdog
       module Monitor
         class RssMemoryLimit
-          attr_reader :memory_limit
+          attr_reader :memory_limit_bytes
 
-          def initialize(memory_limit:)
-            @memory_limit = memory_limit
+          def initialize(memory_limit_bytes:)
+            @memory_limit_bytes = memory_limit_bytes
           end
 
           def call
-            worker_rss = Gitlab::Metrics::System.memory_usage_rss[:total]
+            worker_rss_bytes = Gitlab::Metrics::System.memory_usage_rss[:total]
 
-            return { threshold_violated: false, payload: {} } if worker_rss <= memory_limit
+            return { threshold_violated: false, payload: {} } if worker_rss_bytes <= memory_limit_bytes
 
-            { threshold_violated: true, payload: payload(worker_rss, memory_limit) }
+            { threshold_violated: true, payload: payload(worker_rss_bytes, memory_limit_bytes) }
           end
 
           private
 
-          def payload(worker_rss, memory_limit)
+          def payload(worker_rss_bytes, memory_limit_bytes)
             {
               message: 'rss memory limit exceeded',
-              memwd_rss_bytes: worker_rss,
-              memwd_max_rss_bytes: memory_limit
+              memwd_rss_bytes: worker_rss_bytes,
+              memwd_max_rss_bytes: memory_limit_bytes
             }
           end
         end
diff --git a/lib/gitlab/memory/watchdog/monitor_state.rb b/lib/gitlab/memory/watchdog/monitor_state.rb
index 73be5de3e45..2562599d2ab 100644
--- a/lib/gitlab/memory/watchdog/monitor_state.rb
+++ b/lib/gitlab/memory/watchdog/monitor_state.rb
@@ -5,12 +5,12 @@ module Gitlab
     class Watchdog
       class MonitorState
         class Result
-          attr_reader :payload
+          attr_reader :payload, :monitor_name
 
-          def initialize(strikes_exceeded:, threshold_violated:, monitor_class:, payload: )
+          def initialize(strikes_exceeded:, threshold_violated:, monitor_name:, payload: )
             @strikes_exceeded = strikes_exceeded
             @threshold_violated = threshold_violated
-            @monitor_class = monitor_class
+            @monitor_name = monitor_name.to_s.to_sym
             @payload = payload
           end
 
@@ -21,15 +21,12 @@ module Gitlab
           def threshold_violated?
             @threshold_violated
           end
-
-          def monitor_name
-            @monitor_class.name.demodulize.underscore.to_sym
-          end
         end
 
-        def initialize(monitor, max_strikes:)
+        def initialize(monitor, max_strikes:, monitor_name:)
           @monitor = monitor
           @max_strikes = max_strikes
+          @monitor_name = monitor_name
           @strikes = 0
         end
 
@@ -47,16 +44,12 @@ module Gitlab
           build_result(monitor_result)
         end
 
-        def monitor_class
-          @monitor.class
-        end
-
         private
 
         def build_result(monitor_result)
           Result.new(
             strikes_exceeded: strikes_exceeded?,
-            monitor_class: monitor_class,
+            monitor_name: @monitor_name,
             threshold_violated: monitor_result[:threshold_violated],
             payload: payload.merge(monitor_result[:payload]))
         end