Wait 15 minutes before Sidekiq MemoryKiller action

1 files changed, 33 insertions, 15 deletions

diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb
index 0fb09d3f228..df8968cf677 100644
--- a/lib/gitlab/sidekiq_middleware/memory_killer.rb
+++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb
@@ -1,26 +1,38 @@
 module Gitlab
   module SidekiqMiddleware
     class MemoryKiller
+      # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit
+      GRACE_TIME = 15 * 60
       # Wait 30 seconds for running jobs to finish during graceful shutdown
-      GRACEFUL_SHUTDOWN_WAIT = 30
+      SHUTDOWN_WAIT = 30
+      # Create a mutex so that there will be only one thread waiting to shut
+      # Sidekiq down
+      MUTEX = Mutex.new
 
       def call(worker, job, queue)
         yield
         current_rss = get_rss
+
         return unless max_rss > 0 && current_rss > max_rss
 
-        Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\
-         "#{max_rss}"
-        Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}"
-        # SIGUSR1 tells Sidekiq to stop accepting new jobs
-        Process.kill('SIGUSR1', Process.pid)
-
-        Sidekiq.logger.warn "spawning thread that will send SIGTERM to PID "\
-          "#{Process.pid} in #{graceful_shutdown_wait} seconds"
-        # Send the final shutdown signal to Sidekiq from a separate thread so
-        # that the current job can finish
-        Thread.new do
-          sleep(graceful_shutdown_wait)
+        Tread.new do
+          # Return if another thread is already waiting to shut Sidekiq down
+          return unless MUTEX.try_lock
+
+          Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\
+            "#{max_rss}"
+          Sidekiq.logger.warn "spawned thread that will shut down PID "\
+            "#{Process.pid} in #{grace_time} seconds"
+          sleep(grace_time)
+
+          Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}"
+          Process.kill('SIGUSR1', Process.pid)
+
+          Sidekiq.logger.warn "waiting #{shutdown_wait} seconds before sending "\
+            "SIGTERM to PID #{Process.pid}"
+          sleep(shutdown_wait)
+
+          Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid}"
           Process.kill('SIGTERM', Process.pid)
         end
       end
@@ -38,9 +50,15 @@ module Gitlab
         @max_rss ||= ENV['SIDEKIQ_MAX_RSS'].to_s.to_i
       end
 
-      def graceful_shutdown_wait
+      def shutdown_wait
         @graceful_shutdown_wait ||= (
-          ENV['SIDEKIQ_GRACEFUL_SHUTDOWN_WAIT'] || GRACEFUL_SHUTDOWN_WAIT
+          ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || SHUTDOWN_WAIT
+        ).to_i
+      end
+
+      def grace_time
+        @grace_time ||= (
+          ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || GRACE_TIME
         ).to_i
       end
     end