diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2019-10-17 12:07:07 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2019-10-17 12:07:07 +0300 |
commit | 4226aca420920c1844e8eade4798a2dff188a6fc (patch) | |
tree | 2a09417654e9617a13798d97ae32c495c66d3353 /lib/gitlab/sidekiq_daemon | |
parent | 74b0522ed8b86edee204643c5484da1724a5a31b (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib/gitlab/sidekiq_daemon')
-rw-r--r-- | lib/gitlab/sidekiq_daemon/memory_killer.rb | 58 |
1 files changed, 50 insertions, 8 deletions
diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index 804170169e8..9d0d67a488f 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -21,14 +21,46 @@ module Gitlab # In case not set, default to 300M. This is for extra-safe. DEFAULT_MAX_MEMORY_GROWTH_KB = 300_000 + # Phases of memory killer + PHASE = { + running: 1, + above_soft_limit: 2, + stop_fetching_new_jobs: 3, + shutting_down: 4, + killing_sidekiq: 5 + }.freeze + def initialize super @enabled = true + @metrics = init_metrics end private + def init_metrics + { + sidekiq_current_rss: ::Gitlab::Metrics.gauge(:sidekiq_current_rss, 'Current RSS of Sidekiq Worker'), + sidekiq_memory_killer_soft_limit_rss: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_soft_limit_rss, 'Current soft_limit_rss of Sidekiq Worker'), + sidekiq_memory_killer_hard_limit_rss: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_hard_limit_rss, 'Current hard_limit_rss of Sidekiq Worker'), + sidekiq_memory_killer_phase: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_phase, 'Current phase of Sidekiq Worker') + } + end + + def refresh_state(phase) + @phase = PHASE.fetch(phase) + @current_rss = get_rss + @soft_limit_rss = get_soft_limit_rss + @hard_limit_rss = get_hard_limit_rss + + # track the current state as prometheus gauges + @metrics[:sidekiq_memory_killer_phase].set({}, @phase) + @metrics[:sidekiq_current_rss].set({}, @current_rss) + @metrics[:sidekiq_memory_killer_soft_limit_rss].set({}, @soft_limit_rss) + @metrics[:sidekiq_memory_killer_hard_limit_rss].set({}, @hard_limit_rss) + end + def run_thread Sidekiq.logger.info( class: self.class.to_s, @@ -77,41 +109,51 @@ module Gitlab # Tell Sidekiq to stop fetching new jobs # We first SIGNAL and then wait given time # We also monitor a number of running jobs and allow to restart early + refresh_state(:stop_fetching_new_jobs) signal_and_wait(SHUTDOWN_TIMEOUT_SECONDS, 'SIGTSTP', 'stop fetching new jobs') return unless enabled? # Tell sidekiq to restart itself # Keep extra safe to wait `Sidekiq.options[:timeout] + 2` seconds before SIGKILL + refresh_state(:shutting_down) signal_and_wait(Sidekiq.options[:timeout] + 2, 'SIGTERM', 'gracefully shut down') return unless enabled? # Ideally we should never reach this condition # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't # Kill the whole pgroup, so we can be sure no children are left behind + refresh_state(:killing_sidekiq) signal_pgroup('SIGKILL', 'die') end def rss_within_range? - current_rss = nil + refresh_state(:running) + deadline = Gitlab::Metrics::System.monotonic_time + GRACE_BALLOON_SECONDS.seconds loop do return true unless enabled? - current_rss = get_rss - # RSS go above hard limit should trigger forcible shutdown right away - break if current_rss > hard_limit_rss + break if @current_rss > @hard_limit_rss # RSS go below the soft limit - return true if current_rss < soft_limit_rss + return true if @current_rss < @soft_limit_rss # RSS did not go below the soft limit within deadline, restart break if Gitlab::Metrics::System.monotonic_time > deadline sleep(CHECK_INTERVAL_SECONDS) + + refresh_state(:above_soft_limit) end - log_rss_out_of_range(current_rss, hard_limit_rss, soft_limit_rss) + # There are two chances to break from loop: + # - above hard limit, or + # - above soft limit after deadline + # When `above hard limit`, it immediately go to `stop_fetching_new_jobs` + # So ignore `above hard limit` and always set `above_soft_limit` here + refresh_state(:above_soft_limit) + log_rss_out_of_range(@current_rss, @hard_limit_rss, @soft_limit_rss) false end @@ -143,11 +185,11 @@ module Gitlab output.to_i end - def soft_limit_rss + def get_soft_limit_rss SOFT_LIMIT_RSS_KB + rss_increase_by_jobs end - def hard_limit_rss + def get_hard_limit_rss HARD_LIMIT_RSS_KB end |