diff options
Diffstat (limited to 'spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb')
| -rw-r--r-- | spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb | 128 |
1 files changed, 77 insertions, 51 deletions
diff --git a/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb b/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb index dff04a2e509..62681b21756 100644 --- a/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb +++ b/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb @@ -130,9 +130,10 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do end it 'return true when everything is within limit', :aggregate_failures do - expect(memory_killer).to receive(:get_rss).and_return(100) - expect(memory_killer).to receive(:get_soft_limit_rss).and_return(200) - expect(memory_killer).to receive(:get_hard_limit_rss).and_return(300) + expect(memory_killer).to receive(:get_rss_kb).and_return(100) + expect(memory_killer).to receive(:get_soft_limit_rss_kb).and_return(200) + expect(memory_killer).to receive(:get_hard_limit_rss_kb).and_return(300) + expect(memory_killer).to receive(:get_memory_total_kb).and_return(3072) expect(memory_killer).to receive(:refresh_state) .with(:running) @@ -145,9 +146,10 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do end it 'return false when rss exceeds hard_limit_rss', :aggregate_failures do - expect(memory_killer).to receive(:get_rss).at_least(:once).and_return(400) - expect(memory_killer).to receive(:get_soft_limit_rss).at_least(:once).and_return(200) - expect(memory_killer).to receive(:get_hard_limit_rss).at_least(:once).and_return(300) + expect(memory_killer).to receive(:get_rss_kb).at_least(:once).and_return(400) + expect(memory_killer).to receive(:get_soft_limit_rss_kb).at_least(:once).and_return(200) + expect(memory_killer).to receive(:get_hard_limit_rss_kb).at_least(:once).and_return(300) + expect(memory_killer).to receive(:get_memory_total_kb).at_least(:once).and_return(3072) expect(memory_killer).to receive(:refresh_state) .with(:running) @@ -165,9 +167,10 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do end it 'return false when rss exceed hard_limit_rss after a while', :aggregate_failures do - expect(memory_killer).to receive(:get_rss).and_return(250, 400, 400) - expect(memory_killer).to receive(:get_soft_limit_rss).at_least(:once).and_return(200) - expect(memory_killer).to receive(:get_hard_limit_rss).at_least(:once).and_return(300) + expect(memory_killer).to receive(:get_rss_kb).and_return(250, 400, 400) + expect(memory_killer).to receive(:get_soft_limit_rss_kb).at_least(:once).and_return(200) + expect(memory_killer).to receive(:get_hard_limit_rss_kb).at_least(:once).and_return(300) + expect(memory_killer).to receive(:get_memory_total_kb).at_least(:once).and_return(3072) expect(memory_killer).to receive(:refresh_state) .with(:running) @@ -187,9 +190,10 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do end it 'return true when rss below soft_limit_rss after a while within GRACE_BALLOON_SECONDS', :aggregate_failures do - expect(memory_killer).to receive(:get_rss).and_return(250, 100) - expect(memory_killer).to receive(:get_soft_limit_rss).and_return(200, 200) - expect(memory_killer).to receive(:get_hard_limit_rss).and_return(300, 300) + expect(memory_killer).to receive(:get_rss_kb).and_return(250, 100) + expect(memory_killer).to receive(:get_soft_limit_rss_kb).and_return(200, 200) + expect(memory_killer).to receive(:get_hard_limit_rss_kb).and_return(300, 300) + expect(memory_killer).to receive(:get_memory_total_kb).and_return(3072, 3072) expect(memory_killer).to receive(:refresh_state) .with(:running) @@ -211,9 +215,10 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do let(:grace_balloon_seconds) { 0 } it 'return false when rss exceed soft_limit_rss', :aggregate_failures do - allow(memory_killer).to receive(:get_rss).and_return(250) - allow(memory_killer).to receive(:get_soft_limit_rss).and_return(200) - allow(memory_killer).to receive(:get_hard_limit_rss).and_return(300) + allow(memory_killer).to receive(:get_rss_kb).and_return(250) + allow(memory_killer).to receive(:get_soft_limit_rss_kb).and_return(200) + allow(memory_killer).to receive(:get_hard_limit_rss_kb).and_return(300) + allow(memory_killer).to receive(:get_memory_total_kb).and_return(3072) expect(memory_killer).to receive(:refresh_state) .with(:running) @@ -235,40 +240,57 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do subject { memory_killer.send(:restart_sidekiq) } - before do - stub_const("#{described_class}::SHUTDOWN_TIMEOUT_SECONDS", shutdown_timeout_seconds) - allow(Sidekiq).to receive(:options).and_return(timeout: 9) - allow(memory_killer).to receive(:get_rss).and_return(100) - allow(memory_killer).to receive(:get_soft_limit_rss).and_return(200) - allow(memory_killer).to receive(:get_hard_limit_rss).and_return(300) + context 'when sidekiq_memory_killer_read_only_mode is enabled' do + before do + stub_feature_flags(sidekiq_memory_killer_read_only_mode: true) + end + + it 'does not send signal' do + expect(memory_killer).not_to receive(:refresh_state) + expect(memory_killer).not_to receive(:signal_and_wait) + + subject + end end - it 'send signal' do - expect(memory_killer).to receive(:refresh_state) - .with(:stop_fetching_new_jobs) - .ordered - .and_call_original - expect(memory_killer).to receive(:signal_and_wait) - .with(shutdown_timeout_seconds, 'SIGTSTP', 'stop fetching new jobs') - .ordered + context 'when sidekiq_memory_killer_read_only_mode is disabled' do + before do + stub_const("#{described_class}::SHUTDOWN_TIMEOUT_SECONDS", shutdown_timeout_seconds) + stub_feature_flags(sidekiq_memory_killer_read_only_mode: false) + allow(Sidekiq).to receive(:options).and_return(timeout: 9) + allow(memory_killer).to receive(:get_rss_kb).and_return(100) + allow(memory_killer).to receive(:get_soft_limit_rss_kb).and_return(200) + allow(memory_killer).to receive(:get_hard_limit_rss_kb).and_return(300) + allow(memory_killer).to receive(:get_memory_total_kb).and_return(3072) + end - expect(memory_killer).to receive(:refresh_state) - .with(:shutting_down) - .ordered - .and_call_original - expect(memory_killer).to receive(:signal_and_wait) - .with(11, 'SIGTERM', 'gracefully shut down') - .ordered + it 'send signal' do + expect(memory_killer).to receive(:refresh_state) + .with(:stop_fetching_new_jobs) + .ordered + .and_call_original + expect(memory_killer).to receive(:signal_and_wait) + .with(shutdown_timeout_seconds, 'SIGTSTP', 'stop fetching new jobs') + .ordered - expect(memory_killer).to receive(:refresh_state) - .with(:killing_sidekiq) - .ordered - .and_call_original - expect(memory_killer).to receive(:signal_pgroup) - .with('SIGKILL', 'die') - .ordered + expect(memory_killer).to receive(:refresh_state) + .with(:shutting_down) + .ordered + .and_call_original + expect(memory_killer).to receive(:signal_and_wait) + .with(11, 'SIGTERM', 'gracefully shut down') + .ordered - subject + expect(memory_killer).to receive(:refresh_state) + .with(:killing_sidekiq) + .ordered + .and_call_original + expect(memory_killer).to receive(:signal_pgroup) + .with('SIGKILL', 'die') + .ordered + + subject + end end end @@ -351,6 +373,7 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do let(:current_rss) { 100 } let(:soft_limit_rss) { 200 } let(:hard_limit_rss) { 300 } + let(:memory_total) { 3072 } let(:jid) { 1 } let(:reason) { 'rss out of range reason description' } let(:queue) { 'default' } @@ -369,9 +392,10 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do before do stub_const("DummyWorker", worker) - allow(memory_killer).to receive(:get_rss).and_return(*current_rss) - allow(memory_killer).to receive(:get_soft_limit_rss).and_return(soft_limit_rss) - allow(memory_killer).to receive(:get_hard_limit_rss).and_return(hard_limit_rss) + allow(memory_killer).to receive(:get_rss_kb).and_return(*current_rss) + allow(memory_killer).to receive(:get_soft_limit_rss_kb).and_return(soft_limit_rss) + allow(memory_killer).to receive(:get_hard_limit_rss_kb).and_return(hard_limit_rss) + allow(memory_killer).to receive(:get_memory_total_kb).and_return(memory_total) memory_killer.send(:refresh_state, :running) end @@ -389,7 +413,8 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do hard_limit_rss: hard_limit_rss, soft_limit_rss: soft_limit_rss, reason: reason, - running_jobs: running_jobs) + running_jobs: running_jobs, + memory_total_kb: memory_total) expect(metrics[:sidekiq_memory_killer_running_jobs]).to receive(:increment) .with({ worker_class: "DummyWorker", deadline_exceeded: true }) @@ -525,9 +550,10 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do subject { memory_killer.send(:refresh_state, :shutting_down) } it 'calls gitlab metrics gauge set methods' do - expect(memory_killer).to receive(:get_rss) { 1010 } - expect(memory_killer).to receive(:get_soft_limit_rss) { 1020 } - expect(memory_killer).to receive(:get_hard_limit_rss) { 1040 } + expect(memory_killer).to receive(:get_rss_kb) { 1010 } + expect(memory_killer).to receive(:get_soft_limit_rss_kb) { 1020 } + expect(memory_killer).to receive(:get_hard_limit_rss_kb) { 1040 } + expect(memory_killer).to receive(:get_memory_total_kb) { 3072 } expect(metrics[:sidekiq_memory_killer_phase]).to receive(:set) .with({}, described_class::PHASE[:shutting_down]) |