diff options
Diffstat (limited to 'spec/lib/gitlab/memory/watchdog_spec.rb')
| -rw-r--r-- | spec/lib/gitlab/memory/watchdog_spec.rb | 139 |
1 files changed, 65 insertions, 74 deletions
diff --git a/spec/lib/gitlab/memory/watchdog_spec.rb b/spec/lib/gitlab/memory/watchdog_spec.rb index 8b82078bcb9..010f6884df3 100644 --- a/spec/lib/gitlab/memory/watchdog_spec.rb +++ b/spec/lib/gitlab/memory/watchdog_spec.rb @@ -14,32 +14,57 @@ RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures, :prometheus do let(:sleep_time) { 0.1 } let(:max_heap_fragmentation) { 0.2 } + # Tests should set this to control the number of loop iterations in `call`. + let(:watchdog_iterations) { 1 } + subject(:watchdog) do described_class.new(handler: handler, logger: logger, sleep_time_seconds: sleep_time, - max_strikes: max_strikes, max_heap_fragmentation: max_heap_fragmentation) + max_strikes: max_strikes, max_heap_fragmentation: max_heap_fragmentation).tap do |instance| + # We need to defuse `sleep` and stop the internal loop after N iterations. + iterations = 0 + expect(instance).to receive(:sleep) do + instance.stop if (iterations += 1) >= watchdog_iterations + end.at_most(watchdog_iterations) + end + end + + def stub_prometheus_metrics + allow(Gitlab::Metrics).to receive(:gauge) + .with(:gitlab_memwd_heap_frag_limit, anything) + .and_return(heap_frag_limit_gauge) + allow(Gitlab::Metrics).to receive(:counter) + .with(:gitlab_memwd_heap_frag_violations_total, anything, anything) + .and_return(heap_frag_violations_counter) + allow(Gitlab::Metrics).to receive(:counter) + .with(:gitlab_memwd_heap_frag_violations_handled_total, anything, anything) + .and_return(heap_frag_violations_handled_counter) + + allow(heap_frag_limit_gauge).to receive(:set) + allow(heap_frag_violations_counter).to receive(:increment) + allow(heap_frag_violations_handled_counter).to receive(:increment) end before do + stub_prometheus_metrics + allow(handler).to receive(:on_high_heap_fragmentation).and_return(true) allow(logger).to receive(:warn) allow(logger).to receive(:info) allow(Gitlab::Metrics::Memory).to receive(:gc_heap_fragmentation).and_return(fragmentation) - end - after do - watchdog.stop + allow(::Prometheus::PidProvider).to receive(:worker_id).and_return('worker_1') end - context 'when starting up' do + context 'when created' do let(:fragmentation) { 0 } let(:max_strikes) { 0 } it 'sets the heap fragmentation limit gauge' do - allow(Gitlab::Metrics).to receive(:gauge).and_return(heap_frag_limit_gauge) - expect(heap_frag_limit_gauge).to receive(:set).with({}, max_heap_fragmentation) + + watchdog end context 'when no settings are set in the environment' do @@ -76,77 +101,54 @@ RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures, :prometheus do it 'does not signal the handler' do expect(handler).not_to receive(:on_high_heap_fragmentation) - watchdog.start - - sleep sleep_time * 3 + watchdog.call end end context 'when process exceeds heap fragmentation threshold permanently' do let(:fragmentation) { max_heap_fragmentation + 0.1 } - - before do - allow(Gitlab::Metrics).to receive(:counter) - .with(:gitlab_memwd_heap_frag_violations_total, anything, anything) - .and_return(heap_frag_violations_counter) - allow(Gitlab::Metrics).to receive(:counter) - .with(:gitlab_memwd_heap_frag_violations_handled_total, anything, anything) - .and_return(heap_frag_violations_handled_counter) - allow(heap_frag_violations_counter).to receive(:increment) - allow(heap_frag_violations_handled_counter).to receive(:increment) - end + let(:max_strikes) { 3 } context 'when process has not exceeded allowed number of strikes' do - let(:max_strikes) { 10 } + let(:watchdog_iterations) { max_strikes } it 'does not signal the handler' do expect(handler).not_to receive(:on_high_heap_fragmentation) - watchdog.start - - sleep sleep_time * 3 + watchdog.call end it 'does not log any events' do expect(logger).not_to receive(:warn) - watchdog.start - - sleep sleep_time * 3 + watchdog.call end it 'increments the violations counter' do - expect(heap_frag_violations_counter).to receive(:increment) - - watchdog.start + expect(heap_frag_violations_counter).to receive(:increment).exactly(watchdog_iterations) - sleep sleep_time * 3 + watchdog.call end it 'does not increment violations handled counter' do expect(heap_frag_violations_handled_counter).not_to receive(:increment) - watchdog.start - - sleep sleep_time * 3 + watchdog.call end end context 'when process exceeds the allowed number of strikes' do - let(:max_strikes) { 1 } + let(:watchdog_iterations) { max_strikes + 1 } it 'signals the handler and resets strike counter' do expect(handler).to receive(:on_high_heap_fragmentation).and_return(true) - watchdog.start - - sleep sleep_time * 3 + watchdog.call expect(watchdog.strikes).to eq(0) end it 'logs the event' do - expect(::Prometheus::PidProvider).to receive(:worker_id).at_least(:once).and_return('worker_1') expect(Gitlab::Metrics::System).to receive(:memory_usage_rss).at_least(:once).and_return(1024) expect(logger).to receive(:warn).with({ message: 'heap fragmentation limit exceeded', @@ -161,18 +163,14 @@ RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures, :prometheus do memwd_rss_bytes: 1024 }) - watchdog.start - - sleep sleep_time * 3 + watchdog.call end it 'increments both the violations and violations handled counters' do - expect(heap_frag_violations_counter).to receive(:increment) + expect(heap_frag_violations_counter).to receive(:increment).exactly(watchdog_iterations) expect(heap_frag_violations_handled_counter).to receive(:increment) - watchdog.start - - sleep sleep_time * 3 + watchdog.call end context 'when enforce_memory_watchdog ops toggle is off' do @@ -186,35 +184,31 @@ RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures, :prometheus do receive(:on_high_heap_fragmentation).with(fragmentation).and_return(true) ) - watchdog.start - - sleep sleep_time * 3 + watchdog.call end end - end - - context 'when handler result is true' do - let(:max_strikes) { 1 } - it 'considers the event handled and stops itself' do - expect(handler).to receive(:on_high_heap_fragmentation).once.and_return(true) + context 'when handler result is true' do + it 'considers the event handled and stops itself' do + expect(handler).to receive(:on_high_heap_fragmentation).once.and_return(true) + expect(logger).to receive(:info).with(hash_including(message: 'stopped')) - watchdog.start - - sleep sleep_time * 3 + watchdog.call + end end - end - - context 'when handler result is false' do - let(:max_strikes) { 1 } - it 'keeps running' do - # Return true the third time to terminate the daemon. - expect(handler).to receive(:on_high_heap_fragmentation).and_return(false, false, true) + context 'when handler result is false' do + let(:max_strikes) { 0 } # to make sure the handler fires each iteration + let(:watchdog_iterations) { 3 } - watchdog.start + it 'keeps running' do + expect(heap_frag_violations_counter).to receive(:increment).exactly(watchdog_iterations) + expect(heap_frag_violations_handled_counter).to receive(:increment).exactly(watchdog_iterations) + # Return true the third time to terminate the daemon. + expect(handler).to receive(:on_high_heap_fragmentation).and_return(false, false, true) - sleep sleep_time * 4 + watchdog.call + end end end end @@ -222,6 +216,7 @@ RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures, :prometheus do context 'when process exceeds heap fragmentation threshold temporarily' do let(:fragmentation) { max_heap_fragmentation } let(:max_strikes) { 1 } + let(:watchdog_iterations) { 4 } before do allow(Gitlab::Metrics::Memory).to receive(:gc_heap_fragmentation).and_return( @@ -235,9 +230,7 @@ RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures, :prometheus do it 'does not signal the handler' do expect(handler).not_to receive(:on_high_heap_fragmentation) - watchdog.start - - sleep sleep_time * 4 + watchdog.call end end @@ -252,9 +245,7 @@ RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures, :prometheus do it 'does not monitor heap fragmentation' do expect(Gitlab::Metrics::Memory).not_to receive(:gc_heap_fragmentation) - watchdog.start - - sleep sleep_time * 3 + watchdog.call end end end |