diff options
Diffstat (limited to 'spec/lib/gitlab/memory/watchdog')
4 files changed, 315 insertions, 0 deletions
diff --git a/spec/lib/gitlab/memory/watchdog/configuration_spec.rb b/spec/lib/gitlab/memory/watchdog/configuration_spec.rb new file mode 100644 index 00000000000..892a4b06ad0 --- /dev/null +++ b/spec/lib/gitlab/memory/watchdog/configuration_spec.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require_dependency 'gitlab/cluster/lifecycle_events' + +RSpec.describe Gitlab::Memory::Watchdog::Configuration do + subject(:configuration) { described_class.new } + + describe '#initialize' do + it 'initialize monitors' do + expect(configuration.monitors).to be_an_instance_of(described_class::MonitorStack) + end + end + + describe '#handler' do + context 'when handler is not set' do + it 'defaults to NullHandler' do + expect(configuration.handler).to be(Gitlab::Memory::Watchdog::NullHandler.instance) + end + end + end + + describe '#logger' do + context 'when logger is not set, defaults to stdout logger' do + it 'defaults to Logger' do + expect(configuration.logger).to be_an_instance_of(::Gitlab::Logger) + end + end + end + + describe '#sleep_time_seconds' do + context 'when sleep_time_seconds is not set' do + it 'defaults to SLEEP_TIME_SECONDS' do + expect(configuration.sleep_time_seconds).to eq(described_class::DEFAULT_SLEEP_TIME_SECONDS) + end + end + end + + describe '#monitors' do + context 'when monitors are configured to be used' do + let(:payload1) do + { + message: 'monitor_1_text', + memwd_max_strikes: 5, + memwd_cur_strikes: 0 + } + end + + let(:payload2) do + { + message: 'monitor_2_text', + memwd_max_strikes: 0, + memwd_cur_strikes: 1 + } + end + + let(:monitor_class_1) do + Struct.new(:threshold_violated, :payload) do + def call + { threshold_violated: !!threshold_violated, payload: payload || {} } + end + + def self.name + 'Monitor1' + end + end + end + + let(:monitor_class_2) do + Struct.new(:threshold_violated, :payload) do + def call + { threshold_violated: !!threshold_violated, payload: payload || {} } + end + + def self.name + 'Monitor2' + end + end + end + + context 'when two monitors are configured to be used' do + before do + configuration.monitors.use monitor_class_1, false, { message: 'monitor_1_text' }, max_strikes: 5 + configuration.monitors.use monitor_class_2, true, { message: 'monitor_2_text' }, max_strikes: 0 + end + + it 'calls each monitor and returns correct results', :aggregate_failures do + payloads = [] + thresholds = [] + strikes = [] + monitor_names = [] + + configuration.monitors.call_each do |result| + payloads << result.payload + thresholds << result.threshold_violated? + strikes << result.strikes_exceeded? + monitor_names << result.monitor_name + end + + expect(payloads).to eq([payload1, payload2]) + expect(thresholds).to eq([false, true]) + expect(strikes).to eq([false, true]) + expect(monitor_names).to eq([:monitor1, :monitor2]) + end + end + + context 'when same monitor class is configured to be used twice' do + before do + configuration.monitors.use monitor_class_1, max_strikes: 1 + configuration.monitors.use monitor_class_1, max_strikes: 1 + end + + it 'calls same monitor only once' do + expect do |b| + configuration.monitors.call_each(&b) + end.to yield_control.once + end + end + end + end +end diff --git a/spec/lib/gitlab/memory/watchdog/monitor/heap_fragmentation_spec.rb b/spec/lib/gitlab/memory/watchdog/monitor/heap_fragmentation_spec.rb new file mode 100644 index 00000000000..dad19cfd588 --- /dev/null +++ b/spec/lib/gitlab/memory/watchdog/monitor/heap_fragmentation_spec.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require 'support/shared_examples/lib/gitlab/memory/watchdog/monitor_result_shared_examples' +require 'prometheus/client' + +RSpec.describe Gitlab::Memory::Watchdog::Monitor::HeapFragmentation do + let(:heap_frag_limit_gauge) { instance_double(::Prometheus::Client::Gauge) } + let(:max_heap_fragmentation) { 0.2 } + let(:fragmentation) { 0.3 } + + subject(:monitor) do + described_class.new(max_heap_fragmentation: max_heap_fragmentation) + end + + before do + allow(Gitlab::Metrics).to receive(:gauge) + .with(:gitlab_memwd_heap_frag_limit, anything) + .and_return(heap_frag_limit_gauge) + allow(heap_frag_limit_gauge).to receive(:set) + + allow(Gitlab::Metrics::Memory).to receive(:gc_heap_fragmentation).and_return(fragmentation) + end + + describe '#initialize' do + it 'sets the heap fragmentation limit gauge' do + expect(heap_frag_limit_gauge).to receive(:set).with({}, max_heap_fragmentation) + + monitor + end + end + + describe '#call' do + it 'gets gc_heap_fragmentation' do + expect(Gitlab::Metrics::Memory).to receive(:gc_heap_fragmentation) + + monitor.call + end + + context 'when process exceeds threshold' do + let(:fragmentation) { max_heap_fragmentation + 0.1 } + let(:payload) do + { + message: 'heap fragmentation limit exceeded', + memwd_cur_heap_frag: fragmentation, + memwd_max_heap_frag: max_heap_fragmentation + } + end + + include_examples 'returns Watchdog Monitor result', threshold_violated: true + end + + context 'when process does not exceed threshold' do + let(:fragmentation) { max_heap_fragmentation - 0.1 } + let(:payload) { {} } + + include_examples 'returns Watchdog Monitor result', threshold_violated: false + end + end +end diff --git a/spec/lib/gitlab/memory/watchdog/monitor/unique_memory_growth_spec.rb b/spec/lib/gitlab/memory/watchdog/monitor/unique_memory_growth_spec.rb new file mode 100644 index 00000000000..22494af4425 --- /dev/null +++ b/spec/lib/gitlab/memory/watchdog/monitor/unique_memory_growth_spec.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require 'support/shared_examples/lib/gitlab/memory/watchdog/monitor_result_shared_examples' +require_dependency 'gitlab/cluster/lifecycle_events' + +RSpec.describe Gitlab::Memory::Watchdog::Monitor::UniqueMemoryGrowth do + let(:primary_memory) { 2048 } + let(:worker_memory) { 0 } + let(:max_mem_growth) { 2 } + + subject(:monitor) do + described_class.new(max_mem_growth: max_mem_growth) + end + + before do + allow(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).and_return({ uss: worker_memory }) + allow(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).with( + pid: Gitlab::Cluster::PRIMARY_PID + ).and_return({ uss: primary_memory }) + end + + describe '#call' do + it 'gets memory_usage_uss_pss' do + expect(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).with(no_args) + expect(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).with(pid: Gitlab::Cluster::PRIMARY_PID) + + monitor.call + end + + context 'when monitor is called twice' do + it 'reference memory is calculated only once' do + expect(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).with(no_args).twice + expect(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).with(pid: Gitlab::Cluster::PRIMARY_PID).once + + monitor.call + monitor.call + end + end + + context 'when process exceeds threshold' do + let(:worker_memory) { max_mem_growth * primary_memory + 1 } + let(:payload) do + { + message: 'memory limit exceeded', + memwd_max_uss_bytes: max_mem_growth * primary_memory, + memwd_ref_uss_bytes: primary_memory, + memwd_uss_bytes: worker_memory + } + end + + include_examples 'returns Watchdog Monitor result', threshold_violated: true + end + + context 'when process does not exceed threshold' do + let(:worker_memory) { max_mem_growth * primary_memory - 1 } + let(:payload) { {} } + + include_examples 'returns Watchdog Monitor result', threshold_violated: false + end + end +end diff --git a/spec/lib/gitlab/memory/watchdog/monitor_state_spec.rb b/spec/lib/gitlab/memory/watchdog/monitor_state_spec.rb new file mode 100644 index 00000000000..ace1353c6e3 --- /dev/null +++ b/spec/lib/gitlab/memory/watchdog/monitor_state_spec.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Memory::Watchdog::MonitorState do + let(:max_strikes) { 2 } + let(:payload) { { message: 'DummyMessage' } } + let(:threshold_violated) { true } + let(:monitor) { monitor_class.new(threshold_violated, payload) } + let(:monitor_class) do + Struct.new(:threshold_violated, :payload) do + def call + { threshold_violated: threshold_violated, payload: payload } + end + + def self.name + 'MonitorName' + end + end + end + + subject(:monitor_state) { described_class.new(monitor, max_strikes: max_strikes) } + + shared_examples 'returns correct result' do + it 'returns correct result', :aggregate_failures do + result = monitor_state.call + + expect(result).to be_an_instance_of(described_class::Result) + expect(result.strikes_exceeded?).to eq(strikes_exceeded) + expect(result.threshold_violated?).to eq(threshold_violated) + expect(result.payload).to eq(expected_payload) + expect(result.monitor_name).to eq(:monitor_name) + end + end + + describe '#call' do + let(:strikes_exceeded) { false } + let(:curr_strikes) { 0 } + let(:expected_payload) do + { + memwd_max_strikes: max_strikes, + memwd_cur_strikes: curr_strikes + }.merge(payload) + end + + context 'when threshold is not violated' do + let(:threshold_violated) { false } + + include_examples 'returns correct result' + end + + context 'when threshold is violated' do + let(:curr_strikes) { 1 } + let(:threshold_violated) { true } + + include_examples 'returns correct result' + + context 'when strikes_exceeded' do + let(:max_strikes) { 0 } + let(:strikes_exceeded) { true } + + include_examples 'returns correct result' + end + end + end + + describe '#monitor_class' do + subject { monitor_state.monitor_class } + + it { is_expected.to eq(monitor_class) } + end +end |