diff options
Diffstat (limited to 'spec/lib/gitlab/memory')
| -rw-r--r-- | spec/lib/gitlab/memory/watchdog/configuration_spec.rb | 61 | ||||
| -rw-r--r-- | spec/lib/gitlab/memory/watchdog/configurator_spec.rb | 199 | ||||
| -rw-r--r-- | spec/lib/gitlab/memory/watchdog/monitor/rss_memory_limit_spec.rb | 39 | ||||
| -rw-r--r-- | spec/lib/gitlab/memory/watchdog_spec.rb | 10 |
4 files changed, 283 insertions, 26 deletions
diff --git a/spec/lib/gitlab/memory/watchdog/configuration_spec.rb b/spec/lib/gitlab/memory/watchdog/configuration_spec.rb index 892a4b06ad0..38a39f6a33a 100644 --- a/spec/lib/gitlab/memory/watchdog/configuration_spec.rb +++ b/spec/lib/gitlab/memory/watchdog/configuration_spec.rb @@ -78,36 +78,53 @@ RSpec.describe Gitlab::Memory::Watchdog::Configuration do end end - context 'when two monitors are configured to be used' do - before do - configuration.monitors.use monitor_class_1, false, { message: 'monitor_1_text' }, max_strikes: 5 - configuration.monitors.use monitor_class_2, true, { message: 'monitor_2_text' }, max_strikes: 0 + context 'when two different monitor class are configured' do + shared_examples 'executes monitors and returns correct results' do + it 'calls each monitor and returns correct results', :aggregate_failures do + payloads = [] + thresholds = [] + strikes = [] + monitor_names = [] + + configuration.monitors.call_each do |result| + payloads << result.payload + thresholds << result.threshold_violated? + strikes << result.strikes_exceeded? + monitor_names << result.monitor_name + end + + expect(payloads).to eq([payload1, payload2]) + expect(thresholds).to eq([false, true]) + expect(strikes).to eq([false, true]) + expect(monitor_names).to eq([:monitor1, :monitor2]) + end + end + + context 'when monitors are configured inline' do + before do + configuration.monitors.push monitor_class_1, false, { message: 'monitor_1_text' }, max_strikes: 5 + configuration.monitors.push monitor_class_2, true, { message: 'monitor_2_text' }, max_strikes: 0 + end + + include_examples 'executes monitors and returns correct results' end - it 'calls each monitor and returns correct results', :aggregate_failures do - payloads = [] - thresholds = [] - strikes = [] - monitor_names = [] - - configuration.monitors.call_each do |result| - payloads << result.payload - thresholds << result.threshold_violated? - strikes << result.strikes_exceeded? - monitor_names << result.monitor_name + context 'when monitors are configured in a block' do + before do + configuration.monitors do |stack| + stack.push monitor_class_1, false, { message: 'monitor_1_text' }, max_strikes: 5 + stack.push monitor_class_2, true, { message: 'monitor_2_text' }, max_strikes: 0 + end end - expect(payloads).to eq([payload1, payload2]) - expect(thresholds).to eq([false, true]) - expect(strikes).to eq([false, true]) - expect(monitor_names).to eq([:monitor1, :monitor2]) + include_examples 'executes monitors and returns correct results' end end - context 'when same monitor class is configured to be used twice' do + context 'when same monitor class is configured twice' do before do - configuration.monitors.use monitor_class_1, max_strikes: 1 - configuration.monitors.use monitor_class_1, max_strikes: 1 + configuration.monitors.push monitor_class_1, max_strikes: 1 + configuration.monitors.push monitor_class_1, max_strikes: 1 end it 'calls same monitor only once' do diff --git a/spec/lib/gitlab/memory/watchdog/configurator_spec.rb b/spec/lib/gitlab/memory/watchdog/configurator_spec.rb new file mode 100644 index 00000000000..e6f2d57e9e6 --- /dev/null +++ b/spec/lib/gitlab/memory/watchdog/configurator_spec.rb @@ -0,0 +1,199 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require 'prometheus/client' +require 'sidekiq' +require_dependency 'gitlab/cluster/lifecycle_events' + +RSpec.describe Gitlab::Memory::Watchdog::Configurator do + shared_examples 'as configurator' do |handler_class, sleep_time_env, sleep_time| + it 'configures the correct handler' do + configurator.call(configuration) + + expect(configuration.handler).to be_an_instance_of(handler_class) + end + + it 'configures the correct logger' do + configurator.call(configuration) + + expect(configuration.logger).to eq(logger) + end + + context 'when sleep_time_seconds is not passed through the environment' do + let(:sleep_time_seconds) { sleep_time } + + it 'configures the correct sleep time' do + configurator.call(configuration) + + expect(configuration.sleep_time_seconds).to eq(sleep_time_seconds) + end + end + + context 'when sleep_time_seconds is passed through the environment' do + let(:sleep_time_seconds) { sleep_time - 1 } + + before do + stub_env(sleep_time_env, sleep_time - 1) + end + + it 'configures the correct sleep time' do + configurator.call(configuration) + + expect(configuration.sleep_time_seconds).to eq(sleep_time_seconds) + end + end + end + + shared_examples 'as monitor configurator' do + it 'executes monitors and returns correct results' do + configurator.call(configuration) + + payloads = {} + configuration.monitors.call_each do |result| + payloads[result.monitor_name] = result.payload + end + + expect(payloads).to eq(expected_payloads) + end + end + + let(:configuration) { Gitlab::Memory::Watchdog::Configuration.new } + + # In tests, the Puma constant does not exist so we cannot use a verified double. + # rubocop: disable RSpec/VerifiedDoubles + describe '.configure_for_puma' do + let(:logger) { Gitlab::AppLogger } + let(:puma) do + Class.new do + def self.cli_config + Struct.new(:options).new + end + end + end + + subject(:configurator) { described_class.configure_for_puma } + + def stub_prometheus_metrics + gauge = instance_double(::Prometheus::Client::Gauge) + allow(Gitlab::Metrics).to receive(:gauge).and_return(gauge) + allow(gauge).to receive(:set) + end + + before do + stub_const('Puma', puma) + stub_const('Puma::Cluster::WorkerHandle', double.as_null_object) + stub_prometheus_metrics + end + + it_behaves_like 'as configurator', + Gitlab::Memory::Watchdog::PumaHandler, + 'GITLAB_MEMWD_SLEEP_TIME_SEC', + 60 + + context 'with DISABLE_PUMA_WORKER_KILLER set to true' do + let(:primary_memory) { 2048 } + let(:worker_memory) { max_mem_growth * primary_memory + 1 } + let(:expected_payloads) do + { + heap_fragmentation: { + message: 'heap fragmentation limit exceeded', + memwd_cur_heap_frag: max_heap_fragmentation + 0.1, + memwd_max_heap_frag: max_heap_fragmentation, + memwd_max_strikes: max_strikes, + memwd_cur_strikes: 1 + + }, + unique_memory_growth: { + message: 'memory limit exceeded', + memwd_uss_bytes: worker_memory, + memwd_ref_uss_bytes: primary_memory, + memwd_max_uss_bytes: max_mem_growth * primary_memory, + memwd_max_strikes: max_strikes, + memwd_cur_strikes: 1 + } + } + end + + before do + stub_env('DISABLE_PUMA_WORKER_KILLER', true) + allow(Gitlab::Metrics::Memory).to receive(:gc_heap_fragmentation).and_return(max_heap_fragmentation + 0.1) + allow(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).and_return({ uss: worker_memory }) + allow(Gitlab::Metrics::System).to receive(:memory_usage_uss_pss).with( + pid: Gitlab::Cluster::PRIMARY_PID + ).and_return({ uss: primary_memory }) + end + + context 'when settings are set via environment variables' do + let(:max_heap_fragmentation) { 0.4 } + let(:max_mem_growth) { 4.0 } + let(:max_strikes) { 4 } + + before do + stub_env('GITLAB_MEMWD_MAX_HEAP_FRAG', 0.4) + stub_env('GITLAB_MEMWD_MAX_MEM_GROWTH', 4.0) + stub_env('GITLAB_MEMWD_MAX_STRIKES', 4) + end + + it_behaves_like 'as monitor configurator' + end + + context 'when settings are not set via environment variables' do + let(:max_heap_fragmentation) { 0.5 } + let(:max_mem_growth) { 3.0 } + let(:max_strikes) { 5 } + + it_behaves_like 'as monitor configurator' + end + end + + context 'with DISABLE_PUMA_WORKER_KILLER set to false' do + let(:expected_payloads) do + { + rss_memory_limit: { + message: 'rss memory limit exceeded', + memwd_rss_bytes: memory_limit + 1, + memwd_max_rss_bytes: memory_limit, + memwd_max_strikes: max_strikes, + memwd_cur_strikes: 1 + } + } + end + + before do + stub_env('DISABLE_PUMA_WORKER_KILLER', false) + allow(Gitlab::Metrics::System).to receive(:memory_usage_rss).and_return({ total: memory_limit + 1 }) + end + + context 'when settings are set via environment variables' do + let(:memory_limit) { 1300 } + let(:max_strikes) { 4 } + + before do + stub_env('PUMA_WORKER_MAX_MEMORY', 1300) + stub_env('GITLAB_MEMWD_MAX_STRIKES', 4) + end + + it_behaves_like 'as monitor configurator' + end + + context 'when settings are not set via environment variables' do + let(:memory_limit) { 1200 } + let(:max_strikes) { 5 } + + it_behaves_like 'as monitor configurator' + end + end + end + # rubocop: enable RSpec/VerifiedDoubles + + describe '.configure_for_sidekiq' do + let(:logger) { ::Sidekiq.logger } + + subject(:configurator) { described_class.configure_for_sidekiq } + + it_behaves_like 'as configurator', + Gitlab::Memory::Watchdog::TermProcessHandler, + 'SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', + 3 + end +end diff --git a/spec/lib/gitlab/memory/watchdog/monitor/rss_memory_limit_spec.rb b/spec/lib/gitlab/memory/watchdog/monitor/rss_memory_limit_spec.rb new file mode 100644 index 00000000000..9e25cfda782 --- /dev/null +++ b/spec/lib/gitlab/memory/watchdog/monitor/rss_memory_limit_spec.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require 'support/shared_examples/lib/gitlab/memory/watchdog/monitor_result_shared_examples' + +RSpec.describe Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit do + let(:memory_limit) { 2048 } + let(:worker_memory) { 1024 } + + subject(:monitor) do + described_class.new(memory_limit: memory_limit) + end + + before do + allow(Gitlab::Metrics::System).to receive(:memory_usage_rss).and_return({ total: worker_memory }) + end + + describe '#call' do + context 'when process exceeds threshold' do + let(:worker_memory) { memory_limit + 1 } + let(:payload) do + { + message: 'rss memory limit exceeded', + memwd_rss_bytes: worker_memory, + memwd_max_rss_bytes: memory_limit + } + end + + include_examples 'returns Watchdog Monitor result', threshold_violated: true + end + + context 'when process does not exceed threshold' do + let(:worker_memory) { memory_limit - 1 } + let(:payload) { {} } + + include_examples 'returns Watchdog Monitor result', threshold_violated: false + end + end +end diff --git a/spec/lib/gitlab/memory/watchdog_spec.rb b/spec/lib/gitlab/memory/watchdog_spec.rb index 84e9a577afb..5d9599d6eab 100644 --- a/spec/lib/gitlab/memory/watchdog_spec.rb +++ b/spec/lib/gitlab/memory/watchdog_spec.rb @@ -60,14 +60,16 @@ RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures do describe '#call' do before do stub_prometheus_metrics - allow(Gitlab::Metrics::System).to receive(:memory_usage_rss).at_least(:once).and_return(1024) + allow(Gitlab::Metrics::System).to receive(:memory_usage_rss).at_least(:once).and_return( + total: 1024 + ) allow(::Prometheus::PidProvider).to receive(:worker_id).and_return('worker_1') watchdog.configure do |config| config.handler = handler config.logger = logger config.sleep_time_seconds = sleep_time_seconds - config.monitors.use monitor_class, threshold_violated, payload, max_strikes: max_strikes + config.monitors.push monitor_class, threshold_violated, payload, max_strikes: max_strikes end allow(handler).to receive(:call).and_return(true) @@ -203,8 +205,8 @@ RSpec.describe Gitlab::Memory::Watchdog, :aggregate_failures do config.handler = handler config.logger = logger config.sleep_time_seconds = sleep_time_seconds - config.monitors.use monitor_class, threshold_violated, payload, max_strikes: max_strikes - config.monitors.use monitor_class, threshold_violated, payload, max_strikes: max_strikes + config.monitors.push monitor_class, threshold_violated, payload, max_strikes: max_strikes + config.monitors.push monitor_class, threshold_violated, payload, max_strikes: max_strikes end end |