Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/spec/lib/gitlab/sidekiq_middleware/size_limiter/validator_spec.rb
diff options
context:
space:
mode:
Diffstat (limited to 'spec/lib/gitlab/sidekiq_middleware/size_limiter/validator_spec.rb')
-rw-r--r--spec/lib/gitlab/sidekiq_middleware/size_limiter/validator_spec.rb186
1 files changed, 129 insertions, 57 deletions
diff --git a/spec/lib/gitlab/sidekiq_middleware/size_limiter/validator_spec.rb b/spec/lib/gitlab/sidekiq_middleware/size_limiter/validator_spec.rb
index 3140686c908..4fbe59c3c27 100644
--- a/spec/lib/gitlab/sidekiq_middleware/size_limiter/validator_spec.rb
+++ b/spec/lib/gitlab/sidekiq_middleware/size_limiter/validator_spec.rb
@@ -3,6 +3,21 @@
require 'spec_helper'
RSpec.describe Gitlab::SidekiqMiddleware::SizeLimiter::Validator do
+ let(:base_payload) do
+ {
+ "class" => "ARandomWorker",
+ "queue" => "a_worker",
+ "retry" => true,
+ "jid" => "d774900367dc8b2962b2479c",
+ "created_at" => 1234567890,
+ "enqueued_at" => 1234567890
+ }
+ end
+
+ def job_payload(args = {})
+ base_payload.merge('args' => args)
+ end
+
let(:worker_class) do
Class.new do
def self.name
@@ -24,8 +39,8 @@ RSpec.describe Gitlab::SidekiqMiddleware::SizeLimiter::Validator do
it 'does not log a warning message' do
expect(::Sidekiq.logger).not_to receive(:warn)
- described_class.new(TestSizeLimiterWorker, {}, mode: 'track')
- described_class.new(TestSizeLimiterWorker, {}, mode: 'raise')
+ described_class.new(TestSizeLimiterWorker, job_payload, mode: 'track')
+ described_class.new(TestSizeLimiterWorker, job_payload, mode: 'compress')
end
end
@@ -33,7 +48,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::SizeLimiter::Validator do
it 'defaults to track mode and logs a warning message' do
expect(::Sidekiq.logger).to receive(:warn).with('Invalid Sidekiq size limiter mode: invalid. Fallback to track mode.')
- validator = described_class.new(TestSizeLimiterWorker, {}, mode: 'invalid')
+ validator = described_class.new(TestSizeLimiterWorker, job_payload, mode: 'invalid')
expect(validator.mode).to eql('track')
end
@@ -43,7 +58,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::SizeLimiter::Validator do
it 'defaults to track mode' do
expect(::Sidekiq.logger).not_to receive(:warn)
- validator = described_class.new(TestSizeLimiterWorker, {})
+ validator = described_class.new(TestSizeLimiterWorker, job_payload)
expect(validator.mode).to eql('track')
end
@@ -53,8 +68,8 @@ RSpec.describe Gitlab::SidekiqMiddleware::SizeLimiter::Validator do
it 'does not log a warning message' do
expect(::Sidekiq.logger).not_to receive(:warn)
- described_class.new(TestSizeLimiterWorker, {}, size_limit: 300)
- described_class.new(TestSizeLimiterWorker, {}, size_limit: 0)
+ described_class.new(TestSizeLimiterWorker, job_payload, size_limit: 300)
+ described_class.new(TestSizeLimiterWorker, job_payload, size_limit: 0)
end
end
@@ -62,7 +77,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::SizeLimiter::Validator do
it 'defaults to 0 and logs a warning message' do
expect(::Sidekiq.logger).to receive(:warn).with('Invalid Sidekiq size limiter limit: -1')
- described_class.new(TestSizeLimiterWorker, {}, size_limit: -1)
+ described_class.new(TestSizeLimiterWorker, job_payload, size_limit: -1)
end
end
@@ -70,15 +85,63 @@ RSpec.describe Gitlab::SidekiqMiddleware::SizeLimiter::Validator do
it 'defaults to 0' do
expect(::Sidekiq.logger).not_to receive(:warn)
- validator = described_class.new(TestSizeLimiterWorker, {})
+ validator = described_class.new(TestSizeLimiterWorker, job_payload)
expect(validator.size_limit).to be(0)
end
end
+
+ context 'when the compression threshold is valid' do
+ it 'does not log a warning message' do
+ expect(::Sidekiq.logger).not_to receive(:warn)
+
+ described_class.new(TestSizeLimiterWorker, job_payload, compression_threshold: 300)
+ described_class.new(TestSizeLimiterWorker, job_payload, compression_threshold: 1)
+ end
+ end
+
+ context 'when the compression threshold is negative' do
+ it 'logs a warning message' do
+ expect(::Sidekiq.logger).to receive(:warn).with('Invalid Sidekiq size limiter compression threshold: -1')
+
+ described_class.new(TestSizeLimiterWorker, job_payload, compression_threshold: -1)
+ end
+
+ it 'falls back to the default' do
+ validator = described_class.new(TestSizeLimiterWorker, job_payload, compression_threshold: -1)
+
+ expect(validator.compression_threshold).to be(100_000)
+ end
+ end
+
+ context 'when the compression threshold is zero' do
+ it 'logs a warning message' do
+ expect(::Sidekiq.logger).to receive(:warn).with('Invalid Sidekiq size limiter compression threshold: 0')
+
+ described_class.new(TestSizeLimiterWorker, job_payload, compression_threshold: 0)
+ end
+
+ it 'falls back to the default' do
+ validator = described_class.new(TestSizeLimiterWorker, job_payload, compression_threshold: 0)
+
+ expect(validator.compression_threshold).to be(100_000)
+ end
+ end
+
+ context 'when the compression threshold is empty' do
+ it 'defaults to 100_000' do
+ expect(::Sidekiq.logger).not_to receive(:warn)
+
+ validator = described_class.new(TestSizeLimiterWorker, job_payload)
+
+ expect(validator.compression_threshold).to be(100_000)
+ end
+ end
end
shared_examples 'validate limit job payload size' do
context 'in track mode' do
+ let(:compression_threshold) { nil }
let(:mode) { 'track' }
context 'when size limit negative' do
@@ -87,11 +150,11 @@ RSpec.describe Gitlab::SidekiqMiddleware::SizeLimiter::Validator do
it 'does not track jobs' do
expect(Gitlab::ErrorTracking).not_to receive(:track_exception)
- validate.call(TestSizeLimiterWorker, { a: 'a' * 300 })
+ validate.call(TestSizeLimiterWorker, job_payload(a: 'a' * 300))
end
it 'does not raise exception' do
- expect { validate.call(TestSizeLimiterWorker, { a: 'a' * 300 }) }.not_to raise_error
+ expect { validate.call(TestSizeLimiterWorker, job_payload(a: 'a' * 300)) }.not_to raise_error
end
end
@@ -101,11 +164,13 @@ RSpec.describe Gitlab::SidekiqMiddleware::SizeLimiter::Validator do
it 'does not track jobs' do
expect(Gitlab::ErrorTracking).not_to receive(:track_exception)
- validate.call(TestSizeLimiterWorker, { a: 'a' * 300 })
+ validate.call(TestSizeLimiterWorker, job_payload(a: 'a' * 300))
end
it 'does not raise exception' do
- expect { validate.call(TestSizeLimiterWorker, { a: 'a' * 300 }) }.not_to raise_error
+ expect do
+ validate.call(TestSizeLimiterWorker, job_payload(a: 'a' * 300))
+ end.not_to raise_error
end
end
@@ -117,11 +182,13 @@ RSpec.describe Gitlab::SidekiqMiddleware::SizeLimiter::Validator do
be_a(Gitlab::SidekiqMiddleware::SizeLimiter::ExceedLimitError)
)
- validate.call(TestSizeLimiterWorker, { a: 'a' * 100 })
+ validate.call(TestSizeLimiterWorker, job_payload(a: 'a' * 100))
end
it 'does not raise an exception' do
- expect { validate.call(TestSizeLimiterWorker, { a: 'a' * 300 }) }.not_to raise_error
+ expect do
+ validate.call(TestSizeLimiterWorker, job_payload(a: 'a' * 300))
+ end.not_to raise_error
end
context 'when the worker has big_payload attribute' do
@@ -132,13 +199,17 @@ RSpec.describe Gitlab::SidekiqMiddleware::SizeLimiter::Validator do
it 'does not track jobs' do
expect(Gitlab::ErrorTracking).not_to receive(:track_exception)
- validate.call(TestSizeLimiterWorker, { a: 'a' * 300 })
- validate.call('TestSizeLimiterWorker', { a: 'a' * 300 })
+ validate.call(TestSizeLimiterWorker, job_payload(a: 'a' * 300))
+ validate.call('TestSizeLimiterWorker', job_payload(a: 'a' * 300))
end
it 'does not raise an exception' do
- expect { validate.call(TestSizeLimiterWorker, { a: 'a' * 300 }) }.not_to raise_error
- expect { validate.call('TestSizeLimiterWorker', { a: 'a' * 300 }) }.not_to raise_error
+ expect do
+ validate.call(TestSizeLimiterWorker, job_payload(a: 'a' * 300))
+ end.not_to raise_error
+ expect do
+ validate.call('TestSizeLimiterWorker', job_payload(a: 'a' * 300))
+ end.not_to raise_error
end
end
end
@@ -149,63 +220,60 @@ RSpec.describe Gitlab::SidekiqMiddleware::SizeLimiter::Validator do
it 'does not track job' do
expect(Gitlab::ErrorTracking).not_to receive(:track_exception)
- validate.call(TestSizeLimiterWorker, { a: 'a' })
+ validate.call(TestSizeLimiterWorker, job_payload(a: 'a'))
end
it 'does not raise an exception' do
- expect { validate.call(TestSizeLimiterWorker, { a: 'a' }) }.not_to raise_error
+ expect { validate.call(TestSizeLimiterWorker, job_payload(a: 'a')) }.not_to raise_error
end
end
end
- context 'in raise mode' do
- let(:mode) { 'raise' }
-
- context 'when size limit is negative' do
- let(:size_limit) { -1 }
-
- it 'does not raise exception' do
- expect { validate.call(TestSizeLimiterWorker, { a: 'a' * 300 }) }.not_to raise_error
- end
- end
+ context 'in compress mode' do
+ let(:mode) { 'compress' }
- context 'when size limit is 0' do
- let(:size_limit) { 0 }
+ context 'when job size is less than compression threshold' do
+ let(:size_limit) { 50 }
+ let(:compression_threshold) { 30 }
+ let(:job) { job_payload(a: 'a' * 10) }
- it 'does not raise exception' do
- expect { validate.call(TestSizeLimiterWorker, { a: 'a' * 300 }) }.not_to raise_error
+ it 'does not raise an exception' do
+ expect(::Gitlab::SidekiqMiddleware::SizeLimiter::Compressor).not_to receive(:compress)
+ expect { validate.call(TestSizeLimiterWorker, job_payload(a: 'a')) }.not_to raise_error
end
end
- context 'when job size is bigger than size limit' do
+ context 'when job size is bigger than compression threshold and less than size limit after compressed' do
let(:size_limit) { 50 }
+ let(:compression_threshold) { 30 }
+ let(:args) { { a: 'a' * 300 } }
+ let(:job) { job_payload(args) }
- it 'raises an exception' do
- expect do
- validate.call(TestSizeLimiterWorker, { a: 'a' * 300 })
- end.to raise_error(
- Gitlab::SidekiqMiddleware::SizeLimiter::ExceedLimitError,
- /TestSizeLimiterWorker job exceeds payload size limit/i
- )
- end
-
- context 'when the worker has big_payload attribute' do
- before do
- worker_class.big_payload!
- end
+ it 'does not raise an exception' do
+ expect(::Gitlab::SidekiqMiddleware::SizeLimiter::Compressor).to receive(:compress).with(
+ job, Sidekiq.dump_json(args)
+ ).and_return('a' * 40)
- it 'does not raise an exception' do
- expect { validate.call(TestSizeLimiterWorker, { a: 'a' * 300 }) }.not_to raise_error
- expect { validate.call('TestSizeLimiterWorker', { a: 'a' * 300 }) }.not_to raise_error
- end
+ expect do
+ validate.call(TestSizeLimiterWorker, job)
+ end.not_to raise_error
end
end
- context 'when job size is less than size limit' do
+ context 'when job size is bigger than compression threshold and bigger than size limit after compressed' do
let(:size_limit) { 50 }
+ let(:compression_threshold) { 30 }
+ let(:args) { { a: 'a' * 3000 } }
+ let(:job) { job_payload(args) }
it 'does not raise an exception' do
- expect { validate.call(TestSizeLimiterWorker, { a: 'a' }) }.not_to raise_error
+ expect(::Gitlab::SidekiqMiddleware::SizeLimiter::Compressor).to receive(:compress).with(
+ job, Sidekiq.dump_json(args)
+ ).and_return('a' * 60)
+
+ expect do
+ validate.call(TestSizeLimiterWorker, job)
+ end.to raise_error(Gitlab::SidekiqMiddleware::SizeLimiter::ExceedLimitError)
end
end
end
@@ -218,6 +286,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::SizeLimiter::Validator do
before do
stub_env('GITLAB_SIDEKIQ_SIZE_LIMITER_MODE', mode)
stub_env('GITLAB_SIDEKIQ_SIZE_LIMITER_LIMIT_BYTES', size_limit)
+ stub_env('GITLAB_SIDEKIQ_SIZE_LIMITER_COMPRESSION_THRESHOLD_BYTES', compression_threshold)
end
it_behaves_like 'validate limit job payload size'
@@ -226,14 +295,14 @@ RSpec.describe Gitlab::SidekiqMiddleware::SizeLimiter::Validator do
context 'when creating an instance with the related ENV variables' do
let(:validate) do
->(worker_clas, job) do
- validator = described_class.new(worker_class, job, mode: mode, size_limit: size_limit)
- validator.validate!
+ described_class.new(worker_class, job).validate!
end
end
before do
stub_env('GITLAB_SIDEKIQ_SIZE_LIMITER_MODE', mode)
stub_env('GITLAB_SIDEKIQ_SIZE_LIMITER_LIMIT_BYTES', size_limit)
+ stub_env('GITLAB_SIDEKIQ_SIZE_LIMITER_COMPRESSION_THRESHOLD_BYTES', compression_threshold)
end
it_behaves_like 'validate limit job payload size'
@@ -242,7 +311,10 @@ RSpec.describe Gitlab::SidekiqMiddleware::SizeLimiter::Validator do
context 'when creating an instance with mode and size limit' do
let(:validate) do
->(worker_clas, job) do
- validator = described_class.new(worker_class, job, mode: mode, size_limit: size_limit)
+ validator = described_class.new(
+ worker_class, job,
+ mode: mode, size_limit: size_limit, compression_threshold: compression_threshold
+ )
validator.validate!
end
end
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-16 05:52:58 +0300